commit 010f5c445a8acb9d28a1bc26a483b7bacc8a09e9 Author: jiajie555 Date: Fri Apr 18 14:41:53 2025 +0800 回传数据解析，兼容v5和v10 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f9e4e3c --- /dev/null +++ b/.gitignore @@ -0,0 +1,74 @@ +# Repo-specific GitIgnore ---------------------------------------------------------------------------------------------- +*.jpg +*.jpeg +*.png +*.bmp +*.tif +*.tiff +*.heic +*.JPG +*.JPEG +*.PNG +*.BMP +*.TIF +*.TIFF +*.HEIC +*.mp4 +*.mov +*.MOV +*.avi +*.data +*.json +*.cfg + +*.rar +*.pkl +*.pickle +*.npy +*.csv + + + +# for tracking --------------------------------------------------------------- +tracking/.git +tracking/bakeup +tracking/.gitignore +tracking/result/**/*.mp4 +tracking/result/**/*.png +tracking/data/boxes_imgs/* +tracking/data/trackfeats/* +tracking/data/tracks/* +tracking/data/handlocal/* +ckpts/* +doc + + + + +# Datasets ------------------------------------------------------------------------------------------------------------- +coco/ +coco128/ +VOC/ + + + +# Neural Network weights ----------------------------------------------------------------------------------------------- +*.weights +*.pt +*.pth +*.pb +*.onnx +*.engine +*.mlmodel +*.torchscript +*.tflite +*.h5 +*.caffemodel +*_saved_model/ +*_web_model/ +*_openvino_model/ +*_paddle_model/ + + + + diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..35b88f6 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,131 @@ + + + + + + + + + + + + + + + + { + "associatedIndex": 2 +} + + + + { + "keyToString": { + "RunOnceActivity.OpenProjectViewOnStart": "true", + "RunOnceActivity.ShowReadmeOnStart": "true", + "WebServerToolWindowFactoryState": "true", + "node.js.detected.package.eslint": "true", + "node.js.detected.package.tslint": "true", + "node.js.selected.package.eslint": "(autodetect)", + "node.js.selected.package.tslint": "(autodetect)", + "settings.editor.selected.configurable": "com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable", + "vue.rearranger.settings.migration": "true" + } +} + + + + + + + + + + + + + + + + + + + 1744852597949 + + + + + + + + + + file://$PROJECT_DIR$/pipeline.py + 314 + + + + + + + + + \ No newline at end of file diff --git a/README.en.md b/README.en.md new file mode 100644 index 0000000..644a40d --- /dev/null +++ b/README.en.md @@ -0,0 +1,36 @@ +# YoloV5_track + +#### Description +{**When you're done, you can delete the content in this README and update the file with details for others getting started with your repository**} + +#### Software Architecture +Software architecture description + +#### Installation + +1. xxxx +2. xxxx +3. xxxx + +#### Instructions + +1. xxxx +2. xxxx +3. xxxx + +#### Contribution + +1. Fork the repository +2. Create Feat_xxx branch +3. Commit your code +4. Create Pull Request + + +#### Gitee Feature + +1. You can use Readme\_XXX.md to support different languages, such as Readme\_en.md, Readme\_zh.md +2. Gitee blog [blog.gitee.com](https://blog.gitee.com) +3. Explore open source project [https://gitee.com/explore](https://gitee.com/explore) +4. The most valuable open source project [GVP](https://gitee.com/gvp) +5. The manual of Gitee [https://gitee.com/help](https://gitee.com/help) +6. The most popular members [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/) diff --git a/README.md b/README.md new file mode 100644 index 0000000..7eec15e --- /dev/null +++ b/README.md @@ -0,0 +1,7 @@ +说明： + 该代码仓库是yolov5_track文件夹下代码的备份，这里的yolov5代码来至：https://github.com/ultralytics/yolov5 + gitee地址为：https://gitee.com/nanjing-yimao-information/dettrack +核心模块： + track_reid.py实现： + 1. yolov5检测 + 2. Bot-SORT用于目标跟踪 \ No newline at end of file diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/__pycache__/event_time_specify.cpython-39.pyc b/__pycache__/event_time_specify.cpython-39.pyc new file mode 100644 index 0000000..dd44679 Binary files /dev/null and b/__pycache__/event_time_specify.cpython-39.pyc differ diff --git a/__pycache__/export.cpython-312.pyc b/__pycache__/export.cpython-312.pyc new file mode 100644 index 0000000..2bfe215 Binary files /dev/null and b/__pycache__/export.cpython-312.pyc differ diff --git a/__pycache__/export.cpython-39.pyc b/__pycache__/export.cpython-39.pyc new file mode 100644 index 0000000..999dfae Binary files /dev/null and b/__pycache__/export.cpython-39.pyc differ diff --git a/__pycache__/imgs_inference.cpython-39.pyc b/__pycache__/imgs_inference.cpython-39.pyc new file mode 100644 index 0000000..981322a Binary files /dev/null and b/__pycache__/imgs_inference.cpython-39.pyc differ diff --git a/__pycache__/move_detect.cpython-312.pyc b/__pycache__/move_detect.cpython-312.pyc new file mode 100644 index 0000000..65fb932 Binary files /dev/null and b/__pycache__/move_detect.cpython-312.pyc differ diff --git a/__pycache__/move_detect.cpython-39.pyc b/__pycache__/move_detect.cpython-39.pyc new file mode 100644 index 0000000..02e5739 Binary files /dev/null and b/__pycache__/move_detect.cpython-39.pyc differ diff --git a/__pycache__/pipeline_01.cpython-312.pyc b/__pycache__/pipeline_01.cpython-312.pyc new file mode 100644 index 0000000..9900981 Binary files /dev/null and b/__pycache__/pipeline_01.cpython-312.pyc differ diff --git a/__pycache__/pipeline_01.cpython-39.pyc b/__pycache__/pipeline_01.cpython-39.pyc new file mode 100644 index 0000000..595214a Binary files /dev/null and b/__pycache__/pipeline_01.cpython-39.pyc differ diff --git a/__pycache__/track_reid.cpython-312.pyc b/__pycache__/track_reid.cpython-312.pyc new file mode 100644 index 0000000..78488df Binary files /dev/null and b/__pycache__/track_reid.cpython-312.pyc differ diff --git a/__pycache__/track_reid.cpython-39.pyc b/__pycache__/track_reid.cpython-39.pyc new file mode 100644 index 0000000..4b57508 Binary files /dev/null and b/__pycache__/track_reid.cpython-39.pyc differ diff --git a/bakeup/pipeline.py b/bakeup/pipeline.py new file mode 100644 index 0000000..5ae56f5 --- /dev/null +++ b/bakeup/pipeline.py @@ -0,0 +1,359 @@ +# -*- coding: utf-8 -*- +""" +Created on Sun Sep 29 08:59:21 2024 + +@author: ym +""" +import os +# import sys +import cv2 +import pickle +import numpy as np +from pathlib import Path +from scipy.spatial.distance import cdist +from track_reid import yolo_resnet_tracker, yolov10_resnet_tracker + +from tracking.dotrack.dotracks_back import doBackTracks +from tracking.dotrack.dotracks_front import doFrontTracks +from tracking.utils.drawtracks import plot_frameID_y2, draw_all_trajectories +from utils.getsource import get_image_pairs, get_video_pairs +from tracking.utils.read_data import read_similar + + +def save_subimgs(imgdict, boxes, spath, ctype, featdict = None): + ''' + 当前 box 特征和该轨迹前一个 box 特征的相似度，可用于和跟踪序列中的相似度进行比较 + ''' + boxes = boxes[np.argsort(boxes[:, 7])] + for i in range(len(boxes)): + simi = None + tid, fid, bid = int(boxes[i, 4]), int(boxes[i, 7]), int(boxes[i, 8]) + + if i>0: + _, fid0, bid0 = int(boxes[i-1, 4]), int(boxes[i-1, 7]), int(boxes[i-1, 8]) + if f"{fid0}_{bid0}" in featdict.keys() and f"{fid}_{bid}" in featdict.keys(): + feat0 = featdict[f"{fid0}_{bid0}"] + feat1 = featdict[f"{fid}_{bid}"] + simi = 1 - np.maximum(0.0, cdist(feat0[None, :], feat1[None, :], "cosine"))[0][0] + + img = imgdict[f"{fid}_{bid}"] + imgpath = spath / f"{ctype}_tid{tid}-{fid}-{bid}.png" + if simi is not None: + imgpath = spath / f"{ctype}_tid{tid}-{fid}-{bid}_sim{simi:.2f}.png" + + cv2.imwrite(imgpath, img) + + +def save_subimgs_1(imgdict, boxes, spath, ctype, simidict = None): + ''' + 当前 box 特征和该轨迹 smooth_feat 特征的相似度, yolo_resnet_tracker 函数中， + 采用该方式记录特征相似度 + ''' + for i in range(len(boxes)): + tid, fid, bid = int(boxes[i, 4]), int(boxes[i, 7]), int(boxes[i, 8]) + + key = f"{fid}_{bid}" + img = imgdict[key] + imgpath = spath / f"{ctype}_tid{tid}-{fid}-{bid}.png" + if simidict is not None and key in simidict.keys(): + imgpath = spath / f"{ctype}_tid{tid}-{fid}-{bid}_sim{simidict[key]:.2f}.png" + + cv2.imwrite(imgpath, img) + + +def pipeline( + eventpath, + savepath, + SourceType, + weights, + YoloVersion="V5" + ): + ''' + eventpath: 单个事件的存储路径 + + ''' + optdict = {} + optdict["weights"] = weights + + if SourceType == "video": + vpaths = get_video_pairs(eventpath) + elif SourceType == "image": + vpaths = get_image_pairs(eventpath) + event_tracks = [] + + ## 构造购物事件字典 + evtname = Path(eventpath).stem + barcode = evtname.split('_')[-1] if len(evtname.split('_'))>=2 \ + and len(evtname.split('_')[-1])>=8 \ + and evtname.split('_')[-1].isdigit() else '' + '''事件结果存储文件夹''' + if not savepath: + savepath = Path(__file__).resolve().parents[0] / "events_result" + + savepath_pipeline = Path(savepath) / Path("Yolos_Tracking") / evtname + + + """ShoppingDict pickle 文件保存地址 """ + savepath_spdict = Path(savepath) / "ShoppingDict_pkfile" + if not savepath_spdict.exists(): + savepath_spdict.mkdir(parents=True, exist_ok=True) + pf_path = Path(savepath_spdict) / Path(str(evtname)+".pickle") + + # if pf_path.exists(): + # print(f"Pickle file have saved: {evtname}.pickle") + # return + + '''====================== 构造 ShoppingDict 模块 =======================''' + ShoppingDict = {"eventPath": eventpath, + "eventName": evtname, + "barcode": barcode, + "eventType": '', # "input", "output", "other" + "frontCamera": {}, + "backCamera": {}, + "one2n": [] # + } + yrtDict = {} + + + procpath = Path(eventpath).joinpath('process.data') + if procpath.is_file(): + SimiDict = read_similar(procpath) + ShoppingDict["one2n"] = SimiDict['one2n'] + + + for vpath in vpaths: + '''================= 1. 构造相机事件字典 =================''' + CameraEvent = {"cameraType": '', # "front", "back" + "videoPath": '', + "imagePaths": [], + "yoloResnetTracker": [], + "tracking": [], + } + + if isinstance(vpath, list): + CameraEvent["imagePaths"] = vpath + bname = os.path.basename(vpath[0]) + if not isinstance(vpath, list): + CameraEvent["videoPath"] = vpath + bname = os.path.basename(vpath).split('.')[0] + if bname.split('_')[0] == "0" or bname.find('back')>=0: + CameraEvent["cameraType"] = "back" + if bname.split('_')[0] == "1" or bname.find('front')>=0: + CameraEvent["cameraType"] = "front" + + '''================= 2. 事件结果存储文件夹 =================''' + if isinstance(vpath, list): + savepath_pipeline_imgs = savepath_pipeline / Path("images") + else: + savepath_pipeline_imgs = savepath_pipeline / Path(str(Path(vpath).stem)) + + if not savepath_pipeline_imgs.exists(): + savepath_pipeline_imgs.mkdir(parents=True, exist_ok=True) + + savepath_pipeline_subimgs = savepath_pipeline / Path("subimgs") + if not savepath_pipeline_subimgs.exists(): + savepath_pipeline_subimgs.mkdir(parents=True, exist_ok=True) + + + '''================= 3. Yolo + Resnet + Tracker =================''' + optdict["source"] = vpath + optdict["save_dir"] = savepath_pipeline_imgs + optdict["is_save_img"] = True + optdict["is_save_video"] = True + + + if YoloVersion == "V5": + yrtOut = yolo_resnet_tracker(**optdict) + elif YoloVersion == "V10": + yrtOut = yolov10_resnet_tracker(**optdict) + + + yrtOut_save = [] + for frdict in yrtOut: + fr_dict = {} + for k, v in frdict.items(): + if k != "imgs": + fr_dict[k]=v + yrtOut_save.append(fr_dict) + CameraEvent["yoloResnetTracker"] = yrtOut_save + + # CameraEvent["yoloResnetTracker"] = yrtOut + + '''================= 4. tracking =================''' + '''(1) 生成用于 tracking 模块的 boxes、feats''' + bboxes = np.empty((0, 6), dtype=np.float64) + trackerboxes = np.empty((0, 9), dtype=np.float64) + trackefeats = {} + for frameDict in yrtOut: + tboxes = frameDict["tboxes"] + ffeats = frameDict["feats"] + + boxes = frameDict["bboxes"] + bboxes = np.concatenate((bboxes, np.array(boxes)), axis=0) + trackerboxes = np.concatenate((trackerboxes, np.array(tboxes)), axis=0) + for i in range(len(tboxes)): + fid, bid = int(tboxes[i, 7]), int(tboxes[i, 8]) + trackefeats.update({f"{fid}_{bid}": ffeats[f"{fid}_{bid}"]}) + + + '''(2) tracking, 后摄''' + if CameraEvent["cameraType"] == "back": + vts = doBackTracks(trackerboxes, trackefeats) + vts.classify() + event_tracks.append(("back", vts)) + + CameraEvent["tracking"] = vts + ShoppingDict["backCamera"] = CameraEvent + + yrtDict["backyrt"] = yrtOut + + '''(2) tracking, 前摄''' + if CameraEvent["cameraType"] == "front": + vts = doFrontTracks(trackerboxes, trackefeats) + vts.classify() + event_tracks.append(("front", vts)) + + CameraEvent["tracking"] = vts + ShoppingDict["frontCamera"] = CameraEvent + + yrtDict["frontyrt"] = yrtOut + + '''========================== 保存模块 =================================''' + '''(1) 保存 ShoppingDict 事件''' + with open(str(pf_path), 'wb') as f: + pickle.dump(ShoppingDict, f) + + '''(2) 保存 Tracking 输出的运动轨迹子图，并记录相似度''' + for CamerType, vts in event_tracks: + if len(vts.tracks)==0: continue + if CamerType == 'front': + # yolos = ShoppingDict["frontCamera"]["yoloResnetTracker"] + + yolos = yrtDict["frontyrt"] + ctype = 1 + if CamerType == 'back': + # yolos = ShoppingDict["backCamera"]["yoloResnetTracker"] + + yolos = yrtDict["backyrt"] + ctype = 0 + + imgdict, featdict, simidict = {}, {}, {} + for y in yolos: + imgdict.update(y["imgs"]) + featdict.update(y["feats"]) + simidict.update(y["featsimi"]) + + for track in vts.Residual: + if isinstance(track, np.ndarray): + save_subimgs(imgdict, track, savepath_pipeline_subimgs, ctype, featdict) + else: + save_subimgs(imgdict, track.slt_boxes, savepath_pipeline_subimgs, ctype, featdict) + + '''(3) 轨迹显示与保存''' + illus = [None, None] + for CamerType, vts in event_tracks: + if len(vts.tracks)==0: continue + + if CamerType == 'front': + edgeline = cv2.imread("./tracking/shopcart/cart_tempt/board_ftmp_line.png") + + h, w = edgeline.shape[:2] + # nh, nw = h//2, w//2 + # edgeline = cv2.resize(edgeline, (nw, nh), interpolation=cv2.INTER_AREA) + + img_tracking = draw_all_trajectories(vts, edgeline, savepath_pipeline, CamerType, draw5p=True) + illus[0] = img_tracking + + plt = plot_frameID_y2(vts) + plt.savefig(os.path.join(savepath_pipeline, "front_y2.png")) + + if CamerType == 'back': + edgeline = cv2.imread("./tracking/shopcart/cart_tempt/edgeline.png") + + h, w = edgeline.shape[:2] + # nh, nw = h//2, w//2 + # edgeline = cv2.resize(edgeline, (nw, nh), interpolation=cv2.INTER_AREA) + + img_tracking = draw_all_trajectories(vts, edgeline, savepath_pipeline, CamerType, draw5p=True) + illus[1] = img_tracking + + illus = [im for im in illus if im is not None] + if len(illus): + img_cat = np.concatenate(illus, axis = 1) + if len(illus)==2: + H, W = img_cat.shape[:2] + cv2.line(img_cat, (int(W/2), 0), (int(W/2), int(H)), (128, 128, 255), 3) + + trajpath = os.path.join(savepath_pipeline, "trajectory.png") + cv2.imwrite(trajpath, img_cat) + +def execute_pipeline(evtdir = r"D:\datasets\ym\后台数据\unzip", + source_type = "video", # video, image, + save_path = r"D:\work\result_pipeline", + yolo_ver = "V10", # V10, V5 + + weight_yolo_v5 = r'./ckpts/best_cls10_0906.pt' , + weight_yolo_v10 = r'./ckpts/best_v10s_width0375_1205.pt', + k=0 + ): + ''' + 运行函数 pipeline()，遍历事件文件夹，每个文件夹是一个事件 + ''' + parmDict = {} + parmDict["SourceType"] = source_type + parmDict["savepath"] = save_path + parmDict["YoloVersion"] = yolo_ver + if parmDict["YoloVersion"] == "V5": + parmDict["weights"] = weight_yolo_v5 + elif parmDict["YoloVersion"] == "V10": + parmDict["weights"] = weight_yolo_v10 + + evtdir = Path(evtdir) + errEvents = [] + for item in evtdir.iterdir(): + if item.is_dir(): + item = evtdir/Path("20250310-175352-741") + parmDict["eventpath"] = item + pipeline(**parmDict) + # try: + # pipeline(**parmDict) + # except Exception as e: + # errEvents.append(str(item)) + k+=1 + if k==1: + break + + errfile = os.path.join(parmDict["savepath"], 'error_events.txt') + with open(errfile, 'w', encoding='utf-8') as f: + for line in errEvents: + f.write(line + '\n') + +if __name__ == "__main__": + execute_pipeline() + + # spath_v10 = r"D:\work\result_pipeline_v10" + # spath_v5 = r"D:\work\result_pipeline_v5" + # execute_pipeline(save_path=spath_v10, yolo_ver="V10") + # execute_pipeline(save_path=spath_v5, yolo_ver="V5") + + datapath = r'/home/wqg/dataset/test_dataset/base_dataset/single_event/source/' + savepath = r'/home/wqg/dataset/pipeline/contrast/single_event_V5' + + + + + execute_pipeline(evtdir = datapath, + DataType = "raw", # raw, pkl + kk=1, + source_type = "video", # video, image, + save_path = savepath, + yolo_ver = "V10", # V10, V5 + weight_yolo_v5 = r'./ckpts/best_cls10_0906.pt' , + weight_yolo_v10 = r'./ckpts/best_v10s_width0375_1205.pt', + saveimages = False + ) + + + + + \ No newline at end of file diff --git a/bakeup/track_reid_20240515.py b/bakeup/track_reid_20240515.py new file mode 100644 index 0000000..5564cdc --- /dev/null +++ b/bakeup/track_reid_20240515.py @@ -0,0 +1,629 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Run YOLOv5 detection inference on images, videos, directories, globs, YouTube, webcam, streams, etc. + +Usage - sources: + $ python detect.py --weights yolov5s.pt --source 0 # webcam + img.jpg # image + vid.mp4 # video + screen # screenshot + path/ # directory + list.txt # list of images + list.streams # list of streams + 'path/*.jpg' # glob + 'https://youtu.be/Zgi9g1ksQHc' # YouTube + 'rtsp://example.com/media.mp4' # RTSP, RTMP, HTTP stream + +Usage - formats: + $ python detect.py --weights yolov5s.pt # PyTorch + yolov5s.torchscript # TorchScript + yolov5s.onnx # ONNX Runtime or OpenCV DNN with --dnn + yolov5s_openvino_model # OpenVINO + yolov5s.engine # TensorRT + yolov5s.mlmodel # CoreML (macOS-only) + yolov5s_saved_model # TensorFlow SavedModel + yolov5s.pb # TensorFlow GraphDef + yolov5s.tflite # TensorFlow Lite + yolov5s_edgetpu.tflite # TensorFlow Edge TPU + yolov5s_paddle_model # PaddlePaddle +""" + +import argparse +import csv +import os +import platform +import sys +from pathlib import Path +import glob +import numpy as np +import pickle +import torch + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[0] # YOLOv5 root directory +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) # add ROOT to PATH +ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative + +from models.common import DetectMultiBackend +from utils.dataloaders import IMG_FORMATS, VID_FORMATS, LoadImages, LoadScreenshots, LoadStreams +from utils.general import (LOGGER, Profile, check_file, check_img_size, check_imshow, check_requirements, colorstr, cv2, + increment_path, non_max_suppression, print_args, scale_boxes, strip_optimizer, xyxy2xywh) +from utils.torch_utils import select_device, smart_inference_mode + +'''集成跟踪模块，输出跟踪结果文件 .npy''' +# from ultralytics.engine.results import Boxes # Results +# from ultralytics.utils import IterableSimpleNamespace, yaml_load +from tracking.utils.plotting import Annotator, colors +from tracking.utils import Boxes, IterableSimpleNamespace, yaml_load, boxes_add_fid +from tracking.trackers import BOTSORT, BYTETracker +from tracking.utils.showtrack import drawtracks +from hands.hand_inference import hand_pose + +# ============================================================================= +# from tracking.trackers.reid.reid_interface import ReIDInterface +# from tracking.trackers.reid.config import config as ReIDConfig +# ReIDEncoder = ReIDInterface(ReIDConfig) +# ============================================================================= + +# tracker_yaml = r"./tracking/trackers/cfg/botsort.yaml" + +def init_trackers(tracker_yaml = None, bs=1): + """ + Initialize trackers for object tracking during prediction. + """ + # tracker_yaml = r"./tracking/trackers/cfg/botsort.yaml" + + TRACKER_MAP = {'bytetrack': BYTETracker, 'botsort': BOTSORT} + + cfg = IterableSimpleNamespace(**yaml_load(tracker_yaml)) + trackers = [] + for _ in range(bs): + tracker = TRACKER_MAP[cfg.tracker_type](args=cfg, frame_rate=30) + trackers.append(tracker) + + return trackers + + +@smart_inference_mode() +def run( + weights=ROOT / 'yolov5s.pt', # model path or triton URL + source=ROOT / 'data/images', # file/dir/URL/glob/screen/0(webcam) + + project=ROOT / 'runs/detect', # save results to project/name + name='exp', # save results to project/name + + tracker_yaml = "./tracking/trackers/cfg/botsort.yaml", + imgsz=(640, 640), # inference size (height, width) + conf_thres=0.25, # confidence threshold + iou_thres=0.45, # NMS IOU threshold + max_det=1000, # maximum detections per image + device='', # cuda device, i.e. 0 or 0,1,2,3 or cpu + view_img=False, # show results + save_txt=False, # save results to *.txt + save_csv=False, # save results in CSV format + save_conf=False, # save confidences in --save-txt labels + save_crop=False, # save cropped prediction boxes + nosave=False, # do not save images/videos + classes=None, # filter by class: --class 0, or --class 0 2 3 + agnostic_nms=False, # class-agnostic NMS + augment=False, # augmented inference + visualize=False, # visualize features + update=False, # update all models + exist_ok=False, # existing project/name ok, do not increment + line_thickness=3, # bounding box thickness (pixels) + hide_labels=False, # hide labels + hide_conf=False, # hide confidencesL + half=False, # use FP16 half-precision inference + dnn=False, # use OpenCV DNN for ONNX inference + vid_stride=1, # video frame-rate stride + data=ROOT / 'data/coco128.yaml', # dataset.yaml path +): + source = str(source) + # filename = os.path.split(source)[-1] + + save_img = not nosave and not source.endswith('.txt') # save inference images + is_file = Path(source).suffix[1:] in (IMG_FORMATS + VID_FORMATS) + is_url = source.lower().startswith(('rtsp://', 'rtmp://', 'http://', 'https://')) + webcam = source.isnumeric() or source.endswith('.streams') or (is_url and not is_file) + screenshot = source.lower().startswith('screen') + if is_url and is_file: + source = check_file(source) # download + + save_dir = Path(project) / Path(source).stem + if save_dir.exists(): + print(Path(source).stem) + # return + + save_dir = increment_path(Path(project) / name, exist_ok=exist_ok) # increment run + (save_dir / 'labels' if save_txt else save_dir).mkdir(parents=True, exist_ok=True) # make dir + else: + save_dir.mkdir(parents=True, exist_ok=True) + + # Load model + device = select_device(device) + model = DetectMultiBackend(weights, device=device, dnn=dnn, data=data, fp16=half) + stride, names, pt = model.stride, model.names, model.pt + imgsz = check_img_size(imgsz, s=stride) # check image size + + # Dataloader + bs = 1 # batch_size + + dataset = LoadImages(source, img_size=imgsz, stride=stride, auto=pt, vid_stride=vid_stride) + vid_path, vid_writer = [None] * bs, [None] * bs + + + # Run inference + model.warmup(imgsz=(1 if pt or model.triton else bs, 3, *imgsz)) # warmup + seen, dt = 0, (Profile(), Profile(), Profile()) + + tracker = init_trackers(tracker_yaml, bs)[0] + + handpose = hand_pose() + handlocals_dict = {} + + boxes_and_imgs = [] + track_boxes = np.empty((0, 9), dtype = np.float32) + det_boxes = np.empty((0, 9), dtype = np.float32) + + + features_dict = {} + for path, im, im0s, vid_cap, s in dataset: + if save_img and 'imgshow' not in locals().keys(): + imgshow = im0s.copy() + + ## ============================= tracking 功能只处理视频，writed by WQG + if dataset.mode == 'image': + continue + + with dt[0]: + im = torch.from_numpy(im).to(model.device) + im = im.half() if model.fp16 else im.float() # uint8 to fp16/32 + im /= 255 # 0 - 255 to 0.0 - 1.0 + if len(im.shape) == 3: + im = im[None] # expand for batch dim + + # Inference + with dt[1]: + visualize = increment_path(save_dir / Path(path).stem, mkdir=True) if visualize else False + pred = model(im, augment=augment, visualize=visualize) + + # NMS + with dt[2]: + pred = non_max_suppression(pred, conf_thres, iou_thres, classes, agnostic_nms, max_det=max_det) + + # Second-stage classifier (optional) + # pred = utils.general.apply_classifier(pred, classifier_model, im, im0s) + + # Define the path for the CSV file + # csv_path = save_dir / 'predictions.csv' + + # Create or append to the CSV file + # def write_to_csv(image_name, prediction, confidence): + # data = {'Image Name': image_name, 'Prediction': prediction, 'Confidence': confidence} + # with open(csv_path, mode='a', newline='') as f: + # writer = csv.DictWriter(f, fieldnames=data.keys()) + # if not csv_path.is_file(): + # writer.writeheader() + # writer.writerow(data) + + # Process predictions + for i, det in enumerate(pred): # per image + seen += 1 + if webcam: # batch_size >= 1 + p, im0, frame = path[i], im0s[i].copy(), dataset.count + s += f'{i}: ' + else: + p, im0, frame = path, im0s.copy(), getattr(dataset, 'frame', 0) + + im0_ant = im0.copy() + + p = Path(p) # to Path + save_path = str(save_dir / p.name) # im.jpg + s += '%gx%g ' % im.shape[2:] # print string + + annotator = Annotator(im0_ant, line_width=line_thickness, example=str(names)) + + nd = len(det) + if nd: + # Rescale boxes from img_size to im0 size + det[:, :4] = scale_boxes(im.shape[2:], det[:, :4], im0.shape).round() + + det = det.cpu().numpy() + det = np.concatenate([det[:, :4], np.arange(nd).reshape(-1, 1), det[:, 4:]], axis=-1) + + '''FeatFlag为相对于上一帧boxes，当前boxes是否为静止的标志。''' + # def static_estimate(box1, box2, TH1=8, TH2=12): + # dij_abs = max(np.abs(box1 - box2)) + # dij_euc = max([np.linalg.norm((box1[:2] - box2[:2])), + # np.linalg.norm((box1[2:4] - box2[2:4])) + # ]) + # if dij_abs < TH1 and dij_euc < TH2: + # return True + # else: + # return False + + # FeatFlag = [-1] * nd + # if len(boxes_and_imgs): + # detj = boxes_and_imgs[-1][0] + # frmj = boxes_and_imgs[-1][-1] + # for ii in range(nd): + # ## flag 中保存的是box索引 + # condt1 = frame-frmj==1 + # flag = [idx for jj, idx in enumerate(detj[:, 4]) if condt1 and static_estimate(det[ii, :4], detj[jj, :4])] + # if len(flag) == 1: + # FeatFlag[ii] = flag[0] + + + boxes_and_imgs.append((det, im0, frame)) + + ## ================================================================ writed by WQG + '''tracks: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] + 这里，frame_index 也可以用视频的 帧ID 代替, box_index 保持不变 + ''' + + det_tracking = Boxes(det, im0.shape).cpu().numpy() + tracks = tracker.update(det_tracking, im0) + + # detbox = [tlwh2tlbr(x._tlwh).tolist() + [x.track_id, x.score, x.cls, x.frame_id, x.idx] + # for x in tracker.tracked_stracks if x.is_activated] + if len(tracks): + ''' + tracks: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] + 0 1 2 3 4 5 6 7 8 + ''' + tracks[:, 7] = dataset.frame + + + '''================== 1. 提取手势位置 ===================''' + # idx_0 = tracks[:, 6].astype(np.int_) == 0 + # hn = 0 + # for j, index in enumerate(idx_0): + # if index: + # track = tracks[j, :] + # hand_local, imgshow = handpose.get_hand_local(track, im0) + # handlocals_dict.update({int(track[7]): {int(track[8]): hand_local}}) + + # # '''yoloV5和手势检测的召回率并不一直，用hand_local代替tracks中手部的(x1, y1, x2, y2)，会使得两种坐标方式混淆''' + # # if hand_local: tracks[j, :4] = hand_local + + # hn += 1 + # cv2.imwrite(f"D:\DeepLearning\yolov5\hands\images\{Path(source).stem}_{int(track[7])}_{hn}.png", imgshow) + + + '''================== 2. 存储轨迹信息 ===================''' + track_boxes = np.concatenate([track_boxes, tracks], axis=0) + + # det_boxes = np.concatenate([det_boxes, detbox], axis=0) + '''================== 3. 存储轨迹 REID 特征 =============''' + + def crop_img(track, image): + tlbr = track.tlwh_to_tlbr(track._tlwh).astype(np.int_) + + H, W = image.shape[:2] + tlbr[0] = max(0, tlbr[0]) + tlbr[1] = max(0, tlbr[1]) + tlbr[2] = min(W - 1, tlbr[2]) + tlbr[3] = min(H - 1, tlbr[3]) + + img = image[tlbr[1]:tlbr[3], tlbr[0]:tlbr[2], :] + # cv2.imwrite(f"./runs/imgs/{int(track.idx)}.png", img) + return img + + feat_dict_1 = {f'{int(x.idx)}_img': crop_img(x, im0) for x in tracker.tracked_stracks if x.is_activated} + feat_dict = {int(x.idx): x.curr_feat for x in tracker.tracked_stracks if x.is_activated} + feat_dict.update(feat_dict_1) + + features_dict.update({int(dataset.frame): feat_dict}) + + # det_anno = tracks.copy() + # else: + # idmark = -1 * np.ones([det.shape[0], 1]) + # det_anno = np.concatenate([det[:,:4], idmark, det[:, 4:]], axis=1) + + + + + for *xyxy, id, conf, cls, fid, bid in reversed(tracks): + name = ('' if id==-1 else f'id:{int(id)} ') + names[int(cls)] + label = None if hide_labels else (name if hide_conf else f'{name} {conf:.2f}') + + if id >=0 and cls==0: + color = colors(int(cls), True) + elif id >=0 and cls!=0: + color = colors(int(id), True) + else: + color = colors(19, True) # 19为调色板的最后一个元素 + + annotator.box_label(xyxy, label, color=color) + + # Save results (image and video with tracking) + im0 = annotator.result() + save_path_img, ext = os.path.splitext(save_path) + if save_img: + if dataset.mode == 'image': + imgpath = save_path_img + f"_{dataset}.png" + else: + imgpath = save_path_img + f"_{dataset.frame}.png" + + cv2.imwrite(Path(imgpath), im0) + + if vid_path[i] != save_path: # new video + vid_path[i] = save_path + if isinstance(vid_writer[i], cv2.VideoWriter): + vid_writer[i].release() # release previous video writer + if vid_cap: # video + fps = vid_cap.get(cv2.CAP_PROP_FPS) + w = int(vid_cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + h = int(vid_cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + else: # stream + fps, w, h = 30, im0.shape[1], im0.shape[0] + save_path = str(Path(save_path).with_suffix('.mp4')) # force *.mp4 suffix on results videos + vid_writer[i] = cv2.VideoWriter(save_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h)) + vid_writer[i].write(im0) + + # Print time (inference-only) + LOGGER.info(f"{s}{'' if len(det) else '(no detections), '}{dt[1].dt * 1E3:.1f}ms") + + + ## ======================================================================== written by WQG + ## track_boxes: Array, [x1, y1, x2, y2, track_id, score, cls, frame_index, box_id] + + '''上面保存了检测结果是视频和图像，以下还保存五种类型的数据''' + filename = os.path.split(save_path_img)[-1] + # file, ext = os.path.splitext(filename) +# ============================================================================= +# fileElements = filename.split('_') +# if len(fileElements) == 6 and len(fileElements[3])==1: +# barcode = fileElements[1] +# camera = fileElements[3] +# elif len(fileElements) == 7 and len(fileElements[3])==1: +# barcode = fileElements[2] +# camera = fileElements[4] +# else: +# barcode = '' +# camera = '' +# ============================================================================= + + + '''======================== 1. save in './run/detect/' ====================''' + if source.find("front") >= 0: + carttemp = cv2.imread("./tracking/shopcart/cart_tempt/board_ftmp_line.png") + else: + carttemp = cv2.imread("./tracking/shopcart/cart_tempt/edgeline.png") + + + imgshow = drawtracks(track_boxes, carttemp) + showpath_1 = save_path_img + "_show.png" + cv2.imwrite(Path(showpath_1), imgshow) + + + '''======================== 2. save boxes and raw images ==================''' + # boxes_imgs_dir = Path('./tracking/data/boxes_imgs/') + # if not boxes_imgs_dir.exists(): + # boxes_imgs_dir.mkdir(parents=True, exist_ok=True) + # boxes_imgs_path = boxes_imgs_dir.joinpath(f'{filename}.pkl') + # with open(boxes_imgs_path, 'wb') as file: + # pickle.dump(boxes_and_imgs, file) + + '''======================== 3. save tracks data ===========================''' + tracks_dir = Path('./tracking/data/tracks/') + if not tracks_dir.exists(): + tracks_dir.mkdir(parents=True, exist_ok=True) + tracks_path = tracks_dir.joinpath(filename + "_track.npy") + np.save(tracks_path, track_boxes) + + detect_path = tracks_dir.joinpath(filename + "_detect.npy") + np.save(detect_path, det_boxes) + + '''======================== 4. save reid features data ====================''' + feats_dir = Path('./tracking/data/trackfeats/') + if not feats_dir.exists(): + feats_dir.mkdir(parents=True, exist_ok=True) + feats_path = feats_dir.joinpath(f'{filename}.pkl') + with open(feats_path, 'wb') as file: + pickle.dump(features_dict, file) + + '''======================== 5. save hand_local data ==================''' + # handlocal_dir = Path('./tracking/data/handlocal/') + # if not handlocal_dir.exists(): + # handlocal_dir.mkdir(parents=True, exist_ok=True) + # handlocal_path = handlocal_dir.joinpath(f'{filename}.pkl') + # with open(handlocal_path, 'wb') as file: + # pickle.dump(handlocals_dict, file) + + + + + # Print results + t = tuple(x.t / seen * 1E3 for x in dt) # speeds per image + LOGGER.info(f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {(1, 3, *imgsz)}' % t) + if save_txt or save_img: + s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else '' + LOGGER.info(f"Results saved to {colorstr('bold', save_dir)}{s}") + if update: + strip_optimizer(weights[0]) # update model (to fix SourceChangeWarning) + + +def parse_opt(): + modelpath = ROOT / 'ckpts/best_yolov5m_250000.pt' # 'ckpts/best_15000_0908.pt', 'ckpts/yolov5s.pt', 'ckpts/best_20000_cls30.pt' + + '''datapath为视频文件目录或视频文件''' + datapath = r"D:/datasets/ym/videos/标记视频/" # ROOT/'data/videos', ROOT/'data/images' images + # datapath = r"D:\datasets\ym\highvalue\videos" + # datapath = r"D:/dcheng/videos/" + # modelpath = ROOT / 'ckpts/yolov5s.pt' + + parser = argparse.ArgumentParser() + parser.add_argument('--weights', nargs='+', type=str, default=modelpath, help='model path or triton URL') # 'yolov5s.pt', best_15000_0908.pt + parser.add_argument('--source', type=str, default=datapath, help='file/dir/URL/glob/screen/0(webcam)') # images, videos + parser.add_argument('--data', type=str, default=ROOT / 'data/coco128.yaml', help='(optional) dataset.yaml path') + parser.add_argument('--imgsz', '--img', '--img-size', nargs='+', type=int, default=[640], help='inference size h,w') + parser.add_argument('--conf-thres', type=float, default=0.25, help='confidence threshold') + parser.add_argument('--iou-thres', type=float, default=0.45, help='NMS IoU threshold') + parser.add_argument('--max-det', type=int, default=1000, help='maximum detections per image') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--view-img', action='store_true', help='show results') + parser.add_argument('--save-txt', action='store_true', help='save results to *.txt') + parser.add_argument('--save-csv', action='store_true', help='save results in CSV format') + parser.add_argument('--save-conf', action='store_true', help='save confidences in --save-txt labels') + parser.add_argument('--save-crop', action='store_true', help='save cropped prediction boxes') + parser.add_argument('--nosave', action='store_true', help='do not save images/videos') + parser.add_argument('--classes', nargs='+', type=int, help='filter by class: --classes 0, or --classes 0 2 3') + parser.add_argument('--agnostic-nms', action='store_true', help='class-agnostic NMS') + parser.add_argument('--augment', action='store_true', help='augmented inference') + parser.add_argument('--visualize', action='store_true', help='visualize features') + parser.add_argument('--update', action='store_true', help='update all models') + parser.add_argument('--project', default=ROOT / 'runs/detect', help='save results to project/name') + parser.add_argument('--name', default='exp', help='save results to project/name') + parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment') + parser.add_argument('--line-thickness', default=3, type=int, help='bounding box thickness (pixels)') + parser.add_argument('--hide-labels', default=False, action='store_true', help='hide labels') + parser.add_argument('--hide-conf', default=False, action='store_true', help='hide confidences') + parser.add_argument('--half', action='store_true', help='use FP16 half-precision inference') + parser.add_argument('--dnn', action='store_true', help='use OpenCV DNN for ONNX inference') + parser.add_argument('--vid-stride', type=int, default=1, help='video frame-rate stride') + opt = parser.parse_args() + opt.imgsz *= 2 if len(opt.imgsz) == 1 else 1 # expand + print_args(vars(opt)) + return opt + +def main_loop_folders(opt): + check_requirements(ROOT / 'requirements.txt', exclude=('tensorboard', 'thop')) + + # path1 = r"D:\datasets\ym\videos\标记视频" + + path2 = r"D:\datasets\ym\永辉双摄视频\加购_前摄\videos_front" + # path3 = r"D:\datasets\ym\永辉双摄视频\加购_后摄\videos_back" + # path4 = r"D:\datasets\ym\永辉双摄视频\退购_前摄\videos_front" + # path5 = r"D:\datasets\ym\永辉双摄视频\退购_后摄\videos_back" + + path6 = r"D:\datasets\ym\测试数据20240328\front" + path7 = r"D:\datasets\ym\测试数据20240328\back" + + '''列表paths内的元素为视频文件夹，该文件夹下元素为视频文件''' + paths = [path2, path7] # [path1, path2, path3, path4, path5] + + optdict = vars(opt) + k1, k2 = 0, 0 + for p in paths: + files = [] + if os.path.isdir(p): + files.extend(sorted(glob.glob(os.path.join(p, '*.*')))) + for file in files: + file = r"D:\datasets\ym\测试数据20240328\front\112954521-7dd5ddad-922a-427b-b59e-a593e95e6ff4_front.mp4" + + optdict["source"] = file + run(**optdict) + + k2 += 1 + if k2 == 1: + break + + elif os.path.isfile(p): + run(**optdict) + + k1 += 1 + if k1 == 1: + break + +def find_files_in_nested_dirs(root_dir): + all_files = [] + extensions = ['.mp4'] + for dirpath, dirnames, filenames in os.walk(root_dir): + for filename in filenames: + file, ext = os.path.splitext(filename) + if ext in extensions: + all_files.append(os.path.join(dirpath, filename)) + return all_files + +print('=======') + +def main(opt): + check_requirements(ROOT / 'requirements.txt', exclude=('tensorboard', 'thop')) + + p = r"D:\datasets\ym\永辉测试数据_202404\20240402" + optdict = vars(opt) + files = [] + k = 0 + + all_files = find_files_in_nested_dirs(p) + + if os.path.isdir(p): + files.extend(sorted(glob.glob(os.path.join(p, '*.*')))) + for file in files: + optdict["source"] = file + run(**optdict) + + k += 1 + if k == 2: + break + elif os.path.isfile(p): + run(**vars(opt)) + +def main_loop(opt): + check_requirements(ROOT / 'requirements.txt', exclude=('tensorboard', 'thop')) + + optdict = vars(opt) + + # p = r"D:\datasets\ym\永辉测试数据_比对" + p = r"D:\datasets\ym\广告板遮挡测试\8" + # p = r"D:\datasets\ym\videos\标记视频" + # p = r"D:\datasets\ym\实验室测试" + + k = 0 + if os.path.isdir(p): + files = find_files_in_nested_dirs(p) + + files = [r"D:\datasets\ym\videos\标记视频\test_20240402-173935_6920152400975_back_174037372.mp4", + r"D:\datasets\ym\videos\标记视频\test_20240402-173935_6920152400975_front_174037379.mp4" + ] + + files = [r"D:\datasets\ym\广告板遮挡测试\8\2500441577966_20240508-175946_front_addGood_70f75407b7ae_155_17788571404.mp4"] + for file in files: + optdict["source"] = file + run(**optdict) + + k += 1 + if k == 2: + break + elif os.path.isfile(p): + optdict["source"] = p + run(**vars(opt)) + + + + +if __name__ == '__main__': + opt = parse_opt() + + # main_loop_folders(opt) + # main(opt) + main_loop(opt) + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/bclass.py b/bclass.py new file mode 100644 index 0000000..24cb443 --- /dev/null +++ b/bclass.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +""" +Created on Fri Nov 15 16:23:03 2024 + +@author: ym +""" + +class CamEvent: + def __init__(self, datapath): + self.data_path = datapath + self.bboxes = None + self.bfeats = None + self.tboxes = None + self.tfeats = None + + + +class ShopEvent: + def __init__(self, eventpath, stdpath): + self.barcode = "" + self.event_path = eventpath + self.event_type = self.get_event_type(eventpath) + + self.FrontEvent = "" + self.BackEvent = "" + self.fusion_boxes = None + self.fusion_feats = None + self.stdfeats = self.get_stdfeats(stdpath) + self.weight = None + self.imu = None + + def get_event_type(self, eventpath): + pass + + + + def get_stdfeats(self, stdpath): + pass + + + + + + + + + + + + \ No newline at end of file diff --git a/benchmarks.py b/benchmarks.py new file mode 100644 index 0000000..b590ff6 --- /dev/null +++ b/benchmarks.py @@ -0,0 +1,174 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Run YOLOv5 benchmarks on all supported export formats + +Format | `export.py --include` | Model +--- | --- | --- +PyTorch | - | yolov5s.pt +TorchScript | `torchscript` | yolov5s.torchscript +ONNX | `onnx` | yolov5s.onnx +OpenVINO | `openvino` | yolov5s_openvino_model/ +TensorRT | `engine` | yolov5s.engine +CoreML | `coreml` | yolov5s.mlmodel +TensorFlow SavedModel | `saved_model` | yolov5s_saved_model/ +TensorFlow GraphDef | `pb` | yolov5s.pb +TensorFlow Lite | `tflite` | yolov5s.tflite +TensorFlow Edge TPU | `edgetpu` | yolov5s_edgetpu.tflite +TensorFlow.js | `tfjs` | yolov5s_web_model/ + +Requirements: + $ pip install -r requirements.txt coremltools onnx onnx-simplifier onnxruntime openvino-dev tensorflow-cpu # CPU + $ pip install -r requirements.txt coremltools onnx onnx-simplifier onnxruntime-gpu openvino-dev tensorflow # GPU + $ pip install -U nvidia-tensorrt --index-url https://pypi.ngc.nvidia.com # TensorRT + +Usage: + $ python benchmarks.py --weights yolov5s.pt --img 640 +""" + +import argparse +import platform +import sys +import time +from pathlib import Path + +import pandas as pd + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[0] # YOLOv5 root directory +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) # add ROOT to PATH +# ROOT = ROOT.relative_to(Path.cwd()) # relative + +import export +from models.experimental import attempt_load +from models.yolo import SegmentationModel +from segment.val import run as val_seg +from utils import notebook_init +from utils.general import LOGGER, check_yaml, file_size, print_args +from utils.torch_utils import select_device +from val import run as val_det + + +def run( + weights=ROOT / 'yolov5s.pt', # weights path + imgsz=640, # inference size (pixels) + batch_size=1, # batch size + data=ROOT / 'data/coco128.yaml', # dataset.yaml path + device='', # cuda device, i.e. 0 or 0,1,2,3 or cpu + half=False, # use FP16 half-precision inference + test=False, # test exports only + pt_only=False, # test PyTorch only + hard_fail=False, # throw error on benchmark failure +): + y, t = [], time.time() + device = select_device(device) + model_type = type(attempt_load(weights, fuse=False)) # DetectionModel, SegmentationModel, etc. + for i, (name, f, suffix, cpu, gpu) in export.export_formats().iterrows(): # index, (name, file, suffix, CPU, GPU) + try: + assert i not in (9, 10), 'inference not supported' # Edge TPU and TF.js are unsupported + assert i != 5 or platform.system() == 'Darwin', 'inference only supported on macOS>=10.13' # CoreML + if 'cpu' in device.type: + assert cpu, 'inference not supported on CPU' + if 'cuda' in device.type: + assert gpu, 'inference not supported on GPU' + + # Export + if f == '-': + w = weights # PyTorch format + else: + w = export.run(weights=weights, + imgsz=[imgsz], + include=[f], + batch_size=batch_size, + device=device, + half=half)[-1] # all others + assert suffix in str(w), 'export failed' + + # Validate + if model_type == SegmentationModel: + result = val_seg(data, w, batch_size, imgsz, plots=False, device=device, task='speed', half=half) + metric = result[0][7] # (box(p, r, map50, map), mask(p, r, map50, map), *loss(box, obj, cls)) + else: # DetectionModel: + result = val_det(data, w, batch_size, imgsz, plots=False, device=device, task='speed', half=half) + metric = result[0][3] # (p, r, map50, map, *loss(box, obj, cls)) + speed = result[2][1] # times (preprocess, inference, postprocess) + y.append([name, round(file_size(w), 1), round(metric, 4), round(speed, 2)]) # MB, mAP, t_inference + except Exception as e: + if hard_fail: + assert type(e) is AssertionError, f'Benchmark --hard-fail for {name}: {e}' + LOGGER.warning(f'WARNING ⚠️ Benchmark failure for {name}: {e}') + y.append([name, None, None, None]) # mAP, t_inference + if pt_only and i == 0: + break # break after PyTorch + + # Print results + LOGGER.info('\n') + parse_opt() + notebook_init() # print system info + c = ['Format', 'Size (MB)', 'mAP50-95', 'Inference time (ms)'] if map else ['Format', 'Export', '', ''] + py = pd.DataFrame(y, columns=c) + LOGGER.info(f'\nBenchmarks complete ({time.time() - t:.2f}s)') + LOGGER.info(str(py if map else py.iloc[:, :2])) + if hard_fail and isinstance(hard_fail, str): + metrics = py['mAP50-95'].array # values to compare to floor + floor = eval(hard_fail) # minimum metric floor to pass, i.e. = 0.29 mAP for YOLOv5n + assert all(x > floor for x in metrics if pd.notna(x)), f'HARD FAIL: mAP50-95 < floor {floor}' + return py + + +def test( + weights=ROOT / 'yolov5s.pt', # weights path + imgsz=640, # inference size (pixels) + batch_size=1, # batch size + data=ROOT / 'data/coco128.yaml', # dataset.yaml path + device='', # cuda device, i.e. 0 or 0,1,2,3 or cpu + half=False, # use FP16 half-precision inference + test=False, # test exports only + pt_only=False, # test PyTorch only + hard_fail=False, # throw error on benchmark failure +): + y, t = [], time.time() + device = select_device(device) + for i, (name, f, suffix, gpu) in export.export_formats().iterrows(): # index, (name, file, suffix, gpu-capable) + try: + w = weights if f == '-' else \ + export.run(weights=weights, imgsz=[imgsz], include=[f], device=device, half=half)[-1] # weights + assert suffix in str(w), 'export failed' + y.append([name, True]) + except Exception: + y.append([name, False]) # mAP, t_inference + + # Print results + LOGGER.info('\n') + parse_opt() + notebook_init() # print system info + py = pd.DataFrame(y, columns=['Format', 'Export']) + LOGGER.info(f'\nExports complete ({time.time() - t:.2f}s)') + LOGGER.info(str(py)) + return py + + +def parse_opt(): + parser = argparse.ArgumentParser() + parser.add_argument('--weights', type=str, default=ROOT / 'yolov5s.pt', help='weights path') + parser.add_argument('--imgsz', '--img', '--img-size', type=int, default=640, help='inference size (pixels)') + parser.add_argument('--batch-size', type=int, default=1, help='batch size') + parser.add_argument('--data', type=str, default=ROOT / 'data/coco128.yaml', help='dataset.yaml path') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--half', action='store_true', help='use FP16 half-precision inference') + parser.add_argument('--test', action='store_true', help='test exports only') + parser.add_argument('--pt-only', action='store_true', help='test PyTorch only') + parser.add_argument('--hard-fail', nargs='?', const=True, default=False, help='Exception on error or < min metric') + opt = parser.parse_args() + opt.data = check_yaml(opt.data) # check YAML + print_args(vars(opt)) + return opt + + +def main(opt): + test(**vars(opt)) if opt.test else run(**vars(opt)) + + +if __name__ == '__main__': + opt = parse_opt() + main(opt) diff --git a/contrast/__init__.py b/contrast/__init__.py new file mode 100644 index 0000000..0cd4367 --- /dev/null +++ b/contrast/__init__.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Created on Thu Sep 26 08:53:58 2024 + +@author: ym +""" + diff --git a/contrast/__pycache__/__init__.cpython-312.pyc b/contrast/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..b2010b8 Binary files /dev/null and b/contrast/__pycache__/__init__.cpython-312.pyc differ diff --git a/contrast/__pycache__/__init__.cpython-39.pyc b/contrast/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..6c5ac4a Binary files /dev/null and b/contrast/__pycache__/__init__.cpython-39.pyc differ diff --git a/contrast/__pycache__/config.cpython-39.pyc b/contrast/__pycache__/config.cpython-39.pyc new file mode 100644 index 0000000..79cd9dd Binary files /dev/null and b/contrast/__pycache__/config.cpython-39.pyc differ diff --git a/contrast/__pycache__/event_test.cpython-312.pyc b/contrast/__pycache__/event_test.cpython-312.pyc new file mode 100644 index 0000000..1cfb77a Binary files /dev/null and b/contrast/__pycache__/event_test.cpython-312.pyc differ diff --git a/contrast/__pycache__/event_test.cpython-39.pyc b/contrast/__pycache__/event_test.cpython-39.pyc new file mode 100644 index 0000000..4f87a3f Binary files /dev/null and b/contrast/__pycache__/event_test.cpython-39.pyc differ diff --git a/contrast/__pycache__/feat_inference.cpython-39.pyc b/contrast/__pycache__/feat_inference.cpython-39.pyc new file mode 100644 index 0000000..07eb78f Binary files /dev/null and b/contrast/__pycache__/feat_inference.cpython-39.pyc differ diff --git a/contrast/__pycache__/genfeats.cpython-312.pyc b/contrast/__pycache__/genfeats.cpython-312.pyc new file mode 100644 index 0000000..acc75fa Binary files /dev/null and b/contrast/__pycache__/genfeats.cpython-312.pyc differ diff --git a/contrast/__pycache__/genfeats.cpython-39.pyc b/contrast/__pycache__/genfeats.cpython-39.pyc new file mode 100644 index 0000000..274b653 Binary files /dev/null and b/contrast/__pycache__/genfeats.cpython-39.pyc differ diff --git a/contrast/__pycache__/one2n_contrast.cpython-312.pyc b/contrast/__pycache__/one2n_contrast.cpython-312.pyc new file mode 100644 index 0000000..158320b Binary files /dev/null and b/contrast/__pycache__/one2n_contrast.cpython-312.pyc differ diff --git a/contrast/__pycache__/one2n_contrast.cpython-39.pyc b/contrast/__pycache__/one2n_contrast.cpython-39.pyc new file mode 100644 index 0000000..6130b3f Binary files /dev/null and b/contrast/__pycache__/one2n_contrast.cpython-39.pyc differ diff --git a/contrast/__pycache__/test_ori.cpython-39.pyc b/contrast/__pycache__/test_ori.cpython-39.pyc new file mode 100644 index 0000000..949f1ad Binary files /dev/null and b/contrast/__pycache__/test_ori.cpython-39.pyc differ diff --git a/contrast/event_test.py b/contrast/event_test.py new file mode 100644 index 0000000..d5108fc --- /dev/null +++ b/contrast/event_test.py @@ -0,0 +1,374 @@ +# -*- coding: utf-8 -*- +""" +Created on Mon Dec 16 18:56:18 2024 + +@author: ym +""" +import os +import cv2 +import json +import numpy as np +import matplotlib.pyplot as plt +from pathlib import Path + +from matplotlib import rcParams +from matplotlib.font_manager import FontProperties +from scipy.spatial.distance import cdist +from utils.event import ShoppingEvent, save_data +from utils.calsimi import calsimi_vs_stdfeat_new, get_topk_percent, cluster +from utils.tools import get_evtList +import pickle + +rcParams['font.sans-serif'] = ['SimHei'] # 用黑体显示中文 +rcParams['axes.unicode_minus'] = False # 正确显示负号 + +'''*********** USearch ***********''' +def read_usearch(): + stdFeaturePath = r"D:\contrast\stdlib\v11_test.json" + stdBarcode = [] + stdlib = {} + with open(stdFeaturePath, 'r', encoding='utf-8') as f: + data = json.load(f) + for dic in data['total']: + barcode = dic['key'] + feature = np.array(dic['value']) + stdBarcode.append(barcode) + stdlib[barcode] = feature + + return stdlib + +def get_eventlist_errortxt(evtpaths): + ''' + 读取一次测试中的错误事件 + ''' + text1 = "one_2_Small_n_Error.txt" + text2 = "one_2_Big_N_Error.txt" + events = [] + text = (text1, text2) + for txt in text: + txtfile = os.path.join(evtpaths, txt) + with open(txtfile, "r") as f: + lines = f.readlines() + for i, line in enumerate(lines): + line = line.strip() + if line: + fpath=os.path.join(evtpaths, line) + events.append(fpath) + + + + events = list(set(events)) + + return events + +def save_eventdata(): + evtpaths = r"/home/wqg/dataset/test_dataset/performence_dataset/" + events = get_eventlist_errortxt(evtpaths) + + '''定义当前事件存储地址及生成相应文件件''' + resultPath = r"\\192.168.1.28\share\测试视频数据以及日志\算法全流程测试\202412\result\single_event" + for evtpath in events: + event = ShoppingEvent(evtpath) + save_data(event, resultPath) + + print(event.evtname) + + + +# def get_topk_percent(data, k): +# """ +# 获取数据中最大的 k% 的元素 +# """ +# # 将数据转换为 NumPy 数组 +# if isinstance(data, list): +# data = np.array(data) + +# percentile = np.percentile(data, 100-k) +# top_k_percent = data[data >= percentile] + +# return top_k_percent +# def cluster(data, thresh=0.15): +# # data = np.array([0.1, 0.13, 0.7, 0.2, 0.8, 0.52, 0.3, 0.7, 0.85, 0.58]) +# # data = np.array([0.1, 0.13, 0.2, 0.3]) +# # data = np.array([0.1]) + +# if isinstance(data, list): +# data = np.array(data) + +# data1 = np.sort(data) +# cluter, Cluters, = [data1[0]], [] +# for i in range(1, len(data1)): +# if data1[i] - data1[i-1]< thresh: +# cluter.append(data1[i]) +# else: +# Cluters.append(cluter) +# cluter = [data1[i]] +# Cluters.append(cluter) + +# clt_center = [] +# for clt in Cluters: +# ## 是否应该在此处限制一个聚类中的最小轨迹样本数，应该将该因素放在轨迹分析中 +# # if len(clt)>=3: +# # clt_center.append(np.mean(clt)) +# clt_center.append(np.mean(clt)) + +# # print(clt_center) + +# return clt_center + +# def calsimi_vs_stdfeat_new(event, stdfeat): +# '''事件与标准库的对比策略 +# 该比对策略是否可以拓展到事件与事件的比对？ +# ''' + + +# def calsiml(feat1, feat2, topkp=75, cluth=0.15): +# '''轨迹样本和标准特征集样本相似度的选择策略''' +# matrix = 1 - cdist(feat1, feat2, 'cosine') +# simi_max = [] +# for i in range(len(matrix)): +# sim = np.mean(get_topk_percent(matrix[i, :], topkp)) +# simi_max.append(sim) +# cltc_max = cluster(simi_max, cluth) +# Simi = max(cltc_max) + +# ## cltc_max为空属于编程考虑不周，应予以排查解决 +# # if len(cltc_max): +# # Simi = max(cltc_max) +# # else: +# # Simi = 0 #不应该走到该处 + + +# return Simi + + +# front_boxes = np.empty((0, 9), dtype=np.float64) ##和类doTracks兼容 +# front_feats = np.empty((0, 256), dtype=np.float64) ##和类doTracks兼容 +# for i in range(len(event.front_boxes)): +# front_boxes = np.concatenate((front_boxes, event.front_boxes[i]), axis=0) +# front_feats = np.concatenate((front_feats, event.front_feats[i]), axis=0) + +# back_boxes = np.empty((0, 9), dtype=np.float64) ##和类doTracks兼容 +# back_feats = np.empty((0, 256), dtype=np.float64) ##和类doTracks兼容 +# for i in range(len(event.back_boxes)): +# back_boxes = np.concatenate((back_boxes, event.back_boxes[i]), axis=0) +# back_feats = np.concatenate((back_feats, event.back_feats[i]), axis=0) + +# if len(front_feats): +# front_simi = calsiml(front_feats, stdfeat) +# if len(back_feats): +# back_simi = calsiml(back_feats, stdfeat) + +# '''前后摄相似度融合策略''' +# if len(front_feats) and len(back_feats): +# diff_simi = abs(front_simi - back_simi) +# if diff_simi>0.15: +# Similar = max([front_simi, back_simi]) +# else: +# Similar = (front_simi+back_simi)/2 +# elif len(front_feats) and len(back_feats)==0: +# Similar = front_simi +# elif len(front_feats)==0 and len(back_feats): +# Similar = back_simi +# else: +# Similar = None # 在event.front_feats和event.back_feats同时为空时 + +# return Similar + + + + +def simi_matrix(): + evtpaths = r"/home/wqg/dataset/pipeline/contrast/single_event_V10/evtobjs/" + + stdfeatPath = r"/home/wqg/dataset/test_dataset/total_barcode/features_json/v11_barcode_0304/" + resultPath = r"/home/wqg/dataset/performence_dataset/result/" + + evt_paths, bcdSet = get_evtList(evtpaths) + + ## read std features + stdDict={} + evtDict = {} + for barcode in bcdSet: + stdpath = os.path.join(stdfeatPath, f"{barcode}.json") + if not os.path.isfile(stdpath): + continue + + with open(stdpath, 'r', encoding='utf-8') as f: + stddata = json.load(f) + feat = np.array(stddata["value"]) + stdDict[barcode] = feat + + for evtpath in evt_paths: + barcode = Path(evtpath).stem.split("_")[-1] + + if barcode not in stdDict.keys(): + continue + + # try: + # with open(evtpath, 'rb') as f: + # evtdata = pickle.load(f) + # except Exception as e: + # print(evtname) + + with open(evtpath, 'rb') as f: + event = pickle.load(f) + + stdfeat = stdDict[barcode] + + Similar = calsimi_vs_stdfeat_new(event, stdfeat) + + # 构造 boxes 子图存储路径 + subimgpath = os.path.join(resultPath, f"{event.evtname}", "subimg") + if not os.path.exists(subimgpath): + os.makedirs(subimgpath) + histpath = os.path.join(resultPath, "simi_hist") + if not os.path.exists(histpath): + os.makedirs(histpath) + + mean_values, max_values = [], [] + cameras = ('front', 'back') + fig, ax = plt.subplots(2, 3, figsize=(16, 9), dpi=100) + kpercent = 25 + for camera in cameras: + boxes = np.empty((0, 9), dtype=np.float64) ##和类doTracks兼容 + evtfeat = np.empty((0, 256), dtype=np.float64) ##和类doTracks兼容 + if camera == 'front': + for i in range(len(event.front_boxes)): + boxes = np.concatenate((boxes, event.front_boxes[i]), axis=0) + evtfeat = np.concatenate((evtfeat, event.front_feats[i]), axis=0) + imgpaths = event.front_imgpaths + + else: + for i in range(len(event.back_boxes)): + boxes = np.concatenate((boxes, event.back_boxes[i]), axis=0) + evtfeat = np.concatenate((evtfeat, event.back_feats[i]), axis=0) + imgpaths = event.back_imgpaths + + assert len(boxes)==len(evtfeat), f"Please check the Event: {event.evtname}" + if len(boxes)==0: continue + print(event.evtname) + + matrix = 1 - cdist(evtfeat, stdfeat, 'cosine') + simi_1d = matrix.flatten() + simi_mean = np.mean(matrix, axis=1) + # simi_max = np.max(matrix, axis=1) + + '''以相似度矩阵每一行最大的 k% 的相似度做均值计算''' + simi_max = [] + for i in range(len(matrix)): + sim = np.mean(get_topk_percent(matrix[i, :], kpercent)) + simi_max.append(sim) + + + mean_values.append(np.mean(matrix)) + max_values.append(np.mean(simi_max)) + + diff_max_mean = np.mean(simi_max) - np.mean(matrix) + + '''相似度统计特性图示''' + k =0 + if camera == 'front': k = 1 + + '''********************* 相似度全体数据 *********************''' + ax[k, 0].hist(simi_1d, bins=60, range=(-0.2, 1), edgecolor='black') + ax[k, 0].set_xlim([-0.2, 1]) + ax[k, 0].set_title(camera) + + _, y_max = ax[k, 0].get_ylim() # 获取y轴范围 + '''相似度变动范围''' + ax[k, 0].text(-0.1, 0.15*y_max, f"rng:{max(simi_1d)-min(simi_1d):.3f}", fontsize=18, color='b') + + '''********************* 均值********************************''' + ax[k, 1].hist(simi_mean, bins=24, range=(-0.2, 1), edgecolor='black') + ax[k, 1].set_xlim([-0.2, 1]) + ax[k, 1].set_title("mean") + _, y_max = ax[k, 1].get_ylim() # 获取y轴范围 + '''相似度变动范围''' + ax[k, 1].text(-0.1, 0.15*y_max, f"rng:{max(simi_mean)-min(simi_mean):.3f}", fontsize=18, color='b') + + + '''********************* 最大值 ******************************''' + ax[k, 2].hist(simi_max, bins=24, range=(-0.2, 1), edgecolor='black') + ax[k, 2].set_xlim([-0.2, 1]) + ax[k, 2].set_title("max") + _, y_max = ax[k, 2].get_ylim() # 获取y轴范围 + '''相似度变动范围''' + ax[k, 2].text(-0.1, 0.15*y_max, f"rng:{max(simi_max)-min(simi_max):.3f}", fontsize=18, color='b') + + + '''绘制聚类中心''' + cltc_mean = cluster(simi_mean) + for value in cltc_mean: + ax[k, 1].axvline(x=value, color='m', linestyle='--', linewidth=3) + + cltc_max = cluster(simi_max) + for value in cltc_max: + ax[k, 2].axvline(x=value, color='m', linestyle='--', linewidth=3) + + '''绘制相似度均值与最大值均值''' + ax[k, 1].axvline(x=np.mean(matrix), color='r', linestyle='-', linewidth=3) + ax[k, 2].axvline(x=np.mean(simi_max), color='g', linestyle='-', linewidth=3) + + '''绘制相似度最大值均值 - 均值''' + _, y_max = ax[k, 2].get_ylim() # 获取y轴范围 + ax[k, 2].text(-0.1, 0.05*y_max, f"g-r={diff_max_mean:.3f}", fontsize=18, color='m') + + plt.show() + + # for i, box in enumerate(boxes): + # x1, y1, x2, y2, tid, score, cls, fid, bid = box + # imgpath = imgpaths[int(fid-1)] + # image = cv2.imread(imgpath) + # subimg = image[int(y1/2):int(y2/2), int(x1/2):int(x2/2), :] + # camerType, timeTamp, _, frameID = os.path.basename(imgpath).split('.')[0].split('_') + # subimgName = f"cam{camerType}_{i}_tid{int(tid)}_fid({int(fid)}, {frameID})_{simi_mean[i]:.3f}.png" + # imgpairs.append((subimgName, subimg)) + # spath = os.path.join(subimgpath, subimgName) + # cv2.imwrite(spath, subimg) + + # oldname = f"cam{camerType}_{i}_tid{int(tid)}_fid({int(fid)}, {frameID}).png" + # oldpath = os.path.join(subimgpath, oldname) + # if os.path.exists(oldpath): + # os.remove(oldpath) + + + if len(mean_values)==2: + mean_diff = abs(mean_values[1]-mean_values[0]) + ax[0, 1].set_title(f"mean diff: {mean_diff:.3f}") + if len(max_values)==2: + max_diff = abs(max_values[1]-max_values[0]) + ax[0, 2].set_title(f"max diff: {max_diff:.3f}") + try: + fig.suptitle(f"Similar: {Similar:.3f}", fontsize=16) + except Exception as e: + print(e) + print(f"Similar: {Similar}") + pltpath = os.path.join(subimgpath, f"hist_max_{kpercent}%_.png") + plt.savefig(pltpath) + + pltpath1 = os.path.join(histpath, f"{event.evtname}_.png") + plt.savefig(pltpath1) + + + plt.close() + + +def main(): + simi_matrix() + + + +if __name__ == "__main__": + main() + # cluster() + + + + + + + + + + diff --git a/contrast/feat_extract/__pycache__/config.cpython-312.pyc b/contrast/feat_extract/__pycache__/config.cpython-312.pyc new file mode 100644 index 0000000..3972bf5 Binary files /dev/null and b/contrast/feat_extract/__pycache__/config.cpython-312.pyc differ diff --git a/contrast/feat_extract/__pycache__/config.cpython-39.pyc b/contrast/feat_extract/__pycache__/config.cpython-39.pyc new file mode 100644 index 0000000..b3c7da0 Binary files /dev/null and b/contrast/feat_extract/__pycache__/config.cpython-39.pyc differ diff --git a/contrast/feat_extract/__pycache__/inference.cpython-312.pyc b/contrast/feat_extract/__pycache__/inference.cpython-312.pyc new file mode 100644 index 0000000..3b2ee0e Binary files /dev/null and b/contrast/feat_extract/__pycache__/inference.cpython-312.pyc differ diff --git a/contrast/feat_extract/__pycache__/inference.cpython-39.pyc b/contrast/feat_extract/__pycache__/inference.cpython-39.pyc new file mode 100644 index 0000000..c7eb02c Binary files /dev/null and b/contrast/feat_extract/__pycache__/inference.cpython-39.pyc differ diff --git a/contrast/feat_extract/checkpoints/resnet18_0515/best.rknn b/contrast/feat_extract/checkpoints/resnet18_0515/best.rknn new file mode 100644 index 0000000..62fe8fe Binary files /dev/null and b/contrast/feat_extract/checkpoints/resnet18_0515/best.rknn differ diff --git a/contrast/feat_extract/config.py b/contrast/feat_extract/config.py new file mode 100644 index 0000000..586de69 --- /dev/null +++ b/contrast/feat_extract/config.py @@ -0,0 +1,88 @@ +import torch +import torchvision.transforms as T + + +class Config: + # network settings + backbone = 'resnet18' # [resnet18, mobilevit_s, mobilenet_v2, mobilenetv3_small, mobilenetv3_large, mobilenet_v1, PPLCNET_x1_0, PPLCNET_x0_5, PPLCNET_x2_5] + metric = 'arcface' # [cosface, arcface] + cbam = True + embedding_size = 256 + drop_ratio = 0.5 + img_size = 224 + + batch_size = 8 + + # data preprocess + # input_shape = [1, 128, 128] + """transforms.RandomCrop(size), + transforms.RandomVerticalFlip(p=0.5), + transforms.RandomHorizontalFlip(), + RandomRotate(15, 0.3), + # RandomGaussianBlur()""" + + train_transform = T.Compose([ + T.ToTensor(), + T.Resize((img_size, img_size)), + # T.RandomCrop(img_size), + # T.RandomHorizontalFlip(p=0.5), + T.RandomRotation(180), + T.ColorJitter(brightness=0.5), + T.ConvertImageDtype(torch.float32), + T.Normalize(mean=[0.5], std=[0.5]), + ]) + test_transform = T.Compose([ + T.ToTensor(), + T.Resize((img_size, img_size)), + T.ConvertImageDtype(torch.float32), + T.Normalize(mean=[0.5], std=[0.5]), + ]) + + # dataset + train_root = './data/2250_train/train' # 初始筛选过一次的数据集 + # train_root = './data/0612_train/train' + test_root = "./data/2250_train/val/" + # test_root = "./data/0612_train/val" + test_list = "./data/2250_train/val_pair.txt" + + test_group_json = "./2250_train/cross_same_0508.json" + + + # test_list = "./data/test_data_100/val_pair.txt" + + # training settings + checkpoints = "checkpoints/resnet18_0613/" # [resnet18, mobilevit_s, mobilenet_v2, mobilenetv3] + restore = False + # restore_model = "checkpoints/renet18_2250_0315/best_resnet18_2250_0315.pth" # best_resnet18_1491_0306.pth + restore_model = "checkpoints/resnet18_0515/best.pth" # best_resnet18_1491_0306.pth + + # test_model = "checkpoints/renet18_2250_0314/best_resnet18_2250_0314.pth" + testbackbone = 'resnet18' # [resnet18, mobilevit_s, mobilenet_v2, mobilenetv3_small, mobilenetv3_large, mobilenet_v1, PPLCNET_x1_0, PPLCNET_x0_5] + test_val = "D:/比对/cl" + # test_val = "./data/test_data_100" + + test_model = "checkpoints/best_20250228.pth" + # test_model = "checkpoints/zhanting_res_801.pth" + # test_model = "checkpoints/zhanting_res_abroad_8021.pth" + + + + train_batch_size = 512 # 256 + test_batch_size = 256 # 256 + + epoch = 300 + optimizer = 'sgd' # ['sgd', 'adam'] + lr = 1.5e-2 # 1e-2 + lr_step = 5 # 10 + lr_decay = 0.95 # 0.98 + weight_decay = 5e-4 + loss = 'cross_entropy' # ['focal_loss', 'cross_entropy'] + # device = torch.device('cuda:1' if torch.cuda.is_available() else 'cpu') + device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') + + pin_memory = True # if memory is large, set it True to speed up a bit + num_workers = 4 # dataloader + + group_test = True + +config = Config() diff --git a/contrast/feat_extract/inference.py b/contrast/feat_extract/inference.py new file mode 100644 index 0000000..48acb7d --- /dev/null +++ b/contrast/feat_extract/inference.py @@ -0,0 +1,606 @@ +# -*- coding: utf-8 -*- +""" + +@author: LiChen +""" +# import pdb +# import shutil +import torch.nn as nn +# import statistics +import os +import numpy as np +from scipy.spatial.distance import cdist +import torch +import os.path as osp +from PIL import Image +import json +import matplotlib.pyplot as plt +from pathlib import Path +# import sys +# sys.path.append(r"D:\DetectTracking") +# from contrast.config import config as conf +# from contrast.model import resnet18 + +from .config import config as conf +from .model import resnet18 + +# from model import (mobilevit_s, resnet14, resnet18, resnet34, resnet50, mobilenet_v2, +# MobileNetV3_Small, mobilenet_v1, PPLCNET_x1_0, PPLCNET_x0_5, PPLCNET_x2_5) + +curpath = Path(__file__).resolve().parents[0] + +class FeatsInterface: + def __init__(self, conf): + self.device = conf.device + + # if conf.backbone == 'resnet18': + # model = resnet18().to(conf.device) + + model = resnet18().to(conf.device) + self.transform = conf.test_transform + self.batch_size = conf.batch_size + self.embedding_size = conf.embedding_size + + ###yj 注释 + # if conf.test_model.find("zhanting") == -1: + # model = nn.DataParallel(model).to(conf.device) + self.model = model + + modpath = os.path.join(curpath, conf.test_model) + self.model.load_state_dict(torch.load(modpath, map_location=conf.device)) + self.model.eval() + # print('load model {} '.format(conf.testbackbone)) + + def inference(self, images, detections=None): + ''' + 如果是BGR，需要转变为RGB格式 + ''' + if isinstance(images, np.ndarray): + imgs, features = self.inference_image(images, detections) + return imgs, features + + batch_patches = [] + patches = [] + for i, img in enumerate(images): + img = img.copy() + + ## 对 img 进行补黑边，生成新的图像new_img + width, height = img.size + new_size = max(width, height) + new_img = Image.new("RGB", (new_size, new_size), (0, 0, 0)) + paste_x = (new_size - width) // 2 + paste_y = (new_size - height) // 2 + new_img.paste(img, (paste_x, paste_y)) + + patch = self.transform(new_img) + patch = patch.to(device=self.device) + # if str(self.device) != "cpu": + # patch = patch.to(device=self.device).half() + # else: + # patch = patch.to(device=self.device) + + patches.append(patch) + if (i + 1) % self.batch_size == 0: + patches = torch.stack(patches, dim=0) + batch_patches.append(patches) + patches = [] + + if len(patches): + patches = torch.stack(patches, dim=0) + batch_patches.append(patches) + + features = np.zeros((0, self.embedding_size)) + for patches in batch_patches: + pred=self.model(patches) + pred[torch.isinf(pred)] = 1.0 + feat = pred.cpu().data.numpy() + features = np.vstack((features, feat)) + return features + + def inference_image(self, image, detections): + H, W, _ = np.shape(image) + + batch_patches = [] + patches = [] + imgs = [] + for d in range(np.size(detections, 0)): + tlbr = detections[d, :4].astype(np.int_) + tlbr[0] = max(0, tlbr[0]) + tlbr[1] = max(0, tlbr[1]) + tlbr[2] = min(W - 1, tlbr[2]) + tlbr[3] = min(H - 1, tlbr[3]) + img = image[tlbr[1]:tlbr[3], tlbr[0]:tlbr[2], :] + + imgs.append(img) + + + img1 = img[:, :, ::-1].copy() # the model expects RGB inputs + patch = self.transform(img1) + + # patch = patch.to(device=self.device).half() + # if str(self.device) != "cpu": + # patch = patch.to(device=self.device).half() + # patch = patch.to(device=self.device) + # else: + # patch = patch.to(device=self.device) + patch = patch.to(device=self.device) + + patches.append(patch) + if (d + 1) % self.batch_size == 0: + patches = torch.stack(patches, dim=0) + batch_patches.append(patches) + patches = [] + + if len(patches): + patches = torch.stack(patches, dim=0) + batch_patches.append(patches) + + features = np.zeros((0, self.embedding_size)) + for patches in batch_patches: + pred = self.model(patches) + pred[torch.isinf(pred)] = 1.0 + feat = pred.cpu().data.numpy() + features = np.vstack((features, feat)) + + return imgs, features + + + + + + + + + +def unique_image(pair_list) -> set: + """Return unique image path in pair_list.txt""" + with open(pair_list, 'r') as fd: + pairs = fd.readlines() + unique = set() + for pair in pairs: + id1, id2, _ = pair.split() + unique.add(id1) + unique.add(id2) + return unique + + +def group_image(images: set, batch) -> list: + """Group image paths by batch size""" + images = list(images) + size = len(images) + res = [] + for i in range(0, size, batch): + end = min(batch + i, size) + res.append(images[i: end]) + return res + + +def _preprocess(images: list, transform) -> torch.Tensor: + res = [] + for img in images: + im = Image.open(img) + im = transform(im) + res.append(im) + # data = torch.cat(res, dim=0) # shape: (batch, 128, 128) + # data = data[:, None, :, :] # shape: (batch, 1, 128, 128) + data = torch.stack(res) + return data + + +def test_preprocess(images: list, transform) -> torch.Tensor: + res = [] + for img in images: + im = Image.open(img) + im = transform(im) + res.append(im) + # data = torch.cat(res, dim=0) # shape: (batch, 128, 128) + # data = data[:, None, :, :] # shape: (batch, 1, 128, 128) + data = torch.stack(res) + return data + + +def featurize(images: list, transform, net, device, train=False) -> dict: + """featurize each image and save into a dictionary + Args: + images: image paths + transform: test transform + net: pretrained model + device: cpu or cuda + Returns: + Dict (key: imagePath, value: feature) + """ + if train: + data = _preprocess(images, transform) + data = data.to(device) + net = net.to(device) + with torch.no_grad(): + features = net(data) + res = {img: feature for (img, feature) in zip(images, features)} + else: + data = test_preprocess(images, transform) + data = data.to(device) + net = net.to(device) + with torch.no_grad(): + features = net(data) + res = {img: feature for (img, feature) in zip(images, features)} + return res + +# def inference_image(images: list, transform, net, device, bs=16, embedding_size=256) -> dict: +# batch_patches = [] +# patches = [] +# for d, img in enumerate(images): +# img = Image.open(img) +# patch = transform(img) + +# if str(device) != "cpu": +# patch = patch.to(device).half() +# else: +# patch = patch.to(device) + +# patches.append(patch) +# if (d + 1) % bs == 0: +# patches = torch.stack(patches, dim=0) +# batch_patches.append(patches) +# patches = [] + +# if len(patches): +# patches = torch.stack(patches, dim=0) +# batch_patches.append(patches) + +# features = np.zeros((0, embedding_size), dtype=np.float32) +# for patches in batch_patches: +# pred = net(patches) +# pred[torch.isinf(pred)] = 1.0 +# feat = pred.cpu().data.numpy() +# features = np.vstack((features, feat)) + + + +# return features + + + +def featurize_1(images: list, transform, net, device, train=False) -> dict: + """featurize each image and save into a dictionary + Args: + images: image paths + transform: test transform + net: pretrained model + device: cpu or cuda + Returns: + Dict (key: imagePath, value: feature) + """ + + data = test_preprocess(images, transform) + data = data.to(device) + net = net.to(device) + with torch.no_grad(): + features = net(data).data.numpy() + + return features + + + + +def cosin_metric(x1, x2): + return np.dot(x1, x2) / (np.linalg.norm(x1) * np.linalg.norm(x2)) + + +def threshold_search(y_score, y_true): + y_score = np.asarray(y_score) + y_true = np.asarray(y_true) + best_acc = 0 + best_th = 0 + for i in range(len(y_score)): + th = y_score[i] + y_test = (y_score >= th) + acc = np.mean((y_test == y_true).astype(int)) + if acc > best_acc: + best_acc = acc + best_th = th + return best_acc, best_th + + +def showgrid(recall, recall_TN, PrecisePos, PreciseNeg): + x = np.linspace(start=-1.0, stop=1.0, num=50, endpoint=True).tolist() + plt.figure(figsize=(10, 6)) + plt.plot(x, recall, color='red', label='recall') + plt.plot(x, recall_TN, color='black', label='recall_TN') + plt.plot(x, PrecisePos, color='blue', label='PrecisePos') + plt.plot(x, PreciseNeg, color='green', label='PreciseNeg') + plt.legend() + plt.xlabel('threshold') + # plt.ylabel('Similarity') + plt.grid(True, linestyle='--', alpha=0.5) + plt.savefig('accuracy_recall_grid.png') + plt.show() + plt.close() + + +def compute_accuracy_recall(score, labels): + th = 0.1 + squence = np.linspace(-1, 1, num=50) + # squence = [0.4] + recall, PrecisePos, PreciseNeg, recall_TN = [], [], [], [] + for th in squence: + t_score = (score > th) + t_labels = (labels == 1) + # print(t_score) + # print(t_labels) + TP = np.sum(np.logical_and(t_score, t_labels)) + FN = np.sum(np.logical_and(np.logical_not(t_score), t_labels)) + f_score = (score < th) + f_labels = (labels == 0) + TN = np.sum(np.logical_and(f_score, f_labels)) + FP = np.sum(np.logical_and(np.logical_not(f_score), f_labels)) + print("Threshold:{} TP:{},FP:{},TN:{},FN:{}".format(th, TP, FP, TN, FN)) + + PrecisePos.append(0 if TP / (TP + FP) == 'nan' else TP / (TP + FP)) + PreciseNeg.append(0 if TN == 0 else TN / (TN + FN)) + recall.append(0 if TP == 0 else TP / (TP + FN)) + recall_TN.append(0 if TN == 0 else TN / (TN + FP)) + showgrid(recall, recall_TN, PrecisePos, PreciseNeg) + + +def compute_accuracy(feature_dict, pair_list, test_root): + with open(pair_list, 'r') as f: + pairs = f.readlines() + + similarities = [] + labels = [] + for pair in pairs: + img1, img2, label = pair.split() + img1 = osp.join(test_root, img1) + img2 = osp.join(test_root, img2) + feature1 = feature_dict[img1].cpu().numpy() + feature2 = feature_dict[img2].cpu().numpy() + label = int(label) + + similarity = cosin_metric(feature1, feature2) + similarities.append(similarity) + labels.append(label) + + accuracy, threshold = threshold_search(similarities, labels) + # print('similarities >> {}'.format(similarities)) + # print('labels >> {}'.format(labels)) + compute_accuracy_recall(np.array(similarities), np.array(labels)) + return accuracy, threshold + + +def deal_group_pair(pairList1, pairList2): + allsimilarity = [] + one_similarity = [] + for pair1 in pairList1: + for pair2 in pairList2: + similarity = cosin_metric(pair1.cpu().numpy(), pair2.cpu().numpy()) + one_similarity.append(similarity) + allsimilarity.append(max(one_similarity)) # 最大值 + # allsimilarity.append(sum(one_similarity)/len(one_similarity)) # 均值 + # allsimilarity.append(statistics.median(one_similarity)) # 中位数 + # print(allsimilarity) + # print(labels) + return allsimilarity + +def compute_group_accuracy(content_list_read): + allSimilarity, allLabel= [], [] + for data_loaded in content_list_read: + one_group_list = [] + for i in range(2): + images = [osp.join(conf.test_val, img) for img in data_loaded[i]] + group = group_image(images, conf.test_batch_size) + d = featurize(group[0], conf.test_transform, model, conf.device) + one_group_list.append(d.values()) + similarity = deal_group_pair(one_group_list[0], one_group_list[1]) + allLabel.append(data_loaded[-1]) + allSimilarity.extend(similarity) + # print(allSimilarity) + # print(allLabel) + return allSimilarity, allLabel + +def compute_contrast_accuracy(content_list_read): + + npairs = 50 + + same_folder_pairs = content_list_read['same_folder_pairs'] + cross_folder_pairs = content_list_read['cross_folder_pairs'] + + npairs = min((len(same_folder_pairs), len(cross_folder_pairs))) + + Encoder = FeatsInterface(conf) + + same_pairs = same_folder_pairs[:npairs] + cross_pairs = cross_folder_pairs[:npairs] + + same_pairs_similarity = [] + for i in range(len(same_pairs)): + images_a = [osp.join(conf.test_val, img) for img in same_pairs[i][0]] + images_b = [osp.join(conf.test_val, img) for img in same_pairs[i][1]] + + feats_a = Encoder.inference(images_a) + feats_b = Encoder.inference(images_b) + # matrix = 1- np.maximum(0.0, cdist(feats_a, feats_b, 'cosine')) + matrix = 1 - cdist(feats_a, feats_b, 'cosine') + + feats_am = np.mean(feats_a, axis=0, keepdims=True) + feats_bm = np.mean(feats_b, axis=0, keepdims=True) + matrixm = 1- np.maximum(0.0, cdist(feats_am, feats_bm, 'cosine')) + + same_pairs_similarity.append(np.mean(matrix)) + + '''保存相同 Barcode 图像对''' + # foldi = os.path.join('./result/same', f'{i}') + # if os.path.exists(foldi): + # shutil.rmtree(foldi) + # os.makedirs(foldi) + # else: + # os.makedirs(foldi) + # for ipt in range(len(images_a)): + # source_path = images_a[ipt] + # destination_path = os.path.join(foldi, f'a_{ipt}.png') + # shutil.copy2(source_path, destination_path) + # for ipt in range(len(images_b)): + # source_path = images_b[ipt] + # destination_path = os.path.join(foldi, f'b_{ipt}.png') + # shutil.copy2(source_path, destination_path) + + cross_pairs_similarity = [] + for i in range(len(cross_pairs)): + images_a = [osp.join(conf.test_val, img) for img in cross_pairs[i][0]] + images_b = [osp.join(conf.test_val, img) for img in cross_pairs[i][1]] + + feats_a = Encoder.inference(images_a) + feats_b = Encoder.inference(images_b) + # matrix = 1- np.maximum(0.0, cdist(feats_a, feats_b, 'cosine')) + matrix = 1 - cdist(feats_a, feats_b, 'cosine') + + feats_am = np.mean(feats_a, axis=0, keepdims=True) + feats_bm = np.mean(feats_b, axis=0, keepdims=True) + matrixm = 1- np.maximum(0.0, cdist(feats_am, feats_bm, 'cosine')) + + cross_pairs_similarity.append(np.mean(matrix)) + + '''保存不同 Barcode 图像对''' + # foldi = os.path.join('./result/cross', f'{i}') + # if os.path.exists(foldi): + # shutil.rmtree(foldi) + # os.makedirs(foldi) + # else: + # os.makedirs(foldi) + # for ipt in range(len(images_a)): + # source_path = images_a[ipt] + # destination_path = os.path.join(foldi, f'a_{ipt}.png') + # shutil.copy2(source_path, destination_path) + # for ipt in range(len(images_b)): + # source_path = images_b[ipt] + # destination_path = os.path.join(foldi, f'b_{ipt}.png') + # shutil.copy2(source_path, destination_path) + + + Thresh = np.linspace(-0.2, 1, 100) + + Same = np.array(same_pairs_similarity) + Cross = np.array(cross_pairs_similarity) + + fig, axs = plt.subplots(2, 1) + axs[0].hist(Same, bins=60, edgecolor='black') + axs[0].set_xlim([-0.2, 1]) + axs[0].set_title('Same Barcode') + + axs[1].hist(Cross, bins=60, edgecolor='black') + axs[1].set_xlim([-0.2, 1]) + axs[1].set_title('Cross Barcode') + + TPFN = len(Same) + TNFP = len(Cross) + Recall_Pos, Recall_Neg = [], [] + Precision_Pos, Precision_Neg = [], [] + Correct = [] + for th in Thresh: + TP = np.sum(Same > th) + FN = TPFN - TP + TN = np.sum(Cross < th) + FP = TNFP - TN + + Recall_Pos.append(TP/TPFN) + Recall_Neg.append(TN/TNFP) + Precision_Pos.append(TP/(TP+FP)) + Precision_Neg.append(TN/(TN+FN)) + Correct.append((TN+TP)/(TPFN+TNFP)) + + fig, ax = plt.subplots() + ax.plot(Thresh, Correct, 'r', label='Correct: (TN+TP)/(TPFN+TNFP)') + ax.plot(Thresh, Recall_Pos, 'b', label='Recall_Pos: TP/TPFN') + ax.plot(Thresh, Recall_Neg, 'g', label='Recall_Neg: TN/TNFP') + ax.plot(Thresh, Precision_Pos, 'c', label='Precision_Pos: TP/(TP+FP)') + ax.plot(Thresh, Precision_Neg, 'm', label='Precision_Neg: TN/(TN+FN)') + + ax.set_xlim([0, 1]) + ax.set_ylim([0, 1]) + ax.grid(True) + ax.set_title('PrecisePos & PreciseNeg') + ax.legend() + plt.show() + + print("Haved done!!!") + + + + + +if __name__ == '__main__': + + # Network Setup + if conf.testbackbone == 'resnet18': + # model = ResIRSE(conf.img_size, conf.embedding_size, conf.drop_ratio).to(conf.device) + model = resnet18().to(conf.device) + # elif conf.testbackbone == 'resnet34': + # model = resnet34().to(conf.device) + # elif conf.testbackbone == 'resnet50': + # model = resnet50().to(conf.device) + # elif conf.testbackbone == 'mobilevit_s': + # model = mobilevit_s().to(conf.device) + # elif conf.testbackbone == 'mobilenetv3': + # model = MobileNetV3_Small().to(conf.device) + # elif conf.testbackbone == 'mobilenet_v1': + # model = mobilenet_v1().to(conf.device) + # elif conf.testbackbone == 'PPLCNET_x1_0': + # model = PPLCNET_x1_0().to(conf.device) + # elif conf.testbackbone == 'PPLCNET_x0_5': + # model = PPLCNET_x0_5().to(conf.device) + # elif conf.backbone == 'PPLCNET_x2_5': + # model = PPLCNET_x2_5().to(conf.device) + # elif conf.testbackbone == 'mobilenet_v2': + # model = mobilenet_v2().to(conf.device) + # elif conf.testbackbone == 'resnet14': + # model = resnet14().to(conf.device) + else: + raise ValueError('Have not model {}'.format(conf.backbone)) + + print('load model {} '.format(conf.testbackbone)) + # model = nn.DataParallel(model).to(conf.device) + model.load_state_dict(torch.load(conf.test_model, map_location=conf.device)) + model.eval() + if not conf.group_test: + images = unique_image(conf.test_list) + images = [osp.join(conf.test_val, img) for img in images] + + groups = group_image(images, conf.test_batch_size) ##根据batch_size取图片 + + feature_dict = dict() + for group in groups: + d = featurize(group, conf.test_transform, model, conf.device) + feature_dict.update(d) + # print('feature_dict', feature_dict) + accuracy, threshold = compute_accuracy(feature_dict, conf.test_list, conf.test_val) + + print( + f"Test Model: {conf.test_model}\n" + f"Accuracy: {accuracy:.3f}\n" + f"Threshold: {threshold:.3f}\n" + ) + elif conf.group_test: + """ + conf.test_val: 测试数据集地址 + conf.test_group_json：测试数据分组配置文件 + """ + filename = conf.test_group_json + + filename = "../cl/images_1.json" + with open(filename, 'r', encoding='utf-8') as file: + content_list_read = json.load(file) + + + compute_contrast_accuracy(content_list_read) + + + + + +# ============================================================================= +# Similarity, Label = compute_group_accuracy(content_list_read) +# print('allSimilarity >> {}'.format(Similarity)) +# print('allLabel >> {}'.format(Label)) +# compute_accuracy_recall(np.array(Similarity), np.array(Label)) +# # compute_group_accuracy(data_loaded) +# +# ============================================================================= diff --git a/contrast/feat_extract/model/BAM.py b/contrast/feat_extract/model/BAM.py new file mode 100644 index 0000000..4ac61ae --- /dev/null +++ b/contrast/feat_extract/model/BAM.py @@ -0,0 +1,88 @@ +import torch.nn as nn +import torchvision +from torch.nn import init + + +class Flatten(nn.Module): + def forward(self, x): + return x.view(x.shape[0], -1) + + +class ChannelAttention(nn.Module): + def __int__(self, channel, reduction, num_layers): + super(ChannelAttention, self).__init__() + self.avgpool = nn.AdaptiveAvgPool2d(1) + gate_channels = [channel] + gate_channels += [len(channel) // reduction] * num_layers + gate_channels += [channel] + + self.ca = nn.Sequential() + self.ca.add_module('flatten', Flatten()) + for i in range(len(gate_channels) - 2): + self.ca.add_module('', nn.Linear(gate_channels[i], gate_channels[i + 1])) + self.ca.add_module('', nn.BatchNorm1d(gate_channels[i + 1])) + self.ca.add_module('', nn.ReLU()) + self.ca.add_module('', nn.Linear(gate_channels[-2], gate_channels[-1])) + + def forward(self, x): + res = self.avgpool(x) + res = self.ca(res) + res = res.unsqueeze(-1).unsqueeze(-1).expand_as(x) + return res + + +class SpatialAttention(nn.Module): + def __int__(self, channel, reduction=16, num_lay=3, dilation=2): + super(SpatialAttention).__init__() + self.sa = nn.Sequential() + self.sa.add_module('', nn.Conv2d(kernel_size=1, in_channels=channel, out_channels=(channel // reduction) * 3)) + self.sa.add_module('', nn.BatchNorm2d(num_features=(channel // reduction))) + self.sa.add_module('', nn.ReLU()) + for i in range(num_lay): + self.sa.add_module('', nn.Conv2d(kernel_size=3, + in_channels=(channel // reduction), + out_channels=(channel // reduction), + padding=1, + dilation=2)) + self.sa.add_module('', nn.BatchNorm2d(channel // reduction)) + self.sa.add_module('', nn.ReLU()) + self.sa.add_module('', nn.Conv2d(channel // reduction, 1, kernel_size=1)) + + def forward(self, x): + res = self.sa(x) + res = res.expand_as(x) + return res + + +class BAMblock(nn.Module): + def __init__(self, channel=512, reduction=16, dia_val=2): + super(BAMblock, self).__init__() + self.ca = ChannelAttention(channel, reduction) + self.sa = SpatialAttention(channel, reduction, dia_val) + self.sigmoid = nn.Sigmoid() + + def init_weights(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + init.kaiming_normal(m.weight, mode='fan_out') + if m.bais is not None: + init.constant_(m.bias, 0) + elif isinstance(m, nn.BatchNorm2d): + init.constant_(m.weight, 1) + init.constant_(m.bias, 0) + elif isinstance(m, nn.Linear): + init.normal_(m.weight, std=0.001) + if m.bias is not None: + init.constant_(m.bias, 0) + + def forward(self, x): + b, c, _, _ = x.size() + sa_out = self.sa(x) + ca_out = self.ca(x) + weight = self.sigmoid(sa_out + ca_out) + out = (1 + weight) * x + return out + + +if __name__ == "__main__": + print(512 // 14) diff --git a/contrast/feat_extract/model/CBAM.py b/contrast/feat_extract/model/CBAM.py new file mode 100644 index 0000000..69747e0 --- /dev/null +++ b/contrast/feat_extract/model/CBAM.py @@ -0,0 +1,70 @@ +import torch +import torch.nn as nn +import torch.nn.init as init + +class channelAttention(nn.Module): + def __init__(self, channel, reduction=16): + super(channelAttention, self).__init__() + self.Maxpooling = nn.AdaptiveMaxPool2d(1) + self.Avepooling = nn.AdaptiveAvgPool2d(1) + self.ca = nn.Sequential() + self.ca.add_module('conv1',nn.Conv2d(channel, channel//reduction, 1, bias=False)) + self.ca.add_module('Relu', nn.ReLU()) + self.ca.add_module('conv2',nn.Conv2d(channel//reduction, channel, 1, bias=False)) + self.sigmod = nn.Sigmoid() + + def forward(self, x): + M_out = self.Maxpooling(x) + A_out = self.Avepooling(x) + M_out = self.ca(M_out) + A_out = self.ca(A_out) + out = self.sigmod(M_out+A_out) + return out + +class SpatialAttention(nn.Module): + def __init__(self, kernel_size=7): + super().__init__() + self.conv = nn.Conv2d(in_channels=2, out_channels=1, kernel_size=kernel_size, padding=kernel_size // 2) + self.sigmoid = nn.Sigmoid() + + def forward(self, x): + max_result, _ = torch.max(x, dim=1, keepdim=True) + avg_result = torch.mean(x, dim=1, keepdim=True) + result = torch.cat([max_result, avg_result], dim=1) + output = self.conv(result) + output = self.sigmoid(output) + return output + +class CBAM(nn.Module): + def __init__(self, channel, reduction=16, kernel_size=7): + super().__init__() + self.ca = channelAttention(channel, reduction) + self.sa = SpatialAttention(kernel_size) + + def init_weights(self): + for m in self.modules():#权重初始化 + if isinstance(m, nn.Conv2d): + init.kaiming_normal_(m.weight, mode='fan_out') + if m.bias is not None: + init.constant_(m.bias, 0) + elif isinstance(m, nn.BatchNorm2d): + init.constant_(m.weight, 1) + init.constant_(m.bias, 0) + elif isinstance(m, nn.Linear): + init.normal_(m.weight, std=0.001) + if m.bias is not None: + init.constant_(m.bias, 0) + + def forward(self, x): + # b,c_,_ = x.size() + # residual = x + out = x*self.ca(x) + out = out*self.sa(out) + return out + +if __name__ == '__main__': + input=torch.randn(50,512,7,7) + kernel_size=input.shape[2] + cbam = CBAM(channel=512,reduction=16,kernel_size=kernel_size) + output=cbam(input) + print(output.shape) diff --git a/contrast/feat_extract/model/Tool.py b/contrast/feat_extract/model/Tool.py new file mode 100644 index 0000000..378c11a --- /dev/null +++ b/contrast/feat_extract/model/Tool.py @@ -0,0 +1,33 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +class GeM(nn.Module): + def __init__(self, p=3, eps=1e-6): + super(GeM, self).__init__() + self.p = nn.Parameter(torch.ones(1) * p) + self.eps = eps + + def forward(self, x): + return self.gem(x, p=self.p, eps=self.eps, stride = 2) + + def gem(self, x, p=3, eps=1e-6, stride = 2): + return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1)), stride=2).pow(1. / p) + + def __repr__(self): + return self.__class__.__name__ + \ + '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + \ + ', ' + 'eps=' + str(self.eps) + ')' + +class TripletLoss(nn.Module): + def __init__(self, margin): + super(TripletLoss, self).__init__() + self.margin = margin + + def forward(self, anchor, positive, negative, size_average = True): + distance_positive = (anchor-positive).pow(2).sum(1) + distance_negative = (anchor-negative).pow(2).sum(1) + losses = F.relu(distance_negative-distance_positive+self.margin) + return losses.mean() if size_average else losses.sum() + +if __name__ == '__main__': + print('') diff --git a/contrast/feat_extract/model/__init__.py b/contrast/feat_extract/model/__init__.py new file mode 100644 index 0000000..37a0635 --- /dev/null +++ b/contrast/feat_extract/model/__init__.py @@ -0,0 +1,11 @@ +from .fmobilenet import FaceMobileNet +from .resnet_face import ResIRSE +from .mobilevit import mobilevit_s +from .metric import ArcFace, CosFace +from .loss import FocalLoss +from .resbam import resnet +from .resnet_pre import resnet18, resnet34, resnet50, resnet14 +from .mobilenet_v2 import mobilenet_v2 +from .mobilenet_v3 import MobileNetV3_Small, MobileNetV3_Large +# from .mobilenet_v1 import mobilenet_v1 +from .lcnet import PPLCNET_x0_25, PPLCNET_x0_35, PPLCNET_x0_5, PPLCNET_x0_75, PPLCNET_x1_0, PPLCNET_x1_5, PPLCNET_x2_0, PPLCNET_x2_5 \ No newline at end of file diff --git a/contrast/feat_extract/model/__pycache__/BAM.cpython-38.pyc b/contrast/feat_extract/model/__pycache__/BAM.cpython-38.pyc new file mode 100644 index 0000000..3c10e34 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/BAM.cpython-38.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/CBAM.cpython-312.pyc b/contrast/feat_extract/model/__pycache__/CBAM.cpython-312.pyc new file mode 100644 index 0000000..bec3426 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/CBAM.cpython-312.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/CBAM.cpython-38.pyc b/contrast/feat_extract/model/__pycache__/CBAM.cpython-38.pyc new file mode 100644 index 0000000..3356083 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/CBAM.cpython-38.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/CBAM.cpython-39.pyc b/contrast/feat_extract/model/__pycache__/CBAM.cpython-39.pyc new file mode 100644 index 0000000..9c425a6 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/CBAM.cpython-39.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/Tool.cpython-312.pyc b/contrast/feat_extract/model/__pycache__/Tool.cpython-312.pyc new file mode 100644 index 0000000..1863278 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/Tool.cpython-312.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/Tool.cpython-38.pyc b/contrast/feat_extract/model/__pycache__/Tool.cpython-38.pyc new file mode 100644 index 0000000..8e85335 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/Tool.cpython-38.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/Tool.cpython-39.pyc b/contrast/feat_extract/model/__pycache__/Tool.cpython-39.pyc new file mode 100644 index 0000000..0e3a668 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/Tool.cpython-39.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/__init__.cpython-310.pyc b/contrast/feat_extract/model/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000..828ff83 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/__init__.cpython-310.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/__init__.cpython-312.pyc b/contrast/feat_extract/model/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..4892404 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/__init__.cpython-312.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/__init__.cpython-38.pyc b/contrast/feat_extract/model/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000..653497f Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/__init__.cpython-38.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/__init__.cpython-39.pyc b/contrast/feat_extract/model/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..b79395e Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/__init__.cpython-39.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/fmobilenet.cpython-310.pyc b/contrast/feat_extract/model/__pycache__/fmobilenet.cpython-310.pyc new file mode 100644 index 0000000..2d3a370 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/fmobilenet.cpython-310.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/fmobilenet.cpython-312.pyc b/contrast/feat_extract/model/__pycache__/fmobilenet.cpython-312.pyc new file mode 100644 index 0000000..d809e7c Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/fmobilenet.cpython-312.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/fmobilenet.cpython-38.pyc b/contrast/feat_extract/model/__pycache__/fmobilenet.cpython-38.pyc new file mode 100644 index 0000000..5e2828e Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/fmobilenet.cpython-38.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/fmobilenet.cpython-39.pyc b/contrast/feat_extract/model/__pycache__/fmobilenet.cpython-39.pyc new file mode 100644 index 0000000..890c562 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/fmobilenet.cpython-39.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/lcnet.cpython-312.pyc b/contrast/feat_extract/model/__pycache__/lcnet.cpython-312.pyc new file mode 100644 index 0000000..3e1a42f Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/lcnet.cpython-312.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/lcnet.cpython-38.pyc b/contrast/feat_extract/model/__pycache__/lcnet.cpython-38.pyc new file mode 100644 index 0000000..fa38fe4 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/lcnet.cpython-38.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/lcnet.cpython-39.pyc b/contrast/feat_extract/model/__pycache__/lcnet.cpython-39.pyc new file mode 100644 index 0000000..fc264b4 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/lcnet.cpython-39.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/loss.cpython-312.pyc b/contrast/feat_extract/model/__pycache__/loss.cpython-312.pyc new file mode 100644 index 0000000..ce31fb4 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/loss.cpython-312.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/loss.cpython-38.pyc b/contrast/feat_extract/model/__pycache__/loss.cpython-38.pyc new file mode 100644 index 0000000..5aed3eb Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/loss.cpython-38.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/loss.cpython-39.pyc b/contrast/feat_extract/model/__pycache__/loss.cpython-39.pyc new file mode 100644 index 0000000..b5cb9bd Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/loss.cpython-39.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/metric.cpython-312.pyc b/contrast/feat_extract/model/__pycache__/metric.cpython-312.pyc new file mode 100644 index 0000000..babe348 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/metric.cpython-312.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/metric.cpython-38.pyc b/contrast/feat_extract/model/__pycache__/metric.cpython-38.pyc new file mode 100644 index 0000000..dc77c9d Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/metric.cpython-38.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/metric.cpython-39.pyc b/contrast/feat_extract/model/__pycache__/metric.cpython-39.pyc new file mode 100644 index 0000000..ab1598e Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/metric.cpython-39.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/mobilenet_v1.cpython-38.pyc b/contrast/feat_extract/model/__pycache__/mobilenet_v1.cpython-38.pyc new file mode 100644 index 0000000..8d86f36 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/mobilenet_v1.cpython-38.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/mobilenet_v1.cpython-39.pyc b/contrast/feat_extract/model/__pycache__/mobilenet_v1.cpython-39.pyc new file mode 100644 index 0000000..501c1e4 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/mobilenet_v1.cpython-39.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/mobilenet_v2.cpython-312.pyc b/contrast/feat_extract/model/__pycache__/mobilenet_v2.cpython-312.pyc new file mode 100644 index 0000000..696ba13 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/mobilenet_v2.cpython-312.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/mobilenet_v2.cpython-38.pyc b/contrast/feat_extract/model/__pycache__/mobilenet_v2.cpython-38.pyc new file mode 100644 index 0000000..c51a529 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/mobilenet_v2.cpython-38.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/mobilenet_v2.cpython-39.pyc b/contrast/feat_extract/model/__pycache__/mobilenet_v2.cpython-39.pyc new file mode 100644 index 0000000..9590561 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/mobilenet_v2.cpython-39.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/mobilenet_v3.cpython-312.pyc b/contrast/feat_extract/model/__pycache__/mobilenet_v3.cpython-312.pyc new file mode 100644 index 0000000..d4e74ee Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/mobilenet_v3.cpython-312.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/mobilenet_v3.cpython-38.pyc b/contrast/feat_extract/model/__pycache__/mobilenet_v3.cpython-38.pyc new file mode 100644 index 0000000..85471d2 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/mobilenet_v3.cpython-38.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/mobilenet_v3.cpython-39.pyc b/contrast/feat_extract/model/__pycache__/mobilenet_v3.cpython-39.pyc new file mode 100644 index 0000000..0cf0132 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/mobilenet_v3.cpython-39.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/mobilevit.cpython-310.pyc b/contrast/feat_extract/model/__pycache__/mobilevit.cpython-310.pyc new file mode 100644 index 0000000..39a75ca Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/mobilevit.cpython-310.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/mobilevit.cpython-312.pyc b/contrast/feat_extract/model/__pycache__/mobilevit.cpython-312.pyc new file mode 100644 index 0000000..ec3d4de Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/mobilevit.cpython-312.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/mobilevit.cpython-38.pyc b/contrast/feat_extract/model/__pycache__/mobilevit.cpython-38.pyc new file mode 100644 index 0000000..b7f7051 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/mobilevit.cpython-38.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/mobilevit.cpython-39.pyc b/contrast/feat_extract/model/__pycache__/mobilevit.cpython-39.pyc new file mode 100644 index 0000000..a8f32ba Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/mobilevit.cpython-39.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/resbam.cpython-312.pyc b/contrast/feat_extract/model/__pycache__/resbam.cpython-312.pyc new file mode 100644 index 0000000..2605e1c Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/resbam.cpython-312.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/resbam.cpython-38.pyc b/contrast/feat_extract/model/__pycache__/resbam.cpython-38.pyc new file mode 100644 index 0000000..e694a79 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/resbam.cpython-38.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/resbam.cpython-39.pyc b/contrast/feat_extract/model/__pycache__/resbam.cpython-39.pyc new file mode 100644 index 0000000..053b1ab Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/resbam.cpython-39.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/resnet.cpython-310.pyc b/contrast/feat_extract/model/__pycache__/resnet.cpython-310.pyc new file mode 100644 index 0000000..485ac48 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/resnet.cpython-310.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/resnet.cpython-38.pyc b/contrast/feat_extract/model/__pycache__/resnet.cpython-38.pyc new file mode 100644 index 0000000..a9e8c9f Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/resnet.cpython-38.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/resnet_face.cpython-312.pyc b/contrast/feat_extract/model/__pycache__/resnet_face.cpython-312.pyc new file mode 100644 index 0000000..2c34e5e Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/resnet_face.cpython-312.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/resnet_face.cpython-38.pyc b/contrast/feat_extract/model/__pycache__/resnet_face.cpython-38.pyc new file mode 100644 index 0000000..118efa2 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/resnet_face.cpython-38.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/resnet_face.cpython-39.pyc b/contrast/feat_extract/model/__pycache__/resnet_face.cpython-39.pyc new file mode 100644 index 0000000..a45da2a Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/resnet_face.cpython-39.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/resnet_pre.cpython-312.pyc b/contrast/feat_extract/model/__pycache__/resnet_pre.cpython-312.pyc new file mode 100644 index 0000000..700c724 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/resnet_pre.cpython-312.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/resnet_pre.cpython-38.pyc b/contrast/feat_extract/model/__pycache__/resnet_pre.cpython-38.pyc new file mode 100644 index 0000000..786ef3a Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/resnet_pre.cpython-38.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/resnet_pre.cpython-39.pyc b/contrast/feat_extract/model/__pycache__/resnet_pre.cpython-39.pyc new file mode 100644 index 0000000..9254dcc Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/resnet_pre.cpython-39.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/utils.cpython-312.pyc b/contrast/feat_extract/model/__pycache__/utils.cpython-312.pyc new file mode 100644 index 0000000..20df561 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/utils.cpython-312.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/utils.cpython-38.pyc b/contrast/feat_extract/model/__pycache__/utils.cpython-38.pyc new file mode 100644 index 0000000..6e3f2bb Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/utils.cpython-38.pyc differ diff --git a/contrast/feat_extract/model/__pycache__/utils.cpython-39.pyc b/contrast/feat_extract/model/__pycache__/utils.cpython-39.pyc new file mode 100644 index 0000000..5e9e645 Binary files /dev/null and b/contrast/feat_extract/model/__pycache__/utils.cpython-39.pyc differ diff --git a/contrast/feat_extract/model/fmobilenet.py b/contrast/feat_extract/model/fmobilenet.py new file mode 100644 index 0000000..2e38a44 --- /dev/null +++ b/contrast/feat_extract/model/fmobilenet.py @@ -0,0 +1,124 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + + + +class Flatten(nn.Module): + def forward(self, x): + return x.view(x.shape[0], -1) + +class ConvBn(nn.Module): + + def __init__(self, in_c, out_c, kernel=(1, 1), stride=1, padding=0, groups=1): + super().__init__() + self.net = nn.Sequential( + nn.Conv2d(in_c, out_c, kernel, stride, padding, groups=groups, bias=False), + nn.BatchNorm2d(out_c) + ) + + def forward(self, x): + return self.net(x) + + +class ConvBnPrelu(nn.Module): + + def __init__(self, in_c, out_c, kernel=(1, 1), stride=1, padding=0, groups=1): + super().__init__() + self.net = nn.Sequential( + ConvBn(in_c, out_c, kernel, stride, padding, groups), + nn.PReLU(out_c) + ) + + def forward(self, x): + return self.net(x) + + +class DepthWise(nn.Module): + + def __init__(self, in_c, out_c, kernel=(3, 3), stride=2, padding=1, groups=1): + super().__init__() + self.net = nn.Sequential( + ConvBnPrelu(in_c, groups, kernel=(1, 1), stride=1, padding=0), + ConvBnPrelu(groups, groups, kernel=kernel, stride=stride, padding=padding, groups=groups), + ConvBn(groups, out_c, kernel=(1, 1), stride=1, padding=0), + ) + + def forward(self, x): + return self.net(x) + + +class DepthWiseRes(nn.Module): + """DepthWise with Residual""" + + def __init__(self, in_c, out_c, kernel=(3, 3), stride=2, padding=1, groups=1): + super().__init__() + self.net = DepthWise(in_c, out_c, kernel, stride, padding, groups) + + def forward(self, x): + return self.net(x) + x + + +class MultiDepthWiseRes(nn.Module): + + def __init__(self, num_block, channels, kernel=(3, 3), stride=1, padding=1, groups=1): + super().__init__() + + self.net = nn.Sequential(*[ + DepthWiseRes(channels, channels, kernel, stride, padding, groups) + for _ in range(num_block) + ]) + + def forward(self, x): + return self.net(x) + + +class FaceMobileNet(nn.Module): + + def __init__(self, embedding_size): + super().__init__() + self.conv1 = ConvBnPrelu(1, 64, kernel=(3, 3), stride=2, padding=1) + self.conv2 = ConvBn(64, 64, kernel=(3, 3), stride=1, padding=1, groups=64) + self.conv3 = DepthWise(64, 64, kernel=(3, 3), stride=2, padding=1, groups=128) + self.conv4 = MultiDepthWiseRes(num_block=4, channels=64, kernel=3, stride=1, padding=1, groups=128) + self.conv5 = DepthWise(64, 128, kernel=(3, 3), stride=2, padding=1, groups=256) + self.conv6 = MultiDepthWiseRes(num_block=6, channels=128, kernel=(3, 3), stride=1, padding=1, groups=256) + self.conv7 = DepthWise(128, 128, kernel=(3, 3), stride=2, padding=1, groups=512) + self.conv8 = MultiDepthWiseRes(num_block=2, channels=128, kernel=(3, 3), stride=1, padding=1, groups=256) + self.conv9 = ConvBnPrelu(128, 512, kernel=(1, 1)) + self.conv10 = ConvBn(512, 512, groups=512, kernel=(7, 7)) + self.flatten = Flatten() + self.linear = nn.Linear(2048, embedding_size, bias=False) + self.bn = nn.BatchNorm1d(embedding_size) + + def forward(self, x): + #print('x',x.shape) + out = self.conv1(x) + out = self.conv2(out) + out = self.conv3(out) + out = self.conv4(out) + out = self.conv5(out) + out = self.conv6(out) + out = self.conv7(out) + out = self.conv8(out) + out = self.conv9(out) + out = self.conv10(out) + out = self.flatten(out) + out = self.linear(out) + out = self.bn(out) + return out + +if __name__ == "__main__": + from PIL import Image + import numpy as np + + x = Image.open("../samples/009.jpg").convert('L') + x = x.resize((128, 128)) + x = np.asarray(x, dtype=np.float32) + x = x[None, None, ...] + x = torch.from_numpy(x) + net = FaceMobileNet(512) + net.eval() + with torch.no_grad(): + out = net(x) + print(out.shape) diff --git a/contrast/feat_extract/model/lcnet.py b/contrast/feat_extract/model/lcnet.py new file mode 100644 index 0000000..453e8ef --- /dev/null +++ b/contrast/feat_extract/model/lcnet.py @@ -0,0 +1,233 @@ +import os +import torch +import torch.nn as nn +import thop + +# try: +# import softpool_cuda +# from SoftPool import soft_pool2d, SoftPool2d +# except ImportError: +# print('Please install SoftPool first: https://github.com/alexandrosstergiou/SoftPool') +# exit(0) + +NET_CONFIG = { + # k, in_c, out_c, s, use_se + "blocks2": [[3, 16, 32, 1, False]], + "blocks3": [[3, 32, 64, 2, False], [3, 64, 64, 1, False]], + "blocks4": [[3, 64, 128, 2, False], [3, 128, 128, 1, False]], + "blocks5": [[3, 128, 256, 2, False], [5, 256, 256, 1, False], + [5, 256, 256, 1, False], [5, 256, 256, 1, False], + [5, 256, 256, 1, False], [5, 256, 256, 1, False]], + "blocks6": [[5, 256, 512, 2, True], [5, 512, 512, 1, True]] +} + + +def autopad(k, p=None): + if p is None: + p = k // 2 if isinstance(k, int) else [x // 2 for x in k] + return p + + +def make_divisible(v, divisor=8, min_value=None): + if min_value is None: + min_value = divisor + new_v = max(min_value, int(v + divisor / 2) // divisor * divisor) + if new_v < 0.9 * v: + new_v += divisor + return new_v + + +class HardSwish(nn.Module): + def __init__(self, inplace=True): + super(HardSwish, self).__init__() + self.relu6 = nn.ReLU6(inplace=inplace) + + def forward(self, x): + return x * self.relu6(x+3) / 6 + + +class HardSigmoid(nn.Module): + def __init__(self, inplace=True): + super(HardSigmoid, self).__init__() + self.relu6 = nn.ReLU6(inplace=inplace) + + def forward(self, x): + return (self.relu6(x+3)) / 6 + + +class SELayer(nn.Module): + def __init__(self, channel, reduction=16): + super(SELayer, self).__init__() + self.avgpool = nn.AdaptiveAvgPool2d(1) + self.fc = nn.Sequential( + nn.Linear(channel, channel // reduction, bias=False), + nn.ReLU(inplace=True), + nn.Linear(channel // reduction, channel, bias=False), + HardSigmoid() + ) + + def forward(self, x): + b, c, h, w = x.size() + y = self.avgpool(x).view(b, c) + y = self.fc(y).view(b, c, 1, 1) + return x * y.expand_as(x) + + +class DepthwiseSeparable(nn.Module): + def __init__(self, inp, oup, dw_size, stride, use_se=False): + super(DepthwiseSeparable, self).__init__() + self.use_se = use_se + self.stride = stride + self.inp = inp + self.oup = oup + self.dw_size = dw_size + self.dw_sp = nn.Sequential( + nn.Conv2d(self.inp, self.inp, kernel_size=self.dw_size, stride=self.stride, + padding=autopad(self.dw_size, None), groups=self.inp, bias=False), + nn.BatchNorm2d(self.inp), + HardSwish(), + + nn.Conv2d(self.inp, self.oup, kernel_size=1, stride=1, padding=0, bias=False), + nn.BatchNorm2d(self.oup), + HardSwish(), + ) + self.se = SELayer(self.oup) + + def forward(self, x): + x = self.dw_sp(x) + if self.use_se: + x = self.se(x) + return x + + +class PP_LCNet(nn.Module): + def __init__(self, scale=1.0, class_num=10, class_expand=1280, dropout_prob=0.2): + super(PP_LCNet, self).__init__() + self.scale = scale + self.conv1 = nn.Conv2d(3, out_channels=make_divisible(16 * self.scale), + kernel_size=3, stride=2, padding=1, bias=False) + # k, in_c, out_c, s, use_se inp, oup, dw_size, stride, use_se=False + self.blocks2 = nn.Sequential(*[ + DepthwiseSeparable(inp=make_divisible(in_c * self.scale), + oup=make_divisible(out_c * self.scale), + dw_size=k, stride=s, use_se=use_se) + for i, (k, in_c, out_c, s, use_se) in enumerate(NET_CONFIG["blocks2"]) + ]) + + self.blocks3 = nn.Sequential(*[ + DepthwiseSeparable(inp=make_divisible(in_c * self.scale), + oup=make_divisible(out_c * self.scale), + dw_size=k, stride=s, use_se=use_se) + for i, (k, in_c, out_c, s, use_se) in enumerate(NET_CONFIG["blocks3"]) + ]) + + self.blocks4 = nn.Sequential(*[ + DepthwiseSeparable(inp=make_divisible(in_c * self.scale), + oup=make_divisible(out_c * self.scale), + dw_size=k, stride=s, use_se=use_se) + for i, (k, in_c, out_c, s, use_se) in enumerate(NET_CONFIG["blocks4"]) + ]) + # k, in_c, out_c, s, use_se inp, oup, dw_size, stride, use_se=False + self.blocks5 = nn.Sequential(*[ + DepthwiseSeparable(inp=make_divisible(in_c * self.scale), + oup=make_divisible(out_c * self.scale), + dw_size=k, stride=s, use_se=use_se) + for i, (k, in_c, out_c, s, use_se) in enumerate(NET_CONFIG["blocks5"]) + ]) + + self.blocks6 = nn.Sequential(*[ + DepthwiseSeparable(inp=make_divisible(in_c * self.scale), + oup=make_divisible(out_c * self.scale), + dw_size=k, stride=s, use_se=use_se) + for i, (k, in_c, out_c, s, use_se) in enumerate(NET_CONFIG["blocks6"]) + ]) + + self.GAP = nn.AdaptiveAvgPool2d(1) + + self.last_conv = nn.Conv2d(in_channels=make_divisible(NET_CONFIG["blocks6"][-1][2] * scale), + out_channels=class_expand, + kernel_size=1, stride=1, padding=0, bias=False) + + self.hardswish = HardSwish() + self.dropout = nn.Dropout(p=dropout_prob) + + self.fc = nn.Linear(class_expand, class_num) + + def forward(self, x): + x = self.conv1(x) + print(x.shape) + x = self.blocks2(x) + print(x.shape) + x = self.blocks3(x) + print(x.shape) + x = self.blocks4(x) + print(x.shape) + x = self.blocks5(x) + print(x.shape) + x = self.blocks6(x) + print(x.shape) + + x = self.GAP(x) + x = self.last_conv(x) + x = self.hardswish(x) + x = self.dropout(x) + x = torch.flatten(x, start_dim=1, end_dim=-1) + x = self.fc(x) + return x + + +def PPLCNET_x0_25(**kwargs): + model = PP_LCNet(scale=0.25, **kwargs) + return model + + +def PPLCNET_x0_35(**kwargs): + model = PP_LCNet(scale=0.35, **kwargs) + return model + + +def PPLCNET_x0_5(**kwargs): + model = PP_LCNet(scale=0.5, **kwargs) + return model + + +def PPLCNET_x0_75(**kwargs): + model = PP_LCNet(scale=0.75, **kwargs) + return model + + +def PPLCNET_x1_0(**kwargs): + model = PP_LCNet(scale=1.0, **kwargs) + return model + + +def PPLCNET_x1_5(**kwargs): + model = PP_LCNet(scale=1.5, **kwargs) + return model + + +def PPLCNET_x2_0(**kwargs): + model = PP_LCNet(scale=2.0, **kwargs) + return model + +def PPLCNET_x2_5(**kwargs): + model = PP_LCNet(scale=2.5, **kwargs) + return model + + + + +if __name__ == '__main__': + # input = torch.randn(1, 3, 640, 640) + # model = PPLCNET_x2_5() + # flops, params = thop.profile(model, inputs=(input,)) + # print('flops:', flops / 1000000000) + # print('params:', params / 1000000) + + model = PPLCNET_x1_0() + # model_1 = PW_Conv(3, 16) + input = torch.randn(2, 3, 256, 256) + print(input.shape) + output = model(input) + print(output.shape) # [1, num_class] + diff --git a/contrast/feat_extract/model/loss.py b/contrast/feat_extract/model/loss.py new file mode 100644 index 0000000..8f40c5c --- /dev/null +++ b/contrast/feat_extract/model/loss.py @@ -0,0 +1,18 @@ +import torch +import torch.nn as nn + + +class FocalLoss(nn.Module): + + def __init__(self, gamma=2): + super().__init__() + self.gamma = gamma + self.ce = torch.nn.CrossEntropyLoss() + + def forward(self, input, target): + + #print(f'theta {input.shape, input[0]}, target {target.shape, target}') + logp = self.ce(input, target) + p = torch.exp(-logp) + loss = (1 - p) ** self.gamma * logp + return loss.mean() \ No newline at end of file diff --git a/contrast/feat_extract/model/metric.py b/contrast/feat_extract/model/metric.py new file mode 100644 index 0000000..b5b0189 --- /dev/null +++ b/contrast/feat_extract/model/metric.py @@ -0,0 +1,83 @@ +# Definition of ArcFace loss and CosFace loss + +import math +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class ArcFace(nn.Module): + + def __init__(self, embedding_size, class_num, s=30.0, m=0.50): + """ArcFace formula: + cos(m + theta) = cos(m)cos(theta) - sin(m)sin(theta) + Note that: + 0 <= m + theta <= Pi + So if (m + theta) >= Pi, then theta >= Pi - m. In [0, Pi] + we have: + cos(theta) < cos(Pi - m) + So we can use cos(Pi - m) as threshold to check whether + (m + theta) go out of [0, Pi] + + Args: + embedding_size: usually 128, 256, 512 ... + class_num: num of people when training + s: scale, see normface https://arxiv.org/abs/1704.06369 + m: margin, see SphereFace, CosFace, and ArcFace paper + """ + super().__init__() + self.in_features = embedding_size + self.out_features = class_num + self.s = s + self.m = m + self.weight = nn.Parameter(torch.FloatTensor(class_num, embedding_size)) + nn.init.xavier_uniform_(self.weight) + + self.cos_m = math.cos(m) + self.sin_m = math.sin(m) + self.th = math.cos(math.pi - m) + self.mm = math.sin(math.pi - m) * m + + def forward(self, input, label): + #print(f"embding {self.in_features}, class_num {self.out_features}, input {len(input)}, label {len(label)}") + cosine = F.linear(F.normalize(input), F.normalize(self.weight)) + # print('F.normalize(input)',input.shape) + # print('F.normalize(self.weight)',F.normalize(self.weight).shape) + sine = ((1.0 - cosine.pow(2)).clamp(0, 1)).sqrt() + phi = cosine * self.cos_m - sine * self.sin_m + phi = torch.where(cosine > self.th, phi, cosine - self.mm) # drop to CosFace + #print(f'consine {cosine.shape, cosine}, sine {sine.shape, sine}, phi {phi.shape, phi}') + # update y_i by phi in cosine + output = cosine * 1.0 # make backward works + batch_size = len(output) + output[range(batch_size), label] = phi[range(batch_size), label] + # print(f'output {(output * self.s).shape}') + # print(f'phi[range(batch_size), label] {phi[range(batch_size), label]}') + return output * self.s + + +class CosFace(nn.Module): + + def __init__(self, in_features, out_features, s=30.0, m=0.40): + """ + Args: + embedding_size: usually 128, 256, 512 ... + class_num: num of people when training + s: scale, see normface https://arxiv.org/abs/1704.06369 + m: margin, see SphereFace, CosFace, and ArcFace paper + """ + super().__init__() + self.in_features = in_features + self.out_features = out_features + self.s = s + self.m = m + self.weight = nn.Parameter(torch.FloatTensor(out_features, in_features)) + nn.init.xavier_uniform_(self.weight) + + def forward(self, input, label): + cosine = F.linear(F.normalize(input), F.normalize(self.weight)) + phi = cosine - self.m + output = cosine * 1.0 # make backward works + batch_size = len(output) + output[range(batch_size), label] = phi[range(batch_size), label] + return output * self.s \ No newline at end of file diff --git a/contrast/feat_extract/model/mobilenet_v1.py b/contrast/feat_extract/model/mobilenet_v1.py new file mode 100644 index 0000000..1262d9e --- /dev/null +++ b/contrast/feat_extract/model/mobilenet_v1.py @@ -0,0 +1,148 @@ +# Copyright 2022 Dakewe Biotech Corporation. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +from typing import Callable, Any, Optional + +import torch +from torch import Tensor +from torch import nn +from torchvision.ops.misc import Conv2dNormActivation +from config import config as conf + +__all__ = [ + "MobileNetV1", + "DepthWiseSeparableConv2d", + "mobilenet_v1", +] + + +class MobileNetV1(nn.Module): + + def __init__( + self, + num_classes: int = conf.embedding_size, + ) -> None: + super(MobileNetV1, self).__init__() + self.features = nn.Sequential( + Conv2dNormActivation(3, + 32, + kernel_size=3, + stride=2, + padding=1, + norm_layer=nn.BatchNorm2d, + activation_layer=nn.ReLU, + inplace=True, + bias=False, + ), + + DepthWiseSeparableConv2d(32, 64, 1), + DepthWiseSeparableConv2d(64, 128, 2), + DepthWiseSeparableConv2d(128, 128, 1), + DepthWiseSeparableConv2d(128, 256, 2), + DepthWiseSeparableConv2d(256, 256, 1), + DepthWiseSeparableConv2d(256, 512, 2), + DepthWiseSeparableConv2d(512, 512, 1), + DepthWiseSeparableConv2d(512, 512, 1), + DepthWiseSeparableConv2d(512, 512, 1), + DepthWiseSeparableConv2d(512, 512, 1), + DepthWiseSeparableConv2d(512, 512, 1), + DepthWiseSeparableConv2d(512, 1024, 2), + DepthWiseSeparableConv2d(1024, 1024, 1), + ) + + self.avgpool = nn.AvgPool2d((7, 7)) + + self.classifier = nn.Linear(1024, num_classes) + + # Initialize neural network weights + self._initialize_weights() + + def forward(self, x: Tensor) -> Tensor: + out = self._forward_impl(x) + + return out + + # Support torch.script function + def _forward_impl(self, x: Tensor) -> Tensor: + out = self.features(x) + out = self.avgpool(out) + out = torch.flatten(out, 1) + out = self.classifier(out) + + return out + + def _initialize_weights(self) -> None: + for module in self.modules(): + if isinstance(module, nn.Conv2d): + nn.init.kaiming_normal_(module.weight, mode="fan_out", nonlinearity="relu") + if module.bias is not None: + nn.init.zeros_(module.bias) + elif isinstance(module, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.ones_(module.weight) + nn.init.zeros_(module.bias) + elif isinstance(module, nn.Linear): + nn.init.normal_(module.weight, 0, 0.01) + nn.init.zeros_(module.bias) + + +class DepthWiseSeparableConv2d(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + stride: int, + norm_layer: Optional[Callable[..., nn.Module]] = None + ) -> None: + super(DepthWiseSeparableConv2d, self).__init__() + self.stride = stride + if stride not in [1, 2]: + raise ValueError(f"stride should be 1 or 2 instead of {stride}") + + if norm_layer is None: + norm_layer = nn.BatchNorm2d + + self.conv = nn.Sequential( + Conv2dNormActivation(in_channels, + in_channels, + kernel_size=3, + stride=stride, + padding=1, + groups=in_channels, + norm_layer=norm_layer, + activation_layer=nn.ReLU, + inplace=True, + bias=False, + ), + Conv2dNormActivation(in_channels, + out_channels, + kernel_size=1, + stride=1, + padding=0, + norm_layer=norm_layer, + activation_layer=nn.ReLU, + inplace=True, + bias=False, + ), + + ) + + def forward(self, x: Tensor) -> Tensor: + out = self.conv(x) + + return out + + +def mobilenet_v1(**kwargs: Any) -> MobileNetV1: + model = MobileNetV1(**kwargs) + + return model diff --git a/contrast/feat_extract/model/mobilenet_v2.py b/contrast/feat_extract/model/mobilenet_v2.py new file mode 100644 index 0000000..eec44d4 --- /dev/null +++ b/contrast/feat_extract/model/mobilenet_v2.py @@ -0,0 +1,200 @@ +from torch import nn +from .utils import load_state_dict_from_url +from ..config import config as conf + +__all__ = ['MobileNetV2', 'mobilenet_v2'] + + +model_urls = { + 'mobilenet_v2': 'https://download.pytorch.org/models/mobilenet_v2-b0353104.pth', +} + + +def _make_divisible(v, divisor, min_value=None): + """ + This function is taken from the original tf repo. + It ensures that all layers have a channel number that is divisible by 8 + It can be seen here: + https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py + :param v: + :param divisor: + :param min_value: + :return: + """ + if min_value is None: + min_value = divisor + new_v = max(min_value, int(v + divisor / 2) // divisor * divisor) + # Make sure that round down does not go down by more than 10%. + if new_v < 0.9 * v: + new_v += divisor + return new_v + + +class ConvBNReLU(nn.Sequential): + def __init__(self, in_planes, out_planes, kernel_size=3, stride=1, groups=1, norm_layer=None): + padding = (kernel_size - 1) // 2 + if norm_layer is None: + norm_layer = nn.BatchNorm2d + super(ConvBNReLU, self).__init__( + nn.Conv2d(in_planes, out_planes, kernel_size, stride, padding, groups=groups, bias=False), + norm_layer(out_planes), + nn.ReLU6(inplace=True) + ) + + +class InvertedResidual(nn.Module): + def __init__(self, inp, oup, stride, expand_ratio, norm_layer=None): + super(InvertedResidual, self).__init__() + self.stride = stride + assert stride in [1, 2] + + if norm_layer is None: + norm_layer = nn.BatchNorm2d + + hidden_dim = int(round(inp * expand_ratio)) + self.use_res_connect = self.stride == 1 and inp == oup + + layers = [] + if expand_ratio != 1: + # pw + layers.append(ConvBNReLU(inp, hidden_dim, kernel_size=1, norm_layer=norm_layer)) + layers.extend([ + # dw + ConvBNReLU(hidden_dim, hidden_dim, stride=stride, groups=hidden_dim, norm_layer=norm_layer), + # pw-linear + nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False), + norm_layer(oup), + ]) + self.conv = nn.Sequential(*layers) + + def forward(self, x): + if self.use_res_connect: + return x + self.conv(x) + else: + return self.conv(x) + + +class MobileNetV2(nn.Module): + def __init__(self, + num_classes=conf.embedding_size, + width_mult=1.0, + inverted_residual_setting=None, + round_nearest=8, + block=None, + norm_layer=None): + """ + MobileNet V2 main class + + Args: + num_classes (int): Number of classes + width_mult (float): Width multiplier - adjusts number of channels in each layer by this amount + inverted_residual_setting: Network structure + round_nearest (int): Round the number of channels in each layer to be a multiple of this number + Set to 1 to turn off rounding + block: Module specifying inverted residual building block for mobilenet + norm_layer: Module specifying the normalization layer to use + + """ + super(MobileNetV2, self).__init__() + + if block is None: + block = InvertedResidual + + if norm_layer is None: + norm_layer = nn.BatchNorm2d + + input_channel = 32 + last_channel = 1280 + + if inverted_residual_setting is None: + inverted_residual_setting = [ + # t, c, n, s + [1, 16, 1, 1], + [6, 24, 2, 2], + [6, 32, 3, 2], + [6, 64, 4, 2], + [6, 96, 3, 1], + [6, 160, 3, 2], + [6, 320, 1, 1], + ] + + # only check the first element, assuming user knows t,c,n,s are required + if len(inverted_residual_setting) == 0 or len(inverted_residual_setting[0]) != 4: + raise ValueError("inverted_residual_setting should be non-empty " + "or a 4-element list, got {}".format(inverted_residual_setting)) + + # building first layer + input_channel = _make_divisible(input_channel * width_mult, round_nearest) + self.last_channel = _make_divisible(last_channel * max(1.0, width_mult), round_nearest) + features = [ConvBNReLU(3, input_channel, stride=2, norm_layer=norm_layer)] + # building inverted residual blocks + for t, c, n, s in inverted_residual_setting: + output_channel = _make_divisible(c * width_mult, round_nearest) + for i in range(n): + stride = s if i == 0 else 1 + features.append(block(input_channel, output_channel, stride, expand_ratio=t, norm_layer=norm_layer)) + input_channel = output_channel + # building last several layers + features.append(ConvBNReLU(input_channel, self.last_channel, kernel_size=1, norm_layer=norm_layer)) + # make it nn.Sequential + self.features = nn.Sequential(*features) + + # building classifier + self.classifier = nn.Sequential( + nn.Dropout(0.2), + nn.Linear(self.last_channel, num_classes), + ) + + # weight initialization + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out') + if m.bias is not None: + nn.init.zeros_(m.bias) + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.ones_(m.weight) + nn.init.zeros_(m.bias) + elif isinstance(m, nn.Linear): + nn.init.normal_(m.weight, 0, 0.01) + nn.init.zeros_(m.bias) + + def _forward_impl(self, x): + # This exists since TorchScript doesn't support inheritance, so the superclass method + # (this one) needs to have a name other than `forward` that can be accessed in a subclass + x = self.features(x) + # Cannot use "squeeze" as batch-size can be 1 => must use reshape with x.shape[0] + x = nn.functional.adaptive_avg_pool2d(x, 1).reshape(x.shape[0], -1) + x = self.classifier(x) + return x + + def forward(self, x): + return self._forward_impl(x) + + +def mobilenet_v2(pretrained=True, progress=True, **kwargs): + """ + Constructs a MobileNetV2 architecture from + `"MobileNetV2: Inverted Residuals and Linear Bottlenecks" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + model = MobileNetV2(**kwargs) + if pretrained: + state_dict = load_state_dict_from_url(model_urls['mobilenet_v2'], + progress=progress) + src_state_dict = state_dict + target_state_dict = model.state_dict() + skip_keys = [] + # skip mismatch size tensors in case of pretraining + for k in src_state_dict.keys(): + if k not in target_state_dict: + continue + if src_state_dict[k].size() != target_state_dict[k].size(): + skip_keys.append(k) + for k in skip_keys: + del src_state_dict[k] + missing_keys, unexpected_keys = model.load_state_dict(src_state_dict, strict=False) + #.load_state_dict(state_dict) + return model diff --git a/contrast/feat_extract/model/mobilenet_v3.py b/contrast/feat_extract/model/mobilenet_v3.py new file mode 100644 index 0000000..953db51 --- /dev/null +++ b/contrast/feat_extract/model/mobilenet_v3.py @@ -0,0 +1,200 @@ +'''MobileNetV3 in PyTorch. + +See the paper "Inverted Residuals and Linear Bottlenecks: +Mobile Networks for Classification, Detection and Segmentation" for more details. +''' +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn import init +from ..config import config as conf + + +class hswish(nn.Module): + def forward(self, x): + out = x * F.relu6(x + 3, inplace=True) / 6 + return out + + +class hsigmoid(nn.Module): + def forward(self, x): + out = F.relu6(x + 3, inplace=True) / 6 + return out + + +class SeModule(nn.Module): + def __init__(self, in_size, reduction=4): + super(SeModule, self).__init__() + self.se = nn.Sequential( + nn.AdaptiveAvgPool2d(1), + nn.Conv2d(in_size, in_size // reduction, kernel_size=1, stride=1, padding=0, bias=False), + nn.BatchNorm2d(in_size // reduction), + nn.ReLU(inplace=True), + nn.Conv2d(in_size // reduction, in_size, kernel_size=1, stride=1, padding=0, bias=False), + nn.BatchNorm2d(in_size), + hsigmoid() + ) + + def forward(self, x): + return x * self.se(x) + + +class Block(nn.Module): + '''expand + depthwise + pointwise''' + def __init__(self, kernel_size, in_size, expand_size, out_size, nolinear, semodule, stride): + super(Block, self).__init__() + self.stride = stride + self.se = semodule + + self.conv1 = nn.Conv2d(in_size, expand_size, kernel_size=1, stride=1, padding=0, bias=False) + self.bn1 = nn.BatchNorm2d(expand_size) + self.nolinear1 = nolinear + self.conv2 = nn.Conv2d(expand_size, expand_size, kernel_size=kernel_size, stride=stride, padding=kernel_size//2, groups=expand_size, bias=False) + self.bn2 = nn.BatchNorm2d(expand_size) + self.nolinear2 = nolinear + self.conv3 = nn.Conv2d(expand_size, out_size, kernel_size=1, stride=1, padding=0, bias=False) + self.bn3 = nn.BatchNorm2d(out_size) + + self.shortcut = nn.Sequential() + if stride == 1 and in_size != out_size: + self.shortcut = nn.Sequential( + nn.Conv2d(in_size, out_size, kernel_size=1, stride=1, padding=0, bias=False), + nn.BatchNorm2d(out_size), + ) + + def forward(self, x): + out = self.nolinear1(self.bn1(self.conv1(x))) + out = self.nolinear2(self.bn2(self.conv2(out))) + out = self.bn3(self.conv3(out)) + if self.se != None: + out = self.se(out) + out = out + self.shortcut(x) if self.stride==1 else out + return out + + +class MobileNetV3_Large(nn.Module): + def __init__(self, num_classes=conf.embedding_size): + super(MobileNetV3_Large, self).__init__() + self.conv1 = nn.Conv2d(3, 16, kernel_size=3, stride=2, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(16) + self.hs1 = hswish() + + self.bneck = nn.Sequential( + Block(3, 16, 16, 16, nn.ReLU(inplace=True), None, 1), + Block(3, 16, 64, 24, nn.ReLU(inplace=True), None, 2), + Block(3, 24, 72, 24, nn.ReLU(inplace=True), None, 1), + Block(5, 24, 72, 40, nn.ReLU(inplace=True), SeModule(40), 2), + Block(5, 40, 120, 40, nn.ReLU(inplace=True), SeModule(40), 1), + Block(5, 40, 120, 40, nn.ReLU(inplace=True), SeModule(40), 1), + Block(3, 40, 240, 80, hswish(), None, 2), + Block(3, 80, 200, 80, hswish(), None, 1), + Block(3, 80, 184, 80, hswish(), None, 1), + Block(3, 80, 184, 80, hswish(), None, 1), + Block(3, 80, 480, 112, hswish(), SeModule(112), 1), + Block(3, 112, 672, 112, hswish(), SeModule(112), 1), + Block(5, 112, 672, 160, hswish(), SeModule(160), 1), + Block(5, 160, 672, 160, hswish(), SeModule(160), 2), + Block(5, 160, 960, 160, hswish(), SeModule(160), 1), + ) + + + self.conv2 = nn.Conv2d(160, 960, kernel_size=1, stride=1, padding=0, bias=False) + self.bn2 = nn.BatchNorm2d(960) + self.hs2 = hswish() + self.linear3 = nn.Linear(960, 1280) + self.bn3 = nn.BatchNorm1d(1280) + self.hs3 = hswish() + self.linear4 = nn.Linear(1280, num_classes) + self.init_params() + + def init_params(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + init.kaiming_normal_(m.weight, mode='fan_out') + if m.bias is not None: + init.constant_(m.bias, 0) + elif isinstance(m, nn.BatchNorm2d): + init.constant_(m.weight, 1) + init.constant_(m.bias, 0) + elif isinstance(m, nn.Linear): + init.normal_(m.weight, std=0.001) + if m.bias is not None: + init.constant_(m.bias, 0) + + def forward(self, x): + out = self.hs1(self.bn1(self.conv1(x))) + out = self.bneck(out) + out = self.hs2(self.bn2(self.conv2(out))) + out = F.avg_pool2d(out, conf.img_size // 32) + out = out.view(out.size(0), -1) + out = self.hs3(self.bn3(self.linear3(out))) + out = self.linear4(out) + return out + + + +class MobileNetV3_Small(nn.Module): + def __init__(self, num_classes=conf.embedding_size): + super(MobileNetV3_Small, self).__init__() + self.conv1 = nn.Conv2d(3, 16, kernel_size=3, stride=2, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(16) + self.hs1 = hswish() + + self.bneck = nn.Sequential( + Block(3, 16, 16, 16, nn.ReLU(inplace=True), SeModule(16), 2), + Block(3, 16, 72, 24, nn.ReLU(inplace=True), None, 2), + Block(3, 24, 88, 24, nn.ReLU(inplace=True), None, 1), + Block(5, 24, 96, 40, hswish(), SeModule(40), 2), + Block(5, 40, 240, 40, hswish(), SeModule(40), 1), + Block(5, 40, 240, 40, hswish(), SeModule(40), 1), + Block(5, 40, 120, 48, hswish(), SeModule(48), 1), + Block(5, 48, 144, 48, hswish(), SeModule(48), 1), + Block(5, 48, 288, 96, hswish(), SeModule(96), 2), + Block(5, 96, 576, 96, hswish(), SeModule(96), 1), + Block(5, 96, 576, 96, hswish(), SeModule(96), 1), + ) + + + self.conv2 = nn.Conv2d(96, 576, kernel_size=1, stride=1, padding=0, bias=False) + self.bn2 = nn.BatchNorm2d(576) + self.hs2 = hswish() + self.linear3 = nn.Linear(576, 1280) + self.bn3 = nn.BatchNorm1d(1280) + self.hs3 = hswish() + self.linear4 = nn.Linear(1280, num_classes) + self.init_params() + + def init_params(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + init.kaiming_normal_(m.weight, mode='fan_out') + if m.bias is not None: + init.constant_(m.bias, 0) + elif isinstance(m, nn.BatchNorm2d): + init.constant_(m.weight, 1) + init.constant_(m.bias, 0) + elif isinstance(m, nn.Linear): + init.normal_(m.weight, std=0.001) + if m.bias is not None: + init.constant_(m.bias, 0) + + def forward(self, x): + out = self.hs1(self.bn1(self.conv1(x))) + out = self.bneck(out) + out = self.hs2(self.bn2(self.conv2(out))) + out = F.avg_pool2d(out, conf.img_size // 32) + out = out.view(out.size(0), -1) + + out = self.hs3(self.bn3(self.linear3(out))) + out = self.linear4(out) + return out + + + +def test(): + net = MobileNetV3_Small() + x = torch.randn(2,3,224,224) + y = net(x) + print(y.size()) + +# test() \ No newline at end of file diff --git a/contrast/feat_extract/model/mobilevit.py b/contrast/feat_extract/model/mobilevit.py new file mode 100644 index 0000000..c4a0824 --- /dev/null +++ b/contrast/feat_extract/model/mobilevit.py @@ -0,0 +1,268 @@ +import torch +import torch.nn as nn + +from einops import rearrange + +# import sys +# sys.path.append(r"D:\DetectTracking") +from ..config import config as conf + + +def conv_1x1_bn(inp, oup): + return nn.Sequential( + nn.Conv2d(inp, oup, 1, 1, 0, bias=False), + nn.BatchNorm2d(oup), + nn.SiLU() + ) + + +def conv_nxn_bn(inp, oup, kernal_size=3, stride=1): + return nn.Sequential( + nn.Conv2d(inp, oup, kernal_size, stride, 1, bias=False), + nn.BatchNorm2d(oup), + nn.SiLU() + ) + + +class PreNorm(nn.Module): + def __init__(self, dim, fn): + super().__init__() + self.norm = nn.LayerNorm(dim) + self.fn = fn + + def forward(self, x, **kwargs): + return self.fn(self.norm(x), **kwargs) + + +class FeedForward(nn.Module): + def __init__(self, dim, hidden_dim, dropout=0.): + super().__init__() + self.net = nn.Sequential( + nn.Linear(dim, hidden_dim), + nn.SiLU(), + nn.Dropout(dropout), + nn.Linear(hidden_dim, dim), + nn.Dropout(dropout) + ) + + def forward(self, x): + return self.net(x) + + +class Attention(nn.Module): + def __init__(self, dim, heads=8, dim_head=64, dropout=0.): + super().__init__() + inner_dim = dim_head * heads + project_out = not (heads == 1 and dim_head == dim) + + self.heads = heads + self.scale = dim_head ** -0.5 + + self.attend = nn.Softmax(dim=-1) + self.to_qkv = nn.Linear(dim, inner_dim * 3, bias=False) + + self.to_out = nn.Sequential( + nn.Linear(inner_dim, dim), + nn.Dropout(dropout) + ) if project_out else nn.Identity() + + def forward(self, x): + qkv = self.to_qkv(x).chunk(3, dim=-1) + q, k, v = map(lambda t: rearrange(t, 'b p n (h d) -> b p h n d', h=self.heads), qkv) + + dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale + attn = self.attend(dots) + out = torch.matmul(attn, v) + out = rearrange(out, 'b p h n d -> b p n (h d)') + return self.to_out(out) + + +class Transformer(nn.Module): + def __init__(self, dim, depth, heads, dim_head, mlp_dim, dropout=0.): + super().__init__() + self.layers = nn.ModuleList([]) + for _ in range(depth): + self.layers.append(nn.ModuleList([ + PreNorm(dim, Attention(dim, heads, dim_head, dropout)), + PreNorm(dim, FeedForward(dim, mlp_dim, dropout)) + ])) + + def forward(self, x): + for attn, ff in self.layers: + x = attn(x) + x + x = ff(x) + x + return x + + +class MV2Block(nn.Module): + def __init__(self, inp, oup, stride=1, expansion=4): + super().__init__() + self.stride = stride + assert stride in [1, 2] + + hidden_dim = int(inp * expansion) + self.use_res_connect = self.stride == 1 and inp == oup + + if expansion == 1: + self.conv = nn.Sequential( + # dw + nn.Conv2d(hidden_dim, hidden_dim, 3, stride, 1, groups=hidden_dim, bias=False), + nn.BatchNorm2d(hidden_dim), + nn.SiLU(), + # pw-linear + nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False), + nn.BatchNorm2d(oup), + ) + else: + self.conv = nn.Sequential( + # pw + nn.Conv2d(inp, hidden_dim, 1, 1, 0, bias=False), + nn.BatchNorm2d(hidden_dim), + nn.SiLU(), + # dw + nn.Conv2d(hidden_dim, hidden_dim, 3, stride, 1, groups=hidden_dim, bias=False), + nn.BatchNorm2d(hidden_dim), + nn.SiLU(), + # pw-linear + nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False), + nn.BatchNorm2d(oup), + ) + + def forward(self, x): + if self.use_res_connect: + return x + self.conv(x) + else: + return self.conv(x) + + +class MobileViTBlock(nn.Module): + def __init__(self, dim, depth, channel, kernel_size, patch_size, mlp_dim, dropout=0.): + super().__init__() + self.ph, self.pw = patch_size + + self.conv1 = conv_nxn_bn(channel, channel, kernel_size) + self.conv2 = conv_1x1_bn(channel, dim) + + self.transformer = Transformer(dim, depth, 4, 8, mlp_dim, dropout) + + self.conv3 = conv_1x1_bn(dim, channel) + self.conv4 = conv_nxn_bn(2 * channel, channel, kernel_size) + + def forward(self, x): + y = x.clone() + + # Local representations + x = self.conv1(x) + x = self.conv2(x) + + # Global representations + _, _, h, w = x.shape + x = rearrange(x, 'b d (h ph) (w pw) -> b (ph pw) (h w) d', ph=self.ph, pw=self.pw) + x = self.transformer(x) + x = rearrange(x, 'b (ph pw) (h w) d -> b d (h ph) (w pw)', h=h // self.ph, w=w // self.pw, ph=self.ph, + pw=self.pw) + + # Fusion + x = self.conv3(x) + x = torch.cat((x, y), 1) + x = self.conv4(x) + return x + + +class MobileViT(nn.Module): + def __init__(self, image_size, dims, channels, num_classes, expansion=4, kernel_size=3, patch_size=(2, 2)): + super().__init__() + ih, iw = image_size + ph, pw = patch_size + assert ih % ph == 0 and iw % pw == 0 + + L = [2, 4, 3] + + self.conv1 = conv_nxn_bn(3, channels[0], stride=2) + + self.mv2 = nn.ModuleList([]) + self.mv2.append(MV2Block(channels[0], channels[1], 1, expansion)) + self.mv2.append(MV2Block(channels[1], channels[2], 2, expansion)) + self.mv2.append(MV2Block(channels[2], channels[3], 1, expansion)) + self.mv2.append(MV2Block(channels[2], channels[3], 1, expansion)) # Repeat + self.mv2.append(MV2Block(channels[3], channels[4], 2, expansion)) + self.mv2.append(MV2Block(channels[5], channels[6], 2, expansion)) + self.mv2.append(MV2Block(channels[7], channels[8], 2, expansion)) + + self.mvit = nn.ModuleList([]) + self.mvit.append(MobileViTBlock(dims[0], L[0], channels[5], kernel_size, patch_size, int(dims[0] * 2))) + self.mvit.append(MobileViTBlock(dims[1], L[1], channels[7], kernel_size, patch_size, int(dims[1] * 4))) + self.mvit.append(MobileViTBlock(dims[2], L[2], channels[9], kernel_size, patch_size, int(dims[2] * 4))) + + self.conv2 = conv_1x1_bn(channels[-2], channels[-1]) + + self.pool = nn.AvgPool2d(ih // 32, 1) + self.fc = nn.Linear(channels[-1], num_classes, bias=False) + + def forward(self, x): + #print('x',x.shape) + x = self.conv1(x) + x = self.mv2[0](x) + + x = self.mv2[1](x) + x = self.mv2[2](x) + x = self.mv2[3](x) # Repeat + + x = self.mv2[4](x) + x = self.mvit[0](x) + + x = self.mv2[5](x) + x = self.mvit[1](x) + + x = self.mv2[6](x) + x = self.mvit[2](x) + x = self.conv2(x) + + + #print('pool_before',x.shape) + x = self.pool(x).view(-1, x.shape[1]) + #print('self_pool',self.pool) + #print('pool_after',x.shape) + x = self.fc(x) + return x + + +def mobilevit_xxs(): + dims = [64, 80, 96] + channels = [16, 16, 24, 24, 48, 48, 64, 64, 80, 80, 320] + return MobileViT((256, 256), dims, channels, num_classes=1000, expansion=2) + + +def mobilevit_xs(): + dims = [96, 120, 144] + channels = [16, 32, 48, 48, 64, 64, 80, 80, 96, 96, 384] + return MobileViT((256, 256), dims, channels, num_classes=1000) + + +def mobilevit_s(): + dims = [144, 192, 240] + channels = [16, 32, 64, 64, 96, 96, 128, 128, 160, 160, 640] + return MobileViT((conf.img_size, conf.img_size), dims, channels, num_classes=conf.embedding_size) + + +def count_parameters(model): + return sum(p.numel() for p in model.parameters() if p.requires_grad) + + +if __name__ == '__main__': + img = torch.randn(5, 3, 256, 256) + + vit = mobilevit_xxs() + out = vit(img) + print(out.shape) + print(count_parameters(vit)) + + vit = mobilevit_xs() + out = vit(img) + print(out.shape) + print(count_parameters(vit)) + + vit = mobilevit_s() + out = vit(img) + print(out.shape) + print(count_parameters(vit)) diff --git a/contrast/feat_extract/model/resbam.py b/contrast/feat_extract/model/resbam.py new file mode 100644 index 0000000..15a88e6 --- /dev/null +++ b/contrast/feat_extract/model/resbam.py @@ -0,0 +1,145 @@ +import torch +import torch.nn as nn + +from .CBAM import CBAM +from .Tool import GeM as gem +# from model.CBAM import CBAM +# from model.Tool import GeM as gem + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inchannel, outchannel, stride=1, dowsample=None): + # super(Bottleneck, self).__init__() + super().__init__() + self.conv1 = nn.Conv2d(in_channels=inchannel, out_channels=outchannel, kernel_size=1, stride=1, bias=False) + self.bn1 = nn.BatchNorm2d(outchannel) + self.conv2 = nn.Conv2d(in_channels=outchannel, out_channels=outchannel, kernel_size=3, bias=False, + stride=stride, padding=1) + self.bn2 = nn.BatchNorm2d(outchannel) + self.conv3 = nn.Conv2d(in_channels=outchannel, out_channels=outchannel * self.expansion, stride=1, bias=False, + kernel_size=1) + self.bn3 = nn.BatchNorm2d(outchannel * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = dowsample + + def forward(self, x): + self.identity = x + # print('>>>>>>>>',type(x)) + if self.downsample is not None: + # print('>>>>downsample>>>>', type(self.downsample)) + self.identity = self.downsample(x) + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + out = self.conv3(out) + out = self.bn3(out) + # print('>>>>out>>>identity',out.size(),self.identity.size()) + out = out + self.identity + out = self.relu(out) + return out + + +class resnet(nn.Module): + def __init__(self, block=Bottleneck, block_num=[3, 4, 6, 3], num_class=1000): + super().__init__() + self.in_channel = 64 + self.conv1 = nn.Conv2d(in_channels=3, + out_channels=self.in_channel, + stride=2, + kernel_size=7, + padding=3, + bias=False) + self.bn1 = nn.BatchNorm2d(self.in_channel) + self.relu = nn.ReLU(inplace=True) + self.cbam = CBAM(self.in_channel) + self.cbam1 = CBAM(2048) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, 64, block_num[0], stride=1) + self.layer2 = self._make_layer(block, 128, block_num[1], stride=2) + self.layer3 = self._make_layer(block, 256, block_num[2], stride=2) + self.layer4 = self._make_layer(block, 512, block_num[3], stride=2) + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + self.gem = gem() + self.fc = nn.Linear(512 * block.expansion, num_class) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal(m.weight, mode='fan_out', + nonlinearity='relu') + if isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1.0) + nn.init.constant_(m.bias, 1.0) + + def _make_layer(self, block, channel, block_num, stride=1): + downsample = None + if stride != 1 or self.in_channel != channel * block.expansion: + downsample = nn.Sequential( + nn.Conv2d(self.in_channel, channel * block.expansion, kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(channel * block.expansion)) + layer = [] + layer.append(block(self.in_channel, channel, stride, downsample)) + self.in_channel = channel * block.expansion + for _ in range(1, block_num): + layer.append(block(self.in_channel, channel)) + return nn.Sequential(*layer) + + def forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + x = self.cbam(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.cbam1(x) + # x = self.avgpool(x) + x = self.gem(x) + x = torch.flatten(x, 1) + x = self.fc(x) + return x + + +class TripletNet(nn.Module): + def __init__(self, num_class, flag=True): + super(TripletNet, self).__init__() + self.initnet = rescbam(num_class) + self.flag = flag + + def forward(self, x1, x2=None, x3=None): + if self.flag: + output1 = self.initnet(x1) + output2 = self.initnet(x2) + output3 = self.initnet(x3) + return output1, output2, output3 + else: + output = self.initnet(x1) + return output + + +def rescbam(num_class): + return resnet(block=Bottleneck, block_num=[3, 4, 6, 3], num_class=num_class) + + +if __name__ == '__main__': + input1 = torch.randn(4, 3, 640, 640) + input2 = torch.randn(4, 3, 640, 640) + input3 = torch.randn(4, 3, 640, 640) + + # rescbam测试 + # Resnet50 = rescbam(512) + # output = Resnet50.forward(input1) + # print(Resnet50) + + # trnet测试 + trnet = TripletNet(512) + output = trnet(input1, input2, input3) + print(output) diff --git a/contrast/feat_extract/model/resnet.py b/contrast/feat_extract/model/resnet.py new file mode 100644 index 0000000..3c4fdf0 --- /dev/null +++ b/contrast/feat_extract/model/resnet.py @@ -0,0 +1,189 @@ +"""resnet in pytorch + + + +[1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun. + + Deep Residual Learning for Image Recognition + https://arxiv.org/abs/1512.03385v1 +""" + +import torch +import torch.nn as nn +from config import config as conf +from CBAM import CBAM + +class BasicBlock(nn.Module): + """Basic Block for resnet 18 and resnet 34 + + """ + + #BasicBlock and BottleNeck block + #have different output size + #we use class attribute expansion + #to distinct + expansion = 1 + + def __init__(self, in_channels, out_channels, stride=1): + super().__init__() + + #residual function + self.residual_function = nn.Sequential( + nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False), + nn.BatchNorm2d(out_channels), + nn.ReLU(inplace=True), + nn.Conv2d(out_channels, out_channels * BasicBlock.expansion, kernel_size=3, padding=1, bias=False), + nn.BatchNorm2d(out_channels * BasicBlock.expansion) + ) + + #shortcut + self.shortcut = nn.Sequential() + + #the shortcut output dimension is not the same with residual function + #use 1*1 convolution to match the dimension + if stride != 1 or in_channels != BasicBlock.expansion * out_channels: + self.shortcut = nn.Sequential( + nn.Conv2d(in_channels, out_channels * BasicBlock.expansion, kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(out_channels * BasicBlock.expansion) + ) + + def forward(self, x): + return nn.ReLU(inplace=True)(self.residual_function(x) + self.shortcut(x)) + +class BottleNeck(nn.Module): + """Residual block for resnet over 50 layers + + """ + expansion = 4 + def __init__(self, in_channels, out_channels, stride=1): + super().__init__() + self.residual_function = nn.Sequential( + nn.Conv2d(in_channels, out_channels, kernel_size=1, bias=False), + nn.BatchNorm2d(out_channels), + nn.ReLU(inplace=True), + nn.Conv2d(out_channels, out_channels, stride=stride, kernel_size=3, padding=1, bias=False), + nn.BatchNorm2d(out_channels), + nn.ReLU(inplace=True), + nn.Conv2d(out_channels, out_channels * BottleNeck.expansion, kernel_size=1, bias=False), + nn.BatchNorm2d(out_channels * BottleNeck.expansion), + ) + + self.shortcut = nn.Sequential() + + if stride != 1 or in_channels != out_channels * BottleNeck.expansion: + self.shortcut = nn.Sequential( + nn.Conv2d(in_channels, out_channels * BottleNeck.expansion, stride=stride, kernel_size=1, bias=False), + nn.BatchNorm2d(out_channels * BottleNeck.expansion) + ) + + def forward(self, x): + return nn.ReLU(inplace=True)(self.residual_function(x) + self.shortcut(x)) + +class ResNet(nn.Module): + + def __init__(self, block, num_block, cbam = False, num_classes=conf.embedding_size): + super().__init__() + + self.in_channels = 64 + + # self.conv1 = nn.Sequential( + # nn.Conv2d(3, 64, kernel_size=3, padding=1, bias=False), + # nn.BatchNorm2d(64), + # nn.ReLU(inplace=True)) + + self.conv1 = nn.Sequential( + nn.Conv2d(3, 64,stride=2,kernel_size=7,padding=3,bias=False), + nn.BatchNorm2d(64), + nn.ReLU(inplace=True), + nn.MaxPool2d(kernel_size=3, stride=2, padding=1)) + + self.cbam = CBAM(self.in_channels) + + #we use a different inputsize than the original paper + #so conv2_x's stride is 1 + self.conv2_x = self._make_layer(block, 64, num_block[0], 1) + self.conv3_x = self._make_layer(block, 128, num_block[1], 2) + self.conv4_x = self._make_layer(block, 256, num_block[2], 2) + self.conv5_x = self._make_layer(block, 512, num_block[3], 2) + self.cbam1 = CBAM(self.in_channels) + self.avg_pool = nn.AdaptiveAvgPool2d((1, 1)) + self.fc = nn.Linear(512 * block.expansion, num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal(m.weight,mode = 'fan_out', + nonlinearity='relu') + if isinstance(m, (nn.BatchNorm2d)): + nn.init.constant_(m.weight, 1.0) + nn.init.constant_(m.bias, 1.0) + + def _make_layer(self, block, out_channels, num_blocks, stride): + """make resnet layers(by layer i didnt mean this 'layer' was the + same as a neuron netowork layer, ex. conv layer), one layer may + contain more than one residual block + + Args: + block: block type, basic block or bottle neck block + out_channels: output depth channel number of this layer + num_blocks: how many blocks per layer + stride: the stride of the first block of this layer + + Return: + return a resnet layer + """ + + # we have num_block blocks per layer, the first block + # could be 1 or 2, other blocks would always be 1 + strides = [stride] + [1] * (num_blocks - 1) + layers = [] + for stride in strides: + layers.append(block(self.in_channels, out_channels, stride)) + self.in_channels = out_channels * block.expansion + + return nn.Sequential(*layers) + + def forward(self, x): + output = self.conv1(x) + if cbam: + output = self.cbam(x) + output = self.conv2_x(output) + output = self.conv3_x(output) + output = self.conv4_x(output) + output = self.conv5_x(output) + if cbam: + output = self.cbam1(x) + print('pollBefore',output.shape) + output = self.avg_pool(output) + print('poolAfter',output.shape) + output = output.view(output.size(0), -1) + print('fcBefore',output.shape) + output = self.fc(output) + + return output + +def resnet18(cbam = False): + """ return a ResNet 18 object + """ + return ResNet(BasicBlock, [2, 2, 2, 2], cbam) + +def resnet34(): + """ return a ResNet 34 object + """ + return ResNet(BasicBlock, [3, 4, 6, 3]) + +def resnet50(): + """ return a ResNet 50 object + """ + return ResNet(BottleNeck, [3, 4, 6, 3]) + +def resnet101(): + """ return a ResNet 101 object + """ + return ResNet(BottleNeck, [3, 4, 23, 3]) + +def resnet152(): + """ return a ResNet 152 object + """ + return ResNet(BottleNeck, [3, 8, 36, 3]) + + diff --git a/contrast/feat_extract/model/resnet_face.py b/contrast/feat_extract/model/resnet_face.py new file mode 100644 index 0000000..86f1b2b --- /dev/null +++ b/contrast/feat_extract/model/resnet_face.py @@ -0,0 +1,121 @@ +""" Resnet_IR_SE in ArcFace """ + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class Flatten(nn.Module): + def forward(self, x): + return x.reshape(x.shape[0], -1) + + +class SEConv(nn.Module): + """Use Convolution instead of FullyConnection in SE""" + + def __init__(self, channels, reduction): + super().__init__() + self.net = nn.Sequential( + nn.AdaptiveAvgPool2d(1), + nn.Conv2d(channels, channels // reduction, kernel_size=1, bias=False), + nn.ReLU(inplace=True), + nn.Conv2d(channels // reduction, channels, kernel_size=1, bias=False), + nn.Sigmoid(), + ) + + def forward(self, x): + return self.net(x) * x + + +class SE(nn.Module): + + def __init__(self, channels, reduction): + super().__init__() + self.net = nn.Sequential( + nn.AdaptiveAvgPool2d(1), + nn.Linear(channels, channels // reduction), + nn.ReLU(inplace=True), + nn.Linear(channels // reduction, channels), + nn.Sigmoid(), + ) + + def forward(self, x): + return self.net(x) * x + + +class IRSE(nn.Module): + + def __init__(self, channels, depth, stride): + super().__init__() + if channels == depth: + self.shortcut = nn.MaxPool2d(kernel_size=1, stride=stride) + else: + self.shortcut = nn.Sequential( + nn.Conv2d(channels, depth, (1, 1), stride, bias=False), + nn.BatchNorm2d(depth), + ) + self.residual = nn.Sequential( + nn.BatchNorm2d(channels), + nn.Conv2d(channels, depth, (3, 3), 1, 1, bias=False), + nn.PReLU(depth), + nn.Conv2d(depth, depth, (3, 3), stride, 1, bias=False), + nn.BatchNorm2d(depth), + SEConv(depth, 16), + ) + + def forward(self, x): + return self.shortcut(x) + self.residual(x) + + +class ResIRSE(nn.Module): + """Resnet50-IRSE backbone""" + + def __init__(self, ih, embedding_size, drop_ratio): + super().__init__() + ih_last = ih // 16 + self.input_layer = nn.Sequential( + nn.Conv2d(3, 64, (3, 3), 1, 1, bias=False), + nn.BatchNorm2d(64), + nn.PReLU(64), + ) + self.output_layer = nn.Sequential( + nn.BatchNorm2d(512), + nn.Dropout(drop_ratio), + Flatten(), + nn.Linear(512 * ih_last * ih_last, embedding_size), + nn.BatchNorm1d(embedding_size), + ) + + # ["channels", "depth", "stride"], + self.res50_arch = [ + [64, 64, 2], [64, 64, 1], [64, 64, 1], + [64, 128, 2], [128, 128, 1], [128, 128, 1], [128, 128, 1], + [128, 256, 2], [256, 256, 1], [256, 256, 1], [256, 256, 1], [256, 256, 1], + [256, 256, 1], [256, 256, 1], [256, 256, 1], [256, 256, 1], [256, 256, 1], + [256, 256, 1], [256, 256, 1], [256, 256, 1], [256, 256, 1], + [256, 512, 2], [512, 512, 1], [512, 512, 1], + ] + + self.body = nn.Sequential(*[IRSE(a, b, c) for (a, b, c) in self.res50_arch]) + + def forward(self, x): + x = self.input_layer(x) + x = self.body(x) + x = self.output_layer(x) + return x + + +if __name__ == "__main__": + from PIL import Image + import numpy as np + + x = Image.open("../samples/009.jpg").convert('L') + x = x.resize((128, 128)) + x = np.asarray(x, dtype=np.float32) + x = x[None, None, ...] + x = torch.from_numpy(x) + net = ResIRSE(512, 0.6) + net.eval() + with torch.no_grad(): + out = net(x) + print(out.shape) diff --git a/contrast/feat_extract/model/resnet_pre.py b/contrast/feat_extract/model/resnet_pre.py new file mode 100644 index 0000000..3ea6441 --- /dev/null +++ b/contrast/feat_extract/model/resnet_pre.py @@ -0,0 +1,462 @@ +import torch +import torch.nn as nn +from ..config import config as conf + +try: + from torch.hub import load_state_dict_from_url +except ImportError: + from torch.utils.model_zoo import load_url as load_state_dict_from_url +# from .utils import load_state_dict_from_url + +__all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', + 'resnet152', 'resnext50_32x4d', 'resnext101_32x8d', + 'wide_resnet50_2', 'wide_resnet101_2'] + +model_urls = { + 'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth', + 'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth', + 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth', + 'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth', + 'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth', + 'resnext50_32x4d': 'https://download.pytorch.org/models/resnext50_32x4d-7cdf4587.pth', + 'resnext101_32x8d': 'https://download.pytorch.org/models/resnext101_32x8d-8ba56ff5.pth', + 'wide_resnet50_2': 'https://download.pytorch.org/models/wide_resnet50_2-95faca4d.pth', + 'wide_resnet101_2': 'https://download.pytorch.org/models/wide_resnet101_2-32ee1156.pth', +} + + +def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1): + """3x3 convolution with padding""" + return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=dilation, groups=groups, bias=False, dilation=dilation) + +def conv1x1(in_planes, out_planes, stride=1): + """1x1 convolution""" + return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +class SpatialAttention(nn.Module): + def __init__(self, kernel_size=7): + super(SpatialAttention, self).__init__() + + assert kernel_size in (3, 7), 'kernel size must be 3 or 7' + padding = 3 if kernel_size == 7 else 1 + + self.conv1 = nn.Conv2d(2, 1, kernel_size, padding=padding, bias=False) + self.sigmoid = nn.Sigmoid() + + def forward(self, x): + avg_out = torch.mean(x, dim=1, keepdim=True) + max_out, _ = torch.max(x, dim=1, keepdim=True) + x = torch.cat([avg_out, max_out], dim=1) + x = self.conv1(x) + return self.sigmoid(x) + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1, + base_width=64, dilation=1, norm_layer=None, cam=False, bam=False): + super(BasicBlock, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + if groups != 1 or base_width != 64: + raise ValueError('BasicBlock only supports groups=1 and base_width=64') + if dilation > 1: + raise NotImplementedError("Dilation > 1 not supported in BasicBlock") + self.cam = cam + self.bam = bam + # Both self.conv1 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = norm_layer(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = norm_layer(planes) + self.downsample = downsample + self.stride = stride + if self.cam: + if planes == 64: + self.globalAvgPool = nn.AvgPool2d(56, stride=1) + elif planes == 128: + self.globalAvgPool = nn.AvgPool2d(28, stride=1) + elif planes == 256: + self.globalAvgPool = nn.AvgPool2d(14, stride=1) + elif planes == 512: + self.globalAvgPool = nn.AvgPool2d(7, stride=1) + + self.fc1 = nn.Linear(in_features=planes, out_features=round(planes / 16)) + self.fc2 = nn.Linear(in_features=round(planes / 16), out_features=planes) + self.sigmod = nn.Sigmoid() + if self.bam: + self.bam = SpatialAttention() + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + identity = self.downsample(x) + + if self.cam: + ori_out = self.globalAvgPool(out) + out = out.view(out.size(0), -1) + out = self.fc1(out) + out = self.relu(out) + out = self.fc2(out) + out = self.sigmod(out) + out = out.view(out.size(0), out.size(-1), 1, 1) + out = out * ori_out + + if self.bam: + out = out*self.bam(out) + + out += identity + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2) + # while original implementation places the stride at the first 1x1 convolution(self.conv1) + # according to "Deep residual learning for image recognition"https://arxiv.org/abs/1512.03385. + # This variant is also known as ResNet V1.5 and improves accuracy according to + # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch. + + expansion = 4 + + def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1, + base_width=64, dilation=1, norm_layer=None, cam=False, bam=False): + super(Bottleneck, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + width = int(planes * (base_width / 64.)) * groups + self.cam = cam + self.bam = bam + # Both self.conv2 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv1x1(inplanes, width) + self.bn1 = norm_layer(width) + self.conv2 = conv3x3(width, width, stride, groups, dilation) + self.bn2 = norm_layer(width) + self.conv3 = conv1x1(width, planes * self.expansion) + self.bn3 = norm_layer(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + if self.cam: + if planes == 64: + self.globalAvgPool = nn.AvgPool2d(56, stride=1) + elif planes == 128: + self.globalAvgPool = nn.AvgPool2d(28, stride=1) + elif planes == 256: + self.globalAvgPool = nn.AvgPool2d(14, stride=1) + elif planes == 512: + self.globalAvgPool = nn.AvgPool2d(7, stride=1) + + self.fc1 = nn.Linear(planes * self.expansion, round(planes / 4)) + self.fc2 = nn.Linear(round(planes / 4), planes * self.expansion) + self.sigmod = nn.Sigmoid() + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + if self.cam: + ori_out = self.globalAvgPool(out) + out = out.view(out.size(0), -1) + out = self.fc1(out) + out = self.relu(out) + out = self.fc2(out) + out = self.sigmod(out) + out = out.view(out.size(0), out.size(-1), 1, 1) + out = out * ori_out + out += identity + out = self.relu(out) + return out + + +class ResNet(nn.Module): + + def __init__(self, block, layers, num_classes=conf.embedding_size, zero_init_residual=False, + groups=1, width_per_group=64, replace_stride_with_dilation=None, + norm_layer=None, scale=0.75): + super(ResNet, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + self._norm_layer = norm_layer + + self.inplanes = 64 + self.dilation = 1 + if replace_stride_with_dilation is None: + # each element in the tuple indicates if we should replace + # the 2x2 stride with a dilated convolution instead + replace_stride_with_dilation = [False, False, False] + if len(replace_stride_with_dilation) != 3: + raise ValueError("replace_stride_with_dilation should be None " + "or a 3-element tuple, got {}".format(replace_stride_with_dilation)) + self.groups = groups + self.base_width = width_per_group + self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3, + bias=False) + self.bn1 = norm_layer(self.inplanes) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, int(64*scale), layers[0]) + self.layer2 = self._make_layer(block, int(128*scale), layers[1], stride=2, + dilate=replace_stride_with_dilation[0]) + self.layer3 = self._make_layer(block, int(256*scale), layers[2], stride=2, + dilate=replace_stride_with_dilation[1]) + self.layer4 = self._make_layer(block, int(512*scale), layers[3], stride=2, + dilate=replace_stride_with_dilation[2]) + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + self.fc = nn.Linear(int(512 * block.expansion*scale), num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + # Zero-initialize the last BN in each residual branch, + # so that the residual branch starts with zeros, and each residual block behaves like an identity. + # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 + if zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + nn.init.constant_(m.bn3.weight, 0) + elif isinstance(m, BasicBlock): + nn.init.constant_(m.bn2.weight, 0) + + def _make_layer(self, block, planes, blocks, stride=1, dilate=False): + norm_layer = self._norm_layer + downsample = None + previous_dilation = self.dilation + if dilate: + self.dilation *= stride + stride = 1 + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1(self.inplanes, planes * block.expansion, stride), + norm_layer(planes * block.expansion), + ) + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample, self.groups, + self.base_width, previous_dilation, norm_layer)) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append(block(self.inplanes, planes, groups=self.groups, + base_width=self.base_width, dilation=self.dilation, + norm_layer=norm_layer)) + return nn.Sequential(*layers) + + def _forward_impl(self, x): + # See note [TorchScript super()] + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + # print('poolBefore', x.shape) + x = self.avgpool(x) + # print('poolAfter', x.shape) + x = torch.flatten(x, 1) + # print('fcBefore',x.shape) + x = self.fc(x) + + # print('fcAfter',x.shape) + + return x + + def forward(self, x): + return self._forward_impl(x) + + +# def _resnet(arch, block, layers, pretrained, progress, **kwargs): +# model = ResNet(block, layers, **kwargs) +# if pretrained: +# state_dict = load_state_dict_from_url(model_urls[arch], +# progress=progress) +# model.load_state_dict(state_dict, strict=False) +# return model +def _resnet(arch, block, layers, pretrained, progress, **kwargs): + model = ResNet(block, layers, **kwargs) + if pretrained: + state_dict = load_state_dict_from_url(model_urls[arch], + progress=progress) + + src_state_dict = state_dict + target_state_dict = model.state_dict() + skip_keys = [] + # skip mismatch size tensors in case of pretraining + for k in src_state_dict.keys(): + if k not in target_state_dict: + continue + if src_state_dict[k].size() != target_state_dict[k].size(): + skip_keys.append(k) + for k in skip_keys: + del src_state_dict[k] + missing_keys, unexpected_keys = model.load_state_dict(src_state_dict, strict=False) + + return model + + +def resnet14(pretrained=True, progress=True, **kwargs): + r"""ResNet-14 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet18', BasicBlock, [2, 1, 1, 2], pretrained, progress, + **kwargs) + + +def resnet18(pretrained=True, progress=True, **kwargs): + r"""ResNet-18 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet18', BasicBlock, [2, 2, 2, 2], pretrained, progress, + **kwargs) + + +def resnet34(pretrained=False, progress=True, **kwargs): + r"""ResNet-34 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet34', BasicBlock, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet50(pretrained=False, progress=True, **kwargs): + r"""ResNet-50 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet50', Bottleneck, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet101(pretrained=False, progress=True, **kwargs): + r"""ResNet-101 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet101', Bottleneck, [3, 4, 23, 3], pretrained, progress, + **kwargs) + + +def resnet152(pretrained=False, progress=True, **kwargs): + r"""ResNet-152 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet152', Bottleneck, [3, 8, 36, 3], pretrained, progress, + **kwargs) + + +def resnext50_32x4d(pretrained=False, progress=True, **kwargs): + r"""ResNeXt-50 32x4d model from + `"Aggregated Residual Transformation for Deep Neural Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 4 + return _resnet('resnext50_32x4d', Bottleneck, [3, 4, 6, 3], + pretrained, progress, **kwargs) + + +def resnext101_32x8d(pretrained=False, progress=True, **kwargs): + r"""ResNeXt-101 32x8d model from + `"Aggregated Residual Transformation for Deep Neural Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 8 + return _resnet('resnext101_32x8d', Bottleneck, [3, 4, 23, 3], + pretrained, progress, **kwargs) + + +def wide_resnet50_2(pretrained=False, progress=True, **kwargs): + r"""Wide ResNet-50-2 model from + `"Wide Residual Networks" `_ + + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['width_per_group'] = 64 * 2 + return _resnet('wide_resnet50_2', Bottleneck, [3, 4, 6, 3], + pretrained, progress, **kwargs) + + +def wide_resnet101_2(pretrained=False, progress=True, **kwargs): + r"""Wide ResNet-101-2 model from + `"Wide Residual Networks" `_ + + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['width_per_group'] = 64 * 2 + return _resnet('wide_resnet101_2', Bottleneck, [3, 4, 23, 3], + pretrained, progress, **kwargs) diff --git a/contrast/feat_extract/model/utils.py b/contrast/feat_extract/model/utils.py new file mode 100644 index 0000000..638ef07 --- /dev/null +++ b/contrast/feat_extract/model/utils.py @@ -0,0 +1,4 @@ +try: + from torch.hub import load_state_dict_from_url +except ImportError: + from torch.utils.model_zoo import load_url as load_state_dict_from_url diff --git a/contrast/feat_extract/model/vit.py b/contrast/feat_extract/model/vit.py new file mode 100644 index 0000000..a6231f1 --- /dev/null +++ b/contrast/feat_extract/model/vit.py @@ -0,0 +1,137 @@ +import torch +from vit_pytorch.mobile_vit import MobileViT +from vit_pytorch import vit +from vit_pytorch import SimpleViT +import torch +from torch import nn + +from einops import rearrange, repeat +from einops.layers.torch import Rearrange + + +# helpers + +def pair(t): + return t if isinstance(t, tuple) else (t, t) + + +# classes + +class FeedForward(nn.Module): + def __init__(self, dim, hidden_dim, dropout=0.): + super().__init__() + self.net = nn.Sequential( + nn.LayerNorm(dim), + nn.Linear(dim, hidden_dim), + nn.GELU(), + nn.Dropout(dropout), + nn.Linear(hidden_dim, dim), + nn.Dropout(dropout) + ) + + def forward(self, x): + return self.net(x) + + +class Attention(nn.Module): + def __init__(self, dim, heads=8, dim_head=64, dropout=0.): + super().__init__() + inner_dim = dim_head * heads + project_out = not (heads == 1 and dim_head == dim) + + self.heads = heads + self.scale = dim_head ** -0.5 + + self.norm = nn.LayerNorm(dim) + + self.attend = nn.Softmax(dim=-1) + self.dropout = nn.Dropout(dropout) + + self.to_qkv = nn.Linear(dim, inner_dim * 3, bias=False) + + self.to_out = nn.Sequential( + nn.Linear(inner_dim, dim), + nn.Dropout(dropout) + ) if project_out else nn.Identity() + + def forward(self, x): + x = self.norm(x) + + qkv = self.to_qkv(x).chunk(3, dim=-1) + q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h=self.heads), qkv) + + dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale + + attn = self.attend(dots) + attn = self.dropout(attn) + + out = torch.matmul(attn, v) + out = rearrange(out, 'b h n d -> b n (h d)') + return self.to_out(out) + + +class Transformer(nn.Module): + def __init__(self, dim, depth, heads, dim_head, mlp_dim, dropout=0.): + super().__init__() + self.norm = nn.LayerNorm(dim) + self.layers = nn.ModuleList([]) + for _ in range(depth): + self.layers.append(nn.ModuleList([ + Attention(dim, heads=heads, dim_head=dim_head, dropout=dropout), + FeedForward(dim, mlp_dim, dropout=dropout) + ])) + + def forward(self, x): + for attn, ff in self.layers: + x = attn(x) + x + x = ff(x) + x + + return self.norm(x) + + +class ViT(nn.Module): + def __init__(self, *, image_size, patch_size, num_classes, dim, depth, heads, mlp_dim, pool='cls', channels=3, + dim_head=64, dropout=0., emb_dropout=0.): + super().__init__() + image_height, image_width = pair(image_size) + patch_height, patch_width = pair(patch_size) + + assert image_height % patch_height == 0 and image_width % patch_width == 0, 'Image dimensions must be divisible by the patch size.' + + num_patches = (image_height // patch_height) * (image_width // patch_width) + patch_dim = channels * patch_height * patch_width + assert pool in {'cls', 'mean'}, 'pool type must be either cls (cls token) or mean (mean pooling)' + + self.to_patch_embedding = nn.Sequential( + Rearrange('b c (h p1) (w p2) -> b (h w) (p1 p2 c)', p1=patch_height, p2=patch_width), + nn.LayerNorm(patch_dim), + nn.Linear(patch_dim, dim), + nn.LayerNorm(dim), + ) + + self.pos_embedding = nn.Parameter(torch.randn(1, num_patches + 1, dim)) + self.cls_token = nn.Parameter(torch.randn(1, 1, dim)) + self.dropout = nn.Dropout(emb_dropout) + + self.transformer = Transformer(dim, depth, heads, dim_head, mlp_dim, dropout) + + self.pool = pool + self.to_latent = nn.Identity() + + self.mlp_head = nn.Linear(dim, num_classes) + + def forward(self, img): + x = self.to_patch_embedding(img) + b, n, _ = x.shape + + cls_tokens = repeat(self.cls_token, '1 1 d -> b 1 d', b=b) + x = torch.cat((cls_tokens, x), dim=1) + x += self.pos_embedding[:, :(n + 1)] + x = self.dropout(x) + + x = self.transformer(x) + + x = x.mean(dim=1) if self.pool == 'mean' else x[:, 0] + + x = self.to_latent(x) + return self.mlp_head(x) diff --git a/contrast/feat_extract/resnet_vit/.idea/.gitignore b/contrast/feat_extract/resnet_vit/.idea/.gitignore new file mode 100644 index 0000000..35410ca --- /dev/null +++ b/contrast/feat_extract/resnet_vit/.idea/.gitignore @@ -0,0 +1,8 @@ +# 默认忽略的文件 +/shelf/ +/workspace.xml +# 基于编辑器的 HTTP 客户端请求 +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/contrast/feat_extract/resnet_vit/.idea/contrastInference.iml b/contrast/feat_extract/resnet_vit/.idea/contrastInference.iml new file mode 100644 index 0000000..6d6038d --- /dev/null +++ b/contrast/feat_extract/resnet_vit/.idea/contrastInference.iml @@ -0,0 +1,12 @@ + + + + + + + + + + \ No newline at end of file diff --git a/contrast/feat_extract/resnet_vit/.idea/deployment.xml b/contrast/feat_extract/resnet_vit/.idea/deployment.xml new file mode 100644 index 0000000..b7f9a78 --- /dev/null +++ b/contrast/feat_extract/resnet_vit/.idea/deployment.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/contrast/feat_extract/resnet_vit/.idea/inspectionProfiles/Project_Default.xml b/contrast/feat_extract/resnet_vit/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 0000000..920d523 --- /dev/null +++ b/contrast/feat_extract/resnet_vit/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,12 @@ + + + + \ No newline at end of file diff --git a/contrast/feat_extract/resnet_vit/.idea/inspectionProfiles/profiles_settings.xml b/contrast/feat_extract/resnet_vit/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/contrast/feat_extract/resnet_vit/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/contrast/feat_extract/resnet_vit/.idea/misc.xml b/contrast/feat_extract/resnet_vit/.idea/misc.xml new file mode 100644 index 0000000..3afa107 --- /dev/null +++ b/contrast/feat_extract/resnet_vit/.idea/misc.xml @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/contrast/feat_extract/resnet_vit/.idea/modules.xml b/contrast/feat_extract/resnet_vit/.idea/modules.xml new file mode 100644 index 0000000..316bf04 --- /dev/null +++ b/contrast/feat_extract/resnet_vit/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/contrast/feat_extract/resnet_vit/__init__.py b/contrast/feat_extract/resnet_vit/__init__.py new file mode 100644 index 0000000..e3e0f3f --- /dev/null +++ b/contrast/feat_extract/resnet_vit/__init__.py @@ -0,0 +1 @@ +# from .config import config \ No newline at end of file diff --git a/contrast/feat_extract/resnet_vit/__pycache__/__init__.cpython-39.pyc b/contrast/feat_extract/resnet_vit/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..f894aad Binary files /dev/null and b/contrast/feat_extract/resnet_vit/__pycache__/__init__.cpython-39.pyc differ diff --git a/contrast/feat_extract/resnet_vit/__pycache__/config.cpython-38.pyc b/contrast/feat_extract/resnet_vit/__pycache__/config.cpython-38.pyc new file mode 100644 index 0000000..c6934bc Binary files /dev/null and b/contrast/feat_extract/resnet_vit/__pycache__/config.cpython-38.pyc differ diff --git a/contrast/feat_extract/resnet_vit/__pycache__/config.cpython-39.pyc b/contrast/feat_extract/resnet_vit/__pycache__/config.cpython-39.pyc new file mode 100644 index 0000000..ba23f1d Binary files /dev/null and b/contrast/feat_extract/resnet_vit/__pycache__/config.cpython-39.pyc differ diff --git a/contrast/feat_extract/resnet_vit/__pycache__/inference.cpython-39.pyc b/contrast/feat_extract/resnet_vit/__pycache__/inference.cpython-39.pyc new file mode 100644 index 0000000..d6609ae Binary files /dev/null and b/contrast/feat_extract/resnet_vit/__pycache__/inference.cpython-39.pyc differ diff --git a/contrast/feat_extract/resnet_vit/config.py b/contrast/feat_extract/resnet_vit/config.py new file mode 100644 index 0000000..aea7680 --- /dev/null +++ b/contrast/feat_extract/resnet_vit/config.py @@ -0,0 +1,84 @@ +import torch +import torchvision.transforms as T + + +class Config: + # network settings + backbone = 'vit' # [resnet18, mobilevit_s, mobilenet_v2, mobilenetv3_small, mobilenetv3_large, mobilenet_v1, PPLCNET_x1_0, PPLCNET_x0_5, PPLCNET_x2_5] + metric = 'softmax' # [cosface, arcface, softmax] + cbam = True + embedding_size = 256 # 256 + drop_ratio = 0.5 + img_size = 224 + + teacher = 'vit' # [resnet18, mobilevit_s, mobilenet_v2, mobilenetv3_small, mobilenetv3_large, mobilenet_v1, PPLCNET_x1_0, PPLCNET_x0_5, PPLCNET_x2_5] + student = 'resnet' + # data preprocess + # input_shape = [1, 128, 128] + """transforms.RandomCrop(size), + transforms.RandomVerticalFlip(p=0.5), + transforms.RandomHorizontalFlip(), + RandomRotate(15, 0.3), + # RandomGaussianBlur()""" + + train_transform = T.Compose([ + T.ToTensor(), + T.Resize((img_size, img_size)), + # T.RandomCrop(img_size*4//5), + # T.RandomHorizontalFlip(p=0.5), + T.RandomRotation(180), + T.ColorJitter(brightness=0.5), + T.ConvertImageDtype(torch.float32), + T.Normalize(mean=[0.5], std=[0.5]), + ]) + test_transform = T.Compose([ + T.ToTensor(), + T.Resize((img_size, img_size)), + T.ConvertImageDtype(torch.float32), + T.Normalize(mean=[0.5], std=[0.5]), + ]) + + # dataset + train_root = './data/2250_train/train' # 初始筛选过一次的数据集 + # train_root = './data/0625_train/train' + test_root = "./data/2250_train/val/" + # test_root = "./data/0625_train/val" + + test_list = "./data/2250_train/val_pair.txt" + test_group_json = "./data/2250_train/cross_same.json" + # test_group_json = "./data/0625_train/cross_same.json" + # test_list = "./data/test_data_100/val_pair.txt" + + # training settings + checkpoints = "checkpoints/vit_b_16_0815/" # [resnet18, mobilevit_s, mobilenet_v2, mobilenetv3] + restore = True + # restore_model = "checkpoints/renet18_2250_0315/best_resnet18_2250_0315.pth" # best_resnet18_1491_0306.pth + restore_model = "checkpoints/vit_b_16_0730/best.pth" # best_resnet18_1491_0306.pth + + # test_model = "./checkpoints/renet18_1887_0311/best_resnet18_1887_0311.pth" + testbackbone = 'resnet18' # [resnet18, mobilevit_s, mobilenet_v2, mobilenetv3_small, mobilenetv3_large, mobilenet_v1, PPLCNET_x1_0, PPLCNET_x0_5] + # test_val = "./data/2250_train" + test_val = "./data/0625_train" + test_model = "checkpoints/resnet18_0721/best.pth" + + train_batch_size = 128 # 256 + test_batch_size = 256 # 256 + + + epoch = 300 + optimizer = 'adamw' # ['sgd', 'adam'， 'adamw'] + lr = 1e-3 # 1e-2 + lr_step = 10 # 10 + lr_decay = 0.95 # 0.98 + weight_decay = 5e-4 + loss = 'focal_loss' # ['focal_loss', 'cross_entropy'] + device = torch.device('cuda:1' if torch.cuda.is_available() else 'cpu') + # device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') + + pin_memory = True # if memory is large, set it True to speed up a bit + num_workers = 4 # dataloader + + group_test = True + # group_test = False + +config = Config() \ No newline at end of file diff --git a/contrast/feat_extract/resnet_vit/inference.py b/contrast/feat_extract/resnet_vit/inference.py new file mode 100644 index 0000000..e7a6dbc --- /dev/null +++ b/contrast/feat_extract/resnet_vit/inference.py @@ -0,0 +1,103 @@ +import os +import os.path as osp + +import torch + +import numpy as np +from model import resnet18 +from PIL import Image + +from torch.nn.functional import softmax +from config import config as conf +import time + +embedding_size = conf.embedding_size +img_size = conf.img_size +device = conf.device + +def load_contrast_model(): + model = resnet18().to(conf.device) + model.load_state_dict(torch.load(conf.test_model, map_location=conf.device)) + model.eval() + print('load model {} '.format(conf.testbackbone)) + + return model + + +def group_image(imageDirs, batch) -> list: + images = [] + """Group image paths by batch size""" + with os.scandir(imageDirs) as entries: + for imgpth in entries: + print(imgpth) + images.append(os.sep.join([imageDirs, imgpth.name])) + print(f"{len(images)} images in {imageDirs}") + size = len(images) + res = [] + for i in range(0, size, batch): + end = min(batch + i, size) + res.append(images[i: end]) + return res + +def test_preprocess(images: list, transform) -> torch.Tensor: + res = [] + for img in images: + # print(img) + im = Image.open(img) + im = transform(im) + res.append(im) + # data = torch.cat(res, dim=0) # shape: (batch, 128, 128) + # data = data[:, None, :, :] # shape: (batch, 1, 128, 128) + data = torch.stack(res) + return data + +def featurize(images: list, transform, net, device) -> dict: + """featurize each image and save into a dictionary + Args: + images: image paths + transform: test transform + net: pretrained model + device: cpu or cuda + Returns: + Dict (key: imagePath, value: feature) + """ + data = test_preprocess(images, transform) + data = data.to(device) + net = net.to(device) + with torch.no_grad(): + features = net(data) + # res = {img: feature for (img, feature) in zip(images, features)} + return features + + + +if __name__ == '__main__': + # Network Setup + if conf.testbackbone == 'resnet18': + model = resnet18().to(device) + else: + raise ValueError('Have not model {}'.format(conf.backbone)) + + print('load model {} '.format(conf.testbackbone)) + # model = nn.DataParallel(model).to(conf.device) + model.load_state_dict(torch.load(conf.test_model, map_location=conf.device)) + model.eval() + + # images = unique_image(conf.test_list) + # images = [osp.join(conf.test_val, img) for img in images] + # print('images', images) + # images = ['./data/2250_train/val/6920616313186/6920616313186_6920616313186_20240220-124502_53d2e103-ae3a-4689-b745-9d8723b770fe_front_returnGood_70f75407b7ae_31_01.jpg'] + + + # groups = group_image(conf.test_val, conf.test_batch_size) ##根据batch_size取图片 + groups = group_image('img_test', 1) ##根据batch_size取图片, 默认batch_size = 8 + + feature_dict = dict() + for group in groups: + s = time.time() + features = featurize(group, conf.test_transform, model, conf.device) + e = time.time() + print('time: {}'.format(e - s)) + # out = softmax(features, dim=1).argmax(dim=1) + # print('d >>> {}'. format(out)) + # feature_dict.update(d) diff --git a/contrast/feat_extract/resnet_vit/model/__init__.py b/contrast/feat_extract/resnet_vit/model/__init__.py new file mode 100644 index 0000000..9eebc77 --- /dev/null +++ b/contrast/feat_extract/resnet_vit/model/__init__.py @@ -0,0 +1 @@ +from .resnet_pre import resnet18, resnet34, resnet50, resnet14 diff --git a/contrast/feat_extract/resnet_vit/model/__pycache__/__init__.cpython-38.pyc b/contrast/feat_extract/resnet_vit/model/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000..160bc37 Binary files /dev/null and b/contrast/feat_extract/resnet_vit/model/__pycache__/__init__.cpython-38.pyc differ diff --git a/contrast/feat_extract/resnet_vit/model/__pycache__/__init__.cpython-39.pyc b/contrast/feat_extract/resnet_vit/model/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..539ec67 Binary files /dev/null and b/contrast/feat_extract/resnet_vit/model/__pycache__/__init__.cpython-39.pyc differ diff --git a/contrast/feat_extract/resnet_vit/model/__pycache__/resnet_pre.cpython-38.pyc b/contrast/feat_extract/resnet_vit/model/__pycache__/resnet_pre.cpython-38.pyc new file mode 100644 index 0000000..199085d Binary files /dev/null and b/contrast/feat_extract/resnet_vit/model/__pycache__/resnet_pre.cpython-38.pyc differ diff --git a/contrast/feat_extract/resnet_vit/model/__pycache__/resnet_pre.cpython-39.pyc b/contrast/feat_extract/resnet_vit/model/__pycache__/resnet_pre.cpython-39.pyc new file mode 100644 index 0000000..0301f55 Binary files /dev/null and b/contrast/feat_extract/resnet_vit/model/__pycache__/resnet_pre.cpython-39.pyc differ diff --git a/contrast/feat_extract/resnet_vit/model/resnet_pre.py b/contrast/feat_extract/resnet_vit/model/resnet_pre.py new file mode 100644 index 0000000..5e52ad9 --- /dev/null +++ b/contrast/feat_extract/resnet_vit/model/resnet_pre.py @@ -0,0 +1,462 @@ +import torch +import torch.nn as nn +from config import config as conf + +try: + from torch.hub import load_state_dict_from_url +except ImportError: + from torch.utils.model_zoo import load_url as load_state_dict_from_url +# from .utils import load_state_dict_from_url + +__all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', + 'resnet152', 'resnext50_32x4d', 'resnext101_32x8d', + 'wide_resnet50_2', 'wide_resnet101_2'] + +model_urls = { + 'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth', + 'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth', + 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth', + 'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth', + 'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth', + 'resnext50_32x4d': 'https://download.pytorch.org/models/resnext50_32x4d-7cdf4587.pth', + 'resnext101_32x8d': 'https://download.pytorch.org/models/resnext101_32x8d-8ba56ff5.pth', + 'wide_resnet50_2': 'https://download.pytorch.org/models/wide_resnet50_2-95faca4d.pth', + 'wide_resnet101_2': 'https://download.pytorch.org/models/wide_resnet101_2-32ee1156.pth', +} + + +def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1): + """3x3 convolution with padding""" + return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=dilation, groups=groups, bias=False, dilation=dilation) + +def conv1x1(in_planes, out_planes, stride=1): + """1x1 convolution""" + return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +class SpatialAttention(nn.Module): + def __init__(self, kernel_size=7): + super(SpatialAttention, self).__init__() + + assert kernel_size in (3, 7), 'kernel size must be 3 or 7' + padding = 3 if kernel_size == 7 else 1 + + self.conv1 = nn.Conv2d(2, 1, kernel_size, padding=padding, bias=False) + self.sigmoid = nn.Sigmoid() + + def forward(self, x): + avg_out = torch.mean(x, dim=1, keepdim=True) + max_out, _ = torch.max(x, dim=1, keepdim=True) + x = torch.cat([avg_out, max_out], dim=1) + x = self.conv1(x) + return self.sigmoid(x) + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1, + base_width=64, dilation=1, norm_layer=None, cam=False, bam=False): + super(BasicBlock, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + if groups != 1 or base_width != 64: + raise ValueError('BasicBlock only supports groups=1 and base_width=64') + if dilation > 1: + raise NotImplementedError("Dilation > 1 not supported in BasicBlock") + self.cam = cam + self.bam = bam + # Both self.conv1 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = norm_layer(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = norm_layer(planes) + self.downsample = downsample + self.stride = stride + if self.cam: + if planes == 64: + self.globalAvgPool = nn.AvgPool2d(56, stride=1) + elif planes == 128: + self.globalAvgPool = nn.AvgPool2d(28, stride=1) + elif planes == 256: + self.globalAvgPool = nn.AvgPool2d(14, stride=1) + elif planes == 512: + self.globalAvgPool = nn.AvgPool2d(7, stride=1) + + self.fc1 = nn.Linear(in_features=planes, out_features=round(planes / 16)) + self.fc2 = nn.Linear(in_features=round(planes / 16), out_features=planes) + self.sigmod = nn.Sigmoid() + if self.bam: + self.bam = SpatialAttention() + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + identity = self.downsample(x) + + if self.cam: + ori_out = self.globalAvgPool(out) + out = out.view(out.size(0), -1) + out = self.fc1(out) + out = self.relu(out) + out = self.fc2(out) + out = self.sigmod(out) + out = out.view(out.size(0), out.size(-1), 1, 1) + out = out * ori_out + + if self.bam: + out = out*self.bam(out) + + out += identity + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2) + # while original implementation places the stride at the first 1x1 convolution(self.conv1) + # according to "Deep residual learning for image recognition"https://arxiv.org/abs/1512.03385. + # This variant is also known as ResNet V1.5 and improves accuracy according to + # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch. + + expansion = 4 + + def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1, + base_width=64, dilation=1, norm_layer=None, cam=False, bam=False): + super(Bottleneck, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + width = int(planes * (base_width / 64.)) * groups + self.cam = cam + self.bam = bam + # Both self.conv2 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv1x1(inplanes, width) + self.bn1 = norm_layer(width) + self.conv2 = conv3x3(width, width, stride, groups, dilation) + self.bn2 = norm_layer(width) + self.conv3 = conv1x1(width, planes * self.expansion) + self.bn3 = norm_layer(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + if self.cam: + if planes == 64: + self.globalAvgPool = nn.AvgPool2d(56, stride=1) + elif planes == 128: + self.globalAvgPool = nn.AvgPool2d(28, stride=1) + elif planes == 256: + self.globalAvgPool = nn.AvgPool2d(14, stride=1) + elif planes == 512: + self.globalAvgPool = nn.AvgPool2d(7, stride=1) + + self.fc1 = nn.Linear(planes * self.expansion, round(planes / 4)) + self.fc2 = nn.Linear(round(planes / 4), planes * self.expansion) + self.sigmod = nn.Sigmoid() + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + if self.cam: + ori_out = self.globalAvgPool(out) + out = out.view(out.size(0), -1) + out = self.fc1(out) + out = self.relu(out) + out = self.fc2(out) + out = self.sigmod(out) + out = out.view(out.size(0), out.size(-1), 1, 1) + out = out * ori_out + out += identity + out = self.relu(out) + return out + + +class ResNet(nn.Module): + + def __init__(self, block, layers, num_classes=conf.embedding_size, zero_init_residual=False, + groups=1, width_per_group=64, replace_stride_with_dilation=None, + norm_layer=None, scale=0.75): + super(ResNet, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + self._norm_layer = norm_layer + + self.inplanes = 64 + self.dilation = 1 + if replace_stride_with_dilation is None: + # each element in the tuple indicates if we should replace + # the 2x2 stride with a dilated convolution instead + replace_stride_with_dilation = [False, False, False] + if len(replace_stride_with_dilation) != 3: + raise ValueError("replace_stride_with_dilation should be None " + "or a 3-element tuple, got {}".format(replace_stride_with_dilation)) + self.groups = groups + self.base_width = width_per_group + self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3, + bias=False) + self.bn1 = norm_layer(self.inplanes) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, int(64*scale), layers[0]) + self.layer2 = self._make_layer(block, int(128*scale), layers[1], stride=2, + dilate=replace_stride_with_dilation[0]) + self.layer3 = self._make_layer(block, int(256*scale), layers[2], stride=2, + dilate=replace_stride_with_dilation[1]) + self.layer4 = self._make_layer(block, int(512*scale), layers[3], stride=2, + dilate=replace_stride_with_dilation[2]) + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + self.fc = nn.Linear(int(512 * block.expansion*scale), num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + # Zero-initialize the last BN in each residual branch, + # so that the residual branch starts with zeros, and each residual block behaves like an identity. + # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 + if zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + nn.init.constant_(m.bn3.weight, 0) + elif isinstance(m, BasicBlock): + nn.init.constant_(m.bn2.weight, 0) + + def _make_layer(self, block, planes, blocks, stride=1, dilate=False): + norm_layer = self._norm_layer + downsample = None + previous_dilation = self.dilation + if dilate: + self.dilation *= stride + stride = 1 + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1(self.inplanes, planes * block.expansion, stride), + norm_layer(planes * block.expansion), + ) + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample, self.groups, + self.base_width, previous_dilation, norm_layer)) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append(block(self.inplanes, planes, groups=self.groups, + base_width=self.base_width, dilation=self.dilation, + norm_layer=norm_layer)) + return nn.Sequential(*layers) + + def _forward_impl(self, x): + # See note [TorchScript super()] + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + # print('poolBefore', x.shape) + x = self.avgpool(x) + # print('poolAfter', x.shape) + x = torch.flatten(x, 1) + # print('fcBefore',x.shape) + x = self.fc(x) + + # print('fcAfter',x.shape) + + return x + + def forward(self, x): + return self._forward_impl(x) + + +# def _resnet(arch, block, layers, pretrained, progress, **kwargs): +# model = ResNet(block, layers, **kwargs) +# if pretrained: +# state_dict = load_state_dict_from_url(model_urls[arch], +# progress=progress) +# model.load_state_dict(state_dict, strict=False) +# return model +def _resnet(arch, block, layers, pretrained, progress, **kwargs): + model = ResNet(block, layers, **kwargs) + if pretrained: + state_dict = load_state_dict_from_url(model_urls[arch], + progress=progress) + + src_state_dict = state_dict + target_state_dict = model.state_dict() + skip_keys = [] + # skip mismatch size tensors in case of pretraining + for k in src_state_dict.keys(): + if k not in target_state_dict: + continue + if src_state_dict[k].size() != target_state_dict[k].size(): + skip_keys.append(k) + for k in skip_keys: + del src_state_dict[k] + missing_keys, unexpected_keys = model.load_state_dict(src_state_dict, strict=False) + + return model + + +def resnet14(pretrained=True, progress=True, **kwargs): + r"""ResNet-14 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet18', BasicBlock, [2, 1, 1, 2], pretrained, progress, + **kwargs) + + +def resnet18(pretrained=True, progress=True, **kwargs): + r"""ResNet-18 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet18', BasicBlock, [2, 2, 2, 2], pretrained, progress, + **kwargs) + + +def resnet34(pretrained=False, progress=True, **kwargs): + r"""ResNet-34 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet34', BasicBlock, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet50(pretrained=False, progress=True, **kwargs): + r"""ResNet-50 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet50', Bottleneck, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet101(pretrained=False, progress=True, **kwargs): + r"""ResNet-101 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet101', Bottleneck, [3, 4, 23, 3], pretrained, progress, + **kwargs) + + +def resnet152(pretrained=False, progress=True, **kwargs): + r"""ResNet-152 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet152', Bottleneck, [3, 8, 36, 3], pretrained, progress, + **kwargs) + + +def resnext50_32x4d(pretrained=False, progress=True, **kwargs): + r"""ResNeXt-50 32x4d model from + `"Aggregated Residual Transformation for Deep Neural Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 4 + return _resnet('resnext50_32x4d', Bottleneck, [3, 4, 6, 3], + pretrained, progress, **kwargs) + + +def resnext101_32x8d(pretrained=False, progress=True, **kwargs): + r"""ResNeXt-101 32x8d model from + `"Aggregated Residual Transformation for Deep Neural Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 8 + return _resnet('resnext101_32x8d', Bottleneck, [3, 4, 23, 3], + pretrained, progress, **kwargs) + + +def wide_resnet50_2(pretrained=False, progress=True, **kwargs): + r"""Wide ResNet-50-2 model from + `"Wide Residual Networks" `_ + + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['width_per_group'] = 64 * 2 + return _resnet('wide_resnet50_2', Bottleneck, [3, 4, 6, 3], + pretrained, progress, **kwargs) + + +def wide_resnet101_2(pretrained=False, progress=True, **kwargs): + r"""Wide ResNet-101-2 model from + `"Wide Residual Networks" `_ + + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['width_per_group'] = 64 * 2 + return _resnet('wide_resnet101_2', Bottleneck, [3, 4, 23, 3], + pretrained, progress, **kwargs) diff --git a/contrast/feat_infer.py b/contrast/feat_infer.py new file mode 100644 index 0000000..988e34b --- /dev/null +++ b/contrast/feat_infer.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +""" +Created on Fri Feb 28 16:27:17 2025 + +@author: ym +""" + +import os +import time +import pickle +import numpy as np +from PIL import Image +from scipy.spatial.distance import cdist +from feat_extract.config import config as conf +from feat_extract.inference import FeatsInterface #, inference_image + +Encoder = FeatsInterface(conf) + +def main(): + imgpaths = r"D:\全实时\202502\result\Yolos_Tracking\20250228-160049-188_6921168558018_6921168558018\a" + featDict = {} + imgs, imgfiles = [], [] + for filename in os.listdir(imgpaths): + file, ext = os.path.splitext(filename) + + imgpath = os.path.join(imgpaths, filename) + img = Image.open(imgpath) + + imgs.append(img) + imgfiles.append(filename) + + feature = Encoder.inference([img]) + feature /= np.linalg.norm(feature, axis=1)[:, None] + feature_ft32 = feature.astype(np.float32) + + + + featDict[file] = feature_ft32 + + feature = Encoder.inference(imgs) + feature /= np.linalg.norm(feature, axis=1)[:, None] + feature_ft32 = feature.astype(np.float32) + + + matrix = 1 - cdist(feature, feature, 'cosine') + + print("do") + + + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/contrast/genfeats.py b/contrast/genfeats.py new file mode 100644 index 0000000..41d1254 --- /dev/null +++ b/contrast/genfeats.py @@ -0,0 +1,227 @@ +# -*- coding: utf-8 -*- +""" +Created on Sun Nov 3 12:05:19 2024 + +@author: ym +""" +import os +import time +# import torch +import pickle +# import json +import numpy as np +from PIL import Image +from feat_extract.config import config as conf +# from model import resnet18 as resnet18 +from feat_extract.inference import FeatsInterface #, inference_image + + +IMG_FORMAT = ['.bmp', '.jpg', '.jpeg', '.png'] + +# def model_init(conf, mpath=None): +# '''======= 0. 配置特征提取模型地址 =======''' +# if mpath is None: +# model_path = conf.test_model +# else: +# model_path = mpath + +# ##============ load resnet mdoel +# model = resnet18().to(conf.device) +# # model = nn.DataParallel(model).to(conf.device) +# model.load_state_dict(torch.load(model_path, map_location=conf.device)) +# model.eval() +# print('load model {} '.format(conf.testbackbone)) + +# return model + +def get_std_barcodeDict(bcdpath, savepath, bcdSet): + ''' + inputs: + bcdpath: 已清洗的barcode样本图像，如果barcode下有'base'文件夹，只选用该文件夹下图像 + (default = r'\\\\192.168.1.28\\share\\已标注数据备份\\对比数据\\barcode\\barcode_1771') + 功能： + 生成并保存只有一个key值的字典 {barcode: [imgpath1, imgpath1, ...]}， + savepath: 字典存储地址，文件名格式：barcode.pickle + ''' + + # savepath = r'\\\\192.168.1.28\\share\\测试_202406\\contrast\\std_barcodes' + + '''读取数据集中 barcode 列表''' + stdBarcodeList = [] + for filename in os.listdir(bcdpath): + filepath = os.path.join(bcdpath, filename) + if not os.path.isdir(filepath) or not filename.isdigit() or len(filename)<8: + continue + if bcdSet is None: + stdBarcodeList.append(filename) + elif filename in bcdSet: + stdBarcodeList.append(filename) + + + bcdPaths = [(barcode, os.path.join(bcdpath, barcode)) for barcode in stdBarcodeList] + + '''遍历数据集，针对每一个barcode，生成并保存字典{barcode: [imgpath1, imgpath1, ...]}''' + k = 0 + errbarcodes = [] + for barcode, bpath in bcdPaths: + pickpath = os.path.join(savepath, f"{barcode}.pickle") + if os.path.isfile(pickpath): + continue + + stdBarcodeDict = {} + stdBarcodeDict[barcode] = [] + for root, dirs, files in os.walk(bpath): + imgpaths = [] + if "base" in dirs: + broot = os.path.join(root, "base") + for imgname in os.listdir(broot): + imgpath = os.path.join(broot, imgname) + file, ext = os.path.splitext(imgpath) + + if ext not in IMG_FORMAT: + continue + imgpaths.append(imgpath) + + stdBarcodeDict[barcode].extend(imgpaths) + break + + else: + for imgname in files: + imgpath = os.path.join(root, imgname) + _, ext = os.path.splitext(imgpath) + if ext not in IMG_FORMAT: continue + imgpaths.append(imgpath) + stdBarcodeDict[barcode].extend(imgpaths) + + pickpath = os.path.join(savepath, f"{barcode}.pickle") + with open(pickpath, 'wb') as f: + pickle.dump(stdBarcodeDict, f) + + print(f"Barcode: {barcode}") + + # k += 1 + # if k == 10: + # break + print(f"Len of errbarcodes: {len(errbarcodes)}") + return + + + +def stdfeat_infer(imgPath, featPath, bcdSet=None): + ''' + inputs: + imgPath: 该文件夹下的 pickle 文件格式 {barcode: [imgpath1, imgpath1, ...]} + featPath: imgPath图像对应特征的存储地址 + 功能: + 对 imgPath中图像进行特征提取，生成只有一个key值的字典， + {barcode: features}，features.shape=(nsample, 256)，并保存至 featPath 中 + + ''' + + # imgPath = r"\\192.168.1.28\share\测试_202406\contrast\std_barcodes" + # featPath = r"\\192.168.1.28\share\测试_202406\contrast\std_features" + + + Encoder = FeatsInterface(conf) + + '''4处同名: (1)barcode原始图像文件夹; (2)imgPath中的 .pickle 文件名; + (3)该pickle文件中字典的key值; (4)特征向量字典中的一个key值''' + k = 0 + for filename in os.listdir(imgPath): + bcd, ext = os.path.splitext(filename) + filepath = os.path.join(imgPath, filename) + if ext != ".pickle": continue + if bcdSet is not None and bcd not in bcdSet: + continue + + featpath = os.path.join(featPath, f"{bcd}.pickle") + if os.path.isfile(featpath): + continue + + stdbDict = {} + t1 = time.time() + try: + with open(filepath, 'rb') as f: + bpDict = pickle.load(f) + + for barcode, imgpaths in bpDict.items(): + # feature = batch_inference(imgpaths, 8) #from vit distilled model of LiChen + # feature = inference_image(imgpaths, conf.test_transform, model, conf.device) + + imgs = [] + for d, imgpath in enumerate(imgpaths): + img = Image.open(imgpath) + imgs.append(img) + + feature = Encoder.inference(imgs) + + feature /= np.linalg.norm(feature, axis=1)[:, None] + + feature_ft32 = feature.astype(np.float32) + + # float16 + feature_ft16 = feature.astype(np.float16) + feature_ft16 /= np.linalg.norm(feature_ft16, axis=1)[:, None] + + # uint8, 两种策略，1) 精度损失小, 2) 计算复杂度小 + # feature_uint8, _ = ft16_to_uint8(feature_ft16) + feature_uint8 = (feature_ft16*128).astype(np.int8) + + '''================ 保存单个barcode特征 ================''' + ##================== float32 + stdbDict["barcode"] = barcode + stdbDict["imgpaths"] = imgpaths + stdbDict["feats_ft32"] = feature_ft32 + stdbDict["feats_ft16"] = feature_ft16 + stdbDict["feats_uint8"] = feature_uint8 + + with open(featpath, 'wb') as f: + pickle.dump(stdbDict, f) + + except Exception as e: + print(f"Error accured at: {filename}, with Exception is: {e}") + + + t2 = time.time() + print(f"Barcode: {barcode}, need time: {t2-t1:.1f} secs") + # k += 1 + # if k == 10: + # break + + return + + +def gen_bcd_features(imgpath, bcdpath, featpath, eventSourcePath): + ''' 生成标准特征集 ''' + '''1. 提取 imgpath 中样本地址，生成字典{barcode: [imgpath1, imgpath1, ...]} + 并存储于: bcdpath, 格式为 barcode.pickle''' + + bcdList = [] + for evtname in os.listdir(eventSourcePath): + bname, ext = os.path.splitext(evtname) + evt = bname.split('_') + if len(evt)>=2 and evt[-1].isdigit() and len(evt[-1])>=10: + bcdList.append(evt[-1]) + + bcdSet = set(bcdList) + get_std_barcodeDict(imgpath, bcdpath, bcdSet) + + '''2. 特征提取，并保存至文件夹 featpath 中，也根据 bcdSet 交集执行''' + stdfeat_infer(bcdpath, featpath, bcdSet) + +def main(): + imgpath = r"\\192.168.1.28\share\数据\已完成数据\展厅数据\v2.0_abroad\比对数据\all_base_二筛" + bcdpath = r"D:\exhibition\dataset\bcdpath_abroad" + featpath = r"D:\exhibition\dataset\feats_abroad" + if not os.path.exists(bcdpath): + os.makedirs(bcdpath) + if not os.path.exists(featpath): + os.makedirs(featpath) + + + gen_bcd_features(imgpath, bcdpath, featpath) + + + +if __name__ == '__main__': + main() diff --git a/contrast/input_getout_compare.py b/contrast/input_getout_compare.py new file mode 100644 index 0000000..5d64712 --- /dev/null +++ b/contrast/input_getout_compare.py @@ -0,0 +1,433 @@ +# -*- coding: utf-8 -*- +""" +Created on Sat Jul 27 14:07:25 2024 + +现场测试数据，在不同特征组合情况下的精度、召回率分析程序 + +@author: ym +""" +import os.path +import numpy as np +from scipy.spatial.distance import cdist +import matplotlib.pyplot as plt +import sys +sys.path.append(r"D:\DetectTracking") +from tracking.utils.read_data import extract_data, read_deletedBarcode_file, read_tracking_output +# from tracking.dotrack.dotracks import Track + +from one2n_contrast import compute_recall_precision, show_recall_prec +from one2n_contrast import performance_evaluate, one2n_return, one2n_deleted + +def compute_similar(feat1, feat2): + + if len(feat1)==0 or len(feat2)==0: + return 0 + + similar = 1 - np.maximum(0.0, cdist(feat1, feat2, metric = 'cosine')) + smean = np.mean(similar) + + return smean + + +def update_event(datapath): + '''一次购物事件，包含 8 个keys + back_sole_boxes：后摄boxes + front_sole_boxes：前摄boxes + back_sole_feats：后摄特征 + front_sole_feats：前摄特征 + feats_compose：将前后摄特征进行合并 + feats_select：特征选择，优先选择前摄特征 + ''' + event = {} + # event['front_tracking_boxes'] = [] + # event['front_tracking_feats'] = {} + # event['back_tracking_boxes'] = [] + # event['back_tracking_feats'] = {} + + event['back_sole_boxes'] = np.empty((0, 9), dtype=np.float64) + event['front_sole_boxes'] = np.empty((0, 9), dtype=np.float64) + + event['back_sole_feats'] = np.empty((0, 256), dtype=np.float64) + event['front_sole_feats'] = np.empty((0, 256), dtype=np.float64) + + event['feats_compose'] = np.empty((0, 256), dtype=np.float64) + event['feats_select'] = np.empty((0, 256), dtype=np.float64) + + + # '''读取 track.data 文件中的数据''' + # fpath_0_track = os.path.join(datapath, '0_track.data') + # fpath_1_track = os.path.join(datapath, '1_track.data') + # if os.path.exists(fpath_0_track) and os.path.isfile(fpath_0_track): + # _, _, _, _, tracking_boxes, tracking_feat_dict = extract_data(fpath_0_track) + # event['back_tracking_boxes'] = tracking_boxes + # event['back_tracking_feats'] = tracking_feat_dict + + # if os.path.exists(fpath_1_track) and os.path.isfile(fpath_1_track): + # _, _, _, _, tracking_boxes, tracking_feat_dict = extract_data(fpath_1_track) + # event['front_tracking_boxes'] = tracking_boxes + # event['front_tracking_feats'] = tracking_feat_dict + +# ============================================================================= +# '''================1. 直接指定读取，速度快======================================''' +# '''读取 tracking_output.data 文件中的数据''' +# fpath_0_tracking = os.path.join(datapath, '0_tracking_output.data') +# fpath_1_tracking = os.path.join(datapath, '1_tracking_output.data') +# if os.path.exists(fpath_0_tracking) and os.path.isfile(fpath_0_tracking): +# tracking_output_boxes, tracking_output_feats = read_tracking_output(fpath_0_tracking) +# event['back_sole_boxes'] = tracking_output_boxes +# event['back_sole_feats'] = tracking_output_feats +# +# if os.path.exists(fpath_1_tracking) and os.path.isfile(fpath_1_tracking): +# tracking_output_boxes, tracking_output_feats = read_tracking_output(fpath_1_tracking) +# event['front_sole_boxes'] = tracking_output_boxes +# event['front_sole_feats'] = tracking_output_feats +# ============================================================================= + + + '''================2. 遍历方式读取（与以上方式二选一）======================================''' + '''读取当前事件的 data 文件''' + for filename in os.listdir(datapath): + # filename = '1_track.data' + fpath = os.path.join(datapath, filename) + CamerType = filename.split('_')[0] + # if os.path.isfile(fpath) and filename.find("track.data")>0: + # bboxes, ffeats, trackerboxes, tracker_feat_dict, tracking_boxes, tracking_feat_dict = extract_data(fpath) + + # if CamerType == '0': + # event['back_tracking_boxes'] = tracking_boxes + # event['back_tracking_feats'] = tracking_feat_dict + # elif CamerType == '1': + # event['front_tracking_boxes'] = tracking_boxes + # event['front_tracking_feats'] = tracking_feat_dict + + if os.path.isfile(fpath) and filename.find("tracking_output.data")>0: + tracking_output_boxes, tracking_output_feats = read_tracking_output(fpath) + if CamerType == '0': + event['back_sole_boxes'] = tracking_output_boxes + event['back_sole_feats'] = tracking_output_feats + elif CamerType == '1': + event['front_sole_boxes'] = tracking_output_boxes + event['front_sole_feats'] = tracking_output_feats + + + '''事件的特征表征方式选择''' + fs_feats = event['front_sole_feats'] + bs_feats = event['back_sole_feats'] + + '''1. 如果前后摄均没有轨迹选择输出，返回''' + condt1 = len(fs_feats) + len(bs_feats) == 0 + if condt1: + return event + + + '''2. 构造综合特征''' + feats_compose = np.empty((0, 256), dtype=np.float64) + if len(fs_feats): + feats_compose = np.concatenate((feats_compose, fs_feats), axis=0) + if len(bs_feats): + feats_compose = np.concatenate((feats_compose, bs_feats), axis=0) + event['feats_compose'] = feats_compose + + '''3. 构造前摄特征''' + if len(fs_feats): + event['feats_select'] = fs_feats + return event + + + '''4. 从前摄输出轨迹中选取特定轨迹对应的特征''' +# ============================================================================= +# ftrboxes = event['front_tracking_boxes'] +# ftrfeats = event['front_tracking_feats'] +# +# condt2 = len(ftrboxes) + len(ftrfeats) == 0 +# condt3 = len(ftrfeats) != len(ftrboxes) +# if condt2 or condt3: +# return event +# +# bprops = [] +# for boxes in ftrboxes: +# track = Track(boxes) +# bprops.append(max(track.trajdist)) +# +# index = bprops.index(max(bprops)) +# box_select = ftrboxes[index] +# tid = int(box_select[0, 4]) +# +# feat_select = ftrfeats[f"track_{tid}"] +# feats_select = np.empty((0, 256), dtype=np.float64) +# for fid_bid, feat in feat_select['feats'].items(): +# feats_select = np.concatenate((feats_select, feat[None, :]), axis=0) +# event['feats_select'] = feats_select +# ============================================================================= + + return event + + +def creatd_deletedBarcode_front(filepath): + ''' + 生成deletedBarcodeTest.txt + ''' + + # filepath = r'\\192.168.1.28\share\测试_202406\0723\0723_1\deletedBarcode.txt' + basepath, _ = os.path.split(filepath) + + bcdlist = read_deletedBarcode_file(filepath) + + MatchList = [] + k = 0 + for s_list in bcdlist: + getout_fold = s_list['SeqDir'].strip() + getout_path = os.path.join(basepath, getout_fold) + + '''取出事件文件夹不存在，跳出循环''' + if not os.path.exists(getout_path) and not os.path.isdir(getout_path): + continue + + day, hms = getout_fold.strip('_').split('-') + + ''' 生成取出事件字典 ''' + getout_event = {} + getout_event['barcode'] = s_list['Deleted'].strip() + getout_event['path'] = getout_path + + getout_event['feats_compose'] = np.empty((0, 256), dtype=np.float64) + getout_event['feats_select'] = np.empty((0, 256), dtype=np.float64) + + + InputList = [] + barcodes = [s.strip() for s in s_list['barcode']] + similarity = [float(s.strip()) for s in s_list['similarity']] + for i, barcode in enumerate(barcodes): + + ''' 生成放入事件字典 ''' + input_event = {} + + input_folds, times = [], [] + for pathname in os.listdir(basepath): + if pathname.endswith('_'): continue + if os.path.isfile(os.path.join(basepath, pathname)):continue + + infold = pathname.split('_') + if len(infold)!=2: continue + if len(infold[0])<=14 or len(infold[1])<=10: continue + + day1, hms1 = infold[0].split('-') + + if day1==day and infold[1]==barcode and int(hms1)=0: + typee = typee.split(",")[-1] + + if pattern==1: + evt_similars.append(similar) + if pattern==2 or pattern==3: + o2n_evt = [evt for name, evt in evtDicts.items() if name.find(nname[:15])==0] + if len(o2n_evt)!=1: + continue + + simival = simi_calc(event, o2n_evt[0], pattern, typee) + if simival==None: + continue + evt_similars.append(simival) + + evt_names.append(nname) + evt_barcodes.append(barcode) + evt_types.append(typee) + + # if evtname == "20250226-170321-327_6903244678377": + # print("evtname") + + ## process.data的oneTon的各项中，均不包括当前事件的barcode + if event.barcode not in evt_barcodes: + errorFile_one2n.append(evtname) + continue + else: + one2nFile.append(evtname) + + if len(evt_names)==len(evt_barcodes)==len(evt_similars)==len(evt_types) and len(evt_names)>0: + # maxsim = evt_similars[evt_similars.index(max(evt_similars))] + maxsim = max(evt_similars) + for i in range(len(evt_names)): + bcd, simi = evt_barcodes[i], evt_similars[i] + if bcd==event.barcode and simi==maxsim: + tpsimi.append(simi) + tpevents.append(evtname) + elif bcd==event.barcode and simi!=maxsim: + fnsimi.append(simi) + fnevents.append(evtname) + elif bcd!=event.barcode and simi!=maxsim: + tnsimi.append(simi) + tnevents.append(evtname) + elif bcd!=event.barcode and simi==maxsim and event.barcode in evt_barcodes: + fpsimi.append(simi) + fpevents.append(evtname) + else: + errorFile_one2n_.append(evtname) + + ''' 1:n 数据存储，需根据相似度排序''' + PPrecise, PRecall = [], [] + NPrecise, NRecall = [], [] + Thresh = np.linspace(-0.2, 1, 100) + for th in Thresh: + '''============================= 1:n 计算''' + TP = sum(np.array(tpsimi) >= th) + FP = sum(np.array(fpsimi) >= th) + FN = sum(np.array(fnsimi) < th) + TN = sum(np.array(tnsimi) < th) + + PPrecise.append(TP/(TP+FP+1e-6)) + PRecall.append(TP/(TP+FN+1e-6)) + NPrecise.append(TN/(TN+FN+1e-6)) + NRecall.append(TN/(TN+FP+1e-6)) + + + '''4. ============================= 1:n 曲线,''' + fig, ax = plt.subplots() + ax.plot(Thresh, PPrecise, 'r', label='Precise_Pos: TP/TPFP') + ax.plot(Thresh, PRecall, 'b', label='Recall_Pos: TP/TPFN') + ax.plot(Thresh, NPrecise, 'g', label='Precise_Neg: TN/TNFP') + ax.plot(Thresh, NRecall, 'c', label='Recall_Neg: TN/TNFN') + ax.set_xlim([0, 1]) + ax.set_ylim([0, 1]) + + ax.set_xticks(np.arange(0, 1, 0.1)) + ax.set_yticks(np.arange(0, 1, 0.1)) + ax.grid(True, linestyle='--') + + ax.set_title('1:n Precise & Recall') + ax.set_xlabel(f"Event Num: {len(one2nFile)}") + ax.legend() + plt.show() + ## ============================= 1:n 直方图''' + fig, axes = plt.subplots(2, 2) + axes[0, 0].hist(tpsimi, bins=60, range=(-0.2, 1), edgecolor='black') + axes[0, 0].set_xlim([-0.2, 1]) + axes[0, 0].set_title(f'TP: {len(tpsimi)}') + axes[0, 1].hist(fpsimi, bins=60, range=(-0.2, 1), edgecolor='black') + axes[0, 1].set_xlim([-0.2, 1]) + axes[0, 1].set_title(f'FP: {len(fpsimi)}') + axes[1, 0].hist(tnsimi, bins=60, range=(-0.2, 1), edgecolor='black') + axes[1, 0].set_xlim([-0.2, 1]) + axes[1, 0].set_title(f'TN: {len(tnsimi)}') + axes[1, 1].hist(fnsimi, bins=60, range=(-0.2, 1), edgecolor='black') + axes[1, 1].set_xlim([-0.2, 1]) + axes[1, 1].set_title(f'FN: {len(fnsimi)}') + plt.show() + + return fpevents + +def main(): + '''1. 生成事件字典并保存至 eventDataPath, 只需运行一次 ''' + init_eventDict(eventSourcePath, eventDataPath, stype="realtime") # 'source', 'data', 'realtime' + + # for pfile in os.listdir(eventDataPath): + # evt = os.path.splitext(pfile)[0].split('_') + # cont = len(evt)>=2 and evt[-1].isdigit() and len(evt[-1])>=10 + # if not cont: + # continue + + '''2. 读取事件字典 ''' + evtDicts = read_eventdict(eventDataPath) + + + '''3. 1:n 比对事件评估 ''' + fpevents = one2n_pr(evtDicts, pattern=1) + + fpErrFile = str(Path(resultPath).joinpath("one2n_fp_Error.txt")) + with open(fpErrFile, "w") as file: + for item in fpevents: + file.write(item + "\n") + + + +if __name__ == '__main__': + eventSourcePath = r"\\192.168.1.28\share\测试视频数据以及日志\全实时测试\V12\2025-3-4_2" + resultPath = r"\\192.168.1.28\share\测试视频数据以及日志\全实时测试\testing" + + eventDataPath = os.path.join(resultPath, "evtobjs_wang") + if not os.path.exists(eventDataPath): + os.makedirs(eventDataPath) + + # similPath = os.path.join(resultPath, "simidata") + # if not os.path.exists(similPath): + # os.makedirs(similPath) + + main() + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/contrast/one2n_contrast_old.py b/contrast/one2n_contrast_old.py new file mode 100644 index 0000000..7ad3cb3 --- /dev/null +++ b/contrast/one2n_contrast_old.py @@ -0,0 +1,498 @@ +# -*- coding: utf-8 -*- +""" +Created on Sat Jul ** 14:07:25 2024 + + 现场测试精度、召回率分析程序，是 feat_select.py 的简化版， + 但支持循环计算，并输出总的pr曲线 + +@author: ym +""" + + + +import os.path +import shutil + +import numpy as np +import matplotlib.pyplot as plt +import cv2 +from pathlib import Path +import sys +sys.path.append(r"D:\DetectTracking") +from tracking.utils.plotting import Annotator, colors +from tracking.utils.read_data import extract_data, read_deletedBarcode_file, read_tracking_output, read_returnGoods_file +from tracking.utils.plotting import draw_tracking_boxes, get_subimgs +from contrast.utils.tools import showHist, show_recall_prec, compute_recall_precision + + +# ============================================================================= +# def read_tracking_output(filepath): +# boxes = [] +# feats = [] +# with open(filepath, 'r', encoding='utf-8') as file: +# for line in file: +# line = line.strip() # 去除行尾的换行符和可能的空白字符 +# +# if not line: +# continue +# +# if line.endswith(','): +# line = line[:-1] +# +# data = np.array([float(x) for x in line.split(",")]) +# if data.size == 9: +# boxes.append(data) +# if data.size == 256: +# feats.append(data) +# +# return np.array(boxes), np.array(feats) +# ============================================================================= + +def read_tracking_imgs(imgspath): + ''' + input: + imgspath：该路径中的图像为Yolo算法的输入图像，640x512 + output： + imgs_0：后摄图像，根据 frameId 进行了排序 + imgs_1：前摄图像，根据 frameId 进行了排序 + ''' + imgs_0, frmIDs_0, imgs_1, frmIDs_1 = [], [], [], [] + + for filename in os.listdir(imgspath): + file, ext = os.path.splitext(filename) + flist = file.split('_') + if len(flist)==4 and ext==".jpg": + camID, frmID = flist[0], int(flist[-1]) + imgpath = os.path.join(imgspath, filename) + img = cv2.imread(imgpath) + + if camID=='0': + imgs_0.append(img) + frmIDs_0.append(frmID) + if camID=='1': + imgs_1.append(img) + frmIDs_1.append(frmID) + + if len(frmIDs_0): + indice = np.argsort(np.array(frmIDs_0)) + imgs_0 = [imgs_0[i] for i in indice ] + if len(frmIDs_1): + indice = np.argsort(np.array(frmIDs_1)) + imgs_1 = [imgs_1[i] for i in indice ] + + return imgs_0, imgs_1 + + +# ============================================================================= +# def draw_tracking_boxes(imgs, tracks): +# '''tracks: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] +# 0 1 2 3 4 5 6 7 8 +# 关键：imgs中的次序和 track 中的 fid 对应 +# ''' +# subimgs = [] +# for *xyxy, tid, conf, cls, fid, bid in tracks: +# label = f'id:{int(tid)}_{int(cls)}_{conf:.2f}' +# +# annotator = Annotator(imgs[int(fid-1)].copy()) +# if cls==0: +# color = colors(int(cls), True) +# elif tid>0 and cls!=0: +# color = colors(int(tid), True) +# else: +# color = colors(19, True) # 19为调色板的最后一个元素 +# +# pt2 = [p/2 for p in xyxy] +# annotator.box_label(pt2, label, color=color) +# img0 = annotator.result() +# +# subimgs.append(img0) +# +# return subimgs +# ============================================================================= + +def get_contrast_paths(pair, basepath): + assert(len(pair)==2 or len(pair)==3), "pair: seqdir, delete, barcodes" + + getout_fold = pair[0] # 取出操作对应的文件夹 + relvt_barcode = pair[1] # 取出操作对应放入操作的 Barcode + if len(pair)==3: + error_match = pair[2] # 取出操作错误匹配的 Barcode + else: + error_match = '' + + + getoutpath, inputpath, errorpath = '', '', '' + + day, hms = getout_fold.strip('_').split('-') + + input_folds, times = [], [] + errmatch_folds, errmatch_times = [], [] + for pathname in os.listdir(basepath): + if pathname.endswith('_'): continue + if os.path.isfile(os.path.join(basepath, pathname)):continue + infold = pathname.split('_') + if len(infold)!=2: continue + + day1, hms1 = infold[0].split('-') + + if day1==day and infold[1]==relvt_barcode and int(hms1) simimax + for k in idx: + if similarity[k] > simimax: + idxmax = k + simimax = similarity[k] + if idxmax>-1: + input_event = events[idxmax] + else: + input_event = '' + + errpairs.append((seqdir, input_event, events[index])) + err_similarity.append(max(similarity)) + + return corrpairs, errpairs, corr_similarity, err_similarity + + +def test_rpath_deleted(): + '''deletedBarcode.txt 格式的 1:n 数据结果文件, returnGoods.txt格式数据文件不需要调用该函数''' + + del_bfile = r'\\192.168.1.28\share\测试_202406\709\deletedBarcode.txt' + basepath = r'\\192.168.1.28\share\测试_202406\709' + savepath = r'D:\DetectTracking\contrast\result' + saveimgs = True + + + + relative_paths = [] + + '''1. 读取 deletedBarcode 文件 ''' + all_list = read_deletedBarcode_file(del_bfile) + + '''2. 算法性能评估，并输出 (取出，删除, 错误匹配) 对 ''' + corrpairs, errpairs, _, _ = one2n_deleted(all_list) + + '''3. 构造事件组合(取出，放入并删除, 错误匹配) 对应路径 ''' + for errpair in errpairs: + GetoutPath, InputPath, ErrorPath = get_contrast_paths(errpair, basepath) + + pairs = (GetoutPath, InputPath, ErrorPath) + relative_paths.append(pairs) + + print(InputPath) + '''3. 获取 (取出，放入并删除, 错误匹配) 对应路径，保存相应轨迹图像''' + if saveimgs: + save_tracking_imgpairs(pairs, savepath) + +def test_rpath_return(): + return_bfile = r'\\192.168.1.28\share\测试_202406\1101\images\returnGoods.txt' + basepath = r'\\192.168.1.28\share\测试_202406\1101\images' + savepath = r'D:\DetectTracking\contrast\result' + + all_list = read_returnGoods_file(return_bfile) + corrpairs, errpairs, _, _ = one2n_return(all_list) + for corrpair in corrpairs: + GetoutPath = os.path.join(basepath, corrpair[0]) + InputPath = os.path.join(basepath, corrpair[1]) + + pairs = (GetoutPath, InputPath) + save_tracking_imgpairs(pairs, savepath) + + for errpair in errpairs: + GetoutPath = os.path.join(basepath, errpair[0]) + InputPath = os.path.join(basepath, errpair[1]) + ErrorPath = os.path.join(basepath, errpair[2]) + + pairs = (GetoutPath, InputPath, ErrorPath) + save_tracking_imgpairs(pairs, savepath) + + +def test_one2n(): + ''' + 1:n 性能测试 + 兼容 2 种 txt 文件格式：returnGoods.txt, deletedBarcode.txt + fpath: 文件路径、或文件夹，其中包含多个 txt 文件 + savepath: pr曲线保存路径 + ''' + # fpath = r'\\192.168.1.28\share\测试_202406\deletedBarcode\other' # deletedBarcode.txt + fpath = r'\\192.168.1.28\share\测试_202406\1108_展厅模型v800测试' # returnGoods.txt + savepath = r'\\192.168.1.28\share\测试_202406\deletedBarcode\illustration' + + if os.path.isdir(fpath): + filepaths = [os.path.join(fpath, f) for f in os.listdir(fpath) + if f.find('.txt')>0 + and (f.find('deletedBarcode')>=0 or f.find('returnGoods')>=0)] + elif os.path.isfile(fpath): + filepaths = [fpath] + else: + return + + if not os.path.exists(savepath): + os.mkdir(savepath) + + BarLists, blists = {}, [] + for pth in filepaths: + file = str(Path(pth).stem) + if file.find('deletedBarcode')>=0: + blist = read_deletedBarcode_file(pth) + if file.find('returnGoods')>=0: + blist = read_returnGoods_file(pth) + + BarLists.update({file: blist}) + blists.extend(blist) + + if len(blists): BarLists.update({"Total": blists}) + for file, blist in BarLists.items(): + if all(b['filetype']=="deletedBarcode" for b in blist): + _, _, correct_similarity, err_similarity = one2n_deleted(blist) + if all(b['filetype']=="returnGoods" for b in blists): + _, _, correct_similarity, err_similarity = one2n_return(blist) + + recall, prec, ths = compute_recall_precision(err_similarity, correct_similarity) + + plt1 = show_recall_prec(recall, prec, ths) + # plt1.show() + plt1.xlabel(f'threshold, Num: {len(blist)}') + plt1.savefig(os.path.join(savepath, file+'_pr.png')) + # plt1.close() + + plt2 = showHist(err_similarity, correct_similarity) + plt2.show() + plt2.savefig(os.path.join(savepath, file+'_hist.png')) + # plt.close() + + + +if __name__ == '__main__': + # test_one2n() + test_rpath_return() # returnGoods.txt + # test_rpath_deleted() # deleteBarcode.txt + + + # try: + # test_rpath_return() + # test_rpath_deleted() + # except Exception as e: + # print(e) + + diff --git a/contrast/one2one_contrast.py b/contrast/one2one_contrast.py new file mode 100644 index 0000000..2e79b27 --- /dev/null +++ b/contrast/one2one_contrast.py @@ -0,0 +1,452 @@ +# -*- coding: utf-8 -*- +""" +Created on Fri Aug 30 17:53:03 2024 +功能：1:1比对性能测试程序 + 1. 基于标准特征集所对应的原始图像样本，生成标准特征集并保存。 + func: generate_event_and_stdfeatures(): + (1) get_std_barcodeDict(stdSamplePath, stdBarcodePath) + 提取 stdSamplePath 中样本地址，生成字典{barcode: [imgpath1, imgpath1, ...]} + 并存储为 pickle 文件，barcode.pickle''' + (2) stdfeat_infer(stdBarcodePath, stdFeaturePath, bcdSet=None) + 标准特征提取，并保存至文件夹 stdFeaturePath 中， + 也可在运行过程中根据与购物事件集合 barcodes 交集执行 + 2. 1:1 比对性能测试， + func: one2one_simi() + (1) 求购物事件和标准特征级 Barcode 交集，构造 evtDict、stdDict + (2) 构造扫 A 放 A、扫 A 放 B 组合，mergePairs = AA_list + AB_list + (3) 循环计算 mergePairs 中元素 "(A, A) 或 (A, B)" 相似度; + 对于未保存的轨迹图像或标准 barcode 图像，保存图像 + (4) 保存计算结果 + + + 3. precise、recall等指标计算 + func: compute_one2one_pr(pickpath) + + +@author: ym + +""" +import numpy as np +import os +import sys +import random +import pickle +import json + +from pathlib import Path +import matplotlib.pyplot as plt + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[1] # YOLOv5 root directory +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) + +from utils.calsimi import calsimi_vs_stdfeat, calsimi_vs_stdfeat_new +from utils.tools import get_evtList, init_eventDict +from utils.databits import data_precision_compare +from genfeats import gen_bcd_features + + +def build_std_evt_dict(): + ''' + stdFeaturePath: 标准特征集地址 + eventDataPath: Event对象地址 + ''' + + stdBarcode = [p.stem for p in Path(stdFeaturePath).iterdir() if p.is_file() and (p.suffix=='.json' or p.suffix=='.pickle')] + + '''======1. 购物事件列表，该列表中的 Barcode 存在于标准的 stdBarcode 内 ===''' + evtList = [(p.stem, p.stem.split('_')[-1]) for p in Path(eventDataPath).iterdir() + if p.is_file() + and p.suffix=='.pickle' + and (len(p.stem.split('_'))==2 or len(p.stem.split('_'))==3) + and p.stem.split('_')[-1].isdigit() + and p.stem.split('_')[-1] in stdBarcode + ] + barcodes = set([bcd for _, bcd in evtList]) + + '''======2. 构建用于比对的标准特征字典 =============''' + stdDict = {} + for stdfile in os.listdir(stdFeaturePath): + barcode, ext = os.path.splitext(stdfile) + if barcode not in barcodes: + continue + stdpath = os.path.join(stdFeaturePath, stdfile) + + if ext == ".json": + with open(stdpath, 'r', encoding='utf-8') as f: + stddata = json.load(f) + feat = np.array(stddata["value"]) + stdDict[barcode] = feat + if ext == ".pickle": + with open(stdpath, 'rb') as f: + stddata = pickle.load(f) + feat = stddata["feats_ft32"] + stdDict[barcode] = feat + + '''*********** USearch ***********''' + # stdDict = {} + # for barcode in barcodes: + # stdDict[barcode] = stdlib[barcode] + + '''======3. 构建用于比对的操作事件字典 =============''' + evtDict = {} + for evtname, barcode in evtList: + evtpath = os.path.join(eventDataPath, evtname+'.pickle') + try: + with open(evtpath, 'rb') as f: + evtdata = pickle.load(f) + except Exception as e: + print(evtname) + + evtDict[evtname] = evtdata + + return evtList, evtDict, stdDict + +def one2SN_pr(evtList, evtDict, stdDict, simType="simple"): + + std_barcodes = set([bcd for _, bcd in evtList]) + + + tp_events, fn_events, fp_events, tn_events = [], [], [], [] + tp_simi, fn_simi, tn_simi, fp_simi = [], [], [], [] + errorFile_one2SN = [] + + SN = 9 + for evtname, barcode in evtList: + bcd_selected = [barcode] + + dset = list(std_barcodes - set([barcode])) + if len(dset) > SN: + random.shuffle(dset) + bcd_selected.extend(dset[:SN]) + else: + bcd_selected.extend(dset) + + event = evtDict[evtname] + ## 无轨迹判断 + if len(event.front_feats)+len(event.back_feats)==0: + errorFile_one2SN.append(evtname) + print(f"No trajectory: {evtname}") + continue + + barcodes, similars = [], [] + for stdbcd in bcd_selected: + stdfeat = stdDict[stdbcd] + + if simType=="typea": + simi_mean, simi_max, simi_mfeat = calsimi_vs_stdfeat(event, stdfeat) + elif simType=="typeb": + pass + else: + simi_mean, simi_1, simi_2 = calsimi_vs_stdfeat_new(event, stdfeat) + + + ## 在event.front_feats和event.back_feats同时为空时，此处不需要保护 + # if simi_mean==None: + # continue + + barcodes.append(stdbcd) + similars.append(simi_mean) + + ## 此处不需要保护 + # if len(similars)==0: + # print(evtname) + # continue + + max_idx = similars.index(max(similars)) + max_sim = similars[max_idx] + for i in range(len(barcodes)): + bcd, simi = barcodes[i], similars[i] + if bcd==barcode and simi==max_sim: + tp_simi.append(simi) + tp_events.append(evtname) + elif bcd==barcode and simi!=max_sim: + fn_simi.append(simi) + fn_events.append(evtname) + elif bcd!=barcode and simi!=max_sim: + tn_simi.append(simi) + tn_events.append(evtname) + elif bcd!=barcode and simi==max_sim and barcode in barcodes: + fp_simi.append(simi) + fp_events.append(evtname) + else: + errorFile_one2SN.append(evtname) + + PPreciseX, PRecallX = [], [] + NPreciseX, NRecallX = [], [] + Thresh = np.linspace(-0.2, 1, 100) + for th in Thresh: + '''适用于 (Precise, Recall) 计算方式：多个相似度计算并排序，barcode相等且排名第一为 TP ''' + '''===================================== 1:SN ''' + TPX = sum(np.array(tp_simi) >= th) + FPX = sum(np.array(fp_simi) >= th) + FNX = sum(np.array(fn_simi) < th) + TNX = sum(np.array(tn_simi) < th) + PPreciseX.append(TPX/(TPX+FPX+1e-6)) + PRecallX.append(TPX/(TPX+FNX+1e-6)) + + NPreciseX.append(TNX/(TNX+FNX+1e-6)) + NRecallX.append(TNX/(TNX+FPX+1e-6)) + + fig, ax = plt.subplots() + ax.plot(Thresh, PPreciseX, 'r', label='Precise_Pos: TP/TPFP') + ax.plot(Thresh, PRecallX, 'b', label='Recall_Pos: TP/TPFN') + ax.plot(Thresh, NPreciseX, 'g', label='Precise_Neg: TN/TNFP') + ax.plot(Thresh, NRecallX, 'c', label='Recall_Neg: TN/TNFN') + ax.set_xlim([0, 1]) + ax.set_ylim([0, 1]) + ax.set_xticks(np.arange(0, 1, 0.1)) + ax.set_yticks(np.arange(0, 1, 0.1)) + ax.grid(True, linestyle='--') + ax.set_title('1:SN Precise & Recall') + ax.set_xlabel(f"Event Num: {len(tp_events) + len(fn_events)}") + ax.legend() + plt.show() + + rltpath = os.path.join(similPath, f'pr_1toSN_{simType}.png') + plt.savefig(rltpath) + + ## ============================= 1:N 展厅 直方图''' + fig, axes = plt.subplots(2, 2) + axes[0, 0].hist(tp_simi, bins=60, range=(-0.2, 1), edgecolor='black') + axes[0, 0].set_xlim([-0.2, 1]) + axes[0, 0].set_title(f'TP({len(tp_simi)})') + axes[0, 1].hist(fp_simi, bins=60, range=(-0.2, 1), edgecolor='black') + axes[0, 1].set_xlim([-0.2, 1]) + axes[0, 1].set_title(f'FP({len(fp_simi)})') + axes[1, 0].hist(tn_simi, bins=60, range=(-0.2, 1), edgecolor='black') + axes[1, 0].set_xlim([-0.2, 1]) + axes[1, 0].set_title(f'TN({len(tn_simi)})') + axes[1, 1].hist(fn_simi, bins=60, range=(-0.2, 1), edgecolor='black') + axes[1, 1].set_xlim([-0.2, 1]) + axes[1, 1].set_title(f'FN({len(fn_simi)})') + plt.show() + + rltpath = os.path.join(similPath, f'hist_1toSN_{simType}.png') + plt.savefig(rltpath) + + + + +def one2one_simi(evtList, evtDict, stdDict, simType): + + barcodes = set([bcd for _, bcd in evtList]) + '''======1 构造 3 个事件对: 扫 A 放 A, 扫 A 放 B, 合并 ====================''' + AA_list = [(evtname, barcode, "same") for evtname, barcode in evtList] + AB_list = [] + for evtname, barcode in evtList: + dset = list(barcodes.symmetric_difference(set([barcode]))) + if len(dset): + idx = random.randint(0, len(dset)-1) + AB_list.append((evtname, dset[idx], "diff")) + + mergePairs = AA_list + AB_list + + '''======2 计算事件、标准特征集相似度 ==================''' + rltdata = [] + errorFile_one2one = [] + for i in range(len(mergePairs)): + evtname, stdbcd, label = mergePairs[i] + event = evtDict[evtname] + if len(event.feats_compose)==0: + errorFile_one2one.append(evtname) + + continue + + stdfeat = stdDict[stdbcd] # float32 + + if simType=="typea": + simi_mean, simi_1, simi_2 = calsimi_vs_stdfeat_new(event, stdfeat) + elif simType=="typeb": + pass + else: + simi_mean, simi_1, simi_2 = calsimi_vs_stdfeat(event, stdfeat) + + if simi_mean is None: + continue + + rltdata.append((label, stdbcd, evtname, simi_mean, simi_1, simi_2)) + + '''================ float32、16、int8 精度比较与存储 =============''' + # data_precision_compare(stdfeat, evtfeat, mergePairs[i], similPath, save=True) + + errorFile_one2one = list(set(errorFile_one2one)) + + return rltdata, errorFile_one2one + + +def one2one_pr(evtList, evtDict, stdDict, simType="simple"): + + rltdata, errorFile_one2one = one2one_simi(evtList, evtDict, stdDict, simType) + + Same, Cross = [], [] + + for label, stdbcd, evtname, simi_mean, simi_max, simi_mft in rltdata: + if simType=="simple" and label == "same": + Same.append(simi_max) + if simType=="simple" and label == "diff": + Cross.append(simi_max) + + if simType=="typea" and label == "same": + Same.append(simi_mean) + if simType=="typea" and label == "diff": + Cross.append(simi_mean) + + + # for label, stdbcd, evtname, simi_mean, simi_max, simi_mft in rltdata: + # if label == "same": + # Same.append(simi_mean) + # if label == "diff": + # Cross.append(simi_mean) + + Same = np.array(Same) + Cross = np.array(Cross) + TPFN = len(Same) + TNFP = len(Cross) + + # fig, axs = plt.subplots(2, 1) + # axs[0].hist(Same, bins=60, range=(-0.2, 1), edgecolor='black') + # axs[0].set_xlim([-0.2, 1]) + # axs[0].set_title(f'Same Barcode, Num: {TPFN}') + + # axs[1].hist(Cross, bins=60, range=(-0.2, 1), edgecolor='black') + # axs[1].set_xlim([-0.2, 1]) + # axs[1].set_title(f'Cross Barcode, Num: {TNFP}') + # plt.savefig(f'./result/{file}_hist.png') # svg, png, pdf + + + Recall_Pos, Recall_Neg = [], [] + Precision_Pos, Precision_Neg = [], [] + Correct = [] + Thresh = np.linspace(-0.2, 1, 100) + for th in Thresh: + TP = np.sum(Same >= th) + FN = np.sum(Same < th) + # FN = TPFN - TP + + TN = np.sum(Cross < th) + FP = np.sum(Cross >= th) + # FP = TNFP - TN + + + Precision_Pos.append(TP/(TP+FP+1e-6)) + Precision_Neg.append(TN/(TN+FN+1e-6)) + Recall_Pos.append(TP/(TP+FN+1e-6)) + Recall_Neg.append(TN/(TN+FP+1e-6)) + + # Recall_Pos.append(TP/TPFN) + # Recall_Neg.append(TN/TNFP) + + + Correct.append((TN+TP)/(TPFN+TNFP)) + + fig, ax = plt.subplots() + + ax.plot(Thresh, Precision_Pos, 'r', label='Precision_Pos: TP/(TP+FP)') + ax.plot(Thresh, Recall_Pos, 'b', label='Recall_Pos: TP/TPFN') + ax.plot(Thresh, Recall_Neg, 'g', label='Recall_Neg: TN/TNFP') + ax.plot(Thresh, Correct, 'c', label='Correct: (TN+TP)/(TPFN+TNFP)') + ax.plot(Thresh, Precision_Neg, 'm', label='Precision_Neg: TN/(TN+FN)') + + ax.set_xlim([0, 1]) + ax.set_ylim([0, 1]) + + ax.set_xticks(np.arange(0, 1, 0.1)) + ax.set_yticks(np.arange(0, 1, 0.1)) + ax.grid(True, linestyle='--') + + ax.set_title('PrecisePos & PreciseNeg') + ax.set_xlabel(f"Same Num: {TPFN}, Cross Num: {TNFP}") + ax.legend() + plt.show() + + rltpath = os.path.join(similPath, f'pr_1to1_{simType}.png') + plt.savefig(rltpath) # svg, png, pdf + + + fig, axes = plt.subplots(2,1) + axes[0].hist(Same, bins=60, range=(-0.2, 1), edgecolor='black') + axes[0].set_xlim([-0.2, 1]) + axes[0].set_title(f'TP({len(Same)})') + + axes[1].hist(Cross, bins=60, range=(-0.2, 1), edgecolor='black') + axes[1].set_xlim([-0.2, 1]) + axes[1].set_title(f'TN({len(Cross)})') + + rltpath = os.path.join(similPath, f'hist_1to1_{simType}.png') + plt.savefig(rltpath) + + + plt.show() + + + + + +def test_one2one_one2SN(simType): + '''1:1性能评估''' + + # evtpaths, bcdSet = get_evtList(eventSourcePath) + + '''=== 1. 只需运行一次，生成事件对应的标准特征库字典,如已生成，无需运行 ====''' + # gen_bcd_features(stdSamplePath, stdBarcodePath, stdFeaturePath, eventSourcePath) + + '''==== 2. 生成事件字典, 只需运行一次 ====================''' + # init_eventDict(eventSourcePath, eventDataPath, source_type) + + '''==== 3. 基于事件barcode集和标准库barcode交集构造事件集合 =========''' + evtList, evtDict, stdDict = build_std_evt_dict() + + one2one_pr(evtList, evtDict, stdDict, simType) + + one2SN_pr(evtList, evtDict, stdDict, simType) + +if __name__ == '__main__': + ''' + 共7个地址： + (1) stdSamplePath: 用于生成比对标准特征集的原始图像地址 + (2) stdBarcodePath: 比对标准特征集原始图像地址的pickle文件存储，{barcode: [imgpath1, imgpath1, ...]} + (3) stdFeaturePath: 比对标准特征集特征存储地址 + (4) eventSourcePath: 事件地址, 包含data文件的文件夹或 Yolo-Resnet-Tracker输出的Pickle文件父文件夹 + (5) resultPath: 结果存储地址 + (6) eventDataPath: 用于1:1比对的购物事件存储地址，在resultPath下 + (7) similPath: 1:1比对结果存储地址(事件级)，在resultPath下 + ''' + + stdSamplePath = "/home/wqg/dataset/total_barcode/totalBarcode" + stdBarcodePath = "/home/wqg/dataset/total_barcode/bcdpath" + stdFeaturePath = "/home/wqg/dataset/test_dataset/total_barcode/features_json/v11_barcode_0304/" + + if not os.path.exists(stdBarcodePath): + os.makedirs(stdBarcodePath) + if not os.path.exists(stdFeaturePath): + os.makedirs(stdFeaturePath) + + '''source_type: + "source": eventSourcePath 为 Yolo-Resnet-Tracker 输出的 pickle 文件 + "data": 基于事件切分的原 data 文件版本 + "realtime": 全实时生成的 data 文件 + ''' + source_type = 'source' # 'source', 'data', 'realtime' + simType = "typea" # "simple", "typea", "typeb" + + evttype = "single_event_V10" + # evttype = "single_event_V5" + # evttype = "performence_V10" + # evttype = "performence_V5" + eventSourcePath = "/home/wqg/dataset/pipeline/yrt/{}/shopping_pkl".format(evttype) + + resultPath = "/home/wqg/dataset/pipeline/contrast/{}".format(evttype) + eventDataPath = os.path.join(resultPath, "evtobjs") + similPath = os.path.join(resultPath, "simidata") + if not os.path.exists(eventDataPath): + os.makedirs(eventDataPath) + if not os.path.exists(similPath): + os.makedirs(similPath) + + test_one2one_one2SN(simType) + + + + \ No newline at end of file diff --git a/contrast/onsite_contrast_pr.py b/contrast/onsite_contrast_pr.py new file mode 100644 index 0000000..ffd744a --- /dev/null +++ b/contrast/onsite_contrast_pr.py @@ -0,0 +1,576 @@ + +# -*- coding: utf-8 -*- +""" +Created on Wed Sep 11 11:57:30 2024 + contrast_pr: + 直接利用测试数据中的 data 文件进行 1:1、1:SN、1:n 性能评估 + + test_compare: + 永辉现场试验输出数据的 1:1 性能评估 + 适用于202410前数据保存版本的，需调用 OneToOneCompare.txt +@author: ym +""" +import os +import numpy as np +from pathlib import Path +import matplotlib.pyplot as plt +import sys + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[1] # YOLOv5 root directory +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) + +from tracking.utils.read_data import read_similar + +def read_one2one_data(filepath): + simiList = [] + with open(filepath, 'r', encoding='utf-8') as f: + lines = f.readlines() + + split_flag = False + simi_dict = {} + for i, line in enumerate(lines): + line = line.strip() + if not line: + if len(simi_dict): simiList.append(simi_dict) + simi_dict = {} + continue + + label = line.split(':')[0].strip() + value = line.split(':')[1].strip() + + if label.find("SeqDir") >= 0: + simi_dict["SeqDir"] = value + if label.isdigit() and len(label) >= 8: + + simi_max, simi_min = value.strip(',').split('.') + simi_dict["barcode"] = label + simi_dict["simi_max"] = float(simi_max) / 1000 + simi_dict["simi_min"] = float(simi_min) / 1000 + + if len(simi_dict): simiList.append(simi_dict) + + return simiList + +def plot_pr_curve(matrix): + + simimax, simimean = [], [] + need_analysis = [] + for simidict in matrix: + simimax.append(simidict["simi_max"]) + simimean.append(simidict["simi_min"]) + if simidict["simi_max"]>0.6: + need_analysis.append(simidict) + + simimax = np.array(simimax) + simimean = np.array(simimean) + + TPFN_max = len(simimax) + TPFN_mean = len(simimean) + + fig, axs = plt.subplots(2, 1) + axs[0].hist(simimax, bins=60, edgecolor='black') + axs[0].set_xlim([-0.2, 1]) + axs[0].set_title(f'Same Barcode, Num: {TPFN_max}') + axs[1].hist(simimean, bins=60, edgecolor='black') + axs[1].set_xlim([-0.2, 1]) + axs[1].set_title(f'Cross Barcode, Num: {TPFN_mean}') + # plt.savefig(f'./result/{file}_hist.png') # svg, png, pdf + + Recall_Neg = [] + Thresh = np.linspace(-0.2, 1, 100) + for th in Thresh: + TN = np.sum(simimax < th) + Recall_Neg.append(TN/TPFN_max) + + fig, ax = plt.subplots() + ax.plot(Thresh, Recall_Neg, 'b', label='Recall_Pos: TP/TPFN') + ax.set_xlim([0, 1]) + ax.set_ylim([0, 1]) + ax.grid(True) + ax.set_title('Positive recall') + ax.set_xlabel(f"Num: {TPFN_max}") + ax.legend() + plt.show() + # plt.savefig(f'./result/{file}_pr.png') # svg, png, pdf + + print("Have done!") + pass + + +def test_compare(): + filepaths = [r"\\192.168.1.28\share\测试_202406\0913_扫A放B\0913_1\OneToOneCompare.txt", + r"\\192.168.1.28\share\测试_202406\0913_扫A放B\0913_2\OneToOneCompare.txt", + r"\\192.168.1.28\share\测试_202406\0914_扫A放B\0914_1\OneToOneCompare.txt", + r"\\192.168.1.28\share\测试_202406\0914_扫A放B\0914_2\OneToOneCompare.txt" + ] + + simiList = [] + for fp in filepaths: + slist = read_one2one_data(fp) + simiList.extend(slist) + + plot_pr_curve(simiList) + +def contrast_pr(evtPaths): + ''' + 1:1 + ''' + evtpaths = [] + # date_ = ['2025-3-4_1', '2025-3-5_1', '2025-3-5_2'] + # for dt in date_: + # paths = Path(evtPaths) / dt + abc = [] + for p in Path(evtPaths).iterdir(): + condt1 = p.is_dir() + condt2 = len(p.name.split('_'))>=2 + condt3 = len(p.name.split('_')[-1])>=8 + condt4 = p.name.split('_')[-1].isdigit() + if condt1 and condt2 and condt3 and condt4: + evtpaths.append(p) + elif p.is_dir(): + abc.append(p.stem) + + + # evtpaths = [p for p in paths.iterdir() if p.is_dir() and len(p.name.split('_'))>=2 and len(p.name.split('_')[-1])>8] + # evtpaths = [p for p in paths.iterdir() if p.is_dir()] + + alg_times = [] + + events, similars = [], [] + ##===================================== 扫A放A, 扫A放B场景() + one2oneAA, one2oneAB = [], [] + one2SNAA, one2SNAB = [], [] + + ##===================================== 应用于 1：1 + _tp_events, _fn_events, _fp_events, _tn_events = [], [], [], [] + _tp_simi, _fn_simi, _tn_simi, _fp_simi = [], [], [], [] + + ##===================================== 应用于 1：SN + tp_events, fn_events, fp_events, tn_events = [], [], [], [] + tp_simi, fn_simi, tn_simi, fp_simi = [], [], [], [] + + ##===================================== 应用于1:n + tpevents, fnevents, fpevents, tnevents = [], [], [], [] + tpsimi, fnsimi, tnsimi, fpsimi = [], [], [], [] + + ##===================================== barcodes总数、比对错误事件 + bcdList = [] + one2onePath, one2onePath1 = [], [] + one2SNPath, one2SNPath1 = [], [] + one2nPath = [] + + errorFile_one2one, errorFile_one2SN, errorFile_one2n = [], [], [] + + errorFile = [] + for path in evtpaths: + barcode = path.stem.split('_')[-1] + datapath = path.joinpath('process.data') + + if not barcode.isdigit() or len(barcode)<8: continue + if not datapath.is_file(): continue + + bcdList.append(barcode) + + try: + SimiDict = read_similar(datapath) + except Exception as e: + print(f"{path.stem}, Error: {e}") + + + '''放入为 1:1，相似度取最大值；取出时为 1:SN, 相似度取均值''' + one2one = SimiDict['one2one'] + one2SN = SimiDict['one2SN'] + one2n = SimiDict['one2n'] + + if len(one2one)+len(one2SN)+len(one2n) == 0: + errorFile.append(path.stem) + + dtime = SimiDict["algroStartToEnd"] + if dtime >= 0: + alg_times.append((dtime, path.stem)) + + + '''================== 0. 1:1 ===================''' + barcodes, similars = [], [] + barcodes_ = [] + for dt in one2one: + one2onePath.append((path.stem)) + if dt['similar']==0: + one2onePath1.append((path.stem)) + continue + barcodes.append(dt['barcode']) + similars.append(dt['similar']) + + + barcodes_.append(path.stem) + + + + + + if len(barcodes)==len(similars) and len(barcodes)!=0: + ## 扫A放A, 扫A放B场景 + simAA = [similars[i] for i in range(len(barcodes)) if barcodes[i]==barcode] + simAB = [similars[i] for i in range(len(barcodes)) if barcodes[i]!=barcode] + + one2oneAA.extend(simAA) + one2oneAB.extend(simAB) + + ## 相似度排序，barcode相等且排名第一为TP，适用于多的barcode相似度比较 + max_idx = similars.index(max(similars)) + max_sim = similars[max_idx] + # max_bcd = barcodes[max_idx] + for i in range(len(one2one)): + bcd, simi = barcodes[i], similars[i] + if bcd==barcode and simi==max_sim: + _tp_simi.append(simi) + _tp_events.append(path.stem) + elif bcd==barcode and simi!=max_sim: + _fn_simi.append(simi) + _fn_events.append(path.stem) + elif bcd!=barcode and simi!=max_sim: + _tn_simi.append(simi) + _tn_events.append(path.stem) + elif bcd!=barcode and simi==max_sim and barcode in barcodes: + _fp_simi.append(simi) + _fp_events.append(path.stem) + else: + errorFile_one2one.append(path.stem) + elif len(one2SN)+len(one2n) == 0: + errorFile_one2one.append(path.stem) + + + '''================== 2. 取出场景下的 1 : Small N ===================''' + barcodes, similars = [], [] + for dt in one2SN: + barcodes.append(dt['barcode']) + similars.append(dt['similar']) + + + if len(barcodes)==len(similars) and len(barcodes)!=0: + ## 扫A放A, 扫A放B场景 + simAA = [similars[i] for i in range(len(barcodes)) if barcodes[i]==barcode] + simAB = [similars[i] for i in range(len(barcodes)) if barcodes[i]!=barcode] + + one2SNAA.extend(simAA) + one2SNAB.extend(simAB) + + one2SNPath.append(path.stem) + if len(simAA)==0: + errorFile_one2SN.append(path.stem) + + ## 相似度排序，barcode相等且排名第一为TP，适用于多的barcode相似度比较 + max_idx = similars.index(max(similars)) + max_sim = similars[max_idx] + # max_bcd = barcodes[max_idx] + for i in range(len(one2SN)): + bcd, simi = barcodes[i], similars[i] + if bcd==barcode and simi==max_sim: + tp_simi.append(simi) + tp_events.append(path.stem) + elif bcd==barcode and simi!=max_sim: + fn_simi.append(simi) + fn_events.append(path.stem) + elif bcd!=barcode and simi!=max_sim: + tn_simi.append(simi) + tn_events.append(path.stem) + elif bcd!=barcode and simi==max_sim and barcode in barcodes: + fp_simi.append(simi) + fp_events.append(path.stem) + else: + errorFile_one2SN.append(path.stem) + + + + + '''===================== 3. 取出场景下的 1:n ========================''' + events, evt_barcodes, evt_similars, evt_types = [], [], [], [] + for dt in one2n: + events.append(dt["event"]) + evt_barcodes.append(dt["barcode"]) + evt_similars.append(dt["similar"]) + evt_types.append(dt["type"]) + + + + if len(events)==len(evt_barcodes)==len(evt_similars)==len(evt_types) and len(events)>0: + if not barcode in evt_barcodes: + errorFile_one2n.append(path.stem) + continue + + if len(barcodes_): + print("do") + + one2nPath.append(path.stem) + maxsim = evt_similars[evt_similars.index(max(evt_similars))] + for i in range(len(one2n)): + bcd, simi = evt_barcodes[i], evt_similars[i] + + if bcd==barcode and simi==maxsim: + tpsimi.append(simi) + tpevents.append(path.stem) + elif bcd==barcode and simi!=maxsim: + fnsimi.append(simi) + fnevents.append(path.stem) + elif bcd!=barcode and simi!=maxsim: + tnsimi.append(simi) + tnevents.append(path.stem) + elif bcd!=barcode and simi==maxsim and barcode in evt_barcodes: + fpsimi.append(simi) + fpevents.append(path.stem) + else: + errorFile_one2n.append(path.stem) + + + '''命名规则: + 1:1 (max) 1:1 (max) 1:n 1:N + _TP TP_ TP TPX + _PPrecise PPrecise_ PPrecise PPreciseX + tpsimi tp_simi + ''' + + ''' 1:1 数据存储, 相似度计算方式：最大值、均值''' + _PPrecise, _PRecall = [], [] + _NPrecise, _NRecall = [], [] + PPrecise_, PRecall_ = [], [] + NPrecise_, NRecall_ = [], [] + + ''' 1:SN 数据存储，需根据相似度排序''' + PPreciseX, PRecallX = [], [] + NPreciseX, NRecallX = [], [] + + ''' 1:n 数据存储，需根据相似度排序''' + PPrecise, PRecall = [], [] + NPrecise, NRecall = [], [] + + + + Thresh = np.linspace(-0.2, 1, 100) + for th in Thresh: + '''(Precise, Recall) 计算方式, 若 1:1 与 1:SN 相似度选择方式相同，则可以合并''' + '''===================================== 1:1 最大值''' + _TP = sum(np.array(one2oneAA) >= th) + _FP = sum(np.array(one2oneAB) >= th) + _FN = sum(np.array(one2oneAA) < th) + _TN = sum(np.array(one2oneAB) < th) + + _PPrecise.append(_TP/(_TP+_FP+1e-6)) + _PRecall.append(_TP/(_TP+_FN+1e-6)) + _NPrecise.append(_TN/(_TN+_FN+1e-6)) + _NRecall.append(_TN/(_TN+_FP+1e-6)) + + '''===================================== 1:SN 均值''' + TP_ = sum(np.array(one2SNAA) >= th) + FP_ = sum(np.array(one2SNAB) >= th) + FN_ = sum(np.array(one2SNAA) < th) + TN_ = sum(np.array(one2SNAB) < th) + + PPrecise_.append(TP_/(TP_+FP_+1e-6)) + PRecall_.append(TP_/(len(one2SNAA)+1e-6)) + NPrecise_.append(TN_/(TN_+FN_+1e-6)) + NRecall_.append(TN_/(len(one2SNAB)+1e-6)) + + '''适用于 (Precise, Recall) 计算方式：多个相似度计算并排序，barcode相等且排名第一为 TP ''' + '''===================================== 1:SN ''' + TPX = sum(np.array(tp_simi) >= th) + FPX = sum(np.array(fp_simi) >= th) + FNX = sum(np.array(fn_simi) < th) + TNX = sum(np.array(tn_simi) < th) + PPreciseX.append(TPX/(TPX+FPX+1e-6)) + PRecallX.append(TPX/(TPX+FNX+1e-6)) + + NPreciseX.append(TNX/(TNX+FNX+1e-6)) + NRecallX.append(TNX/(TNX+FPX+1e-6)) + + + '''===================================== 1:n''' + TP = sum(np.array(tpsimi) >= th) + FP = sum(np.array(fpsimi) >= th) + FN = sum(np.array(fnsimi) < th) + TN = sum(np.array(tnsimi) < th) + + PPrecise.append(TP/(TP+FP+1e-6)) + PRecall.append(TP/(TP+FN+1e-6)) + NPrecise.append(TN/(TN+FN+1e-6)) + NRecall.append(TN/(TN+FP+1e-6)) + + algtime = [] + for tm, _ in alg_times: + algtime.append(tm) + fig, ax = plt.subplots() + ax.hist(np.array(algtime), bins=100, edgecolor='black') + ax.set_title('Algorthm Spend Time') + ax.set_xlabel(f"Event Num: {len(alg_times)}") + plt.show() + + '''1. ============================= 1:1 最大值方案 曲线''' + fig, ax = plt.subplots() + ax.plot(Thresh, _PPrecise, 'r', label='Precise_Pos: TP/TPFP') + ax.plot(Thresh, _PRecall, 'b', label='Recall_Pos: TP/TPFN') + ax.plot(Thresh, _NPrecise, 'g', label='Precise_Neg: TN/TNFP') + ax.plot(Thresh, _NRecall, 'c', label='Recall_Neg: TN/TNFN') + ax.set_xlim([0, 1]) + ax.set_ylim([0, 1]) + ax.set_xticks(np.arange(0, 1, 0.1)) + ax.set_yticks(np.arange(0, 1, 0.1)) + ax.grid(True, linestyle='--') + ax.set_title('1:1 Precise & Recall') + ax.set_xlabel(f"Event Num: {len(one2oneAA)+len(one2oneAB)}") + ax.legend() + plt.show() + ## ============================= 1:1 最大值方案 直方图''' + fig, axes = plt.subplots(2, 1) + axes[0].hist(np.array(one2oneAA), bins=60, edgecolor='black') + axes[0].set_xlim([-0.2, 1]) + axes[0].set_title('AA') + axes[1].hist(np.array(one2oneAB), bins=60, edgecolor='black') + axes[1].set_xlim([-0.2, 1]) + axes[1].set_title('BB') + plt.show() + + + + + '''2. ============================= 1:1 均值方案 曲线''' + # fig, ax = plt.subplots() + # ax.plot(Thresh, PPrecise_, 'r', label='Precise_Pos: TP/TPFP') + # ax.plot(Thresh, PRecall_, 'b', label='Recall_Pos: TP/TPFN') + # ax.plot(Thresh, NPrecise_, 'g', label='Precise_Neg: TN/TNFP') + # ax.plot(Thresh, NRecall_, 'c', label='Recall_Neg: TN/TNFN') + # ax.set_xlim([0, 1]) + # ax.set_ylim([0, 1]) + # ax.grid(True) + # ax.set_title('1:1 Precise & Recall') + # ax.set_xlabel(f"Event Num: {len(one2SNAA)}") + # ax.legend() + # plt.show() + # ## ============================= 1:1 均值方案 直方图''' + # fig, axes = plt.subplots(2, 1) + # axes[0].hist(np.array(one2SNAA), bins=60, edgecolor='black') + # axes[0].set_xlim([-0.2, 1]) + # axes[0].set_title('AA') + # axes[0].set_xlabel(f"Event Num: {len(one2SNAA)}") + + # axes[1].hist(np.array(one2SNAB), bins=60, edgecolor='black') + # axes[1].set_xlim([-0.2, 1]) + # axes[1].set_title('BB') + # axes[1].set_xlabel(f"Event Num: {len(one2SNAB)}") + # plt.show() + + ''''3. ============================= 1:SN 曲线''' + fig, ax = plt.subplots() + ax.plot(Thresh, PPreciseX, 'r', label='Precise_Pos: TP/TPFP') + ax.plot(Thresh, PRecallX, 'b', label='Recall_Pos: TP/TPFN') + ax.plot(Thresh, NPreciseX, 'g', label='Precise_Neg: TN/TNFP') + ax.plot(Thresh, NRecallX, 'c', label='Recall_Neg: TN/TNFN') + ax.set_xlim([0, 1]) + ax.set_ylim([0, 1]) + ax.set_xticks(np.arange(0, 1, 0.1)) + ax.set_yticks(np.arange(0, 1, 0.1)) + ax.grid(True, linestyle='--') + ax.set_title('1:SN Precise & Recall') + ax.set_xlabel(f"Event Num: {len(one2SNAA)}") + ax.legend() + plt.show() + ## ============================= 1:N 展厅 直方图''' + fig, axes = plt.subplots(2, 2) + axes[0, 0].hist(tp_simi, bins=60, edgecolor='black') + axes[0, 0].set_xlim([-0.2, 1]) + axes[0, 0].set_title(f'TP({len(tp_simi)})') + axes[0, 1].hist(fp_simi, bins=60, edgecolor='black') + axes[0, 1].set_xlim([-0.2, 1]) + axes[0, 1].set_title(f'FP({len(fp_simi)})') + axes[1, 0].hist(tn_simi, bins=60, edgecolor='black') + axes[1, 0].set_xlim([-0.2, 1]) + axes[1, 0].set_title(f'TN({len(tn_simi)})') + axes[1, 1].hist(fn_simi, bins=60, edgecolor='black') + axes[1, 1].set_xlim([-0.2, 1]) + axes[1, 1].set_title(f'FN({len(fn_simi)})') + plt.show() + + + '''4. ============================= 1:n 曲线,''' + fig, ax = plt.subplots() + ax.plot(Thresh, PPrecise, 'r', label='Precise_Pos: TP/TPFP') + ax.plot(Thresh, PRecall, 'b', label='Recall_Pos: TP/TPFN') + ax.plot(Thresh, NPrecise, 'g', label='Precise_Neg: TN/TNFP') + ax.plot(Thresh, NRecall, 'c', label='Recall_Neg: TN/TNFN') + ax.set_xlim([0, 1]) + ax.set_ylim([0, 1]) + ax.set_xticks(np.arange(0, 1, 0.1)) + ax.set_yticks(np.arange(0, 1, 0.1)) + ax.grid(True, linestyle='--') + ax.set_title('1:n Precise & Recall') + ax.set_xlabel(f"Event Num: {len(tpsimi)+len(fnsimi)}") + ax.legend() + plt.show() + ## ============================= 1:n 直方图''' + fig, axes = plt.subplots(2, 2) + axes[0, 0].hist(tpsimi, bins=60, edgecolor='black') + axes[0, 0].set_xlim([-0.2, 1]) + axes[0, 0].set_title(f'TP({len(tpsimi)})') + axes[0, 1].hist(fpsimi, bins=60, edgecolor='black') + axes[0, 1].set_xlim([-0.2, 1]) + axes[0, 1].set_title(f'FP({len(fpsimi)})') + axes[1, 0].hist(tnsimi, bins=60, edgecolor='black') + axes[1, 0].set_xlim([-0.2, 1]) + axes[1, 0].set_title(f'TN({len(tnsimi)})') + axes[1, 1].hist(fnsimi, bins=60, edgecolor='black') + + axes[1, 1].set_xlim([-0.2, 1]) + axes[1, 1].set_title(f'FN({len(fnsimi)})') + plt.show() + + # fpsnErrFile = str(paths.joinpath("one2SN_Error.txt")) + # with open(fpsnErrFile, "w") as file: + # for item in fp_events: + # file.write(item + "\n") + + # fpErrFile = str(paths.joinpath("one2n_Error.txt")) + # with open(fpErrFile, "w") as file: + # for item in fpevents: + # file.write(item + "\n") + + + # bcdSet = set(bcdList) + + + # one2nErrFile = os.path.join(evtPaths, "one_2_Small_n_Error.txt") + # with open(one2nErrFile, "w") as file: + # for item in fnevents: + # file.write(item + "\n") + + # one2NErrFile = os.path.join(evtPaths, "one_2_Big_N_Error.txt") + # with open(one2NErrFile, "w") as file: + # for item in fn_events: + # file.write(item + "\n") + + print('Done!') + + +if __name__ == "__main__": + evtpaths = r"/home/wqg/dataset/test_base_dataset/single_event/source" + contrast_pr(evtpaths) + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/contrast/result/pickle/20240909_164035.txt b/contrast/result/pickle/20240909_164035.txt new file mode 100644 index 0000000..dc77bfe --- /dev/null +++ b/contrast/result/pickle/20240909_164035.txt @@ -0,0 +1,174 @@ +same, 6954256905319, 20240715-144627_6954256905319, 0.208, 0.746, 0.604 +same, 6902265589617, 20240719-143553_6902265589617, 0.296, 0.719, 0.561 +same, 6930364600025, 20240719-143603_6930364600025, 0.301, 0.772, 0.666 +same, 6924509902839, 20240719-143630_6924509902839, 0.176, 0.660, 0.435 +same, 6902007115524, 20240719-143729_6902007115524, 0.230, 0.635, 0.525 +same, 6930628801472, 20240719-143807_6930628801472, 0.279, 0.730, 0.712 +same, 6902265314615, 20240719-144126_6902265314615, 0.376, 0.897, 0.754 +same, 6928312901324, 20240719-144226_6928312901324, 0.335, 0.782, 0.693 +same, 6908355010450, 20240719-144335_6908355010450, 0.256, 0.748, 0.580 +same, 6924509902754, 20240719-144454_6924509902754, 0.459, 0.832, 0.831 +same, 6902265314615, 20240719-144621_6902265314615, 0.410, 0.847, 0.667 +same, 6941499112529, 20240719-145022_6941499112529, 0.174, 0.736, 0.309 +same, 6972786772775, 20240719-145149_6972786772775, 0.243, 0.698, 0.565 +same, 6920267906706, 20240719-145319_6920267906706, 0.306, 0.799, 0.767 +same, 6974925910116, 20240719-145505_6974925910116, 0.372, 0.844, 0.743 +same, 6928002374551, 20240719-145542_6928002374551, 0.171, 0.773, 0.499 +same, 6920152414071, 20240719-145816_6920152414071, 0.469, 0.816, 0.871 +same, 6973360690409, 20240719-145926_6973360690409, 0.160, 0.636, 0.591 +same, 6930364600001, 20240719-150000_6930364600001, 0.318, 0.790, 0.754 +same, 6937962111526, 20240719-150209_6937962111526, 0.222, 0.660, 0.575 +same, 6924509902839, 20240719-150345_6924509902839, 0.155, 0.757, 0.470 +same, 6924509902839, 20240719-150458_6924509902839, 0.307, 0.670, 0.583 +same, 6924187895423, 20240719-150623_6924187895423, 0.437, 0.882, 0.892 +same, 6916749306102, 20240719-150716_6916749306102, 0.249, 0.725, 0.682 +same, 6951469100067, 20240719-150750_6951469100067, 0.535, 0.938, 0.905 +same, 6973360690409, 20240719-151009_6973360690409, 0.143, 0.578, 0.463 +same, 6923120632521, 20240719-151135_6923120632521, 0.301, 0.865, 0.711 +same, 6958350501101, 20240719-151215_6958350501101, 0.550, 0.902, 0.888 +same, 6951469100067, 20240719-151325_6951469100067, 0.588, 0.927, 0.899 +same, 6937962111526, 20240719-151349_6937962111526, 0.153, 0.657, 0.531 +same, 6916749306102, 20240719-151416_6916749306102, 0.228, 0.677, 0.587 +same, 6930364600001, 20240719-151535_6930364600001, 0.151, 0.692, 0.524 +same, 6911988027191, 20240719-151724_6911988027191, 0.127, 0.525, 0.337 +same, 6972219890632, 20240719-151800_6972219890632, 0.288, 0.872, 0.700 +same, 6901668935908, 20240719-152125_6901668935908, 0.210, 0.806, 0.436 +same, 6920152471517, 20240719-152315_6920152471517, 0.392, 0.836, 0.640 +same, 6951957205274, 20240719-152816_6951957205274, 0.345, 0.810, 0.602 +same, 6971284201244, 20240719-152832_6971284201244, 0.229, 0.817, 0.557 +same, 6930364600001, 20240719-153117_6930364600001, 0.065, 0.743, 0.196 +same, 6972900962273, 20240722-143041_6972900962273, 0.214, 0.596, 0.528 +same, 6902265587712, 20240722-144630_6902265587712, 0.362, 0.816, 0.673 +same, 4892655902237, 20240722-160857_4892655902237, 0.433, 0.783, 0.842 +same, 4892655902015, 20240722-162225_4892655902015, 0.100, 0.668, 0.282 +same, 6938888889896, 20240722-162328_6938888889896, 0.176, 0.779, 0.407 +same, 6920912342002, 20240722-163428_6920912342002, 0.313, 0.825, 0.857 +same, 6971075127784, 20240722-163600_6971075127784, 0.734, 0.923, 0.915 +same, 6926265317384, 20240722-164552_6926265317384, 0.179, 0.715, 0.613 +same, 6916196422325, 20240722-164859_6916196422325, 0.367, 0.773, 0.675 +same, 6901668006011, 20240722-165548_6901668006011, 0.291, 0.718, 0.546 +same, 6926419102347, 20240722-170325_6926419102347, 0.256, 0.785, 0.768 +same, 4897098800133, 20240723-094447_4897098800133, 0.379, 0.846, 0.805 +same, 6971284204597, 20240723-094525_6971284204597, 0.206, 0.732, 0.597 +same, 6970470331529, 20240723-094617_6970470331529, 0.235, 0.924, 0.586 +same, 6918598120014, 20240723-095103_6918598120014, 0.426, 0.905, 0.819 +same, 6902265202318, 20240723-095259_6902265202318, 0.641, 0.907, 0.909 +same, 6938737200650, 20240723-095527_6938737200650, 0.219, 0.825, 0.651 +same, 6902265131212, 20240723-095819_6902265131212, 0.197, 0.899, 0.396 +same, 6902890022558, 20240723-095858_6902890022558, 0.161, 0.713, 0.590 +same, 6925303795535, 20240723-100031_6925303795535, 0.298, 0.838, 0.674 +same, 6920382723257, 20240723-100113_6920382723257, 0.282, 0.815, 0.600 +same, 6976399680069, 20240723-100148_6976399680069, 0.040, 0.554, 0.132 +same, 6921502656660, 20240723-100318_6921502656660, 0.059, 0.424, 0.173 +same, 6938737200650, 20240723-100416_6938737200650, 0.222, 0.701, 0.620 +same, 6918598120014, 20240723-100501_6918598120014, 0.102, 0.742, 0.206 +same, 4897098800133, 20240723-100845_4897098800133, 0.305, 0.834, 0.744 +same, 6971284204597, 20240723-101014_6971284204597, 0.223, 0.714, 0.623 +same, 6902265202318, 20240723-101144_6902265202318, 0.537, 0.875, 0.812 +same, 6902265131212, 20240723-101416_6902265131212, 0.463, 0.901, 0.747 +same, 6918598120014, 20240723-101829_6918598120014, 0.524, 0.901, 0.855 +same, 6902265131212, 20240723-101851_6902265131212, 0.418, 0.899, 0.820 +same, 6920382723257, 20240723-102033_6920382723257, 0.320, 0.780, 0.755 +same, 6902890022558, 20240723-102206_6902890022558, 0.123, 0.707, 0.387 +same, 6970470331529, 20240723-102409_6970470331529, 0.099, 0.641, 0.300 +same, 6938737200650, 20240723-102515_6938737200650, 0.177, 0.695, 0.515 +same, 6937451883132, 20240723-110216_6937451883132, 0.300, 0.817, 0.706 +same, 6973337010216, 20240723-110224_6973337010216, 0.202, 0.727, 0.633 +same, 6972359682937, 20240723-110937_6972359682937, 0.229, 0.792, 0.540 +same, 6937451883132, 20240723-111446_6937451883132, 0.363, 0.796, 0.844 +same, 6973337010216, 20240723-111521_6973337010216, 0.120, 0.764, 0.359 +same, 6924187828544, 20240723-111602_6924187828544, 0.378, 0.857, 0.733 +same, 6934024590268, 20240723-112004_6934024590268, 0.205, 0.724, 0.609 +same, 6972359682937, 20240723-112038_6972359682937, 0.432, 0.839, 0.857 +same, 6924187828544, 20240723-112505_6924187828544, 0.304, 0.835, 0.601 +same, 6974158894528, 20240723-155423_6974158894528, 0.440, 0.854, 0.765 +same, 6973497200045, 20240723-160015_6973497200045, 0.263, 0.712, 0.551 +same, 6970399922518, 20240723-160227_6970399922518, 0.087, 0.564, 0.243 +same, 6974158894528, 20240723-160942_6974158894528, 0.429, 0.847, 0.818 +diff, 6951957205274, 20240715-144627_6954256905319, -0.032, 0.375, -0.064 +diff, 6970399922518, 20240719-143553_6902265589617, -0.040, 0.366, -0.107 +diff, 6926419102347, 20240719-143603_6930364600025, 0.052, 0.455, 0.137 +diff, 6925303795535, 20240719-143630_6924509902839, -0.008, 0.379, -0.021 +diff, 6924509902839, 20240719-143729_6902007115524, 0.060, 0.421, 0.134 +diff, 6924187895423, 20240719-143807_6930628801472, 0.131, 0.416, 0.254 +diff, 6954256905319, 20240719-144126_6902265314615, 0.030, 0.456, 0.088 +diff, 6972786772775, 20240719-144226_6928312901324, -0.000, 0.422, -0.001 +diff, 6902265131212, 20240719-144335_6908355010450, -0.037, 0.308, -0.079 +diff, 6930364600025, 20240719-144454_6924509902754, 0.075, 0.470, 0.156 +diff, 6902265202318, 20240719-144621_6902265314615, 0.212, 0.554, 0.315 +diff, 6925303795535, 20240719-145022_6941499112529, -0.018, 0.474, -0.038 +diff, 6902265131212, 20240719-145149_6972786772775, -0.018, 0.327, -0.032 +diff, 6902265314615, 20240719-145319_6920267906706, 0.103, 0.514, 0.202 +diff, 6951469100067, 20240719-145505_6974925910116, -0.051, 0.274, -0.081 +diff, 6911988027191, 20240719-145542_6928002374551, 0.039, 0.392, 0.134 +diff, 4892655902237, 20240719-145816_6920152414071, -0.144, 0.262, -0.269 +diff, 6924187828544, 20240719-145926_6973360690409, -0.065, 0.287, -0.158 +diff, 6951957205274, 20240719-150000_6930364600001, 0.011, 0.418, 0.019 +diff, 6926419102347, 20240719-150209_6937962111526, -0.009, 0.462, -0.023 +diff, 6930628801472, 20240719-150345_6924509902839, 0.005, 0.429, 0.019 +diff, 6916196422325, 20240719-150458_6924509902839, 0.041, 0.412, 0.068 +diff, 6920912342002, 20240719-150623_6924187895423, -0.008, 0.351, -0.020 +diff, 6925303795535, 20240719-150716_6916749306102, -0.043, 0.329, -0.116 +diff, 4892655902015, 20240719-150750_6951469100067, -0.007, 0.332, -0.015 +diff, 6971284204597, 20240719-151009_6973360690409, -0.058, 0.397, -0.163 +diff, 4892655902015, 20240719-151135_6923120632521, -0.084, 0.425, -0.223 +diff, 6921502656660, 20240719-151215_6958350501101, 0.044, 0.576, 0.095 +diff, 6920152471517, 20240719-151325_6951469100067, 0.065, 0.363, 0.107 +diff, 6970399922518, 20240719-151349_6937962111526, -0.048, 0.405, -0.151 +diff, 6972786772775, 20240719-151416_6916749306102, 0.005, 0.436, 0.015 +diff, 6971075127784, 20240719-151535_6930364600001, -0.070, 0.343, -0.152 +diff, 6916196422325, 20240719-151724_6911988027191, 0.062, 0.373, 0.102 +diff, 6937451883132, 20240719-151800_6972219890632, 0.012, 0.383, 0.033 +diff, 6902265314615, 20240719-152125_6901668935908, -0.055, 0.388, -0.107 +diff, 6928312901324, 20240719-152315_6920152471517, -0.039, 0.481, -0.067 +diff, 6928002374551, 20240719-152816_6951957205274, 0.057, 0.489, 0.129 +diff, 6928002374551, 20240719-152832_6971284201244, 0.010, 0.309, 0.025 +diff, 6970470331529, 20240719-153117_6930364600001, -0.044, 0.358, -0.120 +diff, 6974925910116, 20240722-143041_6972900962273, 0.025, 0.482, 0.064 +diff, 6938737200650, 20240722-144630_6902265587712, 0.019, 0.393, 0.054 +diff, 6941499112529, 20240722-160857_4892655902237, 0.003, 0.373, 0.006 +diff, 6924187895423, 20240722-162225_4892655902015, 0.028, 0.393, 0.066 +diff, 6923120632521, 20240722-162328_6938888889896, -0.022, 0.366, -0.049 +diff, 6972900962273, 20240722-163428_6920912342002, 0.072, 0.450, 0.171 +diff, 6938888889896, 20240722-163600_6971075127784, 0.112, 0.471, 0.189 +diff, 6930364600001, 20240722-164552_6926265317384, -0.033, 0.370, -0.090 +diff, 6923120632521, 20240722-164859_6916196422325, 0.134, 0.528, 0.279 +diff, 6926419102347, 20240722-165548_6901668006011, 0.028, 0.346, 0.071 +diff, 6924187828544, 20240722-170325_6926419102347, 0.011, 0.453, 0.024 +diff, 6926265317384, 20240723-094447_4897098800133, 0.006, 0.355, 0.019 +diff, 6954256905319, 20240723-094525_6971284204597, -0.000, 0.328, -0.000 +diff, 6937451883132, 20240723-094617_6970470331529, -0.013, 0.404, -0.036 +diff, 6973337010216, 20240723-095103_6918598120014, 0.029, 0.402, 0.078 +diff, 6937451883132, 20240723-095259_6902265202318, -0.135, 0.347, -0.280 +diff, 6971284201244, 20240723-095527_6938737200650, -0.023, 0.648, -0.058 +diff, 6972786772775, 20240723-095819_6902265131212, 0.008, 0.502, 0.022 +diff, 6941499112529, 20240723-095858_6902890022558, 0.022, 0.341, 0.049 +diff, 6930628801472, 20240723-100031_6925303795535, 0.020, 0.384, 0.049 +diff, 6970399922518, 20240723-100113_6920382723257, -0.041, 0.394, -0.101 +diff, 6908355010450, 20240723-100148_6976399680069, 0.061, 0.444, 0.150 +diff, 6973497200045, 20240723-100318_6921502656660, 0.007, 0.316, 0.017 +diff, 6951957205274, 20240723-100416_6938737200650, -0.089, 0.437, -0.163 +diff, 6902265131212, 20240723-100501_6918598120014, -0.069, 0.316, -0.133 +diff, 6972359682937, 20240723-100845_4897098800133, -0.017, 0.325, -0.038 +diff, 6972900962273, 20240723-101014_6971284204597, -0.010, 0.373, -0.020 +diff, 6930628801472, 20240723-101144_6902265202318, -0.045, 0.394, -0.101 +diff, 6920152471517, 20240723-101416_6902265131212, -0.002, 0.335, -0.003 +diff, 6934024590268, 20240723-101829_6918598120014, 0.013, 0.373, 0.030 +diff, 4897098800133, 20240723-101851_6902265131212, -0.069, 0.277, -0.157 +diff, 4892655902015, 20240723-102033_6920382723257, 0.012, 0.284, 0.029 +diff, 6921502656660, 20240723-102206_6902890022558, -0.030, 0.487, -0.072 +diff, 6937962111526, 20240723-102409_6970470331529, 0.001, 0.340, 0.004 +diff, 6920267906706, 20240723-102515_6938737200650, 0.094, 0.495, 0.223 +diff, 6926265317384, 20240723-110216_6937451883132, -0.041, 0.358, -0.121 +diff, 6902265202318, 20240723-110224_6973337010216, 0.012, 0.318, 0.021 +diff, 6901668006011, 20240723-110937_6972359682937, -0.060, 0.379, -0.140 +diff, 6926419102347, 20240723-111446_6937451883132, 0.045, 0.421, 0.119 +diff, 6941499112529, 20240723-111521_6973337010216, 0.019, 0.381, 0.039 +diff, 6973337010216, 20240723-111602_6924187828544, 0.052, 0.476, 0.139 +diff, 6920382723257, 20240723-112004_6934024590268, 0.025, 0.366, 0.060 +diff, 6938888889896, 20240723-112038_6972359682937, 0.028, 0.384, 0.057 +diff, 6971284204597, 20240723-112505_6924187828544, -0.028, 0.387, -0.075 +diff, 6926265317384, 20240723-155423_6974158894528, 0.019, 0.334, 0.046 +diff, 6974158894528, 20240723-160015_6973497200045, -0.033, 0.330, -0.075 +diff, 6974925910116, 20240723-160227_6970399922518, 0.010, 0.344, 0.023 +diff, 6972219890632, 20240723-160942_6974158894528, -0.060, 0.369, -0.118 diff --git a/contrast/result/pickle/20240909_164035_ft16.txt b/contrast/result/pickle/20240909_164035_ft16.txt new file mode 100644 index 0000000..ce47233 --- /dev/null +++ b/contrast/result/pickle/20240909_164035_ft16.txt @@ -0,0 +1,174 @@ +same, 6954256905319, 20240715-144627_6954256905319, 0.208, 0.746, 0.605 +same, 6902265589617, 20240719-143553_6902265589617, 0.296, 0.719, 0.561 +same, 6930364600025, 20240719-143603_6930364600025, 0.301, 0.772, 0.666 +same, 6924509902839, 20240719-143630_6924509902839, 0.176, 0.660, 0.435 +same, 6902007115524, 20240719-143729_6902007115524, 0.230, 0.635, 0.525 +same, 6930628801472, 20240719-143807_6930628801472, 0.279, 0.730, 0.712 +same, 6902265314615, 20240719-144126_6902265314615, 0.376, 0.896, 0.754 +same, 6928312901324, 20240719-144226_6928312901324, 0.335, 0.782, 0.693 +same, 6908355010450, 20240719-144335_6908355010450, 0.256, 0.748, 0.580 +same, 6924509902754, 20240719-144454_6924509902754, 0.459, 0.832, 0.831 +same, 6902265314615, 20240719-144621_6902265314615, 0.410, 0.847, 0.667 +same, 6941499112529, 20240719-145022_6941499112529, 0.174, 0.736, 0.309 +same, 6972786772775, 20240719-145149_6972786772775, 0.243, 0.698, 0.565 +same, 6920267906706, 20240719-145319_6920267906706, 0.306, 0.800, 0.767 +same, 6974925910116, 20240719-145505_6974925910116, 0.372, 0.844, 0.743 +same, 6928002374551, 20240719-145542_6928002374551, 0.171, 0.773, 0.499 +same, 6920152414071, 20240719-145816_6920152414071, 0.469, 0.816, 0.871 +same, 6973360690409, 20240719-145926_6973360690409, 0.160, 0.636, 0.591 +same, 6930364600001, 20240719-150000_6930364600001, 0.318, 0.790, 0.754 +same, 6937962111526, 20240719-150209_6937962111526, 0.222, 0.660, 0.575 +same, 6924509902839, 20240719-150345_6924509902839, 0.155, 0.757, 0.470 +same, 6924509902839, 20240719-150458_6924509902839, 0.307, 0.670, 0.583 +same, 6924187895423, 20240719-150623_6924187895423, 0.437, 0.882, 0.892 +same, 6916749306102, 20240719-150716_6916749306102, 0.249, 0.725, 0.682 +same, 6951469100067, 20240719-150750_6951469100067, 0.535, 0.938, 0.905 +same, 6973360690409, 20240719-151009_6973360690409, 0.143, 0.578, 0.463 +same, 6923120632521, 20240719-151135_6923120632521, 0.301, 0.865, 0.711 +same, 6958350501101, 20240719-151215_6958350501101, 0.550, 0.902, 0.888 +same, 6951469100067, 20240719-151325_6951469100067, 0.588, 0.927, 0.899 +same, 6937962111526, 20240719-151349_6937962111526, 0.153, 0.657, 0.531 +same, 6916749306102, 20240719-151416_6916749306102, 0.228, 0.677, 0.587 +same, 6930364600001, 20240719-151535_6930364600001, 0.151, 0.692, 0.524 +same, 6911988027191, 20240719-151724_6911988027191, 0.127, 0.525, 0.337 +same, 6972219890632, 20240719-151800_6972219890632, 0.288, 0.872, 0.700 +same, 6901668935908, 20240719-152125_6901668935908, 0.210, 0.806, 0.436 +same, 6920152471517, 20240719-152315_6920152471517, 0.392, 0.836, 0.640 +same, 6951957205274, 20240719-152816_6951957205274, 0.345, 0.810, 0.602 +same, 6971284201244, 20240719-152832_6971284201244, 0.229, 0.817, 0.557 +same, 6930364600001, 20240719-153117_6930364600001, 0.065, 0.743, 0.196 +same, 6972900962273, 20240722-143041_6972900962273, 0.214, 0.596, 0.528 +same, 6902265587712, 20240722-144630_6902265587712, 0.362, 0.816, 0.673 +same, 4892655902237, 20240722-160857_4892655902237, 0.433, 0.783, 0.842 +same, 4892655902015, 20240722-162225_4892655902015, 0.100, 0.668, 0.282 +same, 6938888889896, 20240722-162328_6938888889896, 0.176, 0.779, 0.407 +same, 6920912342002, 20240722-163428_6920912342002, 0.313, 0.825, 0.857 +same, 6971075127784, 20240722-163600_6971075127784, 0.734, 0.923, 0.915 +same, 6926265317384, 20240722-164552_6926265317384, 0.179, 0.715, 0.613 +same, 6916196422325, 20240722-164859_6916196422325, 0.367, 0.773, 0.675 +same, 6901668006011, 20240722-165548_6901668006011, 0.291, 0.718, 0.546 +same, 6926419102347, 20240722-170325_6926419102347, 0.256, 0.785, 0.768 +same, 4897098800133, 20240723-094447_4897098800133, 0.379, 0.846, 0.805 +same, 6971284204597, 20240723-094525_6971284204597, 0.206, 0.732, 0.597 +same, 6970470331529, 20240723-094617_6970470331529, 0.235, 0.924, 0.586 +same, 6918598120014, 20240723-095103_6918598120014, 0.426, 0.905, 0.819 +same, 6902265202318, 20240723-095259_6902265202318, 0.641, 0.907, 0.909 +same, 6938737200650, 20240723-095527_6938737200650, 0.219, 0.825, 0.651 +same, 6902265131212, 20240723-095819_6902265131212, 0.197, 0.899, 0.396 +same, 6902890022558, 20240723-095858_6902890022558, 0.161, 0.713, 0.590 +same, 6925303795535, 20240723-100031_6925303795535, 0.298, 0.838, 0.674 +same, 6920382723257, 20240723-100113_6920382723257, 0.282, 0.815, 0.600 +same, 6976399680069, 20240723-100148_6976399680069, 0.040, 0.554, 0.132 +same, 6921502656660, 20240723-100318_6921502656660, 0.059, 0.424, 0.173 +same, 6938737200650, 20240723-100416_6938737200650, 0.222, 0.701, 0.620 +same, 6918598120014, 20240723-100501_6918598120014, 0.102, 0.742, 0.206 +same, 4897098800133, 20240723-100845_4897098800133, 0.305, 0.834, 0.744 +same, 6971284204597, 20240723-101014_6971284204597, 0.223, 0.714, 0.623 +same, 6902265202318, 20240723-101144_6902265202318, 0.537, 0.875, 0.812 +same, 6902265131212, 20240723-101416_6902265131212, 0.463, 0.901, 0.747 +same, 6918598120014, 20240723-101829_6918598120014, 0.524, 0.901, 0.855 +same, 6902265131212, 20240723-101851_6902265131212, 0.418, 0.899, 0.820 +same, 6920382723257, 20240723-102033_6920382723257, 0.320, 0.780, 0.755 +same, 6902890022558, 20240723-102206_6902890022558, 0.123, 0.707, 0.387 +same, 6970470331529, 20240723-102409_6970470331529, 0.099, 0.641, 0.300 +same, 6938737200650, 20240723-102515_6938737200650, 0.177, 0.695, 0.515 +same, 6937451883132, 20240723-110216_6937451883132, 0.300, 0.817, 0.706 +same, 6973337010216, 20240723-110224_6973337010216, 0.202, 0.727, 0.633 +same, 6972359682937, 20240723-110937_6972359682937, 0.229, 0.792, 0.540 +same, 6937451883132, 20240723-111446_6937451883132, 0.363, 0.796, 0.844 +same, 6973337010216, 20240723-111521_6973337010216, 0.120, 0.764, 0.359 +same, 6924187828544, 20240723-111602_6924187828544, 0.378, 0.857, 0.733 +same, 6934024590268, 20240723-112004_6934024590268, 0.205, 0.724, 0.609 +same, 6972359682937, 20240723-112038_6972359682937, 0.432, 0.839, 0.857 +same, 6924187828544, 20240723-112505_6924187828544, 0.304, 0.835, 0.601 +same, 6974158894528, 20240723-155423_6974158894528, 0.440, 0.854, 0.765 +same, 6973497200045, 20240723-160015_6973497200045, 0.263, 0.712, 0.551 +same, 6970399922518, 20240723-160227_6970399922518, 0.087, 0.564, 0.243 +same, 6974158894528, 20240723-160942_6974158894528, 0.429, 0.847, 0.818 +diff, 6951957205274, 20240715-144627_6954256905319, -0.032, 0.375, -0.064 +diff, 6970399922518, 20240719-143553_6902265589617, -0.040, 0.366, -0.107 +diff, 6926419102347, 20240719-143603_6930364600025, 0.052, 0.455, 0.137 +diff, 6925303795535, 20240719-143630_6924509902839, -0.008, 0.379, -0.021 +diff, 6924509902839, 20240719-143729_6902007115524, 0.060, 0.421, 0.134 +diff, 6924187895423, 20240719-143807_6930628801472, 0.131, 0.416, 0.254 +diff, 6954256905319, 20240719-144126_6902265314615, 0.030, 0.456, 0.088 +diff, 6972786772775, 20240719-144226_6928312901324, -0.000, 0.422, -0.001 +diff, 6902265131212, 20240719-144335_6908355010450, -0.037, 0.308, -0.079 +diff, 6930364600025, 20240719-144454_6924509902754, 0.075, 0.470, 0.156 +diff, 6902265202318, 20240719-144621_6902265314615, 0.212, 0.554, 0.315 +diff, 6925303795535, 20240719-145022_6941499112529, -0.018, 0.474, -0.038 +diff, 6902265131212, 20240719-145149_6972786772775, -0.018, 0.327, -0.032 +diff, 6902265314615, 20240719-145319_6920267906706, 0.103, 0.514, 0.202 +diff, 6951469100067, 20240719-145505_6974925910116, -0.051, 0.274, -0.081 +diff, 6911988027191, 20240719-145542_6928002374551, 0.039, 0.392, 0.135 +diff, 4892655902237, 20240719-145816_6920152414071, -0.144, 0.262, -0.268 +diff, 6924187828544, 20240719-145926_6973360690409, -0.065, 0.287, -0.158 +diff, 6951957205274, 20240719-150000_6930364600001, 0.011, 0.418, 0.019 +diff, 6926419102347, 20240719-150209_6937962111526, -0.009, 0.462, -0.023 +diff, 6930628801472, 20240719-150345_6924509902839, 0.005, 0.429, 0.019 +diff, 6916196422325, 20240719-150458_6924509902839, 0.041, 0.412, 0.068 +diff, 6920912342002, 20240719-150623_6924187895423, -0.008, 0.351, -0.020 +diff, 6925303795535, 20240719-150716_6916749306102, -0.043, 0.329, -0.116 +diff, 4892655902015, 20240719-150750_6951469100067, -0.007, 0.332, -0.015 +diff, 6971284204597, 20240719-151009_6973360690409, -0.058, 0.397, -0.163 +diff, 4892655902015, 20240719-151135_6923120632521, -0.084, 0.425, -0.223 +diff, 6921502656660, 20240719-151215_6958350501101, 0.044, 0.576, 0.095 +diff, 6920152471517, 20240719-151325_6951469100067, 0.065, 0.363, 0.107 +diff, 6970399922518, 20240719-151349_6937962111526, -0.048, 0.405, -0.151 +diff, 6972786772775, 20240719-151416_6916749306102, 0.005, 0.436, 0.015 +diff, 6971075127784, 20240719-151535_6930364600001, -0.070, 0.343, -0.152 +diff, 6916196422325, 20240719-151724_6911988027191, 0.062, 0.373, 0.102 +diff, 6937451883132, 20240719-151800_6972219890632, 0.012, 0.383, 0.033 +diff, 6902265314615, 20240719-152125_6901668935908, -0.055, 0.388, -0.106 +diff, 6928312901324, 20240719-152315_6920152471517, -0.039, 0.481, -0.067 +diff, 6928002374551, 20240719-152816_6951957205274, 0.057, 0.489, 0.129 +diff, 6928002374551, 20240719-152832_6971284201244, 0.010, 0.309, 0.025 +diff, 6970470331529, 20240719-153117_6930364600001, -0.044, 0.358, -0.120 +diff, 6974925910116, 20240722-143041_6972900962273, 0.025, 0.482, 0.064 +diff, 6938737200650, 20240722-144630_6902265587712, 0.019, 0.393, 0.054 +diff, 6941499112529, 20240722-160857_4892655902237, 0.003, 0.373, 0.006 +diff, 6924187895423, 20240722-162225_4892655902015, 0.028, 0.393, 0.066 +diff, 6923120632521, 20240722-162328_6938888889896, -0.022, 0.366, -0.049 +diff, 6972900962273, 20240722-163428_6920912342002, 0.072, 0.450, 0.171 +diff, 6938888889896, 20240722-163600_6971075127784, 0.112, 0.471, 0.189 +diff, 6930364600001, 20240722-164552_6926265317384, -0.033, 0.370, -0.090 +diff, 6923120632521, 20240722-164859_6916196422325, 0.134, 0.528, 0.279 +diff, 6926419102347, 20240722-165548_6901668006011, 0.028, 0.346, 0.071 +diff, 6924187828544, 20240722-170325_6926419102347, 0.011, 0.453, 0.024 +diff, 6926265317384, 20240723-094447_4897098800133, 0.006, 0.355, 0.019 +diff, 6954256905319, 20240723-094525_6971284204597, -0.000, 0.328, -0.000 +diff, 6937451883132, 20240723-094617_6970470331529, -0.013, 0.404, -0.036 +diff, 6973337010216, 20240723-095103_6918598120014, 0.029, 0.402, 0.078 +diff, 6937451883132, 20240723-095259_6902265202318, -0.135, 0.347, -0.280 +diff, 6971284201244, 20240723-095527_6938737200650, -0.023, 0.648, -0.058 +diff, 6972786772775, 20240723-095819_6902265131212, 0.008, 0.502, 0.022 +diff, 6941499112529, 20240723-095858_6902890022558, 0.022, 0.341, 0.049 +diff, 6930628801472, 20240723-100031_6925303795535, 0.020, 0.384, 0.049 +diff, 6970399922518, 20240723-100113_6920382723257, -0.041, 0.394, -0.101 +diff, 6908355010450, 20240723-100148_6976399680069, 0.061, 0.444, 0.150 +diff, 6973497200045, 20240723-100318_6921502656660, 0.007, 0.316, 0.017 +diff, 6951957205274, 20240723-100416_6938737200650, -0.089, 0.437, -0.163 +diff, 6902265131212, 20240723-100501_6918598120014, -0.069, 0.316, -0.133 +diff, 6972359682937, 20240723-100845_4897098800133, -0.017, 0.325, -0.038 +diff, 6972900962273, 20240723-101014_6971284204597, -0.010, 0.373, -0.020 +diff, 6930628801472, 20240723-101144_6902265202318, -0.045, 0.394, -0.101 +diff, 6920152471517, 20240723-101416_6902265131212, -0.002, 0.335, -0.003 +diff, 6934024590268, 20240723-101829_6918598120014, 0.013, 0.373, 0.030 +diff, 4897098800133, 20240723-101851_6902265131212, -0.069, 0.278, -0.157 +diff, 4892655902015, 20240723-102033_6920382723257, 0.012, 0.284, 0.029 +diff, 6921502656660, 20240723-102206_6902890022558, -0.030, 0.487, -0.072 +diff, 6937962111526, 20240723-102409_6970470331529, 0.001, 0.340, 0.004 +diff, 6920267906706, 20240723-102515_6938737200650, 0.094, 0.495, 0.223 +diff, 6926265317384, 20240723-110216_6937451883132, -0.041, 0.358, -0.121 +diff, 6902265202318, 20240723-110224_6973337010216, 0.012, 0.318, 0.021 +diff, 6901668006011, 20240723-110937_6972359682937, -0.060, 0.379, -0.140 +diff, 6926419102347, 20240723-111446_6937451883132, 0.045, 0.421, 0.119 +diff, 6941499112529, 20240723-111521_6973337010216, 0.019, 0.381, 0.039 +diff, 6973337010216, 20240723-111602_6924187828544, 0.052, 0.476, 0.139 +diff, 6920382723257, 20240723-112004_6934024590268, 0.025, 0.366, 0.060 +diff, 6938888889896, 20240723-112038_6972359682937, 0.028, 0.384, 0.057 +diff, 6971284204597, 20240723-112505_6924187828544, -0.028, 0.387, -0.075 +diff, 6926265317384, 20240723-155423_6974158894528, 0.019, 0.334, 0.046 +diff, 6974158894528, 20240723-160015_6973497200045, -0.033, 0.330, -0.075 +diff, 6974925910116, 20240723-160227_6970399922518, 0.010, 0.344, 0.023 +diff, 6972219890632, 20240723-160942_6974158894528, -0.060, 0.369, -0.118 diff --git a/contrast/result/pickle/20240909_164035_uint8.txt b/contrast/result/pickle/20240909_164035_uint8.txt new file mode 100644 index 0000000..609aaa8 --- /dev/null +++ b/contrast/result/pickle/20240909_164035_uint8.txt @@ -0,0 +1,174 @@ +same, 6954256905319, 20240715-144627_6954256905319, 0.207, 0.740, 0.604 +same, 6902265589617, 20240719-143553_6902265589617, 0.296, 0.719, 0.561 +same, 6930364600025, 20240719-143603_6930364600025, 0.301, 0.772, 0.668 +same, 6924509902839, 20240719-143630_6924509902839, 0.177, 0.662, 0.438 +same, 6902007115524, 20240719-143729_6902007115524, 0.230, 0.632, 0.524 +same, 6930628801472, 20240719-143807_6930628801472, 0.279, 0.726, 0.712 +same, 6902265314615, 20240719-144126_6902265314615, 0.375, 0.895, 0.755 +same, 6928312901324, 20240719-144226_6928312901324, 0.336, 0.779, 0.696 +same, 6908355010450, 20240719-144335_6908355010450, 0.255, 0.743, 0.580 +same, 6924509902754, 20240719-144454_6924509902754, 0.457, 0.828, 0.830 +same, 6902265314615, 20240719-144621_6902265314615, 0.410, 0.849, 0.668 +same, 6941499112529, 20240719-145022_6941499112529, 0.173, 0.732, 0.308 +same, 6972786772775, 20240719-145149_6972786772775, 0.242, 0.693, 0.565 +same, 6920267906706, 20240719-145319_6920267906706, 0.305, 0.798, 0.764 +same, 6974925910116, 20240719-145505_6974925910116, 0.371, 0.844, 0.743 +same, 6928002374551, 20240719-145542_6928002374551, 0.171, 0.769, 0.500 +same, 6920152414071, 20240719-145816_6920152414071, 0.468, 0.812, 0.871 +same, 6973360690409, 20240719-145926_6973360690409, 0.160, 0.642, 0.592 +same, 6930364600001, 20240719-150000_6930364600001, 0.316, 0.790, 0.754 +same, 6937962111526, 20240719-150209_6937962111526, 0.220, 0.658, 0.572 +same, 6924509902839, 20240719-150345_6924509902839, 0.156, 0.752, 0.472 +same, 6924509902839, 20240719-150458_6924509902839, 0.308, 0.664, 0.585 +same, 6924187895423, 20240719-150623_6924187895423, 0.437, 0.881, 0.893 +same, 6916749306102, 20240719-150716_6916749306102, 0.249, 0.728, 0.684 +same, 6951469100067, 20240719-150750_6951469100067, 0.533, 0.935, 0.904 +same, 6973360690409, 20240719-151009_6973360690409, 0.144, 0.579, 0.466 +same, 6923120632521, 20240719-151135_6923120632521, 0.300, 0.863, 0.711 +same, 6958350501101, 20240719-151215_6958350501101, 0.548, 0.897, 0.888 +same, 6951469100067, 20240719-151325_6951469100067, 0.586, 0.925, 0.899 +same, 6937962111526, 20240719-151349_6937962111526, 0.152, 0.657, 0.530 +same, 6916749306102, 20240719-151416_6916749306102, 0.227, 0.682, 0.587 +same, 6930364600001, 20240719-151535_6930364600001, 0.150, 0.693, 0.524 +same, 6911988027191, 20240719-151724_6911988027191, 0.126, 0.529, 0.336 +same, 6972219890632, 20240719-151800_6972219890632, 0.287, 0.868, 0.700 +same, 6901668935908, 20240719-152125_6901668935908, 0.209, 0.802, 0.436 +same, 6920152471517, 20240719-152315_6920152471517, 0.389, 0.829, 0.637 +same, 6951957205274, 20240719-152816_6951957205274, 0.344, 0.808, 0.602 +same, 6971284201244, 20240719-152832_6971284201244, 0.228, 0.814, 0.556 +same, 6930364600001, 20240719-153117_6930364600001, 0.065, 0.744, 0.196 +same, 6972900962273, 20240722-143041_6972900962273, 0.212, 0.596, 0.525 +same, 6902265587712, 20240722-144630_6902265587712, 0.362, 0.812, 0.674 +same, 4892655902237, 20240722-160857_4892655902237, 0.431, 0.784, 0.842 +same, 4892655902015, 20240722-162225_4892655902015, 0.099, 0.659, 0.281 +same, 6938888889896, 20240722-162328_6938888889896, 0.176, 0.780, 0.408 +same, 6920912342002, 20240722-163428_6920912342002, 0.312, 0.830, 0.858 +same, 6971075127784, 20240722-163600_6971075127784, 0.732, 0.919, 0.913 +same, 6926265317384, 20240722-164552_6926265317384, 0.179, 0.709, 0.613 +same, 6916196422325, 20240722-164859_6916196422325, 0.366, 0.770, 0.675 +same, 6901668006011, 20240722-165548_6901668006011, 0.290, 0.711, 0.545 +same, 6926419102347, 20240722-170325_6926419102347, 0.255, 0.786, 0.768 +same, 4897098800133, 20240723-094447_4897098800133, 0.378, 0.845, 0.805 +same, 6971284204597, 20240723-094525_6971284204597, 0.205, 0.732, 0.597 +same, 6970470331529, 20240723-094617_6970470331529, 0.235, 0.922, 0.587 +same, 6918598120014, 20240723-095103_6918598120014, 0.425, 0.904, 0.820 +same, 6902265202318, 20240723-095259_6902265202318, 0.638, 0.904, 0.908 +same, 6938737200650, 20240723-095527_6938737200650, 0.218, 0.821, 0.649 +same, 6902265131212, 20240723-095819_6902265131212, 0.197, 0.897, 0.396 +same, 6902890022558, 20240723-095858_6902890022558, 0.160, 0.710, 0.591 +same, 6925303795535, 20240723-100031_6925303795535, 0.296, 0.834, 0.672 +same, 6920382723257, 20240723-100113_6920382723257, 0.280, 0.808, 0.598 +same, 6976399680069, 20240723-100148_6976399680069, 0.041, 0.553, 0.133 +same, 6921502656660, 20240723-100318_6921502656660, 0.058, 0.428, 0.171 +same, 6938737200650, 20240723-100416_6938737200650, 0.222, 0.705, 0.620 +same, 6918598120014, 20240723-100501_6918598120014, 0.103, 0.737, 0.209 +same, 4897098800133, 20240723-100845_4897098800133, 0.304, 0.828, 0.744 +same, 6971284204597, 20240723-101014_6971284204597, 0.224, 0.705, 0.624 +same, 6902265202318, 20240723-101144_6902265202318, 0.533, 0.872, 0.810 +same, 6902265131212, 20240723-101416_6902265131212, 0.461, 0.897, 0.747 +same, 6918598120014, 20240723-101829_6918598120014, 0.522, 0.898, 0.855 +same, 6902265131212, 20240723-101851_6902265131212, 0.417, 0.895, 0.819 +same, 6920382723257, 20240723-102033_6920382723257, 0.321, 0.778, 0.758 +same, 6902890022558, 20240723-102206_6902890022558, 0.123, 0.704, 0.388 +same, 6970470331529, 20240723-102409_6970470331529, 0.101, 0.640, 0.304 +same, 6938737200650, 20240723-102515_6938737200650, 0.176, 0.697, 0.515 +same, 6937451883132, 20240723-110216_6937451883132, 0.299, 0.815, 0.704 +same, 6973337010216, 20240723-110224_6973337010216, 0.201, 0.729, 0.634 +same, 6972359682937, 20240723-110937_6972359682937, 0.227, 0.786, 0.538 +same, 6937451883132, 20240723-111446_6937451883132, 0.363, 0.799, 0.845 +same, 6973337010216, 20240723-111521_6973337010216, 0.119, 0.764, 0.358 +same, 6924187828544, 20240723-111602_6924187828544, 0.377, 0.853, 0.734 +same, 6934024590268, 20240723-112004_6934024590268, 0.205, 0.720, 0.609 +same, 6972359682937, 20240723-112038_6972359682937, 0.430, 0.832, 0.856 +same, 6924187828544, 20240723-112505_6924187828544, 0.304, 0.834, 0.601 +same, 6974158894528, 20240723-155423_6974158894528, 0.440, 0.849, 0.766 +same, 6973497200045, 20240723-160015_6973497200045, 0.261, 0.710, 0.550 +same, 6970399922518, 20240723-160227_6970399922518, 0.087, 0.563, 0.242 +same, 6974158894528, 20240723-160942_6974158894528, 0.428, 0.843, 0.817 +diff, 6951957205274, 20240715-144627_6954256905319, -0.031, 0.374, -0.062 +diff, 6970399922518, 20240719-143553_6902265589617, -0.040, 0.362, -0.108 +diff, 6926419102347, 20240719-143603_6930364600025, 0.053, 0.457, 0.140 +diff, 6925303795535, 20240719-143630_6924509902839, -0.007, 0.376, -0.018 +diff, 6924509902839, 20240719-143729_6902007115524, 0.060, 0.423, 0.133 +diff, 6924187895423, 20240719-143807_6930628801472, 0.131, 0.415, 0.254 +diff, 6954256905319, 20240719-144126_6902265314615, 0.030, 0.455, 0.089 +diff, 6972786772775, 20240719-144226_6928312901324, 0.000, 0.414, 0.001 +diff, 6902265131212, 20240719-144335_6908355010450, -0.038, 0.309, -0.081 +diff, 6930364600025, 20240719-144454_6924509902754, 0.073, 0.461, 0.153 +diff, 6902265202318, 20240719-144621_6902265314615, 0.208, 0.551, 0.311 +diff, 6925303795535, 20240719-145022_6941499112529, -0.016, 0.472, -0.035 +diff, 6902265131212, 20240719-145149_6972786772775, -0.019, 0.329, -0.034 +diff, 6902265314615, 20240719-145319_6920267906706, 0.104, 0.521, 0.204 +diff, 6951469100067, 20240719-145505_6974925910116, -0.052, 0.274, -0.082 +diff, 6911988027191, 20240719-145542_6928002374551, 0.038, 0.386, 0.133 +diff, 4892655902237, 20240719-145816_6920152414071, -0.144, 0.260, -0.269 +diff, 6924187828544, 20240719-145926_6973360690409, -0.066, 0.285, -0.159 +diff, 6951957205274, 20240719-150000_6930364600001, 0.010, 0.413, 0.018 +diff, 6926419102347, 20240719-150209_6937962111526, -0.009, 0.461, -0.023 +diff, 6930628801472, 20240719-150345_6924509902839, 0.004, 0.425, 0.017 +diff, 6916196422325, 20240719-150458_6924509902839, 0.039, 0.410, 0.065 +diff, 6920912342002, 20240719-150623_6924187895423, -0.009, 0.352, -0.022 +diff, 6925303795535, 20240719-150716_6916749306102, -0.042, 0.324, -0.113 +diff, 4892655902015, 20240719-150750_6951469100067, -0.008, 0.329, -0.019 +diff, 6971284204597, 20240719-151009_6973360690409, -0.057, 0.394, -0.161 +diff, 4892655902015, 20240719-151135_6923120632521, -0.084, 0.423, -0.223 +diff, 6921502656660, 20240719-151215_6958350501101, 0.043, 0.574, 0.094 +diff, 6920152471517, 20240719-151325_6951469100067, 0.063, 0.363, 0.104 +diff, 6970399922518, 20240719-151349_6937962111526, -0.048, 0.404, -0.150 +diff, 6972786772775, 20240719-151416_6916749306102, 0.005, 0.436, 0.013 +diff, 6971075127784, 20240719-151535_6930364600001, -0.072, 0.340, -0.156 +diff, 6916196422325, 20240719-151724_6911988027191, 0.060, 0.370, 0.099 +diff, 6937451883132, 20240719-151800_6972219890632, 0.012, 0.386, 0.033 +diff, 6902265314615, 20240719-152125_6901668935908, -0.054, 0.387, -0.103 +diff, 6928312901324, 20240719-152315_6920152471517, -0.041, 0.481, -0.070 +diff, 6928002374551, 20240719-152816_6951957205274, 0.056, 0.482, 0.127 +diff, 6928002374551, 20240719-152832_6971284201244, 0.011, 0.306, 0.028 +diff, 6970470331529, 20240719-153117_6930364600001, -0.044, 0.355, -0.120 +diff, 6974925910116, 20240722-143041_6972900962273, 0.025, 0.483, 0.066 +diff, 6938737200650, 20240722-144630_6902265587712, 0.019, 0.395, 0.055 +diff, 6941499112529, 20240722-160857_4892655902237, 0.003, 0.371, 0.006 +diff, 6924187895423, 20240722-162225_4892655902015, 0.028, 0.398, 0.066 +diff, 6923120632521, 20240722-162328_6938888889896, -0.022, 0.373, -0.049 +diff, 6972900962273, 20240722-163428_6920912342002, 0.073, 0.457, 0.173 +diff, 6938888889896, 20240722-163600_6971075127784, 0.110, 0.466, 0.186 +diff, 6930364600001, 20240722-164552_6926265317384, -0.034, 0.363, -0.093 +diff, 6923120632521, 20240722-164859_6916196422325, 0.133, 0.530, 0.277 +diff, 6926419102347, 20240722-165548_6901668006011, 0.027, 0.341, 0.069 +diff, 6924187828544, 20240722-170325_6926419102347, 0.010, 0.455, 0.023 +diff, 6926265317384, 20240723-094447_4897098800133, 0.006, 0.355, 0.018 +diff, 6954256905319, 20240723-094525_6971284204597, 0.000, 0.327, 0.001 +diff, 6937451883132, 20240723-094617_6970470331529, -0.013, 0.401, -0.036 +diff, 6973337010216, 20240723-095103_6918598120014, 0.029, 0.403, 0.080 +diff, 6937451883132, 20240723-095259_6902265202318, -0.134, 0.349, -0.278 +diff, 6971284201244, 20240723-095527_6938737200650, -0.022, 0.645, -0.055 +diff, 6972786772775, 20240723-095819_6902265131212, 0.009, 0.503, 0.023 +diff, 6941499112529, 20240723-095858_6902890022558, 0.022, 0.347, 0.049 +diff, 6930628801472, 20240723-100031_6925303795535, 0.021, 0.383, 0.051 +diff, 6970399922518, 20240723-100113_6920382723257, -0.041, 0.389, -0.102 +diff, 6908355010450, 20240723-100148_6976399680069, 0.063, 0.447, 0.154 +diff, 6973497200045, 20240723-100318_6921502656660, 0.005, 0.320, 0.011 +diff, 6951957205274, 20240723-100416_6938737200650, -0.089, 0.445, -0.163 +diff, 6902265131212, 20240723-100501_6918598120014, -0.070, 0.312, -0.135 +diff, 6972359682937, 20240723-100845_4897098800133, -0.017, 0.330, -0.039 +diff, 6972900962273, 20240723-101014_6971284204597, -0.010, 0.376, -0.019 +diff, 6930628801472, 20240723-101144_6902265202318, -0.044, 0.392, -0.101 +diff, 6920152471517, 20240723-101416_6902265131212, -0.001, 0.333, -0.001 +diff, 6934024590268, 20240723-101829_6918598120014, 0.014, 0.378, 0.031 +diff, 4897098800133, 20240723-101851_6902265131212, -0.068, 0.279, -0.155 +diff, 4892655902015, 20240723-102033_6920382723257, 0.012, 0.284, 0.029 +diff, 6921502656660, 20240723-102206_6902890022558, -0.031, 0.486, -0.075 +diff, 6937962111526, 20240723-102409_6970470331529, 0.002, 0.339, 0.007 +diff, 6920267906706, 20240723-102515_6938737200650, 0.094, 0.489, 0.223 +diff, 6926265317384, 20240723-110216_6937451883132, -0.040, 0.347, -0.119 +diff, 6902265202318, 20240723-110224_6973337010216, 0.011, 0.320, 0.021 +diff, 6901668006011, 20240723-110937_6972359682937, -0.059, 0.379, -0.138 +diff, 6926419102347, 20240723-111446_6937451883132, 0.046, 0.417, 0.120 +diff, 6941499112529, 20240723-111521_6973337010216, 0.019, 0.378, 0.038 +diff, 6973337010216, 20240723-111602_6924187828544, 0.051, 0.475, 0.136 +diff, 6920382723257, 20240723-112004_6934024590268, 0.024, 0.367, 0.058 +diff, 6938888889896, 20240723-112038_6972359682937, 0.027, 0.379, 0.055 +diff, 6971284204597, 20240723-112505_6924187828544, -0.027, 0.390, -0.071 +diff, 6926265317384, 20240723-155423_6974158894528, 0.019, 0.327, 0.045 +diff, 6974158894528, 20240723-160015_6973497200045, -0.033, 0.333, -0.075 +diff, 6974925910116, 20240723-160227_6970399922518, 0.012, 0.348, 0.029 +diff, 6972219890632, 20240723-160942_6974158894528, -0.060, 0.365, -0.119 diff --git a/contrast/result/pickle/20240911_183903.txt b/contrast/result/pickle/20240911_183903.txt new file mode 100644 index 0000000..566c0d1 --- /dev/null +++ b/contrast/result/pickle/20240911_183903.txt @@ -0,0 +1,202 @@ +same, 6901668936684, 20240910-173355-1bbf290e-1f14-4ba8-b666-82c990c4eea3_6901668936684, 0.268, 0.659, 0.506 +same, 6902088131437, 20240910-173847-9eedb2ac-e3a5-4d07-94fe-f7e881d67418_6902088131437, 0.582, 0.979, 0.806 +same, 6904682300226, 20240910-171800-76a062fd-409c-480f-94f4-fd0e65d72467_6904682300226, 0.173, 0.830, 0.372 +same, 6970399922365, 20240910-172352-9b79a4d9-092f-477d-a7a4-8af079d1538d_6970399922365, 0.226, 0.774, 0.597 +same, 6902265202318, 20240910-170331-e3ee7cf5-dda2-4d0b-b8c9-4fb411fe78ec_6902265202318, 0.557, 0.922, 0.803 +same, 6907992517780, 20240910-163802-6b9f0129-8497-467f-a506-5708eda436a4_6907992517780, 0.354, 0.761, 0.848 +same, 6902132084337, 20240910-172403-dbc9de02-2811-449c-961f-23e7a16877d7_6902132084337, 0.406, 0.774, 0.850 +same, 6901668934888, 20240910-164315-38c640ba-cdf3-4ac1-8bff-55fe5d0560bb_6901668934888, 0.290, 0.598, 0.621 +same, 8000500023976, 20240910-173323-78dc658e-e4ef-49e1-a2ff-9ada34c27a85_8000500023976, 0.495, 0.825, 0.792 +same, 6904682300219, 20240910-164323-8e9a882a-a502-4a6e-bd99-70deb2130f57_6904682300219, 0.278, 0.782, 0.551 +same, 6903148231623, 20240910-163750-8e13e800-21d0-4bd9-b686-18ed213460cd_6903148231623, 0.320, 0.870, 0.718 +same, 6904682300219, 20240910-170920-dc16c149-06a3-4c2d-9bec-e930274b55ce_6904682300219, 0.217, 0.697, 0.418 +same, 6902890218470, 20240910-172802-0dbe3709-bd0c-45e7-ad36-0cfc9781ef1b_6902890218470, 0.198, 0.690, 0.538 +same, 6901668934888, 20240910-165620-0b870f0d-88a5-4286-bcbf-b0ebb41ddcfc_6901668934888, 0.325, 0.710, 0.689 +same, 6902088131437, 20240910-163846-7793e886-9f09-4744-9e24-eb47d65c09f5_6902088131437, 0.450, 0.983, 0.784 +same, 6901070600142, 20240910-170742-f78b59da-e242-42c9-ac7a-bba23ff11aff_6901070600142, 0.295, 0.728, 0.668 +same, 8993175540667, 20240910-172814-d17bd016-b8e5-4a21-a137-6bce693e0cb0_8993175540667, 0.418, 0.859, 0.687 +same, 6901668929730, 20240910-162930-ec2bb380-53fe-483f-9aab-9038643ebd1f_6901668929730, 0.549, 0.853, 0.888 +same, 6970399922365, 20240910-173332-55f8124d-7ab0-4a7a-8b08-f4dd9ba06502_6970399922365, 0.330, 0.766, 0.817 +same, 6901668929730, 20240910-173214-5b86868f-cb5b-4b7f-8f3a-aff08d89900d_6901668929730, 0.529, 0.849, 0.864 +same, 6903148048801, 20240910-172904-5462ad91-2a07-4116-898f-ff1d2021e6af_6903148048801, 0.444, 0.865, 0.769 +same, 6901668934628, 20240910-171838-c77a6d0d-185b-48e7-9af9-05de561f1172_6901668934628, 0.489, 0.930, 0.758 +same, 6902890218470, 20240910-170934-74c137ee-0689-42d0-9994-da8ba59fd5db_6902890218470, 0.251, 0.738, 0.652 +same, 6949909050041, 20240910-162952-f6ec3a40-9d64-4f20-b122-0b81eb4a2134_6949909050041, 0.384, 0.870, 0.714 +same, 6901668934888, 20240910-172841-9d7b16fb-4200-4089-b4b2-925da10208ed_6901668934888, 0.336, 0.778, 0.751 +same, 6901668936271, 20240910-165632-a1e22655-d9ad-47f5-a467-55718bd1e23e_6901668936271, 0.121, 0.604, 0.257 +same, 6904682300226, 20240910-163718-e1e09ad9-7a7e-4b43-beb7-47080c0a312e_6904682300226, 0.297, 0.847, 0.651 +same, 6903148126677, 20240910-173233-81246d1d-bbf3-4ee2-b6c1-7f8fe5818266_6903148126677, 0.422, 0.814, 0.717 +same, 6924743915848, 20240910-162836-186bdf15-5ebb-4b55-a3a4-47edea86a7ee_6924743915848, 0.285, 0.697, 0.640 +same, 6902132084337, 20240910-173222-8abca736-4b5d-4b8e-8e53-206809f37082_6902132084337, 0.350, 0.819, 0.857 +same, 8993175537322, 20240910-170945-c5a587f8-925a-46c2-b2f4-b8fe0fa41c90_8993175537322, 0.349, 0.832, 0.611 +same, 6902265202318, 20240910-162848-b0d67358-6f68-482a-94cb-d7de7414e32f_6902265202318, 0.392, 0.860, 0.695 +same, 6907992517780, 20240910-173730-c51d9d00-65a2-4212-99f3-701092810919_6907992517780, 0.405, 0.815, 0.865 +same, 6902265160502, 20240910-170318-706146af-c203-459a-b642-da428ce6426a_6902265160502, 0.162, 0.703, 0.531 +same, 6903148347409, 20240910-162902-3de7f2a9-9068-4f61-a150-0bcc47194a43_6903148347409, 0.156, 0.693, 0.470 +same, 6902265202318, 20240910-172023-a9b8c8b4-8030-4aa5-85fe-54cba57e745f_6902265202318, 0.378, 0.865, 0.694 +same, 6903148126677, 20240910-171920-0a6490ce-547f-493d-b76a-4c849ae12a93_6903148126677, 0.496, 0.879, 0.796 +same, 6901668936295, 20240910-164334-09d4e20e-68c8-48ca-b931-50e58428ef2a_6901668936295, 0.170, 0.631, 0.325 +same, 6958104102516, 20240910-165604-0f805f9d-24f7-4729-923a-bff489a09323_6958104102516, 0.235, 0.731, 0.550 +same, 6901668936684, 20240910-164409-053f810b-7369-4a3e-b91b-b7ba99fa5b9c_6901668936684, 0.230, 0.638, 0.450 +same, 6902265150022, 20240910-170349-b357333c-e939-4ce5-8019-7762799a9097_6902265150022, 0.362, 0.927, 0.794 +same, 6902890232216, 20240910-172828-0a20bffd-ede3-4b0c-977b-8652f52518f9_6902890232216, 0.255, 0.761, 0.626 +same, 6902890232216, 20240910-170807-7bc77832-4cf1-4cd8-aa54-994ff164dcc7_6902890232216, 0.296, 0.695, 0.585 +same, 6901668929730, 20240910-171715-a8fc6d8a-87bd-4fbd-b378-85e34193266f_6901668929730, 0.503, 0.848, 0.823 +same, 6903148231623, 20240910-170258-38579506-3874-4d71-b9d2-ac6e47ca75dd_6903148231623, 0.256, 0.720, 0.506 +same, 6902265150022, 20240910-172010-035f68e4-9b7c-40f7-961c-aa8c0f154252_6902265150022, 0.428, 0.940, 0.823 +same, 6901668929518, 20240910-173344-258d27a2-b2e1-468e-8f32-40edcda94486_6901668929518, 0.361, 0.853, 0.721 +same, 6901668934628, 20240910-170431-722e7de7-c7ef-4825-8080-be019c7f4602_6901668934628, 0.444, 0.882, 0.690 +same, 6974158892364, 20240910-163028-418ab174-5722-4e8a-ae12-e8d3c33f70b5_6974158892364, 0.119, 0.684, 0.439 +same, 6902890218470, 20240910-164251-a2a38e17-5532-49a5-9372-5a3ed8dc6972_6902890218470, 0.281, 0.689, 0.666 +same, 6902265150022, 20240910-163814-9fc0324d-134a-46ee-bb79-6b2dfb6388f9_6902265150022, 0.308, 0.899, 0.682 +same, 6901668929518, 20240910-170417-1ac149e8-4ecb-447c-a8b7-8d5b96e77ffa_6901668929518, 0.260, 0.821, 0.586 +same, 6901668936271, 20240910-172745-96dc9808-4157-4806-856f-c7013452f302_6901668936271, 0.156, 0.617, 0.315 +same, 6903148126677, 20240910-163857-736e50b8-eae8-4a6d-af26-ce3a57a073b8_6903148126677, 0.420, 0.891, 0.749 +same, 6901668936684, 20240910-172445-4f28474f-5463-4b19-bc2d-671105764e27_6901668936684, 0.212, 0.675, 0.445 +same, 6901668936295, 20240910-172754-d034ab2f-1b18-4d6a-a936-9fa538066253_6901668936295, 0.130, 0.630, 0.254 +same, 8993175540667, 20240910-165644-2e79a878-caf1-44ca-851c-287848800d35_8993175540667, 0.565, 0.872, 0.821 +same, 6901668929518, 20240910-172039-ebd2a496-c407-4450-b122-0e8f33e07de2_6901668929518, 0.341, 0.826, 0.726 +same, 6902132084337, 20240910-162817-18813894-397a-4c94-8b90-2d7a46319793_6902132084337, 0.438, 0.794, 0.887 +same, 6904682300219, 20240910-172257-9169e95d-ff11-4d31-98af-13df3f071840_6904682300219, 0.365, 0.804, 0.643 +same, 6901668934628, 20240910-173306-a1409202-ea3d-47c4-aa39-9d17dae711cf_6901668934628, 0.489, 0.894, 0.770 +same, 6902088131437, 20240910-172427-781eb94d-efb6-403c-b88f-f4b9df82fee0_6902088131437, 0.536, 0.980, 0.829 +same, 9421903892324, 20240910-173757-b4ed1c60-a96b-48ad-a451-3caecd61c327_9421903892324, 0.421, 0.892, 0.755 +same, 6901668936684, 20240910-170907-0e74383f-0341-4b90-b333-910e5a184296_6901668936684, 0.289, 0.672, 0.569 +same, 8000500023976, 20240910-171014-ee1e7d74-0d89-4014-a125-7c9cdebb15fd_8000500023976, 0.286, 0.872, 0.660 +same, 6901668929518, 20240910-164347-47377bae-2ca6-4d75-a076-e7f6c03d0f2e_6901668929518, 0.446, 0.847, 0.833 +same, 6902265160502, 20240910-173409-55dd7611-7394-4783-9f4e-4639401078ea_6902265160502, 0.212, 0.857, 0.611 +same, 6901668936684, 20240910-165525-e17864c9-e965-4531-be14-be551dad88fb_6901668936684, 0.149, 0.614, 0.344 +same, 6901668934628, 20240910-162805-592cff06-4acb-420f-bc36-bb00f3e0efbb_6901668934628, 0.275, 0.870, 0.521 +same, 6949909050041, 20240910-172919-ab2efd9a-a776-420f-95f5-2f8188f719e4_6949909050041, 0.401, 0.849, 0.792 +same, 6907992517780, 20240910-171723-2f8a7ece-99cb-4d91-b484-67b486599f26_6907992517780, 0.391, 0.848, 0.838 +same, 6902890218470, 20240910-165443-48bad32d-9f2b-499b-907d-c602cf563ee3_6902890218470, 0.281, 0.737, 0.774 +same, 6904682300219, 20240910-165455-d0e36365-f7f2-4f2e-84a7-1ffc24ccc1c7_6904682300219, 0.424, 0.892, 0.792 +same, 6904682300226, 20240910-170231-21568a27-641b-448d-8b8c-9eff4dfe7294_6904682300226, 0.257, 0.725, 0.636 +same, 6903148048801, 20240910-163740-851d23c1-e90f-4947-abc3-f463991c5505_6903148048801, 0.422, 0.826, 0.784 +same, 6902132084337, 20240910-170730-76626a74-34fb-486d-b889-4276552edb0e_6902132084337, 0.379, 0.831, 0.792 +same, 9421903892324, 20240910-172316-ffa74ee4-46d5-4266-b362-ebfebed0c572_9421903892324, 0.304, 0.877, 0.548 +same, 6904682300219, 20240910-173807-afdeec3a-0d6e-4db8-9baf-826b7d6b4660_6904682300219, 0.254, 0.770, 0.477 +same, 6902890232216, 20240910-163838-9e6f0b38-2ffe-4727-9ec7-a02435b8f629_6902890232216, 0.264, 0.786, 0.593 +same, 6901668936295, 20240910-165424-5d55263c-e523-495e-b673-fc53eaa68b05_6901668936295, 0.139, 0.542, 0.239 +same, 6903148126677, 20240910-170403-c1b9db80-7ee0-4508-8858-1e3e1b924648_6903148126677, 0.351, 0.861, 0.602 +same, 6901668929518, 20240910-172500-509a2d1e-e665-4fe6-8ffe-b69117d7b09f_6901668929518, 0.365, 0.821, 0.731 +same, 6903148231623, 20240910-171824-2d3edfcd-c169-4c6e-9734-9325b72cf9fe_6903148231623, 0.176, 0.688, 0.359 +same, 6901668929518, 20240910-173839-e4b3b834-c695-4917-b2f4-7cfaaebb98dc_6901668929518, 0.437, 0.874, 0.772 +same, 6901668929730, 20240910-170447-3b37f76f-5e21-400b-a8a8-2376c0796ae6_6901668929730, 0.461, 0.852, 0.797 +same, 6903148080085, 20240910-173314-d6ac3740-20f2-4aa7-a392-80a96b7607c3_6903148080085, 0.370, 0.860, 0.827 +same, 6901070600142, 20240910-172734-8c23b385-99f7-4e01-819a-78c86611ff48_6901070600142, 0.201, 0.672, 0.442 +same, 6958104102516, 20240910-164452-0f365052-2e4a-4d00-9cf7-0407d731d07e_6958104102516, 0.234, 0.866, 0.583 +same, 6901070600142, 20240910-162749-ab186eb8-6777-489b-8ad0-c1c6e66b285d_6901070600142, 0.269, 0.727, 0.591 +same, 8993175537322, 20240910-164432-008357d7-7ee6-49b9-8d08-3f3a6081c4e1_8993175537322, 0.450, 0.790, 0.785 +same, 6975682480393, 20240910-163007-6dfc085b-42b9-432d-9c41-7bfd294526b6_6975682480393, 0.448, 0.835, 0.828 +same, 6903148080085, 20240910-163825-e4de18e2-fe7c-4ff6-8b51-7ef2a7db7ed3_6903148080085, 0.351, 0.838, 0.766 +same, 6903148231623, 20240910-172854-5fb70036-3089-4258-9346-de25d415f120_6903148231623, 0.423, 0.845, 0.782 +same, 6949909050041, 20240910-170817-c2f8c500-3aa5-4bd2-bf82-787d0cd22585_6949909050041, 0.494, 0.893, 0.885 +same, 6907992517780, 20240910-170246-e773b037-a712-4d78-accd-71c24b675365_6907992517780, 0.338, 0.737, 0.823 +same, 6902265160502, 20240910-163907-1ac881ec-cac4-4811-9cab-1826731e77bd_6902265160502, 0.239, 0.833, 0.706 +same, 6901668936271, 20240910-164239-e4d8f615-8cf3-483d-bc6e-03e470e2110c_6901668936271, 0.230, 0.615, 0.390 +same, 8993175537322, 20240910-172328-48a512b9-4fb1-4abf-bca9-8b3443ce8f2b_8993175537322, 0.456, 0.783, 0.719 +same, 8993175537322, 20240910-173819-226cc352-acdc-4419-9159-c97ae0eb58af_8993175537322, 0.455, 0.766, 0.717 +same, 6901668929518, 20240910-165517-a0000cdf-aa15-42c8-a6be-dbce8cf7cb32_6901668929518, 0.406, 0.861, 0.759 +same, 8000500023976, 20240910-172417-e9d563b9-74e2-4ec1-8f34-331424b48e72_8000500023976, 0.350, 0.853, 0.686 +diff, 8993175537322, 20240910-173355-1bbf290e-1f14-4ba8-b666-82c990c4eea3_6901668936684, 0.017, 0.341, 0.030 +diff, 6904682300226, 20240910-173847-9eedb2ac-e3a5-4d07-94fe-f7e881d67418_6902088131437, 0.007, 0.348, 0.013 +diff, 8993175540667, 20240910-171800-76a062fd-409c-480f-94f4-fd0e65d72467_6904682300226, 0.038, 0.309, 0.067 +diff, 6901668934628, 20240910-172352-9b79a4d9-092f-477d-a7a4-8af079d1538d_6970399922365, -0.003, 0.302, -0.006 +diff, 6901668929518, 20240910-170331-e3ee7cf5-dda2-4d0b-b8c9-4fb411fe78ec_6902265202318, -0.023, 0.273, -0.038 +diff, 6903148080085, 20240910-163802-6b9f0129-8497-467f-a506-5708eda436a4_6907992517780, 0.026, 0.408, 0.061 +diff, 6970399922365, 20240910-172403-dbc9de02-2811-449c-961f-23e7a16877d7_6902132084337, 0.090, 0.479, 0.207 +diff, 6904682300226, 20240910-164315-38c640ba-cdf3-4ac1-8bff-55fe5d0560bb_6901668934888, 0.072, 0.383, 0.142 +diff, 6974158892364, 20240910-173323-78dc658e-e4ef-49e1-a2ff-9ada34c27a85_8000500023976, -0.044, 0.340, -0.117 +diff, 6901668934888, 20240910-164323-8e9a882a-a502-4a6e-bd99-70deb2130f57_6904682300219, -0.017, 0.459, -0.042 +diff, 6907992517780, 20240910-163750-8e13e800-21d0-4bd9-b686-18ed213460cd_6903148231623, 0.019, 0.391, 0.051 +diff, 6901668934628, 20240910-170920-dc16c149-06a3-4c2d-9bec-e930274b55ce_6904682300219, 0.033, 0.331, 0.063 +diff, 6901668936684, 20240910-172802-0dbe3709-bd0c-45e7-ad36-0cfc9781ef1b_6902890218470, -0.072, 0.270, -0.163 +diff, 6907992517780, 20240910-165620-0b870f0d-88a5-4286-bcbf-b0ebb41ddcfc_6901668934888, 0.141, 0.460, 0.292 +diff, 6958104102516, 20240910-163846-7793e886-9f09-4744-9e24-eb47d65c09f5_6902088131437, -0.022, 0.373, -0.053 +diff, 8993175537322, 20240910-170742-f78b59da-e242-42c9-ac7a-bba23ff11aff_6901070600142, -0.018, 0.293, -0.033 +diff, 6903148126677, 20240910-172814-d17bd016-b8e5-4a21-a137-6bce693e0cb0_8993175540667, -0.044, 0.356, -0.082 +diff, 8993175540667, 20240910-162930-ec2bb380-53fe-483f-9aab-9038643ebd1f_6901668929730, -0.021, 0.349, -0.032 +diff, 9421903892324, 20240910-173332-55f8124d-7ab0-4a7a-8b08-f4dd9ba06502_6970399922365, 0.033, 0.383, 0.062 +diff, 6902890232216, 20240910-173214-5b86868f-cb5b-4b7f-8f3a-aff08d89900d_6901668929730, 0.076, 0.420, 0.151 +diff, 6903148231623, 20240910-172904-5462ad91-2a07-4116-898f-ff1d2021e6af_6903148048801, 0.012, 0.309, 0.019 +diff, 6924743915848, 20240910-171838-c77a6d0d-185b-48e7-9af9-05de561f1172_6901668934628, -0.069, 0.326, -0.147 +diff, 6975682480393, 20240910-170934-74c137ee-0689-42d0-9994-da8ba59fd5db_6902890218470, 0.122, 0.628, 0.274 +diff, 6975682480393, 20240910-162952-f6ec3a40-9d64-4f20-b122-0b81eb4a2134_6949909050041, 0.094, 0.647, 0.188 +diff, 6907992517780, 20240910-172841-9d7b16fb-4200-4089-b4b2-925da10208ed_6901668934888, 0.157, 0.646, 0.343 +diff, 6902265202318, 20240910-165632-a1e22655-d9ad-47f5-a467-55718bd1e23e_6901668936271, -0.006, 0.286, -0.011 +diff, 6902890232216, 20240910-163718-e1e09ad9-7a7e-4b43-beb7-47080c0a312e_6904682300226, 0.066, 0.491, 0.157 +diff, 9421903892324, 20240910-173233-81246d1d-bbf3-4ee2-b6c1-7f8fe5818266_6903148126677, -0.038, 0.450, -0.061 +diff, 6902132084337, 20240910-162836-186bdf15-5ebb-4b55-a3a4-47edea86a7ee_6924743915848, -0.061, 0.267, -0.125 +diff, 9421903892324, 20240910-173222-8abca736-4b5d-4b8e-8e53-206809f37082_6902132084337, 0.105, 0.454, 0.213 +diff, 6901668934628, 20240910-170945-c5a587f8-925a-46c2-b2f4-b8fe0fa41c90_8993175537322, -0.089, 0.186, -0.148 +diff, 6901668934888, 20240910-162848-b0d67358-6f68-482a-94cb-d7de7414e32f_6902265202318, -0.038, 0.352, -0.087 +diff, 6902265202318, 20240910-173730-c51d9d00-65a2-4212-99f3-701092810919_6907992517780, 0.025, 0.325, 0.043 +diff, 6902890232216, 20240910-170318-706146af-c203-459a-b642-da428ce6426a_6902265160502, 0.077, 0.540, 0.241 +diff, 6903148126677, 20240910-162902-3de7f2a9-9068-4f61-a150-0bcc47194a43_6903148347409, -0.047, 0.247, -0.113 +diff, 6903148347409, 20240910-172023-a9b8c8b4-8030-4aa5-85fe-54cba57e745f_6902265202318, 0.019, 0.312, 0.049 +diff, 6904682300219, 20240910-171920-0a6490ce-547f-493d-b76a-4c849ae12a93_6903148126677, 0.022, 0.340, 0.033 +diff, 6974158892364, 20240910-164334-09d4e20e-68c8-48ca-b931-50e58428ef2a_6901668936295, 0.035, 0.446, 0.108 +diff, 6901070600142, 20240910-165604-0f805f9d-24f7-4729-923a-bff489a09323_6958104102516, 0.016, 0.385, 0.042 +diff, 6901668934628, 20240910-164409-053f810b-7369-4a3e-b91b-b7ba99fa5b9c_6901668936684, -0.045, 0.563, -0.079 +diff, 6924743915848, 20240910-170349-b357333c-e939-4ce5-8019-7762799a9097_6902265150022, -0.096, 0.342, -0.249 +diff, 6903148126677, 20240910-172828-0a20bffd-ede3-4b0c-977b-8652f52518f9_6902890232216, 0.053, 0.326, 0.112 +diff, 6904682300226, 20240910-170807-7bc77832-4cf1-4cd8-aa54-994ff164dcc7_6902890232216, 0.063, 0.430, 0.115 +diff, 9421903892324, 20240910-171715-a8fc6d8a-87bd-4fbd-b378-85e34193266f_6901668929730, -0.066, 0.306, -0.107 +diff, 6901668936684, 20240910-170258-38579506-3874-4d71-b9d2-ac6e47ca75dd_6903148231623, 0.062, 0.403, 0.131 +diff, 6970399922365, 20240910-172010-035f68e4-9b7c-40f7-961c-aa8c0f154252_6902265150022, -0.044, 0.355, -0.101 +diff, 6903148048801, 20240910-173344-258d27a2-b2e1-468e-8f32-40edcda94486_6901668929518, 0.077, 0.498, 0.147 +diff, 6901668934888, 20240910-170431-722e7de7-c7ef-4825-8080-be019c7f4602_6901668934628, 0.001, 0.441, 0.001 +diff, 6970399922365, 20240910-163028-418ab174-5722-4e8a-ae12-e8d3c33f70b5_6974158892364, 0.072, 0.537, 0.208 +diff, 6975682480393, 20240910-164251-a2a38e17-5532-49a5-9372-5a3ed8dc6972_6902890218470, 0.112, 0.660, 0.231 +diff, 6901668929518, 20240910-163814-9fc0324d-134a-46ee-bb79-6b2dfb6388f9_6902265150022, -0.067, 0.359, -0.146 +diff, 6901070600142, 20240910-170417-1ac149e8-4ecb-447c-a8b7-8d5b96e77ffa_6901668929518, -0.033, 0.306, -0.085 +diff, 6903148126677, 20240910-172745-96dc9808-4157-4806-856f-c7013452f302_6901668936271, 0.008, 0.361, 0.018 +diff, 6903148347409, 20240910-163857-736e50b8-eae8-4a6d-af26-ce3a57a073b8_6903148126677, -0.008, 0.348, -0.017 +diff, 6901668936271, 20240910-172445-4f28474f-5463-4b19-bc2d-671105764e27_6901668936684, 0.064, 0.555, 0.128 +diff, 6901070600142, 20240910-172754-d034ab2f-1b18-4d6a-a936-9fa538066253_6901668936295, 0.189, 0.600, 0.448 +diff, 6902265150022, 20240910-165644-2e79a878-caf1-44ca-851c-287848800d35_8993175540667, 0.036, 0.300, 0.064 +diff, 6901668934888, 20240910-172039-ebd2a496-c407-4450-b122-0e8f33e07de2_6901668929518, 0.047, 0.373, 0.112 +diff, 6958104102516, 20240910-162817-18813894-397a-4c94-8b90-2d7a46319793_6902132084337, -0.068, 0.247, -0.130 +diff, 6902265160502, 20240910-172257-9169e95d-ff11-4d31-98af-13df3f071840_6904682300219, 0.046, 0.467, 0.106 +diff, 6970399922365, 20240910-173306-a1409202-ea3d-47c4-aa39-9d17dae711cf_6901668934628, 0.023, 0.376, 0.049 +diff, 6902265202318, 20240910-172427-781eb94d-efb6-403c-b88f-f4b9df82fee0_6902088131437, 0.017, 0.314, 0.030 +diff, 6907992517780, 20240910-173757-b4ed1c60-a96b-48ad-a451-3caecd61c327_9421903892324, 0.118, 0.551, 0.254 +diff, 6901668936271, 20240910-170907-0e74383f-0341-4b90-b333-910e5a184296_6901668936684, 0.137, 0.498, 0.255 +diff, 6901668934628, 20240910-171014-ee1e7d74-0d89-4014-a125-7c9cdebb15fd_8000500023976, 0.061, 0.324, 0.135 +diff, 6903148126677, 20240910-164347-47377bae-2ca6-4d75-a076-e7f6c03d0f2e_6901668929518, -0.026, 0.332, -0.047 +diff, 6903148048801, 20240910-173409-55dd7611-7394-4783-9f4e-4639401078ea_6902265160502, 0.030, 0.370, 0.070 +diff, 6902132084337, 20240910-165525-e17864c9-e965-4531-be14-be551dad88fb_6901668936684, 0.043, 0.375, 0.112 +diff, 6902890232216, 20240910-162805-592cff06-4acb-420f-bc36-bb00f3e0efbb_6901668934628, -0.067, 0.258, -0.164 +diff, 6903148048801, 20240910-172919-ab2efd9a-a776-420f-95f5-2f8188f719e4_6949909050041, 0.118, 0.397, 0.235 +diff, 6970399922365, 20240910-171723-2f8a7ece-99cb-4d91-b484-67b486599f26_6907992517780, -0.043, 0.339, -0.101 +diff, 6903148048801, 20240910-165443-48bad32d-9f2b-499b-907d-c602cf563ee3_6902890218470, -0.001, 0.482, -0.002 +diff, 6904682300226, 20240910-165455-d0e36365-f7f2-4f2e-84a7-1ffc24ccc1c7_6904682300219, 0.270, 0.813, 0.583 +diff, 6901668936271, 20240910-170231-21568a27-641b-448d-8b8c-9eff4dfe7294_6904682300226, 0.026, 0.369, 0.057 +diff, 6949909050041, 20240910-163740-851d23c1-e90f-4947-abc3-f463991c5505_6903148048801, 0.104, 0.443, 0.192 +diff, 6902890232216, 20240910-170730-76626a74-34fb-486d-b889-4276552edb0e_6902132084337, -0.018, 0.254, -0.040 +diff, 6924743915848, 20240910-172316-ffa74ee4-46d5-4266-b362-ebfebed0c572_9421903892324, 0.076, 0.444, 0.182 +diff, 6901070600142, 20240910-173807-afdeec3a-0d6e-4db8-9baf-826b7d6b4660_6904682300219, 0.010, 0.482, 0.024 +diff, 6924743915848, 20240910-163838-9e6f0b38-2ffe-4727-9ec7-a02435b8f629_6902890232216, -0.025, 0.380, -0.061 +diff, 6902265160502, 20240910-165424-5d55263c-e523-495e-b673-fc53eaa68b05_6901668936295, -0.042, 0.280, -0.088 +diff, 6902088131437, 20240910-170403-c1b9db80-7ee0-4508-8858-1e3e1b924648_6903148126677, -0.019, 0.228, -0.026 +diff, 6903148080085, 20240910-172500-509a2d1e-e665-4fe6-8ffe-b69117d7b09f_6901668929518, 0.064, 0.486, 0.135 +diff, 6901668934888, 20240910-171824-2d3edfcd-c169-4c6e-9734-9325b72cf9fe_6903148231623, 0.014, 0.325, 0.036 +diff, 6901668929730, 20240910-173839-e4b3b834-c695-4917-b2f4-7cfaaebb98dc_6901668929518, -0.066, 0.282, -0.106 +diff, 6901070600142, 20240910-170447-3b37f76f-5e21-400b-a8a8-2376c0796ae6_6901668929730, -0.068, 0.414, -0.148 +diff, 6974158892364, 20240910-173314-d6ac3740-20f2-4aa7-a392-80a96b7607c3_6903148080085, -0.033, 0.303, -0.107 +diff, 6901668936295, 20240910-172734-8c23b385-99f7-4e01-819a-78c86611ff48_6901070600142, 0.008, 0.417, 0.015 +diff, 6975682480393, 20240910-164452-0f365052-2e4a-4d00-9cf7-0407d731d07e_6958104102516, 0.031, 0.405, 0.075 +diff, 6903148080085, 20240910-162749-ab186eb8-6777-489b-8ad0-c1c6e66b285d_6901070600142, -0.015, 0.311, -0.030 +diff, 6901668929730, 20240910-164432-008357d7-7ee6-49b9-8d08-3f3a6081c4e1_8993175537322, 0.020, 0.303, 0.035 +diff, 6902890218470, 20240910-163007-6dfc085b-42b9-432d-9c41-7bfd294526b6_6975682480393, 0.184, 0.633, 0.393 +diff, 6902890232216, 20240910-163825-e4de18e2-fe7c-4ff6-8b51-7ef2a7db7ed3_6903148080085, 0.023, 0.348, 0.053 +diff, 6902890232216, 20240910-172854-5fb70036-3089-4258-9346-de25d415f120_6903148231623, -0.080, 0.324, -0.182 +diff, 6901668936271, 20240910-170817-c2f8c500-3aa5-4bd2-bf82-787d0cd22585_6949909050041, -0.011, 0.324, -0.019 +diff, 6902265160502, 20240910-170246-e773b037-a712-4d78-accd-71c24b675365_6907992517780, -0.094, 0.358, -0.244 +diff, 6902132084337, 20240910-163907-1ac881ec-cac4-4811-9cab-1826731e77bd_6902265160502, -0.007, 0.319, -0.020 +diff, 6970399922365, 20240910-164239-e4d8f615-8cf3-483d-bc6e-03e470e2110c_6901668936271, 0.048, 0.361, 0.105 +diff, 6904682300219, 20240910-172328-48a512b9-4fb1-4abf-bca9-8b3443ce8f2b_8993175537322, -0.014, 0.472, -0.021 +diff, 6901668936271, 20240910-173819-226cc352-acdc-4419-9159-c97ae0eb58af_8993175537322, 0.009, 0.332, 0.014 +diff, 6901668936271, 20240910-165517-a0000cdf-aa15-42c8-a6be-dbce8cf7cb32_6901668929518, 0.089, 0.483, 0.153 +diff, 6901668929730, 20240910-172417-e9d563b9-74e2-4ec1-8f34-331424b48e72_8000500023976, 0.110, 0.465, 0.216 diff --git a/contrast/result/pickle/20240911_183903_ft16.txt b/contrast/result/pickle/20240911_183903_ft16.txt new file mode 100644 index 0000000..fe2872b --- /dev/null +++ b/contrast/result/pickle/20240911_183903_ft16.txt @@ -0,0 +1,202 @@ +same, 6901668936684, 20240910-173355-1bbf290e-1f14-4ba8-b666-82c990c4eea3_6901668936684, 0.268, 0.659, 0.506 +same, 6902088131437, 20240910-173847-9eedb2ac-e3a5-4d07-94fe-f7e881d67418_6902088131437, 0.582, 0.979, 0.806 +same, 6904682300226, 20240910-171800-76a062fd-409c-480f-94f4-fd0e65d72467_6904682300226, 0.173, 0.830, 0.372 +same, 6970399922365, 20240910-172352-9b79a4d9-092f-477d-a7a4-8af079d1538d_6970399922365, 0.226, 0.774, 0.597 +same, 6902265202318, 20240910-170331-e3ee7cf5-dda2-4d0b-b8c9-4fb411fe78ec_6902265202318, 0.557, 0.922, 0.803 +same, 6907992517780, 20240910-163802-6b9f0129-8497-467f-a506-5708eda436a4_6907992517780, 0.354, 0.761, 0.848 +same, 6902132084337, 20240910-172403-dbc9de02-2811-449c-961f-23e7a16877d7_6902132084337, 0.406, 0.774, 0.850 +same, 6901668934888, 20240910-164315-38c640ba-cdf3-4ac1-8bff-55fe5d0560bb_6901668934888, 0.290, 0.598, 0.621 +same, 8000500023976, 20240910-173323-78dc658e-e4ef-49e1-a2ff-9ada34c27a85_8000500023976, 0.495, 0.825, 0.792 +same, 6904682300219, 20240910-164323-8e9a882a-a502-4a6e-bd99-70deb2130f57_6904682300219, 0.278, 0.782, 0.551 +same, 6903148231623, 20240910-163750-8e13e800-21d0-4bd9-b686-18ed213460cd_6903148231623, 0.320, 0.870, 0.718 +same, 6904682300219, 20240910-170920-dc16c149-06a3-4c2d-9bec-e930274b55ce_6904682300219, 0.217, 0.697, 0.418 +same, 6902890218470, 20240910-172802-0dbe3709-bd0c-45e7-ad36-0cfc9781ef1b_6902890218470, 0.198, 0.690, 0.538 +same, 6901668934888, 20240910-165620-0b870f0d-88a5-4286-bcbf-b0ebb41ddcfc_6901668934888, 0.325, 0.710, 0.690 +same, 6902088131437, 20240910-163846-7793e886-9f09-4744-9e24-eb47d65c09f5_6902088131437, 0.450, 0.983, 0.784 +same, 6901070600142, 20240910-170742-f78b59da-e242-42c9-ac7a-bba23ff11aff_6901070600142, 0.295, 0.728, 0.668 +same, 8993175540667, 20240910-172814-d17bd016-b8e5-4a21-a137-6bce693e0cb0_8993175540667, 0.418, 0.859, 0.687 +same, 6901668929730, 20240910-162930-ec2bb380-53fe-483f-9aab-9038643ebd1f_6901668929730, 0.549, 0.853, 0.888 +same, 6970399922365, 20240910-173332-55f8124d-7ab0-4a7a-8b08-f4dd9ba06502_6970399922365, 0.330, 0.766, 0.817 +same, 6901668929730, 20240910-173214-5b86868f-cb5b-4b7f-8f3a-aff08d89900d_6901668929730, 0.529, 0.849, 0.864 +same, 6903148048801, 20240910-172904-5462ad91-2a07-4116-898f-ff1d2021e6af_6903148048801, 0.444, 0.865, 0.769 +same, 6901668934628, 20240910-171838-c77a6d0d-185b-48e7-9af9-05de561f1172_6901668934628, 0.489, 0.930, 0.758 +same, 6902890218470, 20240910-170934-74c137ee-0689-42d0-9994-da8ba59fd5db_6902890218470, 0.251, 0.738, 0.652 +same, 6949909050041, 20240910-162952-f6ec3a40-9d64-4f20-b122-0b81eb4a2134_6949909050041, 0.384, 0.870, 0.714 +same, 6901668934888, 20240910-172841-9d7b16fb-4200-4089-b4b2-925da10208ed_6901668934888, 0.336, 0.778, 0.751 +same, 6901668936271, 20240910-165632-a1e22655-d9ad-47f5-a467-55718bd1e23e_6901668936271, 0.121, 0.604, 0.257 +same, 6904682300226, 20240910-163718-e1e09ad9-7a7e-4b43-beb7-47080c0a312e_6904682300226, 0.297, 0.847, 0.651 +same, 6903148126677, 20240910-173233-81246d1d-bbf3-4ee2-b6c1-7f8fe5818266_6903148126677, 0.422, 0.814, 0.717 +same, 6924743915848, 20240910-162836-186bdf15-5ebb-4b55-a3a4-47edea86a7ee_6924743915848, 0.285, 0.697, 0.640 +same, 6902132084337, 20240910-173222-8abca736-4b5d-4b8e-8e53-206809f37082_6902132084337, 0.350, 0.819, 0.857 +same, 8993175537322, 20240910-170945-c5a587f8-925a-46c2-b2f4-b8fe0fa41c90_8993175537322, 0.349, 0.832, 0.611 +same, 6902265202318, 20240910-162848-b0d67358-6f68-482a-94cb-d7de7414e32f_6902265202318, 0.392, 0.859, 0.695 +same, 6907992517780, 20240910-173730-c51d9d00-65a2-4212-99f3-701092810919_6907992517780, 0.405, 0.815, 0.865 +same, 6902265160502, 20240910-170318-706146af-c203-459a-b642-da428ce6426a_6902265160502, 0.162, 0.703, 0.531 +same, 6903148347409, 20240910-162902-3de7f2a9-9068-4f61-a150-0bcc47194a43_6903148347409, 0.156, 0.693, 0.470 +same, 6902265202318, 20240910-172023-a9b8c8b4-8030-4aa5-85fe-54cba57e745f_6902265202318, 0.378, 0.865, 0.694 +same, 6903148126677, 20240910-171920-0a6490ce-547f-493d-b76a-4c849ae12a93_6903148126677, 0.496, 0.879, 0.796 +same, 6901668936295, 20240910-164334-09d4e20e-68c8-48ca-b931-50e58428ef2a_6901668936295, 0.170, 0.631, 0.325 +same, 6958104102516, 20240910-165604-0f805f9d-24f7-4729-923a-bff489a09323_6958104102516, 0.235, 0.731, 0.550 +same, 6901668936684, 20240910-164409-053f810b-7369-4a3e-b91b-b7ba99fa5b9c_6901668936684, 0.230, 0.638, 0.450 +same, 6902265150022, 20240910-170349-b357333c-e939-4ce5-8019-7762799a9097_6902265150022, 0.362, 0.927, 0.794 +same, 6902890232216, 20240910-172828-0a20bffd-ede3-4b0c-977b-8652f52518f9_6902890232216, 0.255, 0.761, 0.626 +same, 6902890232216, 20240910-170807-7bc77832-4cf1-4cd8-aa54-994ff164dcc7_6902890232216, 0.296, 0.695, 0.585 +same, 6901668929730, 20240910-171715-a8fc6d8a-87bd-4fbd-b378-85e34193266f_6901668929730, 0.503, 0.848, 0.823 +same, 6903148231623, 20240910-170258-38579506-3874-4d71-b9d2-ac6e47ca75dd_6903148231623, 0.256, 0.720, 0.506 +same, 6902265150022, 20240910-172010-035f68e4-9b7c-40f7-961c-aa8c0f154252_6902265150022, 0.428, 0.940, 0.823 +same, 6901668929518, 20240910-173344-258d27a2-b2e1-468e-8f32-40edcda94486_6901668929518, 0.361, 0.853, 0.721 +same, 6901668934628, 20240910-170431-722e7de7-c7ef-4825-8080-be019c7f4602_6901668934628, 0.444, 0.882, 0.690 +same, 6974158892364, 20240910-163028-418ab174-5722-4e8a-ae12-e8d3c33f70b5_6974158892364, 0.119, 0.684, 0.439 +same, 6902890218470, 20240910-164251-a2a38e17-5532-49a5-9372-5a3ed8dc6972_6902890218470, 0.281, 0.689, 0.666 +same, 6902265150022, 20240910-163814-9fc0324d-134a-46ee-bb79-6b2dfb6388f9_6902265150022, 0.308, 0.899, 0.682 +same, 6901668929518, 20240910-170417-1ac149e8-4ecb-447c-a8b7-8d5b96e77ffa_6901668929518, 0.260, 0.821, 0.586 +same, 6901668936271, 20240910-172745-96dc9808-4157-4806-856f-c7013452f302_6901668936271, 0.156, 0.617, 0.315 +same, 6903148126677, 20240910-163857-736e50b8-eae8-4a6d-af26-ce3a57a073b8_6903148126677, 0.420, 0.891, 0.749 +same, 6901668936684, 20240910-172445-4f28474f-5463-4b19-bc2d-671105764e27_6901668936684, 0.212, 0.675, 0.445 +same, 6901668936295, 20240910-172754-d034ab2f-1b18-4d6a-a936-9fa538066253_6901668936295, 0.130, 0.630, 0.254 +same, 8993175540667, 20240910-165644-2e79a878-caf1-44ca-851c-287848800d35_8993175540667, 0.565, 0.872, 0.821 +same, 6901668929518, 20240910-172039-ebd2a496-c407-4450-b122-0e8f33e07de2_6901668929518, 0.341, 0.826, 0.725 +same, 6902132084337, 20240910-162817-18813894-397a-4c94-8b90-2d7a46319793_6902132084337, 0.438, 0.794, 0.887 +same, 6904682300219, 20240910-172257-9169e95d-ff11-4d31-98af-13df3f071840_6904682300219, 0.365, 0.804, 0.643 +same, 6901668934628, 20240910-173306-a1409202-ea3d-47c4-aa39-9d17dae711cf_6901668934628, 0.489, 0.894, 0.770 +same, 6902088131437, 20240910-172427-781eb94d-efb6-403c-b88f-f4b9df82fee0_6902088131437, 0.536, 0.980, 0.829 +same, 9421903892324, 20240910-173757-b4ed1c60-a96b-48ad-a451-3caecd61c327_9421903892324, 0.421, 0.892, 0.755 +same, 6901668936684, 20240910-170907-0e74383f-0341-4b90-b333-910e5a184296_6901668936684, 0.289, 0.672, 0.569 +same, 8000500023976, 20240910-171014-ee1e7d74-0d89-4014-a125-7c9cdebb15fd_8000500023976, 0.286, 0.872, 0.660 +same, 6901668929518, 20240910-164347-47377bae-2ca6-4d75-a076-e7f6c03d0f2e_6901668929518, 0.446, 0.847, 0.834 +same, 6902265160502, 20240910-173409-55dd7611-7394-4783-9f4e-4639401078ea_6902265160502, 0.212, 0.857, 0.611 +same, 6901668936684, 20240910-165525-e17864c9-e965-4531-be14-be551dad88fb_6901668936684, 0.149, 0.614, 0.344 +same, 6901668934628, 20240910-162805-592cff06-4acb-420f-bc36-bb00f3e0efbb_6901668934628, 0.275, 0.870, 0.521 +same, 6949909050041, 20240910-172919-ab2efd9a-a776-420f-95f5-2f8188f719e4_6949909050041, 0.401, 0.849, 0.792 +same, 6907992517780, 20240910-171723-2f8a7ece-99cb-4d91-b484-67b486599f26_6907992517780, 0.391, 0.848, 0.838 +same, 6902890218470, 20240910-165443-48bad32d-9f2b-499b-907d-c602cf563ee3_6902890218470, 0.281, 0.737, 0.774 +same, 6904682300219, 20240910-165455-d0e36365-f7f2-4f2e-84a7-1ffc24ccc1c7_6904682300219, 0.424, 0.892, 0.792 +same, 6904682300226, 20240910-170231-21568a27-641b-448d-8b8c-9eff4dfe7294_6904682300226, 0.257, 0.725, 0.636 +same, 6903148048801, 20240910-163740-851d23c1-e90f-4947-abc3-f463991c5505_6903148048801, 0.422, 0.826, 0.784 +same, 6902132084337, 20240910-170730-76626a74-34fb-486d-b889-4276552edb0e_6902132084337, 0.379, 0.831, 0.792 +same, 9421903892324, 20240910-172316-ffa74ee4-46d5-4266-b362-ebfebed0c572_9421903892324, 0.304, 0.877, 0.548 +same, 6904682300219, 20240910-173807-afdeec3a-0d6e-4db8-9baf-826b7d6b4660_6904682300219, 0.254, 0.769, 0.477 +same, 6902890232216, 20240910-163838-9e6f0b38-2ffe-4727-9ec7-a02435b8f629_6902890232216, 0.264, 0.786, 0.593 +same, 6901668936295, 20240910-165424-5d55263c-e523-495e-b673-fc53eaa68b05_6901668936295, 0.139, 0.542, 0.239 +same, 6903148126677, 20240910-170403-c1b9db80-7ee0-4508-8858-1e3e1b924648_6903148126677, 0.351, 0.861, 0.602 +same, 6901668929518, 20240910-172500-509a2d1e-e665-4fe6-8ffe-b69117d7b09f_6901668929518, 0.365, 0.821, 0.731 +same, 6903148231623, 20240910-171824-2d3edfcd-c169-4c6e-9734-9325b72cf9fe_6903148231623, 0.176, 0.688, 0.359 +same, 6901668929518, 20240910-173839-e4b3b834-c695-4917-b2f4-7cfaaebb98dc_6901668929518, 0.437, 0.874, 0.772 +same, 6901668929730, 20240910-170447-3b37f76f-5e21-400b-a8a8-2376c0796ae6_6901668929730, 0.461, 0.852, 0.797 +same, 6903148080085, 20240910-173314-d6ac3740-20f2-4aa7-a392-80a96b7607c3_6903148080085, 0.370, 0.860, 0.827 +same, 6901070600142, 20240910-172734-8c23b385-99f7-4e01-819a-78c86611ff48_6901070600142, 0.201, 0.672, 0.442 +same, 6958104102516, 20240910-164452-0f365052-2e4a-4d00-9cf7-0407d731d07e_6958104102516, 0.234, 0.866, 0.583 +same, 6901070600142, 20240910-162749-ab186eb8-6777-489b-8ad0-c1c6e66b285d_6901070600142, 0.269, 0.727, 0.591 +same, 8993175537322, 20240910-164432-008357d7-7ee6-49b9-8d08-3f3a6081c4e1_8993175537322, 0.450, 0.790, 0.785 +same, 6975682480393, 20240910-163007-6dfc085b-42b9-432d-9c41-7bfd294526b6_6975682480393, 0.448, 0.835, 0.828 +same, 6903148080085, 20240910-163825-e4de18e2-fe7c-4ff6-8b51-7ef2a7db7ed3_6903148080085, 0.351, 0.838, 0.766 +same, 6903148231623, 20240910-172854-5fb70036-3089-4258-9346-de25d415f120_6903148231623, 0.423, 0.845, 0.782 +same, 6949909050041, 20240910-170817-c2f8c500-3aa5-4bd2-bf82-787d0cd22585_6949909050041, 0.494, 0.893, 0.885 +same, 6907992517780, 20240910-170246-e773b037-a712-4d78-accd-71c24b675365_6907992517780, 0.338, 0.737, 0.823 +same, 6902265160502, 20240910-163907-1ac881ec-cac4-4811-9cab-1826731e77bd_6902265160502, 0.239, 0.833, 0.706 +same, 6901668936271, 20240910-164239-e4d8f615-8cf3-483d-bc6e-03e470e2110c_6901668936271, 0.230, 0.615, 0.390 +same, 8993175537322, 20240910-172328-48a512b9-4fb1-4abf-bca9-8b3443ce8f2b_8993175537322, 0.456, 0.783, 0.719 +same, 8993175537322, 20240910-173819-226cc352-acdc-4419-9159-c97ae0eb58af_8993175537322, 0.455, 0.766, 0.717 +same, 6901668929518, 20240910-165517-a0000cdf-aa15-42c8-a6be-dbce8cf7cb32_6901668929518, 0.406, 0.861, 0.759 +same, 8000500023976, 20240910-172417-e9d563b9-74e2-4ec1-8f34-331424b48e72_8000500023976, 0.350, 0.853, 0.686 +diff, 8993175537322, 20240910-173355-1bbf290e-1f14-4ba8-b666-82c990c4eea3_6901668936684, 0.017, 0.341, 0.030 +diff, 6904682300226, 20240910-173847-9eedb2ac-e3a5-4d07-94fe-f7e881d67418_6902088131437, 0.007, 0.348, 0.013 +diff, 8993175540667, 20240910-171800-76a062fd-409c-480f-94f4-fd0e65d72467_6904682300226, 0.038, 0.309, 0.067 +diff, 6901668934628, 20240910-172352-9b79a4d9-092f-477d-a7a4-8af079d1538d_6970399922365, -0.003, 0.302, -0.006 +diff, 6901668929518, 20240910-170331-e3ee7cf5-dda2-4d0b-b8c9-4fb411fe78ec_6902265202318, -0.023, 0.273, -0.038 +diff, 6903148080085, 20240910-163802-6b9f0129-8497-467f-a506-5708eda436a4_6907992517780, 0.026, 0.408, 0.061 +diff, 6970399922365, 20240910-172403-dbc9de02-2811-449c-961f-23e7a16877d7_6902132084337, 0.090, 0.479, 0.207 +diff, 6904682300226, 20240910-164315-38c640ba-cdf3-4ac1-8bff-55fe5d0560bb_6901668934888, 0.072, 0.383, 0.142 +diff, 6974158892364, 20240910-173323-78dc658e-e4ef-49e1-a2ff-9ada34c27a85_8000500023976, -0.044, 0.340, -0.117 +diff, 6901668934888, 20240910-164323-8e9a882a-a502-4a6e-bd99-70deb2130f57_6904682300219, -0.017, 0.459, -0.042 +diff, 6907992517780, 20240910-163750-8e13e800-21d0-4bd9-b686-18ed213460cd_6903148231623, 0.019, 0.391, 0.051 +diff, 6901668934628, 20240910-170920-dc16c149-06a3-4c2d-9bec-e930274b55ce_6904682300219, 0.033, 0.331, 0.063 +diff, 6901668936684, 20240910-172802-0dbe3709-bd0c-45e7-ad36-0cfc9781ef1b_6902890218470, -0.072, 0.270, -0.163 +diff, 6907992517780, 20240910-165620-0b870f0d-88a5-4286-bcbf-b0ebb41ddcfc_6901668934888, 0.141, 0.461, 0.292 +diff, 6958104102516, 20240910-163846-7793e886-9f09-4744-9e24-eb47d65c09f5_6902088131437, -0.022, 0.373, -0.053 +diff, 8993175537322, 20240910-170742-f78b59da-e242-42c9-ac7a-bba23ff11aff_6901070600142, -0.018, 0.293, -0.033 +diff, 6903148126677, 20240910-172814-d17bd016-b8e5-4a21-a137-6bce693e0cb0_8993175540667, -0.044, 0.356, -0.082 +diff, 8993175540667, 20240910-162930-ec2bb380-53fe-483f-9aab-9038643ebd1f_6901668929730, -0.021, 0.349, -0.032 +diff, 9421903892324, 20240910-173332-55f8124d-7ab0-4a7a-8b08-f4dd9ba06502_6970399922365, 0.033, 0.383, 0.062 +diff, 6902890232216, 20240910-173214-5b86868f-cb5b-4b7f-8f3a-aff08d89900d_6901668929730, 0.076, 0.419, 0.151 +diff, 6903148231623, 20240910-172904-5462ad91-2a07-4116-898f-ff1d2021e6af_6903148048801, 0.012, 0.309, 0.019 +diff, 6924743915848, 20240910-171838-c77a6d0d-185b-48e7-9af9-05de561f1172_6901668934628, -0.069, 0.326, -0.147 +diff, 6975682480393, 20240910-170934-74c137ee-0689-42d0-9994-da8ba59fd5db_6902890218470, 0.122, 0.628, 0.274 +diff, 6975682480393, 20240910-162952-f6ec3a40-9d64-4f20-b122-0b81eb4a2134_6949909050041, 0.094, 0.647, 0.188 +diff, 6907992517780, 20240910-172841-9d7b16fb-4200-4089-b4b2-925da10208ed_6901668934888, 0.157, 0.646, 0.343 +diff, 6902265202318, 20240910-165632-a1e22655-d9ad-47f5-a467-55718bd1e23e_6901668936271, -0.006, 0.286, -0.011 +diff, 6902890232216, 20240910-163718-e1e09ad9-7a7e-4b43-beb7-47080c0a312e_6904682300226, 0.066, 0.491, 0.157 +diff, 9421903892324, 20240910-173233-81246d1d-bbf3-4ee2-b6c1-7f8fe5818266_6903148126677, -0.038, 0.450, -0.061 +diff, 6902132084337, 20240910-162836-186bdf15-5ebb-4b55-a3a4-47edea86a7ee_6924743915848, -0.061, 0.267, -0.125 +diff, 9421903892324, 20240910-173222-8abca736-4b5d-4b8e-8e53-206809f37082_6902132084337, 0.105, 0.454, 0.213 +diff, 6901668934628, 20240910-170945-c5a587f8-925a-46c2-b2f4-b8fe0fa41c90_8993175537322, -0.089, 0.186, -0.148 +diff, 6901668934888, 20240910-162848-b0d67358-6f68-482a-94cb-d7de7414e32f_6902265202318, -0.038, 0.352, -0.087 +diff, 6902265202318, 20240910-173730-c51d9d00-65a2-4212-99f3-701092810919_6907992517780, 0.025, 0.325, 0.043 +diff, 6902890232216, 20240910-170318-706146af-c203-459a-b642-da428ce6426a_6902265160502, 0.077, 0.540, 0.241 +diff, 6903148126677, 20240910-162902-3de7f2a9-9068-4f61-a150-0bcc47194a43_6903148347409, -0.047, 0.247, -0.113 +diff, 6903148347409, 20240910-172023-a9b8c8b4-8030-4aa5-85fe-54cba57e745f_6902265202318, 0.019, 0.312, 0.049 +diff, 6904682300219, 20240910-171920-0a6490ce-547f-493d-b76a-4c849ae12a93_6903148126677, 0.022, 0.340, 0.033 +diff, 6974158892364, 20240910-164334-09d4e20e-68c8-48ca-b931-50e58428ef2a_6901668936295, 0.035, 0.446, 0.108 +diff, 6901070600142, 20240910-165604-0f805f9d-24f7-4729-923a-bff489a09323_6958104102516, 0.016, 0.385, 0.042 +diff, 6901668934628, 20240910-164409-053f810b-7369-4a3e-b91b-b7ba99fa5b9c_6901668936684, -0.045, 0.563, -0.079 +diff, 6924743915848, 20240910-170349-b357333c-e939-4ce5-8019-7762799a9097_6902265150022, -0.096, 0.342, -0.249 +diff, 6903148126677, 20240910-172828-0a20bffd-ede3-4b0c-977b-8652f52518f9_6902890232216, 0.053, 0.326, 0.112 +diff, 6904682300226, 20240910-170807-7bc77832-4cf1-4cd8-aa54-994ff164dcc7_6902890232216, 0.063, 0.430, 0.115 +diff, 9421903892324, 20240910-171715-a8fc6d8a-87bd-4fbd-b378-85e34193266f_6901668929730, -0.066, 0.306, -0.107 +diff, 6901668936684, 20240910-170258-38579506-3874-4d71-b9d2-ac6e47ca75dd_6903148231623, 0.062, 0.403, 0.131 +diff, 6970399922365, 20240910-172010-035f68e4-9b7c-40f7-961c-aa8c0f154252_6902265150022, -0.044, 0.355, -0.101 +diff, 6903148048801, 20240910-173344-258d27a2-b2e1-468e-8f32-40edcda94486_6901668929518, 0.077, 0.498, 0.147 +diff, 6901668934888, 20240910-170431-722e7de7-c7ef-4825-8080-be019c7f4602_6901668934628, 0.001, 0.441, 0.001 +diff, 6970399922365, 20240910-163028-418ab174-5722-4e8a-ae12-e8d3c33f70b5_6974158892364, 0.072, 0.537, 0.208 +diff, 6975682480393, 20240910-164251-a2a38e17-5532-49a5-9372-5a3ed8dc6972_6902890218470, 0.112, 0.660, 0.231 +diff, 6901668929518, 20240910-163814-9fc0324d-134a-46ee-bb79-6b2dfb6388f9_6902265150022, -0.067, 0.359, -0.146 +diff, 6901070600142, 20240910-170417-1ac149e8-4ecb-447c-a8b7-8d5b96e77ffa_6901668929518, -0.033, 0.306, -0.085 +diff, 6903148126677, 20240910-172745-96dc9808-4157-4806-856f-c7013452f302_6901668936271, 0.008, 0.361, 0.018 +diff, 6903148347409, 20240910-163857-736e50b8-eae8-4a6d-af26-ce3a57a073b8_6903148126677, -0.008, 0.348, -0.017 +diff, 6901668936271, 20240910-172445-4f28474f-5463-4b19-bc2d-671105764e27_6901668936684, 0.064, 0.555, 0.128 +diff, 6901070600142, 20240910-172754-d034ab2f-1b18-4d6a-a936-9fa538066253_6901668936295, 0.189, 0.600, 0.448 +diff, 6902265150022, 20240910-165644-2e79a878-caf1-44ca-851c-287848800d35_8993175540667, 0.036, 0.300, 0.064 +diff, 6901668934888, 20240910-172039-ebd2a496-c407-4450-b122-0e8f33e07de2_6901668929518, 0.047, 0.373, 0.112 +diff, 6958104102516, 20240910-162817-18813894-397a-4c94-8b90-2d7a46319793_6902132084337, -0.068, 0.247, -0.130 +diff, 6902265160502, 20240910-172257-9169e95d-ff11-4d31-98af-13df3f071840_6904682300219, 0.046, 0.467, 0.106 +diff, 6970399922365, 20240910-173306-a1409202-ea3d-47c4-aa39-9d17dae711cf_6901668934628, 0.023, 0.376, 0.049 +diff, 6902265202318, 20240910-172427-781eb94d-efb6-403c-b88f-f4b9df82fee0_6902088131437, 0.017, 0.314, 0.030 +diff, 6907992517780, 20240910-173757-b4ed1c60-a96b-48ad-a451-3caecd61c327_9421903892324, 0.118, 0.551, 0.254 +diff, 6901668936271, 20240910-170907-0e74383f-0341-4b90-b333-910e5a184296_6901668936684, 0.137, 0.498, 0.255 +diff, 6901668934628, 20240910-171014-ee1e7d74-0d89-4014-a125-7c9cdebb15fd_8000500023976, 0.061, 0.324, 0.135 +diff, 6903148126677, 20240910-164347-47377bae-2ca6-4d75-a076-e7f6c03d0f2e_6901668929518, -0.026, 0.332, -0.047 +diff, 6903148048801, 20240910-173409-55dd7611-7394-4783-9f4e-4639401078ea_6902265160502, 0.030, 0.370, 0.070 +diff, 6902132084337, 20240910-165525-e17864c9-e965-4531-be14-be551dad88fb_6901668936684, 0.043, 0.375, 0.112 +diff, 6902890232216, 20240910-162805-592cff06-4acb-420f-bc36-bb00f3e0efbb_6901668934628, -0.067, 0.258, -0.164 +diff, 6903148048801, 20240910-172919-ab2efd9a-a776-420f-95f5-2f8188f719e4_6949909050041, 0.118, 0.397, 0.235 +diff, 6970399922365, 20240910-171723-2f8a7ece-99cb-4d91-b484-67b486599f26_6907992517780, -0.043, 0.339, -0.101 +diff, 6903148048801, 20240910-165443-48bad32d-9f2b-499b-907d-c602cf563ee3_6902890218470, -0.001, 0.482, -0.002 +diff, 6904682300226, 20240910-165455-d0e36365-f7f2-4f2e-84a7-1ffc24ccc1c7_6904682300219, 0.270, 0.813, 0.583 +diff, 6901668936271, 20240910-170231-21568a27-641b-448d-8b8c-9eff4dfe7294_6904682300226, 0.026, 0.369, 0.057 +diff, 6949909050041, 20240910-163740-851d23c1-e90f-4947-abc3-f463991c5505_6903148048801, 0.104, 0.443, 0.192 +diff, 6902890232216, 20240910-170730-76626a74-34fb-486d-b889-4276552edb0e_6902132084337, -0.018, 0.254, -0.040 +diff, 6924743915848, 20240910-172316-ffa74ee4-46d5-4266-b362-ebfebed0c572_9421903892324, 0.076, 0.444, 0.182 +diff, 6901070600142, 20240910-173807-afdeec3a-0d6e-4db8-9baf-826b7d6b4660_6904682300219, 0.010, 0.482, 0.024 +diff, 6924743915848, 20240910-163838-9e6f0b38-2ffe-4727-9ec7-a02435b8f629_6902890232216, -0.025, 0.380, -0.061 +diff, 6902265160502, 20240910-165424-5d55263c-e523-495e-b673-fc53eaa68b05_6901668936295, -0.042, 0.280, -0.088 +diff, 6902088131437, 20240910-170403-c1b9db80-7ee0-4508-8858-1e3e1b924648_6903148126677, -0.019, 0.228, -0.026 +diff, 6903148080085, 20240910-172500-509a2d1e-e665-4fe6-8ffe-b69117d7b09f_6901668929518, 0.064, 0.486, 0.135 +diff, 6901668934888, 20240910-171824-2d3edfcd-c169-4c6e-9734-9325b72cf9fe_6903148231623, 0.014, 0.325, 0.036 +diff, 6901668929730, 20240910-173839-e4b3b834-c695-4917-b2f4-7cfaaebb98dc_6901668929518, -0.066, 0.282, -0.106 +diff, 6901070600142, 20240910-170447-3b37f76f-5e21-400b-a8a8-2376c0796ae6_6901668929730, -0.068, 0.414, -0.148 +diff, 6974158892364, 20240910-173314-d6ac3740-20f2-4aa7-a392-80a96b7607c3_6903148080085, -0.033, 0.303, -0.107 +diff, 6901668936295, 20240910-172734-8c23b385-99f7-4e01-819a-78c86611ff48_6901070600142, 0.008, 0.417, 0.015 +diff, 6975682480393, 20240910-164452-0f365052-2e4a-4d00-9cf7-0407d731d07e_6958104102516, 0.031, 0.405, 0.075 +diff, 6903148080085, 20240910-162749-ab186eb8-6777-489b-8ad0-c1c6e66b285d_6901070600142, -0.015, 0.311, -0.030 +diff, 6901668929730, 20240910-164432-008357d7-7ee6-49b9-8d08-3f3a6081c4e1_8993175537322, 0.020, 0.303, 0.035 +diff, 6902890218470, 20240910-163007-6dfc085b-42b9-432d-9c41-7bfd294526b6_6975682480393, 0.184, 0.633, 0.393 +diff, 6902890232216, 20240910-163825-e4de18e2-fe7c-4ff6-8b51-7ef2a7db7ed3_6903148080085, 0.023, 0.348, 0.053 +diff, 6902890232216, 20240910-172854-5fb70036-3089-4258-9346-de25d415f120_6903148231623, -0.080, 0.324, -0.182 +diff, 6901668936271, 20240910-170817-c2f8c500-3aa5-4bd2-bf82-787d0cd22585_6949909050041, -0.011, 0.324, -0.019 +diff, 6902265160502, 20240910-170246-e773b037-a712-4d78-accd-71c24b675365_6907992517780, -0.094, 0.358, -0.244 +diff, 6902132084337, 20240910-163907-1ac881ec-cac4-4811-9cab-1826731e77bd_6902265160502, -0.007, 0.319, -0.020 +diff, 6970399922365, 20240910-164239-e4d8f615-8cf3-483d-bc6e-03e470e2110c_6901668936271, 0.048, 0.361, 0.105 +diff, 6904682300219, 20240910-172328-48a512b9-4fb1-4abf-bca9-8b3443ce8f2b_8993175537322, -0.014, 0.472, -0.021 +diff, 6901668936271, 20240910-173819-226cc352-acdc-4419-9159-c97ae0eb58af_8993175537322, 0.009, 0.332, 0.014 +diff, 6901668936271, 20240910-165517-a0000cdf-aa15-42c8-a6be-dbce8cf7cb32_6901668929518, 0.089, 0.483, 0.153 +diff, 6901668929730, 20240910-172417-e9d563b9-74e2-4ec1-8f34-331424b48e72_8000500023976, 0.110, 0.465, 0.216 diff --git a/contrast/result/pickle/20240911_183903_uint8.txt b/contrast/result/pickle/20240911_183903_uint8.txt new file mode 100644 index 0000000..99ab7df --- /dev/null +++ b/contrast/result/pickle/20240911_183903_uint8.txt @@ -0,0 +1,202 @@ +same, 6901668936684, 20240910-173355-1bbf290e-1f14-4ba8-b666-82c990c4eea3_6901668936684, 0.268, 0.655, 0.507 +same, 6902088131437, 20240910-173847-9eedb2ac-e3a5-4d07-94fe-f7e881d67418_6902088131437, 0.581, 0.977, 0.805 +same, 6904682300226, 20240910-171800-76a062fd-409c-480f-94f4-fd0e65d72467_6904682300226, 0.173, 0.831, 0.372 +same, 6970399922365, 20240910-172352-9b79a4d9-092f-477d-a7a4-8af079d1538d_6970399922365, 0.226, 0.774, 0.596 +same, 6902265202318, 20240910-170331-e3ee7cf5-dda2-4d0b-b8c9-4fb411fe78ec_6902265202318, 0.554, 0.918, 0.802 +same, 6907992517780, 20240910-163802-6b9f0129-8497-467f-a506-5708eda436a4_6907992517780, 0.353, 0.753, 0.849 +same, 6902132084337, 20240910-172403-dbc9de02-2811-449c-961f-23e7a16877d7_6902132084337, 0.405, 0.765, 0.850 +same, 6901668934888, 20240910-164315-38c640ba-cdf3-4ac1-8bff-55fe5d0560bb_6901668934888, 0.289, 0.595, 0.620 +same, 8000500023976, 20240910-173323-78dc658e-e4ef-49e1-a2ff-9ada34c27a85_8000500023976, 0.492, 0.826, 0.792 +same, 6904682300219, 20240910-164323-8e9a882a-a502-4a6e-bd99-70deb2130f57_6904682300219, 0.279, 0.786, 0.554 +same, 6903148231623, 20240910-163750-8e13e800-21d0-4bd9-b686-18ed213460cd_6903148231623, 0.319, 0.870, 0.718 +same, 6904682300219, 20240910-170920-dc16c149-06a3-4c2d-9bec-e930274b55ce_6904682300219, 0.218, 0.692, 0.419 +same, 6902890218470, 20240910-172802-0dbe3709-bd0c-45e7-ad36-0cfc9781ef1b_6902890218470, 0.198, 0.688, 0.541 +same, 6901668934888, 20240910-165620-0b870f0d-88a5-4286-bcbf-b0ebb41ddcfc_6901668934888, 0.322, 0.713, 0.687 +same, 6902088131437, 20240910-163846-7793e886-9f09-4744-9e24-eb47d65c09f5_6902088131437, 0.448, 0.981, 0.782 +same, 6901070600142, 20240910-170742-f78b59da-e242-42c9-ac7a-bba23ff11aff_6901070600142, 0.294, 0.724, 0.666 +same, 8993175540667, 20240910-172814-d17bd016-b8e5-4a21-a137-6bce693e0cb0_8993175540667, 0.419, 0.856, 0.690 +same, 6901668929730, 20240910-162930-ec2bb380-53fe-483f-9aab-9038643ebd1f_6901668929730, 0.549, 0.847, 0.889 +same, 6970399922365, 20240910-173332-55f8124d-7ab0-4a7a-8b08-f4dd9ba06502_6970399922365, 0.328, 0.767, 0.815 +same, 6901668929730, 20240910-173214-5b86868f-cb5b-4b7f-8f3a-aff08d89900d_6901668929730, 0.529, 0.850, 0.865 +same, 6903148048801, 20240910-172904-5462ad91-2a07-4116-898f-ff1d2021e6af_6903148048801, 0.444, 0.861, 0.771 +same, 6901668934628, 20240910-171838-c77a6d0d-185b-48e7-9af9-05de561f1172_6901668934628, 0.486, 0.926, 0.755 +same, 6902890218470, 20240910-170934-74c137ee-0689-42d0-9994-da8ba59fd5db_6902890218470, 0.251, 0.737, 0.653 +same, 6949909050041, 20240910-162952-f6ec3a40-9d64-4f20-b122-0b81eb4a2134_6949909050041, 0.384, 0.866, 0.715 +same, 6901668934888, 20240910-172841-9d7b16fb-4200-4089-b4b2-925da10208ed_6901668934888, 0.335, 0.776, 0.751 +same, 6901668936271, 20240910-165632-a1e22655-d9ad-47f5-a467-55718bd1e23e_6901668936271, 0.118, 0.603, 0.253 +same, 6904682300226, 20240910-163718-e1e09ad9-7a7e-4b43-beb7-47080c0a312e_6904682300226, 0.297, 0.845, 0.653 +same, 6903148126677, 20240910-173233-81246d1d-bbf3-4ee2-b6c1-7f8fe5818266_6903148126677, 0.420, 0.817, 0.717 +same, 6924743915848, 20240910-162836-186bdf15-5ebb-4b55-a3a4-47edea86a7ee_6924743915848, 0.286, 0.706, 0.642 +same, 6902132084337, 20240910-173222-8abca736-4b5d-4b8e-8e53-206809f37082_6902132084337, 0.348, 0.818, 0.856 +same, 8993175537322, 20240910-170945-c5a587f8-925a-46c2-b2f4-b8fe0fa41c90_8993175537322, 0.348, 0.829, 0.611 +same, 6902265202318, 20240910-162848-b0d67358-6f68-482a-94cb-d7de7414e32f_6902265202318, 0.391, 0.858, 0.695 +same, 6907992517780, 20240910-173730-c51d9d00-65a2-4212-99f3-701092810919_6907992517780, 0.405, 0.816, 0.865 +same, 6902265160502, 20240910-170318-706146af-c203-459a-b642-da428ce6426a_6902265160502, 0.161, 0.697, 0.530 +same, 6903148347409, 20240910-162902-3de7f2a9-9068-4f61-a150-0bcc47194a43_6903148347409, 0.156, 0.691, 0.470 +same, 6902265202318, 20240910-172023-a9b8c8b4-8030-4aa5-85fe-54cba57e745f_6902265202318, 0.376, 0.863, 0.692 +same, 6903148126677, 20240910-171920-0a6490ce-547f-493d-b76a-4c849ae12a93_6903148126677, 0.494, 0.875, 0.795 +same, 6901668936295, 20240910-164334-09d4e20e-68c8-48ca-b931-50e58428ef2a_6901668936295, 0.170, 0.632, 0.325 +same, 6958104102516, 20240910-165604-0f805f9d-24f7-4729-923a-bff489a09323_6958104102516, 0.235, 0.726, 0.550 +same, 6901668936684, 20240910-164409-053f810b-7369-4a3e-b91b-b7ba99fa5b9c_6901668936684, 0.228, 0.638, 0.446 +same, 6902265150022, 20240910-170349-b357333c-e939-4ce5-8019-7762799a9097_6902265150022, 0.362, 0.927, 0.794 +same, 6902890232216, 20240910-172828-0a20bffd-ede3-4b0c-977b-8652f52518f9_6902890232216, 0.254, 0.761, 0.625 +same, 6902890232216, 20240910-170807-7bc77832-4cf1-4cd8-aa54-994ff164dcc7_6902890232216, 0.295, 0.692, 0.584 +same, 6901668929730, 20240910-171715-a8fc6d8a-87bd-4fbd-b378-85e34193266f_6901668929730, 0.501, 0.852, 0.823 +same, 6903148231623, 20240910-170258-38579506-3874-4d71-b9d2-ac6e47ca75dd_6903148231623, 0.255, 0.713, 0.505 +same, 6902265150022, 20240910-172010-035f68e4-9b7c-40f7-961c-aa8c0f154252_6902265150022, 0.427, 0.940, 0.823 +same, 6901668929518, 20240910-173344-258d27a2-b2e1-468e-8f32-40edcda94486_6901668929518, 0.361, 0.849, 0.721 +same, 6901668934628, 20240910-170431-722e7de7-c7ef-4825-8080-be019c7f4602_6901668934628, 0.442, 0.879, 0.689 +same, 6974158892364, 20240910-163028-418ab174-5722-4e8a-ae12-e8d3c33f70b5_6974158892364, 0.118, 0.681, 0.437 +same, 6902890218470, 20240910-164251-a2a38e17-5532-49a5-9372-5a3ed8dc6972_6902890218470, 0.281, 0.689, 0.668 +same, 6902265150022, 20240910-163814-9fc0324d-134a-46ee-bb79-6b2dfb6388f9_6902265150022, 0.306, 0.901, 0.680 +same, 6901668929518, 20240910-170417-1ac149e8-4ecb-447c-a8b7-8d5b96e77ffa_6901668929518, 0.260, 0.821, 0.586 +same, 6901668936271, 20240910-172745-96dc9808-4157-4806-856f-c7013452f302_6901668936271, 0.153, 0.609, 0.311 +same, 6903148126677, 20240910-163857-736e50b8-eae8-4a6d-af26-ce3a57a073b8_6903148126677, 0.418, 0.890, 0.749 +same, 6901668936684, 20240910-172445-4f28474f-5463-4b19-bc2d-671105764e27_6901668936684, 0.211, 0.672, 0.444 +same, 6901668936295, 20240910-172754-d034ab2f-1b18-4d6a-a936-9fa538066253_6901668936295, 0.128, 0.628, 0.251 +same, 8993175540667, 20240910-165644-2e79a878-caf1-44ca-851c-287848800d35_8993175540667, 0.565, 0.870, 0.822 +same, 6901668929518, 20240910-172039-ebd2a496-c407-4450-b122-0e8f33e07de2_6901668929518, 0.341, 0.823, 0.726 +same, 6902132084337, 20240910-162817-18813894-397a-4c94-8b90-2d7a46319793_6902132084337, 0.438, 0.795, 0.888 +same, 6904682300219, 20240910-172257-9169e95d-ff11-4d31-98af-13df3f071840_6904682300219, 0.364, 0.800, 0.643 +same, 6901668934628, 20240910-173306-a1409202-ea3d-47c4-aa39-9d17dae711cf_6901668934628, 0.487, 0.889, 0.769 +same, 6902088131437, 20240910-172427-781eb94d-efb6-403c-b88f-f4b9df82fee0_6902088131437, 0.533, 0.980, 0.827 +same, 9421903892324, 20240910-173757-b4ed1c60-a96b-48ad-a451-3caecd61c327_9421903892324, 0.420, 0.892, 0.755 +same, 6901668936684, 20240910-170907-0e74383f-0341-4b90-b333-910e5a184296_6901668936684, 0.289, 0.659, 0.568 +same, 8000500023976, 20240910-171014-ee1e7d74-0d89-4014-a125-7c9cdebb15fd_8000500023976, 0.286, 0.867, 0.660 +same, 6901668929518, 20240910-164347-47377bae-2ca6-4d75-a076-e7f6c03d0f2e_6901668929518, 0.445, 0.846, 0.833 +same, 6902265160502, 20240910-173409-55dd7611-7394-4783-9f4e-4639401078ea_6902265160502, 0.212, 0.857, 0.610 +same, 6901668936684, 20240910-165525-e17864c9-e965-4531-be14-be551dad88fb_6901668936684, 0.149, 0.612, 0.343 +same, 6901668934628, 20240910-162805-592cff06-4acb-420f-bc36-bb00f3e0efbb_6901668934628, 0.274, 0.868, 0.521 +same, 6949909050041, 20240910-172919-ab2efd9a-a776-420f-95f5-2f8188f719e4_6949909050041, 0.400, 0.845, 0.791 +same, 6907992517780, 20240910-171723-2f8a7ece-99cb-4d91-b484-67b486599f26_6907992517780, 0.391, 0.844, 0.837 +same, 6902890218470, 20240910-165443-48bad32d-9f2b-499b-907d-c602cf563ee3_6902890218470, 0.281, 0.738, 0.774 +same, 6904682300219, 20240910-165455-d0e36365-f7f2-4f2e-84a7-1ffc24ccc1c7_6904682300219, 0.423, 0.894, 0.794 +same, 6904682300226, 20240910-170231-21568a27-641b-448d-8b8c-9eff4dfe7294_6904682300226, 0.257, 0.724, 0.634 +same, 6903148048801, 20240910-163740-851d23c1-e90f-4947-abc3-f463991c5505_6903148048801, 0.421, 0.825, 0.785 +same, 6902132084337, 20240910-170730-76626a74-34fb-486d-b889-4276552edb0e_6902132084337, 0.379, 0.832, 0.794 +same, 9421903892324, 20240910-172316-ffa74ee4-46d5-4266-b362-ebfebed0c572_9421903892324, 0.301, 0.875, 0.544 +same, 6904682300219, 20240910-173807-afdeec3a-0d6e-4db8-9baf-826b7d6b4660_6904682300219, 0.254, 0.772, 0.481 +same, 6902890232216, 20240910-163838-9e6f0b38-2ffe-4727-9ec7-a02435b8f629_6902890232216, 0.264, 0.781, 0.592 +same, 6901668936295, 20240910-165424-5d55263c-e523-495e-b673-fc53eaa68b05_6901668936295, 0.138, 0.542, 0.237 +same, 6903148126677, 20240910-170403-c1b9db80-7ee0-4508-8858-1e3e1b924648_6903148126677, 0.351, 0.861, 0.603 +same, 6901668929518, 20240910-172500-509a2d1e-e665-4fe6-8ffe-b69117d7b09f_6901668929518, 0.364, 0.825, 0.731 +same, 6903148231623, 20240910-171824-2d3edfcd-c169-4c6e-9734-9325b72cf9fe_6903148231623, 0.174, 0.689, 0.357 +same, 6901668929518, 20240910-173839-e4b3b834-c695-4917-b2f4-7cfaaebb98dc_6901668929518, 0.436, 0.872, 0.772 +same, 6901668929730, 20240910-170447-3b37f76f-5e21-400b-a8a8-2376c0796ae6_6901668929730, 0.461, 0.851, 0.797 +same, 6903148080085, 20240910-173314-d6ac3740-20f2-4aa7-a392-80a96b7607c3_6903148080085, 0.369, 0.859, 0.826 +same, 6901070600142, 20240910-172734-8c23b385-99f7-4e01-819a-78c86611ff48_6901070600142, 0.200, 0.674, 0.441 +same, 6958104102516, 20240910-164452-0f365052-2e4a-4d00-9cf7-0407d731d07e_6958104102516, 0.233, 0.868, 0.582 +same, 6901070600142, 20240910-162749-ab186eb8-6777-489b-8ad0-c1c6e66b285d_6901070600142, 0.267, 0.725, 0.590 +same, 8993175537322, 20240910-164432-008357d7-7ee6-49b9-8d08-3f3a6081c4e1_8993175537322, 0.448, 0.787, 0.784 +same, 6975682480393, 20240910-163007-6dfc085b-42b9-432d-9c41-7bfd294526b6_6975682480393, 0.447, 0.832, 0.830 +same, 6903148080085, 20240910-163825-e4de18e2-fe7c-4ff6-8b51-7ef2a7db7ed3_6903148080085, 0.350, 0.836, 0.766 +same, 6903148231623, 20240910-172854-5fb70036-3089-4258-9346-de25d415f120_6903148231623, 0.422, 0.843, 0.780 +same, 6949909050041, 20240910-170817-c2f8c500-3aa5-4bd2-bf82-787d0cd22585_6949909050041, 0.493, 0.891, 0.884 +same, 6907992517780, 20240910-170246-e773b037-a712-4d78-accd-71c24b675365_6907992517780, 0.338, 0.738, 0.824 +same, 6902265160502, 20240910-163907-1ac881ec-cac4-4811-9cab-1826731e77bd_6902265160502, 0.238, 0.826, 0.706 +same, 6901668936271, 20240910-164239-e4d8f615-8cf3-483d-bc6e-03e470e2110c_6901668936271, 0.228, 0.610, 0.388 +same, 8993175537322, 20240910-172328-48a512b9-4fb1-4abf-bca9-8b3443ce8f2b_8993175537322, 0.454, 0.780, 0.718 +same, 8993175537322, 20240910-173819-226cc352-acdc-4419-9159-c97ae0eb58af_8993175537322, 0.454, 0.763, 0.718 +same, 6901668929518, 20240910-165517-a0000cdf-aa15-42c8-a6be-dbce8cf7cb32_6901668929518, 0.405, 0.855, 0.759 +same, 8000500023976, 20240910-172417-e9d563b9-74e2-4ec1-8f34-331424b48e72_8000500023976, 0.347, 0.851, 0.684 +diff, 8993175537322, 20240910-173355-1bbf290e-1f14-4ba8-b666-82c990c4eea3_6901668936684, 0.016, 0.342, 0.029 +diff, 6904682300226, 20240910-173847-9eedb2ac-e3a5-4d07-94fe-f7e881d67418_6902088131437, 0.006, 0.344, 0.011 +diff, 8993175540667, 20240910-171800-76a062fd-409c-480f-94f4-fd0e65d72467_6904682300226, 0.038, 0.307, 0.066 +diff, 6901668934628, 20240910-172352-9b79a4d9-092f-477d-a7a4-8af079d1538d_6970399922365, -0.003, 0.305, -0.006 +diff, 6901668929518, 20240910-170331-e3ee7cf5-dda2-4d0b-b8c9-4fb411fe78ec_6902265202318, -0.022, 0.268, -0.036 +diff, 6903148080085, 20240910-163802-6b9f0129-8497-467f-a506-5708eda436a4_6907992517780, 0.026, 0.413, 0.060 +diff, 6970399922365, 20240910-172403-dbc9de02-2811-449c-961f-23e7a16877d7_6902132084337, 0.090, 0.478, 0.206 +diff, 6904682300226, 20240910-164315-38c640ba-cdf3-4ac1-8bff-55fe5d0560bb_6901668934888, 0.071, 0.384, 0.141 +diff, 6974158892364, 20240910-173323-78dc658e-e4ef-49e1-a2ff-9ada34c27a85_8000500023976, -0.044, 0.335, -0.118 +diff, 6901668934888, 20240910-164323-8e9a882a-a502-4a6e-bd99-70deb2130f57_6904682300219, -0.016, 0.459, -0.041 +diff, 6907992517780, 20240910-163750-8e13e800-21d0-4bd9-b686-18ed213460cd_6903148231623, 0.018, 0.399, 0.049 +diff, 6901668934628, 20240910-170920-dc16c149-06a3-4c2d-9bec-e930274b55ce_6904682300219, 0.033, 0.332, 0.062 +diff, 6901668936684, 20240910-172802-0dbe3709-bd0c-45e7-ad36-0cfc9781ef1b_6902890218470, -0.072, 0.269, -0.162 +diff, 6907992517780, 20240910-165620-0b870f0d-88a5-4286-bcbf-b0ebb41ddcfc_6901668934888, 0.141, 0.457, 0.292 +diff, 6958104102516, 20240910-163846-7793e886-9f09-4744-9e24-eb47d65c09f5_6902088131437, -0.023, 0.370, -0.056 +diff, 8993175537322, 20240910-170742-f78b59da-e242-42c9-ac7a-bba23ff11aff_6901070600142, -0.017, 0.289, -0.030 +diff, 6903148126677, 20240910-172814-d17bd016-b8e5-4a21-a137-6bce693e0cb0_8993175540667, -0.044, 0.360, -0.083 +diff, 8993175540667, 20240910-162930-ec2bb380-53fe-483f-9aab-9038643ebd1f_6901668929730, -0.021, 0.359, -0.031 +diff, 9421903892324, 20240910-173332-55f8124d-7ab0-4a7a-8b08-f4dd9ba06502_6970399922365, 0.033, 0.376, 0.063 +diff, 6902890232216, 20240910-173214-5b86868f-cb5b-4b7f-8f3a-aff08d89900d_6901668929730, 0.075, 0.424, 0.151 +diff, 6903148231623, 20240910-172904-5462ad91-2a07-4116-898f-ff1d2021e6af_6903148048801, 0.013, 0.311, 0.021 +diff, 6924743915848, 20240910-171838-c77a6d0d-185b-48e7-9af9-05de561f1172_6901668934628, -0.069, 0.327, -0.147 +diff, 6975682480393, 20240910-170934-74c137ee-0689-42d0-9994-da8ba59fd5db_6902890218470, 0.121, 0.624, 0.273 +diff, 6975682480393, 20240910-162952-f6ec3a40-9d64-4f20-b122-0b81eb4a2134_6949909050041, 0.092, 0.646, 0.185 +diff, 6907992517780, 20240910-172841-9d7b16fb-4200-4089-b4b2-925da10208ed_6901668934888, 0.158, 0.637, 0.344 +diff, 6902265202318, 20240910-165632-a1e22655-d9ad-47f5-a467-55718bd1e23e_6901668936271, -0.006, 0.285, -0.013 +diff, 6902890232216, 20240910-163718-e1e09ad9-7a7e-4b43-beb7-47080c0a312e_6904682300226, 0.066, 0.486, 0.157 +diff, 9421903892324, 20240910-173233-81246d1d-bbf3-4ee2-b6c1-7f8fe5818266_6903148126677, -0.038, 0.451, -0.061 +diff, 6902132084337, 20240910-162836-186bdf15-5ebb-4b55-a3a4-47edea86a7ee_6924743915848, -0.060, 0.275, -0.123 +diff, 9421903892324, 20240910-173222-8abca736-4b5d-4b8e-8e53-206809f37082_6902132084337, 0.105, 0.456, 0.213 +diff, 6901668934628, 20240910-170945-c5a587f8-925a-46c2-b2f4-b8fe0fa41c90_8993175537322, -0.088, 0.187, -0.148 +diff, 6901668934888, 20240910-162848-b0d67358-6f68-482a-94cb-d7de7414e32f_6902265202318, -0.038, 0.355, -0.088 +diff, 6902265202318, 20240910-173730-c51d9d00-65a2-4212-99f3-701092810919_6907992517780, 0.026, 0.318, 0.044 +diff, 6902890232216, 20240910-170318-706146af-c203-459a-b642-da428ce6426a_6902265160502, 0.077, 0.548, 0.244 +diff, 6903148126677, 20240910-162902-3de7f2a9-9068-4f61-a150-0bcc47194a43_6903148347409, -0.047, 0.245, -0.115 +diff, 6903148347409, 20240910-172023-a9b8c8b4-8030-4aa5-85fe-54cba57e745f_6902265202318, 0.019, 0.319, 0.050 +diff, 6904682300219, 20240910-171920-0a6490ce-547f-493d-b76a-4c849ae12a93_6903148126677, 0.022, 0.342, 0.033 +diff, 6974158892364, 20240910-164334-09d4e20e-68c8-48ca-b931-50e58428ef2a_6901668936295, 0.035, 0.449, 0.109 +diff, 6901070600142, 20240910-165604-0f805f9d-24f7-4729-923a-bff489a09323_6958104102516, 0.016, 0.382, 0.041 +diff, 6901668934628, 20240910-164409-053f810b-7369-4a3e-b91b-b7ba99fa5b9c_6901668936684, -0.046, 0.570, -0.081 +diff, 6924743915848, 20240910-170349-b357333c-e939-4ce5-8019-7762799a9097_6902265150022, -0.097, 0.344, -0.250 +diff, 6903148126677, 20240910-172828-0a20bffd-ede3-4b0c-977b-8652f52518f9_6902890232216, 0.051, 0.327, 0.109 +diff, 6904682300226, 20240910-170807-7bc77832-4cf1-4cd8-aa54-994ff164dcc7_6902890232216, 0.062, 0.423, 0.113 +diff, 9421903892324, 20240910-171715-a8fc6d8a-87bd-4fbd-b378-85e34193266f_6901668929730, -0.067, 0.299, -0.108 +diff, 6901668936684, 20240910-170258-38579506-3874-4d71-b9d2-ac6e47ca75dd_6903148231623, 0.062, 0.401, 0.131 +diff, 6970399922365, 20240910-172010-035f68e4-9b7c-40f7-961c-aa8c0f154252_6902265150022, -0.043, 0.358, -0.099 +diff, 6903148048801, 20240910-173344-258d27a2-b2e1-468e-8f32-40edcda94486_6901668929518, 0.079, 0.502, 0.151 +diff, 6901668934888, 20240910-170431-722e7de7-c7ef-4825-8080-be019c7f4602_6901668934628, 0.001, 0.440, 0.001 +diff, 6970399922365, 20240910-163028-418ab174-5722-4e8a-ae12-e8d3c33f70b5_6974158892364, 0.071, 0.539, 0.207 +diff, 6975682480393, 20240910-164251-a2a38e17-5532-49a5-9372-5a3ed8dc6972_6902890218470, 0.112, 0.662, 0.232 +diff, 6901668929518, 20240910-163814-9fc0324d-134a-46ee-bb79-6b2dfb6388f9_6902265150022, -0.067, 0.361, -0.147 +diff, 6901070600142, 20240910-170417-1ac149e8-4ecb-447c-a8b7-8d5b96e77ffa_6901668929518, -0.033, 0.302, -0.086 +diff, 6903148126677, 20240910-172745-96dc9808-4157-4806-856f-c7013452f302_6901668936271, 0.007, 0.366, 0.016 +diff, 6903148347409, 20240910-163857-736e50b8-eae8-4a6d-af26-ce3a57a073b8_6903148126677, -0.007, 0.349, -0.017 +diff, 6901668936271, 20240910-172445-4f28474f-5463-4b19-bc2d-671105764e27_6901668936684, 0.062, 0.556, 0.123 +diff, 6901070600142, 20240910-172754-d034ab2f-1b18-4d6a-a936-9fa538066253_6901668936295, 0.188, 0.602, 0.448 +diff, 6902265150022, 20240910-165644-2e79a878-caf1-44ca-851c-287848800d35_8993175540667, 0.037, 0.303, 0.064 +diff, 6901668934888, 20240910-172039-ebd2a496-c407-4450-b122-0e8f33e07de2_6901668929518, 0.048, 0.374, 0.114 +diff, 6958104102516, 20240910-162817-18813894-397a-4c94-8b90-2d7a46319793_6902132084337, -0.068, 0.248, -0.130 +diff, 6902265160502, 20240910-172257-9169e95d-ff11-4d31-98af-13df3f071840_6904682300219, 0.046, 0.473, 0.106 +diff, 6970399922365, 20240910-173306-a1409202-ea3d-47c4-aa39-9d17dae711cf_6901668934628, 0.022, 0.368, 0.047 +diff, 6902265202318, 20240910-172427-781eb94d-efb6-403c-b88f-f4b9df82fee0_6902088131437, 0.016, 0.316, 0.029 +diff, 6907992517780, 20240910-173757-b4ed1c60-a96b-48ad-a451-3caecd61c327_9421903892324, 0.118, 0.554, 0.257 +diff, 6901668936271, 20240910-170907-0e74383f-0341-4b90-b333-910e5a184296_6901668936684, 0.136, 0.493, 0.253 +diff, 6901668934628, 20240910-171014-ee1e7d74-0d89-4014-a125-7c9cdebb15fd_8000500023976, 0.061, 0.321, 0.135 +diff, 6903148126677, 20240910-164347-47377bae-2ca6-4d75-a076-e7f6c03d0f2e_6901668929518, -0.026, 0.325, -0.048 +diff, 6903148048801, 20240910-173409-55dd7611-7394-4783-9f4e-4639401078ea_6902265160502, 0.030, 0.373, 0.071 +diff, 6902132084337, 20240910-165525-e17864c9-e965-4531-be14-be551dad88fb_6901668936684, 0.045, 0.370, 0.116 +diff, 6902890232216, 20240910-162805-592cff06-4acb-420f-bc36-bb00f3e0efbb_6901668934628, -0.066, 0.262, -0.161 +diff, 6903148048801, 20240910-172919-ab2efd9a-a776-420f-95f5-2f8188f719e4_6949909050041, 0.118, 0.403, 0.235 +diff, 6970399922365, 20240910-171723-2f8a7ece-99cb-4d91-b484-67b486599f26_6907992517780, -0.043, 0.340, -0.102 +diff, 6903148048801, 20240910-165443-48bad32d-9f2b-499b-907d-c602cf563ee3_6902890218470, -0.002, 0.480, -0.004 +diff, 6904682300226, 20240910-165455-d0e36365-f7f2-4f2e-84a7-1ffc24ccc1c7_6904682300219, 0.270, 0.808, 0.584 +diff, 6901668936271, 20240910-170231-21568a27-641b-448d-8b8c-9eff4dfe7294_6904682300226, 0.025, 0.367, 0.055 +diff, 6949909050041, 20240910-163740-851d23c1-e90f-4947-abc3-f463991c5505_6903148048801, 0.102, 0.445, 0.189 +diff, 6902890232216, 20240910-170730-76626a74-34fb-486d-b889-4276552edb0e_6902132084337, -0.019, 0.245, -0.041 +diff, 6924743915848, 20240910-172316-ffa74ee4-46d5-4266-b362-ebfebed0c572_9421903892324, 0.078, 0.441, 0.186 +diff, 6901070600142, 20240910-173807-afdeec3a-0d6e-4db8-9baf-826b7d6b4660_6904682300219, 0.009, 0.483, 0.021 +diff, 6924743915848, 20240910-163838-9e6f0b38-2ffe-4727-9ec7-a02435b8f629_6902890232216, -0.025, 0.388, -0.059 +diff, 6902265160502, 20240910-165424-5d55263c-e523-495e-b673-fc53eaa68b05_6901668936295, -0.041, 0.280, -0.087 +diff, 6902088131437, 20240910-170403-c1b9db80-7ee0-4508-8858-1e3e1b924648_6903148126677, -0.019, 0.230, -0.025 +diff, 6903148080085, 20240910-172500-509a2d1e-e665-4fe6-8ffe-b69117d7b09f_6901668929518, 0.064, 0.492, 0.136 +diff, 6901668934888, 20240910-171824-2d3edfcd-c169-4c6e-9734-9325b72cf9fe_6903148231623, 0.014, 0.328, 0.034 +diff, 6901668929730, 20240910-173839-e4b3b834-c695-4917-b2f4-7cfaaebb98dc_6901668929518, -0.066, 0.278, -0.107 +diff, 6901070600142, 20240910-170447-3b37f76f-5e21-400b-a8a8-2376c0796ae6_6901668929730, -0.069, 0.411, -0.149 +diff, 6974158892364, 20240910-173314-d6ac3740-20f2-4aa7-a392-80a96b7607c3_6903148080085, -0.034, 0.302, -0.111 +diff, 6901668936295, 20240910-172734-8c23b385-99f7-4e01-819a-78c86611ff48_6901070600142, 0.007, 0.424, 0.013 +diff, 6975682480393, 20240910-164452-0f365052-2e4a-4d00-9cf7-0407d731d07e_6958104102516, 0.030, 0.402, 0.074 +diff, 6903148080085, 20240910-162749-ab186eb8-6777-489b-8ad0-c1c6e66b285d_6901070600142, -0.014, 0.308, -0.027 +diff, 6901668929730, 20240910-164432-008357d7-7ee6-49b9-8d08-3f3a6081c4e1_8993175537322, 0.020, 0.298, 0.036 +diff, 6902890218470, 20240910-163007-6dfc085b-42b9-432d-9c41-7bfd294526b6_6975682480393, 0.185, 0.635, 0.394 +diff, 6902890232216, 20240910-163825-e4de18e2-fe7c-4ff6-8b51-7ef2a7db7ed3_6903148080085, 0.024, 0.351, 0.056 +diff, 6902890232216, 20240910-172854-5fb70036-3089-4258-9346-de25d415f120_6903148231623, -0.079, 0.318, -0.181 +diff, 6901668936271, 20240910-170817-c2f8c500-3aa5-4bd2-bf82-787d0cd22585_6949909050041, -0.011, 0.325, -0.019 +diff, 6902265160502, 20240910-170246-e773b037-a712-4d78-accd-71c24b675365_6907992517780, -0.094, 0.362, -0.245 +diff, 6902132084337, 20240910-163907-1ac881ec-cac4-4811-9cab-1826731e77bd_6902265160502, -0.008, 0.321, -0.022 +diff, 6970399922365, 20240910-164239-e4d8f615-8cf3-483d-bc6e-03e470e2110c_6901668936271, 0.047, 0.366, 0.105 +diff, 6904682300219, 20240910-172328-48a512b9-4fb1-4abf-bca9-8b3443ce8f2b_8993175537322, -0.012, 0.472, -0.019 +diff, 6901668936271, 20240910-173819-226cc352-acdc-4419-9159-c97ae0eb58af_8993175537322, 0.010, 0.340, 0.016 +diff, 6901668936271, 20240910-165517-a0000cdf-aa15-42c8-a6be-dbce8cf7cb32_6901668929518, 0.090, 0.480, 0.155 +diff, 6901668929730, 20240910-172417-e9d563b9-74e2-4ec1-8f34-331424b48e72_8000500023976, 0.108, 0.463, 0.214 diff --git a/contrast/result/record/20240907_142813.txt b/contrast/result/record/20240907_142813.txt new file mode 100644 index 0000000..ac84cad --- /dev/null +++ b/contrast/result/record/20240907_142813.txt @@ -0,0 +1,124 @@ +same, 6954256905319, 20240715-144627_6954256905319, 0.211, 0.750, 0.630 +same, 6930364600025, 20240719-143603_6930364600025, 0.283, 0.771, 0.616 +same, 6924509902839, 20240719-143630_6924509902839, 0.182, 0.727, 0.462 +same, 6902007115524, 20240719-143729_6902007115524, 0.221, 0.723, 0.519 +same, 6930628801472, 20240719-143807_6930628801472, 0.249, 0.804, 0.672 +same, 6908355010450, 20240719-144335_6908355010450, 0.255, 0.812, 0.575 +same, 6924509902754, 20240719-144454_6924509902754, 0.458, 0.882, 0.836 +same, 6925843410158, 20240719-144916_6925843410158, 0.229, 0.829, 0.686 +same, 6941499112529, 20240719-145022_6941499112529, 0.190, 0.795, 0.359 +same, 6918149576536, 20240719-145058_6918149576536, 0.493, 0.906, 0.765 +same, 6928002374551, 20240719-145542_6928002374551, 0.165, 0.773, 0.531 +same, 6930364600001, 20240719-150000_6930364600001, 0.322, 0.831, 0.736 +same, 6937962111526, 20240719-150209_6937962111526, 0.225, 0.794, 0.605 +same, 6924509902839, 20240719-150345_6924509902839, 0.161, 0.816, 0.502 +same, 6924509902839, 20240719-150458_6924509902839, 0.319, 0.746, 0.623 +same, 6924187895423, 20240719-150623_6924187895423, 0.430, 0.906, 0.903 +same, 6916749306102, 20240719-150716_6916749306102, 0.269, 0.764, 0.737 +same, 6951469100067, 20240719-150750_6951469100067, 0.547, 0.938, 0.896 +same, 6958350501101, 20240719-151215_6958350501101, 0.578, 0.906, 0.877 +same, 6951469100067, 20240719-151325_6951469100067, 0.610, 0.930, 0.904 +same, 6937962111526, 20240719-151349_6937962111526, 0.138, 0.762, 0.496 +same, 6916749306102, 20240719-151416_6916749306102, 0.243, 0.728, 0.630 +same, 6930364600001, 20240719-151535_6930364600001, 0.144, 0.787, 0.482 +same, 6911988027191, 20240719-151724_6911988027191, 0.035, 0.502, 0.076 +same, 6972219890632, 20240719-151800_6972219890632, 0.315, 0.880, 0.700 +same, 6901668935908, 20240719-152125_6901668935908, 0.203, 0.856, 0.426 +same, 6930364600001, 20240719-153117_6930364600001, 0.057, 0.775, 0.165 +same, 6902265587712, 20240722-144630_6902265587712, 0.364, 0.833, 0.669 +same, 4897073991658, 20240722-160713_4897073991658, 0.101, 0.514, 0.247 +same, 4892655902237, 20240722-160857_4892655902237, 0.459, 0.760, 0.812 +same, 6920907803020, 20240722-161533_6920907803020, 0.211, 0.769, 0.678 +same, 6920912342002, 20240722-163428_6920912342002, 0.281, 0.825, 0.853 +same, 6916196422325, 20240722-164859_6916196422325, 0.392, 0.837, 0.665 +same, 6901668006011, 20240722-165548_6901668006011, 0.284, 0.752, 0.530 +same, 6937221716189, 20240723-094426_6937221716189, 0.407, 0.850, 0.705 +same, 4897098800133, 20240723-094447_4897098800133, 0.382, 0.846, 0.784 +same, 6970470331529, 20240723-094617_6970470331529, 0.238, 0.925, 0.586 +same, 6918598120014, 20240723-095103_6918598120014, 0.407, 0.905, 0.819 +same, 6938737200650, 20240723-095527_6938737200650, 0.208, 0.825, 0.543 +same, 6902890022558, 20240723-095858_6902890022558, 0.162, 0.769, 0.628 +same, 6925303795535, 20240723-100031_6925303795535, 0.304, 0.835, 0.671 +same, 6920382723257, 20240723-100113_6920382723257, 0.274, 0.854, 0.539 +same, 6938737200650, 20240723-100416_6938737200650, 0.237, 0.719, 0.581 +same, 6918598120014, 20240723-100501_6918598120014, 0.099, 0.791, 0.209 +same, 6937221716189, 20240723-100726_6937221716189, 0.347, 0.771, 0.611 +same, 4897098800133, 20240723-100845_4897098800133, 0.304, 0.770, 0.718 +same, 6918598120014, 20240723-101829_6918598120014, 0.502, 0.919, 0.857 +same, 6920382723257, 20240723-102033_6920382723257, 0.343, 0.849, 0.748 +same, 6902890022558, 20240723-102206_6902890022558, 0.117, 0.745, 0.389 +same, 6970470331529, 20240723-102409_6970470331529, 0.102, 0.734, 0.305 +same, 6938737200650, 20240723-102515_6938737200650, 0.183, 0.725, 0.469 +same, 6937221716189, 20240723-102833_6937221716189, 0.067, 0.623, 0.175 +same, 6937451883132, 20240723-110216_6937451883132, 0.323, 0.860, 0.717 +same, 6973337010216, 20240723-110224_6973337010216, 0.202, 0.760, 0.622 +same, 6937451883132, 20240723-111446_6937451883132, 0.367, 0.817, 0.806 +same, 6973337010216, 20240723-111521_6973337010216, 0.140, 0.777, 0.414 +same, 6924187828544, 20240723-111602_6924187828544, 0.359, 0.859, 0.718 +same, 6934024590268, 20240723-112004_6934024590268, 0.184, 0.727, 0.583 +same, 6924187828544, 20240723-112505_6924187828544, 0.281, 0.875, 0.573 +same, 6974158894528, 20240723-155423_6974158894528, 0.433, 0.890, 0.788 +same, 6970399922518, 20240723-160227_6970399922518, 0.078, 0.564, 0.215 +same, 6974158894528, 20240723-160942_6974158894528, 0.417, 0.895, 0.832 +diff, 6970470331529, 20240715-144627_6954256905319, -0.080, 0.368, -0.185 +diff, 6908355010450, 20240719-143603_6930364600025, 0.103, 0.460, 0.201 +diff, 6925843410158, 20240719-143630_6924509902839, -0.080, 0.436, -0.250 +diff, 6925843410158, 20240719-143729_6902007115524, 0.033, 0.522, 0.092 +diff, 6924509902839, 20240719-143807_6930628801472, 0.067, 0.524, 0.144 +diff, 6924187828544, 20240719-144335_6908355010450, -0.064, 0.417, -0.152 +diff, 6972219890632, 20240719-144454_6924509902754, 0.093, 0.537, 0.174 +diff, 6916749306102, 20240719-144916_6925843410158, -0.008, 0.440, -0.022 +diff, 6934024590268, 20240719-145022_6941499112529, -0.097, 0.349, -0.259 +diff, 6902007115524, 20240719-145058_6918149576536, -0.029, 0.393, -0.059 +diff, 6901668006011, 20240719-145542_6928002374551, -0.034, 0.359, -0.083 +diff, 6928002374551, 20240719-150000_6930364600001, -0.069, 0.426, -0.180 +diff, 6924509902839, 20240719-150209_6937962111526, -0.084, 0.357, -0.165 +diff, 6916749306102, 20240719-150345_6924509902839, 0.125, 0.664, 0.441 +diff, 6972219890632, 20240719-150458_6924509902839, 0.143, 0.548, 0.272 +diff, 6970399922518, 20240719-150623_6924187895423, 0.023, 0.438, 0.062 +diff, 6937962111526, 20240719-150716_6916749306102, 0.002, 0.433, 0.008 +diff, 6916749306102, 20240719-150750_6951469100067, 0.139, 0.508, 0.335 +diff, 6901668006011, 20240719-151215_6958350501101, 0.006, 0.406, 0.011 +diff, 6920912342002, 20240719-151325_6951469100067, 0.027, 0.444, 0.063 +diff, 6920382723257, 20240719-151349_6937962111526, 0.001, 0.338, 0.003 +diff, 6925843410158, 20240719-151416_6916749306102, -0.033, 0.431, -0.092 +diff, 6901668006011, 20240719-151535_6930364600001, -0.014, 0.367, -0.041 +diff, 4897098800133, 20240719-151724_6911988027191, 0.025, 0.369, 0.049 +diff, 6937962111526, 20240719-151800_6972219890632, 0.037, 0.447, 0.114 +diff, 6925303795535, 20240719-152125_6901668935908, 0.040, 0.409, 0.096 +diff, 6937221716189, 20240719-153117_6930364600001, -0.072, 0.387, -0.182 +diff, 6951469100067, 20240722-144630_6902265587712, 0.039, 0.409, 0.064 +diff, 6902007115524, 20240722-160713_4897073991658, -0.055, 0.338, -0.141 +diff, 6951469100067, 20240722-160857_4892655902237, -0.006, 0.354, -0.009 +diff, 6954256905319, 20240722-161533_6920907803020, 0.078, 0.658, 0.268 +diff, 6924509902839, 20240722-163428_6920912342002, 0.011, 0.485, 0.028 +diff, 6930364600025, 20240722-164859_6916196422325, 0.003, 0.507, 0.008 +diff, 6930364600001, 20240722-165548_6901668006011, 0.012, 0.426, 0.026 +diff, 4892655902237, 20240723-094426_6937221716189, -0.060, 0.221, -0.095 +diff, 6954256905319, 20240723-094447_4897098800133, 0.029, 0.492, 0.076 +diff, 6925843410158, 20240723-094617_6970470331529, -0.020, 0.361, -0.059 +diff, 6970399922518, 20240723-095103_6918598120014, -0.064, 0.356, -0.154 +diff, 6916196422325, 20240723-095527_6938737200650, -0.051, 0.397, -0.087 +diff, 6920907803020, 20240723-095858_6902890022558, -0.009, 0.418, -0.027 +diff, 6924187895423, 20240723-100031_6925303795535, -0.059, 0.334, -0.112 +diff, 6916196422325, 20240723-100113_6920382723257, 0.055, 0.500, 0.088 +diff, 6901668006011, 20240723-100416_6938737200650, -0.051, 0.309, -0.100 +diff, 6941499112529, 20240723-100501_6918598120014, 0.076, 0.510, 0.157 +diff, 6951469100067, 20240723-100726_6937221716189, -0.036, 0.320, -0.052 +diff, 6970399922518, 20240723-100845_4897098800133, -0.000, 0.440, -0.001 +diff, 6930364600025, 20240723-101829_6918598120014, -0.072, 0.389, -0.132 +diff, 6902890022558, 20240723-102033_6920382723257, -0.079, 0.458, -0.279 +diff, 6954256905319, 20240723-102206_6902890022558, 0.007, 0.479, 0.021 +diff, 6930364600025, 20240723-102409_6970470331529, 0.014, 0.502, 0.043 +diff, 6920382723257, 20240723-102515_6938737200650, 0.057, 0.424, 0.117 +diff, 6951469100067, 20240723-102833_6937221716189, 0.069, 0.453, 0.146 +diff, 6924187828544, 20240723-110216_6937451883132, 0.026, 0.358, 0.053 +diff, 6908355010450, 20240723-110224_6973337010216, -0.060, 0.352, -0.134 +diff, 6937962111526, 20240723-111446_6937451883132, 0.019, 0.431, 0.053 +diff, 6970470331529, 20240723-111521_6973337010216, 0.007, 0.410, 0.016 +diff, 6918598120014, 20240723-111602_6924187828544, -0.004, 0.454, -0.008 +diff, 6901668935908, 20240723-112004_6934024590268, -0.084, 0.389, -0.184 +diff, 6911988027191, 20240723-112505_6924187828544, 0.062, 0.452, 0.139 +diff, 6911988027191, 20240723-155423_6974158894528, -0.016, 0.396, -0.029 +diff, 4897073991658, 20240723-160227_6970399922518, 0.023, 0.295, 0.052 +diff, 4897073991658, 20240723-160942_6974158894528, -0.098, 0.247, -0.176 diff --git a/contrast/result/record/20240907_142813_ft16.txt b/contrast/result/record/20240907_142813_ft16.txt new file mode 100644 index 0000000..29c93f5 --- /dev/null +++ b/contrast/result/record/20240907_142813_ft16.txt @@ -0,0 +1,124 @@ +same, 6954256905319, 20240715-144627_6954256905319, 0.211, 0.751, 0.630 +same, 6930364600025, 20240719-143603_6930364600025, 0.283, 0.771, 0.616 +same, 6924509902839, 20240719-143630_6924509902839, 0.182, 0.727, 0.462 +same, 6902007115524, 20240719-143729_6902007115524, 0.221, 0.723, 0.519 +same, 6930628801472, 20240719-143807_6930628801472, 0.249, 0.804, 0.672 +same, 6908355010450, 20240719-144335_6908355010450, 0.255, 0.812, 0.575 +same, 6924509902754, 20240719-144454_6924509902754, 0.458, 0.882, 0.836 +same, 6925843410158, 20240719-144916_6925843410158, 0.229, 0.829, 0.686 +same, 6941499112529, 20240719-145022_6941499112529, 0.190, 0.795, 0.359 +same, 6918149576536, 20240719-145058_6918149576536, 0.493, 0.906, 0.765 +same, 6928002374551, 20240719-145542_6928002374551, 0.165, 0.773, 0.531 +same, 6930364600001, 20240719-150000_6930364600001, 0.322, 0.831, 0.736 +same, 6937962111526, 20240719-150209_6937962111526, 0.225, 0.794, 0.605 +same, 6924509902839, 20240719-150345_6924509902839, 0.161, 0.816, 0.502 +same, 6924509902839, 20240719-150458_6924509902839, 0.319, 0.746, 0.623 +same, 6924187895423, 20240719-150623_6924187895423, 0.430, 0.906, 0.903 +same, 6916749306102, 20240719-150716_6916749306102, 0.269, 0.764, 0.737 +same, 6951469100067, 20240719-150750_6951469100067, 0.547, 0.938, 0.896 +same, 6958350501101, 20240719-151215_6958350501101, 0.578, 0.906, 0.877 +same, 6951469100067, 20240719-151325_6951469100067, 0.610, 0.930, 0.904 +same, 6937962111526, 20240719-151349_6937962111526, 0.138, 0.762, 0.496 +same, 6916749306102, 20240719-151416_6916749306102, 0.243, 0.728, 0.630 +same, 6930364600001, 20240719-151535_6930364600001, 0.144, 0.787, 0.482 +same, 6911988027191, 20240719-151724_6911988027191, 0.035, 0.502, 0.076 +same, 6972219890632, 20240719-151800_6972219890632, 0.315, 0.880, 0.700 +same, 6901668935908, 20240719-152125_6901668935908, 0.203, 0.856, 0.426 +same, 6930364600001, 20240719-153117_6930364600001, 0.057, 0.775, 0.165 +same, 6902265587712, 20240722-144630_6902265587712, 0.364, 0.833, 0.669 +same, 4897073991658, 20240722-160713_4897073991658, 0.101, 0.514, 0.247 +same, 4892655902237, 20240722-160857_4892655902237, 0.459, 0.760, 0.812 +same, 6920907803020, 20240722-161533_6920907803020, 0.211, 0.769, 0.678 +same, 6920912342002, 20240722-163428_6920912342002, 0.281, 0.825, 0.853 +same, 6916196422325, 20240722-164859_6916196422325, 0.392, 0.837, 0.665 +same, 6901668006011, 20240722-165548_6901668006011, 0.284, 0.751, 0.530 +same, 6937221716189, 20240723-094426_6937221716189, 0.407, 0.850, 0.705 +same, 4897098800133, 20240723-094447_4897098800133, 0.382, 0.846, 0.784 +same, 6970470331529, 20240723-094617_6970470331529, 0.238, 0.925, 0.586 +same, 6918598120014, 20240723-095103_6918598120014, 0.407, 0.905, 0.819 +same, 6938737200650, 20240723-095527_6938737200650, 0.208, 0.825, 0.543 +same, 6902890022558, 20240723-095858_6902890022558, 0.162, 0.769, 0.628 +same, 6925303795535, 20240723-100031_6925303795535, 0.304, 0.835, 0.671 +same, 6920382723257, 20240723-100113_6920382723257, 0.274, 0.854, 0.539 +same, 6938737200650, 20240723-100416_6938737200650, 0.237, 0.719, 0.581 +same, 6918598120014, 20240723-100501_6918598120014, 0.099, 0.791, 0.209 +same, 6937221716189, 20240723-100726_6937221716189, 0.347, 0.771, 0.611 +same, 4897098800133, 20240723-100845_4897098800133, 0.304, 0.770, 0.717 +same, 6918598120014, 20240723-101829_6918598120014, 0.502, 0.919, 0.857 +same, 6920382723257, 20240723-102033_6920382723257, 0.343, 0.849, 0.748 +same, 6902890022558, 20240723-102206_6902890022558, 0.117, 0.745, 0.389 +same, 6970470331529, 20240723-102409_6970470331529, 0.102, 0.734, 0.305 +same, 6938737200650, 20240723-102515_6938737200650, 0.183, 0.725, 0.469 +same, 6937221716189, 20240723-102833_6937221716189, 0.067, 0.623, 0.175 +same, 6937451883132, 20240723-110216_6937451883132, 0.323, 0.860, 0.717 +same, 6973337010216, 20240723-110224_6973337010216, 0.202, 0.760, 0.622 +same, 6937451883132, 20240723-111446_6937451883132, 0.367, 0.817, 0.806 +same, 6973337010216, 20240723-111521_6973337010216, 0.140, 0.777, 0.414 +same, 6924187828544, 20240723-111602_6924187828544, 0.359, 0.859, 0.718 +same, 6934024590268, 20240723-112004_6934024590268, 0.184, 0.727, 0.583 +same, 6924187828544, 20240723-112505_6924187828544, 0.281, 0.875, 0.573 +same, 6974158894528, 20240723-155423_6974158894528, 0.433, 0.890, 0.788 +same, 6970399922518, 20240723-160227_6970399922518, 0.078, 0.564, 0.215 +same, 6974158894528, 20240723-160942_6974158894528, 0.417, 0.895, 0.832 +diff, 6970470331529, 20240715-144627_6954256905319, -0.080, 0.368, -0.185 +diff, 6908355010450, 20240719-143603_6930364600025, 0.103, 0.460, 0.201 +diff, 6925843410158, 20240719-143630_6924509902839, -0.080, 0.436, -0.250 +diff, 6925843410158, 20240719-143729_6902007115524, 0.033, 0.522, 0.092 +diff, 6924509902839, 20240719-143807_6930628801472, 0.067, 0.524, 0.144 +diff, 6924187828544, 20240719-144335_6908355010450, -0.064, 0.417, -0.152 +diff, 6972219890632, 20240719-144454_6924509902754, 0.093, 0.537, 0.174 +diff, 6916749306102, 20240719-144916_6925843410158, -0.008, 0.440, -0.022 +diff, 6934024590268, 20240719-145022_6941499112529, -0.097, 0.349, -0.259 +diff, 6902007115524, 20240719-145058_6918149576536, -0.029, 0.393, -0.058 +diff, 6901668006011, 20240719-145542_6928002374551, -0.034, 0.359, -0.083 +diff, 6928002374551, 20240719-150000_6930364600001, -0.069, 0.426, -0.180 +diff, 6924509902839, 20240719-150209_6937962111526, -0.084, 0.357, -0.165 +diff, 6916749306102, 20240719-150345_6924509902839, 0.125, 0.664, 0.441 +diff, 6972219890632, 20240719-150458_6924509902839, 0.143, 0.548, 0.272 +diff, 6970399922518, 20240719-150623_6924187895423, 0.023, 0.438, 0.062 +diff, 6937962111526, 20240719-150716_6916749306102, 0.002, 0.433, 0.008 +diff, 6916749306102, 20240719-150750_6951469100067, 0.139, 0.508, 0.335 +diff, 6901668006011, 20240719-151215_6958350501101, 0.006, 0.406, 0.011 +diff, 6920912342002, 20240719-151325_6951469100067, 0.027, 0.444, 0.063 +diff, 6920382723257, 20240719-151349_6937962111526, 0.001, 0.338, 0.003 +diff, 6925843410158, 20240719-151416_6916749306102, -0.033, 0.431, -0.092 +diff, 6901668006011, 20240719-151535_6930364600001, -0.014, 0.367, -0.041 +diff, 4897098800133, 20240719-151724_6911988027191, 0.025, 0.369, 0.049 +diff, 6937962111526, 20240719-151800_6972219890632, 0.037, 0.447, 0.114 +diff, 6925303795535, 20240719-152125_6901668935908, 0.040, 0.409, 0.096 +diff, 6937221716189, 20240719-153117_6930364600001, -0.072, 0.387, -0.182 +diff, 6951469100067, 20240722-144630_6902265587712, 0.039, 0.409, 0.064 +diff, 6902007115524, 20240722-160713_4897073991658, -0.055, 0.338, -0.141 +diff, 6951469100067, 20240722-160857_4892655902237, -0.006, 0.354, -0.009 +diff, 6954256905319, 20240722-161533_6920907803020, 0.078, 0.658, 0.268 +diff, 6924509902839, 20240722-163428_6920912342002, 0.011, 0.485, 0.028 +diff, 6930364600025, 20240722-164859_6916196422325, 0.003, 0.507, 0.008 +diff, 6930364600001, 20240722-165548_6901668006011, 0.012, 0.425, 0.026 +diff, 4892655902237, 20240723-094426_6937221716189, -0.060, 0.221, -0.095 +diff, 6954256905319, 20240723-094447_4897098800133, 0.029, 0.492, 0.076 +diff, 6925843410158, 20240723-094617_6970470331529, -0.020, 0.361, -0.059 +diff, 6970399922518, 20240723-095103_6918598120014, -0.064, 0.356, -0.154 +diff, 6916196422325, 20240723-095527_6938737200650, -0.051, 0.397, -0.087 +diff, 6920907803020, 20240723-095858_6902890022558, -0.009, 0.418, -0.027 +diff, 6924187895423, 20240723-100031_6925303795535, -0.059, 0.334, -0.112 +diff, 6916196422325, 20240723-100113_6920382723257, 0.055, 0.500, 0.088 +diff, 6901668006011, 20240723-100416_6938737200650, -0.051, 0.309, -0.100 +diff, 6941499112529, 20240723-100501_6918598120014, 0.076, 0.510, 0.157 +diff, 6951469100067, 20240723-100726_6937221716189, -0.036, 0.320, -0.052 +diff, 6970399922518, 20240723-100845_4897098800133, -0.000, 0.440, -0.001 +diff, 6930364600025, 20240723-101829_6918598120014, -0.072, 0.389, -0.132 +diff, 6902890022558, 20240723-102033_6920382723257, -0.079, 0.458, -0.279 +diff, 6954256905319, 20240723-102206_6902890022558, 0.007, 0.479, 0.021 +diff, 6930364600025, 20240723-102409_6970470331529, 0.014, 0.502, 0.043 +diff, 6920382723257, 20240723-102515_6938737200650, 0.057, 0.424, 0.117 +diff, 6951469100067, 20240723-102833_6937221716189, 0.069, 0.453, 0.146 +diff, 6924187828544, 20240723-110216_6937451883132, 0.026, 0.358, 0.053 +diff, 6908355010450, 20240723-110224_6973337010216, -0.060, 0.352, -0.134 +diff, 6937962111526, 20240723-111446_6937451883132, 0.019, 0.431, 0.053 +diff, 6970470331529, 20240723-111521_6973337010216, 0.007, 0.410, 0.016 +diff, 6918598120014, 20240723-111602_6924187828544, -0.004, 0.454, -0.008 +diff, 6901668935908, 20240723-112004_6934024590268, -0.084, 0.389, -0.184 +diff, 6911988027191, 20240723-112505_6924187828544, 0.062, 0.452, 0.139 +diff, 6911988027191, 20240723-155423_6974158894528, -0.016, 0.396, -0.029 +diff, 4897073991658, 20240723-160227_6970399922518, 0.023, 0.295, 0.052 +diff, 4897073991658, 20240723-160942_6974158894528, -0.098, 0.247, -0.176 diff --git a/contrast/result/record/20240907_171156.txt b/contrast/result/record/20240907_171156.txt new file mode 100644 index 0000000..e5c684f --- /dev/null +++ b/contrast/result/record/20240907_171156.txt @@ -0,0 +1,124 @@ +same, 6954256905319, 20240715-144627_6954256905319, 0.211, 0.750, 0.630 +same, 6930364600025, 20240719-143603_6930364600025, 0.283, 0.771, 0.616 +same, 6924509902839, 20240719-143630_6924509902839, 0.182, 0.727, 0.462 +same, 6902007115524, 20240719-143729_6902007115524, 0.221, 0.723, 0.519 +same, 6930628801472, 20240719-143807_6930628801472, 0.249, 0.804, 0.672 +same, 6908355010450, 20240719-144335_6908355010450, 0.255, 0.812, 0.575 +same, 6924509902754, 20240719-144454_6924509902754, 0.458, 0.882, 0.836 +same, 6925843410158, 20240719-144916_6925843410158, 0.229, 0.829, 0.686 +same, 6941499112529, 20240719-145022_6941499112529, 0.190, 0.795, 0.359 +same, 6918149576536, 20240719-145058_6918149576536, 0.493, 0.906, 0.765 +same, 6928002374551, 20240719-145542_6928002374551, 0.165, 0.773, 0.531 +same, 6930364600001, 20240719-150000_6930364600001, 0.322, 0.831, 0.736 +same, 6937962111526, 20240719-150209_6937962111526, 0.225, 0.794, 0.605 +same, 6924509902839, 20240719-150345_6924509902839, 0.161, 0.816, 0.502 +same, 6924509902839, 20240719-150458_6924509902839, 0.319, 0.746, 0.623 +same, 6924187895423, 20240719-150623_6924187895423, 0.430, 0.906, 0.903 +same, 6916749306102, 20240719-150716_6916749306102, 0.269, 0.764, 0.737 +same, 6951469100067, 20240719-150750_6951469100067, 0.547, 0.938, 0.896 +same, 6958350501101, 20240719-151215_6958350501101, 0.578, 0.906, 0.877 +same, 6951469100067, 20240719-151325_6951469100067, 0.610, 0.930, 0.904 +same, 6937962111526, 20240719-151349_6937962111526, 0.138, 0.762, 0.496 +same, 6916749306102, 20240719-151416_6916749306102, 0.243, 0.728, 0.630 +same, 6930364600001, 20240719-151535_6930364600001, 0.144, 0.787, 0.482 +same, 6911988027191, 20240719-151724_6911988027191, 0.035, 0.502, 0.076 +same, 6972219890632, 20240719-151800_6972219890632, 0.315, 0.880, 0.700 +same, 6901668935908, 20240719-152125_6901668935908, 0.203, 0.856, 0.426 +same, 6930364600001, 20240719-153117_6930364600001, 0.057, 0.775, 0.165 +same, 6902265587712, 20240722-144630_6902265587712, 0.364, 0.833, 0.669 +same, 4897073991658, 20240722-160713_4897073991658, 0.101, 0.514, 0.247 +same, 4892655902237, 20240722-160857_4892655902237, 0.459, 0.760, 0.812 +same, 6920907803020, 20240722-161533_6920907803020, 0.211, 0.769, 0.678 +same, 6920912342002, 20240722-163428_6920912342002, 0.281, 0.825, 0.853 +same, 6916196422325, 20240722-164859_6916196422325, 0.392, 0.837, 0.665 +same, 6901668006011, 20240722-165548_6901668006011, 0.284, 0.752, 0.530 +same, 6937221716189, 20240723-094426_6937221716189, 0.407, 0.850, 0.705 +same, 4897098800133, 20240723-094447_4897098800133, 0.382, 0.846, 0.784 +same, 6970470331529, 20240723-094617_6970470331529, 0.238, 0.925, 0.586 +same, 6918598120014, 20240723-095103_6918598120014, 0.407, 0.905, 0.819 +same, 6938737200650, 20240723-095527_6938737200650, 0.208, 0.825, 0.543 +same, 6902890022558, 20240723-095858_6902890022558, 0.162, 0.769, 0.628 +same, 6925303795535, 20240723-100031_6925303795535, 0.304, 0.835, 0.671 +same, 6920382723257, 20240723-100113_6920382723257, 0.274, 0.854, 0.539 +same, 6938737200650, 20240723-100416_6938737200650, 0.237, 0.719, 0.581 +same, 6918598120014, 20240723-100501_6918598120014, 0.099, 0.791, 0.209 +same, 6937221716189, 20240723-100726_6937221716189, 0.347, 0.771, 0.611 +same, 4897098800133, 20240723-100845_4897098800133, 0.304, 0.770, 0.718 +same, 6918598120014, 20240723-101829_6918598120014, 0.502, 0.919, 0.857 +same, 6920382723257, 20240723-102033_6920382723257, 0.343, 0.849, 0.748 +same, 6902890022558, 20240723-102206_6902890022558, 0.117, 0.745, 0.389 +same, 6970470331529, 20240723-102409_6970470331529, 0.102, 0.734, 0.305 +same, 6938737200650, 20240723-102515_6938737200650, 0.183, 0.725, 0.469 +same, 6937221716189, 20240723-102833_6937221716189, 0.067, 0.623, 0.175 +same, 6937451883132, 20240723-110216_6937451883132, 0.323, 0.860, 0.717 +same, 6973337010216, 20240723-110224_6973337010216, 0.202, 0.760, 0.622 +same, 6937451883132, 20240723-111446_6937451883132, 0.367, 0.817, 0.806 +same, 6973337010216, 20240723-111521_6973337010216, 0.140, 0.777, 0.414 +same, 6924187828544, 20240723-111602_6924187828544, 0.359, 0.859, 0.718 +same, 6934024590268, 20240723-112004_6934024590268, 0.184, 0.727, 0.583 +same, 6924187828544, 20240723-112505_6924187828544, 0.281, 0.875, 0.573 +same, 6974158894528, 20240723-155423_6974158894528, 0.433, 0.890, 0.788 +same, 6970399922518, 20240723-160227_6970399922518, 0.078, 0.564, 0.215 +same, 6974158894528, 20240723-160942_6974158894528, 0.417, 0.895, 0.832 +diff, 6951469100067, 20240715-144627_6954256905319, 0.007, 0.399, 0.011 +diff, 6920907803020, 20240719-143603_6930364600025, -0.025, 0.443, -0.063 +diff, 6918149576536, 20240719-143630_6924509902839, 0.031, 0.324, 0.062 +diff, 6916749306102, 20240719-143729_6902007115524, 0.039, 0.456, 0.102 +diff, 6925843410158, 20240719-143807_6930628801472, 0.057, 0.633, 0.150 +diff, 6958350501101, 20240719-144335_6908355010450, -0.067, 0.326, -0.124 +diff, 6972219890632, 20240719-144454_6924509902754, 0.093, 0.537, 0.174 +diff, 6925303795535, 20240719-144916_6925843410158, -0.022, 0.354, -0.058 +diff, 6951469100067, 20240719-145022_6941499112529, -0.048, 0.320, -0.073 +diff, 6941499112529, 20240719-145058_6918149576536, -0.187, 0.359, -0.360 +diff, 6901668006011, 20240719-145542_6928002374551, -0.034, 0.359, -0.083 +diff, 6924187828544, 20240719-150000_6930364600001, -0.025, 0.438, -0.052 +diff, 6902890022558, 20240719-150209_6937962111526, 0.014, 0.594, 0.042 +diff, 6918149576536, 20240719-150345_6924509902839, -0.035, 0.353, -0.084 +diff, 6941499112529, 20240719-150458_6924509902839, -0.012, 0.403, -0.023 +diff, 6970470331529, 20240719-150623_6924187895423, 0.042, 0.436, 0.098 +diff, 6930364600025, 20240719-150716_6916749306102, 0.090, 0.552, 0.234 +diff, 6918598120014, 20240719-150750_6951469100067, 0.090, 0.498, 0.191 +diff, 6937962111526, 20240719-151215_6958350501101, -0.083, 0.468, -0.217 +diff, 6902890022558, 20240719-151325_6951469100067, 0.008, 0.489, 0.024 +diff, 6930364600025, 20240719-151349_6937962111526, -0.029, 0.336, -0.081 +diff, 6920382723257, 20240719-151416_6916749306102, 0.085, 0.444, 0.187 +diff, 6918149576536, 20240719-151535_6930364600001, -0.035, 0.398, -0.082 +diff, 6973337010216, 20240719-151724_6911988027191, 0.008, 0.515, 0.020 +diff, 6954256905319, 20240719-151800_6972219890632, 0.024, 0.495, 0.072 +diff, 6930628801472, 20240719-152125_6901668935908, -0.047, 0.380, -0.130 +diff, 6973337010216, 20240719-153117_6930364600001, -0.023, 0.542, -0.081 +diff, 6941499112529, 20240722-144630_6902265587712, -0.000, 0.397, -0.000 +diff, 6918149576536, 20240722-160713_4897073991658, -0.043, 0.421, -0.083 +diff, 6951469100067, 20240722-160857_4892655902237, -0.006, 0.354, -0.009 +diff, 6925843410158, 20240722-161533_6920907803020, -0.064, 0.344, -0.205 +diff, 6930364600001, 20240722-163428_6920912342002, -0.070, 0.417, -0.196 +diff, 6970470331529, 20240722-164859_6916196422325, -0.051, 0.435, -0.115 +diff, 6901668935908, 20240722-165548_6901668006011, 0.139, 0.752, 0.257 +diff, 6925843410158, 20240723-094426_6937221716189, -0.034, 0.331, -0.076 +diff, 6941499112529, 20240723-094447_4897098800133, -0.057, 0.399, -0.111 +diff, 4892655902237, 20240723-094617_6970470331529, 0.018, 0.236, 0.037 +diff, 4897073991658, 20240723-095103_6918598120014, 0.007, 0.402, 0.014 +diff, 6924509902754, 20240723-095527_6938737200650, -0.050, 0.364, -0.105 +diff, 6951469100067, 20240723-095858_6902890022558, 0.004, 0.457, 0.007 +diff, 6902007115524, 20240723-100031_6925303795535, -0.021, 0.295, -0.043 +diff, 6934024590268, 20240723-100113_6920382723257, -0.007, 0.420, -0.019 +diff, 6928002374551, 20240723-100416_6938737200650, 0.107, 0.636, 0.280 +diff, 4897098800133, 20240723-100501_6918598120014, 0.054, 0.281, 0.117 +diff, 6951469100067, 20240723-100726_6937221716189, -0.036, 0.320, -0.052 +diff, 6934024590268, 20240723-100845_4897098800133, -0.037, 0.369, -0.118 +diff, 6916749306102, 20240723-101829_6918598120014, -0.022, 0.483, -0.043 +diff, 4892655902237, 20240723-102033_6920382723257, -0.027, 0.246, -0.054 +diff, 6920382723257, 20240723-102206_6902890022558, 0.009, 0.288, 0.019 +diff, 6920912342002, 20240723-102409_6970470331529, 0.062, 0.533, 0.216 +diff, 6911988027191, 20240723-102515_6938737200650, 0.015, 0.461, 0.036 +diff, 6902265587712, 20240723-102833_6937221716189, 0.012, 0.570, 0.028 +diff, 6954256905319, 20240723-110216_6937451883132, -0.067, 0.341, -0.181 +diff, 6930364600001, 20240723-110224_6973337010216, -0.014, 0.507, -0.036 +diff, 6954256905319, 20240723-111446_6937451883132, -0.019, 0.362, -0.051 +diff, 6937221716189, 20240723-111521_6973337010216, 0.047, 0.475, 0.100 +diff, 6916749306102, 20240723-111602_6924187828544, 0.006, 0.458, 0.013 +diff, 6930364600025, 20240723-112004_6934024590268, 0.010, 0.462, 0.024 +diff, 6951469100067, 20240723-112505_6924187828544, -0.058, 0.269, -0.089 +diff, 4897073991658, 20240723-155423_6974158894528, -0.058, 0.177, -0.096 +diff, 6924187828544, 20240723-160227_6970399922518, 0.044, 0.540, 0.104 +diff, 6920912342002, 20240723-160942_6974158894528, -0.025, 0.377, -0.054 diff --git a/contrast/result/record/20240907_171156_ft16.txt b/contrast/result/record/20240907_171156_ft16.txt new file mode 100644 index 0000000..00c582c --- /dev/null +++ b/contrast/result/record/20240907_171156_ft16.txt @@ -0,0 +1,124 @@ +same, 6954256905319, 20240715-144627_6954256905319, 0.664, 0.766, 0.959 +same, 6930364600025, 20240719-143603_6930364600025, 0.779, 0.853, 0.966 +same, 6924509902839, 20240719-143630_6924509902839, 0.685, 0.781, 0.956 +same, 6902007115524, 20240719-143729_6902007115524, 0.681, 0.788, 0.956 +same, 6930628801472, 20240719-143807_6930628801472, 0.690, 0.813, 0.972 +same, 6908355010450, 20240719-144335_6908355010450, 0.482, 0.640, 0.889 +same, 6924509902754, 20240719-144454_6924509902754, 0.714, 0.839, 0.955 +same, 6925843410158, 20240719-144916_6925843410158, 0.609, 0.721, 0.971 +same, 6941499112529, 20240719-145022_6941499112529, 0.661, 0.774, 0.928 +same, 6918149576536, 20240719-145058_6918149576536, 0.636, 0.742, 0.892 +same, 6928002374551, 20240719-145542_6928002374551, 0.729, 0.822, 0.974 +same, 6930364600001, 20240719-150000_6930364600001, 0.638, 0.751, 0.946 +same, 6937962111526, 20240719-150209_6937962111526, 0.789, 0.855, 0.953 +same, 6924509902839, 20240719-150345_6924509902839, 0.541, 0.710, 0.938 +same, 6924509902839, 20240719-150458_6924509902839, 0.696, 0.799, 0.932 +same, 6924187895423, 20240719-150623_6924187895423, 0.835, 0.930, 0.991 +same, 6916749306102, 20240719-150716_6916749306102, 0.862, 0.909, 0.991 +same, 6951469100067, 20240719-150750_6951469100067, 0.579, 0.707, 0.861 +same, 6958350501101, 20240719-151215_6958350501101, 0.888, 0.924, 0.980 +same, 6951469100067, 20240719-151325_6951469100067, 0.572, 0.683, 0.815 +same, 6937962111526, 20240719-151349_6937962111526, 0.846, 0.898, 0.988 +same, 6916749306102, 20240719-151416_6916749306102, 0.820, 0.879, 0.979 +same, 6930364600001, 20240719-151535_6930364600001, 0.631, 0.732, 0.947 +same, 6911988027191, 20240719-151724_6911988027191, 0.723, 0.797, 0.950 +same, 6972219890632, 20240719-151800_6972219890632, 0.535, 0.686, 0.928 +same, 6901668935908, 20240719-152125_6901668935908, 0.741, 0.804, 0.957 +same, 6930364600001, 20240719-153117_6930364600001, 0.659, 0.745, 0.943 +same, 6902265587712, 20240722-144630_6902265587712, 0.756, 0.843, 0.958 +same, 4897073991658, 20240722-160713_4897073991658, 0.698, 0.770, 0.943 +same, 4892655902237, 20240722-160857_4892655902237, 0.760, 0.819, 0.964 +same, 6920907803020, 20240722-161533_6920907803020, 0.759, 0.839, 0.981 +same, 6920912342002, 20240722-163428_6920912342002, 0.744, 0.850, 0.988 +same, 6916196422325, 20240722-164859_6916196422325, 0.541, 0.690, 0.873 +same, 6901668006011, 20240722-165548_6901668006011, 0.665, 0.778, 0.937 +same, 6937221716189, 20240723-094426_6937221716189, 0.698, 0.817, 0.924 +same, 4897098800133, 20240723-094447_4897098800133, 0.795, 0.845, 0.963 +same, 6970470331529, 20240723-094617_6970470331529, 0.746, 0.831, 0.958 +same, 6918598120014, 20240723-095103_6918598120014, 0.853, 0.929, 0.988 +same, 6938737200650, 20240723-095527_6938737200650, 0.727, 0.805, 0.952 +same, 6902890022558, 20240723-095858_6902890022558, 0.713, 0.813, 0.981 +same, 6925303795535, 20240723-100031_6925303795535, 0.732, 0.831, 0.967 +same, 6920382723257, 20240723-100113_6920382723257, 0.763, 0.825, 0.933 +same, 6938737200650, 20240723-100416_6938737200650, 0.753, 0.824, 0.952 +same, 6918598120014, 20240723-100501_6918598120014, 0.785, 0.847, 0.946 +same, 6937221716189, 20240723-100726_6937221716189, 0.709, 0.803, 0.929 +same, 4897098800133, 20240723-100845_4897098800133, 0.792, 0.834, 0.967 +same, 6918598120014, 20240723-101829_6918598120014, 0.866, 0.922, 0.978 +same, 6920382723257, 20240723-102033_6920382723257, 0.762, 0.830, 0.947 +same, 6902890022558, 20240723-102206_6902890022558, 0.791, 0.860, 0.960 +same, 6970470331529, 20240723-102409_6970470331529, 0.725, 0.822, 0.962 +same, 6938737200650, 20240723-102515_6938737200650, 0.761, 0.825, 0.961 +same, 6937221716189, 20240723-102833_6937221716189, 0.506, 0.656, 0.878 +same, 6937451883132, 20240723-110216_6937451883132, 0.624, 0.710, 0.912 +same, 6973337010216, 20240723-110224_6973337010216, 0.664, 0.770, 0.973 +same, 6937451883132, 20240723-111446_6937451883132, 0.652, 0.756, 0.944 +same, 6973337010216, 20240723-111521_6973337010216, 0.714, 0.795, 0.961 +same, 6924187828544, 20240723-111602_6924187828544, 0.468, 0.600, 0.783 +same, 6934024590268, 20240723-112004_6934024590268, 0.531, 0.659, 0.924 +same, 6924187828544, 20240723-112505_6924187828544, 0.698, 0.838, 0.948 +same, 6974158894528, 20240723-155423_6974158894528, 0.711, 0.807, 0.937 +same, 6970399922518, 20240723-160227_6970399922518, 0.672, 0.764, 0.942 +same, 6974158894528, 20240723-160942_6974158894528, 0.667, 0.803, 0.942 +diff, 6951469100067, 20240715-144627_6954256905319, 0.554, 0.659, 0.813 +diff, 6920907803020, 20240719-143603_6930364600025, 0.746, 0.817, 0.956 +diff, 6918149576536, 20240719-143630_6924509902839, 0.727, 0.803, 0.932 +diff, 6916749306102, 20240719-143729_6902007115524, 0.778, 0.842, 0.963 +diff, 6925843410158, 20240719-143807_6930628801472, 0.604, 0.715, 0.941 +diff, 6958350501101, 20240719-144335_6908355010450, 0.556, 0.649, 0.887 +diff, 6972219890632, 20240719-144454_6924509902754, 0.538, 0.653, 0.883 +diff, 6925303795535, 20240719-144916_6925843410158, 0.677, 0.770, 0.955 +diff, 6951469100067, 20240719-145022_6941499112529, 0.532, 0.646, 0.792 +diff, 6941499112529, 20240719-145058_6918149576536, 0.551, 0.662, 0.851 +diff, 6901668006011, 20240719-145542_6928002374551, 0.672, 0.766, 0.932 +diff, 6924187828544, 20240719-150000_6930364600001, 0.709, 0.784, 0.932 +diff, 6902890022558, 20240719-150209_6937962111526, 0.730, 0.808, 0.944 +diff, 6918149576536, 20240719-150345_6924509902839, 0.565, 0.690, 0.902 +diff, 6941499112529, 20240719-150458_6924509902839, 0.679, 0.765, 0.921 +diff, 6970470331529, 20240719-150623_6924187895423, 0.742, 0.815, 0.961 +diff, 6930364600025, 20240719-150716_6916749306102, 0.785, 0.851, 0.975 +diff, 6918598120014, 20240719-150750_6951469100067, 0.792, 0.857, 0.967 +diff, 6937962111526, 20240719-151215_6958350501101, 0.844, 0.888, 0.973 +diff, 6902890022558, 20240719-151325_6951469100067, 0.763, 0.826, 0.967 +diff, 6930364600025, 20240719-151349_6937962111526, 0.775, 0.838, 0.965 +diff, 6920382723257, 20240719-151416_6916749306102, 0.808, 0.856, 0.969 +diff, 6918149576536, 20240719-151535_6930364600001, 0.747, 0.818, 0.930 +diff, 6973337010216, 20240719-151724_6911988027191, 0.719, 0.794, 0.956 +diff, 6954256905319, 20240719-151800_6972219890632, 0.625, 0.739, 0.945 +diff, 6930628801472, 20240719-152125_6901668935908, 0.755, 0.828, 0.963 +diff, 6973337010216, 20240719-153117_6930364600001, 0.725, 0.800, 0.965 +diff, 6941499112529, 20240722-144630_6902265587712, 0.712, 0.784, 0.949 +diff, 6918149576536, 20240722-160713_4897073991658, 0.734, 0.812, 0.923 +diff, 6951469100067, 20240722-160857_4892655902237, 0.569, 0.669, 0.819 +diff, 6925843410158, 20240722-161533_6920907803020, 0.664, 0.752, 0.955 +diff, 6930364600001, 20240722-163428_6920912342002, 0.608, 0.709, 0.928 +diff, 6970470331529, 20240722-164859_6916196422325, 0.657, 0.753, 0.939 +diff, 6901668935908, 20240722-165548_6901668006011, 0.659, 0.774, 0.926 +diff, 6925843410158, 20240723-094426_6937221716189, 0.609, 0.700, 0.895 +diff, 6941499112529, 20240723-094447_4897098800133, 0.734, 0.805, 0.943 +diff, 4892655902237, 20240723-094617_6970470331529, 0.752, 0.802, 0.934 +diff, 4897073991658, 20240723-095103_6918598120014, 0.721, 0.771, 0.936 +diff, 6924509902754, 20240723-095527_6938737200650, 0.626, 0.728, 0.897 +diff, 6951469100067, 20240723-095858_6902890022558, 0.530, 0.641, 0.819 +diff, 6902007115524, 20240723-100031_6925303795535, 0.684, 0.769, 0.931 +diff, 6934024590268, 20240723-100113_6920382723257, 0.624, 0.726, 0.908 +diff, 6928002374551, 20240723-100416_6938737200650, 0.694, 0.785, 0.941 +diff, 4897098800133, 20240723-100501_6918598120014, 0.714, 0.775, 0.932 +diff, 6951469100067, 20240723-100726_6937221716189, 0.544, 0.645, 0.778 +diff, 6934024590268, 20240723-100845_4897098800133, 0.716, 0.785, 0.965 +diff, 6916749306102, 20240723-101829_6918598120014, 0.858, 0.902, 0.968 +diff, 4892655902237, 20240723-102033_6920382723257, 0.704, 0.773, 0.933 +diff, 6920382723257, 20240723-102206_6902890022558, 0.851, 0.889, 0.957 +diff, 6920912342002, 20240723-102409_6970470331529, 0.743, 0.809, 0.972 +diff, 6911988027191, 20240723-102515_6938737200650, 0.654, 0.764, 0.931 +diff, 6902265587712, 20240723-102833_6937221716189, 0.511, 0.608, 0.866 +diff, 6954256905319, 20240723-110216_6937451883132, 0.673, 0.756, 0.946 +diff, 6930364600001, 20240723-110224_6973337010216, 0.594, 0.729, 0.935 +diff, 6954256905319, 20240723-111446_6937451883132, 0.685, 0.770, 0.953 +diff, 6937221716189, 20240723-111521_6973337010216, 0.734, 0.803, 0.943 +diff, 6916749306102, 20240723-111602_6924187828544, 0.521, 0.629, 0.778 +diff, 6930364600025, 20240723-112004_6934024590268, 0.577, 0.691, 0.907 +diff, 6951469100067, 20240723-112505_6924187828544, 0.521, 0.639, 0.772 +diff, 4897073991658, 20240723-155423_6974158894528, 0.712, 0.763, 0.920 +diff, 6924187828544, 20240723-160227_6970399922518, 0.626, 0.733, 0.917 +diff, 6920912342002, 20240723-160942_6974158894528, 0.676, 0.757, 0.914 diff --git a/contrast/result/record/20240909_085740.txt b/contrast/result/record/20240909_085740.txt new file mode 100644 index 0000000..be87d35 --- /dev/null +++ b/contrast/result/record/20240909_085740.txt @@ -0,0 +1,124 @@ +same, 6954256905319, 20240715-144627_6954256905319, 0.211, 0.750, 0.630 +same, 6930364600025, 20240719-143603_6930364600025, 0.283, 0.771, 0.616 +same, 6924509902839, 20240719-143630_6924509902839, 0.182, 0.727, 0.462 +same, 6902007115524, 20240719-143729_6902007115524, 0.221, 0.723, 0.519 +same, 6930628801472, 20240719-143807_6930628801472, 0.249, 0.804, 0.672 +same, 6908355010450, 20240719-144335_6908355010450, 0.255, 0.812, 0.575 +same, 6924509902754, 20240719-144454_6924509902754, 0.458, 0.882, 0.836 +same, 6925843410158, 20240719-144916_6925843410158, 0.229, 0.829, 0.686 +same, 6941499112529, 20240719-145022_6941499112529, 0.190, 0.795, 0.359 +same, 6918149576536, 20240719-145058_6918149576536, 0.493, 0.906, 0.765 +same, 6928002374551, 20240719-145542_6928002374551, 0.165, 0.773, 0.531 +same, 6930364600001, 20240719-150000_6930364600001, 0.322, 0.831, 0.736 +same, 6937962111526, 20240719-150209_6937962111526, 0.225, 0.794, 0.605 +same, 6924509902839, 20240719-150345_6924509902839, 0.161, 0.816, 0.502 +same, 6924509902839, 20240719-150458_6924509902839, 0.319, 0.746, 0.623 +same, 6924187895423, 20240719-150623_6924187895423, 0.430, 0.906, 0.903 +same, 6916749306102, 20240719-150716_6916749306102, 0.269, 0.764, 0.737 +same, 6951469100067, 20240719-150750_6951469100067, 0.547, 0.938, 0.896 +same, 6958350501101, 20240719-151215_6958350501101, 0.578, 0.906, 0.877 +same, 6951469100067, 20240719-151325_6951469100067, 0.610, 0.930, 0.904 +same, 6937962111526, 20240719-151349_6937962111526, 0.138, 0.762, 0.496 +same, 6916749306102, 20240719-151416_6916749306102, 0.243, 0.728, 0.630 +same, 6930364600001, 20240719-151535_6930364600001, 0.144, 0.787, 0.482 +same, 6911988027191, 20240719-151724_6911988027191, 0.035, 0.502, 0.076 +same, 6972219890632, 20240719-151800_6972219890632, 0.315, 0.880, 0.700 +same, 6901668935908, 20240719-152125_6901668935908, 0.203, 0.856, 0.426 +same, 6930364600001, 20240719-153117_6930364600001, 0.057, 0.775, 0.165 +same, 6902265587712, 20240722-144630_6902265587712, 0.364, 0.833, 0.669 +same, 4897073991658, 20240722-160713_4897073991658, 0.101, 0.514, 0.247 +same, 4892655902237, 20240722-160857_4892655902237, 0.459, 0.760, 0.812 +same, 6920907803020, 20240722-161533_6920907803020, 0.211, 0.769, 0.678 +same, 6920912342002, 20240722-163428_6920912342002, 0.281, 0.825, 0.853 +same, 6916196422325, 20240722-164859_6916196422325, 0.392, 0.837, 0.665 +same, 6901668006011, 20240722-165548_6901668006011, 0.284, 0.752, 0.530 +same, 6937221716189, 20240723-094426_6937221716189, 0.407, 0.850, 0.705 +same, 4897098800133, 20240723-094447_4897098800133, 0.382, 0.846, 0.784 +same, 6970470331529, 20240723-094617_6970470331529, 0.238, 0.925, 0.586 +same, 6918598120014, 20240723-095103_6918598120014, 0.407, 0.905, 0.819 +same, 6938737200650, 20240723-095527_6938737200650, 0.208, 0.825, 0.543 +same, 6902890022558, 20240723-095858_6902890022558, 0.162, 0.769, 0.628 +same, 6925303795535, 20240723-100031_6925303795535, 0.304, 0.835, 0.671 +same, 6920382723257, 20240723-100113_6920382723257, 0.274, 0.854, 0.539 +same, 6938737200650, 20240723-100416_6938737200650, 0.237, 0.719, 0.581 +same, 6918598120014, 20240723-100501_6918598120014, 0.099, 0.791, 0.209 +same, 6937221716189, 20240723-100726_6937221716189, 0.347, 0.771, 0.611 +same, 4897098800133, 20240723-100845_4897098800133, 0.304, 0.770, 0.718 +same, 6918598120014, 20240723-101829_6918598120014, 0.502, 0.919, 0.857 +same, 6920382723257, 20240723-102033_6920382723257, 0.343, 0.849, 0.748 +same, 6902890022558, 20240723-102206_6902890022558, 0.117, 0.745, 0.389 +same, 6970470331529, 20240723-102409_6970470331529, 0.102, 0.734, 0.305 +same, 6938737200650, 20240723-102515_6938737200650, 0.183, 0.725, 0.469 +same, 6937221716189, 20240723-102833_6937221716189, 0.067, 0.623, 0.175 +same, 6937451883132, 20240723-110216_6937451883132, 0.323, 0.860, 0.717 +same, 6973337010216, 20240723-110224_6973337010216, 0.202, 0.760, 0.622 +same, 6937451883132, 20240723-111446_6937451883132, 0.367, 0.817, 0.806 +same, 6973337010216, 20240723-111521_6973337010216, 0.140, 0.777, 0.414 +same, 6924187828544, 20240723-111602_6924187828544, 0.359, 0.859, 0.718 +same, 6934024590268, 20240723-112004_6934024590268, 0.184, 0.727, 0.583 +same, 6924187828544, 20240723-112505_6924187828544, 0.281, 0.875, 0.573 +same, 6974158894528, 20240723-155423_6974158894528, 0.433, 0.890, 0.788 +same, 6970399922518, 20240723-160227_6970399922518, 0.078, 0.564, 0.215 +same, 6974158894528, 20240723-160942_6974158894528, 0.417, 0.895, 0.832 +diff, 6924509902839, 20240715-144627_6954256905319, -0.037, 0.355, -0.084 +diff, 6924187828544, 20240719-143603_6930364600025, 0.019, 0.434, 0.038 +diff, 6924509902754, 20240719-143630_6924509902839, 0.086, 0.575, 0.209 +diff, 6954256905319, 20240719-143729_6902007115524, -0.054, 0.437, -0.165 +diff, 6951469100067, 20240719-143807_6930628801472, 0.002, 0.371, 0.003 +diff, 4897098800133, 20240719-144335_6908355010450, 0.015, 0.299, 0.037 +diff, 6918149576536, 20240719-144454_6924509902754, -0.030, 0.254, -0.045 +diff, 6941499112529, 20240719-144916_6925843410158, 0.023, 0.467, 0.053 +diff, 6924187895423, 20240719-145022_6941499112529, 0.008, 0.411, 0.015 +diff, 6920907803020, 20240719-145058_6918149576536, -0.065, 0.395, -0.161 +diff, 6951469100067, 20240719-145542_6928002374551, -0.033, 0.407, -0.064 +diff, 6911988027191, 20240719-150000_6930364600001, -0.022, 0.391, -0.051 +diff, 6937451883132, 20240719-150209_6937962111526, -0.039, 0.302, -0.085 +diff, 6902890022558, 20240719-150345_6924509902839, 0.036, 0.571, 0.172 +diff, 6937451883132, 20240719-150458_6924509902839, -0.021, 0.399, -0.044 +diff, 6924509902839, 20240719-150623_6924187895423, 0.042, 0.716, 0.095 +diff, 6920382723257, 20240719-150716_6916749306102, 0.083, 0.524, 0.194 +diff, 6930364600001, 20240719-150750_6951469100067, -0.059, 0.492, -0.139 +diff, 6920907803020, 20240719-151215_6958350501101, -0.082, 0.363, -0.194 +diff, 6924509902839, 20240719-151325_6951469100067, 0.054, 0.509, 0.104 +diff, 6925843410158, 20240719-151349_6937962111526, -0.088, 0.325, -0.288 +diff, 6974158894528, 20240719-151416_6916749306102, -0.002, 0.455, -0.005 +diff, 6924187828544, 20240719-151535_6930364600001, 0.049, 0.513, 0.150 +diff, 6924187895423, 20240719-151724_6911988027191, 0.007, 0.373, 0.012 +diff, 6901668006011, 20240719-151800_6972219890632, -0.029, 0.352, -0.062 +diff, 6918598120014, 20240719-152125_6901668935908, 0.003, 0.456, 0.006 +diff, 6902007115524, 20240719-153117_6930364600001, -0.058, 0.466, -0.157 +diff, 6970470331529, 20240722-144630_6902265587712, -0.019, 0.431, -0.042 +diff, 6901668935908, 20240722-160713_4897073991658, 0.029, 0.417, 0.069 +diff, 6916749306102, 20240722-160857_4892655902237, 0.079, 0.549, 0.183 +diff, 6924187895423, 20240722-161533_6920907803020, 0.006, 0.424, 0.015 +diff, 6937962111526, 20240722-163428_6920912342002, 0.019, 0.498, 0.066 +diff, 6902890022558, 20240722-164859_6916196422325, 0.004, 0.464, 0.013 +diff, 6902890022558, 20240722-165548_6901668006011, -0.064, 0.390, -0.195 +diff, 6970399922518, 20240723-094426_6937221716189, -0.053, 0.428, -0.115 +diff, 6954256905319, 20240723-094447_4897098800133, 0.029, 0.492, 0.076 +diff, 6974158894528, 20240723-094617_6970470331529, 0.036, 0.425, 0.094 +diff, 6918149576536, 20240723-095103_6918598120014, 0.001, 0.404, 0.001 +diff, 6930628801472, 20240723-095527_6938737200650, -0.041, 0.444, -0.112 +diff, 6928002374551, 20240723-095858_6902890022558, 0.013, 0.441, 0.041 +diff, 6938737200650, 20240723-100031_6925303795535, -0.004, 0.413, -0.010 +diff, 6918149576536, 20240723-100113_6920382723257, -0.085, 0.294, -0.137 +diff, 6901668006011, 20240723-100416_6938737200650, -0.051, 0.309, -0.100 +diff, 6911988027191, 20240723-100501_6918598120014, 0.003, 0.295, 0.006 +diff, 6958350501101, 20240723-100726_6937221716189, 0.073, 0.374, 0.106 +diff, 6925843410158, 20240723-100845_4897098800133, -0.051, 0.328, -0.144 +diff, 4897098800133, 20240723-101829_6918598120014, 0.079, 0.317, 0.138 +diff, 4897098800133, 20240723-102033_6920382723257, 0.004, 0.298, 0.009 +diff, 6920907803020, 20240723-102206_6902890022558, 0.034, 0.441, 0.091 +diff, 6954256905319, 20240723-102409_6970470331529, 0.029, 0.437, 0.110 +diff, 6925843410158, 20240723-102515_6938737200650, -0.005, 0.406, -0.013 +diff, 6920912342002, 20240723-102833_6937221716189, 0.071, 0.536, 0.235 +diff, 6924187828544, 20240723-110216_6937451883132, 0.026, 0.358, 0.053 +diff, 6937221716189, 20240723-110224_6973337010216, 0.013, 0.382, 0.030 +diff, 6920382723257, 20240723-111446_6937451883132, 0.041, 0.469, 0.079 +diff, 6920907803020, 20240723-111521_6973337010216, 0.040, 0.524, 0.111 +diff, 6908355010450, 20240723-111602_6924187828544, -0.021, 0.475, -0.040 +diff, 6901668935908, 20240723-112004_6934024590268, -0.084, 0.389, -0.184 +diff, 4897073991658, 20240723-112505_6924187828544, 0.021, 0.297, 0.041 +diff, 4897098800133, 20240723-155423_6974158894528, 0.015, 0.287, 0.026 +diff, 6930628801472, 20240723-160227_6970399922518, 0.034, 0.447, 0.099 +diff, 6970470331529, 20240723-160942_6974158894528, 0.003, 0.398, 0.006 diff --git a/contrast/result/record/20240909_085740_ft16.txt b/contrast/result/record/20240909_085740_ft16.txt new file mode 100644 index 0000000..d9e75fd --- /dev/null +++ b/contrast/result/record/20240909_085740_ft16.txt @@ -0,0 +1,124 @@ +same, 6954256905319, 20240715-144627_6954256905319, 0.211, 0.751, 0.630 +same, 6930364600025, 20240719-143603_6930364600025, 0.283, 0.771, 0.616 +same, 6924509902839, 20240719-143630_6924509902839, 0.182, 0.727, 0.462 +same, 6902007115524, 20240719-143729_6902007115524, 0.221, 0.723, 0.519 +same, 6930628801472, 20240719-143807_6930628801472, 0.249, 0.804, 0.672 +same, 6908355010450, 20240719-144335_6908355010450, 0.255, 0.812, 0.575 +same, 6924509902754, 20240719-144454_6924509902754, 0.458, 0.882, 0.836 +same, 6925843410158, 20240719-144916_6925843410158, 0.229, 0.829, 0.686 +same, 6941499112529, 20240719-145022_6941499112529, 0.190, 0.795, 0.359 +same, 6918149576536, 20240719-145058_6918149576536, 0.493, 0.906, 0.765 +same, 6928002374551, 20240719-145542_6928002374551, 0.165, 0.773, 0.531 +same, 6930364600001, 20240719-150000_6930364600001, 0.322, 0.831, 0.736 +same, 6937962111526, 20240719-150209_6937962111526, 0.225, 0.794, 0.605 +same, 6924509902839, 20240719-150345_6924509902839, 0.161, 0.816, 0.502 +same, 6924509902839, 20240719-150458_6924509902839, 0.319, 0.746, 0.623 +same, 6924187895423, 20240719-150623_6924187895423, 0.430, 0.906, 0.903 +same, 6916749306102, 20240719-150716_6916749306102, 0.269, 0.764, 0.737 +same, 6951469100067, 20240719-150750_6951469100067, 0.547, 0.938, 0.896 +same, 6958350501101, 20240719-151215_6958350501101, 0.578, 0.906, 0.877 +same, 6951469100067, 20240719-151325_6951469100067, 0.610, 0.930, 0.904 +same, 6937962111526, 20240719-151349_6937962111526, 0.138, 0.762, 0.496 +same, 6916749306102, 20240719-151416_6916749306102, 0.243, 0.728, 0.630 +same, 6930364600001, 20240719-151535_6930364600001, 0.144, 0.787, 0.482 +same, 6911988027191, 20240719-151724_6911988027191, 0.035, 0.502, 0.076 +same, 6972219890632, 20240719-151800_6972219890632, 0.315, 0.880, 0.700 +same, 6901668935908, 20240719-152125_6901668935908, 0.203, 0.856, 0.426 +same, 6930364600001, 20240719-153117_6930364600001, 0.057, 0.775, 0.165 +same, 6902265587712, 20240722-144630_6902265587712, 0.364, 0.833, 0.669 +same, 4897073991658, 20240722-160713_4897073991658, 0.101, 0.514, 0.247 +same, 4892655902237, 20240722-160857_4892655902237, 0.459, 0.760, 0.812 +same, 6920907803020, 20240722-161533_6920907803020, 0.211, 0.769, 0.678 +same, 6920912342002, 20240722-163428_6920912342002, 0.281, 0.825, 0.853 +same, 6916196422325, 20240722-164859_6916196422325, 0.392, 0.837, 0.665 +same, 6901668006011, 20240722-165548_6901668006011, 0.284, 0.751, 0.530 +same, 6937221716189, 20240723-094426_6937221716189, 0.407, 0.850, 0.705 +same, 4897098800133, 20240723-094447_4897098800133, 0.382, 0.846, 0.784 +same, 6970470331529, 20240723-094617_6970470331529, 0.238, 0.925, 0.586 +same, 6918598120014, 20240723-095103_6918598120014, 0.407, 0.905, 0.819 +same, 6938737200650, 20240723-095527_6938737200650, 0.208, 0.825, 0.543 +same, 6902890022558, 20240723-095858_6902890022558, 0.162, 0.769, 0.628 +same, 6925303795535, 20240723-100031_6925303795535, 0.304, 0.835, 0.671 +same, 6920382723257, 20240723-100113_6920382723257, 0.274, 0.854, 0.539 +same, 6938737200650, 20240723-100416_6938737200650, 0.237, 0.719, 0.581 +same, 6918598120014, 20240723-100501_6918598120014, 0.099, 0.791, 0.209 +same, 6937221716189, 20240723-100726_6937221716189, 0.347, 0.771, 0.611 +same, 4897098800133, 20240723-100845_4897098800133, 0.304, 0.770, 0.717 +same, 6918598120014, 20240723-101829_6918598120014, 0.502, 0.919, 0.857 +same, 6920382723257, 20240723-102033_6920382723257, 0.343, 0.849, 0.748 +same, 6902890022558, 20240723-102206_6902890022558, 0.117, 0.745, 0.389 +same, 6970470331529, 20240723-102409_6970470331529, 0.102, 0.734, 0.305 +same, 6938737200650, 20240723-102515_6938737200650, 0.183, 0.725, 0.469 +same, 6937221716189, 20240723-102833_6937221716189, 0.067, 0.623, 0.175 +same, 6937451883132, 20240723-110216_6937451883132, 0.323, 0.860, 0.717 +same, 6973337010216, 20240723-110224_6973337010216, 0.202, 0.760, 0.622 +same, 6937451883132, 20240723-111446_6937451883132, 0.367, 0.817, 0.806 +same, 6973337010216, 20240723-111521_6973337010216, 0.140, 0.777, 0.414 +same, 6924187828544, 20240723-111602_6924187828544, 0.359, 0.859, 0.718 +same, 6934024590268, 20240723-112004_6934024590268, 0.184, 0.727, 0.583 +same, 6924187828544, 20240723-112505_6924187828544, 0.281, 0.875, 0.573 +same, 6974158894528, 20240723-155423_6974158894528, 0.433, 0.890, 0.788 +same, 6970399922518, 20240723-160227_6970399922518, 0.078, 0.564, 0.215 +same, 6974158894528, 20240723-160942_6974158894528, 0.417, 0.895, 0.832 +diff, 6924509902839, 20240715-144627_6954256905319, -0.037, 0.355, -0.084 +diff, 6924187828544, 20240719-143603_6930364600025, 0.019, 0.434, 0.038 +diff, 6924509902754, 20240719-143630_6924509902839, 0.086, 0.575, 0.209 +diff, 6954256905319, 20240719-143729_6902007115524, -0.054, 0.437, -0.165 +diff, 6951469100067, 20240719-143807_6930628801472, 0.002, 0.371, 0.003 +diff, 4897098800133, 20240719-144335_6908355010450, 0.015, 0.299, 0.037 +diff, 6918149576536, 20240719-144454_6924509902754, -0.030, 0.254, -0.045 +diff, 6941499112529, 20240719-144916_6925843410158, 0.023, 0.467, 0.053 +diff, 6924187895423, 20240719-145022_6941499112529, 0.008, 0.411, 0.015 +diff, 6920907803020, 20240719-145058_6918149576536, -0.065, 0.395, -0.161 +diff, 6951469100067, 20240719-145542_6928002374551, -0.033, 0.407, -0.064 +diff, 6911988027191, 20240719-150000_6930364600001, -0.022, 0.391, -0.051 +diff, 6937451883132, 20240719-150209_6937962111526, -0.039, 0.302, -0.085 +diff, 6902890022558, 20240719-150345_6924509902839, 0.036, 0.571, 0.172 +diff, 6937451883132, 20240719-150458_6924509902839, -0.021, 0.399, -0.044 +diff, 6924509902839, 20240719-150623_6924187895423, 0.042, 0.716, 0.095 +diff, 6920382723257, 20240719-150716_6916749306102, 0.083, 0.524, 0.194 +diff, 6930364600001, 20240719-150750_6951469100067, -0.059, 0.492, -0.139 +diff, 6920907803020, 20240719-151215_6958350501101, -0.082, 0.362, -0.194 +diff, 6924509902839, 20240719-151325_6951469100067, 0.054, 0.509, 0.104 +diff, 6925843410158, 20240719-151349_6937962111526, -0.088, 0.325, -0.288 +diff, 6974158894528, 20240719-151416_6916749306102, -0.002, 0.455, -0.005 +diff, 6924187828544, 20240719-151535_6930364600001, 0.049, 0.513, 0.150 +diff, 6924187895423, 20240719-151724_6911988027191, 0.007, 0.373, 0.012 +diff, 6901668006011, 20240719-151800_6972219890632, -0.029, 0.352, -0.062 +diff, 6918598120014, 20240719-152125_6901668935908, 0.003, 0.456, 0.006 +diff, 6902007115524, 20240719-153117_6930364600001, -0.058, 0.466, -0.157 +diff, 6970470331529, 20240722-144630_6902265587712, -0.019, 0.431, -0.042 +diff, 6901668935908, 20240722-160713_4897073991658, 0.029, 0.417, 0.069 +diff, 6916749306102, 20240722-160857_4892655902237, 0.079, 0.549, 0.183 +diff, 6924187895423, 20240722-161533_6920907803020, 0.006, 0.424, 0.015 +diff, 6937962111526, 20240722-163428_6920912342002, 0.019, 0.498, 0.066 +diff, 6902890022558, 20240722-164859_6916196422325, 0.004, 0.464, 0.013 +diff, 6902890022558, 20240722-165548_6901668006011, -0.064, 0.390, -0.195 +diff, 6970399922518, 20240723-094426_6937221716189, -0.053, 0.428, -0.115 +diff, 6954256905319, 20240723-094447_4897098800133, 0.029, 0.492, 0.076 +diff, 6974158894528, 20240723-094617_6970470331529, 0.036, 0.425, 0.094 +diff, 6918149576536, 20240723-095103_6918598120014, 0.001, 0.404, 0.001 +diff, 6930628801472, 20240723-095527_6938737200650, -0.041, 0.444, -0.112 +diff, 6928002374551, 20240723-095858_6902890022558, 0.013, 0.441, 0.041 +diff, 6938737200650, 20240723-100031_6925303795535, -0.004, 0.413, -0.010 +diff, 6918149576536, 20240723-100113_6920382723257, -0.085, 0.294, -0.137 +diff, 6901668006011, 20240723-100416_6938737200650, -0.051, 0.309, -0.100 +diff, 6911988027191, 20240723-100501_6918598120014, 0.003, 0.295, 0.006 +diff, 6958350501101, 20240723-100726_6937221716189, 0.073, 0.374, 0.106 +diff, 6925843410158, 20240723-100845_4897098800133, -0.051, 0.328, -0.144 +diff, 4897098800133, 20240723-101829_6918598120014, 0.079, 0.317, 0.138 +diff, 4897098800133, 20240723-102033_6920382723257, 0.004, 0.298, 0.009 +diff, 6920907803020, 20240723-102206_6902890022558, 0.034, 0.441, 0.091 +diff, 6954256905319, 20240723-102409_6970470331529, 0.029, 0.437, 0.110 +diff, 6925843410158, 20240723-102515_6938737200650, -0.005, 0.406, -0.013 +diff, 6920912342002, 20240723-102833_6937221716189, 0.071, 0.536, 0.235 +diff, 6924187828544, 20240723-110216_6937451883132, 0.026, 0.358, 0.053 +diff, 6937221716189, 20240723-110224_6973337010216, 0.013, 0.382, 0.030 +diff, 6920382723257, 20240723-111446_6937451883132, 0.041, 0.469, 0.079 +diff, 6920907803020, 20240723-111521_6973337010216, 0.040, 0.524, 0.111 +diff, 6908355010450, 20240723-111602_6924187828544, -0.021, 0.475, -0.040 +diff, 6901668935908, 20240723-112004_6934024590268, -0.084, 0.389, -0.184 +diff, 4897073991658, 20240723-112505_6924187828544, 0.021, 0.297, 0.041 +diff, 4897098800133, 20240723-155423_6974158894528, 0.015, 0.287, 0.026 +diff, 6930628801472, 20240723-160227_6970399922518, 0.034, 0.447, 0.099 +diff, 6970470331529, 20240723-160942_6974158894528, 0.003, 0.398, 0.006 diff --git a/contrast/result/record/20240909_085740_uint8.txt b/contrast/result/record/20240909_085740_uint8.txt new file mode 100644 index 0000000..4afbd9e --- /dev/null +++ b/contrast/result/record/20240909_085740_uint8.txt @@ -0,0 +1,124 @@ +same, 6954256905319, 20240715-144627_6954256905319, 0.213, 0.751, 0.632 +same, 6930364600025, 20240719-143603_6930364600025, 0.283, 0.771, 0.617 +same, 6924509902839, 20240719-143630_6924509902839, 0.182, 0.726, 0.462 +same, 6902007115524, 20240719-143729_6902007115524, 0.221, 0.722, 0.518 +same, 6930628801472, 20240719-143807_6930628801472, 0.250, 0.804, 0.673 +same, 6908355010450, 20240719-144335_6908355010450, 0.256, 0.812, 0.576 +same, 6924509902754, 20240719-144454_6924509902754, 0.457, 0.881, 0.835 +same, 6925843410158, 20240719-144916_6925843410158, 0.228, 0.829, 0.686 +same, 6941499112529, 20240719-145022_6941499112529, 0.191, 0.795, 0.360 +same, 6918149576536, 20240719-145058_6918149576536, 0.494, 0.906, 0.766 +same, 6928002374551, 20240719-145542_6928002374551, 0.166, 0.772, 0.532 +same, 6930364600001, 20240719-150000_6930364600001, 0.322, 0.831, 0.735 +same, 6937962111526, 20240719-150209_6937962111526, 0.225, 0.794, 0.604 +same, 6924509902839, 20240719-150345_6924509902839, 0.161, 0.817, 0.502 +same, 6924509902839, 20240719-150458_6924509902839, 0.319, 0.746, 0.623 +same, 6924187895423, 20240719-150623_6924187895423, 0.429, 0.906, 0.903 +same, 6916749306102, 20240719-150716_6916749306102, 0.268, 0.763, 0.737 +same, 6951469100067, 20240719-150750_6951469100067, 0.546, 0.937, 0.895 +same, 6958350501101, 20240719-151215_6958350501101, 0.580, 0.907, 0.878 +same, 6951469100067, 20240719-151325_6951469100067, 0.610, 0.930, 0.903 +same, 6937962111526, 20240719-151349_6937962111526, 0.138, 0.761, 0.495 +same, 6916749306102, 20240719-151416_6916749306102, 0.242, 0.727, 0.629 +same, 6930364600001, 20240719-151535_6930364600001, 0.144, 0.788, 0.482 +same, 6911988027191, 20240719-151724_6911988027191, 0.037, 0.502, 0.079 +same, 6972219890632, 20240719-151800_6972219890632, 0.314, 0.880, 0.699 +same, 6901668935908, 20240719-152125_6901668935908, 0.204, 0.856, 0.428 +same, 6930364600001, 20240719-153117_6930364600001, 0.056, 0.775, 0.164 +same, 6902265587712, 20240722-144630_6902265587712, 0.365, 0.833, 0.669 +same, 4897073991658, 20240722-160713_4897073991658, 0.103, 0.516, 0.251 +same, 4892655902237, 20240722-160857_4892655902237, 0.460, 0.760, 0.812 +same, 6920907803020, 20240722-161533_6920907803020, 0.212, 0.770, 0.679 +same, 6920912342002, 20240722-163428_6920912342002, 0.282, 0.825, 0.853 +same, 6916196422325, 20240722-164859_6916196422325, 0.392, 0.836, 0.665 +same, 6901668006011, 20240722-165548_6901668006011, 0.284, 0.751, 0.530 +same, 6937221716189, 20240723-094426_6937221716189, 0.408, 0.850, 0.706 +same, 4897098800133, 20240723-094447_4897098800133, 0.382, 0.846, 0.784 +same, 6970470331529, 20240723-094617_6970470331529, 0.237, 0.925, 0.585 +same, 6918598120014, 20240723-095103_6918598120014, 0.408, 0.905, 0.820 +same, 6938737200650, 20240723-095527_6938737200650, 0.208, 0.824, 0.544 +same, 6902890022558, 20240723-095858_6902890022558, 0.163, 0.770, 0.629 +same, 6925303795535, 20240723-100031_6925303795535, 0.305, 0.835, 0.672 +same, 6920382723257, 20240723-100113_6920382723257, 0.274, 0.853, 0.539 +same, 6938737200650, 20240723-100416_6938737200650, 0.237, 0.720, 0.582 +same, 6918598120014, 20240723-100501_6918598120014, 0.099, 0.790, 0.210 +same, 6937221716189, 20240723-100726_6937221716189, 0.347, 0.772, 0.612 +same, 4897098800133, 20240723-100845_4897098800133, 0.304, 0.770, 0.717 +same, 6918598120014, 20240723-101829_6918598120014, 0.502, 0.919, 0.857 +same, 6920382723257, 20240723-102033_6920382723257, 0.343, 0.849, 0.748 +same, 6902890022558, 20240723-102206_6902890022558, 0.118, 0.745, 0.391 +same, 6970470331529, 20240723-102409_6970470331529, 0.103, 0.734, 0.307 +same, 6938737200650, 20240723-102515_6938737200650, 0.184, 0.724, 0.470 +same, 6937221716189, 20240723-102833_6937221716189, 0.067, 0.623, 0.175 +same, 6937451883132, 20240723-110216_6937451883132, 0.323, 0.861, 0.717 +same, 6973337010216, 20240723-110224_6973337010216, 0.202, 0.760, 0.623 +same, 6937451883132, 20240723-111446_6937451883132, 0.367, 0.817, 0.806 +same, 6973337010216, 20240723-111521_6973337010216, 0.140, 0.777, 0.415 +same, 6924187828544, 20240723-111602_6924187828544, 0.358, 0.860, 0.718 +same, 6934024590268, 20240723-112004_6934024590268, 0.186, 0.728, 0.585 +same, 6924187828544, 20240723-112505_6924187828544, 0.281, 0.875, 0.573 +same, 6974158894528, 20240723-155423_6974158894528, 0.434, 0.889, 0.789 +same, 6970399922518, 20240723-160227_6970399922518, 0.078, 0.564, 0.216 +same, 6974158894528, 20240723-160942_6974158894528, 0.418, 0.895, 0.833 +diff, 6924509902839, 20240715-144627_6954256905319, -0.036, 0.356, -0.082 +diff, 6924187828544, 20240719-143603_6930364600025, 0.019, 0.435, 0.039 +diff, 6924509902754, 20240719-143630_6924509902839, 0.086, 0.574, 0.208 +diff, 6954256905319, 20240719-143729_6902007115524, -0.054, 0.437, -0.164 +diff, 6951469100067, 20240719-143807_6930628801472, 0.002, 0.371, 0.003 +diff, 4897098800133, 20240719-144335_6908355010450, 0.015, 0.298, 0.036 +diff, 6918149576536, 20240719-144454_6924509902754, -0.031, 0.252, -0.046 +diff, 6941499112529, 20240719-144916_6925843410158, 0.022, 0.468, 0.053 +diff, 6924187895423, 20240719-145022_6941499112529, 0.008, 0.410, 0.015 +diff, 6920907803020, 20240719-145058_6918149576536, -0.064, 0.396, -0.157 +diff, 6951469100067, 20240719-145542_6928002374551, -0.033, 0.407, -0.064 +diff, 6911988027191, 20240719-150000_6930364600001, -0.022, 0.389, -0.051 +diff, 6937451883132, 20240719-150209_6937962111526, -0.040, 0.303, -0.085 +diff, 6902890022558, 20240719-150345_6924509902839, 0.036, 0.570, 0.173 +diff, 6937451883132, 20240719-150458_6924509902839, -0.021, 0.401, -0.044 +diff, 6924509902839, 20240719-150623_6924187895423, 0.041, 0.715, 0.093 +diff, 6920382723257, 20240719-150716_6916749306102, 0.083, 0.524, 0.192 +diff, 6930364600001, 20240719-150750_6951469100067, -0.060, 0.492, -0.142 +diff, 6920907803020, 20240719-151215_6958350501101, -0.079, 0.363, -0.187 +diff, 6924509902839, 20240719-151325_6951469100067, 0.053, 0.506, 0.102 +diff, 6925843410158, 20240719-151349_6937962111526, -0.089, 0.325, -0.291 +diff, 6974158894528, 20240719-151416_6916749306102, -0.003, 0.454, -0.007 +diff, 6924187828544, 20240719-151535_6930364600001, 0.049, 0.513, 0.149 +diff, 6924187895423, 20240719-151724_6911988027191, 0.008, 0.375, 0.014 +diff, 6901668006011, 20240719-151800_6972219890632, -0.029, 0.353, -0.063 +diff, 6918598120014, 20240719-152125_6901668935908, 0.004, 0.458, 0.008 +diff, 6902007115524, 20240719-153117_6930364600001, -0.059, 0.466, -0.158 +diff, 6970470331529, 20240722-144630_6902265587712, -0.018, 0.431, -0.041 +diff, 6901668935908, 20240722-160713_4897073991658, 0.031, 0.419, 0.074 +diff, 6916749306102, 20240722-160857_4892655902237, 0.079, 0.549, 0.183 +diff, 6924187895423, 20240722-161533_6920907803020, 0.006, 0.424, 0.014 +diff, 6937962111526, 20240722-163428_6920912342002, 0.020, 0.498, 0.068 +diff, 6902890022558, 20240722-164859_6916196422325, 0.004, 0.463, 0.013 +diff, 6902890022558, 20240722-165548_6901668006011, -0.064, 0.389, -0.193 +diff, 6970399922518, 20240723-094426_6937221716189, -0.052, 0.428, -0.112 +diff, 6954256905319, 20240723-094447_4897098800133, 0.029, 0.491, 0.078 +diff, 6974158894528, 20240723-094617_6970470331529, 0.036, 0.426, 0.094 +diff, 6918149576536, 20240723-095103_6918598120014, 0.002, 0.406, 0.003 +diff, 6930628801472, 20240723-095527_6938737200650, -0.040, 0.446, -0.110 +diff, 6928002374551, 20240723-095858_6902890022558, 0.014, 0.442, 0.043 +diff, 6938737200650, 20240723-100031_6925303795535, -0.004, 0.415, -0.009 +diff, 6918149576536, 20240723-100113_6920382723257, -0.085, 0.293, -0.136 +diff, 6901668006011, 20240723-100416_6938737200650, -0.050, 0.310, -0.098 +diff, 6911988027191, 20240723-100501_6918598120014, 0.004, 0.296, 0.008 +diff, 6958350501101, 20240723-100726_6937221716189, 0.075, 0.374, 0.109 +diff, 6925843410158, 20240723-100845_4897098800133, -0.051, 0.328, -0.146 +diff, 4897098800133, 20240723-101829_6918598120014, 0.078, 0.316, 0.137 +diff, 4897098800133, 20240723-102033_6920382723257, 0.003, 0.298, 0.007 +diff, 6920907803020, 20240723-102206_6902890022558, 0.036, 0.443, 0.095 +diff, 6954256905319, 20240723-102409_6970470331529, 0.030, 0.437, 0.116 +diff, 6925843410158, 20240723-102515_6938737200650, -0.005, 0.404, -0.013 +diff, 6920912342002, 20240723-102833_6937221716189, 0.071, 0.536, 0.235 +diff, 6924187828544, 20240723-110216_6937451883132, 0.026, 0.360, 0.053 +diff, 6937221716189, 20240723-110224_6973337010216, 0.014, 0.382, 0.031 +diff, 6920382723257, 20240723-111446_6937451883132, 0.041, 0.470, 0.079 +diff, 6920907803020, 20240723-111521_6973337010216, 0.041, 0.524, 0.113 +diff, 6908355010450, 20240723-111602_6924187828544, -0.021, 0.474, -0.039 +diff, 6901668935908, 20240723-112004_6934024590268, -0.083, 0.390, -0.180 +diff, 4897073991658, 20240723-112505_6924187828544, 0.022, 0.295, 0.043 +diff, 4897098800133, 20240723-155423_6974158894528, 0.016, 0.287, 0.027 +diff, 6930628801472, 20240723-160227_6970399922518, 0.034, 0.447, 0.099 +diff, 6970470331529, 20240723-160942_6974158894528, 0.004, 0.398, 0.007 diff --git a/contrast/result/record/20240909_094404.txt b/contrast/result/record/20240909_094404.txt new file mode 100644 index 0000000..9245ab8 --- /dev/null +++ b/contrast/result/record/20240909_094404.txt @@ -0,0 +1,124 @@ +same, 6954256905319, 20240715-144627_6954256905319, 0.211, 0.750, 0.630 +same, 6930364600025, 20240719-143603_6930364600025, 0.283, 0.771, 0.616 +same, 6924509902839, 20240719-143630_6924509902839, 0.182, 0.727, 0.462 +same, 6902007115524, 20240719-143729_6902007115524, 0.221, 0.723, 0.519 +same, 6930628801472, 20240719-143807_6930628801472, 0.249, 0.804, 0.672 +same, 6908355010450, 20240719-144335_6908355010450, 0.255, 0.812, 0.575 +same, 6924509902754, 20240719-144454_6924509902754, 0.458, 0.882, 0.836 +same, 6925843410158, 20240719-144916_6925843410158, 0.229, 0.829, 0.686 +same, 6941499112529, 20240719-145022_6941499112529, 0.190, 0.795, 0.359 +same, 6918149576536, 20240719-145058_6918149576536, 0.493, 0.906, 0.765 +same, 6928002374551, 20240719-145542_6928002374551, 0.165, 0.773, 0.531 +same, 6930364600001, 20240719-150000_6930364600001, 0.322, 0.831, 0.736 +same, 6937962111526, 20240719-150209_6937962111526, 0.225, 0.794, 0.605 +same, 6924509902839, 20240719-150345_6924509902839, 0.161, 0.816, 0.502 +same, 6924509902839, 20240719-150458_6924509902839, 0.319, 0.746, 0.623 +same, 6924187895423, 20240719-150623_6924187895423, 0.430, 0.906, 0.903 +same, 6916749306102, 20240719-150716_6916749306102, 0.269, 0.764, 0.737 +same, 6951469100067, 20240719-150750_6951469100067, 0.547, 0.938, 0.896 +same, 6958350501101, 20240719-151215_6958350501101, 0.578, 0.906, 0.877 +same, 6951469100067, 20240719-151325_6951469100067, 0.610, 0.930, 0.904 +same, 6937962111526, 20240719-151349_6937962111526, 0.138, 0.762, 0.496 +same, 6916749306102, 20240719-151416_6916749306102, 0.243, 0.728, 0.630 +same, 6930364600001, 20240719-151535_6930364600001, 0.144, 0.787, 0.482 +same, 6911988027191, 20240719-151724_6911988027191, 0.035, 0.502, 0.076 +same, 6972219890632, 20240719-151800_6972219890632, 0.315, 0.880, 0.700 +same, 6901668935908, 20240719-152125_6901668935908, 0.203, 0.856, 0.426 +same, 6930364600001, 20240719-153117_6930364600001, 0.057, 0.775, 0.165 +same, 6902265587712, 20240722-144630_6902265587712, 0.364, 0.833, 0.669 +same, 4897073991658, 20240722-160713_4897073991658, 0.101, 0.514, 0.247 +same, 4892655902237, 20240722-160857_4892655902237, 0.459, 0.760, 0.812 +same, 6920907803020, 20240722-161533_6920907803020, 0.211, 0.769, 0.678 +same, 6920912342002, 20240722-163428_6920912342002, 0.281, 0.825, 0.853 +same, 6916196422325, 20240722-164859_6916196422325, 0.392, 0.837, 0.665 +same, 6901668006011, 20240722-165548_6901668006011, 0.284, 0.752, 0.530 +same, 6937221716189, 20240723-094426_6937221716189, 0.407, 0.850, 0.705 +same, 4897098800133, 20240723-094447_4897098800133, 0.382, 0.846, 0.784 +same, 6970470331529, 20240723-094617_6970470331529, 0.238, 0.925, 0.586 +same, 6918598120014, 20240723-095103_6918598120014, 0.407, 0.905, 0.819 +same, 6938737200650, 20240723-095527_6938737200650, 0.208, 0.825, 0.543 +same, 6902890022558, 20240723-095858_6902890022558, 0.162, 0.769, 0.628 +same, 6925303795535, 20240723-100031_6925303795535, 0.304, 0.835, 0.671 +same, 6920382723257, 20240723-100113_6920382723257, 0.274, 0.854, 0.539 +same, 6938737200650, 20240723-100416_6938737200650, 0.237, 0.719, 0.581 +same, 6918598120014, 20240723-100501_6918598120014, 0.099, 0.791, 0.209 +same, 6937221716189, 20240723-100726_6937221716189, 0.347, 0.771, 0.611 +same, 4897098800133, 20240723-100845_4897098800133, 0.304, 0.770, 0.718 +same, 6918598120014, 20240723-101829_6918598120014, 0.502, 0.919, 0.857 +same, 6920382723257, 20240723-102033_6920382723257, 0.343, 0.849, 0.748 +same, 6902890022558, 20240723-102206_6902890022558, 0.117, 0.745, 0.389 +same, 6970470331529, 20240723-102409_6970470331529, 0.102, 0.734, 0.305 +same, 6938737200650, 20240723-102515_6938737200650, 0.183, 0.725, 0.469 +same, 6937221716189, 20240723-102833_6937221716189, 0.067, 0.623, 0.175 +same, 6937451883132, 20240723-110216_6937451883132, 0.323, 0.860, 0.717 +same, 6973337010216, 20240723-110224_6973337010216, 0.202, 0.760, 0.622 +same, 6937451883132, 20240723-111446_6937451883132, 0.367, 0.817, 0.806 +same, 6973337010216, 20240723-111521_6973337010216, 0.140, 0.777, 0.414 +same, 6924187828544, 20240723-111602_6924187828544, 0.359, 0.859, 0.718 +same, 6934024590268, 20240723-112004_6934024590268, 0.184, 0.727, 0.583 +same, 6924187828544, 20240723-112505_6924187828544, 0.281, 0.875, 0.573 +same, 6974158894528, 20240723-155423_6974158894528, 0.433, 0.890, 0.788 +same, 6970399922518, 20240723-160227_6970399922518, 0.078, 0.564, 0.215 +same, 6974158894528, 20240723-160942_6974158894528, 0.417, 0.895, 0.832 +diff, 6916749306102, 20240715-144627_6954256905319, 0.010, 0.449, 0.026 +diff, 6911988027191, 20240719-143603_6930364600025, -0.004, 0.426, -0.008 +diff, 6934024590268, 20240719-143630_6924509902839, -0.003, 0.922, -0.011 +diff, 6916196422325, 20240719-143729_6902007115524, -0.023, 0.470, -0.042 +diff, 6918598120014, 20240719-143807_6930628801472, -0.010, 0.627, -0.021 +diff, 6938737200650, 20240719-144335_6908355010450, 0.040, 0.456, 0.113 +diff, 6972219890632, 20240719-144454_6924509902754, 0.093, 0.537, 0.174 +diff, 6918149576536, 20240719-144916_6925843410158, 0.021, 0.354, 0.039 +diff, 6908355010450, 20240719-145022_6941499112529, -0.066, 0.370, -0.124 +diff, 6920912342002, 20240719-145058_6918149576536, -0.055, 0.362, -0.131 +diff, 6924187895423, 20240719-145542_6928002374551, -0.016, 0.421, -0.037 +diff, 4892655902237, 20240719-150000_6930364600001, 0.051, 0.353, 0.091 +diff, 6901668935908, 20240719-150209_6937962111526, -0.044, 0.357, -0.081 +diff, 6972219890632, 20240719-150345_6924509902839, 0.061, 0.596, 0.187 +diff, 6901668935908, 20240719-150458_6924509902839, 0.035, 0.394, 0.064 +diff, 6902265587712, 20240719-150623_6924187895423, 0.005, 0.433, 0.010 +diff, 6901668006011, 20240719-150716_6916749306102, 0.020, 0.361, 0.047 +diff, 4897098800133, 20240719-150750_6951469100067, 0.157, 0.474, 0.340 +diff, 6920907803020, 20240719-151215_6958350501101, -0.082, 0.363, -0.194 +diff, 6925303795535, 20240719-151325_6951469100067, -0.012, 0.283, -0.024 +diff, 6974158894528, 20240719-151349_6937962111526, -0.060, 0.423, -0.172 +diff, 6937221716189, 20240719-151416_6916749306102, -0.053, 0.459, -0.116 +diff, 6937962111526, 20240719-151535_6930364600001, -0.011, 0.399, -0.046 +diff, 6941499112529, 20240719-151724_6911988027191, -0.013, 0.463, -0.025 +diff, 6958350501101, 20240719-151800_6972219890632, -0.029, 0.358, -0.053 +diff, 6924187895423, 20240719-152125_6901668935908, -0.048, 0.352, -0.099 +diff, 6918598120014, 20240719-153117_6930364600001, -0.034, 0.470, -0.090 +diff, 6924509902839, 20240722-144630_6902265587712, -0.025, 0.366, -0.053 +diff, 6954256905319, 20240722-160713_4897073991658, 0.122, 0.598, 0.401 +diff, 6938737200650, 20240722-160857_4892655902237, 0.028, 0.456, 0.069 +diff, 6916749306102, 20240722-161533_6920907803020, 0.047, 0.574, 0.137 +diff, 6924187895423, 20240722-163428_6920912342002, 0.013, 0.513, 0.031 +diff, 6901668935908, 20240722-164859_6916196422325, -0.060, 0.385, -0.124 +diff, 6938737200650, 20240722-165548_6901668006011, 0.046, 0.439, 0.107 +diff, 6930364600001, 20240723-094426_6937221716189, -0.089, 0.377, -0.179 +diff, 6930364600001, 20240723-094447_4897098800133, 0.009, 0.429, 0.019 +diff, 6925303795535, 20240723-094617_6970470331529, 0.006, 0.361, 0.016 +diff, 6972219890632, 20240723-095103_6918598120014, 0.058, 0.526, 0.115 +diff, 4897098800133, 20240723-095527_6938737200650, 0.011, 0.247, 0.026 +diff, 6925303795535, 20240723-095858_6902890022558, 0.014, 0.519, 0.037 +diff, 6916749306102, 20240723-100031_6925303795535, -0.066, 0.361, -0.152 +diff, 6901668006011, 20240723-100113_6920382723257, -0.037, 0.492, -0.072 +diff, 6920382723257, 20240723-100416_6938737200650, 0.084, 0.402, 0.165 +diff, 6901668935908, 20240723-100501_6918598120014, 0.046, 0.356, 0.092 +diff, 6916196422325, 20240723-100726_6937221716189, 0.037, 0.288, 0.053 +diff, 6902007115524, 20240723-100845_4897098800133, -0.022, 0.368, -0.052 +diff, 6924509902839, 20240723-101829_6918598120014, 0.019, 0.417, 0.033 +diff, 6972219890632, 20240723-102033_6920382723257, 0.027, 0.520, 0.061 +diff, 6916749306102, 20240723-102206_6902890022558, -0.019, 0.343, -0.046 +diff, 6916196422325, 20240723-102409_6970470331529, -0.009, 0.288, -0.020 +diff, 6918598120014, 20240723-102515_6938737200650, 0.025, 0.390, 0.054 +diff, 6930628801472, 20240723-102833_6937221716189, 0.014, 0.360, 0.047 +diff, 6973337010216, 20240723-110216_6937451883132, -0.013, 0.385, -0.036 +diff, 6901668935908, 20240723-110224_6973337010216, -0.080, 0.356, -0.175 +diff, 6941499112529, 20240723-111446_6937451883132, -0.053, 0.360, -0.103 +diff, 6916749306102, 20240723-111521_6973337010216, -0.064, 0.535, -0.161 +diff, 6937451883132, 20240723-111602_6924187828544, 0.024, 0.351, 0.051 +diff, 6908355010450, 20240723-112004_6934024590268, 0.048, 0.426, 0.107 +diff, 6934024590268, 20240723-112505_6924187828544, -0.017, 0.395, -0.046 +diff, 6937962111526, 20240723-155423_6974158894528, -0.056, 0.364, -0.129 +diff, 6916196422325, 20240723-160227_6970399922518, 0.048, 0.390, 0.087 +diff, 6958350501101, 20240723-160942_6974158894528, 0.042, 0.426, 0.062 diff --git a/contrast/result/record/20240909_094404_ft16.txt b/contrast/result/record/20240909_094404_ft16.txt new file mode 100644 index 0000000..d4463a8 --- /dev/null +++ b/contrast/result/record/20240909_094404_ft16.txt @@ -0,0 +1,124 @@ +same, 6954256905319, 20240715-144627_6954256905319, 0.211, 0.751, 0.630 +same, 6930364600025, 20240719-143603_6930364600025, 0.283, 0.771, 0.616 +same, 6924509902839, 20240719-143630_6924509902839, 0.182, 0.727, 0.462 +same, 6902007115524, 20240719-143729_6902007115524, 0.221, 0.723, 0.519 +same, 6930628801472, 20240719-143807_6930628801472, 0.249, 0.804, 0.672 +same, 6908355010450, 20240719-144335_6908355010450, 0.255, 0.812, 0.575 +same, 6924509902754, 20240719-144454_6924509902754, 0.458, 0.882, 0.836 +same, 6925843410158, 20240719-144916_6925843410158, 0.229, 0.829, 0.686 +same, 6941499112529, 20240719-145022_6941499112529, 0.190, 0.795, 0.359 +same, 6918149576536, 20240719-145058_6918149576536, 0.493, 0.906, 0.765 +same, 6928002374551, 20240719-145542_6928002374551, 0.165, 0.773, 0.531 +same, 6930364600001, 20240719-150000_6930364600001, 0.322, 0.831, 0.736 +same, 6937962111526, 20240719-150209_6937962111526, 0.225, 0.794, 0.605 +same, 6924509902839, 20240719-150345_6924509902839, 0.161, 0.816, 0.502 +same, 6924509902839, 20240719-150458_6924509902839, 0.319, 0.746, 0.623 +same, 6924187895423, 20240719-150623_6924187895423, 0.430, 0.906, 0.903 +same, 6916749306102, 20240719-150716_6916749306102, 0.269, 0.764, 0.737 +same, 6951469100067, 20240719-150750_6951469100067, 0.547, 0.938, 0.896 +same, 6958350501101, 20240719-151215_6958350501101, 0.578, 0.906, 0.877 +same, 6951469100067, 20240719-151325_6951469100067, 0.610, 0.930, 0.904 +same, 6937962111526, 20240719-151349_6937962111526, 0.138, 0.762, 0.496 +same, 6916749306102, 20240719-151416_6916749306102, 0.243, 0.728, 0.630 +same, 6930364600001, 20240719-151535_6930364600001, 0.144, 0.787, 0.482 +same, 6911988027191, 20240719-151724_6911988027191, 0.035, 0.502, 0.076 +same, 6972219890632, 20240719-151800_6972219890632, 0.315, 0.880, 0.700 +same, 6901668935908, 20240719-152125_6901668935908, 0.203, 0.856, 0.426 +same, 6930364600001, 20240719-153117_6930364600001, 0.057, 0.775, 0.165 +same, 6902265587712, 20240722-144630_6902265587712, 0.364, 0.833, 0.669 +same, 4897073991658, 20240722-160713_4897073991658, 0.101, 0.514, 0.247 +same, 4892655902237, 20240722-160857_4892655902237, 0.459, 0.760, 0.812 +same, 6920907803020, 20240722-161533_6920907803020, 0.211, 0.769, 0.678 +same, 6920912342002, 20240722-163428_6920912342002, 0.281, 0.825, 0.853 +same, 6916196422325, 20240722-164859_6916196422325, 0.392, 0.837, 0.665 +same, 6901668006011, 20240722-165548_6901668006011, 0.284, 0.751, 0.530 +same, 6937221716189, 20240723-094426_6937221716189, 0.407, 0.850, 0.705 +same, 4897098800133, 20240723-094447_4897098800133, 0.382, 0.846, 0.784 +same, 6970470331529, 20240723-094617_6970470331529, 0.238, 0.925, 0.586 +same, 6918598120014, 20240723-095103_6918598120014, 0.407, 0.905, 0.819 +same, 6938737200650, 20240723-095527_6938737200650, 0.208, 0.825, 0.543 +same, 6902890022558, 20240723-095858_6902890022558, 0.162, 0.769, 0.628 +same, 6925303795535, 20240723-100031_6925303795535, 0.304, 0.835, 0.671 +same, 6920382723257, 20240723-100113_6920382723257, 0.274, 0.854, 0.539 +same, 6938737200650, 20240723-100416_6938737200650, 0.237, 0.719, 0.581 +same, 6918598120014, 20240723-100501_6918598120014, 0.099, 0.791, 0.209 +same, 6937221716189, 20240723-100726_6937221716189, 0.347, 0.771, 0.611 +same, 4897098800133, 20240723-100845_4897098800133, 0.304, 0.770, 0.717 +same, 6918598120014, 20240723-101829_6918598120014, 0.502, 0.919, 0.857 +same, 6920382723257, 20240723-102033_6920382723257, 0.343, 0.849, 0.748 +same, 6902890022558, 20240723-102206_6902890022558, 0.117, 0.745, 0.389 +same, 6970470331529, 20240723-102409_6970470331529, 0.102, 0.734, 0.305 +same, 6938737200650, 20240723-102515_6938737200650, 0.183, 0.725, 0.469 +same, 6937221716189, 20240723-102833_6937221716189, 0.067, 0.623, 0.175 +same, 6937451883132, 20240723-110216_6937451883132, 0.323, 0.860, 0.717 +same, 6973337010216, 20240723-110224_6973337010216, 0.202, 0.760, 0.622 +same, 6937451883132, 20240723-111446_6937451883132, 0.367, 0.817, 0.806 +same, 6973337010216, 20240723-111521_6973337010216, 0.140, 0.777, 0.414 +same, 6924187828544, 20240723-111602_6924187828544, 0.359, 0.859, 0.718 +same, 6934024590268, 20240723-112004_6934024590268, 0.184, 0.727, 0.583 +same, 6924187828544, 20240723-112505_6924187828544, 0.281, 0.875, 0.573 +same, 6974158894528, 20240723-155423_6974158894528, 0.433, 0.890, 0.788 +same, 6970399922518, 20240723-160227_6970399922518, 0.078, 0.564, 0.215 +same, 6974158894528, 20240723-160942_6974158894528, 0.417, 0.895, 0.832 +diff, 6916749306102, 20240715-144627_6954256905319, 0.010, 0.449, 0.026 +diff, 6911988027191, 20240719-143603_6930364600025, -0.004, 0.426, -0.008 +diff, 6934024590268, 20240719-143630_6924509902839, -0.003, 0.922, -0.011 +diff, 6916196422325, 20240719-143729_6902007115524, -0.023, 0.470, -0.042 +diff, 6918598120014, 20240719-143807_6930628801472, -0.010, 0.627, -0.021 +diff, 6938737200650, 20240719-144335_6908355010450, 0.040, 0.456, 0.113 +diff, 6972219890632, 20240719-144454_6924509902754, 0.093, 0.537, 0.174 +diff, 6918149576536, 20240719-144916_6925843410158, 0.021, 0.354, 0.039 +diff, 6908355010450, 20240719-145022_6941499112529, -0.066, 0.370, -0.124 +diff, 6920912342002, 20240719-145058_6918149576536, -0.055, 0.362, -0.131 +diff, 6924187895423, 20240719-145542_6928002374551, -0.016, 0.421, -0.037 +diff, 4892655902237, 20240719-150000_6930364600001, 0.051, 0.353, 0.091 +diff, 6901668935908, 20240719-150209_6937962111526, -0.044, 0.357, -0.081 +diff, 6972219890632, 20240719-150345_6924509902839, 0.061, 0.596, 0.187 +diff, 6901668935908, 20240719-150458_6924509902839, 0.035, 0.394, 0.064 +diff, 6902265587712, 20240719-150623_6924187895423, 0.005, 0.433, 0.010 +diff, 6901668006011, 20240719-150716_6916749306102, 0.020, 0.361, 0.047 +diff, 4897098800133, 20240719-150750_6951469100067, 0.157, 0.474, 0.340 +diff, 6920907803020, 20240719-151215_6958350501101, -0.082, 0.362, -0.194 +diff, 6925303795535, 20240719-151325_6951469100067, -0.012, 0.283, -0.024 +diff, 6974158894528, 20240719-151349_6937962111526, -0.060, 0.423, -0.173 +diff, 6937221716189, 20240719-151416_6916749306102, -0.053, 0.459, -0.116 +diff, 6937962111526, 20240719-151535_6930364600001, -0.011, 0.399, -0.046 +diff, 6941499112529, 20240719-151724_6911988027191, -0.013, 0.463, -0.025 +diff, 6958350501101, 20240719-151800_6972219890632, -0.029, 0.358, -0.053 +diff, 6924187895423, 20240719-152125_6901668935908, -0.048, 0.352, -0.099 +diff, 6918598120014, 20240719-153117_6930364600001, -0.034, 0.470, -0.090 +diff, 6924509902839, 20240722-144630_6902265587712, -0.025, 0.366, -0.053 +diff, 6954256905319, 20240722-160713_4897073991658, 0.122, 0.598, 0.401 +diff, 6938737200650, 20240722-160857_4892655902237, 0.028, 0.456, 0.069 +diff, 6916749306102, 20240722-161533_6920907803020, 0.047, 0.574, 0.137 +diff, 6924187895423, 20240722-163428_6920912342002, 0.013, 0.513, 0.031 +diff, 6901668935908, 20240722-164859_6916196422325, -0.060, 0.385, -0.124 +diff, 6938737200650, 20240722-165548_6901668006011, 0.046, 0.439, 0.107 +diff, 6930364600001, 20240723-094426_6937221716189, -0.089, 0.377, -0.179 +diff, 6930364600001, 20240723-094447_4897098800133, 0.009, 0.429, 0.019 +diff, 6925303795535, 20240723-094617_6970470331529, 0.006, 0.361, 0.016 +diff, 6972219890632, 20240723-095103_6918598120014, 0.058, 0.526, 0.115 +diff, 4897098800133, 20240723-095527_6938737200650, 0.011, 0.247, 0.026 +diff, 6925303795535, 20240723-095858_6902890022558, 0.014, 0.519, 0.037 +diff, 6916749306102, 20240723-100031_6925303795535, -0.066, 0.361, -0.152 +diff, 6901668006011, 20240723-100113_6920382723257, -0.037, 0.492, -0.072 +diff, 6920382723257, 20240723-100416_6938737200650, 0.084, 0.402, 0.165 +diff, 6901668935908, 20240723-100501_6918598120014, 0.046, 0.356, 0.092 +diff, 6916196422325, 20240723-100726_6937221716189, 0.037, 0.288, 0.053 +diff, 6902007115524, 20240723-100845_4897098800133, -0.022, 0.368, -0.052 +diff, 6924509902839, 20240723-101829_6918598120014, 0.019, 0.417, 0.033 +diff, 6972219890632, 20240723-102033_6920382723257, 0.027, 0.520, 0.061 +diff, 6916749306102, 20240723-102206_6902890022558, -0.019, 0.343, -0.046 +diff, 6916196422325, 20240723-102409_6970470331529, -0.009, 0.288, -0.020 +diff, 6918598120014, 20240723-102515_6938737200650, 0.025, 0.390, 0.054 +diff, 6930628801472, 20240723-102833_6937221716189, 0.014, 0.360, 0.047 +diff, 6973337010216, 20240723-110216_6937451883132, -0.013, 0.385, -0.036 +diff, 6901668935908, 20240723-110224_6973337010216, -0.080, 0.356, -0.175 +diff, 6941499112529, 20240723-111446_6937451883132, -0.053, 0.360, -0.103 +diff, 6916749306102, 20240723-111521_6973337010216, -0.064, 0.535, -0.161 +diff, 6937451883132, 20240723-111602_6924187828544, 0.024, 0.351, 0.051 +diff, 6908355010450, 20240723-112004_6934024590268, 0.048, 0.426, 0.107 +diff, 6934024590268, 20240723-112505_6924187828544, -0.017, 0.395, -0.046 +diff, 6937962111526, 20240723-155423_6974158894528, -0.056, 0.364, -0.129 +diff, 6916196422325, 20240723-160227_6970399922518, 0.048, 0.390, 0.087 +diff, 6958350501101, 20240723-160942_6974158894528, 0.042, 0.426, 0.062 diff --git a/contrast/result/record/20240909_094404_uint8.txt b/contrast/result/record/20240909_094404_uint8.txt new file mode 100644 index 0000000..2a2021d --- /dev/null +++ b/contrast/result/record/20240909_094404_uint8.txt @@ -0,0 +1,124 @@ +same, 6954256905319, 20240715-144627_6954256905319, 0.211, 0.750, 0.630 +same, 6930364600025, 20240719-143603_6930364600025, 0.283, 0.772, 0.618 +same, 6924509902839, 20240719-143630_6924509902839, 0.182, 0.727, 0.464 +same, 6902007115524, 20240719-143729_6902007115524, 0.221, 0.721, 0.518 +same, 6930628801472, 20240719-143807_6930628801472, 0.249, 0.799, 0.672 +same, 6908355010450, 20240719-144335_6908355010450, 0.255, 0.809, 0.575 +same, 6924509902754, 20240719-144454_6924509902754, 0.456, 0.879, 0.835 +same, 6925843410158, 20240719-144916_6925843410158, 0.228, 0.820, 0.686 +same, 6941499112529, 20240719-145022_6941499112529, 0.190, 0.790, 0.360 +same, 6918149576536, 20240719-145058_6918149576536, 0.493, 0.905, 0.766 +same, 6928002374551, 20240719-145542_6928002374551, 0.165, 0.769, 0.530 +same, 6930364600001, 20240719-150000_6930364600001, 0.320, 0.834, 0.734 +same, 6937962111526, 20240719-150209_6937962111526, 0.224, 0.791, 0.604 +same, 6924509902839, 20240719-150345_6924509902839, 0.162, 0.812, 0.504 +same, 6924509902839, 20240719-150458_6924509902839, 0.320, 0.745, 0.625 +same, 6924187895423, 20240719-150623_6924187895423, 0.430, 0.907, 0.904 +same, 6916749306102, 20240719-150716_6916749306102, 0.268, 0.761, 0.738 +same, 6951469100067, 20240719-150750_6951469100067, 0.544, 0.935, 0.894 +same, 6958350501101, 20240719-151215_6958350501101, 0.576, 0.904, 0.876 +same, 6951469100067, 20240719-151325_6951469100067, 0.608, 0.928, 0.903 +same, 6937962111526, 20240719-151349_6937962111526, 0.138, 0.757, 0.497 +same, 6916749306102, 20240719-151416_6916749306102, 0.243, 0.722, 0.630 +same, 6930364600001, 20240719-151535_6930364600001, 0.143, 0.789, 0.481 +same, 6911988027191, 20240719-151724_6911988027191, 0.036, 0.499, 0.077 +same, 6972219890632, 20240719-151800_6972219890632, 0.314, 0.881, 0.699 +same, 6901668935908, 20240719-152125_6901668935908, 0.202, 0.853, 0.425 +same, 6930364600001, 20240719-153117_6930364600001, 0.057, 0.779, 0.166 +same, 6902265587712, 20240722-144630_6902265587712, 0.364, 0.833, 0.670 +same, 4897073991658, 20240722-160713_4897073991658, 0.102, 0.505, 0.250 +same, 4892655902237, 20240722-160857_4892655902237, 0.457, 0.754, 0.811 +same, 6920907803020, 20240722-161533_6920907803020, 0.211, 0.765, 0.678 +same, 6920912342002, 20240722-163428_6920912342002, 0.281, 0.830, 0.853 +same, 6916196422325, 20240722-164859_6916196422325, 0.391, 0.831, 0.664 +same, 6901668006011, 20240722-165548_6901668006011, 0.282, 0.748, 0.529 +same, 6937221716189, 20240723-094426_6937221716189, 0.405, 0.845, 0.703 +same, 4897098800133, 20240723-094447_4897098800133, 0.381, 0.845, 0.784 +same, 6970470331529, 20240723-094617_6970470331529, 0.238, 0.924, 0.587 +same, 6918598120014, 20240723-095103_6918598120014, 0.406, 0.904, 0.819 +same, 6938737200650, 20240723-095527_6938737200650, 0.207, 0.821, 0.541 +same, 6902890022558, 20240723-095858_6902890022558, 0.162, 0.773, 0.629 +same, 6925303795535, 20240723-100031_6925303795535, 0.303, 0.834, 0.670 +same, 6920382723257, 20240723-100113_6920382723257, 0.273, 0.859, 0.538 +same, 6938737200650, 20240723-100416_6938737200650, 0.237, 0.723, 0.582 +same, 6918598120014, 20240723-100501_6918598120014, 0.100, 0.790, 0.213 +same, 6937221716189, 20240723-100726_6937221716189, 0.344, 0.765, 0.608 +same, 4897098800133, 20240723-100845_4897098800133, 0.303, 0.764, 0.717 +same, 6918598120014, 20240723-101829_6918598120014, 0.500, 0.917, 0.857 +same, 6920382723257, 20240723-102033_6920382723257, 0.344, 0.844, 0.751 +same, 6902890022558, 20240723-102206_6902890022558, 0.117, 0.746, 0.390 +same, 6970470331529, 20240723-102409_6970470331529, 0.103, 0.740, 0.308 +same, 6938737200650, 20240723-102515_6938737200650, 0.183, 0.720, 0.469 +same, 6937221716189, 20240723-102833_6937221716189, 0.067, 0.620, 0.176 +same, 6937451883132, 20240723-110216_6937451883132, 0.321, 0.859, 0.716 +same, 6973337010216, 20240723-110224_6973337010216, 0.201, 0.757, 0.622 +same, 6937451883132, 20240723-111446_6937451883132, 0.367, 0.818, 0.806 +same, 6973337010216, 20240723-111521_6973337010216, 0.140, 0.769, 0.414 +same, 6924187828544, 20240723-111602_6924187828544, 0.359, 0.856, 0.720 +same, 6934024590268, 20240723-112004_6934024590268, 0.184, 0.724, 0.583 +same, 6924187828544, 20240723-112505_6924187828544, 0.281, 0.873, 0.574 +same, 6974158894528, 20240723-155423_6974158894528, 0.433, 0.887, 0.788 +same, 6970399922518, 20240723-160227_6970399922518, 0.077, 0.563, 0.215 +same, 6974158894528, 20240723-160942_6974158894528, 0.416, 0.893, 0.832 +diff, 6916749306102, 20240715-144627_6954256905319, 0.010, 0.455, 0.027 +diff, 6911988027191, 20240719-143603_6930364600025, -0.004, 0.425, -0.008 +diff, 6934024590268, 20240719-143630_6924509902839, -0.003, 0.922, -0.011 +diff, 6916196422325, 20240719-143729_6902007115524, -0.023, 0.467, -0.042 +diff, 6918598120014, 20240719-143807_6930628801472, -0.010, 0.630, -0.021 +diff, 6938737200650, 20240719-144335_6908355010450, 0.040, 0.451, 0.111 +diff, 6972219890632, 20240719-144454_6924509902754, 0.093, 0.531, 0.174 +diff, 6918149576536, 20240719-144916_6925843410158, 0.020, 0.353, 0.038 +diff, 6908355010450, 20240719-145022_6941499112529, -0.066, 0.370, -0.123 +diff, 6920912342002, 20240719-145058_6918149576536, -0.055, 0.360, -0.132 +diff, 6924187895423, 20240719-145542_6928002374551, -0.015, 0.424, -0.036 +diff, 4892655902237, 20240719-150000_6930364600001, 0.051, 0.359, 0.091 +diff, 6901668935908, 20240719-150209_6937962111526, -0.043, 0.357, -0.080 +diff, 6972219890632, 20240719-150345_6924509902839, 0.061, 0.597, 0.186 +diff, 6901668935908, 20240719-150458_6924509902839, 0.034, 0.389, 0.062 +diff, 6902265587712, 20240719-150623_6924187895423, 0.005, 0.428, 0.009 +diff, 6901668006011, 20240719-150716_6916749306102, 0.019, 0.358, 0.044 +diff, 4897098800133, 20240719-150750_6951469100067, 0.156, 0.472, 0.340 +diff, 6920907803020, 20240719-151215_6958350501101, -0.081, 0.355, -0.192 +diff, 6925303795535, 20240719-151325_6951469100067, -0.011, 0.292, -0.024 +diff, 6974158894528, 20240719-151349_6937962111526, -0.061, 0.428, -0.173 +diff, 6937221716189, 20240719-151416_6916749306102, -0.052, 0.453, -0.115 +diff, 6937962111526, 20240719-151535_6930364600001, -0.011, 0.405, -0.047 +diff, 6941499112529, 20240719-151724_6911988027191, -0.013, 0.463, -0.024 +diff, 6958350501101, 20240719-151800_6972219890632, -0.028, 0.363, -0.050 +diff, 6924187895423, 20240719-152125_6901668935908, -0.048, 0.357, -0.100 +diff, 6918598120014, 20240719-153117_6930364600001, -0.034, 0.466, -0.090 +diff, 6924509902839, 20240722-144630_6902265587712, -0.025, 0.373, -0.054 +diff, 6954256905319, 20240722-160713_4897073991658, 0.121, 0.598, 0.400 +diff, 6938737200650, 20240722-160857_4892655902237, 0.028, 0.461, 0.069 +diff, 6916749306102, 20240722-161533_6920907803020, 0.047, 0.581, 0.139 +diff, 6924187895423, 20240722-163428_6920912342002, 0.014, 0.513, 0.032 +diff, 6901668935908, 20240722-164859_6916196422325, -0.060, 0.381, -0.123 +diff, 6938737200650, 20240722-165548_6901668006011, 0.046, 0.434, 0.108 +diff, 6930364600001, 20240723-094426_6937221716189, -0.089, 0.374, -0.179 +diff, 6930364600001, 20240723-094447_4897098800133, 0.009, 0.432, 0.019 +diff, 6925303795535, 20240723-094617_6970470331529, 0.005, 0.360, 0.014 +diff, 6972219890632, 20240723-095103_6918598120014, 0.058, 0.523, 0.116 +diff, 4897098800133, 20240723-095527_6938737200650, 0.013, 0.247, 0.030 +diff, 6925303795535, 20240723-095858_6902890022558, 0.013, 0.517, 0.036 +diff, 6916749306102, 20240723-100031_6925303795535, -0.066, 0.360, -0.151 +diff, 6901668006011, 20240723-100113_6920382723257, -0.037, 0.491, -0.072 +diff, 6920382723257, 20240723-100416_6938737200650, 0.083, 0.396, 0.164 +diff, 6901668935908, 20240723-100501_6918598120014, 0.046, 0.362, 0.092 +diff, 6916196422325, 20240723-100726_6937221716189, 0.039, 0.284, 0.056 +diff, 6902007115524, 20240723-100845_4897098800133, -0.022, 0.366, -0.053 +diff, 6924509902839, 20240723-101829_6918598120014, 0.019, 0.427, 0.033 +diff, 6972219890632, 20240723-102033_6920382723257, 0.029, 0.522, 0.065 +diff, 6916749306102, 20240723-102206_6902890022558, -0.019, 0.345, -0.046 +diff, 6916196422325, 20240723-102409_6970470331529, -0.008, 0.294, -0.017 +diff, 6918598120014, 20240723-102515_6938737200650, 0.026, 0.389, 0.055 +diff, 6930628801472, 20240723-102833_6937221716189, 0.014, 0.363, 0.049 +diff, 6973337010216, 20240723-110216_6937451883132, -0.013, 0.389, -0.036 +diff, 6901668935908, 20240723-110224_6973337010216, -0.080, 0.351, -0.177 +diff, 6941499112529, 20240723-111446_6937451883132, -0.053, 0.359, -0.103 +diff, 6916749306102, 20240723-111521_6973337010216, -0.065, 0.527, -0.163 +diff, 6937451883132, 20240723-111602_6924187828544, 0.024, 0.352, 0.052 +diff, 6908355010450, 20240723-112004_6934024590268, 0.048, 0.425, 0.107 +diff, 6934024590268, 20240723-112505_6924187828544, -0.017, 0.397, -0.048 +diff, 6937962111526, 20240723-155423_6974158894528, -0.056, 0.356, -0.130 +diff, 6916196422325, 20240723-160227_6970399922518, 0.048, 0.391, 0.088 +diff, 6958350501101, 20240723-160942_6974158894528, 0.043, 0.428, 0.063 diff --git a/contrast/result/record/20240909_094945.txt b/contrast/result/record/20240909_094945.txt new file mode 100644 index 0000000..fa20b99 --- /dev/null +++ b/contrast/result/record/20240909_094945.txt @@ -0,0 +1,124 @@ +same, 6954256905319, 20240715-144627_6954256905319, 0.211, 0.750, 0.630 +same, 6930364600025, 20240719-143603_6930364600025, 0.283, 0.771, 0.616 +same, 6924509902839, 20240719-143630_6924509902839, 0.182, 0.727, 0.462 +same, 6902007115524, 20240719-143729_6902007115524, 0.221, 0.723, 0.519 +same, 6930628801472, 20240719-143807_6930628801472, 0.249, 0.804, 0.672 +same, 6908355010450, 20240719-144335_6908355010450, 0.255, 0.812, 0.575 +same, 6924509902754, 20240719-144454_6924509902754, 0.458, 0.882, 0.836 +same, 6925843410158, 20240719-144916_6925843410158, 0.229, 0.829, 0.686 +same, 6941499112529, 20240719-145022_6941499112529, 0.190, 0.795, 0.359 +same, 6918149576536, 20240719-145058_6918149576536, 0.493, 0.906, 0.765 +same, 6928002374551, 20240719-145542_6928002374551, 0.165, 0.773, 0.531 +same, 6930364600001, 20240719-150000_6930364600001, 0.322, 0.831, 0.736 +same, 6937962111526, 20240719-150209_6937962111526, 0.225, 0.794, 0.605 +same, 6924509902839, 20240719-150345_6924509902839, 0.161, 0.816, 0.502 +same, 6924509902839, 20240719-150458_6924509902839, 0.319, 0.746, 0.623 +same, 6924187895423, 20240719-150623_6924187895423, 0.430, 0.906, 0.903 +same, 6916749306102, 20240719-150716_6916749306102, 0.269, 0.764, 0.737 +same, 6951469100067, 20240719-150750_6951469100067, 0.547, 0.938, 0.896 +same, 6958350501101, 20240719-151215_6958350501101, 0.578, 0.906, 0.877 +same, 6951469100067, 20240719-151325_6951469100067, 0.610, 0.930, 0.904 +same, 6937962111526, 20240719-151349_6937962111526, 0.138, 0.762, 0.496 +same, 6916749306102, 20240719-151416_6916749306102, 0.243, 0.728, 0.630 +same, 6930364600001, 20240719-151535_6930364600001, 0.144, 0.787, 0.482 +same, 6911988027191, 20240719-151724_6911988027191, 0.035, 0.502, 0.076 +same, 6972219890632, 20240719-151800_6972219890632, 0.315, 0.880, 0.700 +same, 6901668935908, 20240719-152125_6901668935908, 0.203, 0.856, 0.426 +same, 6930364600001, 20240719-153117_6930364600001, 0.057, 0.775, 0.165 +same, 6902265587712, 20240722-144630_6902265587712, 0.364, 0.833, 0.669 +same, 4897073991658, 20240722-160713_4897073991658, 0.101, 0.514, 0.247 +same, 4892655902237, 20240722-160857_4892655902237, 0.459, 0.760, 0.812 +same, 6920907803020, 20240722-161533_6920907803020, 0.211, 0.769, 0.678 +same, 6920912342002, 20240722-163428_6920912342002, 0.281, 0.825, 0.853 +same, 6916196422325, 20240722-164859_6916196422325, 0.392, 0.837, 0.665 +same, 6901668006011, 20240722-165548_6901668006011, 0.284, 0.752, 0.530 +same, 6937221716189, 20240723-094426_6937221716189, 0.407, 0.850, 0.705 +same, 4897098800133, 20240723-094447_4897098800133, 0.382, 0.846, 0.784 +same, 6970470331529, 20240723-094617_6970470331529, 0.238, 0.925, 0.586 +same, 6918598120014, 20240723-095103_6918598120014, 0.407, 0.905, 0.819 +same, 6938737200650, 20240723-095527_6938737200650, 0.208, 0.825, 0.543 +same, 6902890022558, 20240723-095858_6902890022558, 0.162, 0.769, 0.628 +same, 6925303795535, 20240723-100031_6925303795535, 0.304, 0.835, 0.671 +same, 6920382723257, 20240723-100113_6920382723257, 0.274, 0.854, 0.539 +same, 6938737200650, 20240723-100416_6938737200650, 0.237, 0.719, 0.581 +same, 6918598120014, 20240723-100501_6918598120014, 0.099, 0.791, 0.209 +same, 6937221716189, 20240723-100726_6937221716189, 0.347, 0.771, 0.611 +same, 4897098800133, 20240723-100845_4897098800133, 0.304, 0.770, 0.718 +same, 6918598120014, 20240723-101829_6918598120014, 0.502, 0.919, 0.857 +same, 6920382723257, 20240723-102033_6920382723257, 0.343, 0.849, 0.748 +same, 6902890022558, 20240723-102206_6902890022558, 0.117, 0.745, 0.389 +same, 6970470331529, 20240723-102409_6970470331529, 0.102, 0.734, 0.305 +same, 6938737200650, 20240723-102515_6938737200650, 0.183, 0.725, 0.469 +same, 6937221716189, 20240723-102833_6937221716189, 0.067, 0.623, 0.175 +same, 6937451883132, 20240723-110216_6937451883132, 0.323, 0.860, 0.717 +same, 6973337010216, 20240723-110224_6973337010216, 0.202, 0.760, 0.622 +same, 6937451883132, 20240723-111446_6937451883132, 0.367, 0.817, 0.806 +same, 6973337010216, 20240723-111521_6973337010216, 0.140, 0.777, 0.414 +same, 6924187828544, 20240723-111602_6924187828544, 0.359, 0.859, 0.718 +same, 6934024590268, 20240723-112004_6934024590268, 0.184, 0.727, 0.583 +same, 6924187828544, 20240723-112505_6924187828544, 0.281, 0.875, 0.573 +same, 6974158894528, 20240723-155423_6974158894528, 0.433, 0.890, 0.788 +same, 6970399922518, 20240723-160227_6970399922518, 0.078, 0.564, 0.215 +same, 6974158894528, 20240723-160942_6974158894528, 0.417, 0.895, 0.832 +diff, 6970399922518, 20240715-144627_6954256905319, -0.043, 0.434, -0.116 +diff, 6925843410158, 20240719-143603_6930364600025, 0.026, 0.502, 0.065 +diff, 6974158894528, 20240719-143630_6924509902839, 0.043, 0.564, 0.119 +diff, 6916749306102, 20240719-143729_6902007115524, 0.039, 0.456, 0.102 +diff, 6930364600025, 20240719-143807_6930628801472, 0.072, 0.465, 0.167 +diff, 6934024590268, 20240719-144335_6908355010450, 0.011, 0.421, 0.036 +diff, 6934024590268, 20240719-144454_6924509902754, 0.043, 0.483, 0.113 +diff, 6970399922518, 20240719-144916_6925843410158, 0.016, 0.512, 0.046 +diff, 6902265587712, 20240719-145022_6941499112529, -0.159, 0.340, -0.267 +diff, 6928002374551, 20240719-145058_6918149576536, 0.077, 0.603, 0.194 +diff, 6920907803020, 20240719-145542_6928002374551, -0.025, 0.430, -0.079 +diff, 4897098800133, 20240719-150000_6930364600001, 0.043, 0.384, 0.090 +diff, 6918149576536, 20240719-150209_6937962111526, -0.113, 0.293, -0.173 +diff, 6930628801472, 20240719-150345_6924509902839, 0.001, 0.509, 0.004 +diff, 6970399922518, 20240719-150458_6924509902839, 0.007, 0.398, 0.017 +diff, 6920907803020, 20240719-150623_6924187895423, -0.031, 0.457, -0.086 +diff, 6902890022558, 20240719-150716_6916749306102, -0.007, 0.472, -0.025 +diff, 6902265587712, 20240719-150750_6951469100067, 0.023, 0.410, 0.043 +diff, 6925303795535, 20240719-151215_6958350501101, -0.083, 0.345, -0.173 +diff, 6920912342002, 20240719-151325_6951469100067, 0.027, 0.444, 0.063 +diff, 6924187895423, 20240719-151349_6937962111526, 0.025, 0.414, 0.061 +diff, 6902007115524, 20240719-151416_6916749306102, -0.020, 0.390, -0.046 +diff, 6934024590268, 20240719-151535_6930364600001, -0.010, 0.419, -0.041 +diff, 6937962111526, 20240719-151724_6911988027191, 0.022, 0.453, 0.059 +diff, 6920912342002, 20240719-151800_6972219890632, 0.018, 0.501, 0.048 +diff, 6924187895423, 20240719-152125_6901668935908, -0.048, 0.352, -0.099 +diff, 6951469100067, 20240719-153117_6930364600001, -0.007, 0.400, -0.015 +diff, 6972219890632, 20240722-144630_6902265587712, -0.046, 0.481, -0.097 +diff, 6970399922518, 20240722-160713_4897073991658, -0.015, 0.373, -0.045 +diff, 6937962111526, 20240722-160857_4892655902237, -0.026, 0.351, -0.073 +diff, 6920382723257, 20240722-161533_6920907803020, 0.022, 0.425, 0.055 +diff, 6970470331529, 20240722-163428_6920912342002, 0.052, 0.447, 0.135 +diff, 6901668006011, 20240722-164859_6916196422325, 0.011, 0.432, 0.024 +diff, 6902265587712, 20240722-165548_6901668006011, 0.011, 0.397, 0.019 +diff, 6901668935908, 20240723-094426_6937221716189, -0.051, 0.324, -0.088 +diff, 6937221716189, 20240723-094447_4897098800133, 0.093, 0.447, 0.178 +diff, 6918149576536, 20240723-094617_6970470331529, -0.010, 0.280, -0.018 +diff, 4897098800133, 20240723-095103_6918598120014, 0.094, 0.327, 0.194 +diff, 6974158894528, 20240723-095527_6938737200650, 0.056, 0.388, 0.134 +diff, 6951469100067, 20240723-095858_6902890022558, 0.004, 0.457, 0.007 +diff, 4897073991658, 20240723-100031_6925303795535, 0.008, 0.387, 0.016 +diff, 6928002374551, 20240723-100113_6920382723257, 0.043, 0.552, 0.112 +diff, 6972219890632, 20240723-100416_6938737200650, 0.026, 0.403, 0.053 +diff, 6970470331529, 20240723-100501_6918598120014, 0.032, 0.351, 0.070 +diff, 6924509902839, 20240723-100726_6937221716189, -0.038, 0.402, -0.071 +diff, 6918598120014, 20240723-100845_4897098800133, 0.026, 0.441, 0.060 +diff, 6920912342002, 20240723-101829_6918598120014, 0.123, 0.575, 0.254 +diff, 6902265587712, 20240723-102033_6920382723257, 0.024, 0.453, 0.047 +diff, 6941499112529, 20240723-102206_6902890022558, -0.029, 0.392, -0.061 +diff, 6920912342002, 20240723-102409_6970470331529, 0.062, 0.533, 0.216 +diff, 6973337010216, 20240723-102515_6938737200650, 0.055, 0.469, 0.156 +diff, 6920382723257, 20240723-102833_6937221716189, 0.079, 0.460, 0.207 +diff, 6901668006011, 20240723-110216_6937451883132, 0.079, 0.427, 0.153 +diff, 6924187828544, 20240723-110224_6973337010216, 0.033, 0.452, 0.078 +diff, 4892655902237, 20240723-111446_6937451883132, -0.001, 0.416, -0.002 +diff, 4897098800133, 20240723-111521_6973337010216, 0.067, 0.353, 0.151 +diff, 6908355010450, 20240723-111602_6924187828544, -0.021, 0.475, -0.040 +diff, 4897098800133, 20240723-112004_6934024590268, 0.010, 0.268, 0.025 +diff, 6901668935908, 20240723-112505_6924187828544, 0.039, 0.415, 0.074 +diff, 6924509902839, 20240723-155423_6974158894528, -0.038, 0.291, -0.064 +diff, 4892655902237, 20240723-160227_6970399922518, -0.027, 0.407, -0.054 +diff, 6925843410158, 20240723-160942_6974158894528, -0.027, 0.350, -0.060 diff --git a/contrast/result/record/20240909_094945_ft16.txt b/contrast/result/record/20240909_094945_ft16.txt new file mode 100644 index 0000000..b8d21b3 --- /dev/null +++ b/contrast/result/record/20240909_094945_ft16.txt @@ -0,0 +1,124 @@ +same, 6954256905319, 20240715-144627_6954256905319, 0.211, 0.751, 0.630 +same, 6930364600025, 20240719-143603_6930364600025, 0.283, 0.771, 0.616 +same, 6924509902839, 20240719-143630_6924509902839, 0.182, 0.727, 0.462 +same, 6902007115524, 20240719-143729_6902007115524, 0.221, 0.723, 0.519 +same, 6930628801472, 20240719-143807_6930628801472, 0.249, 0.804, 0.672 +same, 6908355010450, 20240719-144335_6908355010450, 0.255, 0.812, 0.575 +same, 6924509902754, 20240719-144454_6924509902754, 0.458, 0.882, 0.836 +same, 6925843410158, 20240719-144916_6925843410158, 0.229, 0.829, 0.686 +same, 6941499112529, 20240719-145022_6941499112529, 0.190, 0.795, 0.359 +same, 6918149576536, 20240719-145058_6918149576536, 0.493, 0.906, 0.765 +same, 6928002374551, 20240719-145542_6928002374551, 0.165, 0.773, 0.531 +same, 6930364600001, 20240719-150000_6930364600001, 0.322, 0.831, 0.736 +same, 6937962111526, 20240719-150209_6937962111526, 0.225, 0.794, 0.605 +same, 6924509902839, 20240719-150345_6924509902839, 0.161, 0.816, 0.502 +same, 6924509902839, 20240719-150458_6924509902839, 0.319, 0.746, 0.623 +same, 6924187895423, 20240719-150623_6924187895423, 0.430, 0.906, 0.903 +same, 6916749306102, 20240719-150716_6916749306102, 0.269, 0.764, 0.737 +same, 6951469100067, 20240719-150750_6951469100067, 0.547, 0.938, 0.896 +same, 6958350501101, 20240719-151215_6958350501101, 0.578, 0.906, 0.877 +same, 6951469100067, 20240719-151325_6951469100067, 0.610, 0.930, 0.904 +same, 6937962111526, 20240719-151349_6937962111526, 0.138, 0.762, 0.496 +same, 6916749306102, 20240719-151416_6916749306102, 0.243, 0.728, 0.630 +same, 6930364600001, 20240719-151535_6930364600001, 0.144, 0.787, 0.482 +same, 6911988027191, 20240719-151724_6911988027191, 0.035, 0.502, 0.076 +same, 6972219890632, 20240719-151800_6972219890632, 0.315, 0.880, 0.700 +same, 6901668935908, 20240719-152125_6901668935908, 0.203, 0.856, 0.426 +same, 6930364600001, 20240719-153117_6930364600001, 0.057, 0.775, 0.165 +same, 6902265587712, 20240722-144630_6902265587712, 0.364, 0.833, 0.669 +same, 4897073991658, 20240722-160713_4897073991658, 0.101, 0.514, 0.247 +same, 4892655902237, 20240722-160857_4892655902237, 0.459, 0.760, 0.812 +same, 6920907803020, 20240722-161533_6920907803020, 0.211, 0.769, 0.678 +same, 6920912342002, 20240722-163428_6920912342002, 0.281, 0.825, 0.853 +same, 6916196422325, 20240722-164859_6916196422325, 0.392, 0.837, 0.665 +same, 6901668006011, 20240722-165548_6901668006011, 0.284, 0.751, 0.530 +same, 6937221716189, 20240723-094426_6937221716189, 0.407, 0.850, 0.705 +same, 4897098800133, 20240723-094447_4897098800133, 0.382, 0.846, 0.784 +same, 6970470331529, 20240723-094617_6970470331529, 0.238, 0.925, 0.586 +same, 6918598120014, 20240723-095103_6918598120014, 0.407, 0.905, 0.819 +same, 6938737200650, 20240723-095527_6938737200650, 0.208, 0.825, 0.543 +same, 6902890022558, 20240723-095858_6902890022558, 0.162, 0.769, 0.628 +same, 6925303795535, 20240723-100031_6925303795535, 0.304, 0.835, 0.671 +same, 6920382723257, 20240723-100113_6920382723257, 0.274, 0.854, 0.539 +same, 6938737200650, 20240723-100416_6938737200650, 0.237, 0.719, 0.581 +same, 6918598120014, 20240723-100501_6918598120014, 0.099, 0.791, 0.209 +same, 6937221716189, 20240723-100726_6937221716189, 0.347, 0.771, 0.611 +same, 4897098800133, 20240723-100845_4897098800133, 0.304, 0.770, 0.717 +same, 6918598120014, 20240723-101829_6918598120014, 0.502, 0.919, 0.857 +same, 6920382723257, 20240723-102033_6920382723257, 0.343, 0.849, 0.748 +same, 6902890022558, 20240723-102206_6902890022558, 0.117, 0.745, 0.389 +same, 6970470331529, 20240723-102409_6970470331529, 0.102, 0.734, 0.305 +same, 6938737200650, 20240723-102515_6938737200650, 0.183, 0.725, 0.469 +same, 6937221716189, 20240723-102833_6937221716189, 0.067, 0.623, 0.175 +same, 6937451883132, 20240723-110216_6937451883132, 0.323, 0.860, 0.717 +same, 6973337010216, 20240723-110224_6973337010216, 0.202, 0.760, 0.622 +same, 6937451883132, 20240723-111446_6937451883132, 0.367, 0.817, 0.806 +same, 6973337010216, 20240723-111521_6973337010216, 0.140, 0.777, 0.414 +same, 6924187828544, 20240723-111602_6924187828544, 0.359, 0.859, 0.718 +same, 6934024590268, 20240723-112004_6934024590268, 0.184, 0.727, 0.583 +same, 6924187828544, 20240723-112505_6924187828544, 0.281, 0.875, 0.573 +same, 6974158894528, 20240723-155423_6974158894528, 0.433, 0.890, 0.788 +same, 6970399922518, 20240723-160227_6970399922518, 0.078, 0.564, 0.215 +same, 6974158894528, 20240723-160942_6974158894528, 0.417, 0.895, 0.832 +diff, 6970399922518, 20240715-144627_6954256905319, -0.043, 0.434, -0.116 +diff, 6925843410158, 20240719-143603_6930364600025, 0.026, 0.502, 0.065 +diff, 6974158894528, 20240719-143630_6924509902839, 0.043, 0.564, 0.119 +diff, 6916749306102, 20240719-143729_6902007115524, 0.039, 0.456, 0.102 +diff, 6930364600025, 20240719-143807_6930628801472, 0.072, 0.465, 0.167 +diff, 6934024590268, 20240719-144335_6908355010450, 0.011, 0.421, 0.036 +diff, 6934024590268, 20240719-144454_6924509902754, 0.043, 0.483, 0.113 +diff, 6970399922518, 20240719-144916_6925843410158, 0.016, 0.512, 0.046 +diff, 6902265587712, 20240719-145022_6941499112529, -0.159, 0.340, -0.267 +diff, 6928002374551, 20240719-145058_6918149576536, 0.077, 0.603, 0.194 +diff, 6920907803020, 20240719-145542_6928002374551, -0.025, 0.430, -0.079 +diff, 4897098800133, 20240719-150000_6930364600001, 0.043, 0.384, 0.090 +diff, 6918149576536, 20240719-150209_6937962111526, -0.113, 0.293, -0.173 +diff, 6930628801472, 20240719-150345_6924509902839, 0.001, 0.509, 0.004 +diff, 6970399922518, 20240719-150458_6924509902839, 0.007, 0.398, 0.017 +diff, 6920907803020, 20240719-150623_6924187895423, -0.031, 0.457, -0.086 +diff, 6902890022558, 20240719-150716_6916749306102, -0.007, 0.472, -0.025 +diff, 6902265587712, 20240719-150750_6951469100067, 0.023, 0.410, 0.043 +diff, 6925303795535, 20240719-151215_6958350501101, -0.083, 0.345, -0.173 +diff, 6920912342002, 20240719-151325_6951469100067, 0.027, 0.444, 0.063 +diff, 6924187895423, 20240719-151349_6937962111526, 0.025, 0.414, 0.061 +diff, 6902007115524, 20240719-151416_6916749306102, -0.020, 0.390, -0.046 +diff, 6934024590268, 20240719-151535_6930364600001, -0.010, 0.419, -0.041 +diff, 6937962111526, 20240719-151724_6911988027191, 0.022, 0.453, 0.059 +diff, 6920912342002, 20240719-151800_6972219890632, 0.018, 0.501, 0.048 +diff, 6924187895423, 20240719-152125_6901668935908, -0.048, 0.352, -0.099 +diff, 6951469100067, 20240719-153117_6930364600001, -0.007, 0.400, -0.015 +diff, 6972219890632, 20240722-144630_6902265587712, -0.046, 0.481, -0.097 +diff, 6970399922518, 20240722-160713_4897073991658, -0.015, 0.373, -0.045 +diff, 6937962111526, 20240722-160857_4892655902237, -0.026, 0.351, -0.073 +diff, 6920382723257, 20240722-161533_6920907803020, 0.022, 0.425, 0.055 +diff, 6970470331529, 20240722-163428_6920912342002, 0.052, 0.446, 0.135 +diff, 6901668006011, 20240722-164859_6916196422325, 0.011, 0.432, 0.024 +diff, 6902265587712, 20240722-165548_6901668006011, 0.011, 0.397, 0.019 +diff, 6901668935908, 20240723-094426_6937221716189, -0.051, 0.324, -0.088 +diff, 6937221716189, 20240723-094447_4897098800133, 0.093, 0.447, 0.178 +diff, 6918149576536, 20240723-094617_6970470331529, -0.010, 0.280, -0.018 +diff, 4897098800133, 20240723-095103_6918598120014, 0.094, 0.327, 0.194 +diff, 6974158894528, 20240723-095527_6938737200650, 0.056, 0.388, 0.134 +diff, 6951469100067, 20240723-095858_6902890022558, 0.004, 0.457, 0.007 +diff, 4897073991658, 20240723-100031_6925303795535, 0.008, 0.387, 0.016 +diff, 6928002374551, 20240723-100113_6920382723257, 0.043, 0.552, 0.112 +diff, 6972219890632, 20240723-100416_6938737200650, 0.026, 0.403, 0.053 +diff, 6970470331529, 20240723-100501_6918598120014, 0.032, 0.351, 0.070 +diff, 6924509902839, 20240723-100726_6937221716189, -0.038, 0.402, -0.071 +diff, 6918598120014, 20240723-100845_4897098800133, 0.026, 0.441, 0.060 +diff, 6920912342002, 20240723-101829_6918598120014, 0.123, 0.575, 0.254 +diff, 6902265587712, 20240723-102033_6920382723257, 0.024, 0.453, 0.047 +diff, 6941499112529, 20240723-102206_6902890022558, -0.029, 0.392, -0.061 +diff, 6920912342002, 20240723-102409_6970470331529, 0.062, 0.533, 0.216 +diff, 6973337010216, 20240723-102515_6938737200650, 0.055, 0.469, 0.156 +diff, 6920382723257, 20240723-102833_6937221716189, 0.079, 0.460, 0.207 +diff, 6901668006011, 20240723-110216_6937451883132, 0.079, 0.427, 0.153 +diff, 6924187828544, 20240723-110224_6973337010216, 0.033, 0.452, 0.078 +diff, 4892655902237, 20240723-111446_6937451883132, -0.001, 0.416, -0.002 +diff, 4897098800133, 20240723-111521_6973337010216, 0.067, 0.353, 0.151 +diff, 6908355010450, 20240723-111602_6924187828544, -0.021, 0.475, -0.040 +diff, 4897098800133, 20240723-112004_6934024590268, 0.010, 0.268, 0.025 +diff, 6901668935908, 20240723-112505_6924187828544, 0.039, 0.415, 0.074 +diff, 6924509902839, 20240723-155423_6974158894528, -0.038, 0.291, -0.064 +diff, 4892655902237, 20240723-160227_6970399922518, -0.027, 0.407, -0.054 +diff, 6925843410158, 20240723-160942_6974158894528, -0.027, 0.350, -0.060 diff --git a/contrast/result/record/20240909_094945_uint8.txt b/contrast/result/record/20240909_094945_uint8.txt new file mode 100644 index 0000000..e155d50 --- /dev/null +++ b/contrast/result/record/20240909_094945_uint8.txt @@ -0,0 +1,124 @@ +same, 6954256905319, 20240715-144627_6954256905319, 0.211, 0.750, 0.630 +same, 6930364600025, 20240719-143603_6930364600025, 0.283, 0.772, 0.618 +same, 6924509902839, 20240719-143630_6924509902839, 0.182, 0.727, 0.464 +same, 6902007115524, 20240719-143729_6902007115524, 0.221, 0.721, 0.518 +same, 6930628801472, 20240719-143807_6930628801472, 0.249, 0.799, 0.672 +same, 6908355010450, 20240719-144335_6908355010450, 0.255, 0.809, 0.575 +same, 6924509902754, 20240719-144454_6924509902754, 0.456, 0.879, 0.835 +same, 6925843410158, 20240719-144916_6925843410158, 0.228, 0.820, 0.686 +same, 6941499112529, 20240719-145022_6941499112529, 0.190, 0.790, 0.360 +same, 6918149576536, 20240719-145058_6918149576536, 0.493, 0.905, 0.766 +same, 6928002374551, 20240719-145542_6928002374551, 0.165, 0.769, 0.530 +same, 6930364600001, 20240719-150000_6930364600001, 0.320, 0.834, 0.734 +same, 6937962111526, 20240719-150209_6937962111526, 0.224, 0.791, 0.604 +same, 6924509902839, 20240719-150345_6924509902839, 0.162, 0.812, 0.504 +same, 6924509902839, 20240719-150458_6924509902839, 0.320, 0.745, 0.625 +same, 6924187895423, 20240719-150623_6924187895423, 0.430, 0.907, 0.904 +same, 6916749306102, 20240719-150716_6916749306102, 0.268, 0.761, 0.738 +same, 6951469100067, 20240719-150750_6951469100067, 0.544, 0.935, 0.894 +same, 6958350501101, 20240719-151215_6958350501101, 0.576, 0.904, 0.876 +same, 6951469100067, 20240719-151325_6951469100067, 0.608, 0.928, 0.903 +same, 6937962111526, 20240719-151349_6937962111526, 0.138, 0.757, 0.497 +same, 6916749306102, 20240719-151416_6916749306102, 0.243, 0.722, 0.630 +same, 6930364600001, 20240719-151535_6930364600001, 0.143, 0.789, 0.481 +same, 6911988027191, 20240719-151724_6911988027191, 0.036, 0.499, 0.077 +same, 6972219890632, 20240719-151800_6972219890632, 0.314, 0.881, 0.699 +same, 6901668935908, 20240719-152125_6901668935908, 0.202, 0.853, 0.425 +same, 6930364600001, 20240719-153117_6930364600001, 0.057, 0.779, 0.166 +same, 6902265587712, 20240722-144630_6902265587712, 0.364, 0.833, 0.670 +same, 4897073991658, 20240722-160713_4897073991658, 0.102, 0.505, 0.250 +same, 4892655902237, 20240722-160857_4892655902237, 0.457, 0.754, 0.811 +same, 6920907803020, 20240722-161533_6920907803020, 0.211, 0.765, 0.678 +same, 6920912342002, 20240722-163428_6920912342002, 0.281, 0.830, 0.853 +same, 6916196422325, 20240722-164859_6916196422325, 0.391, 0.831, 0.664 +same, 6901668006011, 20240722-165548_6901668006011, 0.282, 0.748, 0.529 +same, 6937221716189, 20240723-094426_6937221716189, 0.405, 0.845, 0.703 +same, 4897098800133, 20240723-094447_4897098800133, 0.381, 0.845, 0.784 +same, 6970470331529, 20240723-094617_6970470331529, 0.238, 0.924, 0.587 +same, 6918598120014, 20240723-095103_6918598120014, 0.406, 0.904, 0.819 +same, 6938737200650, 20240723-095527_6938737200650, 0.207, 0.821, 0.541 +same, 6902890022558, 20240723-095858_6902890022558, 0.162, 0.773, 0.629 +same, 6925303795535, 20240723-100031_6925303795535, 0.303, 0.834, 0.670 +same, 6920382723257, 20240723-100113_6920382723257, 0.273, 0.859, 0.538 +same, 6938737200650, 20240723-100416_6938737200650, 0.237, 0.723, 0.582 +same, 6918598120014, 20240723-100501_6918598120014, 0.100, 0.790, 0.213 +same, 6937221716189, 20240723-100726_6937221716189, 0.344, 0.765, 0.608 +same, 4897098800133, 20240723-100845_4897098800133, 0.303, 0.764, 0.717 +same, 6918598120014, 20240723-101829_6918598120014, 0.500, 0.917, 0.857 +same, 6920382723257, 20240723-102033_6920382723257, 0.344, 0.844, 0.751 +same, 6902890022558, 20240723-102206_6902890022558, 0.117, 0.746, 0.390 +same, 6970470331529, 20240723-102409_6970470331529, 0.103, 0.740, 0.308 +same, 6938737200650, 20240723-102515_6938737200650, 0.183, 0.720, 0.469 +same, 6937221716189, 20240723-102833_6937221716189, 0.067, 0.620, 0.176 +same, 6937451883132, 20240723-110216_6937451883132, 0.321, 0.859, 0.716 +same, 6973337010216, 20240723-110224_6973337010216, 0.201, 0.757, 0.622 +same, 6937451883132, 20240723-111446_6937451883132, 0.367, 0.818, 0.806 +same, 6973337010216, 20240723-111521_6973337010216, 0.140, 0.769, 0.414 +same, 6924187828544, 20240723-111602_6924187828544, 0.359, 0.856, 0.720 +same, 6934024590268, 20240723-112004_6934024590268, 0.184, 0.724, 0.583 +same, 6924187828544, 20240723-112505_6924187828544, 0.281, 0.873, 0.574 +same, 6974158894528, 20240723-155423_6974158894528, 0.433, 0.887, 0.788 +same, 6970399922518, 20240723-160227_6970399922518, 0.077, 0.563, 0.215 +same, 6974158894528, 20240723-160942_6974158894528, 0.416, 0.893, 0.832 +diff, 6970399922518, 20240715-144627_6954256905319, -0.044, 0.431, -0.119 +diff, 6925843410158, 20240719-143603_6930364600025, 0.025, 0.496, 0.062 +diff, 6974158894528, 20240719-143630_6924509902839, 0.042, 0.556, 0.116 +diff, 6916749306102, 20240719-143729_6902007115524, 0.039, 0.452, 0.101 +diff, 6930364600025, 20240719-143807_6930628801472, 0.072, 0.472, 0.167 +diff, 6934024590268, 20240719-144335_6908355010450, 0.011, 0.417, 0.035 +diff, 6934024590268, 20240719-144454_6924509902754, 0.044, 0.481, 0.114 +diff, 6970399922518, 20240719-144916_6925843410158, 0.017, 0.517, 0.048 +diff, 6902265587712, 20240719-145022_6941499112529, -0.160, 0.340, -0.269 +diff, 6928002374551, 20240719-145058_6918149576536, 0.076, 0.606, 0.193 +diff, 6920907803020, 20240719-145542_6928002374551, -0.025, 0.429, -0.079 +diff, 4897098800133, 20240719-150000_6930364600001, 0.043, 0.385, 0.091 +diff, 6918149576536, 20240719-150209_6937962111526, -0.110, 0.297, -0.170 +diff, 6930628801472, 20240719-150345_6924509902839, 0.001, 0.506, 0.002 +diff, 6970399922518, 20240719-150458_6924509902839, 0.008, 0.397, 0.019 +diff, 6920907803020, 20240719-150623_6924187895423, -0.030, 0.453, -0.084 +diff, 6902890022558, 20240719-150716_6916749306102, -0.006, 0.475, -0.024 +diff, 6902265587712, 20240719-150750_6951469100067, 0.023, 0.409, 0.041 +diff, 6925303795535, 20240719-151215_6958350501101, -0.082, 0.345, -0.172 +diff, 6920912342002, 20240719-151325_6951469100067, 0.027, 0.442, 0.061 +diff, 6924187895423, 20240719-151349_6937962111526, 0.024, 0.409, 0.058 +diff, 6902007115524, 20240719-151416_6916749306102, -0.020, 0.394, -0.047 +diff, 6934024590268, 20240719-151535_6930364600001, -0.010, 0.419, -0.042 +diff, 6937962111526, 20240719-151724_6911988027191, 0.022, 0.448, 0.059 +diff, 6920912342002, 20240719-151800_6972219890632, 0.018, 0.500, 0.048 +diff, 6924187895423, 20240719-152125_6901668935908, -0.048, 0.357, -0.100 +diff, 6951469100067, 20240719-153117_6930364600001, -0.008, 0.400, -0.015 +diff, 6972219890632, 20240722-144630_6902265587712, -0.047, 0.483, -0.099 +diff, 6970399922518, 20240722-160713_4897073991658, -0.016, 0.372, -0.047 +diff, 6937962111526, 20240722-160857_4892655902237, -0.026, 0.348, -0.072 +diff, 6920382723257, 20240722-161533_6920907803020, 0.023, 0.430, 0.057 +diff, 6970470331529, 20240722-163428_6920912342002, 0.052, 0.453, 0.135 +diff, 6901668006011, 20240722-164859_6916196422325, 0.012, 0.427, 0.026 +diff, 6902265587712, 20240722-165548_6901668006011, 0.013, 0.402, 0.021 +diff, 6901668935908, 20240723-094426_6937221716189, -0.051, 0.322, -0.088 +diff, 6937221716189, 20240723-094447_4897098800133, 0.091, 0.444, 0.176 +diff, 6918149576536, 20240723-094617_6970470331529, -0.010, 0.280, -0.019 +diff, 4897098800133, 20240723-095103_6918598120014, 0.094, 0.330, 0.195 +diff, 6974158894528, 20240723-095527_6938737200650, 0.057, 0.388, 0.136 +diff, 6951469100067, 20240723-095858_6902890022558, 0.003, 0.460, 0.006 +diff, 4897073991658, 20240723-100031_6925303795535, 0.008, 0.387, 0.016 +diff, 6928002374551, 20240723-100113_6920382723257, 0.043, 0.550, 0.112 +diff, 6972219890632, 20240723-100416_6938737200650, 0.027, 0.407, 0.054 +diff, 6970470331529, 20240723-100501_6918598120014, 0.031, 0.350, 0.068 +diff, 6924509902839, 20240723-100726_6937221716189, -0.040, 0.399, -0.074 +diff, 6918598120014, 20240723-100845_4897098800133, 0.024, 0.443, 0.057 +diff, 6920912342002, 20240723-101829_6918598120014, 0.123, 0.579, 0.253 +diff, 6902265587712, 20240723-102033_6920382723257, 0.024, 0.456, 0.046 +diff, 6941499112529, 20240723-102206_6902890022558, -0.030, 0.390, -0.061 +diff, 6920912342002, 20240723-102409_6970470331529, 0.062, 0.532, 0.215 +diff, 6973337010216, 20240723-102515_6938737200650, 0.055, 0.467, 0.157 +diff, 6920382723257, 20240723-102833_6937221716189, 0.079, 0.458, 0.209 +diff, 6901668006011, 20240723-110216_6937451883132, 0.078, 0.429, 0.152 +diff, 6924187828544, 20240723-110224_6973337010216, 0.033, 0.450, 0.079 +diff, 4892655902237, 20240723-111446_6937451883132, -0.003, 0.419, -0.005 +diff, 4897098800133, 20240723-111521_6973337010216, 0.066, 0.352, 0.151 +diff, 6908355010450, 20240723-111602_6924187828544, -0.022, 0.482, -0.042 +diff, 4897098800133, 20240723-112004_6934024590268, 0.012, 0.276, 0.027 +diff, 6901668935908, 20240723-112505_6924187828544, 0.040, 0.422, 0.076 +diff, 6924509902839, 20240723-155423_6974158894528, -0.041, 0.290, -0.069 +diff, 4892655902237, 20240723-160227_6970399922518, -0.027, 0.417, -0.055 +diff, 6925843410158, 20240723-160942_6974158894528, -0.025, 0.358, -0.057 diff --git a/contrast/result/record/20240909_152012.txt b/contrast/result/record/20240909_152012.txt new file mode 100644 index 0000000..f7a8211 --- /dev/null +++ b/contrast/result/record/20240909_152012.txt @@ -0,0 +1,174 @@ +same, 6954256905319, 20240715-144627_6954256905319, 0.208, 0.746, 0.604 +same, 6902265589617, 20240719-143553_6902265589617, 0.296, 0.719, 0.561 +same, 6930364600025, 20240719-143603_6930364600025, 0.301, 0.772, 0.666 +same, 6924509902839, 20240719-143630_6924509902839, 0.176, 0.660, 0.435 +same, 6902007115524, 20240719-143729_6902007115524, 0.230, 0.635, 0.525 +same, 6930628801472, 20240719-143807_6930628801472, 0.279, 0.730, 0.712 +same, 6902265314615, 20240719-144126_6902265314615, 0.376, 0.897, 0.754 +same, 6928312901324, 20240719-144226_6928312901324, 0.335, 0.782, 0.693 +same, 6908355010450, 20240719-144335_6908355010450, 0.256, 0.748, 0.580 +same, 6924509902754, 20240719-144454_6924509902754, 0.459, 0.832, 0.831 +same, 6902265314615, 20240719-144621_6902265314615, 0.410, 0.847, 0.667 +same, 6941499112529, 20240719-145022_6941499112529, 0.174, 0.736, 0.309 +same, 6972786772775, 20240719-145149_6972786772775, 0.243, 0.698, 0.565 +same, 6920267906706, 20240719-145319_6920267906706, 0.306, 0.799, 0.767 +same, 6974925910116, 20240719-145505_6974925910116, 0.372, 0.844, 0.743 +same, 6928002374551, 20240719-145542_6928002374551, 0.171, 0.773, 0.499 +same, 6920152414071, 20240719-145816_6920152414071, 0.469, 0.816, 0.871 +same, 6973360690409, 20240719-145926_6973360690409, 0.160, 0.636, 0.591 +same, 6930364600001, 20240719-150000_6930364600001, 0.318, 0.790, 0.754 +same, 6937962111526, 20240719-150209_6937962111526, 0.222, 0.660, 0.575 +same, 6924509902839, 20240719-150345_6924509902839, 0.155, 0.757, 0.470 +same, 6924509902839, 20240719-150458_6924509902839, 0.307, 0.670, 0.583 +same, 6924187895423, 20240719-150623_6924187895423, 0.437, 0.882, 0.892 +same, 6916749306102, 20240719-150716_6916749306102, 0.249, 0.725, 0.682 +same, 6951469100067, 20240719-150750_6951469100067, 0.535, 0.938, 0.905 +same, 6973360690409, 20240719-151009_6973360690409, 0.143, 0.578, 0.463 +same, 6923120632521, 20240719-151135_6923120632521, 0.301, 0.865, 0.711 +same, 6958350501101, 20240719-151215_6958350501101, 0.550, 0.902, 0.888 +same, 6951469100067, 20240719-151325_6951469100067, 0.588, 0.927, 0.899 +same, 6937962111526, 20240719-151349_6937962111526, 0.153, 0.657, 0.531 +same, 6916749306102, 20240719-151416_6916749306102, 0.228, 0.677, 0.587 +same, 6930364600001, 20240719-151535_6930364600001, 0.151, 0.692, 0.524 +same, 6911988027191, 20240719-151724_6911988027191, 0.127, 0.525, 0.337 +same, 6972219890632, 20240719-151800_6972219890632, 0.288, 0.872, 0.700 +same, 6901668935908, 20240719-152125_6901668935908, 0.210, 0.806, 0.436 +same, 6920152471517, 20240719-152315_6920152471517, 0.392, 0.836, 0.640 +same, 6951957205274, 20240719-152816_6951957205274, 0.345, 0.810, 0.602 +same, 6971284201244, 20240719-152832_6971284201244, 0.229, 0.817, 0.557 +same, 6930364600001, 20240719-153117_6930364600001, 0.065, 0.743, 0.196 +same, 6972900962273, 20240722-143041_6972900962273, 0.214, 0.596, 0.528 +same, 6902265587712, 20240722-144630_6902265587712, 0.362, 0.816, 0.673 +same, 4892655902237, 20240722-160857_4892655902237, 0.433, 0.783, 0.842 +same, 4892655902015, 20240722-162225_4892655902015, 0.100, 0.668, 0.282 +same, 6938888889896, 20240722-162328_6938888889896, 0.176, 0.779, 0.407 +same, 6920912342002, 20240722-163428_6920912342002, 0.313, 0.825, 0.857 +same, 6971075127784, 20240722-163600_6971075127784, 0.734, 0.923, 0.915 +same, 6926265317384, 20240722-164552_6926265317384, 0.179, 0.715, 0.613 +same, 6916196422325, 20240722-164859_6916196422325, 0.367, 0.773, 0.675 +same, 6901668006011, 20240722-165548_6901668006011, 0.291, 0.718, 0.546 +same, 6926419102347, 20240722-170325_6926419102347, 0.256, 0.785, 0.768 +same, 4897098800133, 20240723-094447_4897098800133, 0.379, 0.846, 0.805 +same, 6971284204597, 20240723-094525_6971284204597, 0.206, 0.732, 0.597 +same, 6970470331529, 20240723-094617_6970470331529, 0.235, 0.924, 0.586 +same, 6918598120014, 20240723-095103_6918598120014, 0.426, 0.905, 0.819 +same, 6902265202318, 20240723-095259_6902265202318, 0.641, 0.907, 0.909 +same, 6938737200650, 20240723-095527_6938737200650, 0.219, 0.825, 0.651 +same, 6902265131212, 20240723-095819_6902265131212, 0.197, 0.899, 0.396 +same, 6902890022558, 20240723-095858_6902890022558, 0.161, 0.713, 0.590 +same, 6925303795535, 20240723-100031_6925303795535, 0.298, 0.838, 0.674 +same, 6920382723257, 20240723-100113_6920382723257, 0.282, 0.815, 0.600 +same, 6976399680069, 20240723-100148_6976399680069, 0.040, 0.554, 0.132 +same, 6921502656660, 20240723-100318_6921502656660, 0.059, 0.424, 0.173 +same, 6938737200650, 20240723-100416_6938737200650, 0.222, 0.701, 0.620 +same, 6918598120014, 20240723-100501_6918598120014, 0.102, 0.742, 0.206 +same, 4897098800133, 20240723-100845_4897098800133, 0.305, 0.834, 0.744 +same, 6971284204597, 20240723-101014_6971284204597, 0.223, 0.714, 0.623 +same, 6902265202318, 20240723-101144_6902265202318, 0.537, 0.875, 0.812 +same, 6902265131212, 20240723-101416_6902265131212, 0.463, 0.901, 0.747 +same, 6918598120014, 20240723-101829_6918598120014, 0.524, 0.901, 0.855 +same, 6902265131212, 20240723-101851_6902265131212, 0.418, 0.899, 0.820 +same, 6920382723257, 20240723-102033_6920382723257, 0.320, 0.780, 0.755 +same, 6902890022558, 20240723-102206_6902890022558, 0.123, 0.707, 0.387 +same, 6970470331529, 20240723-102409_6970470331529, 0.099, 0.641, 0.300 +same, 6938737200650, 20240723-102515_6938737200650, 0.177, 0.695, 0.515 +same, 6937451883132, 20240723-110216_6937451883132, 0.300, 0.817, 0.706 +same, 6973337010216, 20240723-110224_6973337010216, 0.202, 0.727, 0.633 +same, 6972359682937, 20240723-110937_6972359682937, 0.229, 0.792, 0.540 +same, 6937451883132, 20240723-111446_6937451883132, 0.363, 0.796, 0.844 +same, 6973337010216, 20240723-111521_6973337010216, 0.120, 0.764, 0.359 +same, 6924187828544, 20240723-111602_6924187828544, 0.378, 0.857, 0.733 +same, 6934024590268, 20240723-112004_6934024590268, 0.205, 0.724, 0.609 +same, 6972359682937, 20240723-112038_6972359682937, 0.432, 0.839, 0.857 +same, 6924187828544, 20240723-112505_6924187828544, 0.304, 0.835, 0.601 +same, 6974158894528, 20240723-155423_6974158894528, 0.440, 0.854, 0.765 +same, 6973497200045, 20240723-160015_6973497200045, 0.263, 0.712, 0.551 +same, 6970399922518, 20240723-160227_6970399922518, 0.087, 0.564, 0.243 +same, 6974158894528, 20240723-160942_6974158894528, 0.429, 0.847, 0.818 +diff, 6973360690409, 20240715-144627_6954256905319, 0.091, 0.515, 0.307 +diff, 6924509902839, 20240719-143553_6902265589617, -0.039, 0.330, -0.084 +diff, 6973360690409, 20240719-143603_6930364600025, 0.011, 0.343, 0.034 +diff, 4897098800133, 20240719-143630_6924509902839, 0.038, 0.340, 0.103 +diff, 6902265202318, 20240719-143729_6902007115524, -0.069, 0.293, -0.125 +diff, 4892655902015, 20240719-143807_6930628801472, -0.059, 0.480, -0.135 +diff, 6908355010450, 20240719-144126_6902265314615, -0.045, 0.261, -0.100 +diff, 4892655902015, 20240719-144226_6928312901324, 0.017, 0.356, 0.042 +diff, 6920912342002, 20240719-144335_6908355010450, -0.013, 0.327, -0.034 +diff, 6970470331529, 20240719-144454_6924509902754, 0.087, 0.379, 0.171 +diff, 6902265131212, 20240719-144621_6902265314615, -0.000, 0.349, -0.000 +diff, 6954256905319, 20240719-145022_6941499112529, -0.030, 0.390, -0.075 +diff, 6970399922518, 20240719-145149_6972786772775, -0.096, 0.544, -0.224 +diff, 6916749306102, 20240719-145319_6920267906706, -0.063, 0.323, -0.163 +diff, 6923120632521, 20240719-145505_6974925910116, 0.163, 0.519, 0.311 +diff, 6972900962273, 20240719-145542_6928002374551, -0.001, 0.406, -0.003 +diff, 6970470331529, 20240719-145816_6920152414071, -0.042, 0.323, -0.085 +diff, 6973337010216, 20240719-145926_6973360690409, 0.060, 0.590, 0.200 +diff, 6902265131212, 20240719-150000_6930364600001, -0.159, 0.404, -0.298 +diff, 6918598120014, 20240719-150209_6937962111526, 0.233, 0.614, 0.438 +diff, 6938888889896, 20240719-150345_6924509902839, -0.086, 0.286, -0.262 +diff, 6971075127784, 20240719-150458_6924509902839, -0.079, 0.224, -0.112 +diff, 6921502656660, 20240719-150623_6924187895423, 0.002, 0.386, 0.006 +diff, 6924509902754, 20240719-150716_6916749306102, 0.015, 0.384, 0.034 +diff, 6902265202318, 20240719-150750_6951469100067, -0.139, 0.325, -0.234 +diff, 4892655902237, 20240719-151009_6973360690409, 0.041, 0.366, 0.084 +diff, 6938888889896, 20240719-151135_6923120632521, 0.018, 0.333, 0.043 +diff, 6923120632521, 20240719-151215_6958350501101, 0.026, 0.349, 0.047 +diff, 6941499112529, 20240719-151325_6951469100067, -0.006, 0.270, -0.010 +diff, 6974158894528, 20240719-151349_6937962111526, -0.076, 0.322, -0.206 +diff, 4897098800133, 20240719-151416_6916749306102, -0.061, 0.343, -0.146 +diff, 6958350501101, 20240719-151535_6930364600001, 0.003, 0.361, 0.006 +diff, 6972219890632, 20240719-151724_6911988027191, 0.005, 0.356, 0.010 +diff, 6972900962273, 20240719-151800_6972219890632, 0.010, 0.356, 0.022 +diff, 6974925910116, 20240719-152125_6901668935908, 0.040, 0.441, 0.090 +diff, 6972786772775, 20240719-152315_6920152471517, -0.049, 0.318, -0.111 +diff, 6971284201244, 20240719-152816_6951957205274, 0.039, 0.382, 0.088 +diff, 6925303795535, 20240719-152832_6971284201244, 0.021, 0.405, 0.050 +diff, 6970399922518, 20240719-153117_6930364600001, -0.039, 0.415, -0.123 +diff, 6925303795535, 20240722-143041_6972900962273, 0.016, 0.368, 0.047 +diff, 6973360690409, 20240722-144630_6902265587712, 0.027, 0.415, 0.087 +diff, 6925303795535, 20240722-160857_4892655902237, -0.033, 0.313, -0.075 +diff, 6902265202318, 20240722-162225_4892655902015, -0.023, 0.348, -0.048 +diff, 6971075127784, 20240722-162328_6938888889896, -0.010, 0.239, -0.017 +diff, 6937451883132, 20240722-163428_6920912342002, -0.009, 0.407, -0.025 +diff, 6970470331529, 20240722-163600_6971075127784, 0.045, 0.316, 0.081 +diff, 6901668006011, 20240722-164552_6926265317384, -0.059, 0.381, -0.135 +diff, 6920152471517, 20240722-164859_6916196422325, 0.100, 0.390, 0.186 +diff, 4892655902237, 20240722-165548_6901668006011, 0.031, 0.443, 0.057 +diff, 6958350501101, 20240722-170325_6926419102347, 0.034, 0.369, 0.065 +diff, 6938737200650, 20240723-094447_4897098800133, 0.058, 0.430, 0.156 +diff, 6972219890632, 20240723-094525_6971284204597, -0.005, 0.451, -0.013 +diff, 6924187828544, 20240723-094617_6970470331529, -0.016, 0.411, -0.037 +diff, 6941499112529, 20240723-095103_6918598120014, 0.029, 0.430, 0.054 +diff, 6921502656660, 20240723-095259_6902265202318, 0.001, 0.342, 0.002 +diff, 6902265314615, 20240723-095527_6938737200650, 0.015, 0.412, 0.028 +diff, 6970399922518, 20240723-095819_6902265131212, -0.062, 0.362, -0.162 +diff, 6921502656660, 20240723-095858_6902890022558, 0.025, 0.432, 0.072 +diff, 6902265587712, 20240723-100031_6925303795535, -0.033, 0.254, -0.059 +diff, 6972359682937, 20240723-100113_6920382723257, -0.044, 0.268, -0.087 +diff, 6973497200045, 20240723-100148_6976399680069, 0.014, 0.357, 0.034 +diff, 6928312901324, 20240723-100318_6921502656660, -0.043, 0.526, -0.100 +diff, 4892655902015, 20240723-100416_6938737200650, 0.041, 0.482, 0.092 +diff, 6924509902754, 20240723-100501_6918598120014, -0.072, 0.297, -0.146 +diff, 6972219890632, 20240723-100845_4897098800133, 0.017, 0.395, 0.041 +diff, 6974925910116, 20240723-101014_6971284204597, 0.046, 0.496, 0.098 +diff, 6972786772775, 20240723-101144_6902265202318, 0.062, 0.427, 0.142 +diff, 6902890022558, 20240723-101416_6902265131212, -0.037, 0.333, -0.096 +diff, 6901668006011, 20240723-101829_6918598120014, 0.017, 0.290, 0.027 +diff, 6930364600001, 20240723-101851_6902265131212, -0.016, 0.420, -0.041 +diff, 6974925910116, 20240723-102033_6920382723257, 0.102, 0.510, 0.232 +diff, 6937962111526, 20240723-102206_6902890022558, 0.059, 0.466, 0.166 +diff, 6924187895423, 20240723-102409_6970470331529, 0.021, 0.352, 0.054 +diff, 6973337010216, 20240723-102515_6938737200650, 0.072, 0.497, 0.208 +diff, 6974925910116, 20240723-110216_6937451883132, 0.090, 0.428, 0.182 +diff, 6901668935908, 20240723-110224_6973337010216, -0.074, 0.351, -0.161 +diff, 6974925910116, 20240723-110937_6972359682937, 0.085, 0.450, 0.207 +diff, 6925303795535, 20240723-111446_6937451883132, -0.009, 0.335, -0.021 +diff, 6928002374551, 20240723-111521_6973337010216, -0.012, 0.401, -0.030 +diff, 6972359682937, 20240723-111602_6924187828544, -0.095, 0.443, -0.182 +diff, 6941499112529, 20240723-112004_6934024590268, -0.008, 0.425, -0.017 +diff, 6928002374551, 20240723-112038_6972359682937, 0.083, 0.499, 0.195 +diff, 6918598120014, 20240723-112505_6924187828544, 0.005, 0.371, 0.009 +diff, 6925303795535, 20240723-155423_6974158894528, 0.058, 0.399, 0.109 +diff, 6916196422325, 20240723-160015_6973497200045, -0.003, 0.316, -0.005 +diff, 6924187895423, 20240723-160227_6970399922518, 0.073, 0.368, 0.153 +diff, 6973497200045, 20240723-160942_6974158894528, -0.093, 0.334, -0.164 diff --git a/contrast/result/record/20240909_152012_ft16.txt b/contrast/result/record/20240909_152012_ft16.txt new file mode 100644 index 0000000..e4ee78f --- /dev/null +++ b/contrast/result/record/20240909_152012_ft16.txt @@ -0,0 +1,174 @@ +same, 6954256905319, 20240715-144627_6954256905319, 0.208, 0.746, 0.605 +same, 6902265589617, 20240719-143553_6902265589617, 0.296, 0.719, 0.561 +same, 6930364600025, 20240719-143603_6930364600025, 0.301, 0.772, 0.666 +same, 6924509902839, 20240719-143630_6924509902839, 0.176, 0.660, 0.435 +same, 6902007115524, 20240719-143729_6902007115524, 0.230, 0.635, 0.525 +same, 6930628801472, 20240719-143807_6930628801472, 0.279, 0.730, 0.712 +same, 6902265314615, 20240719-144126_6902265314615, 0.376, 0.896, 0.754 +same, 6928312901324, 20240719-144226_6928312901324, 0.335, 0.782, 0.693 +same, 6908355010450, 20240719-144335_6908355010450, 0.256, 0.748, 0.580 +same, 6924509902754, 20240719-144454_6924509902754, 0.459, 0.832, 0.831 +same, 6902265314615, 20240719-144621_6902265314615, 0.410, 0.847, 0.667 +same, 6941499112529, 20240719-145022_6941499112529, 0.174, 0.736, 0.309 +same, 6972786772775, 20240719-145149_6972786772775, 0.243, 0.698, 0.565 +same, 6920267906706, 20240719-145319_6920267906706, 0.306, 0.800, 0.767 +same, 6974925910116, 20240719-145505_6974925910116, 0.372, 0.844, 0.743 +same, 6928002374551, 20240719-145542_6928002374551, 0.171, 0.773, 0.499 +same, 6920152414071, 20240719-145816_6920152414071, 0.469, 0.816, 0.871 +same, 6973360690409, 20240719-145926_6973360690409, 0.160, 0.636, 0.591 +same, 6930364600001, 20240719-150000_6930364600001, 0.318, 0.790, 0.754 +same, 6937962111526, 20240719-150209_6937962111526, 0.222, 0.660, 0.575 +same, 6924509902839, 20240719-150345_6924509902839, 0.155, 0.757, 0.470 +same, 6924509902839, 20240719-150458_6924509902839, 0.307, 0.670, 0.583 +same, 6924187895423, 20240719-150623_6924187895423, 0.437, 0.882, 0.892 +same, 6916749306102, 20240719-150716_6916749306102, 0.249, 0.725, 0.682 +same, 6951469100067, 20240719-150750_6951469100067, 0.535, 0.938, 0.905 +same, 6973360690409, 20240719-151009_6973360690409, 0.143, 0.578, 0.463 +same, 6923120632521, 20240719-151135_6923120632521, 0.301, 0.865, 0.711 +same, 6958350501101, 20240719-151215_6958350501101, 0.550, 0.902, 0.888 +same, 6951469100067, 20240719-151325_6951469100067, 0.588, 0.927, 0.899 +same, 6937962111526, 20240719-151349_6937962111526, 0.153, 0.657, 0.531 +same, 6916749306102, 20240719-151416_6916749306102, 0.228, 0.677, 0.587 +same, 6930364600001, 20240719-151535_6930364600001, 0.151, 0.692, 0.524 +same, 6911988027191, 20240719-151724_6911988027191, 0.127, 0.525, 0.337 +same, 6972219890632, 20240719-151800_6972219890632, 0.288, 0.872, 0.700 +same, 6901668935908, 20240719-152125_6901668935908, 0.210, 0.806, 0.436 +same, 6920152471517, 20240719-152315_6920152471517, 0.392, 0.836, 0.640 +same, 6951957205274, 20240719-152816_6951957205274, 0.345, 0.810, 0.602 +same, 6971284201244, 20240719-152832_6971284201244, 0.229, 0.817, 0.557 +same, 6930364600001, 20240719-153117_6930364600001, 0.065, 0.743, 0.196 +same, 6972900962273, 20240722-143041_6972900962273, 0.214, 0.596, 0.528 +same, 6902265587712, 20240722-144630_6902265587712, 0.362, 0.816, 0.673 +same, 4892655902237, 20240722-160857_4892655902237, 0.433, 0.783, 0.842 +same, 4892655902015, 20240722-162225_4892655902015, 0.100, 0.668, 0.282 +same, 6938888889896, 20240722-162328_6938888889896, 0.176, 0.779, 0.407 +same, 6920912342002, 20240722-163428_6920912342002, 0.313, 0.825, 0.857 +same, 6971075127784, 20240722-163600_6971075127784, 0.734, 0.923, 0.915 +same, 6926265317384, 20240722-164552_6926265317384, 0.179, 0.715, 0.613 +same, 6916196422325, 20240722-164859_6916196422325, 0.367, 0.773, 0.675 +same, 6901668006011, 20240722-165548_6901668006011, 0.291, 0.718, 0.546 +same, 6926419102347, 20240722-170325_6926419102347, 0.256, 0.785, 0.768 +same, 4897098800133, 20240723-094447_4897098800133, 0.379, 0.846, 0.805 +same, 6971284204597, 20240723-094525_6971284204597, 0.206, 0.732, 0.597 +same, 6970470331529, 20240723-094617_6970470331529, 0.235, 0.924, 0.586 +same, 6918598120014, 20240723-095103_6918598120014, 0.426, 0.905, 0.819 +same, 6902265202318, 20240723-095259_6902265202318, 0.641, 0.907, 0.909 +same, 6938737200650, 20240723-095527_6938737200650, 0.219, 0.825, 0.651 +same, 6902265131212, 20240723-095819_6902265131212, 0.197, 0.899, 0.396 +same, 6902890022558, 20240723-095858_6902890022558, 0.161, 0.713, 0.590 +same, 6925303795535, 20240723-100031_6925303795535, 0.298, 0.838, 0.674 +same, 6920382723257, 20240723-100113_6920382723257, 0.282, 0.815, 0.600 +same, 6976399680069, 20240723-100148_6976399680069, 0.040, 0.554, 0.132 +same, 6921502656660, 20240723-100318_6921502656660, 0.059, 0.424, 0.173 +same, 6938737200650, 20240723-100416_6938737200650, 0.222, 0.701, 0.620 +same, 6918598120014, 20240723-100501_6918598120014, 0.102, 0.742, 0.206 +same, 4897098800133, 20240723-100845_4897098800133, 0.305, 0.834, 0.744 +same, 6971284204597, 20240723-101014_6971284204597, 0.223, 0.714, 0.623 +same, 6902265202318, 20240723-101144_6902265202318, 0.537, 0.875, 0.812 +same, 6902265131212, 20240723-101416_6902265131212, 0.463, 0.901, 0.747 +same, 6918598120014, 20240723-101829_6918598120014, 0.524, 0.901, 0.855 +same, 6902265131212, 20240723-101851_6902265131212, 0.418, 0.899, 0.820 +same, 6920382723257, 20240723-102033_6920382723257, 0.320, 0.780, 0.755 +same, 6902890022558, 20240723-102206_6902890022558, 0.123, 0.707, 0.387 +same, 6970470331529, 20240723-102409_6970470331529, 0.099, 0.641, 0.300 +same, 6938737200650, 20240723-102515_6938737200650, 0.177, 0.695, 0.515 +same, 6937451883132, 20240723-110216_6937451883132, 0.300, 0.817, 0.706 +same, 6973337010216, 20240723-110224_6973337010216, 0.202, 0.727, 0.633 +same, 6972359682937, 20240723-110937_6972359682937, 0.229, 0.792, 0.540 +same, 6937451883132, 20240723-111446_6937451883132, 0.363, 0.796, 0.844 +same, 6973337010216, 20240723-111521_6973337010216, 0.120, 0.764, 0.359 +same, 6924187828544, 20240723-111602_6924187828544, 0.378, 0.857, 0.733 +same, 6934024590268, 20240723-112004_6934024590268, 0.205, 0.724, 0.609 +same, 6972359682937, 20240723-112038_6972359682937, 0.432, 0.839, 0.857 +same, 6924187828544, 20240723-112505_6924187828544, 0.304, 0.835, 0.601 +same, 6974158894528, 20240723-155423_6974158894528, 0.440, 0.854, 0.765 +same, 6973497200045, 20240723-160015_6973497200045, 0.263, 0.712, 0.551 +same, 6970399922518, 20240723-160227_6970399922518, 0.087, 0.564, 0.243 +same, 6974158894528, 20240723-160942_6974158894528, 0.429, 0.847, 0.818 +diff, 6973360690409, 20240715-144627_6954256905319, 0.091, 0.515, 0.307 +diff, 6924509902839, 20240719-143553_6902265589617, -0.039, 0.330, -0.084 +diff, 6973360690409, 20240719-143603_6930364600025, 0.011, 0.343, 0.034 +diff, 4897098800133, 20240719-143630_6924509902839, 0.038, 0.340, 0.103 +diff, 6902265202318, 20240719-143729_6902007115524, -0.069, 0.293, -0.125 +diff, 4892655902015, 20240719-143807_6930628801472, -0.059, 0.480, -0.135 +diff, 6908355010450, 20240719-144126_6902265314615, -0.045, 0.261, -0.100 +diff, 4892655902015, 20240719-144226_6928312901324, 0.017, 0.356, 0.042 +diff, 6920912342002, 20240719-144335_6908355010450, -0.013, 0.328, -0.034 +diff, 6970470331529, 20240719-144454_6924509902754, 0.087, 0.379, 0.171 +diff, 6902265131212, 20240719-144621_6902265314615, -0.000, 0.349, -0.000 +diff, 6954256905319, 20240719-145022_6941499112529, -0.030, 0.390, -0.075 +diff, 6970399922518, 20240719-145149_6972786772775, -0.096, 0.544, -0.224 +diff, 6916749306102, 20240719-145319_6920267906706, -0.063, 0.323, -0.163 +diff, 6923120632521, 20240719-145505_6974925910116, 0.163, 0.519, 0.311 +diff, 6972900962273, 20240719-145542_6928002374551, -0.001, 0.406, -0.003 +diff, 6970470331529, 20240719-145816_6920152414071, -0.042, 0.323, -0.085 +diff, 6973337010216, 20240719-145926_6973360690409, 0.060, 0.590, 0.200 +diff, 6902265131212, 20240719-150000_6930364600001, -0.159, 0.404, -0.298 +diff, 6918598120014, 20240719-150209_6937962111526, 0.233, 0.614, 0.438 +diff, 6938888889896, 20240719-150345_6924509902839, -0.086, 0.286, -0.262 +diff, 6971075127784, 20240719-150458_6924509902839, -0.079, 0.224, -0.112 +diff, 6921502656660, 20240719-150623_6924187895423, 0.002, 0.386, 0.006 +diff, 6924509902754, 20240719-150716_6916749306102, 0.015, 0.384, 0.034 +diff, 6902265202318, 20240719-150750_6951469100067, -0.139, 0.325, -0.234 +diff, 4892655902237, 20240719-151009_6973360690409, 0.041, 0.366, 0.084 +diff, 6938888889896, 20240719-151135_6923120632521, 0.018, 0.333, 0.043 +diff, 6923120632521, 20240719-151215_6958350501101, 0.026, 0.349, 0.047 +diff, 6941499112529, 20240719-151325_6951469100067, -0.006, 0.271, -0.010 +diff, 6974158894528, 20240719-151349_6937962111526, -0.076, 0.322, -0.206 +diff, 4897098800133, 20240719-151416_6916749306102, -0.061, 0.343, -0.146 +diff, 6958350501101, 20240719-151535_6930364600001, 0.003, 0.361, 0.006 +diff, 6972219890632, 20240719-151724_6911988027191, 0.005, 0.356, 0.010 +diff, 6972900962273, 20240719-151800_6972219890632, 0.010, 0.356, 0.022 +diff, 6974925910116, 20240719-152125_6901668935908, 0.040, 0.441, 0.090 +diff, 6972786772775, 20240719-152315_6920152471517, -0.049, 0.318, -0.111 +diff, 6971284201244, 20240719-152816_6951957205274, 0.039, 0.382, 0.088 +diff, 6925303795535, 20240719-152832_6971284201244, 0.021, 0.405, 0.050 +diff, 6970399922518, 20240719-153117_6930364600001, -0.039, 0.414, -0.123 +diff, 6925303795535, 20240722-143041_6972900962273, 0.016, 0.368, 0.047 +diff, 6973360690409, 20240722-144630_6902265587712, 0.027, 0.415, 0.087 +diff, 6925303795535, 20240722-160857_4892655902237, -0.033, 0.313, -0.075 +diff, 6902265202318, 20240722-162225_4892655902015, -0.023, 0.348, -0.048 +diff, 6971075127784, 20240722-162328_6938888889896, -0.010, 0.239, -0.017 +diff, 6937451883132, 20240722-163428_6920912342002, -0.009, 0.407, -0.025 +diff, 6970470331529, 20240722-163600_6971075127784, 0.045, 0.316, 0.081 +diff, 6901668006011, 20240722-164552_6926265317384, -0.059, 0.381, -0.135 +diff, 6920152471517, 20240722-164859_6916196422325, 0.100, 0.390, 0.186 +diff, 4892655902237, 20240722-165548_6901668006011, 0.031, 0.443, 0.057 +diff, 6958350501101, 20240722-170325_6926419102347, 0.034, 0.369, 0.065 +diff, 6938737200650, 20240723-094447_4897098800133, 0.058, 0.430, 0.156 +diff, 6972219890632, 20240723-094525_6971284204597, -0.005, 0.451, -0.013 +diff, 6924187828544, 20240723-094617_6970470331529, -0.016, 0.411, -0.037 +diff, 6941499112529, 20240723-095103_6918598120014, 0.029, 0.430, 0.054 +diff, 6921502656660, 20240723-095259_6902265202318, 0.001, 0.342, 0.002 +diff, 6902265314615, 20240723-095527_6938737200650, 0.015, 0.412, 0.028 +diff, 6970399922518, 20240723-095819_6902265131212, -0.062, 0.362, -0.162 +diff, 6921502656660, 20240723-095858_6902890022558, 0.025, 0.432, 0.072 +diff, 6902265587712, 20240723-100031_6925303795535, -0.033, 0.254, -0.059 +diff, 6972359682937, 20240723-100113_6920382723257, -0.044, 0.268, -0.087 +diff, 6973497200045, 20240723-100148_6976399680069, 0.014, 0.357, 0.034 +diff, 6928312901324, 20240723-100318_6921502656660, -0.043, 0.526, -0.100 +diff, 4892655902015, 20240723-100416_6938737200650, 0.041, 0.482, 0.091 +diff, 6924509902754, 20240723-100501_6918598120014, -0.072, 0.297, -0.146 +diff, 6972219890632, 20240723-100845_4897098800133, 0.017, 0.395, 0.041 +diff, 6974925910116, 20240723-101014_6971284204597, 0.046, 0.496, 0.098 +diff, 6972786772775, 20240723-101144_6902265202318, 0.062, 0.427, 0.142 +diff, 6902890022558, 20240723-101416_6902265131212, -0.037, 0.334, -0.096 +diff, 6901668006011, 20240723-101829_6918598120014, 0.017, 0.290, 0.028 +diff, 6930364600001, 20240723-101851_6902265131212, -0.016, 0.420, -0.041 +diff, 6974925910116, 20240723-102033_6920382723257, 0.102, 0.510, 0.231 +diff, 6937962111526, 20240723-102206_6902890022558, 0.059, 0.466, 0.166 +diff, 6924187895423, 20240723-102409_6970470331529, 0.021, 0.352, 0.054 +diff, 6973337010216, 20240723-102515_6938737200650, 0.072, 0.497, 0.208 +diff, 6974925910116, 20240723-110216_6937451883132, 0.090, 0.428, 0.182 +diff, 6901668935908, 20240723-110224_6973337010216, -0.074, 0.351, -0.161 +diff, 6974925910116, 20240723-110937_6972359682937, 0.085, 0.450, 0.207 +diff, 6925303795535, 20240723-111446_6937451883132, -0.009, 0.335, -0.021 +diff, 6928002374551, 20240723-111521_6973337010216, -0.012, 0.401, -0.030 +diff, 6972359682937, 20240723-111602_6924187828544, -0.095, 0.443, -0.182 +diff, 6941499112529, 20240723-112004_6934024590268, -0.008, 0.425, -0.017 +diff, 6928002374551, 20240723-112038_6972359682937, 0.083, 0.499, 0.195 +diff, 6918598120014, 20240723-112505_6924187828544, 0.005, 0.371, 0.009 +diff, 6925303795535, 20240723-155423_6974158894528, 0.058, 0.399, 0.109 +diff, 6916196422325, 20240723-160015_6973497200045, -0.003, 0.316, -0.005 +diff, 6924187895423, 20240723-160227_6970399922518, 0.073, 0.368, 0.153 +diff, 6973497200045, 20240723-160942_6974158894528, -0.093, 0.334, -0.164 diff --git a/contrast/result/record/20240909_152012_uint8.txt b/contrast/result/record/20240909_152012_uint8.txt new file mode 100644 index 0000000..358ee98 --- /dev/null +++ b/contrast/result/record/20240909_152012_uint8.txt @@ -0,0 +1,174 @@ +same, 6954256905319, 20240715-144627_6954256905319, 0.207, 0.740, 0.604 +same, 6902265589617, 20240719-143553_6902265589617, 0.296, 0.719, 0.561 +same, 6930364600025, 20240719-143603_6930364600025, 0.301, 0.772, 0.668 +same, 6924509902839, 20240719-143630_6924509902839, 0.177, 0.662, 0.438 +same, 6902007115524, 20240719-143729_6902007115524, 0.230, 0.632, 0.524 +same, 6930628801472, 20240719-143807_6930628801472, 0.279, 0.726, 0.712 +same, 6902265314615, 20240719-144126_6902265314615, 0.375, 0.895, 0.755 +same, 6928312901324, 20240719-144226_6928312901324, 0.336, 0.779, 0.696 +same, 6908355010450, 20240719-144335_6908355010450, 0.255, 0.743, 0.580 +same, 6924509902754, 20240719-144454_6924509902754, 0.457, 0.828, 0.830 +same, 6902265314615, 20240719-144621_6902265314615, 0.410, 0.849, 0.668 +same, 6941499112529, 20240719-145022_6941499112529, 0.173, 0.732, 0.308 +same, 6972786772775, 20240719-145149_6972786772775, 0.242, 0.693, 0.565 +same, 6920267906706, 20240719-145319_6920267906706, 0.305, 0.798, 0.764 +same, 6974925910116, 20240719-145505_6974925910116, 0.371, 0.844, 0.743 +same, 6928002374551, 20240719-145542_6928002374551, 0.171, 0.769, 0.500 +same, 6920152414071, 20240719-145816_6920152414071, 0.468, 0.812, 0.871 +same, 6973360690409, 20240719-145926_6973360690409, 0.160, 0.642, 0.592 +same, 6930364600001, 20240719-150000_6930364600001, 0.316, 0.790, 0.754 +same, 6937962111526, 20240719-150209_6937962111526, 0.220, 0.658, 0.572 +same, 6924509902839, 20240719-150345_6924509902839, 0.156, 0.752, 0.472 +same, 6924509902839, 20240719-150458_6924509902839, 0.308, 0.664, 0.585 +same, 6924187895423, 20240719-150623_6924187895423, 0.437, 0.881, 0.893 +same, 6916749306102, 20240719-150716_6916749306102, 0.249, 0.728, 0.684 +same, 6951469100067, 20240719-150750_6951469100067, 0.533, 0.935, 0.904 +same, 6973360690409, 20240719-151009_6973360690409, 0.144, 0.579, 0.466 +same, 6923120632521, 20240719-151135_6923120632521, 0.300, 0.863, 0.711 +same, 6958350501101, 20240719-151215_6958350501101, 0.548, 0.897, 0.888 +same, 6951469100067, 20240719-151325_6951469100067, 0.586, 0.925, 0.899 +same, 6937962111526, 20240719-151349_6937962111526, 0.152, 0.657, 0.530 +same, 6916749306102, 20240719-151416_6916749306102, 0.227, 0.682, 0.587 +same, 6930364600001, 20240719-151535_6930364600001, 0.150, 0.693, 0.524 +same, 6911988027191, 20240719-151724_6911988027191, 0.126, 0.529, 0.336 +same, 6972219890632, 20240719-151800_6972219890632, 0.287, 0.868, 0.700 +same, 6901668935908, 20240719-152125_6901668935908, 0.209, 0.802, 0.436 +same, 6920152471517, 20240719-152315_6920152471517, 0.389, 0.829, 0.637 +same, 6951957205274, 20240719-152816_6951957205274, 0.344, 0.808, 0.602 +same, 6971284201244, 20240719-152832_6971284201244, 0.228, 0.814, 0.556 +same, 6930364600001, 20240719-153117_6930364600001, 0.065, 0.744, 0.196 +same, 6972900962273, 20240722-143041_6972900962273, 0.212, 0.596, 0.525 +same, 6902265587712, 20240722-144630_6902265587712, 0.362, 0.812, 0.674 +same, 4892655902237, 20240722-160857_4892655902237, 0.431, 0.784, 0.842 +same, 4892655902015, 20240722-162225_4892655902015, 0.099, 0.659, 0.281 +same, 6938888889896, 20240722-162328_6938888889896, 0.176, 0.780, 0.408 +same, 6920912342002, 20240722-163428_6920912342002, 0.312, 0.830, 0.858 +same, 6971075127784, 20240722-163600_6971075127784, 0.732, 0.919, 0.913 +same, 6926265317384, 20240722-164552_6926265317384, 0.179, 0.709, 0.613 +same, 6916196422325, 20240722-164859_6916196422325, 0.366, 0.770, 0.675 +same, 6901668006011, 20240722-165548_6901668006011, 0.290, 0.711, 0.545 +same, 6926419102347, 20240722-170325_6926419102347, 0.255, 0.786, 0.768 +same, 4897098800133, 20240723-094447_4897098800133, 0.378, 0.845, 0.805 +same, 6971284204597, 20240723-094525_6971284204597, 0.205, 0.732, 0.597 +same, 6970470331529, 20240723-094617_6970470331529, 0.235, 0.922, 0.587 +same, 6918598120014, 20240723-095103_6918598120014, 0.425, 0.904, 0.820 +same, 6902265202318, 20240723-095259_6902265202318, 0.638, 0.904, 0.908 +same, 6938737200650, 20240723-095527_6938737200650, 0.218, 0.821, 0.649 +same, 6902265131212, 20240723-095819_6902265131212, 0.197, 0.897, 0.396 +same, 6902890022558, 20240723-095858_6902890022558, 0.160, 0.710, 0.591 +same, 6925303795535, 20240723-100031_6925303795535, 0.296, 0.834, 0.672 +same, 6920382723257, 20240723-100113_6920382723257, 0.280, 0.808, 0.598 +same, 6976399680069, 20240723-100148_6976399680069, 0.041, 0.553, 0.133 +same, 6921502656660, 20240723-100318_6921502656660, 0.058, 0.428, 0.171 +same, 6938737200650, 20240723-100416_6938737200650, 0.222, 0.705, 0.620 +same, 6918598120014, 20240723-100501_6918598120014, 0.103, 0.737, 0.209 +same, 4897098800133, 20240723-100845_4897098800133, 0.304, 0.828, 0.744 +same, 6971284204597, 20240723-101014_6971284204597, 0.224, 0.705, 0.624 +same, 6902265202318, 20240723-101144_6902265202318, 0.533, 0.872, 0.810 +same, 6902265131212, 20240723-101416_6902265131212, 0.461, 0.897, 0.747 +same, 6918598120014, 20240723-101829_6918598120014, 0.522, 0.898, 0.855 +same, 6902265131212, 20240723-101851_6902265131212, 0.417, 0.895, 0.819 +same, 6920382723257, 20240723-102033_6920382723257, 0.321, 0.778, 0.758 +same, 6902890022558, 20240723-102206_6902890022558, 0.123, 0.704, 0.388 +same, 6970470331529, 20240723-102409_6970470331529, 0.101, 0.640, 0.304 +same, 6938737200650, 20240723-102515_6938737200650, 0.176, 0.697, 0.515 +same, 6937451883132, 20240723-110216_6937451883132, 0.299, 0.815, 0.704 +same, 6973337010216, 20240723-110224_6973337010216, 0.201, 0.729, 0.634 +same, 6972359682937, 20240723-110937_6972359682937, 0.227, 0.786, 0.538 +same, 6937451883132, 20240723-111446_6937451883132, 0.363, 0.799, 0.845 +same, 6973337010216, 20240723-111521_6973337010216, 0.119, 0.764, 0.358 +same, 6924187828544, 20240723-111602_6924187828544, 0.377, 0.853, 0.734 +same, 6934024590268, 20240723-112004_6934024590268, 0.205, 0.720, 0.609 +same, 6972359682937, 20240723-112038_6972359682937, 0.430, 0.832, 0.856 +same, 6924187828544, 20240723-112505_6924187828544, 0.304, 0.834, 0.601 +same, 6974158894528, 20240723-155423_6974158894528, 0.440, 0.849, 0.766 +same, 6973497200045, 20240723-160015_6973497200045, 0.261, 0.710, 0.550 +same, 6970399922518, 20240723-160227_6970399922518, 0.087, 0.563, 0.242 +same, 6974158894528, 20240723-160942_6974158894528, 0.428, 0.843, 0.817 +diff, 6973360690409, 20240715-144627_6954256905319, 0.091, 0.512, 0.307 +diff, 6924509902839, 20240719-143553_6902265589617, -0.039, 0.319, -0.085 +diff, 6973360690409, 20240719-143603_6930364600025, 0.011, 0.340, 0.034 +diff, 4897098800133, 20240719-143630_6924509902839, 0.038, 0.338, 0.101 +diff, 6902265202318, 20240719-143729_6902007115524, -0.068, 0.282, -0.124 +diff, 4892655902015, 20240719-143807_6930628801472, -0.059, 0.480, -0.135 +diff, 6908355010450, 20240719-144126_6902265314615, -0.044, 0.258, -0.100 +diff, 4892655902015, 20240719-144226_6928312901324, 0.016, 0.353, 0.039 +diff, 6920912342002, 20240719-144335_6908355010450, -0.013, 0.321, -0.034 +diff, 6970470331529, 20240719-144454_6924509902754, 0.086, 0.377, 0.171 +diff, 6902265131212, 20240719-144621_6902265314615, -0.001, 0.355, -0.001 +diff, 6954256905319, 20240719-145022_6941499112529, -0.029, 0.392, -0.073 +diff, 6970399922518, 20240719-145149_6972786772775, -0.093, 0.542, -0.219 +diff, 6916749306102, 20240719-145319_6920267906706, -0.063, 0.324, -0.163 +diff, 6923120632521, 20240719-145505_6974925910116, 0.161, 0.517, 0.308 +diff, 6972900962273, 20240719-145542_6928002374551, -0.001, 0.409, -0.001 +diff, 6970470331529, 20240719-145816_6920152414071, -0.041, 0.320, -0.083 +diff, 6973337010216, 20240719-145926_6973360690409, 0.060, 0.594, 0.200 +diff, 6902265131212, 20240719-150000_6930364600001, -0.159, 0.400, -0.299 +diff, 6918598120014, 20240719-150209_6937962111526, 0.233, 0.610, 0.439 +diff, 6938888889896, 20240719-150345_6924509902839, -0.086, 0.293, -0.262 +diff, 6971075127784, 20240719-150458_6924509902839, -0.078, 0.228, -0.109 +diff, 6921502656660, 20240719-150623_6924187895423, 0.003, 0.386, 0.008 +diff, 6924509902754, 20240719-150716_6916749306102, 0.015, 0.383, 0.035 +diff, 6902265202318, 20240719-150750_6951469100067, -0.138, 0.332, -0.233 +diff, 4892655902237, 20240719-151009_6973360690409, 0.040, 0.360, 0.083 +diff, 6938888889896, 20240719-151135_6923120632521, 0.017, 0.340, 0.040 +diff, 6923120632521, 20240719-151215_6958350501101, 0.025, 0.347, 0.047 +diff, 6941499112529, 20240719-151325_6951469100067, -0.005, 0.271, -0.008 +diff, 6974158894528, 20240719-151349_6937962111526, -0.075, 0.323, -0.205 +diff, 4897098800133, 20240719-151416_6916749306102, -0.061, 0.344, -0.146 +diff, 6958350501101, 20240719-151535_6930364600001, 0.002, 0.359, 0.004 +diff, 6972219890632, 20240719-151724_6911988027191, 0.005, 0.357, 0.010 +diff, 6972900962273, 20240719-151800_6972219890632, 0.011, 0.357, 0.023 +diff, 6974925910116, 20240719-152125_6901668935908, 0.039, 0.441, 0.087 +diff, 6972786772775, 20240719-152315_6920152471517, -0.050, 0.321, -0.113 +diff, 6971284201244, 20240719-152816_6951957205274, 0.039, 0.378, 0.088 +diff, 6925303795535, 20240719-152832_6971284201244, 0.022, 0.400, 0.051 +diff, 6970399922518, 20240719-153117_6930364600001, -0.039, 0.410, -0.122 +diff, 6925303795535, 20240722-143041_6972900962273, 0.016, 0.374, 0.048 +diff, 6973360690409, 20240722-144630_6902265587712, 0.028, 0.414, 0.090 +diff, 6925303795535, 20240722-160857_4892655902237, -0.032, 0.306, -0.073 +diff, 6902265202318, 20240722-162225_4892655902015, -0.024, 0.347, -0.050 +diff, 6971075127784, 20240722-162328_6938888889896, -0.010, 0.238, -0.017 +diff, 6937451883132, 20240722-163428_6920912342002, -0.008, 0.407, -0.024 +diff, 6970470331529, 20240722-163600_6971075127784, 0.045, 0.318, 0.081 +diff, 6901668006011, 20240722-164552_6926265317384, -0.060, 0.373, -0.136 +diff, 6920152471517, 20240722-164859_6916196422325, 0.099, 0.388, 0.185 +diff, 4892655902237, 20240722-165548_6901668006011, 0.033, 0.448, 0.062 +diff, 6958350501101, 20240722-170325_6926419102347, 0.034, 0.362, 0.067 +diff, 6938737200650, 20240723-094447_4897098800133, 0.058, 0.437, 0.158 +diff, 6972219890632, 20240723-094525_6971284204597, -0.005, 0.448, -0.012 +diff, 6924187828544, 20240723-094617_6970470331529, -0.015, 0.409, -0.036 +diff, 6941499112529, 20240723-095103_6918598120014, 0.030, 0.430, 0.056 +diff, 6921502656660, 20240723-095259_6902265202318, 0.001, 0.351, 0.003 +diff, 6902265314615, 20240723-095527_6938737200650, 0.015, 0.413, 0.029 +diff, 6970399922518, 20240723-095819_6902265131212, -0.062, 0.357, -0.164 +diff, 6921502656660, 20240723-095858_6902890022558, 0.026, 0.426, 0.074 +diff, 6902265587712, 20240723-100031_6925303795535, -0.035, 0.245, -0.061 +diff, 6972359682937, 20240723-100113_6920382723257, -0.043, 0.267, -0.086 +diff, 6973497200045, 20240723-100148_6976399680069, 0.014, 0.356, 0.033 +diff, 6928312901324, 20240723-100318_6921502656660, -0.042, 0.529, -0.099 +diff, 4892655902015, 20240723-100416_6938737200650, 0.041, 0.483, 0.091 +diff, 6924509902754, 20240723-100501_6918598120014, -0.073, 0.294, -0.148 +diff, 6972219890632, 20240723-100845_4897098800133, 0.015, 0.397, 0.036 +diff, 6974925910116, 20240723-101014_6971284204597, 0.045, 0.495, 0.097 +diff, 6972786772775, 20240723-101144_6902265202318, 0.061, 0.428, 0.141 +diff, 6902890022558, 20240723-101416_6902265131212, -0.037, 0.327, -0.098 +diff, 6901668006011, 20240723-101829_6918598120014, 0.016, 0.290, 0.027 +diff, 6930364600001, 20240723-101851_6902265131212, -0.016, 0.432, -0.040 +diff, 6974925910116, 20240723-102033_6920382723257, 0.102, 0.512, 0.233 +diff, 6937962111526, 20240723-102206_6902890022558, 0.059, 0.464, 0.167 +diff, 6924187895423, 20240723-102409_6970470331529, 0.021, 0.350, 0.055 +diff, 6973337010216, 20240723-102515_6938737200650, 0.072, 0.489, 0.209 +diff, 6974925910116, 20240723-110216_6937451883132, 0.092, 0.423, 0.187 +diff, 6901668935908, 20240723-110224_6973337010216, -0.075, 0.350, -0.163 +diff, 6974925910116, 20240723-110937_6972359682937, 0.086, 0.451, 0.209 +diff, 6925303795535, 20240723-111446_6937451883132, -0.010, 0.333, -0.023 +diff, 6928002374551, 20240723-111521_6973337010216, -0.011, 0.395, -0.029 +diff, 6972359682937, 20240723-111602_6924187828544, -0.093, 0.439, -0.180 +diff, 6941499112529, 20240723-112004_6934024590268, -0.007, 0.423, -0.015 +diff, 6928002374551, 20240723-112038_6972359682937, 0.084, 0.497, 0.199 +diff, 6918598120014, 20240723-112505_6924187828544, 0.006, 0.377, 0.011 +diff, 6925303795535, 20240723-155423_6974158894528, 0.058, 0.395, 0.110 +diff, 6916196422325, 20240723-160015_6973497200045, -0.001, 0.319, -0.002 +diff, 6924187895423, 20240723-160227_6970399922518, 0.073, 0.367, 0.155 +diff, 6973497200045, 20240723-160942_6974158894528, -0.093, 0.339, -0.165 diff --git a/contrast/result/record/20240909_163328.txt b/contrast/result/record/20240909_163328.txt new file mode 100644 index 0000000..9e6db87 --- /dev/null +++ b/contrast/result/record/20240909_163328.txt @@ -0,0 +1,174 @@ +same, 6954256905319, 20240715-144627_6954256905319, 0.208, 0.746, 0.604 +same, 6902265589617, 20240719-143553_6902265589617, 0.296, 0.719, 0.561 +same, 6930364600025, 20240719-143603_6930364600025, 0.301, 0.772, 0.666 +same, 6924509902839, 20240719-143630_6924509902839, 0.176, 0.660, 0.435 +same, 6902007115524, 20240719-143729_6902007115524, 0.230, 0.635, 0.525 +same, 6930628801472, 20240719-143807_6930628801472, 0.279, 0.730, 0.712 +same, 6902265314615, 20240719-144126_6902265314615, 0.376, 0.897, 0.754 +same, 6928312901324, 20240719-144226_6928312901324, 0.335, 0.782, 0.693 +same, 6908355010450, 20240719-144335_6908355010450, 0.256, 0.748, 0.580 +same, 6924509902754, 20240719-144454_6924509902754, 0.459, 0.832, 0.831 +same, 6902265314615, 20240719-144621_6902265314615, 0.410, 0.847, 0.667 +same, 6941499112529, 20240719-145022_6941499112529, 0.174, 0.736, 0.309 +same, 6972786772775, 20240719-145149_6972786772775, 0.243, 0.698, 0.565 +same, 6920267906706, 20240719-145319_6920267906706, 0.306, 0.799, 0.767 +same, 6974925910116, 20240719-145505_6974925910116, 0.372, 0.844, 0.743 +same, 6928002374551, 20240719-145542_6928002374551, 0.171, 0.773, 0.499 +same, 6920152414071, 20240719-145816_6920152414071, 0.469, 0.816, 0.871 +same, 6973360690409, 20240719-145926_6973360690409, 0.160, 0.636, 0.591 +same, 6930364600001, 20240719-150000_6930364600001, 0.318, 0.790, 0.754 +same, 6937962111526, 20240719-150209_6937962111526, 0.222, 0.660, 0.575 +same, 6924509902839, 20240719-150345_6924509902839, 0.155, 0.757, 0.470 +same, 6924509902839, 20240719-150458_6924509902839, 0.307, 0.670, 0.583 +same, 6924187895423, 20240719-150623_6924187895423, 0.437, 0.882, 0.892 +same, 6916749306102, 20240719-150716_6916749306102, 0.249, 0.725, 0.682 +same, 6951469100067, 20240719-150750_6951469100067, 0.535, 0.938, 0.905 +same, 6973360690409, 20240719-151009_6973360690409, 0.143, 0.578, 0.463 +same, 6923120632521, 20240719-151135_6923120632521, 0.301, 0.865, 0.711 +same, 6958350501101, 20240719-151215_6958350501101, 0.550, 0.902, 0.888 +same, 6951469100067, 20240719-151325_6951469100067, 0.588, 0.927, 0.899 +same, 6937962111526, 20240719-151349_6937962111526, 0.153, 0.657, 0.531 +same, 6916749306102, 20240719-151416_6916749306102, 0.228, 0.677, 0.587 +same, 6930364600001, 20240719-151535_6930364600001, 0.151, 0.692, 0.524 +same, 6911988027191, 20240719-151724_6911988027191, 0.127, 0.525, 0.337 +same, 6972219890632, 20240719-151800_6972219890632, 0.288, 0.872, 0.700 +same, 6901668935908, 20240719-152125_6901668935908, 0.210, 0.806, 0.436 +same, 6920152471517, 20240719-152315_6920152471517, 0.392, 0.836, 0.640 +same, 6951957205274, 20240719-152816_6951957205274, 0.345, 0.810, 0.602 +same, 6971284201244, 20240719-152832_6971284201244, 0.229, 0.817, 0.557 +same, 6930364600001, 20240719-153117_6930364600001, 0.065, 0.743, 0.196 +same, 6972900962273, 20240722-143041_6972900962273, 0.214, 0.596, 0.528 +same, 6902265587712, 20240722-144630_6902265587712, 0.362, 0.816, 0.673 +same, 4892655902237, 20240722-160857_4892655902237, 0.433, 0.783, 0.842 +same, 4892655902015, 20240722-162225_4892655902015, 0.100, 0.668, 0.282 +same, 6938888889896, 20240722-162328_6938888889896, 0.176, 0.779, 0.407 +same, 6920912342002, 20240722-163428_6920912342002, 0.313, 0.825, 0.857 +same, 6971075127784, 20240722-163600_6971075127784, 0.734, 0.923, 0.915 +same, 6926265317384, 20240722-164552_6926265317384, 0.179, 0.715, 0.613 +same, 6916196422325, 20240722-164859_6916196422325, 0.367, 0.773, 0.675 +same, 6901668006011, 20240722-165548_6901668006011, 0.291, 0.718, 0.546 +same, 6926419102347, 20240722-170325_6926419102347, 0.256, 0.785, 0.768 +same, 4897098800133, 20240723-094447_4897098800133, 0.379, 0.846, 0.805 +same, 6971284204597, 20240723-094525_6971284204597, 0.206, 0.732, 0.597 +same, 6970470331529, 20240723-094617_6970470331529, 0.235, 0.924, 0.586 +same, 6918598120014, 20240723-095103_6918598120014, 0.426, 0.905, 0.819 +same, 6902265202318, 20240723-095259_6902265202318, 0.641, 0.907, 0.909 +same, 6938737200650, 20240723-095527_6938737200650, 0.219, 0.825, 0.651 +same, 6902265131212, 20240723-095819_6902265131212, 0.197, 0.899, 0.396 +same, 6902890022558, 20240723-095858_6902890022558, 0.161, 0.713, 0.590 +same, 6925303795535, 20240723-100031_6925303795535, 0.298, 0.838, 0.674 +same, 6920382723257, 20240723-100113_6920382723257, 0.282, 0.815, 0.600 +same, 6976399680069, 20240723-100148_6976399680069, 0.040, 0.554, 0.132 +same, 6921502656660, 20240723-100318_6921502656660, 0.059, 0.424, 0.173 +same, 6938737200650, 20240723-100416_6938737200650, 0.222, 0.701, 0.620 +same, 6918598120014, 20240723-100501_6918598120014, 0.102, 0.742, 0.206 +same, 4897098800133, 20240723-100845_4897098800133, 0.305, 0.834, 0.744 +same, 6971284204597, 20240723-101014_6971284204597, 0.223, 0.714, 0.623 +same, 6902265202318, 20240723-101144_6902265202318, 0.537, 0.875, 0.812 +same, 6902265131212, 20240723-101416_6902265131212, 0.463, 0.901, 0.747 +same, 6918598120014, 20240723-101829_6918598120014, 0.524, 0.901, 0.855 +same, 6902265131212, 20240723-101851_6902265131212, 0.418, 0.899, 0.820 +same, 6920382723257, 20240723-102033_6920382723257, 0.320, 0.780, 0.755 +same, 6902890022558, 20240723-102206_6902890022558, 0.123, 0.707, 0.387 +same, 6970470331529, 20240723-102409_6970470331529, 0.099, 0.641, 0.300 +same, 6938737200650, 20240723-102515_6938737200650, 0.177, 0.695, 0.515 +same, 6937451883132, 20240723-110216_6937451883132, 0.300, 0.817, 0.706 +same, 6973337010216, 20240723-110224_6973337010216, 0.202, 0.727, 0.633 +same, 6972359682937, 20240723-110937_6972359682937, 0.229, 0.792, 0.540 +same, 6937451883132, 20240723-111446_6937451883132, 0.363, 0.796, 0.844 +same, 6973337010216, 20240723-111521_6973337010216, 0.120, 0.764, 0.359 +same, 6924187828544, 20240723-111602_6924187828544, 0.378, 0.857, 0.733 +same, 6934024590268, 20240723-112004_6934024590268, 0.205, 0.724, 0.609 +same, 6972359682937, 20240723-112038_6972359682937, 0.432, 0.839, 0.857 +same, 6924187828544, 20240723-112505_6924187828544, 0.304, 0.835, 0.601 +same, 6974158894528, 20240723-155423_6974158894528, 0.440, 0.854, 0.765 +same, 6973497200045, 20240723-160015_6973497200045, 0.263, 0.712, 0.551 +same, 6970399922518, 20240723-160227_6970399922518, 0.087, 0.564, 0.243 +same, 6974158894528, 20240723-160942_6974158894528, 0.429, 0.847, 0.818 +diff, 6930364600001, 20240715-144627_6954256905319, -0.064, 0.332, -0.166 +diff, 6971075127784, 20240719-143553_6902265589617, 0.172, 0.479, 0.276 +diff, 6920152414071, 20240719-143603_6930364600025, 0.035, 0.379, 0.068 +diff, 6976399680069, 20240719-143630_6924509902839, 0.006, 0.355, 0.020 +diff, 6924509902754, 20240719-143729_6902007115524, 0.021, 0.430, 0.047 +diff, 6971284201244, 20240719-143807_6930628801472, 0.014, 0.384, 0.034 +diff, 6926265317384, 20240719-144126_6902265314615, -0.021, 0.354, -0.069 +diff, 6971284201244, 20240719-144226_6928312901324, 0.015, 0.411, 0.039 +diff, 6902265202318, 20240719-144335_6908355010450, 0.030, 0.276, 0.056 +diff, 6923120632521, 20240719-144454_6924509902754, -0.005, 0.310, -0.009 +diff, 6920152414071, 20240719-144621_6902265314615, -0.173, 0.216, -0.311 +diff, 6902890022558, 20240719-145022_6941499112529, -0.047, 0.308, -0.135 +diff, 6911988027191, 20240719-145149_6972786772775, 0.000, 0.427, 0.000 +diff, 4892655902237, 20240719-145319_6920267906706, 0.078, 0.454, 0.170 +diff, 6901668935908, 20240719-145505_6974925910116, 0.055, 0.341, 0.102 +diff, 6973497200045, 20240719-145542_6928002374551, 0.053, 0.359, 0.130 +diff, 6924509902754, 20240719-145816_6920152414071, -0.178, 0.256, -0.329 +diff, 4897098800133, 20240719-145926_6973360690409, -0.003, 0.473, -0.008 +diff, 6970470331529, 20240719-150000_6930364600001, 0.004, 0.297, 0.010 +diff, 6920912342002, 20240719-150209_6937962111526, 0.021, 0.329, 0.044 +diff, 6930628801472, 20240719-150345_6924509902839, 0.005, 0.429, 0.019 +diff, 6930364600025, 20240719-150458_6924509902839, 0.105, 0.450, 0.223 +diff, 6920152471517, 20240719-150623_6924187895423, -0.043, 0.378, -0.084 +diff, 6928002374551, 20240719-150716_6916749306102, -0.009, 0.451, -0.026 +diff, 6930364600025, 20240719-150750_6951469100067, -0.015, 0.387, -0.035 +diff, 6934024590268, 20240719-151009_6973360690409, -0.001, 0.433, -0.004 +diff, 6954256905319, 20240719-151135_6923120632521, 0.049, 0.461, 0.156 +diff, 6971284201244, 20240719-151215_6958350501101, -0.022, 0.413, -0.049 +diff, 6928002374551, 20240719-151325_6951469100067, 0.023, 0.386, 0.052 +diff, 6976399680069, 20240719-151349_6937962111526, -0.004, 0.359, -0.013 +diff, 6920152414071, 20240719-151416_6916749306102, 0.049, 0.375, 0.107 +diff, 6926419102347, 20240719-151535_6930364600001, 0.047, 0.448, 0.184 +diff, 6972359682937, 20240719-151724_6911988027191, -0.062, 0.344, -0.116 +diff, 6970399922518, 20240719-151800_6972219890632, 0.038, 0.494, 0.102 +diff, 6951469100067, 20240719-152125_6901668935908, -0.071, 0.355, -0.126 +diff, 6902890022558, 20240719-152315_6920152471517, -0.042, 0.340, -0.117 +diff, 6973337010216, 20240719-152816_6951957205274, -0.009, 0.361, -0.024 +diff, 6928002374551, 20240719-152832_6971284201244, 0.010, 0.309, 0.025 +diff, 6973497200045, 20240719-153117_6930364600001, -0.032, 0.308, -0.080 +diff, 6916196422325, 20240722-143041_6972900962273, 0.005, 0.400, 0.011 +diff, 6920152471517, 20240722-144630_6902265587712, 0.020, 0.316, 0.037 +diff, 6920382723257, 20240722-160857_4892655902237, -0.023, 0.348, -0.050 +diff, 6902890022558, 20240722-162225_4892655902015, 0.048, 0.533, 0.187 +diff, 6970470331529, 20240722-162328_6938888889896, 0.000, 0.464, 0.000 +diff, 6902007115524, 20240722-163428_6920912342002, 0.055, 0.483, 0.139 +diff, 6921502656660, 20240722-163600_6971075127784, -0.018, 0.380, -0.036 +diff, 6920382723257, 20240722-164552_6926265317384, 0.040, 0.346, 0.099 +diff, 6924509902754, 20240722-164859_6916196422325, -0.047, 0.426, -0.098 +diff, 6902890022558, 20240722-165548_6901668006011, -0.079, 0.319, -0.227 +diff, 6901668006011, 20240722-170325_6926419102347, -0.045, 0.335, -0.100 +diff, 6930364600001, 20240723-094447_4897098800133, 0.008, 0.349, 0.019 +diff, 6916749306102, 20240723-094525_6971284204597, 0.031, 0.360, 0.078 +diff, 6902265589617, 20240723-094617_6970470331529, -0.056, 0.382, -0.113 +diff, 6951957205274, 20240723-095103_6918598120014, 0.112, 0.447, 0.201 +diff, 6902007115524, 20240723-095259_6902265202318, -0.101, 0.273, -0.180 +diff, 6972359682937, 20240723-095527_6938737200650, -0.005, 0.349, -0.010 +diff, 6911988027191, 20240723-095819_6902265131212, -0.077, 0.393, -0.233 +diff, 6916196422325, 20240723-095858_6902890022558, 0.024, 0.318, 0.051 +diff, 6902007115524, 20240723-100031_6925303795535, -0.019, 0.272, -0.038 +diff, 6970399922518, 20240723-100113_6920382723257, -0.041, 0.394, -0.101 +diff, 6923120632521, 20240723-100148_6976399680069, -0.052, 0.272, -0.126 +diff, 6951957205274, 20240723-100318_6921502656660, -0.017, 0.340, -0.039 +diff, 6970470331529, 20240723-100416_6938737200650, -0.016, 0.308, -0.035 +diff, 6972900962273, 20240723-100501_6918598120014, -0.003, 0.441, -0.006 +diff, 6924509902839, 20240723-100845_4897098800133, -0.030, 0.348, -0.068 +diff, 6928312901324, 20240723-101014_6971284204597, 0.026, 0.370, 0.050 +diff, 6920152414071, 20240723-101144_6902265202318, -0.022, 0.291, -0.041 +diff, 6920912342002, 20240723-101416_6902265131212, -0.109, 0.174, -0.211 +diff, 6930364600025, 20240723-101829_6918598120014, -0.085, 0.214, -0.159 +diff, 6958350501101, 20240723-101851_6902265131212, -0.005, 0.412, -0.010 +diff, 6951469100067, 20240723-102033_6920382723257, 0.113, 0.602, 0.204 +diff, 6951957205274, 20240723-102206_6902890022558, 0.037, 0.467, 0.070 +diff, 6902265587712, 20240723-102409_6970470331529, -0.019, 0.291, -0.048 +diff, 6972900962273, 20240723-102515_6938737200650, -0.030, 0.330, -0.062 +diff, 6920152471517, 20240723-110216_6937451883132, 0.010, 0.361, 0.018 +diff, 6951469100067, 20240723-110224_6973337010216, -0.064, 0.300, -0.118 +diff, 6901668935908, 20240723-110937_6972359682937, -0.056, 0.275, -0.126 +diff, 6938737200650, 20240723-111446_6937451883132, -0.005, 0.461, -0.015 +diff, 6972359682937, 20240723-111521_6973337010216, -0.105, 0.219, -0.226 +diff, 6951469100067, 20240723-111602_6924187828544, -0.081, 0.228, -0.127 +diff, 6902265131212, 20240723-112004_6934024590268, -0.001, 0.243, -0.003 +diff, 6920267906706, 20240723-112038_6972359682937, 0.014, 0.334, 0.031 +diff, 6925303795535, 20240723-112505_6924187828544, -0.032, 0.382, -0.073 +diff, 6920912342002, 20240723-155423_6974158894528, -0.095, 0.278, -0.174 +diff, 6941499112529, 20240723-160015_6973497200045, 0.006, 0.466, 0.011 +diff, 6902007115524, 20240723-160227_6970399922518, 0.077, 0.413, 0.179 +diff, 6972219890632, 20240723-160942_6974158894528, -0.060, 0.369, -0.118 diff --git a/contrast/result/record/20240909_163328_ft16.txt b/contrast/result/record/20240909_163328_ft16.txt new file mode 100644 index 0000000..af62e96 --- /dev/null +++ b/contrast/result/record/20240909_163328_ft16.txt @@ -0,0 +1,174 @@ +same, 6954256905319, 20240715-144627_6954256905319, 0.208, 0.746, 0.605 +same, 6902265589617, 20240719-143553_6902265589617, 0.296, 0.719, 0.561 +same, 6930364600025, 20240719-143603_6930364600025, 0.301, 0.772, 0.666 +same, 6924509902839, 20240719-143630_6924509902839, 0.176, 0.660, 0.435 +same, 6902007115524, 20240719-143729_6902007115524, 0.230, 0.635, 0.525 +same, 6930628801472, 20240719-143807_6930628801472, 0.279, 0.730, 0.712 +same, 6902265314615, 20240719-144126_6902265314615, 0.376, 0.896, 0.754 +same, 6928312901324, 20240719-144226_6928312901324, 0.335, 0.782, 0.693 +same, 6908355010450, 20240719-144335_6908355010450, 0.256, 0.748, 0.580 +same, 6924509902754, 20240719-144454_6924509902754, 0.459, 0.832, 0.831 +same, 6902265314615, 20240719-144621_6902265314615, 0.410, 0.847, 0.667 +same, 6941499112529, 20240719-145022_6941499112529, 0.174, 0.736, 0.309 +same, 6972786772775, 20240719-145149_6972786772775, 0.243, 0.698, 0.565 +same, 6920267906706, 20240719-145319_6920267906706, 0.306, 0.800, 0.767 +same, 6974925910116, 20240719-145505_6974925910116, 0.372, 0.844, 0.743 +same, 6928002374551, 20240719-145542_6928002374551, 0.171, 0.773, 0.499 +same, 6920152414071, 20240719-145816_6920152414071, 0.469, 0.816, 0.871 +same, 6973360690409, 20240719-145926_6973360690409, 0.160, 0.636, 0.591 +same, 6930364600001, 20240719-150000_6930364600001, 0.318, 0.790, 0.754 +same, 6937962111526, 20240719-150209_6937962111526, 0.222, 0.660, 0.575 +same, 6924509902839, 20240719-150345_6924509902839, 0.155, 0.757, 0.470 +same, 6924509902839, 20240719-150458_6924509902839, 0.307, 0.670, 0.583 +same, 6924187895423, 20240719-150623_6924187895423, 0.437, 0.882, 0.892 +same, 6916749306102, 20240719-150716_6916749306102, 0.249, 0.725, 0.682 +same, 6951469100067, 20240719-150750_6951469100067, 0.535, 0.938, 0.905 +same, 6973360690409, 20240719-151009_6973360690409, 0.143, 0.578, 0.463 +same, 6923120632521, 20240719-151135_6923120632521, 0.301, 0.865, 0.711 +same, 6958350501101, 20240719-151215_6958350501101, 0.550, 0.902, 0.888 +same, 6951469100067, 20240719-151325_6951469100067, 0.588, 0.927, 0.899 +same, 6937962111526, 20240719-151349_6937962111526, 0.153, 0.657, 0.531 +same, 6916749306102, 20240719-151416_6916749306102, 0.228, 0.677, 0.587 +same, 6930364600001, 20240719-151535_6930364600001, 0.151, 0.692, 0.524 +same, 6911988027191, 20240719-151724_6911988027191, 0.127, 0.525, 0.337 +same, 6972219890632, 20240719-151800_6972219890632, 0.288, 0.872, 0.700 +same, 6901668935908, 20240719-152125_6901668935908, 0.210, 0.806, 0.436 +same, 6920152471517, 20240719-152315_6920152471517, 0.392, 0.836, 0.640 +same, 6951957205274, 20240719-152816_6951957205274, 0.345, 0.810, 0.602 +same, 6971284201244, 20240719-152832_6971284201244, 0.229, 0.817, 0.557 +same, 6930364600001, 20240719-153117_6930364600001, 0.065, 0.743, 0.196 +same, 6972900962273, 20240722-143041_6972900962273, 0.214, 0.596, 0.528 +same, 6902265587712, 20240722-144630_6902265587712, 0.362, 0.816, 0.673 +same, 4892655902237, 20240722-160857_4892655902237, 0.433, 0.783, 0.842 +same, 4892655902015, 20240722-162225_4892655902015, 0.100, 0.668, 0.282 +same, 6938888889896, 20240722-162328_6938888889896, 0.176, 0.779, 0.407 +same, 6920912342002, 20240722-163428_6920912342002, 0.313, 0.825, 0.857 +same, 6971075127784, 20240722-163600_6971075127784, 0.734, 0.923, 0.915 +same, 6926265317384, 20240722-164552_6926265317384, 0.179, 0.715, 0.613 +same, 6916196422325, 20240722-164859_6916196422325, 0.367, 0.773, 0.675 +same, 6901668006011, 20240722-165548_6901668006011, 0.291, 0.718, 0.546 +same, 6926419102347, 20240722-170325_6926419102347, 0.256, 0.785, 0.768 +same, 4897098800133, 20240723-094447_4897098800133, 0.379, 0.846, 0.805 +same, 6971284204597, 20240723-094525_6971284204597, 0.206, 0.732, 0.597 +same, 6970470331529, 20240723-094617_6970470331529, 0.235, 0.924, 0.586 +same, 6918598120014, 20240723-095103_6918598120014, 0.426, 0.905, 0.819 +same, 6902265202318, 20240723-095259_6902265202318, 0.641, 0.907, 0.909 +same, 6938737200650, 20240723-095527_6938737200650, 0.219, 0.825, 0.651 +same, 6902265131212, 20240723-095819_6902265131212, 0.197, 0.899, 0.396 +same, 6902890022558, 20240723-095858_6902890022558, 0.161, 0.713, 0.590 +same, 6925303795535, 20240723-100031_6925303795535, 0.298, 0.838, 0.674 +same, 6920382723257, 20240723-100113_6920382723257, 0.282, 0.815, 0.600 +same, 6976399680069, 20240723-100148_6976399680069, 0.040, 0.554, 0.132 +same, 6921502656660, 20240723-100318_6921502656660, 0.059, 0.424, 0.173 +same, 6938737200650, 20240723-100416_6938737200650, 0.222, 0.701, 0.620 +same, 6918598120014, 20240723-100501_6918598120014, 0.102, 0.742, 0.206 +same, 4897098800133, 20240723-100845_4897098800133, 0.305, 0.834, 0.744 +same, 6971284204597, 20240723-101014_6971284204597, 0.223, 0.714, 0.623 +same, 6902265202318, 20240723-101144_6902265202318, 0.537, 0.875, 0.812 +same, 6902265131212, 20240723-101416_6902265131212, 0.463, 0.901, 0.747 +same, 6918598120014, 20240723-101829_6918598120014, 0.524, 0.901, 0.855 +same, 6902265131212, 20240723-101851_6902265131212, 0.418, 0.899, 0.820 +same, 6920382723257, 20240723-102033_6920382723257, 0.320, 0.780, 0.755 +same, 6902890022558, 20240723-102206_6902890022558, 0.123, 0.707, 0.387 +same, 6970470331529, 20240723-102409_6970470331529, 0.099, 0.641, 0.300 +same, 6938737200650, 20240723-102515_6938737200650, 0.177, 0.695, 0.515 +same, 6937451883132, 20240723-110216_6937451883132, 0.300, 0.817, 0.706 +same, 6973337010216, 20240723-110224_6973337010216, 0.202, 0.727, 0.633 +same, 6972359682937, 20240723-110937_6972359682937, 0.229, 0.792, 0.540 +same, 6937451883132, 20240723-111446_6937451883132, 0.363, 0.796, 0.844 +same, 6973337010216, 20240723-111521_6973337010216, 0.120, 0.764, 0.359 +same, 6924187828544, 20240723-111602_6924187828544, 0.378, 0.857, 0.733 +same, 6934024590268, 20240723-112004_6934024590268, 0.205, 0.724, 0.609 +same, 6972359682937, 20240723-112038_6972359682937, 0.432, 0.839, 0.857 +same, 6924187828544, 20240723-112505_6924187828544, 0.304, 0.835, 0.601 +same, 6974158894528, 20240723-155423_6974158894528, 0.440, 0.854, 0.765 +same, 6973497200045, 20240723-160015_6973497200045, 0.263, 0.712, 0.551 +same, 6970399922518, 20240723-160227_6970399922518, 0.087, 0.564, 0.243 +same, 6974158894528, 20240723-160942_6974158894528, 0.429, 0.847, 0.818 +diff, 6930364600001, 20240715-144627_6954256905319, -0.064, 0.332, -0.166 +diff, 6971075127784, 20240719-143553_6902265589617, 0.172, 0.479, 0.276 +diff, 6920152414071, 20240719-143603_6930364600025, 0.035, 0.379, 0.068 +diff, 6976399680069, 20240719-143630_6924509902839, 0.006, 0.355, 0.021 +diff, 6924509902754, 20240719-143729_6902007115524, 0.021, 0.430, 0.047 +diff, 6971284201244, 20240719-143807_6930628801472, 0.014, 0.384, 0.034 +diff, 6926265317384, 20240719-144126_6902265314615, -0.021, 0.354, -0.069 +diff, 6971284201244, 20240719-144226_6928312901324, 0.015, 0.411, 0.039 +diff, 6902265202318, 20240719-144335_6908355010450, 0.030, 0.276, 0.056 +diff, 6923120632521, 20240719-144454_6924509902754, -0.005, 0.310, -0.009 +diff, 6920152414071, 20240719-144621_6902265314615, -0.173, 0.217, -0.311 +diff, 6902890022558, 20240719-145022_6941499112529, -0.047, 0.308, -0.135 +diff, 6911988027191, 20240719-145149_6972786772775, 0.000, 0.427, 0.000 +diff, 4892655902237, 20240719-145319_6920267906706, 0.078, 0.454, 0.170 +diff, 6901668935908, 20240719-145505_6974925910116, 0.055, 0.341, 0.102 +diff, 6973497200045, 20240719-145542_6928002374551, 0.053, 0.359, 0.130 +diff, 6924509902754, 20240719-145816_6920152414071, -0.178, 0.256, -0.329 +diff, 4897098800133, 20240719-145926_6973360690409, -0.003, 0.473, -0.008 +diff, 6970470331529, 20240719-150000_6930364600001, 0.004, 0.297, 0.010 +diff, 6920912342002, 20240719-150209_6937962111526, 0.021, 0.329, 0.044 +diff, 6930628801472, 20240719-150345_6924509902839, 0.005, 0.429, 0.019 +diff, 6930364600025, 20240719-150458_6924509902839, 0.105, 0.450, 0.223 +diff, 6920152471517, 20240719-150623_6924187895423, -0.043, 0.378, -0.084 +diff, 6928002374551, 20240719-150716_6916749306102, -0.009, 0.451, -0.026 +diff, 6930364600025, 20240719-150750_6951469100067, -0.015, 0.387, -0.035 +diff, 6934024590268, 20240719-151009_6973360690409, -0.001, 0.433, -0.004 +diff, 6954256905319, 20240719-151135_6923120632521, 0.049, 0.461, 0.156 +diff, 6971284201244, 20240719-151215_6958350501101, -0.022, 0.413, -0.049 +diff, 6928002374551, 20240719-151325_6951469100067, 0.023, 0.386, 0.052 +diff, 6976399680069, 20240719-151349_6937962111526, -0.004, 0.359, -0.013 +diff, 6920152414071, 20240719-151416_6916749306102, 0.049, 0.375, 0.107 +diff, 6926419102347, 20240719-151535_6930364600001, 0.047, 0.448, 0.184 +diff, 6972359682937, 20240719-151724_6911988027191, -0.062, 0.344, -0.116 +diff, 6970399922518, 20240719-151800_6972219890632, 0.038, 0.494, 0.102 +diff, 6951469100067, 20240719-152125_6901668935908, -0.071, 0.355, -0.126 +diff, 6902890022558, 20240719-152315_6920152471517, -0.042, 0.340, -0.117 +diff, 6973337010216, 20240719-152816_6951957205274, -0.009, 0.361, -0.024 +diff, 6928002374551, 20240719-152832_6971284201244, 0.010, 0.309, 0.025 +diff, 6973497200045, 20240719-153117_6930364600001, -0.032, 0.308, -0.080 +diff, 6916196422325, 20240722-143041_6972900962273, 0.005, 0.400, 0.011 +diff, 6920152471517, 20240722-144630_6902265587712, 0.020, 0.315, 0.037 +diff, 6920382723257, 20240722-160857_4892655902237, -0.023, 0.348, -0.050 +diff, 6902890022558, 20240722-162225_4892655902015, 0.048, 0.533, 0.187 +diff, 6970470331529, 20240722-162328_6938888889896, 0.000, 0.464, 0.000 +diff, 6902007115524, 20240722-163428_6920912342002, 0.055, 0.483, 0.139 +diff, 6921502656660, 20240722-163600_6971075127784, -0.018, 0.380, -0.036 +diff, 6920382723257, 20240722-164552_6926265317384, 0.040, 0.346, 0.099 +diff, 6924509902754, 20240722-164859_6916196422325, -0.047, 0.426, -0.098 +diff, 6902890022558, 20240722-165548_6901668006011, -0.079, 0.319, -0.227 +diff, 6901668006011, 20240722-170325_6926419102347, -0.045, 0.335, -0.100 +diff, 6930364600001, 20240723-094447_4897098800133, 0.008, 0.348, 0.019 +diff, 6916749306102, 20240723-094525_6971284204597, 0.031, 0.360, 0.078 +diff, 6902265589617, 20240723-094617_6970470331529, -0.056, 0.382, -0.113 +diff, 6951957205274, 20240723-095103_6918598120014, 0.112, 0.447, 0.201 +diff, 6902007115524, 20240723-095259_6902265202318, -0.101, 0.273, -0.180 +diff, 6972359682937, 20240723-095527_6938737200650, -0.005, 0.349, -0.010 +diff, 6911988027191, 20240723-095819_6902265131212, -0.077, 0.393, -0.233 +diff, 6916196422325, 20240723-095858_6902890022558, 0.024, 0.318, 0.051 +diff, 6902007115524, 20240723-100031_6925303795535, -0.019, 0.272, -0.038 +diff, 6970399922518, 20240723-100113_6920382723257, -0.041, 0.394, -0.101 +diff, 6923120632521, 20240723-100148_6976399680069, -0.052, 0.272, -0.126 +diff, 6951957205274, 20240723-100318_6921502656660, -0.017, 0.340, -0.039 +diff, 6970470331529, 20240723-100416_6938737200650, -0.016, 0.308, -0.035 +diff, 6972900962273, 20240723-100501_6918598120014, -0.003, 0.441, -0.006 +diff, 6924509902839, 20240723-100845_4897098800133, -0.030, 0.348, -0.068 +diff, 6928312901324, 20240723-101014_6971284204597, 0.026, 0.370, 0.050 +diff, 6920152414071, 20240723-101144_6902265202318, -0.022, 0.291, -0.041 +diff, 6920912342002, 20240723-101416_6902265131212, -0.109, 0.174, -0.211 +diff, 6930364600025, 20240723-101829_6918598120014, -0.085, 0.214, -0.159 +diff, 6958350501101, 20240723-101851_6902265131212, -0.005, 0.412, -0.010 +diff, 6951469100067, 20240723-102033_6920382723257, 0.113, 0.602, 0.204 +diff, 6951957205274, 20240723-102206_6902890022558, 0.037, 0.467, 0.070 +diff, 6902265587712, 20240723-102409_6970470331529, -0.019, 0.291, -0.048 +diff, 6972900962273, 20240723-102515_6938737200650, -0.030, 0.330, -0.062 +diff, 6920152471517, 20240723-110216_6937451883132, 0.010, 0.361, 0.018 +diff, 6951469100067, 20240723-110224_6973337010216, -0.064, 0.300, -0.118 +diff, 6901668935908, 20240723-110937_6972359682937, -0.056, 0.275, -0.126 +diff, 6938737200650, 20240723-111446_6937451883132, -0.005, 0.461, -0.015 +diff, 6972359682937, 20240723-111521_6973337010216, -0.105, 0.219, -0.226 +diff, 6951469100067, 20240723-111602_6924187828544, -0.081, 0.228, -0.127 +diff, 6902265131212, 20240723-112004_6934024590268, -0.001, 0.243, -0.003 +diff, 6920267906706, 20240723-112038_6972359682937, 0.014, 0.334, 0.031 +diff, 6925303795535, 20240723-112505_6924187828544, -0.032, 0.382, -0.073 +diff, 6920912342002, 20240723-155423_6974158894528, -0.095, 0.278, -0.174 +diff, 6941499112529, 20240723-160015_6973497200045, 0.006, 0.466, 0.012 +diff, 6902007115524, 20240723-160227_6970399922518, 0.077, 0.413, 0.179 +diff, 6972219890632, 20240723-160942_6974158894528, -0.060, 0.369, -0.118 diff --git a/contrast/result/record/20240909_163328_uint8.txt b/contrast/result/record/20240909_163328_uint8.txt new file mode 100644 index 0000000..62b49c7 --- /dev/null +++ b/contrast/result/record/20240909_163328_uint8.txt @@ -0,0 +1,174 @@ +same, 6954256905319, 20240715-144627_6954256905319, 0.207, 0.740, 0.604 +same, 6902265589617, 20240719-143553_6902265589617, 0.296, 0.719, 0.561 +same, 6930364600025, 20240719-143603_6930364600025, 0.301, 0.772, 0.668 +same, 6924509902839, 20240719-143630_6924509902839, 0.177, 0.662, 0.438 +same, 6902007115524, 20240719-143729_6902007115524, 0.230, 0.632, 0.524 +same, 6930628801472, 20240719-143807_6930628801472, 0.279, 0.726, 0.712 +same, 6902265314615, 20240719-144126_6902265314615, 0.375, 0.895, 0.755 +same, 6928312901324, 20240719-144226_6928312901324, 0.336, 0.779, 0.696 +same, 6908355010450, 20240719-144335_6908355010450, 0.255, 0.743, 0.580 +same, 6924509902754, 20240719-144454_6924509902754, 0.457, 0.828, 0.830 +same, 6902265314615, 20240719-144621_6902265314615, 0.410, 0.849, 0.668 +same, 6941499112529, 20240719-145022_6941499112529, 0.173, 0.732, 0.308 +same, 6972786772775, 20240719-145149_6972786772775, 0.242, 0.693, 0.565 +same, 6920267906706, 20240719-145319_6920267906706, 0.305, 0.798, 0.764 +same, 6974925910116, 20240719-145505_6974925910116, 0.371, 0.844, 0.743 +same, 6928002374551, 20240719-145542_6928002374551, 0.171, 0.769, 0.500 +same, 6920152414071, 20240719-145816_6920152414071, 0.468, 0.812, 0.871 +same, 6973360690409, 20240719-145926_6973360690409, 0.160, 0.642, 0.592 +same, 6930364600001, 20240719-150000_6930364600001, 0.316, 0.790, 0.754 +same, 6937962111526, 20240719-150209_6937962111526, 0.220, 0.658, 0.572 +same, 6924509902839, 20240719-150345_6924509902839, 0.156, 0.752, 0.472 +same, 6924509902839, 20240719-150458_6924509902839, 0.308, 0.664, 0.585 +same, 6924187895423, 20240719-150623_6924187895423, 0.437, 0.881, 0.893 +same, 6916749306102, 20240719-150716_6916749306102, 0.249, 0.728, 0.684 +same, 6951469100067, 20240719-150750_6951469100067, 0.533, 0.935, 0.904 +same, 6973360690409, 20240719-151009_6973360690409, 0.144, 0.579, 0.466 +same, 6923120632521, 20240719-151135_6923120632521, 0.300, 0.863, 0.711 +same, 6958350501101, 20240719-151215_6958350501101, 0.548, 0.897, 0.888 +same, 6951469100067, 20240719-151325_6951469100067, 0.586, 0.925, 0.899 +same, 6937962111526, 20240719-151349_6937962111526, 0.152, 0.657, 0.530 +same, 6916749306102, 20240719-151416_6916749306102, 0.227, 0.682, 0.587 +same, 6930364600001, 20240719-151535_6930364600001, 0.150, 0.693, 0.524 +same, 6911988027191, 20240719-151724_6911988027191, 0.126, 0.529, 0.336 +same, 6972219890632, 20240719-151800_6972219890632, 0.287, 0.868, 0.700 +same, 6901668935908, 20240719-152125_6901668935908, 0.209, 0.802, 0.436 +same, 6920152471517, 20240719-152315_6920152471517, 0.389, 0.829, 0.637 +same, 6951957205274, 20240719-152816_6951957205274, 0.344, 0.808, 0.602 +same, 6971284201244, 20240719-152832_6971284201244, 0.228, 0.814, 0.556 +same, 6930364600001, 20240719-153117_6930364600001, 0.065, 0.744, 0.196 +same, 6972900962273, 20240722-143041_6972900962273, 0.212, 0.596, 0.525 +same, 6902265587712, 20240722-144630_6902265587712, 0.362, 0.812, 0.674 +same, 4892655902237, 20240722-160857_4892655902237, 0.431, 0.784, 0.842 +same, 4892655902015, 20240722-162225_4892655902015, 0.099, 0.659, 0.281 +same, 6938888889896, 20240722-162328_6938888889896, 0.176, 0.780, 0.408 +same, 6920912342002, 20240722-163428_6920912342002, 0.312, 0.830, 0.858 +same, 6971075127784, 20240722-163600_6971075127784, 0.732, 0.919, 0.913 +same, 6926265317384, 20240722-164552_6926265317384, 0.179, 0.709, 0.613 +same, 6916196422325, 20240722-164859_6916196422325, 0.366, 0.770, 0.675 +same, 6901668006011, 20240722-165548_6901668006011, 0.290, 0.711, 0.545 +same, 6926419102347, 20240722-170325_6926419102347, 0.255, 0.786, 0.768 +same, 4897098800133, 20240723-094447_4897098800133, 0.378, 0.845, 0.805 +same, 6971284204597, 20240723-094525_6971284204597, 0.205, 0.732, 0.597 +same, 6970470331529, 20240723-094617_6970470331529, 0.235, 0.922, 0.587 +same, 6918598120014, 20240723-095103_6918598120014, 0.425, 0.904, 0.820 +same, 6902265202318, 20240723-095259_6902265202318, 0.638, 0.904, 0.908 +same, 6938737200650, 20240723-095527_6938737200650, 0.218, 0.821, 0.649 +same, 6902265131212, 20240723-095819_6902265131212, 0.197, 0.897, 0.396 +same, 6902890022558, 20240723-095858_6902890022558, 0.160, 0.710, 0.591 +same, 6925303795535, 20240723-100031_6925303795535, 0.296, 0.834, 0.672 +same, 6920382723257, 20240723-100113_6920382723257, 0.280, 0.808, 0.598 +same, 6976399680069, 20240723-100148_6976399680069, 0.041, 0.553, 0.133 +same, 6921502656660, 20240723-100318_6921502656660, 0.058, 0.428, 0.171 +same, 6938737200650, 20240723-100416_6938737200650, 0.222, 0.705, 0.620 +same, 6918598120014, 20240723-100501_6918598120014, 0.103, 0.737, 0.209 +same, 4897098800133, 20240723-100845_4897098800133, 0.304, 0.828, 0.744 +same, 6971284204597, 20240723-101014_6971284204597, 0.224, 0.705, 0.624 +same, 6902265202318, 20240723-101144_6902265202318, 0.533, 0.872, 0.810 +same, 6902265131212, 20240723-101416_6902265131212, 0.461, 0.897, 0.747 +same, 6918598120014, 20240723-101829_6918598120014, 0.522, 0.898, 0.855 +same, 6902265131212, 20240723-101851_6902265131212, 0.417, 0.895, 0.819 +same, 6920382723257, 20240723-102033_6920382723257, 0.321, 0.778, 0.758 +same, 6902890022558, 20240723-102206_6902890022558, 0.123, 0.704, 0.388 +same, 6970470331529, 20240723-102409_6970470331529, 0.101, 0.640, 0.304 +same, 6938737200650, 20240723-102515_6938737200650, 0.176, 0.697, 0.515 +same, 6937451883132, 20240723-110216_6937451883132, 0.299, 0.815, 0.704 +same, 6973337010216, 20240723-110224_6973337010216, 0.201, 0.729, 0.634 +same, 6972359682937, 20240723-110937_6972359682937, 0.227, 0.786, 0.538 +same, 6937451883132, 20240723-111446_6937451883132, 0.363, 0.799, 0.845 +same, 6973337010216, 20240723-111521_6973337010216, 0.119, 0.764, 0.358 +same, 6924187828544, 20240723-111602_6924187828544, 0.377, 0.853, 0.734 +same, 6934024590268, 20240723-112004_6934024590268, 0.205, 0.720, 0.609 +same, 6972359682937, 20240723-112038_6972359682937, 0.430, 0.832, 0.856 +same, 6924187828544, 20240723-112505_6924187828544, 0.304, 0.834, 0.601 +same, 6974158894528, 20240723-155423_6974158894528, 0.440, 0.849, 0.766 +same, 6973497200045, 20240723-160015_6973497200045, 0.261, 0.710, 0.550 +same, 6970399922518, 20240723-160227_6970399922518, 0.087, 0.563, 0.242 +same, 6974158894528, 20240723-160942_6974158894528, 0.428, 0.843, 0.817 +diff, 6930364600001, 20240715-144627_6954256905319, -0.064, 0.326, -0.166 +diff, 6971075127784, 20240719-143553_6902265589617, 0.174, 0.479, 0.280 +diff, 6920152414071, 20240719-143603_6930364600025, 0.035, 0.382, 0.068 +diff, 6976399680069, 20240719-143630_6924509902839, 0.007, 0.359, 0.023 +diff, 6924509902754, 20240719-143729_6902007115524, 0.022, 0.425, 0.048 +diff, 6971284201244, 20240719-143807_6930628801472, 0.015, 0.384, 0.036 +diff, 6926265317384, 20240719-144126_6902265314615, -0.020, 0.353, -0.067 +diff, 6971284201244, 20240719-144226_6928312901324, 0.015, 0.406, 0.039 +diff, 6902265202318, 20240719-144335_6908355010450, 0.031, 0.280, 0.057 +diff, 6923120632521, 20240719-144454_6924509902754, -0.005, 0.306, -0.010 +diff, 6920152414071, 20240719-144621_6902265314615, -0.172, 0.217, -0.311 +diff, 6902890022558, 20240719-145022_6941499112529, -0.046, 0.308, -0.134 +diff, 6911988027191, 20240719-145149_6972786772775, 0.000, 0.434, 0.001 +diff, 4892655902237, 20240719-145319_6920267906706, 0.078, 0.448, 0.171 +diff, 6901668935908, 20240719-145505_6974925910116, 0.055, 0.343, 0.103 +diff, 6973497200045, 20240719-145542_6928002374551, 0.053, 0.365, 0.128 +diff, 6924509902754, 20240719-145816_6920152414071, -0.177, 0.256, -0.328 +diff, 4897098800133, 20240719-145926_6973360690409, -0.002, 0.471, -0.006 +diff, 6970470331529, 20240719-150000_6930364600001, 0.004, 0.306, 0.008 +diff, 6920912342002, 20240719-150209_6937962111526, 0.022, 0.337, 0.047 +diff, 6930628801472, 20240719-150345_6924509902839, 0.004, 0.425, 0.017 +diff, 6930364600025, 20240719-150458_6924509902839, 0.106, 0.446, 0.226 +diff, 6920152471517, 20240719-150623_6924187895423, -0.043, 0.379, -0.085 +diff, 6928002374551, 20240719-150716_6916749306102, -0.009, 0.449, -0.026 +diff, 6930364600025, 20240719-150750_6951469100067, -0.014, 0.389, -0.034 +diff, 6934024590268, 20240719-151009_6973360690409, -0.001, 0.425, -0.002 +diff, 6954256905319, 20240719-151135_6923120632521, 0.049, 0.467, 0.157 +diff, 6971284201244, 20240719-151215_6958350501101, -0.022, 0.414, -0.049 +diff, 6928002374551, 20240719-151325_6951469100067, 0.023, 0.382, 0.051 +diff, 6976399680069, 20240719-151349_6937962111526, -0.004, 0.352, -0.015 +diff, 6920152414071, 20240719-151416_6916749306102, 0.050, 0.384, 0.109 +diff, 6926419102347, 20240719-151535_6930364600001, 0.047, 0.446, 0.186 +diff, 6972359682937, 20240719-151724_6911988027191, -0.063, 0.342, -0.119 +diff, 6970399922518, 20240719-151800_6972219890632, 0.038, 0.503, 0.102 +diff, 6951469100067, 20240719-152125_6901668935908, -0.072, 0.351, -0.128 +diff, 6902890022558, 20240719-152315_6920152471517, -0.041, 0.346, -0.115 +diff, 6973337010216, 20240719-152816_6951957205274, -0.008, 0.360, -0.022 +diff, 6928002374551, 20240719-152832_6971284201244, 0.011, 0.306, 0.028 +diff, 6973497200045, 20240719-153117_6930364600001, -0.032, 0.315, -0.080 +diff, 6916196422325, 20240722-143041_6972900962273, 0.004, 0.400, 0.009 +diff, 6920152471517, 20240722-144630_6902265587712, 0.020, 0.312, 0.036 +diff, 6920382723257, 20240722-160857_4892655902237, -0.024, 0.356, -0.050 +diff, 6902890022558, 20240722-162225_4892655902015, 0.049, 0.533, 0.190 +diff, 6970470331529, 20240722-162328_6938888889896, -0.001, 0.459, -0.002 +diff, 6902007115524, 20240722-163428_6920912342002, 0.055, 0.487, 0.137 +diff, 6921502656660, 20240722-163600_6971075127784, -0.018, 0.373, -0.036 +diff, 6920382723257, 20240722-164552_6926265317384, 0.040, 0.347, 0.098 +diff, 6924509902754, 20240722-164859_6916196422325, -0.047, 0.433, -0.098 +diff, 6902890022558, 20240722-165548_6901668006011, -0.078, 0.317, -0.226 +diff, 6901668006011, 20240722-170325_6926419102347, -0.046, 0.338, -0.101 +diff, 6930364600001, 20240723-094447_4897098800133, 0.008, 0.346, 0.020 +diff, 6916749306102, 20240723-094525_6971284204597, 0.032, 0.358, 0.079 +diff, 6902265589617, 20240723-094617_6970470331529, -0.056, 0.383, -0.113 +diff, 6951957205274, 20240723-095103_6918598120014, 0.111, 0.445, 0.200 +diff, 6902007115524, 20240723-095259_6902265202318, -0.101, 0.269, -0.181 +diff, 6972359682937, 20240723-095527_6938737200650, -0.005, 0.349, -0.010 +diff, 6911988027191, 20240723-095819_6902265131212, -0.077, 0.389, -0.234 +diff, 6916196422325, 20240723-095858_6902890022558, 0.024, 0.326, 0.050 +diff, 6902007115524, 20240723-100031_6925303795535, -0.019, 0.261, -0.038 +diff, 6970399922518, 20240723-100113_6920382723257, -0.041, 0.389, -0.102 +diff, 6923120632521, 20240723-100148_6976399680069, -0.053, 0.270, -0.130 +diff, 6951957205274, 20240723-100318_6921502656660, -0.017, 0.346, -0.039 +diff, 6970470331529, 20240723-100416_6938737200650, -0.014, 0.319, -0.030 +diff, 6972900962273, 20240723-100501_6918598120014, -0.002, 0.445, -0.004 +diff, 6924509902839, 20240723-100845_4897098800133, -0.030, 0.354, -0.068 +diff, 6928312901324, 20240723-101014_6971284204597, 0.025, 0.367, 0.049 +diff, 6920152414071, 20240723-101144_6902265202318, -0.023, 0.286, -0.042 +diff, 6920912342002, 20240723-101416_6902265131212, -0.109, 0.177, -0.211 +diff, 6930364600025, 20240723-101829_6918598120014, -0.085, 0.217, -0.159 +diff, 6958350501101, 20240723-101851_6902265131212, -0.006, 0.413, -0.010 +diff, 6951469100067, 20240723-102033_6920382723257, 0.113, 0.603, 0.205 +diff, 6951957205274, 20240723-102206_6902890022558, 0.037, 0.468, 0.070 +diff, 6902265587712, 20240723-102409_6970470331529, -0.020, 0.293, -0.050 +diff, 6972900962273, 20240723-102515_6938737200650, -0.030, 0.339, -0.062 +diff, 6920152471517, 20240723-110216_6937451883132, 0.013, 0.363, 0.022 +diff, 6951469100067, 20240723-110224_6973337010216, -0.063, 0.305, -0.117 +diff, 6901668935908, 20240723-110937_6972359682937, -0.055, 0.272, -0.124 +diff, 6938737200650, 20240723-111446_6937451883132, -0.006, 0.462, -0.016 +diff, 6972359682937, 20240723-111521_6973337010216, -0.104, 0.220, -0.225 +diff, 6951469100067, 20240723-111602_6924187828544, -0.080, 0.224, -0.127 +diff, 6902265131212, 20240723-112004_6934024590268, -0.002, 0.237, -0.004 +diff, 6920267906706, 20240723-112038_6972359682937, 0.013, 0.328, 0.030 +diff, 6925303795535, 20240723-112505_6924187828544, -0.031, 0.389, -0.071 +diff, 6920912342002, 20240723-155423_6974158894528, -0.095, 0.283, -0.173 +diff, 6941499112529, 20240723-160015_6973497200045, 0.006, 0.470, 0.012 +diff, 6902007115524, 20240723-160227_6970399922518, 0.076, 0.415, 0.177 +diff, 6972219890632, 20240723-160942_6974158894528, -0.060, 0.365, -0.119 diff --git a/contrast/select_subimgs.py b/contrast/select_subimgs.py new file mode 100644 index 0000000..cfbcf62 --- /dev/null +++ b/contrast/select_subimgs.py @@ -0,0 +1,117 @@ +# -*- coding: utf-8 -*- +""" +Created on Mon Dec 23 13:58:13 2024 + +writting for selectting std subimgs to Wuhuaqi + +@author: ym +""" + +import os +import time +# import torch +import pickle +# import json +import numpy as np +from PIL import Image +import matplotlib.pyplot as plt +from scipy.spatial.distance import cdist +from feat_extract.config import config as conf +# from model import resnet18 as resnet18 +from feat_extract.inference import FeatsInterface #, inference_image + + +IMG_FORMAT = ['.bmp', '.jpg', '.jpeg', '.png'] + + + +def gen_features(imgpath): + + Encoder = FeatsInterface(conf) + imgs, imgnames = [], [] + for filename in os.listdir(imgpath): + file, ext = os.path.splitext(filename) + if ext not in IMG_FORMAT: continue + + fpath = os.path.join(imgpath, filename) + img = Image.open(fpath) + + imgs.append(img) + + filelist = file.split("_") + newname = "_".join([filelist[0],filelist[1], filelist[2], filelist[-3], filelist[-2], filelist[-1]]) + # imgnames.append(newname) + imgnames.append(file) + + features = Encoder.inference(imgs) + features /= np.linalg.norm(features, axis=1)[:, None] + + return features, imgnames + +def top_p_percent_indices(matrix, p): + """ + Finds the indices of the top p% largest elements in a 2D matrix. + Args: + matrix (np.ndarray): A 2D NumPy array. + p: int, 0-100 + Returns: + List[Tuple[int, int]]: A list of indices (row, column) for the top 10% largest elements. + """ + # Flatten the matrix + flat_matrix = matrix.flatten() + + # Calculate the threshold for the top 10% + num_elements = len(flat_matrix) + threshold_index = int(num_elements * 0.01*p) # Top 10% + threshold_index = max(1, threshold_index) # Ensure at least one element is considered + threshold_value = np.partition(flat_matrix, -threshold_index)[-threshold_index] + + # Create a mask for elements >= threshold + mask = matrix >= threshold_value + + # Get the indices of elements that satisfy the mask + indices = np.argwhere(mask) + + return list(map(tuple, indices)) + + +def main(): + imgpath = r"\\192.168.1.28\share\数据\已完成数据\展厅数据\v1.0\比对数据\整理\zhantingBase\6923555210479" + + feats, imgnames = gen_features(imgpath) + n = len(feats) + + matrix = 1 - cdist(feats, feats, 'cosine') + + nmatrix = np.array([[matrix[i][j] for j in range(n) if i != j] for i in range(n)]) + + top_p_large_index = top_p_percent_indices(nmatrix, 1) + top_p_small_index = top_p_percent_indices(-1*nmatrix, 1) + + simi_mean = np.mean(nmatrix, axis=1) + max_simi = np.max(nmatrix) + max_index = np.where(nmatrix==max_simi) + min_simi = np.min(nmatrix) + min_index = np.where(nmatrix==min_simi) + + fig, ax = plt.subplots() + + simils = [matrix[i][j] for j in range(n) for i in range(n) if j>i] + ax.hist(simils, bins=60, range=(-0.2, 1), edgecolor='black') + ax.set_xlim([-0.2, 1]) + ax.set_title("Similarity") + + + print("done!") + + + + + + + + + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/contrast/seqfeat_compare.py b/contrast/seqfeat_compare.py new file mode 100644 index 0000000..b29b3a2 --- /dev/null +++ b/contrast/seqfeat_compare.py @@ -0,0 +1,200 @@ +# -*- coding: utf-8 -*- +""" +Created on Fri Aug 9 10:36:45 2024 +分析图像对间的相似度 + +@author: ym +""" +import os +import cv2 +import numpy as np + +import torch +import sys +from scipy.spatial.distance import cdist + +''' 加载 LC 定义的模型形式''' +from config import config as conf +from model import resnet18 as resnet18 +from test_ori import inference_image +##============ load resnet mdoel +model = resnet18().to(conf.device) +# model = nn.DataParallel(model).to(conf.device) +model.load_state_dict(torch.load(conf.test_model, map_location=conf.device)) +model.eval() +print('load model {} '.format(conf.testbackbone)) + +IMG_FORMAT = ['.bmp', '.jpg', '.JPG', '.jpeg', '.png'] + +# ============================================================================= +# ''' 加载REID中定义的模型形式''' +# sys.path.append(r"D:\DetectTracking") +# from tracking.trackers.reid.reid_interface import ReIDInterface +# from tracking.trackers.reid.config import config as ReIDConfig +# ReIDEncoder = ReIDInterface(ReIDConfig) +# +# def inference_image_ReID(images): +# batch_patches = [] +# patches = [] +# for d, img1 in enumerate(images): +# +# +# img = img1[:, :, ::-1].copy() # the model expects RGB inputs +# patch = ReIDEncoder.transform(img) +# +# # patch = patch.to(device=self.device).half() +# if str(ReIDEncoder.device) != "cpu": +# patch = patch.to(device=ReIDEncoder.device).half() +# else: +# patch = patch.to(device=ReIDEncoder.device) +# +# patches.append(patch) +# if (d + 1) % ReIDEncoder.batch_size == 0: +# patches = torch.stack(patches, dim=0) +# batch_patches.append(patches) +# patches = [] +# +# if len(patches): +# patches = torch.stack(patches, dim=0) +# batch_patches.append(patches) +# +# features = np.zeros((0, ReIDEncoder.embedding_size)) +# for patches in batch_patches: +# pred = ReIDEncoder.model(patches) +# pred[torch.isinf(pred)] = 1.0 +# feat = pred.cpu().data.numpy() +# features = np.vstack((features, feat)) +# +# return features +# ============================================================================= + + +def silimarity_compare(): + + imgpaths = r"D:\DetectTracking\contrast\images\2" + + + filepaths = [] + for root, dirs, filenames in os.walk(imgpaths): + for filename in filenames: + file, ext = os.path.splitext(filename) + if ext not in IMG_FORMAT: continue + + file_path = os.path.join(root, filename) + filepaths.append(file_path) + + feature = inference_image(filepaths, conf.test_transform, model, conf.device) + feature /= np.linalg.norm(feature, axis=1)[:, None] + + similar = 1 - np.maximum(0.0, cdist(feature, feature, metric='cosine')) + + + print("Done!") + + + +def similarity_compare_sequence(root_dir): + ''' + root_dir：包含 "subimgs"字段的文件夹中图像为 subimg子图 + 功能：相邻帧子图间相似度比较 + + + ''' + + all_files = [] + extensions = ['.png', '.jpg'] + for dirpath, dirnames, filenames in os.walk(root_dir): + filepaths = [] + for filename in filenames: + if os.path.basename(dirpath).find('subimgs') < 0: + continue + file, ext = os.path.splitext(filename) + if ext in extensions: + imgpath = os.path.join(dirpath, filename) + filepaths.append(imgpath) + nf = len(filepaths) + if nf==0: + continue + + fnma = os.path.basename(filepaths[0]).split('.')[0] + imga = cv2.imread(filepaths[0]) + ha, wa = imga.shape[:2] + + for i in range(1, nf): + fnmb = os.path.basename(filepaths[i]).split('.')[0] + + imgb = cv2.imread(filepaths[i]) + hb, wb = imgb.shape[:2] + + + feats = inference_image_ReID(((imga, imgb))) + + similar = 1 - np.maximum(0.0, cdist(feats, feats, metric='cosine')) + + + h, w = max((ha, hb)), max((wa, wb)) + img = np.zeros(((h, 2*w, 3)), np.uint8) + img[0:ha, 0:wa], img[0:hb, w:(w+wb)] = imga, imgb + + linewidth = max(round(((h+2*w))/2 * 0.001), 2) + cv2.putText(img, + text=f'{similar[0,1]:.2f}', # Text string to be drawn + org=(max(w-20, 10), h-10), # Bottom-left corner of the text string + fontFace=0, # Font type + fontScale=linewidth/3, # Font scale factor + color=(0, 0, 255), # Text color + thickness=linewidth, # Thickness of the lines used to draw a text + lineType=cv2.LINE_AA, # Line type + ) + spath = os.path.join(dirpath, 's'+fnma+'-vs-'+fnmb+'.png') + cv2.imwrite(spath, img) + + + fnma = os.path.basename(filepaths[i]).split('.')[0] + imga = imgb.copy() + ha, wa = imga.shape[:2] + + + return + + +def main(): + root_dir = r"D:\contrast\dataset\result\20240723-112242_6923790709882" + + try: + similarity_compare_sequence(root_dir) + except Exception as e: + print(f'Error: {e}') + + + + +if __name__ == '__main__': + # main() + + silimarity_compare() + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/contrast/stdfeat_analys.py b/contrast/stdfeat_analys.py new file mode 100644 index 0000000..38d58aa --- /dev/null +++ b/contrast/stdfeat_analys.py @@ -0,0 +1,166 @@ +# -*- coding: utf-8 -*- +""" +Created on Wed Nov 20 11:17:29 2024 + +@author: ym +""" + +import os +import cv2 +import pickle +import numpy as np +from scipy.spatial.distance import cdist +import matplotlib.pyplot as plt + +def save_imgpairs(barcode, imgpaths, matrix, savepath, thresh=(0.4, 0.6), ctype="intra"): + if ctype=="intra": + rows, cols = np.triu_indices(matrix.shape[0], k=1) # k=1 表示不包括对角线 + mask = matrix[rows, cols] < thresh[1] + indices = list(zip(rows[mask], cols[mask])) + else: + rows, cols = np.where(matrix > thresh[0]) + indices = list(zip(rows, cols)) + + + if len(indices): + savepath = os.path.join(savepath, barcode) + if not os.path.exists(savepath): + os.makedirs (savepath) + + + for idx1, idx2 in indices: + if len(imgpaths) == 1: + img1 = cv2.imread(imgpaths[0][idx1]) + img2 = cv2.imread(imgpaths[0][idx2]) + elif len(imgpaths) == 2: + img1 = cv2.imread(imgpaths[0][idx1]) + img2 = cv2.imread(imgpaths[1][idx2]) + + + + simi = matrix[idx1, idx2] + + H1, W1 = img1.shape[:2] + H2, W2 = img2.shape[:2] + H, W = max((H1, H2)), max((W1, W2)) + img = np.ones((H, 2*W, 3), dtype=np.uint8) *np.array([255, 128, 128]) + + img[0:H1, 0:W1, :] = img1 + img[0:H2, (2*W-W2):, :] = img2 + + text = f"sim: {simi:.2f}" + org = (10, H-10) + cv2.putText(img, text, org, fontFace=cv2.FONT_HERSHEY_SIMPLEX, fontScale=0.75, + color=(0, 0, 255), thickness=2, lineType=cv2.LINE_AA) + imgpath = os.path.join(savepath, f"{simi:.2f}_{barcode}_{idx1}_{idx2}.png") + cv2.imwrite(imgpath, img) + + + +def feat_analysis(featpath): + ''' + 标准特征集中样本类内、类间相似度分布 + ''' + + + + savepath = r"D:\exhibition\result\stdfeat" + + InterThresh = (0.4, 0.6) + + featDict, features= [], [] + for filename in os.listdir(featpath): + file, ext = os.path.splitext(filename) + if ext != ".pickle": continue + + filepath = os.path.join(featpath, filename) + with open(filepath, 'rb') as f: + bpDict = pickle.load(f) + + feat = bpDict["feats_ft32"] + + featDict.append(bpDict) + features.append(feat) + N = len(features) + + simMatrix = [] + intra_simi = np.empty(0) + low_simi_index = {} + for i, feats in enumerate(features): + matrix = 1 - cdist(feats, feats, 'cosine') + + simMatrix.append(matrix) + + '''提取相似矩阵上三角元素''' + rows, cols = np.triu_indices(matrix.shape[0], k=1) # k=1 表示不包括对角线 + upper_tri= matrix[rows, cols] + intra_simi = np.concatenate((intra_simi, upper_tri)) + + '''保存相似度小于阈值的图像对''' + barcode = featDict[i]["barcode"] + imgpaths = featDict[i]["imgpaths"] + # save_imgpairs(barcode, [imgpaths], matrix, savepath, InterThresh, "intra") + print(f"{barcode} have done!") + + Matrix = np.zeros((N, N)) + inter_bcds = [] + inter_simi = np.empty(0) + for i, feati in enumerate(features): + bcdi = featDict[i]["barcode"] + imgpathi = featDict[i]["imgpaths"] + for j, featj in enumerate(features): + bcdj = featDict[j]["barcode"] + imgpathj = featDict[j]["imgpaths"] + + matrix = 1 - cdist(feati, featj, 'cosine') + + inter_bcds.append((i, j, bcdi, bcdj)) + Matrix[i, j] = np.mean(matrix) + if j>i: + bcd_ij = bcdi+'_'+bcdj + # save_imgpairs(bcd_ij, [imgpathi, imgpathj], matrix, savepath, InterThresh, "inter") + inter_simi = np.concatenate((inter_simi, matrix.ravel())) + + print(f"{bcd_ij} have done!") + + fig, axs = plt.subplots(2, 1) + axs[0].hist(intra_simi, bins=100, color='blue', edgecolor='black', alpha=0.7) + axs[0].set_xlim(0, 1) + axs[0].set_xlabel('Performance') + axs[0].set_title("inter similarity") + + axs[1].hist(inter_simi, bins=100, color='green', edgecolor='black', alpha=0.7) + axs[1].set_xlim(0, 1) + axs[1].set_xlabel('Performance') + axs[1].set_title("inter similarity") + + + + + + print("Done！") + + + + + + + + +def main(): + stdpath = r"D:\exhibition\dataset\feats" + + feat_analysis(stdpath) + + +if __name__ == '__main__': + main() + + + + + + + + + \ No newline at end of file diff --git a/contrast/trail2trail.py b/contrast/trail2trail.py new file mode 100644 index 0000000..cbd364a --- /dev/null +++ b/contrast/trail2trail.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" + 取出再放回场景下商品轨迹特征比对方式与性能分析 + +Created on Tue Apr 1 17:17:47 2025 +@author: wqg +""" +import os +import pickle +import random +import numpy as np +from pathlib import Path +import matplotlib.pyplot as plt +from scipy.spatial.distance import cdist +from utils.calsimi import calsiml, calsimi_vs_evts + +def read_eventdict(evtpaths): + evtDict = {} + for filename in os.listdir(evtpaths): + evtname, ext = os.path.splitext(filename) + if ext != ".pickle": continue + + evtpath = os.path.join(evtpaths, filename) + with open(evtpath, 'rb') as f: + evtdata = pickle.load(f) + evtDict[evtname] = evtdata + + + return evtDict + + + +def compute_show_pr(Same, Cross): + TPFN = len(Same) + TNFP = len(Cross) + + Recall_Pos, Recall_Neg = [], [] + Precision_Pos, Precision_Neg = [], [] + Correct = [] + Thresh = np.linspace(-0.2, 1, 100) + for th in Thresh: + TP = np.sum(Same >= th) + FN = np.sum(Same < th) + # FN = TPFN - TP + + TN = np.sum(Cross < th) + FP = np.sum(Cross >= th) + # FP = TNFP - TN + + + Precision_Pos.append(TP/(TP+FP+1e-6)) + Precision_Neg.append(TN/(TN+FN+1e-6)) + Recall_Pos.append(TP/(TP+FN+1e-6)) + Recall_Neg.append(TN/(TN+FP+1e-6)) + + # Recall_Pos.append(TP/TPFN) + # Recall_Neg.append(TN/TNFP) + + + Correct.append((TN+TP)/(TPFN+TNFP)) + + fig, ax = plt.subplots() + + ax.plot(Thresh, Precision_Pos, 'r', label='Precision_Pos: TP/(TP+FP)') + ax.plot(Thresh, Recall_Pos, 'b', label='Recall_Pos: TP/TPFN') + ax.plot(Thresh, Recall_Neg, 'g', label='Recall_Neg: TN/TNFP') + ax.plot(Thresh, Correct, 'c', label='Correct: (TN+TP)/(TPFN+TNFP)') + ax.plot(Thresh, Precision_Neg, 'm', label='Precision_Neg: TN/(TN+FN)') + + ax.set_xlim([0, 1]) + ax.set_ylim([0, 1]) + + ax.set_xticks(np.arange(0, 1, 0.1)) + ax.set_yticks(np.arange(0, 1, 0.1)) + ax.grid(True, linestyle='--') + + ax.set_title('PrecisePos & PreciseNeg') + ax.set_xlabel(f"Same Num: {TPFN}, Cross Num: {TNFP}") + ax.legend() + plt.show() + + # rltpath = os.path.join(similPath, f'pr_1to1_{simType}.png') + # plt.savefig(rltpath) # svg, png, pdf + + + fig, axes = plt.subplots(2,1) + axes[0].hist(Same, bins=60, range=(-0.2, 1), edgecolor='black') + axes[0].set_xlim([-0.2, 1]) + axes[0].set_title(f'TP({len(Same)})') + + axes[1].hist(Cross, bins=60, range=(-0.2, 1), edgecolor='black') + axes[1].set_xlim([-0.2, 1]) + axes[1].set_title(f'TN({len(Cross)})') + + # rltpath = os.path.join(similPath, f'hist_1to1_{simType}.png') + # plt.savefig(rltpath) + + + plt.show() + + + +def trail_to_trail(evtpaths, rltpaths): + # select the method type of how to calculate the feat similarity of trail + simType = 2 + + ##1. read all the ShoppingEvent object in the dir 'evtpaths' + evtDicts = read_eventdict(evtpaths) + + ##2. Combine event object with the same barcode + barcodes, evtpairDict = [], {} + for k in evtDicts.keys(): + evt = k.split('_') + condt = len(evt)>=2 and evt[-1].isdigit() and len(evt[-1])>=10 + if not condt: continue + + barcode = evt[-1] + if barcode not in evtpairDict.keys(): + evtpairDict[barcode] = [] + barcodes.append(barcode) + + evtpairDict[barcode].append(evtDicts[k]) + barcodes = set(barcodes) + + AA_list, AB_list = [], [] + for barcode in evtpairDict.keys(): + events = evtpairDict[barcode] + if len(events)>1: + evta, evtb = random.sample(events, 2) + AA_list.append((evta, evtb, "same")) + + evtc = random.sample(events, 1)[0] + + dset = list(barcodes.symmetric_difference(set([barcode]))) + bcd = random.sample(dset, 1)[0] + evtd = random.sample(evtpairDict[bcd], 1)[0] + AB_list.append((evtc, evtd, "diff")) + + mergePairs = AA_list + AB_list + + ##3. calculate the similar of two event: evta, evtb + new_pirs = [] + for evta, evtb, label in mergePairs: + similar = calsimi_vs_evts(evta, evtb, simType) + if similar is None: + continue + new_pirs.append((label, round(similar, 3), evta.evtname[:15], evtb.evtname[:15])) + + ##4. compute PR and showing + Same = np.array([s for label, s, _, _ in new_pirs if label=="same"]) + Cross = np.array([s for label, s, _, _ in new_pirs if label=="diff"]) + compute_show_pr(Same, Cross) + + +def main(): + evttypes = ["single_event_V10", "single_event_V5", "performence_V10", "performence_V5"] + # evttypes = ["single_event_V10"] + + for evttype in evttypes: + evtpaths = "/home/wqg/dataset/pipeline/contrast/{}/evtobjs/".format(evttype) + rltpaths = "/home/wqg/dataset/pipeline/yrt/{}/yolos_tracking".format(evttype) + + trail_to_trail(evtpaths, rltpaths) + + + +if __name__ == '__main__': + main() + + + \ No newline at end of file diff --git a/contrast/utils/__init__.py b/contrast/utils/__init__.py new file mode 100644 index 0000000..0cd4367 --- /dev/null +++ b/contrast/utils/__init__.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Created on Thu Sep 26 08:53:58 2024 + +@author: ym +""" + diff --git a/contrast/utils/__pycache__/__init__.cpython-312.pyc b/contrast/utils/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..0804b41 Binary files /dev/null and b/contrast/utils/__pycache__/__init__.cpython-312.pyc differ diff --git a/contrast/utils/__pycache__/__init__.cpython-39.pyc b/contrast/utils/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..c7e60b0 Binary files /dev/null and b/contrast/utils/__pycache__/__init__.cpython-39.pyc differ diff --git a/contrast/utils/__pycache__/calsimi.cpython-312.pyc b/contrast/utils/__pycache__/calsimi.cpython-312.pyc new file mode 100644 index 0000000..fe0388d Binary files /dev/null and b/contrast/utils/__pycache__/calsimi.cpython-312.pyc differ diff --git a/contrast/utils/__pycache__/databits.cpython-312.pyc b/contrast/utils/__pycache__/databits.cpython-312.pyc new file mode 100644 index 0000000..c3bdc62 Binary files /dev/null and b/contrast/utils/__pycache__/databits.cpython-312.pyc differ diff --git a/contrast/utils/__pycache__/event.cpython-312.pyc b/contrast/utils/__pycache__/event.cpython-312.pyc new file mode 100644 index 0000000..eb34dc6 Binary files /dev/null and b/contrast/utils/__pycache__/event.cpython-312.pyc differ diff --git a/contrast/utils/__pycache__/event.cpython-39.pyc b/contrast/utils/__pycache__/event.cpython-39.pyc new file mode 100644 index 0000000..9ab95b6 Binary files /dev/null and b/contrast/utils/__pycache__/event.cpython-39.pyc differ diff --git a/contrast/utils/__pycache__/tools.cpython-312.pyc b/contrast/utils/__pycache__/tools.cpython-312.pyc new file mode 100644 index 0000000..1157e1b Binary files /dev/null and b/contrast/utils/__pycache__/tools.cpython-312.pyc differ diff --git a/contrast/utils/__pycache__/tools.cpython-39.pyc b/contrast/utils/__pycache__/tools.cpython-39.pyc new file mode 100644 index 0000000..fcf268f Binary files /dev/null and b/contrast/utils/__pycache__/tools.cpython-39.pyc differ diff --git a/contrast/utils/barcode_set_operate.py b/contrast/utils/barcode_set_operate.py new file mode 100644 index 0000000..293a6fc --- /dev/null +++ b/contrast/utils/barcode_set_operate.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +""" +Created on Fri Sep 13 16:49:05 2024 + + 比较 stdBcdpath 和 filepath 中的 barcodes 列表，求出二者的并集和为包含在 + stdBcdpath 中的 barcodes 清单 + +@author: ym +""" +import os +from openpyxl import load_workbook, Workbook + +def read_xlsx(): + stdBcdpath = r"\\192.168.1.28\share\已标注数据备份\对比数据\barcode\total_barcode_6588" + filepath = r"\\192.168.1.28\share\联华中环店\中环店商品信息.xlsx" + + existingPath = r'\\192.168.1.28\share\联华中环店\中环店商品信息_已有商品.xlsx' + lackingPath = r'\\192.168.1.28\share\联华中环店\中环店商品信息_未包含商品.xlsx' + + workbook = load_workbook(filename=filepath) + sheet = workbook['Sheet1'] + barcodeCol = [sheet.cell(row=r, column=1).value for r in range(1, sheet.max_row+1)] + + zhBarcodeList = [barcodeCol[i] for i in range(1, len(barcodeCol))] + + stdBarcodeList = [] + for filename in os.listdir(stdBcdpath): + filepath = os.path.join(stdBcdpath, filename) + if not os.path.isdir(filepath) or not filename.isdigit(): + continue + stdBarcodeList.append(int(filename)) + + + stdBarcodeSet = set(stdBarcodeList) + zhBarcodeSet = set(zhBarcodeList) + interBarcodes = list(zhBarcodeSet.intersection(stdBarcodeSet)) + + print(len(interBarcodes)) + + dest_wb1 = Workbook() + dest_sheet1 = dest_wb1.active + for row in sheet.iter_rows(min_row=1, max_col=sheet.max_column, values_only=True): + if str(row[0]).find("商品条码")>=0: + dest_sheet1.append(row) + + if row[0] in interBarcodes: + dest_sheet1.append(row) + + dest_wb1.save(filename=existingPath) + dest_wb1.close() + + + diffBarcodes = list(zhBarcodeSet.difference(stdBarcodeSet)) + + dest_wb2 = Workbook() + dest_sheet2 = dest_wb2.active + for row in sheet.iter_rows(min_row=1, max_col=sheet.max_column, values_only=True): + if str(row[0]).find("商品条码")>=0: + dest_sheet2.append(row) + + if row[0] in diffBarcodes: + dest_sheet2.append(row) + + dest_wb2.save(filename=lackingPath) + dest_wb2.close() + + + workbook.close() + + +if __name__ == '__main__': + # main() + + read_xlsx() + + + + + + + + + \ No newline at end of file diff --git a/contrast/utils/calsimi.py b/contrast/utils/calsimi.py new file mode 100644 index 0000000..a965707 --- /dev/null +++ b/contrast/utils/calsimi.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Mon Mar 31 16:25:43 2025 + +@author: wqg +""" +import numpy as np +from scipy.spatial.distance import cdist + + +def get_topk_percent(data, k): + """ + 获取数据中最大的 k% 的元素 + """ + # 将数据转换为 NumPy 数组 + if isinstance(data, list): + data = np.array(data) + + percentile = np.percentile(data, 100-k) + top_k_percent = data[data >= percentile] + + return top_k_percent +def cluster(data, thresh=0.15): + # data = np.array([0.1, 0.13, 0.7, 0.2, 0.8, 0.52, 0.3, 0.7, 0.85, 0.58]) + # data = np.array([0.1, 0.13, 0.2, 0.3]) + # data = np.array([0.1]) + + if isinstance(data, list): + data = np.array(data) + + data1 = np.sort(data) + cluter, Cluters, = [data1[0]], [] + for i in range(1, len(data1)): + if data1[i] - data1[i-1]< thresh: + cluter.append(data1[i]) + else: + Cluters.append(cluter) + cluter = [data1[i]] + Cluters.append(cluter) + + clt_center = [] + for clt in Cluters: + ## 是否应该在此处限制一个聚类中的最小轨迹样本数，应该将该因素放在轨迹分析中 + # if len(clt)>=3: + # clt_center.append(np.mean(clt)) + clt_center.append(np.mean(clt)) + + # print(clt_center) + + return clt_center + +def calsiml(feat1, feat2, topkp=75, cluth=0.15): + '''轨迹样本和标准特征集样本相似度的选择策略''' + matrix = 1 - cdist(feat1, feat2, 'cosine') + simi_max = [] + for i in range(len(matrix)): + sim = np.mean(get_topk_percent(matrix[i, :], topkp)) + simi_max.append(sim) + cltc_max = cluster(simi_max, cluth) + Simi = max(cltc_max) + + ## cltc_max为空属于编程考虑不周，应予以排查解决 + # if len(cltc_max): + # Simi = max(cltc_max) + # else: + # Simi = 0 #不应该走到该处 + + return Simi + + +def calsimi_vs_stdfeat_new(event, stdfeat): + '''事件与标准库的对比策略 + 该比对策略是否可以拓展到事件与事件的比对？ + ''' + front_boxes = np.empty((0, 9), dtype=np.float64) ##和类doTracks兼容 + front_feats = np.empty((0, 256), dtype=np.float64) ##和类doTracks兼容 + for i in range(len(event.front_boxes)): + front_boxes = np.concatenate((front_boxes, event.front_boxes[i]), axis=0) + front_feats = np.concatenate((front_feats, event.front_feats[i]), axis=0) + + back_boxes = np.empty((0, 9), dtype=np.float64) ##和类doTracks兼容 + back_feats = np.empty((0, 256), dtype=np.float64) ##和类doTracks兼容 + for i in range(len(event.back_boxes)): + back_boxes = np.concatenate((back_boxes, event.back_boxes[i]), axis=0) + back_feats = np.concatenate((back_feats, event.back_feats[i]), axis=0) + + front_simi, back_simi = None, None + if len(front_feats): + front_simi = calsiml(front_feats, stdfeat) + if len(back_feats): + back_simi = calsiml(back_feats, stdfeat) + + '''前后摄相似度融合策略''' + if len(front_feats) and len(back_feats): + diff_simi = abs(front_simi - back_simi) + if diff_simi>0.15: + Similar = max([front_simi, back_simi]) + else: + Similar = (front_simi+back_simi)/2 + elif len(front_feats) and len(back_feats)==0: + Similar = front_simi + elif len(front_feats)==0 and len(back_feats): + Similar = back_simi + else: + Similar = None # 在event.front_feats和event.back_feats同时为空时 + + return Similar, front_simi, back_simi + + + + + +def calsimi_vs_stdfeat(event, stdfeat): + evtfeat = event.feats_compose + if isinstance(event.feats_select, list): + if len(event.feats_select) and len(event.feats_select[0]): + evtfeat = event.feats_select[0] + else: + return None, None, None + else: + evtfeat = event.feats_select + + if len(evtfeat)==0 or len(stdfeat)==0: + return None, None, None + + + evtfeat /= np.linalg.norm(evtfeat, axis=1)[:, None] + stdfeat /= np.linalg.norm(stdfeat, axis=1)[:, None] + + matrix = 1 - cdist(evtfeat, stdfeat, 'cosine') + matrix[matrix < 0] = 0 + + simi_mean = np.mean(matrix) + simi_max = np.max(matrix) + stdfeatm = np.mean(stdfeat, axis=0, keepdims=True) + evtfeatm = np.mean(evtfeat, axis=0, keepdims=True) + simi_mfeat = 1- np.maximum(0.0, cdist(stdfeatm, evtfeatm, 'cosine')) + + return simi_mean, simi_max, simi_mfeat[0,0] + + +def calsimi_vs_evts(evta, evtb, simType=1): + if simType==1: + if len(evta.feats_compose) and len(evtb.feats_compose): + feata = evta.feats_compose + featb = evtb.feats_compose + matrix = 1 - cdist(feata, featb, 'cosine') + similar = np.mean(matrix) + else: + similar = None + return similar + + if simType==2: + if len(evta.feats_compose) and len(evtb.feats_compose): + feata = evta.feats_compose + featb = evtb.feats_compose + matrix = 1 - cdist(feata, featb, 'cosine') + similar = np.max(matrix) + else: + similar = None + return similar + + if simType==3: + if len(evta.feats_compose) and len(evtb.feats_compose): + feata = evta.feats_compose + featb = evtb.feats_compose + similar = calsiml(feata, featb) + else: + similar = None + return similar + + + ##1. the front feats of evta, evtb + fr_feata = np.empty((0, 256), dtype=np.float64) ##和类doTracks兼容 + for i in range(len(evta.front_feats)): + fr_feata = np.concatenate((fr_feata, evta.front_feats[i]), axis=0) + + fr_featb = np.empty((0, 256), dtype=np.float64) ##和类doTracks兼容 + for i in range(len(evtb.front_feats)): + fr_featb = np.concatenate((fr_featb, evtb.front_feats[i]), axis=0) + + ##2. the back feats of evta, evtb + bk_feata = np.empty((0, 256), dtype=np.float64) ##和类doTracks兼容 + for i in range(len(evta.back_feats)): + bk_feata = np.concatenate((bk_feata, evta.back_feats[i]), axis=0) + + bk_featb = np.empty((0, 256), dtype=np.float64) ##和类doTracks兼容 + for i in range(len(evtb.back_feats)): + bk_featb = np.concatenate((bk_featb, evtb.back_feats[i]), axis=0) + + + front_simi, back_simi = None, None + if len(fr_feata) and len(fr_featb): + front_simi = calsiml(fr_feata, fr_featb) + + if len(bk_feata) and len(bk_featb): + back_simi = calsiml(bk_feata, bk_featb) + + '''前后摄相似度融合策略''' + if front_simi is not None and back_simi is not None: + diff_simi = abs(front_simi - back_simi) + if diff_simi>0.15: + similar = max([front_simi, back_simi]) + else: + similar = (front_simi+back_simi)/2 + elif front_simi is not None and back_simi is None: + similar = front_simi + elif front_simi is None and back_simi is not None: + similar = back_simi + else: + similar = None # 在event.front_feats和event.back_feats同时为空时 + + return similar + + diff --git a/contrast/utils/databits.py b/contrast/utils/databits.py new file mode 100644 index 0000000..9d59e2b --- /dev/null +++ b/contrast/utils/databits.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Tue Apr 1 16:43:04 2025 + +@author: wqg +""" +import os +import pickle +import numpy as np +from scipy.spatial.distance import cdist + + +def int8_to_ft16(arr_uint8, amin, amax): + arr_ft16 = (arr_uint8 / 255 * (amax-amin) + amin).astype(np.float16) + + return arr_ft16 + +def ft16_to_uint8(arr_ft16): + # pickpath = r"\\192.168.1.28\share\测试_202406\contrast\std_features_ft32vsft16\6902265587712_ft16.pickle" + + # with open(pickpath, 'rb') as f: + # edict = pickle.load(f) + + # arr_ft16 = edict['feats'] + + amin = np.min(arr_ft16) + amax = np.max(arr_ft16) + arr_ft255 = (arr_ft16 - amin) * 255 / (amax-amin) + arr_uint8 = arr_ft255.astype(np.uint8) + + arr_ft16_ = int8_to_ft16(arr_uint8, amin, amax) + + arrDistNorm = np.linalg.norm(arr_ft16_ - arr_ft16) / arr_ft16_.size + + return arr_uint8, arr_ft16_ + + +def data_precision_compare(stdfeat, evtfeat, evtMessage, similPath='', save=True): + evt, stdbcd, label = evtMessage + rltdata, rltdata_ft16, rltdata_ft16_ = [], [], [] + + matrix = 1 - cdist(stdfeat, evtfeat, 'cosine') + simi_mean = np.mean(matrix) + simi_max = np.max(matrix) + stdfeatm = np.mean(stdfeat, axis=0, keepdims=True) + evtfeatm = np.mean(evtfeat, axis=0, keepdims=True) + simi_mfeat = 1- np.maximum(0.0, cdist(stdfeatm, evtfeatm, 'cosine')) + rltdata = [label, stdbcd, evt, simi_mean, simi_max, simi_mfeat[0,0]] + + + ##================================================================= float16 + stdfeat_ft16 = stdfeat.astype(np.float16) + evtfeat_ft16 = evtfeat.astype(np.float16) + stdfeat_ft16 /= np.linalg.norm(stdfeat_ft16, axis=1)[:, None] + evtfeat_ft16 /= np.linalg.norm(evtfeat_ft16, axis=1)[:, None] + + + matrix_ft16 = 1 - cdist(stdfeat_ft16, evtfeat_ft16, 'cosine') + simi_mean_ft16 = np.mean(matrix_ft16) + simi_max_ft16 = np.max(matrix_ft16) + stdfeatm_ft16 = np.mean(stdfeat_ft16, axis=0, keepdims=True) + evtfeatm_ft16 = np.mean(evtfeat_ft16, axis=0, keepdims=True) + simi_mfeat_ft16 = 1- np.maximum(0.0, cdist(stdfeatm_ft16, evtfeatm_ft16, 'cosine')) + rltdata_ft16 = [label, stdbcd, evt, simi_mean_ft16, simi_max_ft16, simi_mfeat_ft16[0,0]] + + '''****************** uint8 is ok!!!!!! ******************''' + ##=================================================================== uint8 + # stdfeat_uint8, stdfeat_ft16_ = ft16_to_uint8(stdfeat_ft16) + # evtfeat_uint8, evtfeat_ft16_ = ft16_to_uint8(evtfeat_ft16) + + stdfeat_uint8 = (stdfeat_ft16*128).astype(np.int8) + evtfeat_uint8 = (evtfeat_ft16*128).astype(np.int8) + stdfeat_ft16_ = stdfeat_uint8.astype(np.float16)/128 + evtfeat_ft16_ = evtfeat_uint8.astype(np.float16)/128 + + absdiff = np.linalg.norm(stdfeat_ft16_ - stdfeat) / stdfeat.size + + matrix_ft16_ = 1 - cdist(stdfeat_ft16_, evtfeat_ft16_, 'cosine') + simi_mean_ft16_ = np.mean(matrix_ft16_) + simi_max_ft16_ = np.max(matrix_ft16_) + stdfeatm_ft16_ = np.mean(stdfeat_ft16_, axis=0, keepdims=True) + evtfeatm_ft16_ = np.mean(evtfeat_ft16_, axis=0, keepdims=True) + simi_mfeat_ft16_ = 1- np.maximum(0.0, cdist(stdfeatm_ft16_, evtfeatm_ft16_, 'cosine')) + rltdata_ft16_ = [label, stdbcd, evt, simi_mean_ft16_, simi_max_ft16_, simi_mfeat_ft16_[0,0]] + + if not save: + return + + + ##========================================================= save as float32 + rppath = os.path.join(similPath, f'{evt}_ft32.pickle') + with open(rppath, 'wb') as f: + pickle.dump(rltdata, f) + + rtpath = os.path.join(similPath, f'{evt}_ft32.txt') + with open(rtpath, 'w', encoding='utf-8') as f: + for result in rltdata: + part = [f"{x:.3f}" if isinstance(x, float) else str(x) for x in result] + line = ', '.join(part) + f.write(line + '\n') + + + ##========================================================= save as float16 + rppath_ft16 = os.path.join(similPath, f'{evt}_ft16.pickle') + with open(rppath_ft16, 'wb') as f: + pickle.dump(rltdata_ft16, f) + + rtpath_ft16 = os.path.join(similPath, f'{evt}_ft16.txt') + with open(rtpath_ft16, 'w', encoding='utf-8') as f: + for result in rltdata_ft16: + part = [f"{x:.3f}" if isinstance(x, float) else str(x) for x in result] + line = ', '.join(part) + f.write(line + '\n') + + + ##=========================================================== save as uint8 + rppath_uint8 = os.path.join(similPath, f'{evt}_uint8.pickle') + with open(rppath_uint8, 'wb') as f: + pickle.dump(rltdata_ft16_, f) + + rtpath_uint8 = os.path.join(similPath, f'{evt}_uint8.txt') + with open(rtpath_uint8, 'w', encoding='utf-8') as f: + for result in rltdata_ft16_: + part = [f"{x:.3f}" if isinstance(x, float) else str(x) for x in result] + line = ', '.join(part) + f.write(line + '\n') \ No newline at end of file diff --git a/contrast/utils/dotest.py b/contrast/utils/dotest.py new file mode 100644 index 0000000..569b861 --- /dev/null +++ b/contrast/utils/dotest.py @@ -0,0 +1,155 @@ +# -*- coding: utf-8 -*- +""" +Created on Tue Dec 10 14:30:16 2024 + +@author: ym +""" +import os +import sys +import numpy as np +sys.path.append(r"D:\DetectTracking") +from tracking.utils.read_data import read_tracking_output, read_similar #, extract_data, read_deletedBarcode_file + + +IMG_FORMAT = ['.bmp', '.jpg', '.jpeg', '.png'] + + +def creat_shopping_event(eventPath): + '''构造放入商品事件字典，这些事件需满足条件： + 1) 前后摄至少有一条轨迹输出 + 2) 保存有帧图像，以便裁剪出 boxe 子图 + ''' + + '''evtName 为一次购物事件''' + evtName = os.path.basename(eventPath) + evtList = evtName.split('_') + + '''================ 0. 检查 evtName 及 eventPath 正确性和有效性 ================''' + if evtName.find('2024')<0 and len(evtList[0])!=15: + return + if not os.path.isdir(eventPath): + return + + if len(evtList)==1 or (len(evtList)==2 and len(evtList[1])==0): + barcode = '' + else: + barcode = evtList[-1] + + if len(evtList)==3 and evtList[-1]== evtList[-2]: + evtType = 'input' + else: + evtType = 'other' + + '''================ 1. 构造事件描述字典，暂定 9 items ===============''' + + + + + event = {} + event['barcode'] = barcode + event['type'] = evtType + event['filepath'] = eventPath + event['back_imgpaths'] = [] + event['front_imgpaths'] = [] + event['back_boxes'] = np.empty((0, 9), dtype=np.float64) + event['front_boxes'] = np.empty((0, 9), dtype=np.float64) + event['back_feats'] = np.empty((0, 256), dtype=np.float64) + event['front_feats'] = np.empty((0, 256), dtype=np.float64) + event['feats_compose'] = np.empty((0, 256), dtype=np.float64) + event['one2one'] = None + event['one2n'] = None + event['feats_select'] = np.empty((0, 256), dtype=np.float64) + + + '''================= 2. 读取 data 文件 =============================''' + for dataname in os.listdir(eventPath): + # filename = '1_track.data' + datapath = os.path.join(eventPath, dataname) + if not os.path.isfile(datapath): continue + + CamerType = dataname.split('_')[0] + ''' 2.1 读取 0/1_track.data 中数据，暂不考虑''' + # if dataname.find("_track.data")>0: + # bboxes, ffeats, trackerboxes, tracker_feat_dict, trackingboxes, tracking_feat_dict = extract_data(datapath) + + ''' 2.2 读取 0/1_tracking_output.data 中数据''' + if dataname.find("_tracking_output.data")>0: + tracking_output_boxes, tracking_output_feats = read_tracking_output(datapath) + if len(tracking_output_boxes) != len(tracking_output_feats): continue + if CamerType == '0': + event['back_boxes'] = tracking_output_boxes + event['back_feats'] = tracking_output_feats + elif CamerType == '1': + event['front_boxes'] = tracking_output_boxes + event['front_feats'] = tracking_output_feats + + if dataname.find("process.data")==0: + simiDict = read_similar(datapath) + event['one2one'] = simiDict['one2one'] + event['one2n'] = simiDict['one2n'] + + + if len(event['back_boxes'])==0 or len(event['front_boxes'])==0: + return None + + '''2.3 事件的特征表征方式: 特征选择、特征集成''' + bk_feats = event['back_feats'] + ft_feats = event['front_feats'] + + '''2.3.1 特征集成''' + feats_compose = np.empty((0, 256), dtype=np.float64) + if len(ft_feats): + feats_compose = np.concatenate((feats_compose, ft_feats), axis=0) + if len(bk_feats): + feats_compose = np.concatenate((feats_compose, bk_feats), axis=0) + event['feats_compose'] = feats_compose + + '''2.3.1 特征选择''' + if len(ft_feats): + event['feats_select'] = ft_feats + + + '''================ 3. 读取图像文件地址，并按照帧ID排序 =============''' + frontImgs, frontFid = [], [] + backImgs, backFid = [], [] + for imgname in os.listdir(eventPath): + name, ext = os.path.splitext(imgname) + if ext not in IMG_FORMAT or name.find('frameId')<0: continue + + CamerType = name.split('_')[0] + frameId = int(name.split('_')[3]) + imgpath = os.path.join(eventPath, imgname) + if CamerType == '0': + backImgs.append(imgpath) + backFid.append(frameId) + if CamerType == '1': + frontImgs.append(imgpath) + frontFid.append(frameId) + + frontIdx = np.argsort(np.array(frontFid)) + backIdx = np.argsort(np.array(backFid)) + + '''3.1 生成依据帧 ID 排序的前后摄图像地址列表''' + frontImgs = [frontImgs[i] for i in frontIdx] + backImgs = [backImgs[i] for i in backIdx] + + '''3.2 将前、后摄图像路径添加至事件字典''' + + + bfid = event['back_boxes'][:, 7].astype(np.int64) + ffid = event['front_boxes'][:, 7].astype(np.int64) + if len(bfid) and max(bfid) <= len(backImgs): + event['back_imgpaths'] = [backImgs[i-1] for i in bfid] + if len(ffid) and max(ffid) <= len(frontImgs): + event['front_imgpaths'] = [frontImgs[i-1] for i in ffid] + + + '''================ 4. 判断当前事件有效性，并添加至事件列表 ==========''' + condt1 = len(event['back_imgpaths'])==0 or len(event['front_imgpaths'])==0 + condt2 = len(event['front_feats'])==0 and len(event['back_feats'])==0 + + if condt1 or condt2: + print(f"Event: {evtName}, Error, condt1: {condt1}, condt2: {condt2}") + return None + + return event \ No newline at end of file diff --git a/contrast/utils/event.py b/contrast/utils/event.py new file mode 100644 index 0000000..02df620 --- /dev/null +++ b/contrast/utils/event.py @@ -0,0 +1,551 @@ +# -*- coding: utf-8 -*- +""" +Created on Tue Nov 26 17:35:05 2024 + +@author: ym +""" +import os +import sys +import cv2 +import pickle +import numpy as np +from pathlib import Path + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[2] # YOLOv5 root directory +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) + +from tracking.utils.plotting import Annotator, colors +from tracking.utils.drawtracks import drawTrack +from tracking.utils.read_data import extract_data, read_tracking_output, read_similar +from tracking.utils.read_data import extract_data_realtime, read_tracking_output_realtime + + + + +# import platform +# import pathlib +# plt = platform.system() + + +IMG_FORMAT = ['.bmp', '.jpg', '.jpeg', '.png'] +VID_FORMAT = ['.mp4', '.avi'] + +def save_data(event, resultPath=None): + '''事件轨迹子图保存''' + if resultPath is None: + resultPath = os.path.dirname(os.path.abspath(__file__)) + + + subimgpath = os.path.join(resultPath, f"{event.evtname}", "subimg") + imgspath = os.path.join(resultPath, f"{event.evtname}", "imgs") + if not os.path.exists(subimgpath): + os.makedirs(subimgpath) + if not os.path.exists(imgspath): + os.makedirs(imgspath) + ##(2) 保存轨迹中的子图 + subimgpairs = event.save_event_subimg(subimgpath) + for subimgName, subimg in subimgpairs: + spath = os.path.join(subimgpath, subimgName) + cv2.imwrite(spath, subimg) + + ##(3) 保存序列图像 + imgpairs = event.plot_save_image(imgspath) + for imgname, img in imgpairs: + spath = os.path.join(imgspath, imgname) + cv2.imwrite(spath, img) + ##(4) 保存轨迹散点图 + img_cat = event.draw_tracks() + trajpath = os.path.join(resultPath, "trajectory") + if not os.path.exists(trajpath): + os.makedirs(trajpath) + traj_imgpath = os.path.join(trajpath, event.evtname+".png") + cv2.imwrite(traj_imgpath, img_cat) + + +def array2list(bboxes): + ''' + 将 bboxes 变换为 track 列表 + bboxes: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] + Return： + lboxes：列表，列表中元素具有同一 track_id，x1y1x2y2 格式 + [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] + ''' + lboxes = [] + if len(bboxes)==0: + return [] + + trackID = np.unique(bboxes[:, 4].astype(int)) + track_ids = bboxes[:, 4].astype(int) + for t_id in trackID: + idx = np.where(track_ids == t_id)[0] + box = bboxes[idx, :] + lboxes.append(box) + + return lboxes + + +class ShoppingEvent: + def __init__(self, eventpath, stype="data"): + '''stype: str, 'source', 'data', 'realtime', 共三种 + source: 前后摄视频经 pipeline 生成的文件 + data: 基于事件切分的原 data 文件版本 + realtime: 全实时生成的 data 文件 + ''' + + self.eventpath = eventpath + self.evtname = str(Path(eventpath).stem) + self.barcode = '' + self.evtType = '' + + '''=========== path of image and video =========== ''' + self.back_videopath = '' + self.front_videopath = '' + self.back_imgpaths = [] + self.front_imgpaths = [] + + '''=========== process.data ===============================''' + self.one2one = None + self.one2n = None + self.one2SN = None + + '''=========== 0/1_track.data =============================''' + self.back_yolobboxes = [] + self.back_yolofeats = [] + self.back_trackerboxes = np.empty((0, 9), dtype=np.float64) ##和类doTracks兼容 + self.back_trackerfeats = {} + self.back_trackingboxes = [] + self.back_trackingfeats = [] + + self.front_yolobboxes = [] + self.front_yolofeats = [] + self.front_trackerboxes = np.empty((0, 9), dtype=np.float64) ##和类doTracks兼容 + self.front_trackerfeats = {} + self.front_trackingboxes = [] + self.front_trackingfeats = [] + + '''=========== 0/1_tracking_output.data ===================''' + self.back_boxes = [] + self.back_feats = [] + self.front_boxes = [] + self.front_feats = [] + + + if stype=="data": + self.from_datafile(eventpath) + if stype=="realtime": + self.from_realtime_datafile(eventpath) + if stype=="source": + self.from_source_pkl(eventpath) + + self.feats_select = np.empty((0, 256), dtype=np.float64) + self.feats_compose = np.empty((0, 256), dtype=np.float64) + self.select_feats() + self.compose_feats() + + # if stype=="image": + # self.from_image(eventpath) + + def kerndata(self, ShoppingDict, camtype="backCamera"): + ''' + camtype: str, "backCamera" or "frontCamera" + ''' + yoloboxes, resfeats = [], [] + trackerboxes = np.empty((0, 9), dtype=np.float64) + trackefeats = {} + trackingboxes, trackingfeats = [], [] + + frameDictList = ShoppingDict[camtype]["yoloResnetTracker"] + for frameDict in frameDictList: + yoloboxes.append(frameDict["bboxes"]) + + tboxes = frameDict["tboxes"] + trackefeats.update(frameDict["feats"]) + + trackerboxes = np.concatenate((trackerboxes, np.array(tboxes)), axis=0) + + Residual = ShoppingDict[camtype]["tracking"].Residual + for track in Residual: + trackingboxes.append(track.boxes) + trackingfeats.append(track.features) + kdata = (yoloboxes, resfeats, trackerboxes, trackefeats, trackingboxes, trackingfeats) + + + tracking_out_boxes, tracking_out_feats = [], [] + Confirmed = ShoppingDict[camtype]["tracking"].Confirmed + for track in Confirmed: + tracking_out_boxes.append(track.boxes) + tracking_out_feats.append(track.features) + outdata = (tracking_out_boxes, tracking_out_feats) + + return kdata, outdata + + + def from_source_pkl(self, eventpath): + # if plt == 'Windows': + # pathlib.PosixPath = pathlib.WindowsPath + with open(eventpath, 'rb') as f: + ShoppingDict = pickle.load(f) + + self.eventpath = ShoppingDict["eventPath"] + self.evtname = ShoppingDict["eventName"] + self.barcode = ShoppingDict["barcode"] + + if len(ShoppingDict["one2n"]): + self.one2n = ShoppingDict["one2n"] + + '''=========== path of image and video =========== ''' + self.back_videopath = ShoppingDict["backCamera"]["videoPath"] + self.front_videopath = ShoppingDict["frontCamera"]["videoPath"] + self.back_imgpaths = ShoppingDict["backCamera"]["imagePaths"] + self.front_imgpaths = ShoppingDict["frontCamera"]["imagePaths"] + + + '''===========对应于 0/1_track.data =============================''' + backdata, back_outdata = self.kerndata(ShoppingDict, "backCamera") + frontdata, front_outdata = self.kerndata(ShoppingDict, "frontCamera") + self.back_yolobboxes = backdata[0] + self.back_yolofeats = backdata[1] + self.back_trackerboxes = backdata[2] + self.back_trackerfeats = [3] + self.back_trackingboxes = [4] + self.back_trackingfeats = [5] + + self.front_yolobboxes = frontdata[0] + self.front_yolofeats = frontdata[1] + self.front_trackerboxes = frontdata[2] + self.front_trackerfeats = frontdata[3] + self.front_trackingboxes = frontdata[4] + self.front_trackingfeats = frontdata[5] + + '''===========对应于 0/1_tracking_output.data =============================''' + self.back_boxes = back_outdata[0] + self.back_feats = back_outdata[1] + self.front_boxes = front_outdata[0] + self.front_feats = front_outdata[1] + + + def from_datafile(self, eventpath): + evtList = self.evtname.split('_') + if len(evtList)>=2 and len(evtList[-1])>=10 and evtList[-1].isdigit(): + self.barcode = evtList[-1] + if len(evtList)==3 and evtList[-1]== evtList[-2]: + self.evtType = 'input' + else: + self.evtType = 'other' + + '''================ path of image =============''' + frontImgs, frontFid = [], [] + backImgs, backFid = [], [] + for imgname in os.listdir(eventpath): + name, ext = os.path.splitext(imgname) + if ext not in IMG_FORMAT or name.find('frameId') < 0: continue + if len(name.split('_')) != 3 and not name.split('_')[3].isdigit(): continue + + CamerType = name.split('_')[0] + frameId = int(name.split('_')[3]) + imgpath = os.path.join(eventpath, imgname) + if CamerType == '0': + backImgs.append(imgpath) + backFid.append(frameId) + if CamerType == '1': + frontImgs.append(imgpath) + frontFid.append(frameId) + ## 生成依据帧 ID 排序的前后摄图像地址列表 + frontIdx = np.argsort(np.array(frontFid)) + backIdx = np.argsort(np.array(backFid)) + self.front_imgpaths = [frontImgs[i] for i in frontIdx] + self.back_imgpaths = [backImgs[i] for i in backIdx] + + + '''================ path of video =============''' + for vidname in os.listdir(eventpath): + name, ext = os.path.splitext(vidname) + if ext not in VID_FORMAT: continue + vidpath = os.path.join(eventpath, vidname) + + CamerType = name.split('_')[0] + if CamerType == '0': + self.back_videopath = vidpath + if CamerType == '1': + self.front_videopath = vidpath + + '''================ process.data =============''' + procpath = Path(eventpath).joinpath('process.data') + if procpath.is_file(): + SimiDict = read_similar(procpath) + self.one2one = SimiDict['one2one'] + self.one2n = SimiDict['one2n'] + self.one2SN = SimiDict['one2SN'] + + '''=========== 0/1_track.data & 0/1_tracking_output.data =======''' + for dataname in os.listdir(eventpath): + datapath = os.path.join(eventpath, dataname) + if not os.path.isfile(datapath): continue + CamerType = dataname.split('_')[0] + + '''========== 0/1_track.data ==========''' + if dataname.find("_track.data")>0: + bboxes, ffeats, trackerboxes, trackerfeats, trackingboxes, trackingfeats = extract_data(datapath) + if CamerType == '0': + self.back_yolobboxes = bboxes + self.back_yolofeats = ffeats + self.back_trackerboxes = trackerboxes + self.back_trackerfeats = trackerfeats + self.back_trackingboxes = trackingboxes + self.back_trackingfeats = trackingfeats + if CamerType == '1': + self.front_yolobboxes = bboxes + self.front_yolofeats = ffeats + self.front_trackerboxes = trackerboxes + self.front_trackerfeats = trackerfeats + self.front_trackingboxes = trackingboxes + self.front_trackingfeats = trackingfeats + + '''========== 0/1_tracking_output.data ==========''' + if dataname.find("_tracking_output.data")>0: + tracking_output_boxes, tracking_output_feats = read_tracking_output(datapath) + if CamerType == '0': + self.back_boxes = tracking_output_boxes + self.back_feats = tracking_output_feats + elif CamerType == '1': + self.front_boxes = tracking_output_boxes + self.front_feats = tracking_output_feats + + def from_realtime_datafile(self, eventpath): + evtList = self.evtname.split('_') + if len(evtList)>=2 and len(evtList[-1])>=10 and evtList[-1].isdigit(): + self.barcode = evtList[-1] + if len(evtList)==3 and evtList[-1]== evtList[-2]: + self.evtType = 'input' + else: + self.evtType = 'other' + + '''================ path of video =============''' + for vidname in os.listdir(eventpath): + name, ext = os.path.splitext(vidname) + if ext not in VID_FORMAT: continue + vidpath = os.path.join(eventpath, vidname) + + CamerType = name.split('_')[0] + if CamerType == '0': + self.back_videopath = vidpath + if CamerType == '1': + self.front_videopath = vidpath + + '''================ process.data =============''' + procpath = Path(eventpath).joinpath('process.data') + if procpath.is_file(): + SimiDict = read_similar(procpath) + self.one2one = SimiDict['one2one'] + self.one2n = SimiDict['one2n'] + self.one2SN = SimiDict['one2SN'] + + '''=========== 0/1_track.data & 0/1_tracking_output.data =======''' + for dataname in os.listdir(eventpath): + datapath = os.path.join(eventpath, dataname) + if not os.path.isfile(datapath): continue + CamerType = dataname.split('_')[0] + '''========== 0/1_track.data ==========''' + if dataname.find("_tracker.data")>0: + trackerboxes, trackerfeats = extract_data_realtime(datapath) + if CamerType == '0': + self.back_trackerboxes = trackerboxes + self.back_trackerfeats = trackerfeats + + if CamerType == '1': + self.front_trackerboxes = trackerboxes + self.front_trackerfeats = trackerfeats + '''========== 0/1_tracking_output.data ==========''' + if dataname.find("_tracking_output.data")>0: + trackingboxes, trackingfeats, tracking_outboxes, tracking_outfeats = read_tracking_output_realtime(datapath) + if CamerType == '0': + self.back_trackingboxes = trackingboxes + self.back_trackingfeats = trackingfeats + self.back_boxes = tracking_outboxes + self.back_feats = tracking_outfeats + elif CamerType == '1': + self.front_trackingboxes = trackingboxes + self.front_trackingfeats = trackingfeats + self.front_boxes = tracking_outboxes + self.front_feats = tracking_outfeats + + + + + + def compose_feats(self): + '''事件的特征集成''' + feats_compose = np.empty((0, 256), dtype=np.float64) + if len(self.front_feats): + for feat in self.front_feats: + feats_compose = np.concatenate((feats_compose, feat), axis=0) + if len(self.back_feats): + for feat in self.back_feats: + feats_compose = np.concatenate((feats_compose, feat), axis=0) + self.feats_compose = feats_compose + + def select_feats(self): + '''事件的特征选择''' + if len(self.front_feats): + self.feats_select = self.front_feats[0] + elif len(self.back_feats): + self.feats_select = self.back_feats[0] + + def plot_save_image(self, savepath): + + def array2list(bboxes): + '''[x1, y1, x2, y2, track_id, score, cls, frame_index, box_index]''' + frame_ids = bboxes[:, 7].astype(int) + fID = np.unique(bboxes[:, 7].astype(int)) + fboxes = [] + for f_id in fID: + idx = np.where(frame_ids==f_id)[0] + box = bboxes[idx, :] + fboxes.append((f_id, box)) + return fboxes + + imgpairs = [] + cameras = ('front', 'back') + for camera in cameras: + if camera == 'front': + boxes = self.front_trackerboxes + imgpaths = self.front_imgpaths + else: + boxes = self.back_trackerboxes + imgpaths = self.back_imgpaths + + fboxes = array2list(boxes) + for fid, fbox in fboxes: + imgpath = imgpaths[int(fid-1)] + + image = cv2.imread(imgpath) + + annotator = Annotator(image.copy(), line_width=2) + for i, box in enumerate(fbox): + x1, y1, x2, y2, tid, score, cls, fid, bid = box + label = f'{int(tid), int(cls)}' + if tid >=0 and cls==0: + color = colors(int(cls), True) + elif tid >=0 and cls!=0: + color = colors(int(tid), True) + else: + color = colors(19, True) # 19为调色板的最后一个元素 + xyxy = (x1/2, y1/2, x2/2, y2/2) + annotator.box_label(xyxy, label, color=color) + + im0 = annotator.result() + + imgpairs.append((Path(imgpath).name, im0)) + + # spath = os.path.join(savepath, Path(imgpath).name) + + + # cv2.imwrite(spath, im0) + return imgpairs + + + def save_event_subimg(self, savepath): + ''' + 功能: 保存一次购物事件的轨迹子图 + 9 items: barcode, type, filepath, back_imgpaths, front_imgpaths, + back_boxes, front_boxes, back_feats, front_feats, + feats_compose, feats_select + 子图保存次序：先前摄、后后摄，以 k 为编号，和 "feats_compose" 中次序相同 + ''' + imgpairs = [] + cameras = ('front', 'back') + for camera in cameras: + boxes = np.empty((0, 9), dtype=np.float64) ##和类doTracks兼容 + if camera == 'front': + for b in self.front_boxes: + boxes = np.concatenate((boxes, b), axis=0) + imgpaths = self.front_imgpaths + else: + for b in self.back_boxes: + boxes = np.concatenate((boxes, b), axis=0) + imgpaths = self.back_imgpaths + + for i, box in enumerate(boxes): + x1, y1, x2, y2, tid, score, cls, fid, bid = box + + imgpath = imgpaths[int(fid-1)] + image = cv2.imread(imgpath) + + subimg = image[int(y1/2):int(y2/2), int(x1/2):int(x2/2), :] + + camerType, timeTamp, _, frameID = os.path.basename(imgpath).split('.')[0].split('_') + subimgName = f"cam{camerType}_{i}_tid{int(tid)}_fid({int(fid)}, {frameID}).png" + + imgpairs.append((subimgName, subimg)) + + # spath = os.path.join(savepath, subimgName) + + # cv2.imwrite(spath, subimg) + return imgpairs + # basename = os.path.basename(event['filepath']) + print(f"Image saved: {os.path.basename(self.eventpath)}") + + def draw_tracks(self): + front_edge = cv2.imread(r"D:\DetectTracking\tracking\shopcart\cart_tempt\board_ftmp_line.png") + back_edge = cv2.imread(r"D:\DetectTracking\tracking\shopcart\cart_tempt\edgeline.png") + + front_trackerboxes = array2list(self.front_trackerboxes) + back_trackerboxes = array2list(self.back_trackerboxes) + + # img1, img2 = edgeline.copy(), edgeline.copy() + img1 = drawTrack(front_trackerboxes, front_edge.copy()) + img2 = drawTrack(self.front_trackingboxes, front_edge.copy()) + + img3 = drawTrack(back_trackerboxes, back_edge.copy()) + img4 = drawTrack(self.back_trackingboxes, back_edge.copy()) + + + + imgcat1 = np.concatenate((img1, img2), axis = 1) + H, W = imgcat1.shape[:2] + cv2.line(imgcat1, (int(W/2), 0), (int(W/2), H), (128, 255, 128), 2) + + imgcat2 = np.concatenate((img3, img4), axis = 1) + H, W = imgcat2.shape[:2] + cv2.line(imgcat2, (int(W/2), 0), (int(W/2), H), (128, 255, 128), 2) + + + illus = [imgcat1, imgcat2] + if len(illus): + img_cat = np.concatenate(illus, axis = 1) + if len(illus)==2: + H, W = img_cat.shape[:2] + cv2.line(img_cat, (int(W/2), 0), (int(W/2), int(H)), (128, 128, 255), 3) + + return img_cat + + + + + +def main(): + # pklpath = r"D:\DetectTracking\evtresult\images2\ShoppingDict.pkl" + # evt = ShoppingEvent(pklpath, stype='source') + + evtpath = r"\\192.168.1.28\share\测试视频数据以及日志\算法全流程测试\202412\images\20241209-160248-08edd5f6-1806-45ad-babf-7a4dd11cea60_6973226721445" + evt = ShoppingEvent(evtpath, stype='data') + + img_cat = evt.draw_tracks() + + cv2.imwrite("a.png", img_cat) + + + + + +if __name__ == "__main__": + main() + # main1() + + + + + + \ No newline at end of file diff --git a/contrast/utils/tools.py b/contrast/utils/tools.py new file mode 100644 index 0000000..f3214e7 --- /dev/null +++ b/contrast/utils/tools.py @@ -0,0 +1,129 @@ +# -*- coding: utf-8 -*- +""" +Created on Thu Oct 31 15:17:01 2024 + +@author: ym +""" +import os +import numpy as np +import pickle +from pathlib import Path +import matplotlib.pyplot as plt +from .event import ShoppingEvent + +def init_eventDict(sourcePath, eventDataPath, stype="data"): + ''' + stype: str, + 'source': 由 videos 或 images 生成的 pickle 文件 + 'data': 从 data 文件中读取的现场运行数据 + "realtime": 全实时数据，从 data 文件中读取的现场运行数据 + + sourcePath:事件文件夹，事件类型包含2种： + (1) pipeline生成的 pickle 文件 + (2) 直接采集的事件文件夹 + ''' + k, errEvents = 0, [] + for evtname in os.listdir(sourcePath): + bname, ext = os.path.splitext(evtname) + source_path = os.path.join(sourcePath, evtname) + + if stype=="source" and ext not in ['.pkl', '.pickle']: continue + if stype=="data" and os.path.isfile(source_path): continue + if stype=="realtime" and os.path.isfile(source_path): continue + + evt = bname.split('_') + condt = len(evt)>=2 and evt[-1].isdigit() and len(evt[-1])>=10 + if not condt: continue + + pickpath = os.path.join(eventDataPath, f"{bname}.pickle") + if os.path.isfile(pickpath): continue + + # event = ShoppingEvent(source_path, stype) + try: + event = ShoppingEvent(source_path, stype) + with open(pickpath, 'wb') as f: + pickle.dump(event, f) + print(evtname) + except Exception as e: + errEvents.append(source_path) + print(f"Error: {evtname}, {e}") + # k += 1 + # if k==1: + # break + + errfile = Path(eventDataPath).parent / 'error_events.txt' + with open(str(errfile), 'a', encoding='utf-8') as f: + for line in errEvents: + f.write(line + '\n') + + +def get_evtList(evtpath): + '''==== 0. 生成事件列表和对应的 Barcodes 集合 ===========''' + bcdList, evtpaths = [], [] + for evtname in os.listdir(evtpath): + bname, ext = os.path.splitext(evtname) + + ## 处理事件的两种情况：文件夹 和 Yolo-Resnet-Tracker 的输出 + fpath = os.path.join(evtpath, evtname) + if os.path.isfile(fpath) and (ext==".pkl" or ext==".pickle"): + evt = bname.split('_') + elif os.path.isdir(fpath): + evt = evtname.split('_') + else: + continue + + if len(evt)>=2 and evt[-1].isdigit() and len(evt[-1])>=10: + bcdList.append(evt[-1]) + evtpaths.append(fpath) + + bcdSet = set(bcdList) + + return evtpaths, bcdSet + + + +def showHist(err, correct): + err = np.array(err) + correct = np.array(correct) + + fig, axs = plt.subplots(2, 1) + axs[0].hist(err, bins=50, edgecolor='black') + axs[0].set_xlim([0, 1]) + axs[0].set_title('err') + + axs[1].hist(correct, bins=50, edgecolor='black') + axs[1].set_xlim([0, 1]) + axs[1].set_title('correct') + # plt.show() + + return plt + +def show_recall_prec(recall, prec, ths): + # x = np.linspace(start=-0, stop=1, num=11, endpoint=True).tolist() + fig = plt.figure(figsize=(10, 6)) + plt.plot(ths, recall, color='red', label='recall') + plt.plot(ths, prec, color='blue', label='PrecisePos') + plt.legend() + plt.xlabel(f'threshold') + # plt.ylabel('Similarity') + plt.grid(True, linestyle='--', alpha=0.5) + # plt.savefig('accuracy_recall_grid.png') + # plt.show() + # plt.close() + + return plt + + +def compute_recall_precision(err_similarity, correct_similarity): + ths = np.linspace(0, 1, 51) + recall, prec = [], [] + for th in ths: + TP = len([num for num in correct_similarity if num >= th]) + FP = len([num for num in err_similarity if num >= th]) + if (TP+FP) == 0: + prec.append(1) + recall.append(0) + else: + prec.append(TP / (TP + FP)) + recall.append(TP / (len(err_similarity) + len(correct_similarity))) + return recall, prec, ths \ No newline at end of file diff --git a/contrast/utils/write_feature_json.py b/contrast/utils/write_feature_json.py new file mode 100644 index 0000000..b007caa --- /dev/null +++ b/contrast/utils/write_feature_json.py @@ -0,0 +1,182 @@ +# -*- coding: utf-8 -*- +""" + +@author: LiChen +""" +import json +import os +import pickle +import numpy as np + +import sys +sys.path.append(r"D:\DetectTracking\contrast") + +from config import config as conf +# from img_data import library_imgs, temp_imgs, main_library_imgs, main_imgs_2 +# from test_logic import initModel,getFeatureList +from model import resnet18 +import torch +from PIL import Image + +device = conf.device + +def initModel(): + model = resnet18().to(device) + model.load_state_dict(torch.load(conf.test_model, map_location=conf.device)) + model.eval() + return model + + +from PIL import Image + + +def convert_rgba_to_rgb(image_path, output_path=None): + """ + 将给定路径的4通道PNG图像转换为3通道，并保存到指定输出路径。 + + :param image_path: 输入图像的路径 + :param output_path: 转换后的图像保存路径 + """ + # 打开图像 + img = Image.open(image_path) + # 转换图像模式从RGBA到RGB + # .convert('RGB')会丢弃Alpha通道并转换为纯RGB图像 + if img.mode == 'RGBA': + # 转换为RGB模式 + img_rgb = img.convert('RGB') + # 保存转换后的图像 + img_rgb.save(image_path) + print(f"Image converted from RGBA to RGB and saved to {image_path}") + # else: + # # 如果已经是RGB或其他模式，直接保存 + # img.save(image_path) + # print(f"Image already in {img.mode} mode, saved to {image_path}") + + +def test_preprocess(images: list, actionModel=False) -> torch.Tensor: + res = [] + for img in images: + try: + print(img) + im = conf.test_transform(img) if actionModel else conf.test_transform(Image.open(img)) + res.append(im) + except: + continue + data = torch.stack(res) + return data + + +def inference(images, model, actionModel=False): + data = test_preprocess(images, actionModel) + if torch.cuda.is_available(): + data = data.to(conf.device) + features = model(data) + return features + + +def group_image(images, batch=64) -> list: + """Group image paths by batch size""" + size = len(images) + res = [] + for i in range(0, size, batch): + end = min(batch + i, size) + res.append(images[i:end]) + return res + +def getFeatureList(barList, imgList, model): + featList = [[] for i in range(len(barList))] + for index, feat in enumerate(imgList): + groups = group_image(feat) + for group in groups: + feat_tensor = inference(group, model) + for fe in feat_tensor: + if fe.device == 'cpu': + fe_np = fe.squeeze().detach().numpy() + else: + fe_np = fe.squeeze().detach().cpu().numpy() + featList[index].append(fe_np) + return featList + +def get_files(folder): + file_dict = {} + cnt = 0 + # barcode_list = ['6944649700065', '6924743915848', '6920459905012', '6901285991219', '6924882406269'] + for root, dirs, files in os.walk(folder): + + folder_name = os.path.basename(root) # 获取当前文件夹名称 + print(folder_name) + # with open('main_barcode.txt','a') as f: + # f.write(folder_name + '\n') + + # if len(dirs) == 0 and len(files) > 0 and folder_name in barcode_list: # 如果该文件夹没有子文件夹且有文件 + if len(dirs) == 0 and len(files) > 0: # 如果该文件夹没有子文件夹且有文件 + + file_names = [os.path.join(root, file) for file in files] # 获取所有文件名 + for file_name in file_names: + try: + convert_rgba_to_rgb(file_name) + except: + file_names.remove(file_name) + cnt += len(file_names) + file_dict[folder_name] = file_names + print(cnt) + + return file_dict + +def normalize(queFeatList): + for num1 in range(len(queFeatList)): + for num2 in range(len(queFeatList[num1])): + queFeatList[num1][num2] = queFeatList[num1][num2] / np.linalg.norm(queFeatList[num1][num2]) + return queFeatList +def img2feature(imgs_dict, model, barcode_flag): + if not len(imgs_dict) > 0: + raise ValueError("No imgs files provided") + queBarIdList = list(imgs_dict.keys()) + queImgsList = list(imgs_dict.values()) + queFeatList = getFeatureList(queBarIdList, queImgsList, model) + queFeatList = normalize(queFeatList) + return queBarIdList, queFeatList + + +def createFeatureDict(imgs_dict, model, + barcode_flag=False): ##imgs->{barcode1:[img1_1...img1_n], barcode2:[img2_1...img2_n]} + dicts_all = {} + value_list = [] + barcode_list, imgs_list = img2feature(imgs_dict, model, barcode_flag=False) + for i in range(len(barcode_list)): + dicts = {} + + imgs_list_ = [] + for j in range(len(imgs_list[i])): + imgs_list_.append(imgs_list[i][j].tolist()) + # with open('feature.txt','a') as f: + # f.write(str(imgs_list[i][j].tolist())+'\n') + + dicts['key'] = barcode_list[i] + dicts['value'] = imgs_list_ + value_list.append(dicts) + dicts_all['total'] = value_list + print('dicts_all', dicts_all) + with open('data_0909.json', 'a') as json_file: + json.dump(dicts_all, json_file) + + +def read_pkl_file(file_path): + with open(file_path, 'rb') as file: + data = pickle.load(file) + return data + + +if __name__ == "__main__": + ###将图片名称和模型推理特征向量字典存为json文件 + img_path = 'data/2000_train/base' + imgs_dict = get_files(img_path) + # print('imgs_dict', imgs_dict) + model = initModel() + createFeatureDict(imgs_dict, model, barcode_flag=False) + ###======================================================= + # ## =========pkl转json================ + # contents = read_pkl_file('dicts_list_1887.pkl') + # print(contents) + # with open('data_1887.json', 'w') as json_file: + # json.dump(contents, json_file) diff --git a/data.txt b/data.txt new file mode 100644 index 0000000..2dba6cb --- /dev/null +++ b/data.txt @@ -0,0 +1 @@ +-0.011240 0.008260 -0.006586 -0.001030 -0.013720 -0.002563 -0.046692 0.001246 -0.004448 -0.000575 0.011596 -0.008257 0.013804 0.013636 -0.015231 0.009997 0.002080 -0.004984 -0.008263 -0.017190 0.015428 -0.023105 -0.032034 0.014638 0.017993 -0.020305 -0.026571 -0.013695 0.019300 -0.008173 -0.014615 -0.029739 -0.005307 0.004259 0.027220 -0.003702 0.014700 -0.007800 -0.003245 -0.033670 -0.046784 -0.014097 0.022587 -0.007240 0.020345 -0.007308 -0.032579 -0.026351 0.007981 -0.004793 -0.003228 -0.001727 -0.019257 -0.004215 -0.018438 0.022703 -0.007637 -0.009849 -0.039515 -0.063543 0.038563 0.024434 -0.020794 -0.013288 0.024293 0.005290 -0.023577 0.043458 -0.008223 0.009316 0.015879 -0.016333 0.008900 0.014967 0.023980 -0.007785 0.011072 0.014957 -0.014673 -0.015173 0.001238 -0.002713 -0.010376 -0.009099 -0.019973 0.036330 -0.025961 -0.004987 -0.003738 -0.029137 0.027006 0.013193 -0.013956 -0.010477 0.006946 0.003161 0.006327 -0.010057 -0.000473 0.039186 -0.017787 0.030310 0.001994 0.021185 -0.010288 -0.026407 -0.007325 -0.035419 0.013209 0.040455 0.022204 0.011113 0.013391 -0.025687 -0.008719 0.018367 -0.044993 0.000359 -0.027293 0.017411 0.016963 -0.012727 0.004216 -0.007090 0.020172 -0.007653 -0.003869 0.000472 -0.006563 -0.010175 -0.012288 0.030884 0.021227 -0.008667 0.001995 0.002351 0.001223 0.024315 0.048389 -0.016056 0.015207 0.035997 -0.017303 0.029428 -0.018798 0.009189 -0.008502 0.036859 -0.003675 -0.003153 -0.017599 -0.020731 0.023639 0.019200 0.017236 0.015245 0.000899 0.013015 -0.026410 0.003367 0.007493 0.006190 -0.008258 0.017456 0.007086 -0.015679 -0.035943 -0.028529 -0.029751 0.000321 0.027217 0.002749 -0.016362 0.003308 0.011506 0.005780 0.008492 -0.002685 -0.006707 -0.001248 -0.005391 -0.010571 0.000716 -0.015180 -0.008275 -0.002362 -0.002915 -0.011054 -0.007975 0.016847 -0.003256 0.004353 -0.015026 -0.007171 -0.019375 -0.002358 -0.029985 -0.004786 0.008605 0.000120 0.016673 -0.029609 -0.030924 0.004636 -0.022859 -0.002508 0.028345 -0.007889 0.017705 0.012368 -0.020287 -0.001889 0.008966 0.017198 0.031740 -0.016312 -0.029071 0.014328 -0.029138 -0.006111 -0.038278 -0.006854 -0.006448 -0.016257 -0.003441 -0.003229 -0.012162 -0.014835 0.011474 -0.010222 -0.017947 0.021293 -0.006472 0.003448 0.005727 -0.033055 0.005207 -0.008356 -0.015410 0.029921 -0.030446 0.000284 -0.029262 0.034003 0.017720 -0.029013 0.010341 -0.037284 0.006937 0.060924 0.015401 0.023268 0.009021 -0.028368 -0.019817 0.034878 diff --git a/dataset/multi-trajs.py b/dataset/multi-trajs.py new file mode 100644 index 0000000..107f408 --- /dev/null +++ b/dataset/multi-trajs.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +""" +Created on Mon Mar 10 09:33:35 2025 +基准数据集筛选，选取tracking输出多个轨迹的事件 + +@author: ym +""" +import os +import numpy as np + +import sys +sys.path.append(r"D:\DetectTracking") +from tracking.utils.read_data import extract_data, read_tracking_output_realtime + + +def get_multitraj_file(spath, pattern): + multi_traj_events = [] + n = 0 + for evtname in os.listdir(spath): + name, ext = os.path.splitext(evtname) + eventpath = os.path.join(spath, evtname) + + evt = name.split('_') + condt = len(evt)>=2 and evt[-1].isdigit() and len(evt[-1])>=8 + if not condt: continue + if not os.path.isdir(eventpath): continue + + trackingboxes = [] + for dataname in os.listdir(eventpath): + if os.path.splitext(dataname)[-1] in [".jpg", ".png"]: + continue + + datapath = os.path.join(eventpath, dataname) + if not os.path.isfile(datapath): continue + CamerType = dataname.split('_')[0] + + if pattern=="realtime" and dataname.find("_tracking_output.data")>0: + trackingboxes, trackingfeats, tracking_outboxes, tracking_outfeats = read_tracking_output_realtime(datapath) + if pattern=="evtsplit" and dataname.find("_track.data")>0: + bboxes, ffeats, trackerboxes, trackerfeats, trackingboxes, trackingfeats = extract_data(datapath) + + if len(trackingboxes)>=2: + multi_traj_events.append(evtname) + n += 1 + print(f"{n}: {evtname}") + break + + + multi_traj_file = os.path.join(spath, "multi_traj_file.txt") + with open(multi_traj_file, "w") as file: + for item in multi_traj_events: + file.write(item + "\n") + +def main(): + spaths = [r"\\192.168.1.28\share\测试视频数据以及日志\各模块测试记录\比对测试\1212", + r"\\192.168.1.28\share\测试视频数据以及日志\各模块测试记录\比对测试\1216", + r"\\192.168.1.28\share\测试视频数据以及日志\各模块测试记录\比对测试\1218", + r"\\192.168.1.28\share\测试视频数据以及日志\各模块测试记录\比对测试\202412" + ] + + pattern = "evtsplit" # realtime # 全实时版、事件切分版数据读取方式 + for spath in spaths: + get_multitraj_file(spath, pattern) + + + + + + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/detect.py b/detect.py new file mode 100644 index 0000000..f577cca --- /dev/null +++ b/detect.py @@ -0,0 +1,308 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Run YOLOv5 detection inference on images, videos, directories, globs, YouTube, webcam, streams, etc. + +Usage - sources: + $ python detect.py --weights yolov5s.pt --source 0 # webcam + img.jpg # image + vid.mp4 # video + screen # screenshot + path/ # directory + list.txt # list of images + list.streams # list of streams + 'path/*.jpg' # glob + 'https://youtu.be/Zgi9g1ksQHc' # YouTube + 'rtsp://example.com/media.mp4' # RTSP, RTMP, HTTP stream + +Usage - formats: + $ python detect.py --weights yolov5s.pt # PyTorch + yolov5s.torchscript # TorchScript + yolov5s.onnx # ONNX Runtime or OpenCV DNN with --dnn + yolov5s_openvino_model # OpenVINO + yolov5s.engine # TensorRT + yolov5s.mlmodel # CoreML (macOS-only) + yolov5s_saved_model # TensorFlow SavedModel + yolov5s.pb # TensorFlow GraphDef + yolov5s.tflite # TensorFlow Lite + yolov5s_edgetpu.tflite # TensorFlow Edge TPU + yolov5s_paddle_model # PaddlePaddle +""" + +import argparse +import csv +import os +import platform +import sys +from pathlib import Path + +import torch + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[0] # YOLOv5 root directory +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) # add ROOT to PATH +ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative + +from ultralytics.utils.plotting import Annotator, colors, save_one_box + + +from models.common import DetectMultiBackend +from utils.dataloaders import IMG_FORMATS, VID_FORMATS, LoadImages, LoadScreenshots, LoadStreams +from utils.general import (LOGGER, Profile, check_file, check_img_size, check_imshow, check_requirements, colorstr, cv2, + increment_path, non_max_suppression, print_args, scale_boxes, strip_optimizer, xyxy2xywh) +from utils.torch_utils import select_device, smart_inference_mode + + + +@smart_inference_mode() +def run( + weights=ROOT / 'yolov5s.pt', # model path or triton URL + source=ROOT / 'data/images', # file/dir/URL/glob/screen/0(webcam) + data=ROOT / 'data/coco128.yaml', # dataset.yaml path + imgsz=(640, 640), # inference size (height, width) + conf_thres=0.25, # confidence threshold + iou_thres=0.45, # NMS IOU threshold + max_det=1000, # maximum detections per image + device='', # cuda device, i.e. 0 or 0,1,2,3 or cpu + view_img=False, # show results + save_txt=False, # save results to *.txt + save_csv=False, # save results in CSV format + save_conf=False, # save confidences in --save-txt labels + save_crop=False, # save cropped prediction boxes + nosave=False, # do not save images/videos + classes=None, # filter by class: --class 0, or --class 0 2 3 + agnostic_nms=False, # class-agnostic NMS + augment=False, # augmented inference + visualize=False, # visualize features + update=False, # update all models + project=ROOT / 'runs/detect', # save results to project/name + name='exp', # save results to project/name + exist_ok=False, # existing project/name ok, do not increment + line_thickness=3, # bounding box thickness (pixels) + hide_labels=False, # hide labels + hide_conf=False, # hide confidences + half=False, # use FP16 half-precision inference + dnn=False, # use OpenCV DNN for ONNX inference + vid_stride=1, # video frame-rate stride +): + source = str(source) + save_img = not nosave and not source.endswith('.txt') # save inference images + is_file = Path(source).suffix[1:] in (IMG_FORMATS + VID_FORMATS) + is_url = source.lower().startswith(('rtsp://', 'rtmp://', 'http://', 'https://')) + webcam = source.isnumeric() or source.endswith('.streams') or (is_url and not is_file) + screenshot = source.lower().startswith('screen') + if is_url and is_file: + source = check_file(source) # download + + # Directories + save_dir = increment_path(Path(project) / name, exist_ok=exist_ok) # increment run + (save_dir / 'labels' if save_txt else save_dir).mkdir(parents=True, exist_ok=True) # make dir + + # Load model + device = select_device(device) + model = DetectMultiBackend(weights, device=device, dnn=dnn, data=data, fp16=half) + stride, names, pt = model.stride, model.names, model.pt + imgsz = check_img_size(imgsz, s=stride) # check image size + + # Dataloader + bs = 1 # batch_size + if webcam: + view_img = check_imshow(warn=True) + dataset = LoadStreams(source, img_size=imgsz, stride=stride, auto=pt, vid_stride=vid_stride) + bs = len(dataset) + elif screenshot: + dataset = LoadScreenshots(source, img_size=imgsz, stride=stride, auto=pt) + else: + dataset = LoadImages(source, img_size=imgsz, stride=stride, auto=pt, vid_stride=vid_stride) + vid_path, vid_writer = [None] * bs, [None] * bs + + + # Run inference + model.warmup(imgsz=(1 if pt or model.triton else bs, 3, *imgsz)) # warmup + seen, windows, dt = 0, [], (Profile(), Profile(), Profile()) + + + for path, im, im0s, vid_cap, s in dataset: + ## ============================= tracking 功能只处理视频，writed by WQG + if dataset.mode == 'image': + continue + + + with dt[0]: + im = torch.from_numpy(im).to(model.device) + im = im.half() if model.fp16 else im.float() # uint8 to fp16/32 + im /= 255 # 0 - 255 to 0.0 - 1.0 + if len(im.shape) == 3: + im = im[None] # expand for batch dim + + # Inference + with dt[1]: + visualize = increment_path(save_dir / Path(path).stem, mkdir=True) if visualize else False + pred = model(im, augment=augment, visualize=visualize) + + # NMS + with dt[2]: + pred = non_max_suppression(pred, conf_thres, iou_thres, classes, agnostic_nms, max_det=max_det) + + # Second-stage classifier (optional) + # pred = utils.general.apply_classifier(pred, classifier_model, im, im0s) + + # Define the path for the CSV file + csv_path = save_dir / 'predictions.csv' + + # Create or append to the CSV file + def write_to_csv(image_name, prediction, confidence): + data = {'Image Name': image_name, 'Prediction': prediction, 'Confidence': confidence} + with open(csv_path, mode='a', newline='') as f: + writer = csv.DictWriter(f, fieldnames=data.keys()) + if not csv_path.is_file(): + writer.writeheader() + writer.writerow(data) + + + + # Process predictions + for i, det in enumerate(pred): # per image + seen += 1 + if webcam: # batch_size >= 1 + p, im0, frame = path[i], im0s[i].copy(), dataset.count + s += f'{i}: ' + else: + p, im0, frame = path, im0s.copy(), getattr(dataset, 'frame', 0) + + p = Path(p) # to Path + save_path = str(save_dir / p.name) # im.jpg + txt_path = str(save_dir / 'labels' / p.stem) + ('' if dataset.mode == 'image' else f'_{frame}') # im.txt + s += '%gx%g ' % im.shape[2:] # print string + gn = torch.tensor(im0.shape)[[1, 0, 1, 0]] # normalization gain whwh + imc = im0.copy() if save_crop else im0 # for save_crop + + annotator = Annotator(im0, line_width=line_thickness, example=str(names)) + + ## ================================================================================== writed by WQG + annotator_track = Annotator(im0, line_width=line_thickness, example=str(names)) + if len(det): + # Rescale boxes from img_size to im0 size + det[:, :4] = scale_boxes(im.shape[2:], det[:, :4], im0.shape).round() + + + # Print results + for c in det[:, 5].unique(): + n = (det[:, 5] == c).sum() # detections per class + s += f"{n} {names[int(c)]}{'s' * (n > 1)}, " # add to string + + # Write results + for *xyxy, conf, cls in reversed(det): + c = int(cls) # integer class + label = names[c] if hide_conf else f'{names[c]}' + confidence = float(conf) + confidence_str = f'{confidence:.2f}' + + if save_csv: + write_to_csv(p.name, label, confidence_str) + + if save_txt: # Write to file + xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh + line = (cls, *xywh, conf) if save_conf else (cls, *xywh) # label format + with open(f'{txt_path}.txt', 'a') as f: + f.write(('%g ' * len(line)).rstrip() % line + '\n') + + if save_img or save_crop or view_img: # Add bbox to image + c = int(cls) # integer class + label = None if hide_labels else (names[c] if hide_conf else f'{names[c]} {conf:.2f}') + annotator.box_label(xyxy, label, color=colors(c, True)) + if save_crop: + save_one_box(xyxy, imc, file=save_dir / 'crops' / names[c] / f'{p.stem}.jpg', BGR=True) + + + # Stream results + im0 = annotator.result() + if view_img: + if platform.system() == 'Linux' and p not in windows: + windows.append(p) + cv2.namedWindow(str(p), cv2.WINDOW_NORMAL | cv2.WINDOW_KEEPRATIO) # allow window resize (Linux) + cv2.resizeWindow(str(p), im0.shape[1], im0.shape[0]) + cv2.imshow(str(p), im0) + cv2.waitKey(1) # 1 millisecond + + # Save results (image with detections) + if save_img: + if dataset.mode == 'image': + cv2.imwrite(save_path, im0) + else: # 'video' or 'stream' + if vid_path[i] != save_path: # new video + vid_path[i] = save_path + if isinstance(vid_writer[i], cv2.VideoWriter): + vid_writer[i].release() # release previous video writer + if vid_cap: # video + fps = vid_cap.get(cv2.CAP_PROP_FPS) + w = int(vid_cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + h = int(vid_cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + else: # stream + fps, w, h = 30, im0.shape[1], im0.shape[0] + save_path = str(Path(save_path).with_suffix('.mp4')) # force *.mp4 suffix on results videos + vid_writer[i] = cv2.VideoWriter(save_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h)) + vid_writer[i].write(im0) + + # Print time (inference-only) + LOGGER.info(f"{s}{'' if len(det) else '(no detections), '}{dt[1].dt * 1E3:.1f}ms") + + + + # Print results + t = tuple(x.t / seen * 1E3 for x in dt) # speeds per image + LOGGER.info(f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {(1, 3, *imgsz)}' % t) + if save_txt or save_img: + s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else '' + LOGGER.info(f"Results saved to {colorstr('bold', save_dir)}{s}") + if update: + strip_optimizer(weights[0]) # update model (to fix SourceChangeWarning) + + +def parse_opt(): + modelpath = ROOT / 'ckpts/best_15000_0908.pt' # ROOT/'ckpts/best_15000_0908.pt', ROOT/'ckpts/yolov5s.pt' + datapath = r"D:/datasets/ym/videos/标记视频" # ROOT/'data/videos', ROOT/'data/images' images + + parser = argparse.ArgumentParser() + parser.add_argument('--weights', nargs='+', type=str, default=modelpath, help='model path or triton URL') # 'yolov5s.pt', best_15000_0908.pt + parser.add_argument('--source', type=str, default=datapath, help='file/dir/URL/glob/screen/0(webcam)') # images, videos + parser.add_argument('--data', type=str, default=ROOT / 'data/coco128.yaml', help='(optional) dataset.yaml path') + parser.add_argument('--imgsz', '--img', '--img-size', nargs='+', type=int, default=[640], help='inference size h,w') + parser.add_argument('--conf-thres', type=float, default=0.25, help='confidence threshold') + parser.add_argument('--iou-thres', type=float, default=0.45, help='NMS IoU threshold') + parser.add_argument('--max-det', type=int, default=1000, help='maximum detections per image') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--view-img', action='store_true', help='show results') + parser.add_argument('--save-txt', action='store_true', help='save results to *.txt') + parser.add_argument('--save-csv', action='store_true', help='save results in CSV format') + parser.add_argument('--save-conf', action='store_true', help='save confidences in --save-txt labels') + parser.add_argument('--save-crop', action='store_true', help='save cropped prediction boxes') + parser.add_argument('--nosave', action='store_true', help='do not save images/videos') + parser.add_argument('--classes', nargs='+', type=int, help='filter by class: --classes 0, or --classes 0 2 3') + parser.add_argument('--agnostic-nms', action='store_true', help='class-agnostic NMS') + parser.add_argument('--augment', action='store_true', help='augmented inference') + parser.add_argument('--visualize', action='store_true', help='visualize features') + parser.add_argument('--update', action='store_true', help='update all models') + parser.add_argument('--project', default=ROOT / 'runs/detect', help='save results to project/name') + parser.add_argument('--name', default='exp', help='save results to project/name') + parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment') + parser.add_argument('--line-thickness', default=3, type=int, help='bounding box thickness (pixels)') + parser.add_argument('--hide-labels', default=False, action='store_true', help='hide labels') + parser.add_argument('--hide-conf', default=False, action='store_true', help='hide confidences') + parser.add_argument('--half', action='store_true', help='use FP16 half-precision inference') + parser.add_argument('--dnn', action='store_true', help='use OpenCV DNN for ONNX inference') + parser.add_argument('--vid-stride', type=int, default=1, help='video frame-rate stride') + opt = parser.parse_args() + opt.imgsz *= 2 if len(opt.imgsz) == 1 else 1 # expand + print_args(vars(opt)) + return opt + + +def main(opt): + check_requirements(ROOT / 'requirements.txt', exclude=('tensorboard', 'thop')) + run(**vars(opt)) + + +if __name__ == '__main__': + opt = parse_opt() + main(opt) diff --git a/events/evt_rename.py b/events/evt_rename.py new file mode 100644 index 0000000..d350773 --- /dev/null +++ b/events/evt_rename.py @@ -0,0 +1,111 @@ +# -*- coding: utf-8 -*- +""" +Created on Thu Jan 9 14:34:41 2025 + +@author: ym +""" +import os +def read_deletedBarcode_file(filePath): + with open(filePath, 'r', encoding='utf-8') as f: + lines = f.readlines() + + split_flag, all_list = False, [] + dict, barcode_list, similarity_list = {}, [], [] + clean_lines = [line.strip().replace("'", '').replace('"', '') for line in lines] + + for i, line in enumerate(clean_lines): + if line.endswith(','): + line = line[:-1] + + stripped_line = line.strip() + if not stripped_line: + if len(barcode_list): dict['barcode'] = barcode_list + if len(similarity_list): dict['similarity'] = similarity_list + if len(dict): all_list.append(dict) + + split_flag = False + dict, barcode_list, similarity_list = {}, [], [] + continue + + if line.find(':')<0: continue + label = line.split(':')[0] + value = line.split(':')[1] + + if label == 'SeqDir': + dict['SeqDir'] = value + dict['filetype'] = "deletedBarcode" + if label == 'Deleted': + dict['Deleted'] = value + if label == 'List': + split_flag = True + continue + if split_flag: + barcode_list.append(label) + similarity_list.append(value) + + if len(barcode_list): dict['barcode'] = barcode_list + if len(similarity_list): dict['similarity'] = similarity_list + if len(dict): all_list.append(dict) + return all_list + + +def event_rename(path): + for filename in os.listdir(path): + fpath = os.path.join(path, filename) + if os.path.isfile(fpath): continue + flist = filename.split("_") + + if len(flist)==3 and flist[-1].isdigit() and len(flist[-1])>=8: + print(f"{path}: renamed!") + return + + dpath = os.path.join(path, "deletedBarcode.txt") + alist = read_deletedBarcode_file(dpath) + + input_events, getout_events = [], [] + for filename in os.listdir(path): + fpath = os.path.join(path, filename) + if os.path.isfile(fpath): continue + flist = filename.split("_") + + bcd = flist[-1] + if len(flist)==2 and flist[0].find("2024")==0 and len(bcd)==0: + bcds = [dt['Deleted'] for dt in alist if dt['SeqDir'].strip()==filename] + if len(bcds)==1: + getout_events.append((filename, filename + bcds[0].strip())) + + if len(flist)==2 and flist[0].find("2024")==0 and len(bcd)>=10: + input_events.append((filename, filename + '_' + bcd)) + + events = (input_events, getout_events) + for evts in events: + for name_old, name_new in evts: + path_old = os.path.join(path, name_old) + path_new = os.path.join(path, name_new) + try: + os.rename(path_old, path_new) + except Exception as e: + print(f"发生错误：{e}") + + + + +def main(): + + path = r"\\192.168.1.28\share\测试视频数据以及日志\测试_202406\0910\images" + event_rename(path) + + + +if __name__ == "__main__": + main() + + + + + + + + + + \ No newline at end of file diff --git a/execute_pipeline.py b/execute_pipeline.py new file mode 100644 index 0000000..39ed7ae --- /dev/null +++ b/execute_pipeline.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +""" +Created on Fri Mar 28 11:35:28 2025 + +@author: ym +""" + +from pipeline import execute_pipeline + + +def execute(datapath, savepath_v5, savepath_v10): + execute_pipeline(evtdir = datapath, + DataType = "raw", # raw, pkl + kk=None, + source_type = "video", # video, image, + save_path = savepath_v5, + yolo_ver = "V5", # V10, V5 + weight_yolo_v5 = r'./ckpts/best_cls10_0906.pt' , + weight_yolo_v10 = r'./ckpts/best_v10s_width0375_1205.pt', + saveimages = False + ) + execute_pipeline(evtdir = datapath, + DataType = "raw", # raw, pkl + kk=None, + source_type = "video", # video, image, + save_path = savepath_v10, + yolo_ver = "V10", # V10, V5 + weight_yolo_v5 = r'./ckpts/best_cls10_0906.pt' , + weight_yolo_v10 = r'./ckpts/best_v10s_width0375_1205.pt', + saveimages = False + ) + +datapath = r'/home/wqg/dataset/test_dataset/base_dataset/single_event/source/' +savepath_v5 = r'/home/wqg/dataset/pipeline/contrast/single_event_V5' +savepath_v10 = r'/home/wqg/dataset/pipeline/contrast/single_event_V10' +execute(datapath, savepath_v5, savepath_v10) + +datapath = r'/home/wqg/dataset/test_performence_dataset/' +savepath_v5 = r'/home/wqg/dataset/pipeline/contrast/performence_V5' +savepath_v10 = r'/home/wqg/dataset/pipeline/contrast/performence_V10' +execute(datapath, savepath_v5, savepath_v10) diff --git a/export.py b/export.py new file mode 100644 index 0000000..92d4247 --- /dev/null +++ b/export.py @@ -0,0 +1,863 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Export a YOLOv5 PyTorch model to other formats. TensorFlow exports authored by https://github.com/zldrobit + +Format | `export.py --include` | Model +--- | --- | --- +PyTorch | - | yolov5s.pt +TorchScript | `torchscript` | yolov5s.torchscript +ONNX | `onnx` | yolov5s.onnx +OpenVINO | `openvino` | yolov5s_openvino_model/ +TensorRT | `engine` | yolov5s.engine +CoreML | `coreml` | yolov5s.mlmodel +TensorFlow SavedModel | `saved_model` | yolov5s_saved_model/ +TensorFlow GraphDef | `pb` | yolov5s.pb +TensorFlow Lite | `tflite` | yolov5s.tflite +TensorFlow Edge TPU | `edgetpu` | yolov5s_edgetpu.tflite +TensorFlow.js | `tfjs` | yolov5s_web_model/ +PaddlePaddle | `paddle` | yolov5s_paddle_model/ + +Requirements: + $ pip install -r requirements.txt coremltools onnx onnx-simplifier onnxruntime openvino-dev tensorflow-cpu # CPU + $ pip install -r requirements.txt coremltools onnx onnx-simplifier onnxruntime-gpu openvino-dev tensorflow # GPU + +Usage: + $ python export.py --weights yolov5s.pt --include torchscript onnx openvino engine coreml tflite ... + +Inference: + $ python detect.py --weights yolov5s.pt # PyTorch + yolov5s.torchscript # TorchScript + yolov5s.onnx # ONNX Runtime or OpenCV DNN with --dnn + yolov5s_openvino_model # OpenVINO + yolov5s.engine # TensorRT + yolov5s.mlmodel # CoreML (macOS-only) + yolov5s_saved_model # TensorFlow SavedModel + yolov5s.pb # TensorFlow GraphDef + yolov5s.tflite # TensorFlow Lite + yolov5s_edgetpu.tflite # TensorFlow Edge TPU + yolov5s_paddle_model # PaddlePaddle + +TensorFlow.js: + $ cd .. && git clone https://github.com/zldrobit/tfjs-yolov5-example.git && cd tfjs-yolov5-example + $ npm install + $ ln -s ../../yolov5/yolov5s_web_model public/yolov5s_web_model + $ npm start +""" + +import argparse +import contextlib +import json +import os +import platform +import re +import subprocess +import sys +import time +import warnings +from pathlib import Path + +import pandas as pd +import torch +from torch.utils.mobile_optimizer import optimize_for_mobile + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[0] # YOLOv5 root directory +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) # add ROOT to PATH +if platform.system() != 'Windows': + ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative + +from models.experimental import attempt_load +from models.yolo import ClassificationModel, Detect, DetectionModel, SegmentationModel +from utils.dataloaders import LoadImages +from utils.general import (LOGGER, Profile, check_dataset, check_img_size, check_requirements, check_version, + check_yaml, colorstr, file_size, get_default_args, print_args, url2file, yaml_save) +from utils.torch_utils import select_device, smart_inference_mode + +MACOS = platform.system() == 'Darwin' # macOS environment + + +class iOSModel(torch.nn.Module): + + def __init__(self, model, im): + super().__init__() + b, c, h, w = im.shape # batch, channel, height, width + self.model = model + self.nc = model.nc # number of classes + if w == h: + self.normalize = 1. / w + else: + self.normalize = torch.tensor([1. / w, 1. / h, 1. / w, 1. / h]) # broadcast (slower, smaller) + # np = model(im)[0].shape[1] # number of points + # self.normalize = torch.tensor([1. / w, 1. / h, 1. / w, 1. / h]).expand(np, 4) # explicit (faster, larger) + + def forward(self, x): + xywh, conf, cls = self.model(x)[0].squeeze().split((4, 1, self.nc), 1) + return cls * conf, xywh * self.normalize # confidence (3780, 80), coordinates (3780, 4) + + +def export_formats(): + # YOLOv5 export formats + x = [ + ['PyTorch', '-', '.pt', True, True], + ['TorchScript', 'torchscript', '.torchscript', True, True], + ['ONNX', 'onnx', '.onnx', True, True], + ['OpenVINO', 'openvino', '_openvino_model', True, False], + ['TensorRT', 'engine', '.engine', False, True], + ['CoreML', 'coreml', '.mlmodel', True, False], + ['TensorFlow SavedModel', 'saved_model', '_saved_model', True, True], + ['TensorFlow GraphDef', 'pb', '.pb', True, True], + ['TensorFlow Lite', 'tflite', '.tflite', True, False], + ['TensorFlow Edge TPU', 'edgetpu', '_edgetpu.tflite', False, False], + ['TensorFlow.js', 'tfjs', '_web_model', False, False], + ['PaddlePaddle', 'paddle', '_paddle_model', True, True], ] + return pd.DataFrame(x, columns=['Format', 'Argument', 'Suffix', 'CPU', 'GPU']) + + +def try_export(inner_func): + # YOLOv5 export decorator, i..e @try_export + inner_args = get_default_args(inner_func) + + def outer_func(*args, **kwargs): + prefix = inner_args['prefix'] + try: + with Profile() as dt: + f, model = inner_func(*args, **kwargs) + LOGGER.info(f'{prefix} export success ✅ {dt.t:.1f}s, saved as {f} ({file_size(f):.1f} MB)') + return f, model + except Exception as e: + LOGGER.info(f'{prefix} export failure ❌ {dt.t:.1f}s: {e}') + return None, None + + return outer_func + + +@try_export +def export_torchscript(model, im, file, optimize, prefix=colorstr('TorchScript:')): + # YOLOv5 TorchScript model export + LOGGER.info(f'\n{prefix} starting export with torch {torch.__version__}...') + f = file.with_suffix('.torchscript') + + ts = torch.jit.trace(model, im, strict=False) + d = {'shape': im.shape, 'stride': int(max(model.stride)), 'names': model.names} + extra_files = {'config.txt': json.dumps(d)} # torch._C.ExtraFilesMap() + if optimize: # https://pytorch.org/tutorials/recipes/mobile_interpreter.html + optimize_for_mobile(ts)._save_for_lite_interpreter(str(f), _extra_files=extra_files) + else: + ts.save(str(f), _extra_files=extra_files) + return f, None + + +@try_export +def export_onnx(model, im, file, opset, dynamic, simplify, prefix=colorstr('ONNX:')): + # YOLOv5 ONNX export + check_requirements('onnx>=1.12.0') + import onnx + + LOGGER.info(f'\n{prefix} starting export with onnx {onnx.__version__}...') + f = file.with_suffix('.onnx') + + output_names = ['output0', 'output1'] if isinstance(model, SegmentationModel) else ['output0'] + if dynamic: + dynamic = {'images': {0: 'batch', 2: 'height', 3: 'width'}} # shape(1,3,640,640) + if isinstance(model, SegmentationModel): + dynamic['output0'] = {0: 'batch', 1: 'anchors'} # shape(1,25200,85) + dynamic['output1'] = {0: 'batch', 2: 'mask_height', 3: 'mask_width'} # shape(1,32,160,160) + elif isinstance(model, DetectionModel): + dynamic['output0'] = {0: 'batch', 1: 'anchors'} # shape(1,25200,85) + + torch.onnx.export( + model.cpu() if dynamic else model, # --dynamic only compatible with cpu + im.cpu() if dynamic else im, + f, + verbose=False, + opset_version=opset, + do_constant_folding=True, # WARNING: DNN inference with torch>=1.12 may require do_constant_folding=False + input_names=['images'], + output_names=output_names, + dynamic_axes=dynamic or None) + + # Checks + model_onnx = onnx.load(f) # load onnx model + onnx.checker.check_model(model_onnx) # check onnx model + + # Metadata + d = {'stride': int(max(model.stride)), 'names': model.names} + for k, v in d.items(): + meta = model_onnx.metadata_props.add() + meta.key, meta.value = k, str(v) + onnx.save(model_onnx, f) + + # Simplify + if simplify: + try: + cuda = torch.cuda.is_available() + check_requirements(('onnxruntime-gpu' if cuda else 'onnxruntime', 'onnx-simplifier>=0.4.1')) + import onnxsim + + LOGGER.info(f'{prefix} simplifying with onnx-simplifier {onnxsim.__version__}...') + model_onnx, check = onnxsim.simplify(model_onnx) + assert check, 'assert check failed' + onnx.save(model_onnx, f) + except Exception as e: + LOGGER.info(f'{prefix} simplifier failure: {e}') + return f, model_onnx + + +@try_export +def export_openvino(file, metadata, half, int8, data, prefix=colorstr('OpenVINO:')): + # YOLOv5 OpenVINO export + check_requirements('openvino-dev>=2023.0') # requires openvino-dev: https://pypi.org/project/openvino-dev/ + import openvino.runtime as ov # noqa + from openvino.tools import mo # noqa + + LOGGER.info(f'\n{prefix} starting export with openvino {ov.__version__}...') + f = str(file).replace(file.suffix, f'_openvino_model{os.sep}') + f_onnx = file.with_suffix('.onnx') + f_ov = str(Path(f) / file.with_suffix('.xml').name) + if int8: + check_requirements('nncf>=2.4.0') # requires at least version 2.4.0 to use the post-training quantization + import nncf + import numpy as np + from openvino.runtime import Core + + from utils.dataloaders import create_dataloader + core = Core() + onnx_model = core.read_model(f_onnx) # export + + def prepare_input_tensor(image: np.ndarray): + input_tensor = image.astype(np.float32) # uint8 to fp16/32 + input_tensor /= 255.0 # 0 - 255 to 0.0 - 1.0 + + if input_tensor.ndim == 3: + input_tensor = np.expand_dims(input_tensor, 0) + return input_tensor + + def gen_dataloader(yaml_path, task='train', imgsz=640, workers=4): + data_yaml = check_yaml(yaml_path) + data = check_dataset(data_yaml) + dataloader = create_dataloader(data[task], + imgsz=imgsz, + batch_size=1, + stride=32, + pad=0.5, + single_cls=False, + rect=False, + workers=workers)[0] + return dataloader + + # noqa: F811 + + def transform_fn(data_item): + """ + Quantization transform function. Extracts and preprocess input data from dataloader item for quantization. + Parameters: + data_item: Tuple with data item produced by DataLoader during iteration + Returns: + input_tensor: Input data for quantization + """ + img = data_item[0].numpy() + input_tensor = prepare_input_tensor(img) + return input_tensor + + ds = gen_dataloader(data) + quantization_dataset = nncf.Dataset(ds, transform_fn) + ov_model = nncf.quantize(onnx_model, quantization_dataset, preset=nncf.QuantizationPreset.MIXED) + else: + ov_model = mo.convert_model(f_onnx, model_name=file.stem, framework='onnx', compress_to_fp16=half) # export + + ov.serialize(ov_model, f_ov) # save + yaml_save(Path(f) / file.with_suffix('.yaml').name, metadata) # add metadata.yaml + return f, None + + +@try_export +def export_paddle(model, im, file, metadata, prefix=colorstr('PaddlePaddle:')): + # YOLOv5 Paddle export + check_requirements(('paddlepaddle', 'x2paddle')) + import x2paddle + from x2paddle.convert import pytorch2paddle + + LOGGER.info(f'\n{prefix} starting export with X2Paddle {x2paddle.__version__}...') + f = str(file).replace('.pt', f'_paddle_model{os.sep}') + + pytorch2paddle(module=model, save_dir=f, jit_type='trace', input_examples=[im]) # export + yaml_save(Path(f) / file.with_suffix('.yaml').name, metadata) # add metadata.yaml + return f, None + + +@try_export +def export_coreml(model, im, file, int8, half, nms, prefix=colorstr('CoreML:')): + # YOLOv5 CoreML export + check_requirements('coremltools') + import coremltools as ct + + LOGGER.info(f'\n{prefix} starting export with coremltools {ct.__version__}...') + f = file.with_suffix('.mlmodel') + + if nms: + model = iOSModel(model, im) + ts = torch.jit.trace(model, im, strict=False) # TorchScript model + ct_model = ct.convert(ts, inputs=[ct.ImageType('image', shape=im.shape, scale=1 / 255, bias=[0, 0, 0])]) + bits, mode = (8, 'kmeans_lut') if int8 else (16, 'linear') if half else (32, None) + if bits < 32: + if MACOS: # quantization only supported on macOS + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', category=DeprecationWarning) # suppress numpy==1.20 float warning + ct_model = ct.models.neural_network.quantization_utils.quantize_weights(ct_model, bits, mode) + else: + print(f'{prefix} quantization only supported on macOS, skipping...') + ct_model.save(f) + return f, ct_model + + +@try_export +def export_engine(model, im, file, half, dynamic, simplify, workspace=4, verbose=False, prefix=colorstr('TensorRT:')): + # YOLOv5 TensorRT export https://developer.nvidia.com/tensorrt + assert im.device.type != 'cpu', 'export running on CPU but must be on GPU, i.e. `python export.py --device 0`' + try: + import tensorrt as trt + except Exception: + if platform.system() == 'Linux': + check_requirements('nvidia-tensorrt', cmds='-U --index-url https://pypi.ngc.nvidia.com') + import tensorrt as trt + + if trt.__version__[0] == '7': # TensorRT 7 handling https://github.com/ultralytics/yolov5/issues/6012 + grid = model.model[-1].anchor_grid + model.model[-1].anchor_grid = [a[..., :1, :1, :] for a in grid] + export_onnx(model, im, file, 12, dynamic, simplify) # opset 12 + model.model[-1].anchor_grid = grid + else: # TensorRT >= 8 + check_version(trt.__version__, '8.0.0', hard=True) # require tensorrt>=8.0.0 + export_onnx(model, im, file, 12, dynamic, simplify) # opset 12 + onnx = file.with_suffix('.onnx') + + LOGGER.info(f'\n{prefix} starting export with TensorRT {trt.__version__}...') + assert onnx.exists(), f'failed to export ONNX file: {onnx}' + f = file.with_suffix('.engine') # TensorRT engine file + logger = trt.Logger(trt.Logger.INFO) + if verbose: + logger.min_severity = trt.Logger.Severity.VERBOSE + + builder = trt.Builder(logger) + config = builder.create_builder_config() + config.max_workspace_size = workspace * 1 << 30 + # config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, workspace << 30) # fix TRT 8.4 deprecation notice + + flag = (1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)) + network = builder.create_network(flag) + parser = trt.OnnxParser(network, logger) + if not parser.parse_from_file(str(onnx)): + raise RuntimeError(f'failed to load ONNX file: {onnx}') + + inputs = [network.get_input(i) for i in range(network.num_inputs)] + outputs = [network.get_output(i) for i in range(network.num_outputs)] + for inp in inputs: + LOGGER.info(f'{prefix} input "{inp.name}" with shape{inp.shape} {inp.dtype}') + for out in outputs: + LOGGER.info(f'{prefix} output "{out.name}" with shape{out.shape} {out.dtype}') + + if dynamic: + if im.shape[0] <= 1: + LOGGER.warning(f'{prefix} WARNING ⚠️ --dynamic model requires maximum --batch-size argument') + profile = builder.create_optimization_profile() + for inp in inputs: + profile.set_shape(inp.name, (1, *im.shape[1:]), (max(1, im.shape[0] // 2), *im.shape[1:]), im.shape) + config.add_optimization_profile(profile) + + LOGGER.info(f'{prefix} building FP{16 if builder.platform_has_fast_fp16 and half else 32} engine as {f}') + if builder.platform_has_fast_fp16 and half: + config.set_flag(trt.BuilderFlag.FP16) + with builder.build_engine(network, config) as engine, open(f, 'wb') as t: + t.write(engine.serialize()) + return f, None + + +@try_export +def export_saved_model(model, + im, + file, + dynamic, + tf_nms=False, + agnostic_nms=False, + topk_per_class=100, + topk_all=100, + iou_thres=0.45, + conf_thres=0.25, + keras=False, + prefix=colorstr('TensorFlow SavedModel:')): + # YOLOv5 TensorFlow SavedModel export + try: + import tensorflow as tf + except Exception: + check_requirements(f"tensorflow{'' if torch.cuda.is_available() else '-macos' if MACOS else '-cpu'}") + import tensorflow as tf + from tensorflow.python.framework.convert_to_constants import convert_variables_to_constants_v2 + + from models.tf import TFModel + + LOGGER.info(f'\n{prefix} starting export with tensorflow {tf.__version__}...') + f = str(file).replace('.pt', '_saved_model') + batch_size, ch, *imgsz = list(im.shape) # BCHW + + tf_model = TFModel(cfg=model.yaml, model=model, nc=model.nc, imgsz=imgsz) + im = tf.zeros((batch_size, *imgsz, ch)) # BHWC order for TensorFlow + _ = tf_model.predict(im, tf_nms, agnostic_nms, topk_per_class, topk_all, iou_thres, conf_thres) + inputs = tf.keras.Input(shape=(*imgsz, ch), batch_size=None if dynamic else batch_size) + outputs = tf_model.predict(inputs, tf_nms, agnostic_nms, topk_per_class, topk_all, iou_thres, conf_thres) + keras_model = tf.keras.Model(inputs=inputs, outputs=outputs) + keras_model.trainable = False + keras_model.summary() + if keras: + keras_model.save(f, save_format='tf') + else: + spec = tf.TensorSpec(keras_model.inputs[0].shape, keras_model.inputs[0].dtype) + m = tf.function(lambda x: keras_model(x)) # full model + m = m.get_concrete_function(spec) + frozen_func = convert_variables_to_constants_v2(m) + tfm = tf.Module() + tfm.__call__ = tf.function(lambda x: frozen_func(x)[:4] if tf_nms else frozen_func(x), [spec]) + tfm.__call__(im) + tf.saved_model.save(tfm, + f, + options=tf.saved_model.SaveOptions(experimental_custom_gradients=False) if check_version( + tf.__version__, '2.6') else tf.saved_model.SaveOptions()) + return f, keras_model + + +@try_export +def export_pb(keras_model, file, prefix=colorstr('TensorFlow GraphDef:')): + # YOLOv5 TensorFlow GraphDef *.pb export https://github.com/leimao/Frozen_Graph_TensorFlow + import tensorflow as tf + from tensorflow.python.framework.convert_to_constants import convert_variables_to_constants_v2 + + LOGGER.info(f'\n{prefix} starting export with tensorflow {tf.__version__}...') + f = file.with_suffix('.pb') + + m = tf.function(lambda x: keras_model(x)) # full model + m = m.get_concrete_function(tf.TensorSpec(keras_model.inputs[0].shape, keras_model.inputs[0].dtype)) + frozen_func = convert_variables_to_constants_v2(m) + frozen_func.graph.as_graph_def() + tf.io.write_graph(graph_or_graph_def=frozen_func.graph, logdir=str(f.parent), name=f.name, as_text=False) + return f, None + + +@try_export +def export_tflite(keras_model, im, file, int8, data, nms, agnostic_nms, prefix=colorstr('TensorFlow Lite:')): + # YOLOv5 TensorFlow Lite export + import tensorflow as tf + + LOGGER.info(f'\n{prefix} starting export with tensorflow {tf.__version__}...') + batch_size, ch, *imgsz = list(im.shape) # BCHW + f = str(file).replace('.pt', '-fp16.tflite') + + converter = tf.lite.TFLiteConverter.from_keras_model(keras_model) + converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS] + converter.target_spec.supported_types = [tf.float16] + converter.optimizations = [tf.lite.Optimize.DEFAULT] + if int8: + from models.tf import representative_dataset_gen + dataset = LoadImages(check_dataset(check_yaml(data))['train'], img_size=imgsz, auto=False) + converter.representative_dataset = lambda: representative_dataset_gen(dataset, ncalib=100) + converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8] + converter.target_spec.supported_types = [] + converter.inference_input_type = tf.uint8 # or tf.int8 + converter.inference_output_type = tf.uint8 # or tf.int8 + converter.experimental_new_quantizer = True + f = str(file).replace('.pt', '-int8.tflite') + if nms or agnostic_nms: + converter.target_spec.supported_ops.append(tf.lite.OpsSet.SELECT_TF_OPS) + + tflite_model = converter.convert() + open(f, 'wb').write(tflite_model) + return f, None + + +@try_export +def export_edgetpu(file, prefix=colorstr('Edge TPU:')): + # YOLOv5 Edge TPU export https://coral.ai/docs/edgetpu/models-intro/ + cmd = 'edgetpu_compiler --version' + help_url = 'https://coral.ai/docs/edgetpu/compiler/' + assert platform.system() == 'Linux', f'export only supported on Linux. See {help_url}' + if subprocess.run(f'{cmd} > /dev/null 2>&1', shell=True).returncode != 0: + LOGGER.info(f'\n{prefix} export requires Edge TPU compiler. Attempting install from {help_url}') + sudo = subprocess.run('sudo --version >/dev/null', shell=True).returncode == 0 # sudo installed on system + for c in ( + 'curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -', + 'echo "deb https://packages.cloud.google.com/apt coral-edgetpu-stable main" | sudo tee /etc/apt/sources.list.d/coral-edgetpu.list', + 'sudo apt-get update', 'sudo apt-get install edgetpu-compiler'): + subprocess.run(c if sudo else c.replace('sudo ', ''), shell=True, check=True) + ver = subprocess.run(cmd, shell=True, capture_output=True, check=True).stdout.decode().split()[-1] + + LOGGER.info(f'\n{prefix} starting export with Edge TPU compiler {ver}...') + f = str(file).replace('.pt', '-int8_edgetpu.tflite') # Edge TPU model + f_tfl = str(file).replace('.pt', '-int8.tflite') # TFLite model + + subprocess.run([ + 'edgetpu_compiler', + '-s', + '-d', + '-k', + '10', + '--out_dir', + str(file.parent), + f_tfl, ], check=True) + return f, None + + +@try_export +def export_tfjs(file, int8, prefix=colorstr('TensorFlow.js:')): + # YOLOv5 TensorFlow.js export + check_requirements('tensorflowjs') + import tensorflowjs as tfjs + + LOGGER.info(f'\n{prefix} starting export with tensorflowjs {tfjs.__version__}...') + f = str(file).replace('.pt', '_web_model') # js dir + f_pb = file.with_suffix('.pb') # *.pb path + f_json = f'{f}/model.json' # *.json path + + args = [ + 'tensorflowjs_converter', + '--input_format=tf_frozen_model', + '--quantize_uint8' if int8 else '', + '--output_node_names=Identity,Identity_1,Identity_2,Identity_3', + str(f_pb), + str(f), ] + subprocess.run([arg for arg in args if arg], check=True) + + json = Path(f_json).read_text() + with open(f_json, 'w') as j: # sort JSON Identity_* in ascending order + subst = re.sub( + r'{"outputs": {"Identity.?.?": {"name": "Identity.?.?"}, ' + r'"Identity.?.?": {"name": "Identity.?.?"}, ' + r'"Identity.?.?": {"name": "Identity.?.?"}, ' + r'"Identity.?.?": {"name": "Identity.?.?"}}}', r'{"outputs": {"Identity": {"name": "Identity"}, ' + r'"Identity_1": {"name": "Identity_1"}, ' + r'"Identity_2": {"name": "Identity_2"}, ' + r'"Identity_3": {"name": "Identity_3"}}}', json) + j.write(subst) + return f, None + + +def add_tflite_metadata(file, metadata, num_outputs): + # Add metadata to *.tflite models per https://www.tensorflow.org/lite/models/convert/metadata + with contextlib.suppress(ImportError): + # check_requirements('tflite_support') + from tflite_support import flatbuffers + from tflite_support import metadata as _metadata + from tflite_support import metadata_schema_py_generated as _metadata_fb + + tmp_file = Path('/tmp/meta.txt') + with open(tmp_file, 'w') as meta_f: + meta_f.write(str(metadata)) + + model_meta = _metadata_fb.ModelMetadataT() + label_file = _metadata_fb.AssociatedFileT() + label_file.name = tmp_file.name + model_meta.associatedFiles = [label_file] + + subgraph = _metadata_fb.SubGraphMetadataT() + subgraph.inputTensorMetadata = [_metadata_fb.TensorMetadataT()] + subgraph.outputTensorMetadata = [_metadata_fb.TensorMetadataT()] * num_outputs + model_meta.subgraphMetadata = [subgraph] + + b = flatbuffers.Builder(0) + b.Finish(model_meta.Pack(b), _metadata.MetadataPopulator.METADATA_FILE_IDENTIFIER) + metadata_buf = b.Output() + + populator = _metadata.MetadataPopulator.with_model_file(file) + populator.load_metadata_buffer(metadata_buf) + populator.load_associated_files([str(tmp_file)]) + populator.populate() + tmp_file.unlink() + + +def pipeline_coreml(model, im, file, names, y, prefix=colorstr('CoreML Pipeline:')): + # YOLOv5 CoreML pipeline + import coremltools as ct + from PIL import Image + + print(f'{prefix} starting pipeline with coremltools {ct.__version__}...') + batch_size, ch, h, w = list(im.shape) # BCHW + t = time.time() + + # YOLOv5 Output shapes + spec = model.get_spec() + out0, out1 = iter(spec.description.output) + if platform.system() == 'Darwin': + img = Image.new('RGB', (w, h)) # img(192 width, 320 height) + # img = torch.zeros((*opt.img_size, 3)).numpy() # img size(320,192,3) iDetection + out = model.predict({'image': img}) + out0_shape, out1_shape = out[out0.name].shape, out[out1.name].shape + else: # linux and windows can not run model.predict(), get sizes from pytorch output y + s = tuple(y[0].shape) + out0_shape, out1_shape = (s[1], s[2] - 5), (s[1], 4) # (3780, 80), (3780, 4) + + # Checks + nx, ny = spec.description.input[0].type.imageType.width, spec.description.input[0].type.imageType.height + na, nc = out0_shape + # na, nc = out0.type.multiArrayType.shape # number anchors, classes + assert len(names) == nc, f'{len(names)} names found for nc={nc}' # check + + # Define output shapes (missing) + out0.type.multiArrayType.shape[:] = out0_shape # (3780, 80) + out1.type.multiArrayType.shape[:] = out1_shape # (3780, 4) + # spec.neuralNetwork.preprocessing[0].featureName = '0' + + # Flexible input shapes + # from coremltools.models.neural_network import flexible_shape_utils + # s = [] # shapes + # s.append(flexible_shape_utils.NeuralNetworkImageSize(320, 192)) + # s.append(flexible_shape_utils.NeuralNetworkImageSize(640, 384)) # (height, width) + # flexible_shape_utils.add_enumerated_image_sizes(spec, feature_name='image', sizes=s) + # r = flexible_shape_utils.NeuralNetworkImageSizeRange() # shape ranges + # r.add_height_range((192, 640)) + # r.add_width_range((192, 640)) + # flexible_shape_utils.update_image_size_range(spec, feature_name='image', size_range=r) + + # Print + print(spec.description) + + # Model from spec + model = ct.models.MLModel(spec) + + # 3. Create NMS protobuf + nms_spec = ct.proto.Model_pb2.Model() + nms_spec.specificationVersion = 5 + for i in range(2): + decoder_output = model._spec.description.output[i].SerializeToString() + nms_spec.description.input.add() + nms_spec.description.input[i].ParseFromString(decoder_output) + nms_spec.description.output.add() + nms_spec.description.output[i].ParseFromString(decoder_output) + + nms_spec.description.output[0].name = 'confidence' + nms_spec.description.output[1].name = 'coordinates' + + output_sizes = [nc, 4] + for i in range(2): + ma_type = nms_spec.description.output[i].type.multiArrayType + ma_type.shapeRange.sizeRanges.add() + ma_type.shapeRange.sizeRanges[0].lowerBound = 0 + ma_type.shapeRange.sizeRanges[0].upperBound = -1 + ma_type.shapeRange.sizeRanges.add() + ma_type.shapeRange.sizeRanges[1].lowerBound = output_sizes[i] + ma_type.shapeRange.sizeRanges[1].upperBound = output_sizes[i] + del ma_type.shape[:] + + nms = nms_spec.nonMaximumSuppression + nms.confidenceInputFeatureName = out0.name # 1x507x80 + nms.coordinatesInputFeatureName = out1.name # 1x507x4 + nms.confidenceOutputFeatureName = 'confidence' + nms.coordinatesOutputFeatureName = 'coordinates' + nms.iouThresholdInputFeatureName = 'iouThreshold' + nms.confidenceThresholdInputFeatureName = 'confidenceThreshold' + nms.iouThreshold = 0.45 + nms.confidenceThreshold = 0.25 + nms.pickTop.perClass = True + nms.stringClassLabels.vector.extend(names.values()) + nms_model = ct.models.MLModel(nms_spec) + + # 4. Pipeline models together + pipeline = ct.models.pipeline.Pipeline(input_features=[('image', ct.models.datatypes.Array(3, ny, nx)), + ('iouThreshold', ct.models.datatypes.Double()), + ('confidenceThreshold', ct.models.datatypes.Double())], + output_features=['confidence', 'coordinates']) + pipeline.add_model(model) + pipeline.add_model(nms_model) + + # Correct datatypes + pipeline.spec.description.input[0].ParseFromString(model._spec.description.input[0].SerializeToString()) + pipeline.spec.description.output[0].ParseFromString(nms_model._spec.description.output[0].SerializeToString()) + pipeline.spec.description.output[1].ParseFromString(nms_model._spec.description.output[1].SerializeToString()) + + # Update metadata + pipeline.spec.specificationVersion = 5 + pipeline.spec.description.metadata.versionString = 'https://github.com/ultralytics/yolov5' + pipeline.spec.description.metadata.shortDescription = 'https://github.com/ultralytics/yolov5' + pipeline.spec.description.metadata.author = 'glenn.jocher@ultralytics.com' + pipeline.spec.description.metadata.license = 'https://github.com/ultralytics/yolov5/blob/master/LICENSE' + pipeline.spec.description.metadata.userDefined.update({ + 'classes': ','.join(names.values()), + 'iou_threshold': str(nms.iouThreshold), + 'confidence_threshold': str(nms.confidenceThreshold)}) + + # Save the model + f = file.with_suffix('.mlmodel') # filename + model = ct.models.MLModel(pipeline.spec) + model.input_description['image'] = 'Input image' + model.input_description['iouThreshold'] = f'(optional) IOU Threshold override (default: {nms.iouThreshold})' + model.input_description['confidenceThreshold'] = \ + f'(optional) Confidence Threshold override (default: {nms.confidenceThreshold})' + model.output_description['confidence'] = 'Boxes × Class confidence (see user-defined metadata "classes")' + model.output_description['coordinates'] = 'Boxes × [x, y, width, height] (relative to image size)' + model.save(f) # pipelined + print(f'{prefix} pipeline success ({time.time() - t:.2f}s), saved as {f} ({file_size(f):.1f} MB)') + + +@smart_inference_mode() +def run( + data=ROOT / 'data/coco128.yaml', # 'dataset.yaml path' + weights=ROOT / 'yolov5s.pt', # weights path + imgsz=(640, 640), # image (height, width) + batch_size=1, # batch size + device='cpu', # cuda device, i.e. 0 or 0,1,2,3 or cpu + include=('torchscript', 'onnx'), # include formats + half=False, # FP16 half-precision export + inplace=False, # set YOLOv5 Detect() inplace=True + keras=False, # use Keras + optimize=False, # TorchScript: optimize for mobile + int8=False, # CoreML/TF INT8 quantization + dynamic=False, # ONNX/TF/TensorRT: dynamic axes + simplify=False, # ONNX: simplify model + opset=12, # ONNX: opset version + verbose=False, # TensorRT: verbose log + workspace=4, # TensorRT: workspace size (GB) + nms=False, # TF: add NMS to model + agnostic_nms=False, # TF: add agnostic NMS to model + topk_per_class=100, # TF.js NMS: topk per class to keep + topk_all=100, # TF.js NMS: topk for all classes to keep + iou_thres=0.45, # TF.js NMS: IoU threshold + conf_thres=0.25, # TF.js NMS: confidence threshold +): + t = time.time() + include = [x.lower() for x in include] # to lowercase + fmts = tuple(export_formats()['Argument'][1:]) # --include arguments + flags = [x in include for x in fmts] + assert sum(flags) == len(include), f'ERROR: Invalid --include {include}, valid --include arguments are {fmts}' + jit, onnx, xml, engine, coreml, saved_model, pb, tflite, edgetpu, tfjs, paddle = flags # export booleans + file = Path(url2file(weights) if str(weights).startswith(('http:/', 'https:/')) else weights) # PyTorch weights + + # Load PyTorch model + device = select_device(device) + if half: + assert device.type != 'cpu' or coreml, '--half only compatible with GPU export, i.e. use --device 0' + assert not dynamic, '--half not compatible with --dynamic, i.e. use either --half or --dynamic but not both' + model = attempt_load(weights, device=device, inplace=True, fuse=True) # load FP32 model + + # Checks + imgsz *= 2 if len(imgsz) == 1 else 1 # expand + if optimize: + assert device.type == 'cpu', '--optimize not compatible with cuda devices, i.e. use --device cpu' + + # Input + gs = int(max(model.stride)) # grid size (max stride) + imgsz = [check_img_size(x, gs) for x in imgsz] # verify img_size are gs-multiples + im = torch.zeros(batch_size, 3, *imgsz).to(device) # image size(1,3,320,192) BCHW iDetection + + # Update model + model.eval() + for k, m in model.named_modules(): + if isinstance(m, Detect): + m.inplace = inplace + m.dynamic = dynamic + m.export = True + + for _ in range(2): + y = model(im) # dry runs + if half and not coreml: + im, model = im.half(), model.half() # to FP16 + shape = tuple((y[0] if isinstance(y, tuple) else y).shape) # model output shape + metadata = {'stride': int(max(model.stride)), 'names': model.names} # model metadata + LOGGER.info(f"\n{colorstr('PyTorch:')} starting from {file} with output shape {shape} ({file_size(file):.1f} MB)") + + # Exports + f = [''] * len(fmts) # exported filenames + warnings.filterwarnings(action='ignore', category=torch.jit.TracerWarning) # suppress TracerWarning + if jit: # TorchScript + f[0], _ = export_torchscript(model, im, file, optimize) + if engine: # TensorRT required before ONNX + f[1], _ = export_engine(model, im, file, half, dynamic, simplify, workspace, verbose) + if onnx or xml: # OpenVINO requires ONNX + f[2], _ = export_onnx(model, im, file, opset, dynamic, simplify) + if xml: # OpenVINO + f[3], _ = export_openvino(file, metadata, half, int8, data) + if coreml: # CoreML + f[4], ct_model = export_coreml(model, im, file, int8, half, nms) + if nms: + pipeline_coreml(ct_model, im, file, model.names, y) + if any((saved_model, pb, tflite, edgetpu, tfjs)): # TensorFlow formats + assert not tflite or not tfjs, 'TFLite and TF.js models must be exported separately, please pass only one type.' + assert not isinstance(model, ClassificationModel), 'ClassificationModel export to TF formats not yet supported.' + f[5], s_model = export_saved_model(model.cpu(), + im, + file, + dynamic, + tf_nms=nms or agnostic_nms or tfjs, + agnostic_nms=agnostic_nms or tfjs, + topk_per_class=topk_per_class, + topk_all=topk_all, + iou_thres=iou_thres, + conf_thres=conf_thres, + keras=keras) + if pb or tfjs: # pb prerequisite to tfjs + f[6], _ = export_pb(s_model, file) + if tflite or edgetpu: + f[7], _ = export_tflite(s_model, im, file, int8 or edgetpu, data=data, nms=nms, agnostic_nms=agnostic_nms) + if edgetpu: + f[8], _ = export_edgetpu(file) + add_tflite_metadata(f[8] or f[7], metadata, num_outputs=len(s_model.outputs)) + if tfjs: + f[9], _ = export_tfjs(file, int8) + if paddle: # PaddlePaddle + f[10], _ = export_paddle(model, im, file, metadata) + + # Finish + f = [str(x) for x in f if x] # filter out '' and None + if any(f): + cls, det, seg = (isinstance(model, x) for x in (ClassificationModel, DetectionModel, SegmentationModel)) # type + det &= not seg # segmentation models inherit from SegmentationModel(DetectionModel) + dir = Path('segment' if seg else 'classify' if cls else '') + h = '--half' if half else '' # --half FP16 inference arg + s = '# WARNING ⚠️ ClassificationModel not yet supported for PyTorch Hub AutoShape inference' if cls else \ + '# WARNING ⚠️ SegmentationModel not yet supported for PyTorch Hub AutoShape inference' if seg else '' + LOGGER.info(f'\nExport complete ({time.time() - t:.1f}s)' + f"\nResults saved to {colorstr('bold', file.parent.resolve())}" + f"\nDetect: python {dir / ('detect.py' if det else 'predict.py')} --weights {f[-1]} {h}" + f"\nValidate: python {dir / 'val.py'} --weights {f[-1]} {h}" + f"\nPyTorch Hub: model = torch.hub.load('ultralytics/yolov5', 'custom', '{f[-1]}') {s}" + f'\nVisualize: https://netron.app') + return f # return list of exported files/dirs + + +def parse_opt(known=False): + parser = argparse.ArgumentParser() + parser.add_argument('--data', type=str, default=ROOT / 'data/coco128.yaml', help='dataset.yaml path') + parser.add_argument('--weights', nargs='+', type=str, default=ROOT / 'yolov5s.pt', help='model.pt path(s)') + parser.add_argument('--imgsz', '--img', '--img-size', nargs='+', type=int, default=[640, 640], help='image (h, w)') + parser.add_argument('--batch-size', type=int, default=1, help='batch size') + parser.add_argument('--device', default='cpu', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--half', action='store_true', help='FP16 half-precision export') + parser.add_argument('--inplace', action='store_true', help='set YOLOv5 Detect() inplace=True') + parser.add_argument('--keras', action='store_true', help='TF: use Keras') + parser.add_argument('--optimize', action='store_true', help='TorchScript: optimize for mobile') + parser.add_argument('--int8', action='store_true', help='CoreML/TF/OpenVINO INT8 quantization') + parser.add_argument('--dynamic', action='store_true', help='ONNX/TF/TensorRT: dynamic axes') + parser.add_argument('--simplify', action='store_true', help='ONNX: simplify model') + parser.add_argument('--opset', type=int, default=17, help='ONNX: opset version') + parser.add_argument('--verbose', action='store_true', help='TensorRT: verbose log') + parser.add_argument('--workspace', type=int, default=4, help='TensorRT: workspace size (GB)') + parser.add_argument('--nms', action='store_true', help='TF: add NMS to model') + parser.add_argument('--agnostic-nms', action='store_true', help='TF: add agnostic NMS to model') + parser.add_argument('--topk-per-class', type=int, default=100, help='TF.js NMS: topk per class to keep') + parser.add_argument('--topk-all', type=int, default=100, help='TF.js NMS: topk for all classes to keep') + parser.add_argument('--iou-thres', type=float, default=0.45, help='TF.js NMS: IoU threshold') + parser.add_argument('--conf-thres', type=float, default=0.25, help='TF.js NMS: confidence threshold') + parser.add_argument( + '--include', + nargs='+', + default=['torchscript'], + help='torchscript, onnx, openvino, engine, coreml, saved_model, pb, tflite, edgetpu, tfjs, paddle') + opt = parser.parse_known_args()[0] if known else parser.parse_args() + print_args(vars(opt)) + return opt + + +def main(opt): + for opt.weights in (opt.weights if isinstance(opt.weights, list) else [opt.weights]): + run(**vars(opt)) + + +if __name__ == '__main__': + opt = parse_opt() + main(opt) diff --git a/featureVal.py b/featureVal.py new file mode 100644 index 0000000..dd9fe3a --- /dev/null +++ b/featureVal.py @@ -0,0 +1,285 @@ +# -*- coding: utf-8 -*- +""" +Created on Fri May 31 14:50:21 2024 + +@author: ym +""" +import cv2 +import numpy as np +import torch +from scipy.spatial.distance import cdist +from tracking.trackers.reid.config import config as ReIDConfig +from tracking.trackers.reid.reid_interface import ReIDInterface +ReIDEncoder = ReIDInterface(ReIDConfig) + +def read_data_file(datapath): + + with open(datapath, 'r') as file: + lines = file.readlines() + Videos = [] + FrameBoxes, FrameFeats = [], [] + boxes, feats = [], [] + + bboxes, ffeats = [], [] + timestamp = [] + t1 = None + for line in lines: + if line.find('CameraId') >= 0: + t = int(line.split(',')[1].split(':')[1]) + timestamp.append(t) + + if len(boxes) and len(feats): + FrameBoxes.append(np.array(boxes, dtype = np.float32)) + FrameFeats.append(np.array(feats, dtype = np.float32)) + + boxes, feats = [], [] + + if t1 and t - t1 > 1e4: + Videos.append((FrameBoxes, FrameFeats)) + FrameBoxes, FrameFeats = [], [] + t1 = int(line.split(',')[1].split(':')[1]) + + if line.find('box') >= 0: + box = line.split(':', )[1].split(',')[:-1] + boxes.append(box) + bboxes.append(boxes) + + + if line.find('feat') >= 0: + feat = line.split(':', )[1].split(',')[:-1] + feats.append(feat) + ffeats.append(feat) + + + + + FrameBoxes.append(np.array(boxes, dtype = np.float32)) + FrameFeats.append(np.array(feats, dtype = np.float32)) + Videos.append((FrameBoxes, FrameFeats)) + + TimeStamp = np.array(timestamp, dtype = np.float32) + DimesDiff = np.diff((timestamp)) + + return Videos + +def inference_image(image, detections): + H, W, _ = np.shape(image) + imgs = [] + batch_patches = [] + patches = [] + for d in range(np.size(detections, 0)): + tlbr = detections[d, :4].astype(np.int_) + tlbr[0] = max(0, tlbr[0]) + tlbr[1] = max(0, tlbr[1]) + tlbr[2] = min(W - 1, tlbr[2]) + tlbr[3] = min(H - 1, tlbr[3]) + img1 = image[tlbr[1]:tlbr[3], tlbr[0]:tlbr[2], :] + + img = img1[:, :, ::-1].copy() # the model expects RGB inputs + patch = ReIDEncoder.transform(img) + + imgs.append(img1) + # patch = patch.to(device=self.device).half() + if str(ReIDEncoder.device) != "cpu": + patch = patch.to(device=ReIDEncoder.device).half() + else: + patch = patch.to(device=ReIDEncoder.device) + + patches.append(patch) + if (d + 1) % ReIDEncoder.batch_size == 0: + patches = torch.stack(patches, dim=0) + batch_patches.append(patches) + patches = [] + + if len(patches): + patches = torch.stack(patches, dim=0) + batch_patches.append(patches) + + features = np.zeros((0, ReIDEncoder.embedding_size)) + for patches in batch_patches: + pred = ReIDEncoder.model(patches) + pred[torch.isinf(pred)] = 1.0 + feat = pred.cpu().data.numpy() + features = np.vstack((features, feat)) + + return imgs, features + +def readimg(): + imgpath = r"D:\datasets\ym\Img_ResnetData\result\0.png" + image = cv2.imread(imgpath) + + img = cv2.resize(image, (224, 224)) + + cv2.imwrite('0_224x224.jpg', img) + + +def readdata(datapath): + + with open(datapath, 'r') as file: + lines = file.readlines() + dlist = lines[0].split(',') + dfloat = [float(d) for d in dlist] + afeat = np.array(dfloat).reshape(1, -1) + + return afeat +def readrawimg(datapath): + with open(datapath, 'r') as file: + llines = file.readlines() + imgs = [] + + row = 224 + + for i in range(8): + lines = llines[i*224 : (i+1)*224] + + + + img = np.empty((224, 224, 0), dtype=np.float32) + imgr = np.empty((0, 224), dtype=np.float32) + imgg = np.empty((0, 224), dtype=np.float32) + imgb = np.empty((0, 224), dtype=np.float32) + + for line in lines: + dlist = line.split(' ')[0:224] + + img_r = np.array([float(s.split(',')[0]) for s in dlist], dtype=np.float32).reshape(1, -1) + img_g = np.array([float(s.split(',')[1]) for s in dlist], dtype=np.float32).reshape(1, -1) + img_b = np.array([float(s.split(',')[2]) for s in dlist], dtype=np.float32).reshape(1, -1) + + # img_r = [float(s.split(',')[0]) for s in dlist if len(s.split(',')[0].encode('utf-8')) == 4] + # img_g = [float(s.split(',')[1]) for s in dlist if len(s.split(',')[1].encode('utf-8')) == 4] + # img_b = [float(s.split(',')[2]) for s in dlist if len(s.split(',')[2].encode('utf-8')) == 4] + + imgr = np.concatenate((imgr, img_r), axis=0) + imgg = np.concatenate((imgg, img_g), axis=0) + imgb = np.concatenate((imgb, img_b), axis=0) + + imgr = imgr[:, :, None] + imgg = imgg[:, :, None] + imgb = imgb[:, :, None] + + img = np.concatenate((imgb, imgg, imgr), axis=2).astype(np.uint8) + + imgs.append(img) + + return imgs + + + + + +def inference(image): + + patches = [] + + image = image[:, :, ::-1].copy() # the model expects RGB inputs + patch = ReIDEncoder.transform(image) + + patch = patch.to(device=ReIDEncoder.device) + + patches.append(patch) + patches = torch.stack(patches, dim=0) + pred = ReIDEncoder.model(patches) + pred[torch.isinf(pred)] = 1.0 + bfeat = pred.cpu().data.numpy() + + return bfeat + + + +def test_img_feat(): + # datapath = r"D:\datasets\ym\Img_ResnetData\aa\aa.txt" + # afeat = readdata(datapath) + + imgpath = r"D:\datasets\ym\Img_ResnetData\aa\aa.jpg" + img = cv2.imread(imgpath) + bfeat = inference(img) + + + datapath = r"D:\datasets\ym\Img_ResnetData\rawimg\7.txt" + afeat = readdata(datapath) + + rawpath = r"D:\datasets\ym\Img_ResnetData\rawimg\28950640607_mat_rgb" + imgx = readrawimg(rawpath) + cv2.imwrite("rawimg.png", imgx[7]) + bfeatx = inference(imgx[7]) + + cost_matrix = 1 - np.maximum(0.0, cdist(afeat, bfeatx, 'cosine')) + + imgpath1 = r"D:\datasets\ym\Img_ResnetData\result\0_224x224.png" + img1 = cv2.imread(imgpath1) + bfeat1 = inference(img1) + + aafeat = afeat / np.linalg.norm(afeat, ord=2, axis=1, keepdims=True) + bbfeat = bfeat / np.linalg.norm(bfeat, ord=2, axis=1, keepdims=True) + + cost_matrix = 1 - np.maximum(0.0, cdist(aafeat, bbfeat, 'cosine')) + + print("Done!!!") + +def main(): + imgpath = r"D:\datasets\ym\Img_ResnetData\20240531-103547_0354b1cb-53fa-48de-86cd-ac3c5b127ada_6921168593576\3568800050000_0.jpeg" + datapath = r"D:\datasets\ym\Img_ResnetData\20240531-103547_0354b1cb-53fa-48de-86cd-ac3c5b127ada_6921168593576\0_tracker_inout.data" + savepath = r"D:\datasets\ym\Img_ResnetData\result" + + image = cv2.imread(imgpath) + + + Videos = read_data_file(datapath) + + bboxes, afeats = Videos[0][0][0], Videos[0][1][0] + imgs, bfeats = inference_image(image, bboxes) + + + aafeats = afeats / np.linalg.norm(afeats, ord=2, axis=1, keepdims=True) + bbfeats = bfeats / np.linalg.norm(bfeats, ord=2, axis=1, keepdims=True) + + + cost_matrix = 1 - np.maximum(0.0, cdist(aafeats, bbfeats, 'cosine')) + + + for i, img in enumerate(imgs): + cv2.imwrite(savepath + f"\{i}.png", img) + + + + print("Done!!!!") + + + + +if __name__ == '__main__': + # main() + + # readimg() + + test_img_feat() + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/hands/__pycache__/hand_inference.cpython-312.pyc b/hands/__pycache__/hand_inference.cpython-312.pyc new file mode 100644 index 0000000..35e1f47 Binary files /dev/null and b/hands/__pycache__/hand_inference.cpython-312.pyc differ diff --git a/hands/__pycache__/hand_inference.cpython-39.pyc b/hands/__pycache__/hand_inference.cpython-39.pyc new file mode 100644 index 0000000..0960b17 Binary files /dev/null and b/hands/__pycache__/hand_inference.cpython-39.pyc differ diff --git a/hands/hand_inference.py b/hands/hand_inference.py new file mode 100644 index 0000000..3328861 --- /dev/null +++ b/hands/hand_inference.py @@ -0,0 +1,107 @@ +import cv2 +import os +import numpy as np +import mediapipe as mp + +class hand_pose: + def __init__(self, min_dc=0.45, min_tc=0.45, max_nh=1): + self.mp_drawing = mp.solutions.drawing_utils + self.mp_hands = mp.solutions.hands + self.hands = self.mp_hands.Hands( + static_image_mode=False, + max_num_hands=max_nh, + min_detection_confidence=min_dc, + min_tracking_confidence=min_tc + ) + + @staticmethod + def img_show(img): + cv2.imshow('MediaPipe Hands', img) + cv2.waitKey(1) + + def draw(self, img): + hand_locals = [] + h_re = self.hands.process(img) + if h_re.multi_hand_landmarks: + + for hand in h_re.multi_handedness: + hand_position = 'Left' if hand.classification[0].label == "Left" else 'Right' + + hand_local = [] + for hand_landmarks in h_re.multi_hand_landmarks: + self.mp_drawing.draw_landmarks(img, + hand_landmarks, + self.mp_hands.HAND_CONNECTIONS) + imgshow = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) + # self.img_show(imgshow) + cv2.imwrite(f"./images/{k}.png", imgshow) + + '''获取手部关键点坐标''' + for i in range(21): + x = hand_landmarks.landmark[i].x * img.shape[1] + y = hand_landmarks.landmark[i].y * img.shape[0] + hand_local.append((x, y)) + + hand_locals.append(hand_local) + return hand_locals + + def get_hand_local(self, track, image): + '''tracks: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index]''' + H, W, _ = np.shape(image) + tlbr = track[:4].astype(np.int_) + x1 = max(0, tlbr[0]-50) + y1 = max(0, tlbr[1]-50) + x2 = min(W - 1, tlbr[2]+50) + y2 = min(H - 1, tlbr[3]+50) + img = image[y1:y2, x1:x2, :] + imgshow = img.copy() + + # hand_local = np.empty((0,2), dtype = np.int_) + min_x, max_x = W-1, 0 + min_y, max_y = H-1, 0 + + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + h_re = self.hands.process(img) + if h_re.multi_hand_landmarks: + for hand_landmarks in h_re.multi_hand_landmarks: + for i in range(21): + x = hand_landmarks.landmark[i].x * img.shape[1] + y = hand_landmarks.landmark[i].y * img.shape[0] + # hand_local = np.concatenate([hand_local, np.array([[x1+x, y1+y]]).astype(np.int_)], axis=0) + + if min_x>x1+x: min_x=x1+x + if max_xy1+y: min_y=y1+y + if max_y=0 and cls==0: + color = colors(int(cls), True) + elif id >=0 and cls!=0: + color = colors(int(id), True) + else: + color = colors(19, True) # 19为调色板的最后一个元素 + + annotator.box_label(xyxy, label, color=color) + + # Save results (image and video with tracking) + im0 = annotator.result() + p = Path(path) # to Path + if save_img: + imgpath = str(save_dir/p.stem) + f"_{dataset.count}.png" + cv2.imwrite(Path(imgpath), im0) + if vid_path[i] != save_path_video: # new video + vid_path[i] = save_path_video + if isinstance(vid_writer[i], cv2.VideoWriter): + vid_writer[i].release() # release previous video writer + fps, w, h = 30, im0.shape[1], im0.shape[0] + vpath = str(Path(save_path_video).with_suffix('.mp4')) + vid_writer[i] = cv2.VideoWriter(vpath, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h)) + + vid_writer[i].write(im0) + + LOGGER.info(f"{s}{'' if len(det) else '(no detections), '}{dt[1].dt * 1E3:.1f}ms") + + for v in vid_writer: + v.release() + + if track_boxes.size == 0: + return CamerType, [] + + save_path_np = os.path.join(str(fulldir), f"{EventName}_{CamerType}") + np.save(save_path_np, track_boxes) + + # Print results + t = tuple(x.t / seen * 1E3 for x in dt) # speeds per image + LOGGER.info(f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {(1, 3, *imgsz)}' % t) + if save_txt or save_img: + s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else '' + LOGGER.info(f"Results saved to {colorstr('bold', save_dir)}{s}") + if update: + strip_optimizer(weights[0]) # update model (to fix SourceChangeWarning) + + + return CamerType, track_boxes + + +def parse_opt(): + modelpath = ROOT / 'ckpts/best_cls10_0906.pt' # 'ckpts/best_15000_0908.pt', 'ckpts/yolov5s.pt', 'ckpts/best_20000_cls30.pt, best_yolov5m_250000' + + parser = argparse.ArgumentParser() + parser.add_argument('--weights', nargs='+', type=str, default=modelpath, help='model path or triton URL') # 'yolov5s.pt', best_15000_0908.pt + parser.add_argument('--source', type=str, default='', help='file/dir/URL/glob/screen/0(webcam)') # images, videos + parser.add_argument('--data', type=str, default=ROOT / 'data/coco128.yaml', help='(optional) dataset.yaml path') + parser.add_argument('--imgsz', '--img', '--img-size', nargs='+', type=int, default=[640], help='inference size h,w') + parser.add_argument('--conf-thres', type=float, default=0.25, help='confidence threshold') + parser.add_argument('--iou-thres', type=float, default=0.45, help='NMS IoU threshold') + parser.add_argument('--max-det', type=int, default=1000, help='maximum detections per image') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--view-img', action='store_true', help='show results') + parser.add_argument('--save-txt', action='store_true', help='save results to *.txt') + parser.add_argument('--save-csv', action='store_true', help='save results in CSV format') + parser.add_argument('--save-conf', action='store_true', help='save confidences in --save-txt labels') + parser.add_argument('--save-crop', action='store_true', help='save cropped prediction boxes') + parser.add_argument('--nosave', action='store_true', help='do not save images/videos') + parser.add_argument('--classes', nargs='+', type=int, help='filter by class: --classes 0, or --classes 0 2 3') + parser.add_argument('--agnostic-nms', action='store_true', help='class-agnostic NMS') + parser.add_argument('--augment', action='store_true', help='augmented inference') + parser.add_argument('--visualize', action='store_true', help='visualize features') + parser.add_argument('--update', action='store_true', help='update all models') + parser.add_argument('--project', default=ROOT / 'runs/detect', help='save results to project/name') + parser.add_argument('--name', default='exp', help='save results to project/name') + parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment') + parser.add_argument('--line-thickness', default=3, type=int, help='bounding box thickness (pixels)') + parser.add_argument('--hide-labels', default=False, action='store_true', help='hide labels') + parser.add_argument('--hide-conf', default=False, action='store_true', help='hide confidences') + parser.add_argument('--half', action='store_true', help='use FP16 half-precision inference') + parser.add_argument('--dnn', action='store_true', help='use OpenCV DNN for ONNX inference') + parser.add_argument('--vid-stride', type=int, default=1, help='video frame-rate stride') + opt = parser.parse_args() + opt.imgsz *= 2 if len(opt.imgsz) == 1 else 1 # expand + print_args(vars(opt)) + return opt + +def run_yolo(eventdir, savedir): + + opt = parse_opt() + optdict = vars(opt) + optdict["project"] = savedir + optdict["source"] = eventdir + run(**vars(opt)) + + + + + diff --git a/imgs_to_video.py b/imgs_to_video.py new file mode 100644 index 0000000..db016d6 --- /dev/null +++ b/imgs_to_video.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- +""" +Created on Tue Jan 30 19:15:05 2024 + +@author: ym +""" +import cv2 +import os +import glob +IMG_FORMATS = "bmp", "dng", "jpeg", "jpg", "mpo", "png", "tif", "tiff", "webp", "pfm" # include image suffixes +VID_FORMATS = "asf", "avi", "gif", "m4v", "mkv", "mov", "mp4", "mpeg", "mpg", "ts", "wmv" # include video suffixes + + +def for_test(): + save_path = video_path + img_path + + fps, w, h = 10, 1024, 1280 + cap = cv2.VideoWriter(save_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h)) + + pathx = path + img_path + imgfiles = [f for f in os.listdir(pathx) if not f.find("_cut") != -1] + + imgfiles.sort(key = lambda x: int(x[:-5])) + imgpaths = [] + for imgfile in imgfiles: + imgpaths.append(os.path.join(pathx, imgfile)) + + center = (1280/2, 1024/2) + rotate_matrix = cv2.getRotationMatrix2D(center=center, angle=-90, scale=1) + k = 0 + for ipath in imgpaths: + img = cv2.imread(ipath) + rotated_image = cv2.warpAffine(src=img, M=rotate_matrix, dsize=(w, h)) + cap.write(rotated_image) + print("Have imgs") + +def test_1(): + + # name = os.path.split(img_path)[-1] + # save_path = video_path + name + '.mp4' + + save_path = video_path + img_path + + + + fps, w, h = 10, 1024, 1280 + cap = cv2.VideoWriter(save_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h)) + + pathx = path + img_path + imgfiles = [f for f in os.listdir(pathx) if not f.find("_cut") != -1] + + imgfiles.sort(key = lambda x: int(x[:-5])) + imgpaths = [] + for imgfile in imgfiles: + imgpaths.append(os.path.join(pathx, imgfile)) + + + + + # ipaths = [os.path.join(pathx, f) for f in os.listdir(pathx) if not f.find("_cut") != -1] + # ipaths = [] + # for f in os.listdir(pathx): + # if not f.find('_cut'): + # ipaths.append(os.path.join(pathx, f)) + # ipaths.sort(key = lambda x: int(x.split('_')[-2])) + + + k = 0 + for ipath in imgpaths: + img = cv2.imread(ipath) + cap.write(img) + + + k += 1 + + cap.release() + + print(img_path + f" have imgs: {k}") + +def img2video(imgpath): + if not os.path.isdir(imgpath): + return + + files = [] + files.extend(sorted(glob.glob(os.path.join(imgpath, "*.*")))) + images = [x for x in files if x.split(".")[-1].lower() in IMG_FORMATS] + + h, w = cv2.imread(images[0]).shape[:2] + fps = 25 + + vidpath = imgpath + '.mp4' + cap = cv2.VideoWriter(vidpath, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h)) + for p in images: + img = cv2.imread(p) + cap.write(img) + cap.release() + + +def main(): + imgpath = r"D:\work\result\202503251112_v10s_result" + + img2video(imgpath) + + + +if __name__ == "__main__": + main() + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/models/__init__.py b/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/models/__pycache__/__init__.cpython-312.pyc b/models/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..0c0f867 Binary files /dev/null and b/models/__pycache__/__init__.cpython-312.pyc differ diff --git a/models/__pycache__/__init__.cpython-39.pyc b/models/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..8360ac3 Binary files /dev/null and b/models/__pycache__/__init__.cpython-39.pyc differ diff --git a/models/__pycache__/common.cpython-312.pyc b/models/__pycache__/common.cpython-312.pyc new file mode 100644 index 0000000..b614dff Binary files /dev/null and b/models/__pycache__/common.cpython-312.pyc differ diff --git a/models/__pycache__/common.cpython-39.pyc b/models/__pycache__/common.cpython-39.pyc new file mode 100644 index 0000000..c088794 Binary files /dev/null and b/models/__pycache__/common.cpython-39.pyc differ diff --git a/models/__pycache__/experimental.cpython-312.pyc b/models/__pycache__/experimental.cpython-312.pyc new file mode 100644 index 0000000..c9c9f6f Binary files /dev/null and b/models/__pycache__/experimental.cpython-312.pyc differ diff --git a/models/__pycache__/experimental.cpython-39.pyc b/models/__pycache__/experimental.cpython-39.pyc new file mode 100644 index 0000000..2423fd1 Binary files /dev/null and b/models/__pycache__/experimental.cpython-39.pyc differ diff --git a/models/__pycache__/yolo.cpython-312.pyc b/models/__pycache__/yolo.cpython-312.pyc new file mode 100644 index 0000000..072f2b0 Binary files /dev/null and b/models/__pycache__/yolo.cpython-312.pyc differ diff --git a/models/__pycache__/yolo.cpython-39.pyc b/models/__pycache__/yolo.cpython-39.pyc new file mode 100644 index 0000000..66f0abf Binary files /dev/null and b/models/__pycache__/yolo.cpython-39.pyc differ diff --git a/models/common.py b/models/common.py new file mode 100644 index 0000000..75cc4e9 --- /dev/null +++ b/models/common.py @@ -0,0 +1,883 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Common modules +""" + +import ast +import contextlib +import json +import math +import platform +import warnings +import zipfile +from collections import OrderedDict, namedtuple +from copy import copy +from pathlib import Path +from urllib.parse import urlparse + +import cv2 +import numpy as np +import pandas as pd +import requests +import torch +import torch.nn as nn +from PIL import Image +from torch.cuda import amp + +# Import 'ultralytics' package or install if if missing +try: + import ultralytics + + assert hasattr(ultralytics, '__version__') # verify package is not directory +except (ImportError, AssertionError): + import os + + os.system('pip install -U ultralytics') + import ultralytics + +from ultralytics.utils.plotting import Annotator, colors, save_one_box + +from utils import TryExcept +from utils.dataloaders import exif_transpose, letterbox +from utils.general import (LOGGER, ROOT, Profile, check_requirements, check_suffix, check_version, colorstr, + increment_path, is_jupyter, make_divisible, non_max_suppression, scale_boxes, xywh2xyxy, + xyxy2xywh, yaml_load) +from utils.torch_utils import copy_attr, smart_inference_mode + + +def autopad(k, p=None, d=1): # kernel, padding, dilation + # Pad to 'same' shape outputs + if d > 1: + k = d * (k - 1) + 1 if isinstance(k, int) else [d * (x - 1) + 1 for x in k] # actual kernel-size + if p is None: + p = k // 2 if isinstance(k, int) else [x // 2 for x in k] # auto-pad + return p + + +class Conv(nn.Module): + # Standard convolution with args(ch_in, ch_out, kernel, stride, padding, groups, dilation, activation) + default_act = nn.SiLU() # default activation + + def __init__(self, c1, c2, k=1, s=1, p=None, g=1, d=1, act=True): + super().__init__() + self.conv = nn.Conv2d(c1, c2, k, s, autopad(k, p, d), groups=g, dilation=d, bias=False) + self.bn = nn.BatchNorm2d(c2) + self.act = self.default_act if act is True else act if isinstance(act, nn.Module) else nn.Identity() + + def forward(self, x): + return self.act(self.bn(self.conv(x))) + + def forward_fuse(self, x): + return self.act(self.conv(x)) + + +class DWConv(Conv): + # Depth-wise convolution + def __init__(self, c1, c2, k=1, s=1, d=1, act=True): # ch_in, ch_out, kernel, stride, dilation, activation + super().__init__(c1, c2, k, s, g=math.gcd(c1, c2), d=d, act=act) + + +class DWConvTranspose2d(nn.ConvTranspose2d): + # Depth-wise transpose convolution + def __init__(self, c1, c2, k=1, s=1, p1=0, p2=0): # ch_in, ch_out, kernel, stride, padding, padding_out + super().__init__(c1, c2, k, s, p1, p2, groups=math.gcd(c1, c2)) + + +class TransformerLayer(nn.Module): + # Transformer layer https://arxiv.org/abs/2010.11929 (LayerNorm layers removed for better performance) + def __init__(self, c, num_heads): + super().__init__() + self.q = nn.Linear(c, c, bias=False) + self.k = nn.Linear(c, c, bias=False) + self.v = nn.Linear(c, c, bias=False) + self.ma = nn.MultiheadAttention(embed_dim=c, num_heads=num_heads) + self.fc1 = nn.Linear(c, c, bias=False) + self.fc2 = nn.Linear(c, c, bias=False) + + def forward(self, x): + x = self.ma(self.q(x), self.k(x), self.v(x))[0] + x + x = self.fc2(self.fc1(x)) + x + return x + + +class TransformerBlock(nn.Module): + # Vision Transformer https://arxiv.org/abs/2010.11929 + def __init__(self, c1, c2, num_heads, num_layers): + super().__init__() + self.conv = None + if c1 != c2: + self.conv = Conv(c1, c2) + self.linear = nn.Linear(c2, c2) # learnable position embedding + self.tr = nn.Sequential(*(TransformerLayer(c2, num_heads) for _ in range(num_layers))) + self.c2 = c2 + + def forward(self, x): + if self.conv is not None: + x = self.conv(x) + b, _, w, h = x.shape + p = x.flatten(2).permute(2, 0, 1) + return self.tr(p + self.linear(p)).permute(1, 2, 0).reshape(b, self.c2, w, h) + + +class Bottleneck(nn.Module): + # Standard bottleneck + def __init__(self, c1, c2, shortcut=True, g=1, e=0.5): # ch_in, ch_out, shortcut, groups, expansion + super().__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = Conv(c_, c2, 3, 1, g=g) + self.add = shortcut and c1 == c2 + + def forward(self, x): + return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x)) + + +class BottleneckCSP(nn.Module): + # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = nn.Conv2d(c1, c_, 1, 1, bias=False) + self.cv3 = nn.Conv2d(c_, c_, 1, 1, bias=False) + self.cv4 = Conv(2 * c_, c2, 1, 1) + self.bn = nn.BatchNorm2d(2 * c_) # applied to cat(cv2, cv3) + self.act = nn.SiLU() + self.m = nn.Sequential(*(Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n))) + + def forward(self, x): + y1 = self.cv3(self.m(self.cv1(x))) + y2 = self.cv2(x) + return self.cv4(self.act(self.bn(torch.cat((y1, y2), 1)))) + + +class CrossConv(nn.Module): + # Cross Convolution Downsample + def __init__(self, c1, c2, k=3, s=1, g=1, e=1.0, shortcut=False): + # ch_in, ch_out, kernel, stride, groups, expansion, shortcut + super().__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = Conv(c1, c_, (1, k), (1, s)) + self.cv2 = Conv(c_, c2, (k, 1), (s, 1), g=g) + self.add = shortcut and c1 == c2 + + def forward(self, x): + return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x)) + + +class C3(nn.Module): + # CSP Bottleneck with 3 convolutions + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = Conv(c1, c_, 1, 1) + self.cv3 = Conv(2 * c_, c2, 1) # optional act=FReLU(c2) + self.m = nn.Sequential(*(Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n))) + + def forward(self, x): + return self.cv3(torch.cat((self.m(self.cv1(x)), self.cv2(x)), 1)) + + +class C3x(C3): + # C3 module with cross-convolutions + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): + super().__init__(c1, c2, n, shortcut, g, e) + c_ = int(c2 * e) + self.m = nn.Sequential(*(CrossConv(c_, c_, 3, 1, g, 1.0, shortcut) for _ in range(n))) + + +class C3TR(C3): + # C3 module with TransformerBlock() + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): + super().__init__(c1, c2, n, shortcut, g, e) + c_ = int(c2 * e) + self.m = TransformerBlock(c_, c_, 4, n) + + +class C3SPP(C3): + # C3 module with SPP() + def __init__(self, c1, c2, k=(5, 9, 13), n=1, shortcut=True, g=1, e=0.5): + super().__init__(c1, c2, n, shortcut, g, e) + c_ = int(c2 * e) + self.m = SPP(c_, c_, k) + + +class C3Ghost(C3): + # C3 module with GhostBottleneck() + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): + super().__init__(c1, c2, n, shortcut, g, e) + c_ = int(c2 * e) # hidden channels + self.m = nn.Sequential(*(GhostBottleneck(c_, c_) for _ in range(n))) + + +class SPP(nn.Module): + # Spatial Pyramid Pooling (SPP) layer https://arxiv.org/abs/1406.4729 + def __init__(self, c1, c2, k=(5, 9, 13)): + super().__init__() + c_ = c1 // 2 # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = Conv(c_ * (len(k) + 1), c2, 1, 1) + self.m = nn.ModuleList([nn.MaxPool2d(kernel_size=x, stride=1, padding=x // 2) for x in k]) + + def forward(self, x): + x = self.cv1(x) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') # suppress torch 1.9.0 max_pool2d() warning + return self.cv2(torch.cat([x] + [m(x) for m in self.m], 1)) + + +class SPPF(nn.Module): + # Spatial Pyramid Pooling - Fast (SPPF) layer for YOLOv5 by Glenn Jocher + def __init__(self, c1, c2, k=5): # equivalent to SPP(k=(5, 9, 13)) + super().__init__() + c_ = c1 // 2 # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = Conv(c_ * 4, c2, 1, 1) + self.m = nn.MaxPool2d(kernel_size=k, stride=1, padding=k // 2) + + def forward(self, x): + x = self.cv1(x) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') # suppress torch 1.9.0 max_pool2d() warning + y1 = self.m(x) + y2 = self.m(y1) + return self.cv2(torch.cat((x, y1, y2, self.m(y2)), 1)) + + +class Focus(nn.Module): + # Focus wh information into c-space + def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True): # ch_in, ch_out, kernel, stride, padding, groups + super().__init__() + self.conv = Conv(c1 * 4, c2, k, s, p, g, act=act) + # self.contract = Contract(gain=2) + + def forward(self, x): # x(b,c,w,h) -> y(b,4c,w/2,h/2) + return self.conv(torch.cat((x[..., ::2, ::2], x[..., 1::2, ::2], x[..., ::2, 1::2], x[..., 1::2, 1::2]), 1)) + # return self.conv(self.contract(x)) + + +class GhostConv(nn.Module): + # Ghost Convolution https://github.com/huawei-noah/ghostnet + def __init__(self, c1, c2, k=1, s=1, g=1, act=True): # ch_in, ch_out, kernel, stride, groups + super().__init__() + c_ = c2 // 2 # hidden channels + self.cv1 = Conv(c1, c_, k, s, None, g, act=act) + self.cv2 = Conv(c_, c_, 5, 1, None, c_, act=act) + + def forward(self, x): + y = self.cv1(x) + return torch.cat((y, self.cv2(y)), 1) + + +class GhostBottleneck(nn.Module): + # Ghost Bottleneck https://github.com/huawei-noah/ghostnet + def __init__(self, c1, c2, k=3, s=1): # ch_in, ch_out, kernel, stride + super().__init__() + c_ = c2 // 2 + self.conv = nn.Sequential( + GhostConv(c1, c_, 1, 1), # pw + DWConv(c_, c_, k, s, act=False) if s == 2 else nn.Identity(), # dw + GhostConv(c_, c2, 1, 1, act=False)) # pw-linear + self.shortcut = nn.Sequential(DWConv(c1, c1, k, s, act=False), Conv(c1, c2, 1, 1, + act=False)) if s == 2 else nn.Identity() + + def forward(self, x): + return self.conv(x) + self.shortcut(x) + + +class Contract(nn.Module): + # Contract width-height into channels, i.e. x(1,64,80,80) to x(1,256,40,40) + def __init__(self, gain=2): + super().__init__() + self.gain = gain + + def forward(self, x): + b, c, h, w = x.size() # assert (h / s == 0) and (W / s == 0), 'Indivisible gain' + s = self.gain + x = x.view(b, c, h // s, s, w // s, s) # x(1,64,40,2,40,2) + x = x.permute(0, 3, 5, 1, 2, 4).contiguous() # x(1,2,2,64,40,40) + return x.view(b, c * s * s, h // s, w // s) # x(1,256,40,40) + + +class Expand(nn.Module): + # Expand channels into width-height, i.e. x(1,64,80,80) to x(1,16,160,160) + def __init__(self, gain=2): + super().__init__() + self.gain = gain + + def forward(self, x): + b, c, h, w = x.size() # assert C / s ** 2 == 0, 'Indivisible gain' + s = self.gain + x = x.view(b, s, s, c // s ** 2, h, w) # x(1,2,2,16,80,80) + x = x.permute(0, 3, 4, 1, 5, 2).contiguous() # x(1,16,80,2,80,2) + return x.view(b, c // s ** 2, h * s, w * s) # x(1,16,160,160) + + +class Concat(nn.Module): + # Concatenate a list of tensors along dimension + def __init__(self, dimension=1): + super().__init__() + self.d = dimension + + def forward(self, x): + return torch.cat(x, self.d) + + +class DetectMultiBackend(nn.Module): + # YOLOv5 MultiBackend class for python inference on various backends + def __init__(self, weights='yolov5s.pt', device=torch.device('cpu'), dnn=False, data=None, fp16=False, fuse=True): + # Usage: + # PyTorch: weights = *.pt + # TorchScript: *.torchscript + # ONNX Runtime: *.onnx + # ONNX OpenCV DNN: *.onnx --dnn + # OpenVINO: *_openvino_model + # CoreML: *.mlmodel + # TensorRT: *.engine + # TensorFlow SavedModel: *_saved_model + # TensorFlow GraphDef: *.pb + # TensorFlow Lite: *.tflite + # TensorFlow Edge TPU: *_edgetpu.tflite + # PaddlePaddle: *_paddle_model + from models.experimental import attempt_download, attempt_load # scoped to avoid circular import + + super().__init__() + w = str(weights[0] if isinstance(weights, list) else weights) + pt, jit, onnx, xml, engine, coreml, saved_model, pb, tflite, edgetpu, tfjs, paddle, triton = self._model_type(w) + fp16 &= pt or jit or onnx or engine or triton # FP16 + nhwc = coreml or saved_model or pb or tflite or edgetpu # BHWC formats (vs torch BCWH) + stride = 32 # default stride + cuda = torch.cuda.is_available() and device.type != 'cpu' # use CUDA + if not (pt or triton): + w = attempt_download(w) # download if not local + + if pt: # PyTorch + model = attempt_load(weights if isinstance(weights, list) else w, device=device, inplace=True, fuse=fuse) + stride = max(int(model.stride.max()), 32) # model stride + names = model.module.names if hasattr(model, 'module') else model.names # get class names + model.half() if fp16 else model.float() + self.model = model # explicitly assign for to(), cpu(), cuda(), half() + elif jit: # TorchScript + LOGGER.info(f'Loading {w} for TorchScript inference...') + extra_files = {'config.txt': ''} # model metadata + model = torch.jit.load(w, _extra_files=extra_files, map_location=device) + model.half() if fp16 else model.float() + if extra_files['config.txt']: # load metadata dict + d = json.loads(extra_files['config.txt'], + object_hook=lambda d: { + int(k) if k.isdigit() else k: v + for k, v in d.items()}) + stride, names = int(d['stride']), d['names'] + elif dnn: # ONNX OpenCV DNN + LOGGER.info(f'Loading {w} for ONNX OpenCV DNN inference...') + check_requirements('opencv-python>=4.5.4') + net = cv2.dnn.readNetFromONNX(w) + elif onnx: # ONNX Runtime + LOGGER.info(f'Loading {w} for ONNX Runtime inference...') + check_requirements(('onnx', 'onnxruntime-gpu' if cuda else 'onnxruntime')) + import onnxruntime + providers = ['CUDAExecutionProvider', 'CPUExecutionProvider'] if cuda else ['CPUExecutionProvider'] + session = onnxruntime.InferenceSession(w, providers=providers) + output_names = [x.name for x in session.get_outputs()] + meta = session.get_modelmeta().custom_metadata_map # metadata + if 'stride' in meta: + stride, names = int(meta['stride']), eval(meta['names']) + elif xml: # OpenVINO + LOGGER.info(f'Loading {w} for OpenVINO inference...') + check_requirements('openvino>=2023.0') # requires openvino-dev: https://pypi.org/project/openvino-dev/ + from openvino.runtime import Core, Layout, get_batch + core = Core() + if not Path(w).is_file(): # if not *.xml + w = next(Path(w).glob('*.xml')) # get *.xml file from *_openvino_model dir + ov_model = core.read_model(model=w, weights=Path(w).with_suffix('.bin')) + if ov_model.get_parameters()[0].get_layout().empty: + ov_model.get_parameters()[0].set_layout(Layout('NCHW')) + batch_dim = get_batch(ov_model) + if batch_dim.is_static: + batch_size = batch_dim.get_length() + ov_compiled_model = core.compile_model(ov_model, device_name='AUTO') # AUTO selects best available device + stride, names = self._load_metadata(Path(w).with_suffix('.yaml')) # load metadata + elif engine: # TensorRT + LOGGER.info(f'Loading {w} for TensorRT inference...') + import tensorrt as trt # https://developer.nvidia.com/nvidia-tensorrt-download + check_version(trt.__version__, '7.0.0', hard=True) # require tensorrt>=7.0.0 + if device.type == 'cpu': + device = torch.device('cuda:0') + Binding = namedtuple('Binding', ('name', 'dtype', 'shape', 'data', 'ptr')) + logger = trt.Logger(trt.Logger.INFO) + with open(w, 'rb') as f, trt.Runtime(logger) as runtime: + model = runtime.deserialize_cuda_engine(f.read()) + context = model.create_execution_context() + bindings = OrderedDict() + output_names = [] + fp16 = False # default updated below + dynamic = False + for i in range(model.num_bindings): + name = model.get_binding_name(i) + dtype = trt.nptype(model.get_binding_dtype(i)) + if model.binding_is_input(i): + if -1 in tuple(model.get_binding_shape(i)): # dynamic + dynamic = True + context.set_binding_shape(i, tuple(model.get_profile_shape(0, i)[2])) + if dtype == np.float16: + fp16 = True + else: # output + output_names.append(name) + shape = tuple(context.get_binding_shape(i)) + im = torch.from_numpy(np.empty(shape, dtype=dtype)).to(device) + bindings[name] = Binding(name, dtype, shape, im, int(im.data_ptr())) + binding_addrs = OrderedDict((n, d.ptr) for n, d in bindings.items()) + batch_size = bindings['images'].shape[0] # if dynamic, this is instead max batch size + elif coreml: # CoreML + LOGGER.info(f'Loading {w} for CoreML inference...') + import coremltools as ct + model = ct.models.MLModel(w) + elif saved_model: # TF SavedModel + LOGGER.info(f'Loading {w} for TensorFlow SavedModel inference...') + import tensorflow as tf + keras = False # assume TF1 saved_model + model = tf.keras.models.load_model(w) if keras else tf.saved_model.load(w) + elif pb: # GraphDef https://www.tensorflow.org/guide/migrate#a_graphpb_or_graphpbtxt + LOGGER.info(f'Loading {w} for TensorFlow GraphDef inference...') + import tensorflow as tf + + def wrap_frozen_graph(gd, inputs, outputs): + x = tf.compat.v1.wrap_function(lambda: tf.compat.v1.import_graph_def(gd, name=''), []) # wrapped + ge = x.graph.as_graph_element + return x.prune(tf.nest.map_structure(ge, inputs), tf.nest.map_structure(ge, outputs)) + + def gd_outputs(gd): + name_list, input_list = [], [] + for node in gd.node: # tensorflow.core.framework.node_def_pb2.NodeDef + name_list.append(node.name) + input_list.extend(node.input) + return sorted(f'{x}:0' for x in list(set(name_list) - set(input_list)) if not x.startswith('NoOp')) + + gd = tf.Graph().as_graph_def() # TF GraphDef + with open(w, 'rb') as f: + gd.ParseFromString(f.read()) + frozen_func = wrap_frozen_graph(gd, inputs='x:0', outputs=gd_outputs(gd)) + elif tflite or edgetpu: # https://www.tensorflow.org/lite/guide/python#install_tensorflow_lite_for_python + try: # https://coral.ai/docs/edgetpu/tflite-python/#update-existing-tf-lite-code-for-the-edge-tpu + from tflite_runtime.interpreter import Interpreter, load_delegate + except ImportError: + import tensorflow as tf + Interpreter, load_delegate = tf.lite.Interpreter, tf.lite.experimental.load_delegate, + if edgetpu: # TF Edge TPU https://coral.ai/software/#edgetpu-runtime + LOGGER.info(f'Loading {w} for TensorFlow Lite Edge TPU inference...') + delegate = { + 'Linux': 'libedgetpu.so.1', + 'Darwin': 'libedgetpu.1.dylib', + 'Windows': 'edgetpu.dll'}[platform.system()] + interpreter = Interpreter(model_path=w, experimental_delegates=[load_delegate(delegate)]) + else: # TFLite + LOGGER.info(f'Loading {w} for TensorFlow Lite inference...') + interpreter = Interpreter(model_path=w) # load TFLite model + interpreter.allocate_tensors() # allocate + input_details = interpreter.get_input_details() # inputs + output_details = interpreter.get_output_details() # outputs + # load metadata + with contextlib.suppress(zipfile.BadZipFile): + with zipfile.ZipFile(w, 'r') as model: + meta_file = model.namelist()[0] + meta = ast.literal_eval(model.read(meta_file).decode('utf-8')) + stride, names = int(meta['stride']), meta['names'] + elif tfjs: # TF.js + raise NotImplementedError('ERROR: YOLOv5 TF.js inference is not supported') + elif paddle: # PaddlePaddle + LOGGER.info(f'Loading {w} for PaddlePaddle inference...') + check_requirements('paddlepaddle-gpu' if cuda else 'paddlepaddle') + import paddle.inference as pdi + if not Path(w).is_file(): # if not *.pdmodel + w = next(Path(w).rglob('*.pdmodel')) # get *.pdmodel file from *_paddle_model dir + weights = Path(w).with_suffix('.pdiparams') + config = pdi.Config(str(w), str(weights)) + if cuda: + config.enable_use_gpu(memory_pool_init_size_mb=2048, device_id=0) + predictor = pdi.create_predictor(config) + input_handle = predictor.get_input_handle(predictor.get_input_names()[0]) + output_names = predictor.get_output_names() + elif triton: # NVIDIA Triton Inference Server + LOGGER.info(f'Using {w} as Triton Inference Server...') + check_requirements('tritonclient[all]') + from utils.triton import TritonRemoteModel + model = TritonRemoteModel(url=w) + nhwc = model.runtime.startswith('tensorflow') + else: + raise NotImplementedError(f'ERROR: {w} is not a supported format') + + # class names + if 'names' not in locals(): + names = yaml_load(data)['names'] if data else {i: f'class{i}' for i in range(999)} + if names[0] == 'n01440764' and len(names) == 1000: # ImageNet + names = yaml_load(ROOT / 'data/ImageNet.yaml')['names'] # human-readable names + + self.__dict__.update(locals()) # assign all variables to self + + def forward(self, im, augment=False, visualize=False): + # YOLOv5 MultiBackend inference + b, ch, h, w = im.shape # batch, channel, height, width + if self.fp16 and im.dtype != torch.float16: + im = im.half() # to FP16 + if self.nhwc: + im = im.permute(0, 2, 3, 1) # torch BCHW to numpy BHWC shape(1,320,192,3) + + if self.pt: # PyTorch + y = self.model(im, augment=augment, visualize=visualize) if augment or visualize else self.model(im) + elif self.jit: # TorchScript + y = self.model(im) + elif self.dnn: # ONNX OpenCV DNN + im = im.cpu().numpy() # torch to numpy + self.net.setInput(im) + y = self.net.forward() + elif self.onnx: # ONNX Runtime + im = im.cpu().numpy() # torch to numpy + y = self.session.run(self.output_names, {self.session.get_inputs()[0].name: im}) + elif self.xml: # OpenVINO + im = im.cpu().numpy() # FP32 + y = list(self.ov_compiled_model(im).values()) + elif self.engine: # TensorRT + if self.dynamic and im.shape != self.bindings['images'].shape: + i = self.model.get_binding_index('images') + self.context.set_binding_shape(i, im.shape) # reshape if dynamic + self.bindings['images'] = self.bindings['images']._replace(shape=im.shape) + for name in self.output_names: + i = self.model.get_binding_index(name) + self.bindings[name].data.resize_(tuple(self.context.get_binding_shape(i))) + s = self.bindings['images'].shape + assert im.shape == s, f"input size {im.shape} {'>' if self.dynamic else 'not equal to'} max model size {s}" + self.binding_addrs['images'] = int(im.data_ptr()) + self.context.execute_v2(list(self.binding_addrs.values())) + y = [self.bindings[x].data for x in sorted(self.output_names)] + elif self.coreml: # CoreML + im = im.cpu().numpy() + im = Image.fromarray((im[0] * 255).astype('uint8')) + # im = im.resize((192, 320), Image.BILINEAR) + y = self.model.predict({'image': im}) # coordinates are xywh normalized + if 'confidence' in y: + box = xywh2xyxy(y['coordinates'] * [[w, h, w, h]]) # xyxy pixels + conf, cls = y['confidence'].max(1), y['confidence'].argmax(1).astype(np.float) + y = np.concatenate((box, conf.reshape(-1, 1), cls.reshape(-1, 1)), 1) + else: + y = list(reversed(y.values())) # reversed for segmentation models (pred, proto) + elif self.paddle: # PaddlePaddle + im = im.cpu().numpy().astype(np.float32) + self.input_handle.copy_from_cpu(im) + self.predictor.run() + y = [self.predictor.get_output_handle(x).copy_to_cpu() for x in self.output_names] + elif self.triton: # NVIDIA Triton Inference Server + y = self.model(im) + else: # TensorFlow (SavedModel, GraphDef, Lite, Edge TPU) + im = im.cpu().numpy() + if self.saved_model: # SavedModel + y = self.model(im, training=False) if self.keras else self.model(im) + elif self.pb: # GraphDef + y = self.frozen_func(x=self.tf.constant(im)) + else: # Lite or Edge TPU + input = self.input_details[0] + int8 = input['dtype'] == np.uint8 # is TFLite quantized uint8 model + if int8: + scale, zero_point = input['quantization'] + im = (im / scale + zero_point).astype(np.uint8) # de-scale + self.interpreter.set_tensor(input['index'], im) + self.interpreter.invoke() + y = [] + for output in self.output_details: + x = self.interpreter.get_tensor(output['index']) + if int8: + scale, zero_point = output['quantization'] + x = (x.astype(np.float32) - zero_point) * scale # re-scale + y.append(x) + y = [x if isinstance(x, np.ndarray) else x.numpy() for x in y] + y[0][..., :4] *= [w, h, w, h] # xywh normalized to pixels + + if isinstance(y, (list, tuple)): + return self.from_numpy(y[0]) if len(y) == 1 else [self.from_numpy(x) for x in y] + else: + return self.from_numpy(y) + + def from_numpy(self, x): + return torch.from_numpy(x).to(self.device) if isinstance(x, np.ndarray) else x + + def warmup(self, imgsz=(1, 3, 640, 640)): + # Warmup model by running inference once + warmup_types = self.pt, self.jit, self.onnx, self.engine, self.saved_model, self.pb, self.triton + if any(warmup_types) and (self.device.type != 'cpu' or self.triton): + im = torch.empty(*imgsz, dtype=torch.half if self.fp16 else torch.float, device=self.device) # input + for _ in range(2 if self.jit else 1): # + self.forward(im) # warmup + + @staticmethod + def _model_type(p='path/to/model.pt'): + # Return model type from model path, i.e. path='path/to/model.onnx' -> type=onnx + # types = [pt, jit, onnx, xml, engine, coreml, saved_model, pb, tflite, edgetpu, tfjs, paddle] + from export import export_formats + from utils.downloads import is_url + sf = list(export_formats().Suffix) # export suffixes + if not is_url(p, check=False): + check_suffix(p, sf) # checks + url = urlparse(p) # if url may be Triton inference server + types = [s in Path(p).name for s in sf] + types[8] &= not types[9] # tflite &= not edgetpu + triton = not any(types) and all([any(s in url.scheme for s in ['http', 'grpc']), url.netloc]) + return types + [triton] + + @staticmethod + def _load_metadata(f=Path('path/to/meta.yaml')): + # Load metadata from meta.yaml if it exists + if f.exists(): + d = yaml_load(f) + return d['stride'], d['names'] # assign stride, names + return None, None + + +class AutoShape(nn.Module): + # YOLOv5 input-robust model wrapper for passing cv2/np/PIL/torch inputs. Includes preprocessing, inference and NMS + conf = 0.25 # NMS confidence threshold + iou = 0.45 # NMS IoU threshold + agnostic = False # NMS class-agnostic + multi_label = False # NMS multiple labels per box + classes = None # (optional list) filter by class, i.e. = [0, 15, 16] for COCO persons, cats and dogs + max_det = 1000 # maximum number of detections per image + amp = False # Automatic Mixed Precision (AMP) inference + + def __init__(self, model, verbose=True): + super().__init__() + if verbose: + LOGGER.info('Adding AutoShape... ') + copy_attr(self, model, include=('yaml', 'nc', 'hyp', 'names', 'stride', 'abc'), exclude=()) # copy attributes + self.dmb = isinstance(model, DetectMultiBackend) # DetectMultiBackend() instance + self.pt = not self.dmb or model.pt # PyTorch model + self.model = model.eval() + if self.pt: + m = self.model.model.model[-1] if self.dmb else self.model.model[-1] # Detect() + m.inplace = False # Detect.inplace=False for safe multithread inference + m.export = True # do not output loss values + + def _apply(self, fn): + # Apply to(), cpu(), cuda(), half() to model tensors that are not parameters or registered buffers + self = super()._apply(fn) + if self.pt: + m = self.model.model.model[-1] if self.dmb else self.model.model[-1] # Detect() + m.stride = fn(m.stride) + m.grid = list(map(fn, m.grid)) + if isinstance(m.anchor_grid, list): + m.anchor_grid = list(map(fn, m.anchor_grid)) + return self + + @smart_inference_mode() + def forward(self, ims, size=640, augment=False, profile=False): + # Inference from various sources. For size(height=640, width=1280), RGB images example inputs are: + # file: ims = 'data/images/zidane.jpg' # str or PosixPath + # URI: = 'https://ultralytics.com/images/zidane.jpg' + # OpenCV: = cv2.imread('image.jpg')[:,:,::-1] # HWC BGR to RGB x(640,1280,3) + # PIL: = Image.open('image.jpg') or ImageGrab.grab() # HWC x(640,1280,3) + # numpy: = np.zeros((640,1280,3)) # HWC + # torch: = torch.zeros(16,3,320,640) # BCHW (scaled to size=640, 0-1 values) + # multiple: = [Image.open('image1.jpg'), Image.open('image2.jpg'), ...] # list of images + + dt = (Profile(), Profile(), Profile()) + with dt[0]: + if isinstance(size, int): # expand + size = (size, size) + p = next(self.model.parameters()) if self.pt else torch.empty(1, device=self.model.device) # param + autocast = self.amp and (p.device.type != 'cpu') # Automatic Mixed Precision (AMP) inference + if isinstance(ims, torch.Tensor): # torch + with amp.autocast(autocast): + return self.model(ims.to(p.device).type_as(p), augment=augment) # inference + + # Pre-process + n, ims = (len(ims), list(ims)) if isinstance(ims, (list, tuple)) else (1, [ims]) # number, list of images + shape0, shape1, files = [], [], [] # image and inference shapes, filenames + for i, im in enumerate(ims): + f = f'image{i}' # filename + if isinstance(im, (str, Path)): # filename or uri + im, f = Image.open(requests.get(im, stream=True).raw if str(im).startswith('http') else im), im + im = np.asarray(exif_transpose(im)) + elif isinstance(im, Image.Image): # PIL Image + im, f = np.asarray(exif_transpose(im)), getattr(im, 'filename', f) or f + files.append(Path(f).with_suffix('.jpg').name) + if im.shape[0] < 5: # image in CHW + im = im.transpose((1, 2, 0)) # reverse dataloader .transpose(2, 0, 1) + im = im[..., :3] if im.ndim == 3 else cv2.cvtColor(im, cv2.COLOR_GRAY2BGR) # enforce 3ch input + s = im.shape[:2] # HWC + shape0.append(s) # image shape + g = max(size) / max(s) # gain + shape1.append([int(y * g) for y in s]) + ims[i] = im if im.data.contiguous else np.ascontiguousarray(im) # update + shape1 = [make_divisible(x, self.stride) for x in np.array(shape1).max(0)] # inf shape + x = [letterbox(im, shape1, auto=False)[0] for im in ims] # pad + x = np.ascontiguousarray(np.array(x).transpose((0, 3, 1, 2))) # stack and BHWC to BCHW + x = torch.from_numpy(x).to(p.device).type_as(p) / 255 # uint8 to fp16/32 + + with amp.autocast(autocast): + # Inference + with dt[1]: + y = self.model(x, augment=augment) # forward + + # Post-process + with dt[2]: + y = non_max_suppression(y if self.dmb else y[0], + self.conf, + self.iou, + self.classes, + self.agnostic, + self.multi_label, + max_det=self.max_det) # NMS + for i in range(n): + scale_boxes(shape1, y[i][:, :4], shape0[i]) + + return Detections(ims, y, files, dt, self.names, x.shape) + + +class Detections: + # YOLOv5 detections class for inference results + def __init__(self, ims, pred, files, times=(0, 0, 0), names=None, shape=None): + super().__init__() + d = pred[0].device # device + gn = [torch.tensor([*(im.shape[i] for i in [1, 0, 1, 0]), 1, 1], device=d) for im in ims] # normalizations + self.ims = ims # list of images as numpy arrays + self.pred = pred # list of tensors pred[0] = (xyxy, conf, cls) + self.names = names # class names + self.files = files # image filenames + self.times = times # profiling times + self.xyxy = pred # xyxy pixels + self.xywh = [xyxy2xywh(x) for x in pred] # xywh pixels + self.xyxyn = [x / g for x, g in zip(self.xyxy, gn)] # xyxy normalized + self.xywhn = [x / g for x, g in zip(self.xywh, gn)] # xywh normalized + self.n = len(self.pred) # number of images (batch size) + self.t = tuple(x.t / self.n * 1E3 for x in times) # timestamps (ms) + self.s = tuple(shape) # inference BCHW shape + + def _run(self, pprint=False, show=False, save=False, crop=False, render=False, labels=True, save_dir=Path('')): + s, crops = '', [] + for i, (im, pred) in enumerate(zip(self.ims, self.pred)): + s += f'\nimage {i + 1}/{len(self.pred)}: {im.shape[0]}x{im.shape[1]} ' # string + if pred.shape[0]: + for c in pred[:, -1].unique(): + n = (pred[:, -1] == c).sum() # detections per class + s += f"{n} {self.names[int(c)]}{'s' * (n > 1)}, " # add to string + s = s.rstrip(', ') + if show or save or render or crop: + annotator = Annotator(im, example=str(self.names)) + for *box, conf, cls in reversed(pred): # xyxy, confidence, class + label = f'{self.names[int(cls)]} {conf:.2f}' + if crop: + file = save_dir / 'crops' / self.names[int(cls)] / self.files[i] if save else None + crops.append({ + 'box': box, + 'conf': conf, + 'cls': cls, + 'label': label, + 'im': save_one_box(box, im, file=file, save=save)}) + else: # all others + annotator.box_label(box, label if labels else '', color=colors(cls)) + im = annotator.im + else: + s += '(no detections)' + + im = Image.fromarray(im.astype(np.uint8)) if isinstance(im, np.ndarray) else im # from np + if show: + if is_jupyter(): + from IPython.display import display + display(im) + else: + im.show(self.files[i]) + if save: + f = self.files[i] + im.save(save_dir / f) # save + if i == self.n - 1: + LOGGER.info(f"Saved {self.n} image{'s' * (self.n > 1)} to {colorstr('bold', save_dir)}") + if render: + self.ims[i] = np.asarray(im) + if pprint: + s = s.lstrip('\n') + return f'{s}\nSpeed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {self.s}' % self.t + if crop: + if save: + LOGGER.info(f'Saved results to {save_dir}\n') + return crops + + @TryExcept('Showing images is not supported in this environment') + def show(self, labels=True): + self._run(show=True, labels=labels) # show results + + def save(self, labels=True, save_dir='runs/detect/exp', exist_ok=False): + save_dir = increment_path(save_dir, exist_ok, mkdir=True) # increment save_dir + self._run(save=True, labels=labels, save_dir=save_dir) # save results + + def crop(self, save=True, save_dir='runs/detect/exp', exist_ok=False): + save_dir = increment_path(save_dir, exist_ok, mkdir=True) if save else None + return self._run(crop=True, save=save, save_dir=save_dir) # crop results + + def render(self, labels=True): + self._run(render=True, labels=labels) # render results + return self.ims + + def pandas(self): + # return detections as pandas DataFrames, i.e. print(results.pandas().xyxy[0]) + new = copy(self) # return copy + ca = 'xmin', 'ymin', 'xmax', 'ymax', 'confidence', 'class', 'name' # xyxy columns + cb = 'xcenter', 'ycenter', 'width', 'height', 'confidence', 'class', 'name' # xywh columns + for k, c in zip(['xyxy', 'xyxyn', 'xywh', 'xywhn'], [ca, ca, cb, cb]): + a = [[x[:5] + [int(x[5]), self.names[int(x[5])]] for x in x.tolist()] for x in getattr(self, k)] # update + setattr(new, k, [pd.DataFrame(x, columns=c) for x in a]) + return new + + def tolist(self): + # return a list of Detections objects, i.e. 'for result in results.tolist():' + r = range(self.n) # iterable + x = [Detections([self.ims[i]], [self.pred[i]], [self.files[i]], self.times, self.names, self.s) for i in r] + # for d in x: + # for k in ['ims', 'pred', 'xyxy', 'xyxyn', 'xywh', 'xywhn']: + # setattr(d, k, getattr(d, k)[0]) # pop out of list + return x + + def print(self): + LOGGER.info(self.__str__()) + + def __len__(self): # override len(results) + return self.n + + def __str__(self): # override print(results) + return self._run(pprint=True) # print results + + def __repr__(self): + return f'YOLOv5 {self.__class__} instance\n' + self.__str__() + + +class Proto(nn.Module): + # YOLOv5 mask Proto module for segmentation models + def __init__(self, c1, c_=256, c2=32): # ch_in, number of protos, number of masks + super().__init__() + self.cv1 = Conv(c1, c_, k=3) + self.upsample = nn.Upsample(scale_factor=2, mode='nearest') + self.cv2 = Conv(c_, c_, k=3) + self.cv3 = Conv(c_, c2) + + def forward(self, x): + return self.cv3(self.cv2(self.upsample(self.cv1(x)))) + + +class Classify(nn.Module): + # YOLOv5 classification head, i.e. x(b,c1,20,20) to x(b,c2) + def __init__(self, + c1, + c2, + k=1, + s=1, + p=None, + g=1, + dropout_p=0.0): # ch_in, ch_out, kernel, stride, padding, groups, dropout probability + super().__init__() + c_ = 1280 # efficientnet_b0 size + self.conv = Conv(c1, c_, k, s, autopad(k, p), g) + self.pool = nn.AdaptiveAvgPool2d(1) # to x(b,c_,1,1) + self.drop = nn.Dropout(p=dropout_p, inplace=True) + self.linear = nn.Linear(c_, c2) # to x(b,c2) + + def forward(self, x): + if isinstance(x, list): + x = torch.cat(x, 1) + return self.linear(self.drop(self.pool(self.conv(x)).flatten(1))) diff --git a/models/experimental.py b/models/experimental.py new file mode 100644 index 0000000..d0fc839 --- /dev/null +++ b/models/experimental.py @@ -0,0 +1,115 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Experimental modules +""" +import math + +import numpy as np +import torch +import torch.nn as nn + +from utils.downloads import attempt_download + + +class Sum(nn.Module): + # Weighted sum of 2 or more layers https://arxiv.org/abs/1911.09070 + def __init__(self, n, weight=False): # n: number of inputs + super().__init__() + self.weight = weight # apply weights boolean + self.iter = range(n - 1) # iter object + if weight: + self.w = nn.Parameter(-torch.arange(1.0, n) / 2, requires_grad=True) # layer weights + + def forward(self, x): + y = x[0] # no weight + if self.weight: + w = torch.sigmoid(self.w) * 2 + for i in self.iter: + y = y + x[i + 1] * w[i] + else: + for i in self.iter: + y = y + x[i + 1] + return y + + +class MixConv2d(nn.Module): + # Mixed Depth-wise Conv https://arxiv.org/abs/1907.09595 + def __init__(self, c1, c2, k=(1, 3), s=1, equal_ch=True): # ch_in, ch_out, kernel, stride, ch_strategy + super().__init__() + n = len(k) # number of convolutions + if equal_ch: # equal c_ per group + i = torch.linspace(0, n - 1E-6, c2).floor() # c2 indices + c_ = [(i == g).sum() for g in range(n)] # intermediate channels + else: # equal weight.numel() per group + b = [c2] + [0] * n + a = np.eye(n + 1, n, k=-1) + a -= np.roll(a, 1, axis=1) + a *= np.array(k) ** 2 + a[0] = 1 + c_ = np.linalg.lstsq(a, b, rcond=None)[0].round() # solve for equal weight indices, ax = b + + self.m = nn.ModuleList([ + nn.Conv2d(c1, int(c_), k, s, k // 2, groups=math.gcd(c1, int(c_)), bias=False) for k, c_ in zip(k, c_)]) + self.bn = nn.BatchNorm2d(c2) + self.act = nn.SiLU() + + def forward(self, x): + return self.act(self.bn(torch.cat([m(x) for m in self.m], 1))) + + +class Ensemble(nn.ModuleList): + # Ensemble of models + def __init__(self): + super().__init__() + + def forward(self, x, augment=False, profile=False, visualize=False): + y = [module(x, augment, profile, visualize)[0] for module in self] + # y = torch.stack(y).max(0)[0] # max ensemble + # y = torch.stack(y).mean(0) # mean ensemble + y = torch.cat(y, 1) # nms ensemble + return y, None # inference, train output + + +def attempt_load(weights, device=None, inplace=True, fuse=True): + # Loads an ensemble of models weights=[a,b,c] or a single model weights=[a] or weights=a + from models.yolo import Detect, Model + + model = Ensemble() + for w in weights if isinstance(weights, list) else [weights]: + if torch.__version__ >= '2.6': + ckpt = torch.load(attempt_download(w), map_location=device, weights_only=False) # load + else: + ckpt = torch.load(attempt_download(w), map_location=device) + + ckpt = (ckpt.get('ema') or ckpt['model']).to(device).float() # FP32 model + + # Model compatibility updates + if not hasattr(ckpt, 'stride'): + ckpt.stride = torch.tensor([32.]) + if hasattr(ckpt, 'names') and isinstance(ckpt.names, (list, tuple)): + ckpt.names = dict(enumerate(ckpt.names)) # convert to dict + + model.append(ckpt.fuse().eval() if fuse and hasattr(ckpt, 'fuse') else ckpt.eval()) # model in eval mode + + # Module updates + for m in model.modules(): + t = type(m) + if t in (nn.Hardswish, nn.LeakyReLU, nn.ReLU, nn.ReLU6, nn.SiLU, Detect, Model): + m.inplace = inplace + if t is Detect and not isinstance(m.anchor_grid, list): + delattr(m, 'anchor_grid') + setattr(m, 'anchor_grid', [torch.zeros(1)] * m.nl) + elif t is nn.Upsample and not hasattr(m, 'recompute_scale_factor'): + m.recompute_scale_factor = None # torch 1.11.0 compatibility + + # Return model + if len(model) == 1: + return model[-1] + + # Return detection ensemble + print(f'Ensemble created with {weights}\n') + for k in 'names', 'nc', 'yaml': + setattr(model, k, getattr(model[0], k)) + model.stride = model[torch.argmax(torch.tensor([m.stride.max() for m in model])).int()].stride # max stride + assert all(model[0].nc == m.nc for m in model), f'Models have different class counts: {[m.nc for m in model]}' + return model diff --git a/models/hub/anchors.yaml b/models/hub/anchors.yaml new file mode 100644 index 0000000..df2f668 --- /dev/null +++ b/models/hub/anchors.yaml @@ -0,0 +1,59 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +# Default anchors for COCO data + + +# P5 ------------------------------------------------------------------------------------------------------------------- +# P5-640: +anchors_p5_640: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + + +# P6 ------------------------------------------------------------------------------------------------------------------- +# P6-640: thr=0.25: 0.9964 BPR, 5.54 anchors past thr, n=12, img_size=640, metric_all=0.281/0.716-mean/best, past_thr=0.469-mean: 9,11, 21,19, 17,41, 43,32, 39,70, 86,64, 65,131, 134,130, 120,265, 282,180, 247,354, 512,387 +anchors_p6_640: + - [9,11, 21,19, 17,41] # P3/8 + - [43,32, 39,70, 86,64] # P4/16 + - [65,131, 134,130, 120,265] # P5/32 + - [282,180, 247,354, 512,387] # P6/64 + +# P6-1280: thr=0.25: 0.9950 BPR, 5.55 anchors past thr, n=12, img_size=1280, metric_all=0.281/0.714-mean/best, past_thr=0.468-mean: 19,27, 44,40, 38,94, 96,68, 86,152, 180,137, 140,301, 303,264, 238,542, 436,615, 739,380, 925,792 +anchors_p6_1280: + - [19,27, 44,40, 38,94] # P3/8 + - [96,68, 86,152, 180,137] # P4/16 + - [140,301, 303,264, 238,542] # P5/32 + - [436,615, 739,380, 925,792] # P6/64 + +# P6-1920: thr=0.25: 0.9950 BPR, 5.55 anchors past thr, n=12, img_size=1920, metric_all=0.281/0.714-mean/best, past_thr=0.468-mean: 28,41, 67,59, 57,141, 144,103, 129,227, 270,205, 209,452, 455,396, 358,812, 653,922, 1109,570, 1387,1187 +anchors_p6_1920: + - [28,41, 67,59, 57,141] # P3/8 + - [144,103, 129,227, 270,205] # P4/16 + - [209,452, 455,396, 358,812] # P5/32 + - [653,922, 1109,570, 1387,1187] # P6/64 + + +# P7 ------------------------------------------------------------------------------------------------------------------- +# P7-640: thr=0.25: 0.9962 BPR, 6.76 anchors past thr, n=15, img_size=640, metric_all=0.275/0.733-mean/best, past_thr=0.466-mean: 11,11, 13,30, 29,20, 30,46, 61,38, 39,92, 78,80, 146,66, 79,163, 149,150, 321,143, 157,303, 257,402, 359,290, 524,372 +anchors_p7_640: + - [11,11, 13,30, 29,20] # P3/8 + - [30,46, 61,38, 39,92] # P4/16 + - [78,80, 146,66, 79,163] # P5/32 + - [149,150, 321,143, 157,303] # P6/64 + - [257,402, 359,290, 524,372] # P7/128 + +# P7-1280: thr=0.25: 0.9968 BPR, 6.71 anchors past thr, n=15, img_size=1280, metric_all=0.273/0.732-mean/best, past_thr=0.463-mean: 19,22, 54,36, 32,77, 70,83, 138,71, 75,173, 165,159, 148,334, 375,151, 334,317, 251,626, 499,474, 750,326, 534,814, 1079,818 +anchors_p7_1280: + - [19,22, 54,36, 32,77] # P3/8 + - [70,83, 138,71, 75,173] # P4/16 + - [165,159, 148,334, 375,151] # P5/32 + - [334,317, 251,626, 499,474] # P6/64 + - [750,326, 534,814, 1079,818] # P7/128 + +# P7-1920: thr=0.25: 0.9968 BPR, 6.71 anchors past thr, n=15, img_size=1920, metric_all=0.273/0.732-mean/best, past_thr=0.463-mean: 29,34, 81,55, 47,115, 105,124, 207,107, 113,259, 247,238, 222,500, 563,227, 501,476, 376,939, 749,711, 1126,489, 801,1222, 1618,1227 +anchors_p7_1920: + - [29,34, 81,55, 47,115] # P3/8 + - [105,124, 207,107, 113,259] # P4/16 + - [247,238, 222,500, 563,227] # P5/32 + - [501,476, 376,939, 749,711] # P6/64 + - [1126,489, 801,1222, 1618,1227] # P7/128 diff --git a/models/hub/yolov3-spp.yaml b/models/hub/yolov3-spp.yaml new file mode 100644 index 0000000..4a71ed4 --- /dev/null +++ b/models/hub/yolov3-spp.yaml @@ -0,0 +1,51 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# Parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# darknet53 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [32, 3, 1]], # 0 + [-1, 1, Conv, [64, 3, 2]], # 1-P1/2 + [-1, 1, Bottleneck, [64]], + [-1, 1, Conv, [128, 3, 2]], # 3-P2/4 + [-1, 2, Bottleneck, [128]], + [-1, 1, Conv, [256, 3, 2]], # 5-P3/8 + [-1, 8, Bottleneck, [256]], + [-1, 1, Conv, [512, 3, 2]], # 7-P4/16 + [-1, 8, Bottleneck, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 9-P5/32 + [-1, 4, Bottleneck, [1024]], # 10 + ] + +# YOLOv3-SPP head +head: + [[-1, 1, Bottleneck, [1024, False]], + [-1, 1, SPP, [512, [5, 9, 13]]], + [-1, 1, Conv, [1024, 3, 1]], + [-1, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [1024, 3, 1]], # 15 (P5/32-large) + + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 8], 1, Concat, [1]], # cat backbone P4 + [-1, 1, Bottleneck, [512, False]], + [-1, 1, Bottleneck, [512, False]], + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [512, 3, 1]], # 22 (P4/16-medium) + + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P3 + [-1, 1, Bottleneck, [256, False]], + [-1, 2, Bottleneck, [256, False]], # 27 (P3/8-small) + + [[27, 22, 15], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/models/hub/yolov3-tiny.yaml b/models/hub/yolov3-tiny.yaml new file mode 100644 index 0000000..50b47e2 --- /dev/null +++ b/models/hub/yolov3-tiny.yaml @@ -0,0 +1,41 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# Parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple +anchors: + - [10,14, 23,27, 37,58] # P4/16 + - [81,82, 135,169, 344,319] # P5/32 + +# YOLOv3-tiny backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [16, 3, 1]], # 0 + [-1, 1, nn.MaxPool2d, [2, 2, 0]], # 1-P1/2 + [-1, 1, Conv, [32, 3, 1]], + [-1, 1, nn.MaxPool2d, [2, 2, 0]], # 3-P2/4 + [-1, 1, Conv, [64, 3, 1]], + [-1, 1, nn.MaxPool2d, [2, 2, 0]], # 5-P3/8 + [-1, 1, Conv, [128, 3, 1]], + [-1, 1, nn.MaxPool2d, [2, 2, 0]], # 7-P4/16 + [-1, 1, Conv, [256, 3, 1]], + [-1, 1, nn.MaxPool2d, [2, 2, 0]], # 9-P5/32 + [-1, 1, Conv, [512, 3, 1]], + [-1, 1, nn.ZeroPad2d, [[0, 1, 0, 1]]], # 11 + [-1, 1, nn.MaxPool2d, [2, 1, 0]], # 12 + ] + +# YOLOv3-tiny head +head: + [[-1, 1, Conv, [1024, 3, 1]], + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [512, 3, 1]], # 15 (P5/32-large) + + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 8], 1, Concat, [1]], # cat backbone P4 + [-1, 1, Conv, [256, 3, 1]], # 19 (P4/16-medium) + + [[19, 15], 1, Detect, [nc, anchors]], # Detect(P4, P5) + ] diff --git a/models/hub/yolov3.yaml b/models/hub/yolov3.yaml new file mode 100644 index 0000000..c5e2109 --- /dev/null +++ b/models/hub/yolov3.yaml @@ -0,0 +1,51 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# Parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# darknet53 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [32, 3, 1]], # 0 + [-1, 1, Conv, [64, 3, 2]], # 1-P1/2 + [-1, 1, Bottleneck, [64]], + [-1, 1, Conv, [128, 3, 2]], # 3-P2/4 + [-1, 2, Bottleneck, [128]], + [-1, 1, Conv, [256, 3, 2]], # 5-P3/8 + [-1, 8, Bottleneck, [256]], + [-1, 1, Conv, [512, 3, 2]], # 7-P4/16 + [-1, 8, Bottleneck, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 9-P5/32 + [-1, 4, Bottleneck, [1024]], # 10 + ] + +# YOLOv3 head +head: + [[-1, 1, Bottleneck, [1024, False]], + [-1, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [1024, 3, 1]], + [-1, 1, Conv, [512, 1, 1]], + [-1, 1, Conv, [1024, 3, 1]], # 15 (P5/32-large) + + [-2, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 8], 1, Concat, [1]], # cat backbone P4 + [-1, 1, Bottleneck, [512, False]], + [-1, 1, Bottleneck, [512, False]], + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, Conv, [512, 3, 1]], # 22 (P4/16-medium) + + [-2, 1, Conv, [128, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P3 + [-1, 1, Bottleneck, [256, False]], + [-1, 2, Bottleneck, [256, False]], # 27 (P3/8-small) + + [[27, 22, 15], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/models/hub/yolov5-bifpn.yaml b/models/hub/yolov5-bifpn.yaml new file mode 100644 index 0000000..9dbdd4e --- /dev/null +++ b/models/hub/yolov5-bifpn.yaml @@ -0,0 +1,48 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# Parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# YOLOv5 v6.0 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [64, 6, 2, 2]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, C3, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 6, C3, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, C3, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 7-P5/32 + [-1, 3, C3, [1024]], + [-1, 1, SPPF, [1024, 5]], # 9 + ] + +# YOLOv5 v6.0 BiFPN head +head: + [[-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, C3, [512, False]], # 13 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, C3, [256, False]], # 17 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 14, 6], 1, Concat, [1]], # cat P4 <--- BiFPN change + [-1, 3, C3, [512, False]], # 20 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 10], 1, Concat, [1]], # cat head P5 + [-1, 3, C3, [1024, False]], # 23 (P5/32-large) + + [[17, 20, 23], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/models/hub/yolov5-fpn.yaml b/models/hub/yolov5-fpn.yaml new file mode 100644 index 0000000..2292eb1 --- /dev/null +++ b/models/hub/yolov5-fpn.yaml @@ -0,0 +1,42 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# Parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# YOLOv5 v6.0 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [64, 6, 2, 2]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, C3, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 6, C3, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, C3, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 7-P5/32 + [-1, 3, C3, [1024]], + [-1, 1, SPPF, [1024, 5]], # 9 + ] + +# YOLOv5 v6.0 FPN head +head: + [[-1, 3, C3, [1024, False]], # 10 (P5/32-large) + + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 1, Conv, [512, 1, 1]], + [-1, 3, C3, [512, False]], # 14 (P4/16-medium) + + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 1, Conv, [256, 1, 1]], + [-1, 3, C3, [256, False]], # 18 (P3/8-small) + + [[18, 14, 10], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/models/hub/yolov5-p2.yaml b/models/hub/yolov5-p2.yaml new file mode 100644 index 0000000..2c0ae44 --- /dev/null +++ b/models/hub/yolov5-p2.yaml @@ -0,0 +1,54 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# Parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple +anchors: 3 # AutoAnchor evolves 3 anchors per P output layer + +# YOLOv5 v6.0 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [64, 6, 2, 2]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, C3, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 6, C3, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, C3, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 7-P5/32 + [-1, 3, C3, [1024]], + [-1, 1, SPPF, [1024, 5]], # 9 + ] + +# YOLOv5 v6.0 head with (P2, P3, P4, P5) outputs +head: + [[-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, C3, [512, False]], # 13 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, C3, [256, False]], # 17 (P3/8-small) + + [-1, 1, Conv, [128, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 2], 1, Concat, [1]], # cat backbone P2 + [-1, 1, C3, [128, False]], # 21 (P2/4-xsmall) + + [-1, 1, Conv, [128, 3, 2]], + [[-1, 18], 1, Concat, [1]], # cat head P3 + [-1, 3, C3, [256, False]], # 24 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 14], 1, Concat, [1]], # cat head P4 + [-1, 3, C3, [512, False]], # 27 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 10], 1, Concat, [1]], # cat head P5 + [-1, 3, C3, [1024, False]], # 30 (P5/32-large) + + [[21, 24, 27, 30], 1, Detect, [nc, anchors]], # Detect(P2, P3, P4, P5) + ] diff --git a/models/hub/yolov5-p34.yaml b/models/hub/yolov5-p34.yaml new file mode 100644 index 0000000..60ae3b4 --- /dev/null +++ b/models/hub/yolov5-p34.yaml @@ -0,0 +1,41 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# Parameters +nc: 80 # number of classes +depth_multiple: 0.33 # model depth multiple +width_multiple: 0.50 # layer channel multiple +anchors: 3 # AutoAnchor evolves 3 anchors per P output layer + +# YOLOv5 v6.0 backbone +backbone: + # [from, number, module, args] + [ [ -1, 1, Conv, [ 64, 6, 2, 2 ] ], # 0-P1/2 + [ -1, 1, Conv, [ 128, 3, 2 ] ], # 1-P2/4 + [ -1, 3, C3, [ 128 ] ], + [ -1, 1, Conv, [ 256, 3, 2 ] ], # 3-P3/8 + [ -1, 6, C3, [ 256 ] ], + [ -1, 1, Conv, [ 512, 3, 2 ] ], # 5-P4/16 + [ -1, 9, C3, [ 512 ] ], + [ -1, 1, Conv, [ 1024, 3, 2 ] ], # 7-P5/32 + [ -1, 3, C3, [ 1024 ] ], + [ -1, 1, SPPF, [ 1024, 5 ] ], # 9 + ] + +# YOLOv5 v6.0 head with (P3, P4) outputs +head: + [ [ -1, 1, Conv, [ 512, 1, 1 ] ], + [ -1, 1, nn.Upsample, [ None, 2, 'nearest' ] ], + [ [ -1, 6 ], 1, Concat, [ 1 ] ], # cat backbone P4 + [ -1, 3, C3, [ 512, False ] ], # 13 + + [ -1, 1, Conv, [ 256, 1, 1 ] ], + [ -1, 1, nn.Upsample, [ None, 2, 'nearest' ] ], + [ [ -1, 4 ], 1, Concat, [ 1 ] ], # cat backbone P3 + [ -1, 3, C3, [ 256, False ] ], # 17 (P3/8-small) + + [ -1, 1, Conv, [ 256, 3, 2 ] ], + [ [ -1, 14 ], 1, Concat, [ 1 ] ], # cat head P4 + [ -1, 3, C3, [ 512, False ] ], # 20 (P4/16-medium) + + [ [ 17, 20 ], 1, Detect, [ nc, anchors ] ], # Detect(P3, P4) + ] diff --git a/models/hub/yolov5-p6.yaml b/models/hub/yolov5-p6.yaml new file mode 100644 index 0000000..a9e1b5f --- /dev/null +++ b/models/hub/yolov5-p6.yaml @@ -0,0 +1,56 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# Parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple +anchors: 3 # AutoAnchor evolves 3 anchors per P output layer + +# YOLOv5 v6.0 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [64, 6, 2, 2]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, C3, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 6, C3, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, C3, [512]], + [-1, 1, Conv, [768, 3, 2]], # 7-P5/32 + [-1, 3, C3, [768]], + [-1, 1, Conv, [1024, 3, 2]], # 9-P6/64 + [-1, 3, C3, [1024]], + [-1, 1, SPPF, [1024, 5]], # 11 + ] + +# YOLOv5 v6.0 head with (P3, P4, P5, P6) outputs +head: + [[-1, 1, Conv, [768, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 8], 1, Concat, [1]], # cat backbone P5 + [-1, 3, C3, [768, False]], # 15 + + [-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, C3, [512, False]], # 19 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, C3, [256, False]], # 23 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 20], 1, Concat, [1]], # cat head P4 + [-1, 3, C3, [512, False]], # 26 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 16], 1, Concat, [1]], # cat head P5 + [-1, 3, C3, [768, False]], # 29 (P5/32-large) + + [-1, 1, Conv, [768, 3, 2]], + [[-1, 12], 1, Concat, [1]], # cat head P6 + [-1, 3, C3, [1024, False]], # 32 (P6/64-xlarge) + + [[23, 26, 29, 32], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5, P6) + ] diff --git a/models/hub/yolov5-p7.yaml b/models/hub/yolov5-p7.yaml new file mode 100644 index 0000000..a502412 --- /dev/null +++ b/models/hub/yolov5-p7.yaml @@ -0,0 +1,67 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# Parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple +anchors: 3 # AutoAnchor evolves 3 anchors per P output layer + +# YOLOv5 v6.0 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [64, 6, 2, 2]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, C3, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 6, C3, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, C3, [512]], + [-1, 1, Conv, [768, 3, 2]], # 7-P5/32 + [-1, 3, C3, [768]], + [-1, 1, Conv, [1024, 3, 2]], # 9-P6/64 + [-1, 3, C3, [1024]], + [-1, 1, Conv, [1280, 3, 2]], # 11-P7/128 + [-1, 3, C3, [1280]], + [-1, 1, SPPF, [1280, 5]], # 13 + ] + +# YOLOv5 v6.0 head with (P3, P4, P5, P6, P7) outputs +head: + [[-1, 1, Conv, [1024, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 10], 1, Concat, [1]], # cat backbone P6 + [-1, 3, C3, [1024, False]], # 17 + + [-1, 1, Conv, [768, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 8], 1, Concat, [1]], # cat backbone P5 + [-1, 3, C3, [768, False]], # 21 + + [-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, C3, [512, False]], # 25 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, C3, [256, False]], # 29 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 26], 1, Concat, [1]], # cat head P4 + [-1, 3, C3, [512, False]], # 32 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 22], 1, Concat, [1]], # cat head P5 + [-1, 3, C3, [768, False]], # 35 (P5/32-large) + + [-1, 1, Conv, [768, 3, 2]], + [[-1, 18], 1, Concat, [1]], # cat head P6 + [-1, 3, C3, [1024, False]], # 38 (P6/64-xlarge) + + [-1, 1, Conv, [1024, 3, 2]], + [[-1, 14], 1, Concat, [1]], # cat head P7 + [-1, 3, C3, [1280, False]], # 41 (P7/128-xxlarge) + + [[29, 32, 35, 38, 41], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5, P6, P7) + ] diff --git a/models/hub/yolov5-panet.yaml b/models/hub/yolov5-panet.yaml new file mode 100644 index 0000000..5595e25 --- /dev/null +++ b/models/hub/yolov5-panet.yaml @@ -0,0 +1,48 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# Parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# YOLOv5 v6.0 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [64, 6, 2, 2]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, C3, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 6, C3, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, C3, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 7-P5/32 + [-1, 3, C3, [1024]], + [-1, 1, SPPF, [1024, 5]], # 9 + ] + +# YOLOv5 v6.0 PANet head +head: + [[-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, C3, [512, False]], # 13 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, C3, [256, False]], # 17 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 14], 1, Concat, [1]], # cat head P4 + [-1, 3, C3, [512, False]], # 20 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 10], 1, Concat, [1]], # cat head P5 + [-1, 3, C3, [1024, False]], # 23 (P5/32-large) + + [[17, 20, 23], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/models/hub/yolov5l6.yaml b/models/hub/yolov5l6.yaml new file mode 100644 index 0000000..651dbb0 --- /dev/null +++ b/models/hub/yolov5l6.yaml @@ -0,0 +1,60 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# Parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple +anchors: + - [19,27, 44,40, 38,94] # P3/8 + - [96,68, 86,152, 180,137] # P4/16 + - [140,301, 303,264, 238,542] # P5/32 + - [436,615, 739,380, 925,792] # P6/64 + +# YOLOv5 v6.0 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [64, 6, 2, 2]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, C3, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 6, C3, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, C3, [512]], + [-1, 1, Conv, [768, 3, 2]], # 7-P5/32 + [-1, 3, C3, [768]], + [-1, 1, Conv, [1024, 3, 2]], # 9-P6/64 + [-1, 3, C3, [1024]], + [-1, 1, SPPF, [1024, 5]], # 11 + ] + +# YOLOv5 v6.0 head +head: + [[-1, 1, Conv, [768, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 8], 1, Concat, [1]], # cat backbone P5 + [-1, 3, C3, [768, False]], # 15 + + [-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, C3, [512, False]], # 19 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, C3, [256, False]], # 23 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 20], 1, Concat, [1]], # cat head P4 + [-1, 3, C3, [512, False]], # 26 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 16], 1, Concat, [1]], # cat head P5 + [-1, 3, C3, [768, False]], # 29 (P5/32-large) + + [-1, 1, Conv, [768, 3, 2]], + [[-1, 12], 1, Concat, [1]], # cat head P6 + [-1, 3, C3, [1024, False]], # 32 (P6/64-xlarge) + + [[23, 26, 29, 32], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5, P6) + ] diff --git a/models/hub/yolov5m6.yaml b/models/hub/yolov5m6.yaml new file mode 100644 index 0000000..059b12b --- /dev/null +++ b/models/hub/yolov5m6.yaml @@ -0,0 +1,60 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# Parameters +nc: 80 # number of classes +depth_multiple: 0.67 # model depth multiple +width_multiple: 0.75 # layer channel multiple +anchors: + - [19,27, 44,40, 38,94] # P3/8 + - [96,68, 86,152, 180,137] # P4/16 + - [140,301, 303,264, 238,542] # P5/32 + - [436,615, 739,380, 925,792] # P6/64 + +# YOLOv5 v6.0 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [64, 6, 2, 2]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, C3, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 6, C3, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, C3, [512]], + [-1, 1, Conv, [768, 3, 2]], # 7-P5/32 + [-1, 3, C3, [768]], + [-1, 1, Conv, [1024, 3, 2]], # 9-P6/64 + [-1, 3, C3, [1024]], + [-1, 1, SPPF, [1024, 5]], # 11 + ] + +# YOLOv5 v6.0 head +head: + [[-1, 1, Conv, [768, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 8], 1, Concat, [1]], # cat backbone P5 + [-1, 3, C3, [768, False]], # 15 + + [-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, C3, [512, False]], # 19 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, C3, [256, False]], # 23 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 20], 1, Concat, [1]], # cat head P4 + [-1, 3, C3, [512, False]], # 26 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 16], 1, Concat, [1]], # cat head P5 + [-1, 3, C3, [768, False]], # 29 (P5/32-large) + + [-1, 1, Conv, [768, 3, 2]], + [[-1, 12], 1, Concat, [1]], # cat head P6 + [-1, 3, C3, [1024, False]], # 32 (P6/64-xlarge) + + [[23, 26, 29, 32], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5, P6) + ] diff --git a/models/hub/yolov5n6.yaml b/models/hub/yolov5n6.yaml new file mode 100644 index 0000000..5052e7c --- /dev/null +++ b/models/hub/yolov5n6.yaml @@ -0,0 +1,60 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# Parameters +nc: 80 # number of classes +depth_multiple: 0.33 # model depth multiple +width_multiple: 0.25 # layer channel multiple +anchors: + - [19,27, 44,40, 38,94] # P3/8 + - [96,68, 86,152, 180,137] # P4/16 + - [140,301, 303,264, 238,542] # P5/32 + - [436,615, 739,380, 925,792] # P6/64 + +# YOLOv5 v6.0 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [64, 6, 2, 2]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, C3, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 6, C3, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, C3, [512]], + [-1, 1, Conv, [768, 3, 2]], # 7-P5/32 + [-1, 3, C3, [768]], + [-1, 1, Conv, [1024, 3, 2]], # 9-P6/64 + [-1, 3, C3, [1024]], + [-1, 1, SPPF, [1024, 5]], # 11 + ] + +# YOLOv5 v6.0 head +head: + [[-1, 1, Conv, [768, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 8], 1, Concat, [1]], # cat backbone P5 + [-1, 3, C3, [768, False]], # 15 + + [-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, C3, [512, False]], # 19 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, C3, [256, False]], # 23 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 20], 1, Concat, [1]], # cat head P4 + [-1, 3, C3, [512, False]], # 26 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 16], 1, Concat, [1]], # cat head P5 + [-1, 3, C3, [768, False]], # 29 (P5/32-large) + + [-1, 1, Conv, [768, 3, 2]], + [[-1, 12], 1, Concat, [1]], # cat head P6 + [-1, 3, C3, [1024, False]], # 32 (P6/64-xlarge) + + [[23, 26, 29, 32], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5, P6) + ] diff --git a/models/hub/yolov5s-LeakyReLU.yaml b/models/hub/yolov5s-LeakyReLU.yaml new file mode 100644 index 0000000..0368a78 --- /dev/null +++ b/models/hub/yolov5s-LeakyReLU.yaml @@ -0,0 +1,49 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# Parameters +nc: 80 # number of classes +activation: nn.LeakyReLU(0.1) # <----- Conv() activation used throughout entire YOLOv5 model +depth_multiple: 0.33 # model depth multiple +width_multiple: 0.50 # layer channel multiple +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# YOLOv5 v6.0 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [64, 6, 2, 2]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, C3, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 6, C3, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, C3, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 7-P5/32 + [-1, 3, C3, [1024]], + [-1, 1, SPPF, [1024, 5]], # 9 + ] + +# YOLOv5 v6.0 head +head: + [[-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, C3, [512, False]], # 13 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, C3, [256, False]], # 17 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 14], 1, Concat, [1]], # cat head P4 + [-1, 3, C3, [512, False]], # 20 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 10], 1, Concat, [1]], # cat head P5 + [-1, 3, C3, [1024, False]], # 23 (P5/32-large) + + [[17, 20, 23], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/models/hub/yolov5s-ghost.yaml b/models/hub/yolov5s-ghost.yaml new file mode 100644 index 0000000..ce5238f --- /dev/null +++ b/models/hub/yolov5s-ghost.yaml @@ -0,0 +1,48 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# Parameters +nc: 80 # number of classes +depth_multiple: 0.33 # model depth multiple +width_multiple: 0.50 # layer channel multiple +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# YOLOv5 v6.0 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [64, 6, 2, 2]], # 0-P1/2 + [-1, 1, GhostConv, [128, 3, 2]], # 1-P2/4 + [-1, 3, C3Ghost, [128]], + [-1, 1, GhostConv, [256, 3, 2]], # 3-P3/8 + [-1, 6, C3Ghost, [256]], + [-1, 1, GhostConv, [512, 3, 2]], # 5-P4/16 + [-1, 9, C3Ghost, [512]], + [-1, 1, GhostConv, [1024, 3, 2]], # 7-P5/32 + [-1, 3, C3Ghost, [1024]], + [-1, 1, SPPF, [1024, 5]], # 9 + ] + +# YOLOv5 v6.0 head +head: + [[-1, 1, GhostConv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, C3Ghost, [512, False]], # 13 + + [-1, 1, GhostConv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, C3Ghost, [256, False]], # 17 (P3/8-small) + + [-1, 1, GhostConv, [256, 3, 2]], + [[-1, 14], 1, Concat, [1]], # cat head P4 + [-1, 3, C3Ghost, [512, False]], # 20 (P4/16-medium) + + [-1, 1, GhostConv, [512, 3, 2]], + [[-1, 10], 1, Concat, [1]], # cat head P5 + [-1, 3, C3Ghost, [1024, False]], # 23 (P5/32-large) + + [[17, 20, 23], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/models/hub/yolov5s-transformer.yaml b/models/hub/yolov5s-transformer.yaml new file mode 100644 index 0000000..f526716 --- /dev/null +++ b/models/hub/yolov5s-transformer.yaml @@ -0,0 +1,48 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# Parameters +nc: 80 # number of classes +depth_multiple: 0.33 # model depth multiple +width_multiple: 0.50 # layer channel multiple +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# YOLOv5 v6.0 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [64, 6, 2, 2]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, C3, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 6, C3, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, C3, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 7-P5/32 + [-1, 3, C3TR, [1024]], # 9 <--- C3TR() Transformer module + [-1, 1, SPPF, [1024, 5]], # 9 + ] + +# YOLOv5 v6.0 head +head: + [[-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, C3, [512, False]], # 13 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, C3, [256, False]], # 17 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 14], 1, Concat, [1]], # cat head P4 + [-1, 3, C3, [512, False]], # 20 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 10], 1, Concat, [1]], # cat head P5 + [-1, 3, C3, [1024, False]], # 23 (P5/32-large) + + [[17, 20, 23], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/models/hub/yolov5s6.yaml b/models/hub/yolov5s6.yaml new file mode 100644 index 0000000..2f39b03 --- /dev/null +++ b/models/hub/yolov5s6.yaml @@ -0,0 +1,60 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# Parameters +nc: 80 # number of classes +depth_multiple: 0.33 # model depth multiple +width_multiple: 0.50 # layer channel multiple +anchors: + - [19,27, 44,40, 38,94] # P3/8 + - [96,68, 86,152, 180,137] # P4/16 + - [140,301, 303,264, 238,542] # P5/32 + - [436,615, 739,380, 925,792] # P6/64 + +# YOLOv5 v6.0 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [64, 6, 2, 2]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, C3, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 6, C3, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, C3, [512]], + [-1, 1, Conv, [768, 3, 2]], # 7-P5/32 + [-1, 3, C3, [768]], + [-1, 1, Conv, [1024, 3, 2]], # 9-P6/64 + [-1, 3, C3, [1024]], + [-1, 1, SPPF, [1024, 5]], # 11 + ] + +# YOLOv5 v6.0 head +head: + [[-1, 1, Conv, [768, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 8], 1, Concat, [1]], # cat backbone P5 + [-1, 3, C3, [768, False]], # 15 + + [-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, C3, [512, False]], # 19 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, C3, [256, False]], # 23 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 20], 1, Concat, [1]], # cat head P4 + [-1, 3, C3, [512, False]], # 26 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 16], 1, Concat, [1]], # cat head P5 + [-1, 3, C3, [768, False]], # 29 (P5/32-large) + + [-1, 1, Conv, [768, 3, 2]], + [[-1, 12], 1, Concat, [1]], # cat head P6 + [-1, 3, C3, [1024, False]], # 32 (P6/64-xlarge) + + [[23, 26, 29, 32], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5, P6) + ] diff --git a/models/hub/yolov5x6.yaml b/models/hub/yolov5x6.yaml new file mode 100644 index 0000000..e1edbcb --- /dev/null +++ b/models/hub/yolov5x6.yaml @@ -0,0 +1,60 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# Parameters +nc: 80 # number of classes +depth_multiple: 1.33 # model depth multiple +width_multiple: 1.25 # layer channel multiple +anchors: + - [19,27, 44,40, 38,94] # P3/8 + - [96,68, 86,152, 180,137] # P4/16 + - [140,301, 303,264, 238,542] # P5/32 + - [436,615, 739,380, 925,792] # P6/64 + +# YOLOv5 v6.0 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [64, 6, 2, 2]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, C3, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 6, C3, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, C3, [512]], + [-1, 1, Conv, [768, 3, 2]], # 7-P5/32 + [-1, 3, C3, [768]], + [-1, 1, Conv, [1024, 3, 2]], # 9-P6/64 + [-1, 3, C3, [1024]], + [-1, 1, SPPF, [1024, 5]], # 11 + ] + +# YOLOv5 v6.0 head +head: + [[-1, 1, Conv, [768, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 8], 1, Concat, [1]], # cat backbone P5 + [-1, 3, C3, [768, False]], # 15 + + [-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, C3, [512, False]], # 19 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, C3, [256, False]], # 23 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 20], 1, Concat, [1]], # cat head P4 + [-1, 3, C3, [512, False]], # 26 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 16], 1, Concat, [1]], # cat head P5 + [-1, 3, C3, [768, False]], # 29 (P5/32-large) + + [-1, 1, Conv, [768, 3, 2]], + [[-1, 12], 1, Concat, [1]], # cat head P6 + [-1, 3, C3, [1024, False]], # 32 (P6/64-xlarge) + + [[23, 26, 29, 32], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5, P6) + ] diff --git a/models/segment/yolov5l-seg.yaml b/models/segment/yolov5l-seg.yaml new file mode 100644 index 0000000..71f80cc --- /dev/null +++ b/models/segment/yolov5l-seg.yaml @@ -0,0 +1,48 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# Parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# YOLOv5 v6.0 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [64, 6, 2, 2]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, C3, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 6, C3, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, C3, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 7-P5/32 + [-1, 3, C3, [1024]], + [-1, 1, SPPF, [1024, 5]], # 9 + ] + +# YOLOv5 v6.0 head +head: + [[-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, C3, [512, False]], # 13 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, C3, [256, False]], # 17 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 14], 1, Concat, [1]], # cat head P4 + [-1, 3, C3, [512, False]], # 20 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 10], 1, Concat, [1]], # cat head P5 + [-1, 3, C3, [1024, False]], # 23 (P5/32-large) + + [[17, 20, 23], 1, Segment, [nc, anchors, 32, 256]], # Detect(P3, P4, P5) + ] diff --git a/models/segment/yolov5m-seg.yaml b/models/segment/yolov5m-seg.yaml new file mode 100644 index 0000000..2b8e1db --- /dev/null +++ b/models/segment/yolov5m-seg.yaml @@ -0,0 +1,48 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# Parameters +nc: 80 # number of classes +depth_multiple: 0.67 # model depth multiple +width_multiple: 0.75 # layer channel multiple +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# YOLOv5 v6.0 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [64, 6, 2, 2]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, C3, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 6, C3, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, C3, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 7-P5/32 + [-1, 3, C3, [1024]], + [-1, 1, SPPF, [1024, 5]], # 9 + ] + +# YOLOv5 v6.0 head +head: + [[-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, C3, [512, False]], # 13 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, C3, [256, False]], # 17 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 14], 1, Concat, [1]], # cat head P4 + [-1, 3, C3, [512, False]], # 20 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 10], 1, Concat, [1]], # cat head P5 + [-1, 3, C3, [1024, False]], # 23 (P5/32-large) + + [[17, 20, 23], 1, Segment, [nc, anchors, 32, 256]], # Detect(P3, P4, P5) + ] diff --git a/models/segment/yolov5n-seg.yaml b/models/segment/yolov5n-seg.yaml new file mode 100644 index 0000000..1f67f8e --- /dev/null +++ b/models/segment/yolov5n-seg.yaml @@ -0,0 +1,48 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# Parameters +nc: 80 # number of classes +depth_multiple: 0.33 # model depth multiple +width_multiple: 0.25 # layer channel multiple +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# YOLOv5 v6.0 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [64, 6, 2, 2]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, C3, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 6, C3, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, C3, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 7-P5/32 + [-1, 3, C3, [1024]], + [-1, 1, SPPF, [1024, 5]], # 9 + ] + +# YOLOv5 v6.0 head +head: + [[-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, C3, [512, False]], # 13 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, C3, [256, False]], # 17 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 14], 1, Concat, [1]], # cat head P4 + [-1, 3, C3, [512, False]], # 20 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 10], 1, Concat, [1]], # cat head P5 + [-1, 3, C3, [1024, False]], # 23 (P5/32-large) + + [[17, 20, 23], 1, Segment, [nc, anchors, 32, 256]], # Detect(P3, P4, P5) + ] diff --git a/models/segment/yolov5s-seg.yaml b/models/segment/yolov5s-seg.yaml new file mode 100644 index 0000000..2ff2524 --- /dev/null +++ b/models/segment/yolov5s-seg.yaml @@ -0,0 +1,48 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# Parameters +nc: 80 # number of classes +depth_multiple: 0.33 # model depth multiple +width_multiple: 0.5 # layer channel multiple +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# YOLOv5 v6.0 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [64, 6, 2, 2]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, C3, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 6, C3, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, C3, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 7-P5/32 + [-1, 3, C3, [1024]], + [-1, 1, SPPF, [1024, 5]], # 9 + ] + +# YOLOv5 v6.0 head +head: + [[-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, C3, [512, False]], # 13 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, C3, [256, False]], # 17 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 14], 1, Concat, [1]], # cat head P4 + [-1, 3, C3, [512, False]], # 20 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 10], 1, Concat, [1]], # cat head P5 + [-1, 3, C3, [1024, False]], # 23 (P5/32-large) + + [[17, 20, 23], 1, Segment, [nc, anchors, 32, 256]], # Detect(P3, P4, P5) + ] diff --git a/models/segment/yolov5x-seg.yaml b/models/segment/yolov5x-seg.yaml new file mode 100644 index 0000000..589f65c --- /dev/null +++ b/models/segment/yolov5x-seg.yaml @@ -0,0 +1,48 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# Parameters +nc: 80 # number of classes +depth_multiple: 1.33 # model depth multiple +width_multiple: 1.25 # layer channel multiple +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# YOLOv5 v6.0 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [64, 6, 2, 2]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, C3, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 6, C3, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, C3, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 7-P5/32 + [-1, 3, C3, [1024]], + [-1, 1, SPPF, [1024, 5]], # 9 + ] + +# YOLOv5 v6.0 head +head: + [[-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, C3, [512, False]], # 13 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, C3, [256, False]], # 17 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 14], 1, Concat, [1]], # cat head P4 + [-1, 3, C3, [512, False]], # 20 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 10], 1, Concat, [1]], # cat head P5 + [-1, 3, C3, [1024, False]], # 23 (P5/32-large) + + [[17, 20, 23], 1, Segment, [nc, anchors, 32, 256]], # Detect(P3, P4, P5) + ] diff --git a/models/tf.py b/models/tf.py new file mode 100644 index 0000000..62ba3eb --- /dev/null +++ b/models/tf.py @@ -0,0 +1,608 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +TensorFlow, Keras and TFLite versions of YOLOv5 +Authored by https://github.com/zldrobit in PR https://github.com/ultralytics/yolov5/pull/1127 + +Usage: + $ python models/tf.py --weights yolov5s.pt + +Export: + $ python export.py --weights yolov5s.pt --include saved_model pb tflite tfjs +""" + +import argparse +import sys +from copy import deepcopy +from pathlib import Path + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[1] # YOLOv5 root directory +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) # add ROOT to PATH +# ROOT = ROOT.relative_to(Path.cwd()) # relative + +import numpy as np +import tensorflow as tf +import torch +import torch.nn as nn +from tensorflow import keras + +from models.common import (C3, SPP, SPPF, Bottleneck, BottleneckCSP, C3x, Concat, Conv, CrossConv, DWConv, + DWConvTranspose2d, Focus, autopad) +from models.experimental import MixConv2d, attempt_load +from models.yolo import Detect, Segment +from utils.activations import SiLU +from utils.general import LOGGER, make_divisible, print_args + + +class TFBN(keras.layers.Layer): + # TensorFlow BatchNormalization wrapper + def __init__(self, w=None): + super().__init__() + self.bn = keras.layers.BatchNormalization( + beta_initializer=keras.initializers.Constant(w.bias.numpy()), + gamma_initializer=keras.initializers.Constant(w.weight.numpy()), + moving_mean_initializer=keras.initializers.Constant(w.running_mean.numpy()), + moving_variance_initializer=keras.initializers.Constant(w.running_var.numpy()), + epsilon=w.eps) + + def call(self, inputs): + return self.bn(inputs) + + +class TFPad(keras.layers.Layer): + # Pad inputs in spatial dimensions 1 and 2 + def __init__(self, pad): + super().__init__() + if isinstance(pad, int): + self.pad = tf.constant([[0, 0], [pad, pad], [pad, pad], [0, 0]]) + else: # tuple/list + self.pad = tf.constant([[0, 0], [pad[0], pad[0]], [pad[1], pad[1]], [0, 0]]) + + def call(self, inputs): + return tf.pad(inputs, self.pad, mode='constant', constant_values=0) + + +class TFConv(keras.layers.Layer): + # Standard convolution + def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True, w=None): + # ch_in, ch_out, weights, kernel, stride, padding, groups + super().__init__() + assert g == 1, "TF v2.2 Conv2D does not support 'groups' argument" + # TensorFlow convolution padding is inconsistent with PyTorch (e.g. k=3 s=2 'SAME' padding) + # see https://stackoverflow.com/questions/52975843/comparing-conv2d-with-padding-between-tensorflow-and-pytorch + conv = keras.layers.Conv2D( + filters=c2, + kernel_size=k, + strides=s, + padding='SAME' if s == 1 else 'VALID', + use_bias=not hasattr(w, 'bn'), + kernel_initializer=keras.initializers.Constant(w.conv.weight.permute(2, 3, 1, 0).numpy()), + bias_initializer='zeros' if hasattr(w, 'bn') else keras.initializers.Constant(w.conv.bias.numpy())) + self.conv = conv if s == 1 else keras.Sequential([TFPad(autopad(k, p)), conv]) + self.bn = TFBN(w.bn) if hasattr(w, 'bn') else tf.identity + self.act = activations(w.act) if act else tf.identity + + def call(self, inputs): + return self.act(self.bn(self.conv(inputs))) + + +class TFDWConv(keras.layers.Layer): + # Depthwise convolution + def __init__(self, c1, c2, k=1, s=1, p=None, act=True, w=None): + # ch_in, ch_out, weights, kernel, stride, padding, groups + super().__init__() + assert c2 % c1 == 0, f'TFDWConv() output={c2} must be a multiple of input={c1} channels' + conv = keras.layers.DepthwiseConv2D( + kernel_size=k, + depth_multiplier=c2 // c1, + strides=s, + padding='SAME' if s == 1 else 'VALID', + use_bias=not hasattr(w, 'bn'), + depthwise_initializer=keras.initializers.Constant(w.conv.weight.permute(2, 3, 1, 0).numpy()), + bias_initializer='zeros' if hasattr(w, 'bn') else keras.initializers.Constant(w.conv.bias.numpy())) + self.conv = conv if s == 1 else keras.Sequential([TFPad(autopad(k, p)), conv]) + self.bn = TFBN(w.bn) if hasattr(w, 'bn') else tf.identity + self.act = activations(w.act) if act else tf.identity + + def call(self, inputs): + return self.act(self.bn(self.conv(inputs))) + + +class TFDWConvTranspose2d(keras.layers.Layer): + # Depthwise ConvTranspose2d + def __init__(self, c1, c2, k=1, s=1, p1=0, p2=0, w=None): + # ch_in, ch_out, weights, kernel, stride, padding, groups + super().__init__() + assert c1 == c2, f'TFDWConv() output={c2} must be equal to input={c1} channels' + assert k == 4 and p1 == 1, 'TFDWConv() only valid for k=4 and p1=1' + weight, bias = w.weight.permute(2, 3, 1, 0).numpy(), w.bias.numpy() + self.c1 = c1 + self.conv = [ + keras.layers.Conv2DTranspose(filters=1, + kernel_size=k, + strides=s, + padding='VALID', + output_padding=p2, + use_bias=True, + kernel_initializer=keras.initializers.Constant(weight[..., i:i + 1]), + bias_initializer=keras.initializers.Constant(bias[i])) for i in range(c1)] + + def call(self, inputs): + return tf.concat([m(x) for m, x in zip(self.conv, tf.split(inputs, self.c1, 3))], 3)[:, 1:-1, 1:-1] + + +class TFFocus(keras.layers.Layer): + # Focus wh information into c-space + def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True, w=None): + # ch_in, ch_out, kernel, stride, padding, groups + super().__init__() + self.conv = TFConv(c1 * 4, c2, k, s, p, g, act, w.conv) + + def call(self, inputs): # x(b,w,h,c) -> y(b,w/2,h/2,4c) + # inputs = inputs / 255 # normalize 0-255 to 0-1 + inputs = [inputs[:, ::2, ::2, :], inputs[:, 1::2, ::2, :], inputs[:, ::2, 1::2, :], inputs[:, 1::2, 1::2, :]] + return self.conv(tf.concat(inputs, 3)) + + +class TFBottleneck(keras.layers.Layer): + # Standard bottleneck + def __init__(self, c1, c2, shortcut=True, g=1, e=0.5, w=None): # ch_in, ch_out, shortcut, groups, expansion + super().__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = TFConv(c1, c_, 1, 1, w=w.cv1) + self.cv2 = TFConv(c_, c2, 3, 1, g=g, w=w.cv2) + self.add = shortcut and c1 == c2 + + def call(self, inputs): + return inputs + self.cv2(self.cv1(inputs)) if self.add else self.cv2(self.cv1(inputs)) + + +class TFCrossConv(keras.layers.Layer): + # Cross Convolution + def __init__(self, c1, c2, k=3, s=1, g=1, e=1.0, shortcut=False, w=None): + super().__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = TFConv(c1, c_, (1, k), (1, s), w=w.cv1) + self.cv2 = TFConv(c_, c2, (k, 1), (s, 1), g=g, w=w.cv2) + self.add = shortcut and c1 == c2 + + def call(self, inputs): + return inputs + self.cv2(self.cv1(inputs)) if self.add else self.cv2(self.cv1(inputs)) + + +class TFConv2d(keras.layers.Layer): + # Substitution for PyTorch nn.Conv2D + def __init__(self, c1, c2, k, s=1, g=1, bias=True, w=None): + super().__init__() + assert g == 1, "TF v2.2 Conv2D does not support 'groups' argument" + self.conv = keras.layers.Conv2D(filters=c2, + kernel_size=k, + strides=s, + padding='VALID', + use_bias=bias, + kernel_initializer=keras.initializers.Constant( + w.weight.permute(2, 3, 1, 0).numpy()), + bias_initializer=keras.initializers.Constant(w.bias.numpy()) if bias else None) + + def call(self, inputs): + return self.conv(inputs) + + +class TFBottleneckCSP(keras.layers.Layer): + # CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5, w=None): + # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = TFConv(c1, c_, 1, 1, w=w.cv1) + self.cv2 = TFConv2d(c1, c_, 1, 1, bias=False, w=w.cv2) + self.cv3 = TFConv2d(c_, c_, 1, 1, bias=False, w=w.cv3) + self.cv4 = TFConv(2 * c_, c2, 1, 1, w=w.cv4) + self.bn = TFBN(w.bn) + self.act = lambda x: keras.activations.swish(x) + self.m = keras.Sequential([TFBottleneck(c_, c_, shortcut, g, e=1.0, w=w.m[j]) for j in range(n)]) + + def call(self, inputs): + y1 = self.cv3(self.m(self.cv1(inputs))) + y2 = self.cv2(inputs) + return self.cv4(self.act(self.bn(tf.concat((y1, y2), axis=3)))) + + +class TFC3(keras.layers.Layer): + # CSP Bottleneck with 3 convolutions + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5, w=None): + # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = TFConv(c1, c_, 1, 1, w=w.cv1) + self.cv2 = TFConv(c1, c_, 1, 1, w=w.cv2) + self.cv3 = TFConv(2 * c_, c2, 1, 1, w=w.cv3) + self.m = keras.Sequential([TFBottleneck(c_, c_, shortcut, g, e=1.0, w=w.m[j]) for j in range(n)]) + + def call(self, inputs): + return self.cv3(tf.concat((self.m(self.cv1(inputs)), self.cv2(inputs)), axis=3)) + + +class TFC3x(keras.layers.Layer): + # 3 module with cross-convolutions + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5, w=None): + # ch_in, ch_out, number, shortcut, groups, expansion + super().__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = TFConv(c1, c_, 1, 1, w=w.cv1) + self.cv2 = TFConv(c1, c_, 1, 1, w=w.cv2) + self.cv3 = TFConv(2 * c_, c2, 1, 1, w=w.cv3) + self.m = keras.Sequential([ + TFCrossConv(c_, c_, k=3, s=1, g=g, e=1.0, shortcut=shortcut, w=w.m[j]) for j in range(n)]) + + def call(self, inputs): + return self.cv3(tf.concat((self.m(self.cv1(inputs)), self.cv2(inputs)), axis=3)) + + +class TFSPP(keras.layers.Layer): + # Spatial pyramid pooling layer used in YOLOv3-SPP + def __init__(self, c1, c2, k=(5, 9, 13), w=None): + super().__init__() + c_ = c1 // 2 # hidden channels + self.cv1 = TFConv(c1, c_, 1, 1, w=w.cv1) + self.cv2 = TFConv(c_ * (len(k) + 1), c2, 1, 1, w=w.cv2) + self.m = [keras.layers.MaxPool2D(pool_size=x, strides=1, padding='SAME') for x in k] + + def call(self, inputs): + x = self.cv1(inputs) + return self.cv2(tf.concat([x] + [m(x) for m in self.m], 3)) + + +class TFSPPF(keras.layers.Layer): + # Spatial pyramid pooling-Fast layer + def __init__(self, c1, c2, k=5, w=None): + super().__init__() + c_ = c1 // 2 # hidden channels + self.cv1 = TFConv(c1, c_, 1, 1, w=w.cv1) + self.cv2 = TFConv(c_ * 4, c2, 1, 1, w=w.cv2) + self.m = keras.layers.MaxPool2D(pool_size=k, strides=1, padding='SAME') + + def call(self, inputs): + x = self.cv1(inputs) + y1 = self.m(x) + y2 = self.m(y1) + return self.cv2(tf.concat([x, y1, y2, self.m(y2)], 3)) + + +class TFDetect(keras.layers.Layer): + # TF YOLOv5 Detect layer + def __init__(self, nc=80, anchors=(), ch=(), imgsz=(640, 640), w=None): # detection layer + super().__init__() + self.stride = tf.convert_to_tensor(w.stride.numpy(), dtype=tf.float32) + self.nc = nc # number of classes + self.no = nc + 5 # number of outputs per anchor + self.nl = len(anchors) # number of detection layers + self.na = len(anchors[0]) // 2 # number of anchors + self.grid = [tf.zeros(1)] * self.nl # init grid + self.anchors = tf.convert_to_tensor(w.anchors.numpy(), dtype=tf.float32) + self.anchor_grid = tf.reshape(self.anchors * tf.reshape(self.stride, [self.nl, 1, 1]), [self.nl, 1, -1, 1, 2]) + self.m = [TFConv2d(x, self.no * self.na, 1, w=w.m[i]) for i, x in enumerate(ch)] + self.training = False # set to False after building model + self.imgsz = imgsz + for i in range(self.nl): + ny, nx = self.imgsz[0] // self.stride[i], self.imgsz[1] // self.stride[i] + self.grid[i] = self._make_grid(nx, ny) + + def call(self, inputs): + z = [] # inference output + x = [] + for i in range(self.nl): + x.append(self.m[i](inputs[i])) + # x(bs,20,20,255) to x(bs,3,20,20,85) + ny, nx = self.imgsz[0] // self.stride[i], self.imgsz[1] // self.stride[i] + x[i] = tf.reshape(x[i], [-1, ny * nx, self.na, self.no]) + + if not self.training: # inference + y = x[i] + grid = tf.transpose(self.grid[i], [0, 2, 1, 3]) - 0.5 + anchor_grid = tf.transpose(self.anchor_grid[i], [0, 2, 1, 3]) * 4 + xy = (tf.sigmoid(y[..., 0:2]) * 2 + grid) * self.stride[i] # xy + wh = tf.sigmoid(y[..., 2:4]) ** 2 * anchor_grid + # Normalize xywh to 0-1 to reduce calibration error + xy /= tf.constant([[self.imgsz[1], self.imgsz[0]]], dtype=tf.float32) + wh /= tf.constant([[self.imgsz[1], self.imgsz[0]]], dtype=tf.float32) + y = tf.concat([xy, wh, tf.sigmoid(y[..., 4:5 + self.nc]), y[..., 5 + self.nc:]], -1) + z.append(tf.reshape(y, [-1, self.na * ny * nx, self.no])) + + return tf.transpose(x, [0, 2, 1, 3]) if self.training else (tf.concat(z, 1), ) + + @staticmethod + def _make_grid(nx=20, ny=20): + # yv, xv = torch.meshgrid([torch.arange(ny), torch.arange(nx)]) + # return torch.stack((xv, yv), 2).view((1, 1, ny, nx, 2)).float() + xv, yv = tf.meshgrid(tf.range(nx), tf.range(ny)) + return tf.cast(tf.reshape(tf.stack([xv, yv], 2), [1, 1, ny * nx, 2]), dtype=tf.float32) + + +class TFSegment(TFDetect): + # YOLOv5 Segment head for segmentation models + def __init__(self, nc=80, anchors=(), nm=32, npr=256, ch=(), imgsz=(640, 640), w=None): + super().__init__(nc, anchors, ch, imgsz, w) + self.nm = nm # number of masks + self.npr = npr # number of protos + self.no = 5 + nc + self.nm # number of outputs per anchor + self.m = [TFConv2d(x, self.no * self.na, 1, w=w.m[i]) for i, x in enumerate(ch)] # output conv + self.proto = TFProto(ch[0], self.npr, self.nm, w=w.proto) # protos + self.detect = TFDetect.call + + def call(self, x): + p = self.proto(x[0]) + # p = TFUpsample(None, scale_factor=4, mode='nearest')(self.proto(x[0])) # (optional) full-size protos + p = tf.transpose(p, [0, 3, 1, 2]) # from shape(1,160,160,32) to shape(1,32,160,160) + x = self.detect(self, x) + return (x, p) if self.training else (x[0], p) + + +class TFProto(keras.layers.Layer): + + def __init__(self, c1, c_=256, c2=32, w=None): + super().__init__() + self.cv1 = TFConv(c1, c_, k=3, w=w.cv1) + self.upsample = TFUpsample(None, scale_factor=2, mode='nearest') + self.cv2 = TFConv(c_, c_, k=3, w=w.cv2) + self.cv3 = TFConv(c_, c2, w=w.cv3) + + def call(self, inputs): + return self.cv3(self.cv2(self.upsample(self.cv1(inputs)))) + + +class TFUpsample(keras.layers.Layer): + # TF version of torch.nn.Upsample() + def __init__(self, size, scale_factor, mode, w=None): # warning: all arguments needed including 'w' + super().__init__() + assert scale_factor % 2 == 0, 'scale_factor must be multiple of 2' + self.upsample = lambda x: tf.image.resize(x, (x.shape[1] * scale_factor, x.shape[2] * scale_factor), mode) + # self.upsample = keras.layers.UpSampling2D(size=scale_factor, interpolation=mode) + # with default arguments: align_corners=False, half_pixel_centers=False + # self.upsample = lambda x: tf.raw_ops.ResizeNearestNeighbor(images=x, + # size=(x.shape[1] * 2, x.shape[2] * 2)) + + def call(self, inputs): + return self.upsample(inputs) + + +class TFConcat(keras.layers.Layer): + # TF version of torch.concat() + def __init__(self, dimension=1, w=None): + super().__init__() + assert dimension == 1, 'convert only NCHW to NHWC concat' + self.d = 3 + + def call(self, inputs): + return tf.concat(inputs, self.d) + + +def parse_model(d, ch, model, imgsz): # model_dict, input_channels(3) + LOGGER.info(f"\n{'':>3}{'from':>18}{'n':>3}{'params':>10} {'module':<40}{'arguments':<30}") + anchors, nc, gd, gw = d['anchors'], d['nc'], d['depth_multiple'], d['width_multiple'] + na = (len(anchors[0]) // 2) if isinstance(anchors, list) else anchors # number of anchors + no = na * (nc + 5) # number of outputs = anchors * (classes + 5) + + layers, save, c2 = [], [], ch[-1] # layers, savelist, ch out + for i, (f, n, m, args) in enumerate(d['backbone'] + d['head']): # from, number, module, args + m_str = m + m = eval(m) if isinstance(m, str) else m # eval strings + for j, a in enumerate(args): + try: + args[j] = eval(a) if isinstance(a, str) else a # eval strings + except NameError: + pass + + n = max(round(n * gd), 1) if n > 1 else n # depth gain + if m in [ + nn.Conv2d, Conv, DWConv, DWConvTranspose2d, Bottleneck, SPP, SPPF, MixConv2d, Focus, CrossConv, + BottleneckCSP, C3, C3x]: + c1, c2 = ch[f], args[0] + c2 = make_divisible(c2 * gw, 8) if c2 != no else c2 + + args = [c1, c2, *args[1:]] + if m in [BottleneckCSP, C3, C3x]: + args.insert(2, n) + n = 1 + elif m is nn.BatchNorm2d: + args = [ch[f]] + elif m is Concat: + c2 = sum(ch[-1 if x == -1 else x + 1] for x in f) + elif m in [Detect, Segment]: + args.append([ch[x + 1] for x in f]) + if isinstance(args[1], int): # number of anchors + args[1] = [list(range(args[1] * 2))] * len(f) + if m is Segment: + args[3] = make_divisible(args[3] * gw, 8) + args.append(imgsz) + else: + c2 = ch[f] + + tf_m = eval('TF' + m_str.replace('nn.', '')) + m_ = keras.Sequential([tf_m(*args, w=model.model[i][j]) for j in range(n)]) if n > 1 \ + else tf_m(*args, w=model.model[i]) # module + + torch_m_ = nn.Sequential(*(m(*args) for _ in range(n))) if n > 1 else m(*args) # module + t = str(m)[8:-2].replace('__main__.', '') # module type + np = sum(x.numel() for x in torch_m_.parameters()) # number params + m_.i, m_.f, m_.type, m_.np = i, f, t, np # attach index, 'from' index, type, number params + LOGGER.info(f'{i:>3}{str(f):>18}{str(n):>3}{np:>10} {t:<40}{str(args):<30}') # print + save.extend(x % i for x in ([f] if isinstance(f, int) else f) if x != -1) # append to savelist + layers.append(m_) + ch.append(c2) + return keras.Sequential(layers), sorted(save) + + +class TFModel: + # TF YOLOv5 model + def __init__(self, cfg='yolov5s.yaml', ch=3, nc=None, model=None, imgsz=(640, 640)): # model, channels, classes + super().__init__() + if isinstance(cfg, dict): + self.yaml = cfg # model dict + else: # is *.yaml + import yaml # for torch hub + self.yaml_file = Path(cfg).name + with open(cfg) as f: + self.yaml = yaml.load(f, Loader=yaml.FullLoader) # model dict + + # Define model + if nc and nc != self.yaml['nc']: + LOGGER.info(f"Overriding {cfg} nc={self.yaml['nc']} with nc={nc}") + self.yaml['nc'] = nc # override yaml value + self.model, self.savelist = parse_model(deepcopy(self.yaml), ch=[ch], model=model, imgsz=imgsz) + + def predict(self, + inputs, + tf_nms=False, + agnostic_nms=False, + topk_per_class=100, + topk_all=100, + iou_thres=0.45, + conf_thres=0.25): + y = [] # outputs + x = inputs + for m in self.model.layers: + if m.f != -1: # if not from previous layer + x = y[m.f] if isinstance(m.f, int) else [x if j == -1 else y[j] for j in m.f] # from earlier layers + + x = m(x) # run + y.append(x if m.i in self.savelist else None) # save output + + # Add TensorFlow NMS + if tf_nms: + boxes = self._xywh2xyxy(x[0][..., :4]) + probs = x[0][:, :, 4:5] + classes = x[0][:, :, 5:] + scores = probs * classes + if agnostic_nms: + nms = AgnosticNMS()((boxes, classes, scores), topk_all, iou_thres, conf_thres) + else: + boxes = tf.expand_dims(boxes, 2) + nms = tf.image.combined_non_max_suppression(boxes, + scores, + topk_per_class, + topk_all, + iou_thres, + conf_thres, + clip_boxes=False) + return (nms, ) + return x # output [1,6300,85] = [xywh, conf, class0, class1, ...] + # x = x[0] # [x(1,6300,85), ...] to x(6300,85) + # xywh = x[..., :4] # x(6300,4) boxes + # conf = x[..., 4:5] # x(6300,1) confidences + # cls = tf.reshape(tf.cast(tf.argmax(x[..., 5:], axis=1), tf.float32), (-1, 1)) # x(6300,1) classes + # return tf.concat([conf, cls, xywh], 1) + + @staticmethod + def _xywh2xyxy(xywh): + # Convert nx4 boxes from [x, y, w, h] to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right + x, y, w, h = tf.split(xywh, num_or_size_splits=4, axis=-1) + return tf.concat([x - w / 2, y - h / 2, x + w / 2, y + h / 2], axis=-1) + + +class AgnosticNMS(keras.layers.Layer): + # TF Agnostic NMS + def call(self, input, topk_all, iou_thres, conf_thres): + # wrap map_fn to avoid TypeSpec related error https://stackoverflow.com/a/65809989/3036450 + return tf.map_fn(lambda x: self._nms(x, topk_all, iou_thres, conf_thres), + input, + fn_output_signature=(tf.float32, tf.float32, tf.float32, tf.int32), + name='agnostic_nms') + + @staticmethod + def _nms(x, topk_all=100, iou_thres=0.45, conf_thres=0.25): # agnostic NMS + boxes, classes, scores = x + class_inds = tf.cast(tf.argmax(classes, axis=-1), tf.float32) + scores_inp = tf.reduce_max(scores, -1) + selected_inds = tf.image.non_max_suppression(boxes, + scores_inp, + max_output_size=topk_all, + iou_threshold=iou_thres, + score_threshold=conf_thres) + selected_boxes = tf.gather(boxes, selected_inds) + padded_boxes = tf.pad(selected_boxes, + paddings=[[0, topk_all - tf.shape(selected_boxes)[0]], [0, 0]], + mode='CONSTANT', + constant_values=0.0) + selected_scores = tf.gather(scores_inp, selected_inds) + padded_scores = tf.pad(selected_scores, + paddings=[[0, topk_all - tf.shape(selected_boxes)[0]]], + mode='CONSTANT', + constant_values=-1.0) + selected_classes = tf.gather(class_inds, selected_inds) + padded_classes = tf.pad(selected_classes, + paddings=[[0, topk_all - tf.shape(selected_boxes)[0]]], + mode='CONSTANT', + constant_values=-1.0) + valid_detections = tf.shape(selected_inds)[0] + return padded_boxes, padded_scores, padded_classes, valid_detections + + +def activations(act=nn.SiLU): + # Returns TF activation from input PyTorch activation + if isinstance(act, nn.LeakyReLU): + return lambda x: keras.activations.relu(x, alpha=0.1) + elif isinstance(act, nn.Hardswish): + return lambda x: x * tf.nn.relu6(x + 3) * 0.166666667 + elif isinstance(act, (nn.SiLU, SiLU)): + return lambda x: keras.activations.swish(x) + else: + raise Exception(f'no matching TensorFlow activation found for PyTorch activation {act}') + + +def representative_dataset_gen(dataset, ncalib=100): + # Representative dataset generator for use with converter.representative_dataset, returns a generator of np arrays + for n, (path, img, im0s, vid_cap, string) in enumerate(dataset): + im = np.transpose(img, [1, 2, 0]) + im = np.expand_dims(im, axis=0).astype(np.float32) + im /= 255 + yield [im] + if n >= ncalib: + break + + +def run( + weights=ROOT / 'yolov5s.pt', # weights path + imgsz=(640, 640), # inference size h,w + batch_size=1, # batch size + dynamic=False, # dynamic batch size +): + # PyTorch model + im = torch.zeros((batch_size, 3, *imgsz)) # BCHW image + model = attempt_load(weights, device=torch.device('cpu'), inplace=True, fuse=False) + _ = model(im) # inference + model.info() + + # TensorFlow model + im = tf.zeros((batch_size, *imgsz, 3)) # BHWC image + tf_model = TFModel(cfg=model.yaml, model=model, nc=model.nc, imgsz=imgsz) + _ = tf_model.predict(im) # inference + + # Keras model + im = keras.Input(shape=(*imgsz, 3), batch_size=None if dynamic else batch_size) + keras_model = keras.Model(inputs=im, outputs=tf_model.predict(im)) + keras_model.summary() + + LOGGER.info('PyTorch, TensorFlow and Keras models successfully verified.\nUse export.py for TF model export.') + + +def parse_opt(): + parser = argparse.ArgumentParser() + parser.add_argument('--weights', type=str, default=ROOT / 'yolov5s.pt', help='weights path') + parser.add_argument('--imgsz', '--img', '--img-size', nargs='+', type=int, default=[640], help='inference size h,w') + parser.add_argument('--batch-size', type=int, default=1, help='batch size') + parser.add_argument('--dynamic', action='store_true', help='dynamic batch size') + opt = parser.parse_args() + opt.imgsz *= 2 if len(opt.imgsz) == 1 else 1 # expand + print_args(vars(opt)) + return opt + + +def main(opt): + run(**vars(opt)) + + +if __name__ == '__main__': + opt = parse_opt() + main(opt) diff --git a/models/yolo.py b/models/yolo.py new file mode 100644 index 0000000..4f4d567 --- /dev/null +++ b/models/yolo.py @@ -0,0 +1,391 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +YOLO-specific modules + +Usage: + $ python models/yolo.py --cfg yolov5s.yaml +""" + +import argparse +import contextlib +import os +import platform +import sys +from copy import deepcopy +from pathlib import Path + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[1] # YOLOv5 root directory +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) # add ROOT to PATH +if platform.system() != 'Windows': + ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative + +from models.common import * # noqa +from models.experimental import * # noqa +from utils.autoanchor import check_anchor_order +from utils.general import LOGGER, check_version, check_yaml, make_divisible, print_args +from utils.plots import feature_visualization +from utils.torch_utils import (fuse_conv_and_bn, initialize_weights, model_info, profile, scale_img, select_device, + time_sync) + +try: + import thop # for FLOPs computation +except ImportError: + thop = None + + +class Detect(nn.Module): + # YOLOv5 Detect head for detection models + stride = None # strides computed during build + dynamic = False # force grid reconstruction + export = False # export mode + + def __init__(self, nc=80, anchors=(), ch=(), inplace=True): # detection layer + super().__init__() + self.nc = nc # number of classes + self.no = nc + 5 # number of outputs per anchor + self.nl = len(anchors) # number of detection layers + self.na = len(anchors[0]) // 2 # number of anchors + self.grid = [torch.empty(0) for _ in range(self.nl)] # init grid + self.anchor_grid = [torch.empty(0) for _ in range(self.nl)] # init anchor grid + self.register_buffer('anchors', torch.tensor(anchors).float().view(self.nl, -1, 2)) # shape(nl,na,2) + self.m = nn.ModuleList(nn.Conv2d(x, self.no * self.na, 1) for x in ch) # output conv + self.inplace = inplace # use inplace ops (e.g. slice assignment) + + def forward(self, x): + z = [] # inference output + for i in range(self.nl): + x[i] = self.m[i](x[i]) # conv + bs, _, ny, nx = x[i].shape # x(bs,255,20,20) to x(bs,3,20,20,85) + x[i] = x[i].view(bs, self.na, self.no, ny, nx).permute(0, 1, 3, 4, 2).contiguous() + + if not self.training: # inference + if self.dynamic or self.grid[i].shape[2:4] != x[i].shape[2:4]: + self.grid[i], self.anchor_grid[i] = self._make_grid(nx, ny, i) + + if isinstance(self, Segment): # (boxes + masks) + xy, wh, conf, mask = x[i].split((2, 2, self.nc + 1, self.no - self.nc - 5), 4) + xy = (xy.sigmoid() * 2 + self.grid[i]) * self.stride[i] # xy + wh = (wh.sigmoid() * 2) ** 2 * self.anchor_grid[i] # wh + y = torch.cat((xy, wh, conf.sigmoid(), mask), 4) + else: # Detect (boxes only) + xy, wh, conf = x[i].sigmoid().split((2, 2, self.nc + 1), 4) + xy = (xy * 2 + self.grid[i]) * self.stride[i] # xy + wh = (wh * 2) ** 2 * self.anchor_grid[i] # wh + y = torch.cat((xy, wh, conf), 4) + z.append(y.view(bs, self.na * nx * ny, self.no)) + + return x if self.training else (torch.cat(z, 1), ) if self.export else (torch.cat(z, 1), x) + + def _make_grid(self, nx=20, ny=20, i=0, torch_1_10=check_version(torch.__version__, '1.10.0')): + d = self.anchors[i].device + t = self.anchors[i].dtype + shape = 1, self.na, ny, nx, 2 # grid shape + y, x = torch.arange(ny, device=d, dtype=t), torch.arange(nx, device=d, dtype=t) + yv, xv = torch.meshgrid(y, x, indexing='ij') if torch_1_10 else torch.meshgrid(y, x) # torch>=0.7 compatibility + grid = torch.stack((xv, yv), 2).expand(shape) - 0.5 # add grid offset, i.e. y = 2.0 * x - 0.5 + anchor_grid = (self.anchors[i] * self.stride[i]).view((1, self.na, 1, 1, 2)).expand(shape) + return grid, anchor_grid + + +class Segment(Detect): + # YOLOv5 Segment head for segmentation models + def __init__(self, nc=80, anchors=(), nm=32, npr=256, ch=(), inplace=True): + super().__init__(nc, anchors, ch, inplace) + self.nm = nm # number of masks + self.npr = npr # number of protos + self.no = 5 + nc + self.nm # number of outputs per anchor + self.m = nn.ModuleList(nn.Conv2d(x, self.no * self.na, 1) for x in ch) # output conv + self.proto = Proto(ch[0], self.npr, self.nm) # protos + self.detect = Detect.forward + + def forward(self, x): + p = self.proto(x[0]) + x = self.detect(self, x) + return (x, p) if self.training else (x[0], p) if self.export else (x[0], p, x[1]) + + +class BaseModel(nn.Module): + # YOLOv5 base model + def forward(self, x, profile=False, visualize=False): + return self._forward_once(x, profile, visualize) # single-scale inference, train + + def _forward_once(self, x, profile=False, visualize=False): + y, dt = [], [] # outputs + for m in self.model: + if m.f != -1: # if not from previous layer + x = y[m.f] if isinstance(m.f, int) else [x if j == -1 else y[j] for j in m.f] # from earlier layers + if profile: + self._profile_one_layer(m, x, dt) + x = m(x) # run + y.append(x if m.i in self.save else None) # save output + if visualize: + feature_visualization(x, m.type, m.i, save_dir=visualize) + return x + + def _profile_one_layer(self, m, x, dt): + c = m == self.model[-1] # is final layer, copy input as inplace fix + o = thop.profile(m, inputs=(x.copy() if c else x, ), verbose=False)[0] / 1E9 * 2 if thop else 0 # FLOPs + t = time_sync() + for _ in range(10): + m(x.copy() if c else x) + dt.append((time_sync() - t) * 100) + if m == self.model[0]: + LOGGER.info(f"{'time (ms)':>10s} {'GFLOPs':>10s} {'params':>10s} module") + LOGGER.info(f'{dt[-1]:10.2f} {o:10.2f} {m.np:10.0f} {m.type}') + if c: + LOGGER.info(f"{sum(dt):10.2f} {'-':>10s} {'-':>10s} Total") + + def fuse(self): # fuse model Conv2d() + BatchNorm2d() layers + LOGGER.info('Fusing layers... ') + for m in self.model.modules(): + if isinstance(m, (Conv, DWConv)) and hasattr(m, 'bn'): + m.conv = fuse_conv_and_bn(m.conv, m.bn) # update conv + delattr(m, 'bn') # remove batchnorm + m.forward = m.forward_fuse # update forward + self.info() + return self + + def info(self, verbose=False, img_size=640): # print model information + model_info(self, verbose, img_size) + + def _apply(self, fn): + # Apply to(), cpu(), cuda(), half() to model tensors that are not parameters or registered buffers + self = super()._apply(fn) + m = self.model[-1] # Detect() + if isinstance(m, (Detect, Segment)): + m.stride = fn(m.stride) + m.grid = list(map(fn, m.grid)) + if isinstance(m.anchor_grid, list): + m.anchor_grid = list(map(fn, m.anchor_grid)) + return self + + +class DetectionModel(BaseModel): + # YOLOv5 detection model + def __init__(self, cfg='yolov5s.yaml', ch=3, nc=None, anchors=None): # model, input channels, number of classes + super().__init__() + if isinstance(cfg, dict): + self.yaml = cfg # model dict + else: # is *.yaml + import yaml # for torch hub + self.yaml_file = Path(cfg).name + with open(cfg, encoding='ascii', errors='ignore') as f: + self.yaml = yaml.safe_load(f) # model dict + + # Define model + ch = self.yaml['ch'] = self.yaml.get('ch', ch) # input channels + if nc and nc != self.yaml['nc']: + LOGGER.info(f"Overriding model.yaml nc={self.yaml['nc']} with nc={nc}") + self.yaml['nc'] = nc # override yaml value + if anchors: + LOGGER.info(f'Overriding model.yaml anchors with anchors={anchors}') + self.yaml['anchors'] = round(anchors) # override yaml value + self.model, self.save = parse_model(deepcopy(self.yaml), ch=[ch]) # model, savelist + self.names = [str(i) for i in range(self.yaml['nc'])] # default names + self.inplace = self.yaml.get('inplace', True) + + # Build strides, anchors + m = self.model[-1] # Detect() + if isinstance(m, (Detect, Segment)): + s = 256 # 2x min stride + m.inplace = self.inplace + forward = lambda x: self.forward(x)[0] if isinstance(m, Segment) else self.forward(x) + m.stride = torch.tensor([s / x.shape[-2] for x in forward(torch.zeros(1, ch, s, s))]) # forward + check_anchor_order(m) + m.anchors /= m.stride.view(-1, 1, 1) + self.stride = m.stride + self._initialize_biases() # only run once + + # Init weights, biases + initialize_weights(self) + self.info() + LOGGER.info('') + + def forward(self, x, augment=False, profile=False, visualize=False): + if augment: + return self._forward_augment(x) # augmented inference, None + return self._forward_once(x, profile, visualize) # single-scale inference, train + + def _forward_augment(self, x): + img_size = x.shape[-2:] # height, width + s = [1, 0.83, 0.67] # scales + f = [None, 3, None] # flips (2-ud, 3-lr) + y = [] # outputs + for si, fi in zip(s, f): + xi = scale_img(x.flip(fi) if fi else x, si, gs=int(self.stride.max())) + yi = self._forward_once(xi)[0] # forward + # cv2.imwrite(f'img_{si}.jpg', 255 * xi[0].cpu().numpy().transpose((1, 2, 0))[:, :, ::-1]) # save + yi = self._descale_pred(yi, fi, si, img_size) + y.append(yi) + y = self._clip_augmented(y) # clip augmented tails + return torch.cat(y, 1), None # augmented inference, train + + def _descale_pred(self, p, flips, scale, img_size): + # de-scale predictions following augmented inference (inverse operation) + if self.inplace: + p[..., :4] /= scale # de-scale + if flips == 2: + p[..., 1] = img_size[0] - p[..., 1] # de-flip ud + elif flips == 3: + p[..., 0] = img_size[1] - p[..., 0] # de-flip lr + else: + x, y, wh = p[..., 0:1] / scale, p[..., 1:2] / scale, p[..., 2:4] / scale # de-scale + if flips == 2: + y = img_size[0] - y # de-flip ud + elif flips == 3: + x = img_size[1] - x # de-flip lr + p = torch.cat((x, y, wh, p[..., 4:]), -1) + return p + + def _clip_augmented(self, y): + # Clip YOLOv5 augmented inference tails + nl = self.model[-1].nl # number of detection layers (P3-P5) + g = sum(4 ** x for x in range(nl)) # grid points + e = 1 # exclude layer count + i = (y[0].shape[1] // g) * sum(4 ** x for x in range(e)) # indices + y[0] = y[0][:, :-i] # large + i = (y[-1].shape[1] // g) * sum(4 ** (nl - 1 - x) for x in range(e)) # indices + y[-1] = y[-1][:, i:] # small + return y + + def _initialize_biases(self, cf=None): # initialize biases into Detect(), cf is class frequency + # https://arxiv.org/abs/1708.02002 section 3.3 + # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1. + m = self.model[-1] # Detect() module + for mi, s in zip(m.m, m.stride): # from + b = mi.bias.view(m.na, -1) # conv.bias(255) to (3,85) + b.data[:, 4] += math.log(8 / (640 / s) ** 2) # obj (8 objects per 640 image) + b.data[:, 5:5 + m.nc] += math.log(0.6 / (m.nc - 0.99999)) if cf is None else torch.log(cf / cf.sum()) # cls + mi.bias = torch.nn.Parameter(b.view(-1), requires_grad=True) + + +Model = DetectionModel # retain YOLOv5 'Model' class for backwards compatibility + + +class SegmentationModel(DetectionModel): + # YOLOv5 segmentation model + def __init__(self, cfg='yolov5s-seg.yaml', ch=3, nc=None, anchors=None): + super().__init__(cfg, ch, nc, anchors) + + +class ClassificationModel(BaseModel): + # YOLOv5 classification model + def __init__(self, cfg=None, model=None, nc=1000, cutoff=10): # yaml, model, number of classes, cutoff index + super().__init__() + self._from_detection_model(model, nc, cutoff) if model is not None else self._from_yaml(cfg) + + def _from_detection_model(self, model, nc=1000, cutoff=10): + # Create a YOLOv5 classification model from a YOLOv5 detection model + if isinstance(model, DetectMultiBackend): + model = model.model # unwrap DetectMultiBackend + model.model = model.model[:cutoff] # backbone + m = model.model[-1] # last layer + ch = m.conv.in_channels if hasattr(m, 'conv') else m.cv1.conv.in_channels # ch into module + c = Classify(ch, nc) # Classify() + c.i, c.f, c.type = m.i, m.f, 'models.common.Classify' # index, from, type + model.model[-1] = c # replace + self.model = model.model + self.stride = model.stride + self.save = [] + self.nc = nc + + def _from_yaml(self, cfg): + # Create a YOLOv5 classification model from a *.yaml file + self.model = None + + +def parse_model(d, ch): # model_dict, input_channels(3) + # Parse a YOLOv5 model.yaml dictionary + LOGGER.info(f"\n{'':>3}{'from':>18}{'n':>3}{'params':>10} {'module':<40}{'arguments':<30}") + anchors, nc, gd, gw, act = d['anchors'], d['nc'], d['depth_multiple'], d['width_multiple'], d.get('activation') + if act: + Conv.default_act = eval(act) # redefine default activation, i.e. Conv.default_act = nn.SiLU() + LOGGER.info(f"{colorstr('activation:')} {act}") # print + na = (len(anchors[0]) // 2) if isinstance(anchors, list) else anchors # number of anchors + no = na * (nc + 5) # number of outputs = anchors * (classes + 5) + + layers, save, c2 = [], [], ch[-1] # layers, savelist, ch out + for i, (f, n, m, args) in enumerate(d['backbone'] + d['head']): # from, number, module, args + m = eval(m) if isinstance(m, str) else m # eval strings + for j, a in enumerate(args): + with contextlib.suppress(NameError): + args[j] = eval(a) if isinstance(a, str) else a # eval strings + + n = n_ = max(round(n * gd), 1) if n > 1 else n # depth gain + if m in { + Conv, GhostConv, Bottleneck, GhostBottleneck, SPP, SPPF, DWConv, MixConv2d, Focus, CrossConv, + BottleneckCSP, C3, C3TR, C3SPP, C3Ghost, nn.ConvTranspose2d, DWConvTranspose2d, C3x}: + c1, c2 = ch[f], args[0] + if c2 != no: # if not output + c2 = make_divisible(c2 * gw, 8) + + args = [c1, c2, *args[1:]] + if m in {BottleneckCSP, C3, C3TR, C3Ghost, C3x}: + args.insert(2, n) # number of repeats + n = 1 + elif m is nn.BatchNorm2d: + args = [ch[f]] + elif m is Concat: + c2 = sum(ch[x] for x in f) + # TODO: channel, gw, gd + elif m in {Detect, Segment}: + args.append([ch[x] for x in f]) + if isinstance(args[1], int): # number of anchors + args[1] = [list(range(args[1] * 2))] * len(f) + if m is Segment: + args[3] = make_divisible(args[3] * gw, 8) + elif m is Contract: + c2 = ch[f] * args[0] ** 2 + elif m is Expand: + c2 = ch[f] // args[0] ** 2 + else: + c2 = ch[f] + + m_ = nn.Sequential(*(m(*args) for _ in range(n))) if n > 1 else m(*args) # module + t = str(m)[8:-2].replace('__main__.', '') # module type + np = sum(x.numel() for x in m_.parameters()) # number params + m_.i, m_.f, m_.type, m_.np = i, f, t, np # attach index, 'from' index, type, number params + LOGGER.info(f'{i:>3}{str(f):>18}{n_:>3}{np:10.0f} {t:<40}{str(args):<30}') # print + save.extend(x % i for x in ([f] if isinstance(f, int) else f) if x != -1) # append to savelist + layers.append(m_) + if i == 0: + ch = [] + ch.append(c2) + return nn.Sequential(*layers), sorted(save) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--cfg', type=str, default='yolov5s.yaml', help='model.yaml') + parser.add_argument('--batch-size', type=int, default=1, help='total batch size for all GPUs') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--profile', action='store_true', help='profile model speed') + parser.add_argument('--line-profile', action='store_true', help='profile model speed layer by layer') + parser.add_argument('--test', action='store_true', help='test all yolo*.yaml') + opt = parser.parse_args() + opt.cfg = check_yaml(opt.cfg) # check YAML + print_args(vars(opt)) + device = select_device(opt.device) + + # Create model + im = torch.rand(opt.batch_size, 3, 640, 640).to(device) + model = Model(opt.cfg).to(device) + + # Options + if opt.line_profile: # profile layer by layer + model(im, profile=True) + + elif opt.profile: # profile forward-backward + results = profile(input=im, ops=[model], n=3) + + elif opt.test: # test all models + for cfg in Path(ROOT / 'models').rglob('yolo*.yaml'): + try: + _ = Model(cfg) + except Exception as e: + print(f'Error in {cfg}: {e}') + + else: # report fused model summary + model.fuse() diff --git a/models/yolov5l.yaml b/models/yolov5l.yaml new file mode 100644 index 0000000..31362f8 --- /dev/null +++ b/models/yolov5l.yaml @@ -0,0 +1,48 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# Parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# YOLOv5 v6.0 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [64, 6, 2, 2]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, C3, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 6, C3, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, C3, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 7-P5/32 + [-1, 3, C3, [1024]], + [-1, 1, SPPF, [1024, 5]], # 9 + ] + +# YOLOv5 v6.0 head +head: + [[-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, C3, [512, False]], # 13 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, C3, [256, False]], # 17 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 14], 1, Concat, [1]], # cat head P4 + [-1, 3, C3, [512, False]], # 20 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 10], 1, Concat, [1]], # cat head P5 + [-1, 3, C3, [1024, False]], # 23 (P5/32-large) + + [[17, 20, 23], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/models/yolov5m.yaml b/models/yolov5m.yaml new file mode 100644 index 0000000..a76900c --- /dev/null +++ b/models/yolov5m.yaml @@ -0,0 +1,48 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# Parameters +nc: 80 # number of classes +depth_multiple: 0.67 # model depth multiple +width_multiple: 0.75 # layer channel multiple +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# YOLOv5 v6.0 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [64, 6, 2, 2]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, C3, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 6, C3, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, C3, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 7-P5/32 + [-1, 3, C3, [1024]], + [-1, 1, SPPF, [1024, 5]], # 9 + ] + +# YOLOv5 v6.0 head +head: + [[-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, C3, [512, False]], # 13 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, C3, [256, False]], # 17 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 14], 1, Concat, [1]], # cat head P4 + [-1, 3, C3, [512, False]], # 20 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 10], 1, Concat, [1]], # cat head P5 + [-1, 3, C3, [1024, False]], # 23 (P5/32-large) + + [[17, 20, 23], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/models/yolov5n.yaml b/models/yolov5n.yaml new file mode 100644 index 0000000..aba96cf --- /dev/null +++ b/models/yolov5n.yaml @@ -0,0 +1,48 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# Parameters +nc: 80 # number of classes +depth_multiple: 0.33 # model depth multiple +width_multiple: 0.25 # layer channel multiple +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# YOLOv5 v6.0 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [64, 6, 2, 2]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, C3, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 6, C3, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, C3, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 7-P5/32 + [-1, 3, C3, [1024]], + [-1, 1, SPPF, [1024, 5]], # 9 + ] + +# YOLOv5 v6.0 head +head: + [[-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, C3, [512, False]], # 13 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, C3, [256, False]], # 17 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 14], 1, Concat, [1]], # cat head P4 + [-1, 3, C3, [512, False]], # 20 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 10], 1, Concat, [1]], # cat head P5 + [-1, 3, C3, [1024, False]], # 23 (P5/32-large) + + [[17, 20, 23], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/models/yolov5s.yaml b/models/yolov5s.yaml new file mode 100644 index 0000000..5d05364 --- /dev/null +++ b/models/yolov5s.yaml @@ -0,0 +1,48 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# Parameters +nc: 80 # number of classes +depth_multiple: 0.33 # model depth multiple +width_multiple: 0.50 # layer channel multiple +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# YOLOv5 v6.0 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [64, 6, 2, 2]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, C3, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 6, C3, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, C3, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 7-P5/32 + [-1, 3, C3, [1024]], + [-1, 1, SPPF, [1024, 5]], # 9 + ] + +# YOLOv5 v6.0 head +head: + [[-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, C3, [512, False]], # 13 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, C3, [256, False]], # 17 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 14], 1, Concat, [1]], # cat head P4 + [-1, 3, C3, [512, False]], # 20 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 10], 1, Concat, [1]], # cat head P5 + [-1, 3, C3, [1024, False]], # 23 (P5/32-large) + + [[17, 20, 23], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/models/yolov5x.yaml b/models/yolov5x.yaml new file mode 100644 index 0000000..4bdd939 --- /dev/null +++ b/models/yolov5x.yaml @@ -0,0 +1,48 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# Parameters +nc: 80 # number of classes +depth_multiple: 1.33 # model depth multiple +width_multiple: 1.25 # layer channel multiple +anchors: + - [10,13, 16,30, 33,23] # P3/8 + - [30,61, 62,45, 59,119] # P4/16 + - [116,90, 156,198, 373,326] # P5/32 + +# YOLOv5 v6.0 backbone +backbone: + # [from, number, module, args] + [[-1, 1, Conv, [64, 6, 2, 2]], # 0-P1/2 + [-1, 1, Conv, [128, 3, 2]], # 1-P2/4 + [-1, 3, C3, [128]], + [-1, 1, Conv, [256, 3, 2]], # 3-P3/8 + [-1, 6, C3, [256]], + [-1, 1, Conv, [512, 3, 2]], # 5-P4/16 + [-1, 9, C3, [512]], + [-1, 1, Conv, [1024, 3, 2]], # 7-P5/32 + [-1, 3, C3, [1024]], + [-1, 1, SPPF, [1024, 5]], # 9 + ] + +# YOLOv5 v6.0 head +head: + [[-1, 1, Conv, [512, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 6], 1, Concat, [1]], # cat backbone P4 + [-1, 3, C3, [512, False]], # 13 + + [-1, 1, Conv, [256, 1, 1]], + [-1, 1, nn.Upsample, [None, 2, 'nearest']], + [[-1, 4], 1, Concat, [1]], # cat backbone P3 + [-1, 3, C3, [256, False]], # 17 (P3/8-small) + + [-1, 1, Conv, [256, 3, 2]], + [[-1, 14], 1, Concat, [1]], # cat head P4 + [-1, 3, C3, [512, False]], # 20 (P4/16-medium) + + [-1, 1, Conv, [512, 3, 2]], + [[-1, 10], 1, Concat, [1]], # cat head P5 + [-1, 3, C3, [1024, False]], # 23 (P5/32-large) + + [[17, 20, 23], 1, Detect, [nc, anchors]], # Detect(P3, P4, P5) + ] diff --git a/move_detect.py b/move_detect.py new file mode 100644 index 0000000..7103ed6 --- /dev/null +++ b/move_detect.py @@ -0,0 +1,249 @@ +# -*- coding: utf-8 -*- +""" +Created on Mon Oct 14 10:01:24 2024 + +@author: ym +""" +import numpy as np +import cv2 +from scipy.spatial.distance import cdist + + +class TrackFrag: + + def __init__(self, boxes, imgshape=(1280, 1024)): + self.boxes = boxes + self.cls = int(boxes[0, 6]) + self.tid = int(boxes[0, 4]) + self.imgshape = imgshape + + + '''轨迹的持续时间，以帧ID表征, （或考虑用时间戳）''' + self.during = (np.min(boxes[:, 7]), np.max(boxes[:, 7])) + + self.groups = [set(np.unique(boxes[:, 7].astype(int)))] + + # '''5个关键点（中心点、左上点、右上点、左下点、右下点 ）坐标''' + self.isCornpoint = self.is_cornpoint(10) + self.compute_cornpoints() + + + def is_cornpoint(self, edge=10): + + isleft = min(self.boxes[:, 0]) < edge + istop = min(self.boxes[:, 1]) < edge + isright = max(self.boxes[:, 2]) > self.imgshape[0] - edge + isbottom = max(self.boxes[:, 3]) > self.imgshape[0] - edge + + isCornpoint = isbottom or istop or isleft or isright + return isCornpoint + + + + def compute_cornpoints(self): + ''' + cornpoints 共10项，分别是个点的坐标值（x, y） + (center, top_left, top_right, bottom_left, bottom_right) + ''' + boxes = self.boxes + cornpoints = np.zeros((len(boxes), 10)) + + cornpoints[:,0] = (boxes[:, 0] + boxes[:, 2]) / 2 + cornpoints[:,1] = (boxes[:, 1] + boxes[:, 3]) / 2 + cornpoints[:,2], cornpoints[:,3] = boxes[:, 0], boxes[:, 1] + cornpoints[:,4], cornpoints[:,5] = boxes[:, 2], boxes[:, 1] + cornpoints[:,6], cornpoints[:,7] = boxes[:, 0], boxes[:, 3] + cornpoints[:,8], cornpoints[:,9] = boxes[:, 2], boxes[:, 3] + + trajdist = [] + for k in range(5): + X = cornpoints[:, 2*k:2*(k+1)] + trajdist.append(np.max(cdist(X, X))) + + idx = trajdist.index(min(trajdist)) + + self.trajdist_min = trajdist[idx] + self.cornpoints = cornpoints + + + + + + def update_groups(self, THRESH=18): + ''' + 对 self.groups 重新赋值 + ''' + + boxes = self.boxes + nbox = len(boxes) + + X = np.zeros((len(boxes), 2)) + X[:,0] = (boxes[:, 0] + boxes[:, 2]) / 2 + X[:,1] = (boxes[:, 1] + boxes[:, 3]) / 2 + + dist2 = cdist(X, X) + # label = np.zeros(nbox, dtype=np.int) + + marked, groups = set(), [] + for k in range(nbox): + if k in marked: + continue + group = set() + + dt = dist2[k, :] + idx = np.where(dt < THRESH)[0] + + if len(idx) == 1: + groups.append({k}) + marked.add(k) + continue + '''初始近邻样本点集合, 并移除当前点''' + seeds = set(idx) + seeds.remove(k) + + group.add(k) + marked.add(k) + while len(seeds) !=0: + pt = seeds.pop() + dt = dist2[pt, :] + + seed = set(np.where(dt < THRESH)[0]) + seed.remove(pt) + + seed.difference_update(marked) + seeds.update(seed) + + group.add(pt) + marked.add(pt) + + groups.append(group) + + self.groups = groups + + + def jump_boxes(self): + gpboxes = [] + for group in self.groups: + box = self.boxes[list(group), :] + gpboxes.append(box) + + return gpboxes + + + + def is_moving(self): + if len(self.groups)>=3: + return True + + return False + + + def is_static(self, THRESH=50): + box1 = self.boxes[0, :4] + box2 = self.boxes[-1, :4] + + ''' 第1帧、最后一帧 boxes 四个角点间的距离 ''' + ptd = box2 - box1 + ptd1 = np.linalg.norm((ptd[0], ptd[1])) + ptd2 = np.linalg.norm((ptd[2], ptd[1])) + ptd3 = np.linalg.norm((ptd[0], ptd[3])) + ptd4 = np.linalg.norm((ptd[2], ptd[3])) + condt1 = ptd1=3] + + + + def draw(self): + pass + + + + def array2list(self): + ''' + 将 bboxes 变换为 track 列表 + bboxes: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] + Return： + lboxes：列表，列表中元素具有同一 track_id，x1y1x2y2 格式 + [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] + ''' + track_ids = self.bboxes[:, 4].astype(int) + lboxes = [] + for t_id in self.trackIDs: + # print(f"The ID is: {t_id}") + idx = np.where(track_ids == t_id)[0] + box = self.bboxes[idx, :] + + lboxes.append(box) + + return lboxes + + @staticmethod + def sub_tracks(tlista, tlistb): + track_ids_b = {t.tid for t in tlistb} + return [t for t in tlista if t.tid not in track_ids_b] + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/pipeline.py b/pipeline.py new file mode 100644 index 0000000..afdc6f0 --- /dev/null +++ b/pipeline.py @@ -0,0 +1,385 @@ +# -*- coding: utf-8 -*- +""" +Created on Sun Sep 29 08:59:21 2024 + +@author: ym +""" +import os +# import sys +import cv2 +import pickle +import numpy as np +from pathlib import Path +from scipy.spatial.distance import cdist +from track_reid import yolo_resnet_tracker, yolov10_resnet_tracker + +from tracking.dotrack.dotracks_back import doBackTracks +from tracking.dotrack.dotracks_front import doFrontTracks +from tracking.utils.drawtracks import plot_frameID_y2, draw_all_trajectories +from utils.getsource import get_image_pairs, get_video_pairs +from tracking.utils.read_data import read_similar + + +def save_subimgs(imgdict, boxes, spath, ctype, featdict = None): + ''' + 当前 box 特征和该轨迹前一个 box 特征的相似度，可用于和跟踪序列中的相似度进行比较 + ''' + boxes = boxes[np.argsort(boxes[:, 7])] + for i in range(len(boxes)): + simi = None + tid, fid, bid = int(boxes[i, 4]), int(boxes[i, 7]), int(boxes[i, 8]) + + if i>0: + _, fid0, bid0 = int(boxes[i-1, 4]), int(boxes[i-1, 7]), int(boxes[i-1, 8]) + if f"{fid0}_{bid0}" in featdict.keys() and f"{fid}_{bid}" in featdict.keys(): + feat0 = featdict[f"{fid0}_{bid0}"] + feat1 = featdict[f"{fid}_{bid}"] + simi = 1 - np.maximum(0.0, cdist(feat0[None, :], feat1[None, :], "cosine"))[0][0] + + img = imgdict[f"{fid}_{bid}"] + imgpath = spath / f"{ctype}_tid{tid}-{fid}-{bid}.png" + if simi is not None: + imgpath = spath / f"{ctype}_tid{tid}-{fid}-{bid}_sim{simi:.2f}.png" + + cv2.imwrite(imgpath, img) + + +def save_subimgs_1(imgdict, boxes, spath, ctype, simidict = None): + ''' + 当前 box 特征和该轨迹 smooth_feat 特征的相似度, yolo_resnet_tracker 函数中， + 采用该方式记录特征相似度 + ''' + for i in range(len(boxes)): + tid, fid, bid = int(boxes[i, 4]), int(boxes[i, 7]), int(boxes[i, 8]) + + key = f"{fid}_{bid}" + img = imgdict[key] + imgpath = spath / f"{ctype}_tid{tid}-{fid}-{bid}.png" + if simidict is not None and key in simidict.keys(): + imgpath = spath / f"{ctype}_tid{tid}-{fid}-{bid}_sim{simidict[key]:.2f}.png" + + cv2.imwrite(imgpath, img) + +def show_result(event_tracks, yrtDict, savepath_pipe): + '''保存 Tracking 输出的运动轨迹子图，并记录相似度''' + + savepath_pipe_subimgs = savepath_pipe / Path("subimgs") + if not savepath_pipe_subimgs.exists(): + savepath_pipe_subimgs.mkdir(parents=True, exist_ok=True) + + + + + for CamerType, vts in event_tracks: + if len(vts.tracks)==0: continue + if CamerType == 'front': + # yolos = ShoppingDict["frontCamera"]["yoloResnetTracker"] + + yolos = yrtDict["frontyrt"] + ctype = 1 + if CamerType == 'back': + # yolos = ShoppingDict["backCamera"]["yoloResnetTracker"] + + yolos = yrtDict["backyrt"] + ctype = 0 + + imgdict, featdict, simidict = {}, {}, {} + for y in yolos: + imgdict.update(y["imgs"]) + featdict.update(y["feats"]) + simidict.update(y["featsimi"]) + + for track in vts.Residual: + if isinstance(track, np.ndarray): + save_subimgs(imgdict, track, savepath_pipe_subimgs, ctype, featdict) + else: + save_subimgs(imgdict, track.slt_boxes, savepath_pipe_subimgs, ctype, featdict) + + '''(3) 轨迹显示与保存''' + illus = [None, None] + for CamerType, vts in event_tracks: + if len(vts.tracks)==0: continue + + if CamerType == 'front': + edgeline = cv2.imread("./tracking/shopcart/cart_tempt/board_ftmp_line.png") + + h, w = edgeline.shape[:2] + # nh, nw = h//2, w//2 + # edgeline = cv2.resize(edgeline, (nw, nh), interpolation=cv2.INTER_AREA) + + img_tracking = draw_all_trajectories(vts, edgeline, savepath_pipe, CamerType, draw5p=True) + illus[0] = img_tracking + + plt = plot_frameID_y2(vts) + plt.savefig(os.path.join(savepath_pipe, "front_y2.png")) + + if CamerType == 'back': + edgeline = cv2.imread("./tracking/shopcart/cart_tempt/edgeline.png") + + h, w = edgeline.shape[:2] + # nh, nw = h//2, w//2 + # edgeline = cv2.resize(edgeline, (nw, nh), interpolation=cv2.INTER_AREA) + + img_tracking = draw_all_trajectories(vts, edgeline, savepath_pipe, CamerType, draw5p=True) + illus[1] = img_tracking + + illus = [im for im in illus if im is not None] + if len(illus): + img_cat = np.concatenate(illus, axis = 1) + if len(illus)==2: + H, W = img_cat.shape[:2] + cv2.line(img_cat, (int(W/2), 0), (int(W/2), int(H)), (128, 128, 255), 3) + + trajpath = os.path.join(savepath_pipe, "trajectory.png") + cv2.imwrite(trajpath, img_cat) + + + + +def pipeline(eventpath, + SourceType, + weights, + DataType = "raw", #raw, pkl: images or videos, pkl, pickle file + YoloVersion="V5", + savepath = None, + saveimages = True + ): + + ## 构造购物事件字典 + evtname = Path(eventpath).stem + barcode = evtname.split('_')[-1] if len(evtname.split('_'))>=2 \ + and len(evtname.split('_')[-1])>=8 \ + and evtname.split('_')[-1].isdigit() else '' + + '''事件结果存储文件夹: savepath_pipe, savepath_pkl''' + if not savepath: + savepath = Path(__file__).resolve().parents[0] / "events_result" + savepath_pipe = Path(savepath) / Path("yolos_tracking") / evtname + + + savepath_pkl = Path(savepath) / "shopping_pkl" + if not savepath_pkl.exists(): + savepath_pkl.mkdir(parents=True, exist_ok=True) + pklpath = Path(savepath_pkl) / Path(str(evtname)+".pickle") + + + + yrt_out = [] + if DataType == "raw": + ### 不重复执行已经过yolo-resnet-tracker + if pklpath.exists(): + print(f"Pickle file have saved: {evtname}.pickle") + return + + if SourceType == "video": + vpaths = get_video_pairs(eventpath) + elif SourceType == "image": + vpaths = get_image_pairs(eventpath) + + + + for vpath in vpaths: + '''================= 2. 事件结果存储文件夹 =================''' + + + if isinstance(vpath, list): + savepath_pipe_imgs = savepath_pipe / Path("images") + else: + savepath_pipe_imgs = savepath_pipe / Path(str(Path(vpath).stem)) + + if not savepath_pipe_imgs.exists(): + savepath_pipe_imgs.mkdir(parents=True, exist_ok=True) + + optdict = {} + optdict["weights"] = weights + optdict["source"] = vpath + optdict["save_dir"] = savepath_pipe_imgs + optdict["is_save_img"] = saveimages + optdict["is_save_video"] = True + + + if YoloVersion == "V5": + yrtOut = yolo_resnet_tracker(**optdict) + elif YoloVersion == "V10": + yrtOut = yolov10_resnet_tracker(**optdict) + + yrt_out.append((vpath, yrtOut)) + + elif DataType == "pkl": + pass + + else: + return + + + + '''====================== 构造 ShoppingDict 模块 =======================''' + ShoppingDict = {"eventPath": eventpath, + "eventName": evtname, + "barcode": barcode, + "eventType": '', # "input", "output", "other" + "frontCamera": {}, + "backCamera": {}, + "one2n": [] # + } + procpath = Path(eventpath).joinpath('process.data') + if procpath.is_file(): + SimiDict = read_similar(procpath) + ShoppingDict["one2n"] = SimiDict['one2n'] + + yrtDict = {} + event_tracks = [] + for vpath, yrtOut in yrt_out: + '''================= 1. 构造相机事件字典 =================''' + CameraEvent = {"cameraType": '', # "front", "back" + "videoPath": '', + "imagePaths": [], + "yoloResnetTracker": [], + "tracking": [], + } + + if isinstance(vpath, list): + CameraEvent["imagePaths"] = vpath + bname = os.path.basename(vpath[0]) + if not isinstance(vpath, list): + CameraEvent["videoPath"] = vpath + bname = os.path.basename(vpath).split('.')[0] + if bname.split('_')[0] == "0" or bname.find('back')>=0: + CameraEvent["cameraType"] = "back" + if bname.split('_')[0] == "1" or bname.find('front')>=0: + CameraEvent["cameraType"] = "front" + + + '''2种保存方式: (1) no save subimg, (2) save img''' + ###(1) save images + yrtOut_save = [] + for frdict in yrtOut: + fr_dict = {} + for k, v in frdict.items(): + if k != "imgs": + fr_dict[k]=v + yrtOut_save.append(fr_dict) + CameraEvent["yoloResnetTracker"] = yrtOut_save + + ###(2) no save images + # CameraEvent["yoloResnetTracker"] = yrtOut + + '''================= 4. tracking =================''' + '''(1) 生成用于 tracking 模块的 boxes、feats''' + bboxes = np.empty((0, 6), dtype=np.float64) + trackerboxes = np.empty((0, 9), dtype=np.float64) + trackefeats = {} + for frameDict in yrtOut: + tboxes = frameDict["tboxes"] + ffeats = frameDict["feats"] + + boxes = frameDict["bboxes"] + bboxes = np.concatenate((bboxes, np.array(boxes)), axis=0) + trackerboxes = np.concatenate((trackerboxes, np.array(tboxes)), axis=0) + for i in range(len(tboxes)): + fid, bid = int(tboxes[i, 7]), int(tboxes[i, 8]) + trackefeats.update({f"{fid}_{bid}": ffeats[f"{fid}_{bid}"]}) + + + '''(2) tracking, 后摄''' + if CameraEvent["cameraType"] == "back": + vts = doBackTracks(trackerboxes, trackefeats) + vts.classify() + event_tracks.append(("back", vts)) + + CameraEvent["tracking"] = vts + ShoppingDict["backCamera"] = CameraEvent + + yrtDict["backyrt"] = yrtOut + + '''(2) tracking, 前摄''' + if CameraEvent["cameraType"] == "front": + vts = doFrontTracks(trackerboxes, trackefeats) + vts.classify() + event_tracks.append(("front", vts)) + + CameraEvent["tracking"] = vts + ShoppingDict["frontCamera"] = CameraEvent + + yrtDict["frontyrt"] = yrtOut + + '''========================== 保存模块 =================================''' + # 保存 ShoppingDict + with open(str(pklpath), 'wb') as f: + pickle.dump(ShoppingDict, f) + + # 绘制并保存轨迹图 + show_result(event_tracks, yrtDict, savepath_pipe) + + + +def execute_pipeline(evtdir = r"D:\datasets\ym\后台数据\unzip", + DataType = "raw", # raw, pkl + save_path = r"D:\work\result_pipeline", + kk=1, + source_type = "video", # video, image, + yolo_ver = "V10", # V10, V5 + weight_yolo_v5 = r'./ckpts/best_cls10_0906.pt' , + weight_yolo_v10 = r'./ckpts/best_v10s_width0375_1205.pt', + saveimages = True + ): + ''' + 运行函数 pipeline()，遍历事件文件夹，每个文件夹是一个事件 + ''' + parmDict = {} + parmDict["DataType"] = DataType + parmDict["savepath"] = save_path + parmDict["SourceType"] = source_type + + parmDict["YoloVersion"] = yolo_ver + if parmDict["YoloVersion"] == "V5": + parmDict["weights"] = weight_yolo_v5 + elif parmDict["YoloVersion"] == "V10": + parmDict["weights"] = weight_yolo_v10 + + parmDict["saveimages"] = saveimages + + + evtdir = Path(evtdir) + errEvents = [] + k = 0 + for item in evtdir.iterdir(): + if item.is_dir(): + # item = evtdir/Path("20241212-171505-f0afe929-fdfe-4efa-94d0-2fa748d65fbb_6907992518930") + parmDict["eventpath"] = item + pipeline(**parmDict) + + # try: + # pipeline(**parmDict) + # except Exception as e: + # errEvents.append(str(item)) + + k+=1 + if kk is not None and k==kk: + break + + errfile = os.path.join(parmDict["savepath"], 'error_events.txt') + with open(errfile, 'w', encoding='utf-8') as f: + for line in errEvents: + f.write(line + '\n') + +if __name__ == "__main__": + datapath = r'/home/wqg/dataset/test_dataset/base_dataset/single_event/source/' + savepath = r'/home/wqg/dataset/pipeline/test_result/single_event_V10' + + execute_pipeline(evtdir = datapath, + DataType = "raw", # raw, pkl + kk=1, + source_type = "video", # video, image, + save_path = savepath, + yolo_ver = "V10", # V10, V5 + weight_yolo_v5 = r'./ckpts/best_cls10_0906.pt' , + weight_yolo_v10 = r'./ckpts/best_v10s_width0375_1205.pt', + saveimages = False + ) + + + + + + \ No newline at end of file diff --git a/pipeline_extract_subimg.py b/pipeline_extract_subimg.py new file mode 100644 index 0000000..8a3cb9c --- /dev/null +++ b/pipeline_extract_subimg.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- +""" +Created on Sun Sep 29 08:59:21 2024 + + 针对现场采集的视频，利用算法pipeline提取运动轨迹内的subimg，代替人工图像筛选、标注 + +@author: ym +""" +import os +import cv2 +import pickle +from pathlib import Path +from track_reid import parse_opt, yolo_resnet_tracker + +from tracking.dotrack.dotracks_back import doBackTracks +from tracking.dotrack.dotracks_front import doFrontTracks + +IMGFORMATS = '.bmp', '.jpeg', '.jpg', 'png', 'tif', 'tiff', 'webp', 'pfm' +VIDFORMATS = '.avi', '.gif', '.m4v', '.mkv', '.mov', '.mp4', '.ts', '.wmv' + +std_feature_path = r"\\192.168.1.28\share\测试_202406\contrast\std_features_2192_ft32vsft16" + + +opt = parse_opt() +optdict = vars(opt) + + +def pipeline(eventpath, savepath): + # eventpath = r"\\192.168.1.28\share\测试_202406\0918\images1\20240918-110822-1bc3902e-5a8e-4e23-8eca-fb3f02738551_6938314601726" + optdict["project"] = savepath + + '''Yolo + Resnet + Tracker''' + optdict["source"] = eventpath + optdict["save_dir"] = savepath + optdict["nosave"] = False + + tracksdict = yolo_resnet_tracker(**optdict) + + bboxes = tracksdict['TrackBoxes'] + + basename = os.path.basename(eventpath) + base, ext = os.path.splitext(basename) + + if base.find('front')>=0: + vts = doFrontTracks(bboxes, tracksdict) + vts.classify() + + if base.find('back')>=0: + vts = doBackTracks(bboxes, tracksdict) + vts.classify() + + tracks = [t for t in vts.tracks if t.cls>0 and not t.is_static()] + # tracks = [t for t in vts.tracks if t.cls>0] + + for track in tracks: + # for track in vts.Residual: + for *xyxy, tid, conf, cls, fid, bid in track.boxes: + img = tracksdict[f'frame_{int(fid)}']["imgs"][int(bid)] + + imgpth = savepath / Path(f'{base}_tid-{int(tid)}_fid-{int(fid)}_bid-{int(bid)}.jpg') + + cv2.imwrite(imgpth, img) + + return len(vts.Residual) + + + + +def main(): + + videopath = r"\\192.168.1.28\share\上海中环店采集视频\21-25\videos\1\back" + savepath = r"D:\contrast\barcodes" + + vpaths = [] + for root, dirs, files in os.walk(videopath): + vpth = [os.path.join(root, f) for f in files if os.path.splitext(f)[-1] in VIDFORMATS] + vpaths.extend(vpth) + + manual_txt = os.path.join(savepath, 'manual_videos.txt') + file = open(manual_txt, 'a', encoding='utf-8') + + + + manual = [] + k = 0 + for vpath in vpaths: + videoname = os.path.basename(vpath) + + vname, ext = os.path.splitext(videoname) + barcode = videoname.split('_')[0] + + subpath = os.path.join(savepath, barcode, vname) + subpath = Path(subpath) + if not subpath.exists(): + subpath.mkdir(parents=True, exist_ok=True) + + ntract = pipeline(vpath, subpath) + + if ntract==0: + manual.append(vpath) + file.write(vpath) + file.write("\n") + + print(f"{videoname} done!!!") + k += 1 + if k==10: + break + + file.close() + + + + + + +if __name__ == "__main__": + main() + + + + + + + + + + \ No newline at end of file diff --git a/pipeline_process_addWrite.py b/pipeline_process_addWrite.py new file mode 100644 index 0000000..6319d8c --- /dev/null +++ b/pipeline_process_addWrite.py @@ -0,0 +1,536 @@ +# -*- coding: utf-8 -*- +""" +Created on Sun Sep 29 08:59:21 2024 + +@author: ym +""" +import os +# import sys +import cv2 +import pickle +import numpy as np +from pathlib import Path +from scipy.spatial.distance import cdist +from track_reid import yolo_resnet_tracker, yolov10_resnet_tracker + +from tracking.dotrack.dotracks_back import doBackTracks +from tracking.dotrack.dotracks_front import doFrontTracks +from tracking.utils.drawtracks import plot_frameID_y2, draw_all_trajectories +from utils.getsource import get_image_pairs, get_video_pairs +from tracking.utils.read_data import read_similar, get_process_csv_data + +from openpyxl import Workbook, load_workbook + + +def save_subimgs(imgdict, boxes, spath, ctype, featdict = None): + ''' + 当前 box 特征和该轨迹前一个 box 特征的相似度，可用于和跟踪序列中的相似度进行比较 + ''' + boxes = boxes[np.argsort(boxes[:, 7])] + for i in range(len(boxes)): + simi = None + tid, fid, bid = int(boxes[i, 4]), int(boxes[i, 7]), int(boxes[i, 8]) + + if i>0: + _, fid0, bid0 = int(boxes[i-1, 4]), int(boxes[i-1, 7]), int(boxes[i-1, 8]) + if f"{fid0}_{bid0}" in featdict.keys() and f"{fid}_{bid}" in featdict.keys(): + feat0 = featdict[f"{fid0}_{bid0}"] + feat1 = featdict[f"{fid}_{bid}"] + simi = 1 - np.maximum(0.0, cdist(feat0[None, :], feat1[None, :], "cosine"))[0][0] + + img = imgdict[f"{fid}_{bid}"] + imgpath = spath / f"{ctype}_tid{tid}-{fid}-{bid}.png" + if simi is not None: + imgpath = spath / f"{ctype}_tid{tid}-{fid}-{bid}_sim{simi:.2f}.png" + + cv2.imwrite(imgpath, img) + + +def save_subimgs_1(imgdict, boxes, spath, ctype, simidict = None): + ''' + 当前 box 特征和该轨迹 smooth_feat 特征的相似度, yolo_resnet_tracker 函数中， + 采用该方式记录特征相似度 + ''' + for i in range(len(boxes)): + tid, fid, bid = int(boxes[i, 4]), int(boxes[i, 7]), int(boxes[i, 8]) + + key = f"{fid}_{bid}" + img = imgdict[key] + imgpath = spath / f"{ctype}_tid{tid}-{fid}-{bid}.png" + if simidict is not None and key in simidict.keys(): + imgpath = spath / f"{ctype}_tid{tid}-{fid}-{bid}_sim{simidict[key]:.2f}.png" + + cv2.imwrite(imgpath, img) + +def show_result(eventpath, event_tracks, yrtDict, savepath_pipe): + '''保存 Tracking 输出的运动轨迹子图，并记录相似度''' + + savepath_pipe_subimgs = savepath_pipe / Path("subimgs") + if not savepath_pipe_subimgs.exists(): + savepath_pipe_subimgs.mkdir(parents=True, exist_ok=True) + + + for CamerType, vts in event_tracks: + if len(vts.tracks)==0: continue + if CamerType == 'front': + # yolos = ShoppingDict["frontCamera"]["yoloResnetTracker"] + + yolos = yrtDict["frontyrt"] + ctype = 1 + if CamerType == 'back': + # yolos = ShoppingDict["backCamera"]["yoloResnetTracker"] + + yolos = yrtDict["backyrt"] + ctype = 0 + + imgdict, featdict, simidict = {}, {}, {} + for y in yolos: + imgdict.update(y["imgs"]) + featdict.update(y["feats"]) + simidict.update(y["featsimi"]) + + for track in vts.Residual: + if isinstance(track, np.ndarray): + save_subimgs(imgdict, track, savepath_pipe_subimgs, ctype, featdict) + else: + save_subimgs(imgdict, track.slt_boxes, savepath_pipe_subimgs, ctype, featdict) + + '''(3) 轨迹显示与保存''' + illus = [None, None] + for CamerType, vts in event_tracks: + if len(vts.tracks)==0: continue + + if CamerType == 'front': + edgeline = cv2.imread("./tracking/shopcart/cart_tempt/board_ftmp_line.png") + + h, w = edgeline.shape[:2] + # nh, nw = h//2, w//2 + # edgeline = cv2.resize(edgeline, (nw, nh), interpolation=cv2.INTER_AREA) + + img_tracking = draw_all_trajectories(vts, edgeline, savepath_pipe, CamerType, draw5p=True) + illus[0] = img_tracking + + plt = plot_frameID_y2(vts) + '''==========yj callbackdata=========''' + plt.savefig(os.path.join(eventpath, "front_y2.png")) + '''========================================''' + + if CamerType == 'back': + edgeline = cv2.imread("./tracking/shopcart/cart_tempt/edgeline.png") + + h, w = edgeline.shape[:2] + # nh, nw = h//2, w//2 + # edgeline = cv2.resize(edgeline, (nw, nh), interpolation=cv2.INTER_AREA) + + img_tracking = draw_all_trajectories(vts, edgeline, savepath_pipe, CamerType, draw5p=True) + illus[1] = img_tracking + + illus = [im for im in illus if im is not None] + if len(illus): + img_cat = np.concatenate(illus, axis = 1) + if len(illus)==2: + H, W = img_cat.shape[:2] + cv2.line(img_cat, (int(W/2), 0), (int(W/2), int(H)), (128, 128, 255), 3) + + '''==========yj callbackdata=========''' + trajpath = os.path.join(eventpath, "trajectory.png") + # trajpath = os.path.join(savepath_pipe, "trajectory.png") + '''======================================''' + cv2.imwrite(trajpath, img_cat) + + + + +def pipeline(dict_data, + eventpath, + SourceType, + weights, + DataType = "raw", #raw, pkl: images or videos, pkl, pickle file + YoloVersion="V5", + savepath = None, + saveimages = True + ): + + ## 构造购物事件字典 + evtname = Path(eventpath).stem + barcode = evtname.split('_')[-1] if len(evtname.split('_'))>=2 \ + and len(evtname.split('_')[-1])>=8 \ + and evtname.split('_')[-1].isdigit() else '' + + '''事件结果存储文件夹: savepath_pipe, savepath_pkl''' + if not savepath: + savepath = Path(__file__).resolve().parents[0] / "events_result" + savepath_pipe = Path(savepath) / Path("yolos_tracking") / evtname + + + savepath_pkl = Path(savepath) / "shopping_pkl" + if not savepath_pkl.exists(): + savepath_pkl.mkdir(parents=True, exist_ok=True) + pklpath = Path(savepath_pkl) / Path(str(evtname)+".pickle") + + + + yrt_out = [] + if DataType == "raw": + ### 不重复执行已经过yolo-resnet-tracker + if pklpath.exists(): + print(f"Pickle file have saved: {evtname}.pickle") + return + + if SourceType == "video": + vpaths = get_video_pairs(eventpath) + elif SourceType == "image": + vpaths = get_image_pairs(eventpath) + + + + for vpath in vpaths: + '''================= 2. 事件结果存储文件夹 =================''' + + + if isinstance(vpath, list): + savepath_pipe_imgs = savepath_pipe / Path("images") + else: + savepath_pipe_imgs = savepath_pipe / Path(str(Path(vpath).stem)) + + if not savepath_pipe_imgs.exists(): + savepath_pipe_imgs.mkdir(parents=True, exist_ok=True) + + optdict = {} + optdict["weights"] = weights + optdict["source"] = vpath + optdict["save_dir"] = savepath_pipe_imgs + optdict["is_save_img"] = saveimages + optdict["is_save_video"] = True + + + if YoloVersion == "V5": + yrtOut = yolo_resnet_tracker(**optdict) + elif YoloVersion == "V10": + yrtOut = yolov10_resnet_tracker(**optdict) + + yrt_out.append((vpath, yrtOut)) + + elif DataType == "pkl": + pass + + else: + return + + + + '''====================== 构造 ShoppingDict 模块 =======================''' + ShoppingDict = {"eventPath": eventpath, + "eventName": evtname, + "barcode": barcode, + "eventType": '', # "input", "output", "other" + "frontCamera": {}, + "backCamera": {}, + "one2n": [] # + } + procpath = Path(eventpath).joinpath('process.data') + if procpath.is_file(): + SimiDict = read_similar(procpath) + ShoppingDict["one2n"] = SimiDict['one2n'] + + yrtDict = {} + event_tracks = [] + for vpath, yrtOut in yrt_out: + '''================= 1. 构造相机事件字典 =================''' + CameraEvent = {"cameraType": '', # "front", "back" + "videoPath": '', + "imagePaths": [], + "yoloResnetTracker": [], + "tracking": [], + } + + if isinstance(vpath, list): + CameraEvent["imagePaths"] = vpath + bname = os.path.basename(vpath[0]) + if not isinstance(vpath, list): + CameraEvent["videoPath"] = vpath + bname = os.path.basename(vpath).split('.')[0] + if bname.split('_')[0] == "0" or bname.find('back')>=0: + CameraEvent["cameraType"] = "back" + if bname.split('_')[0] == "1" or bname.find('front')>=0: + CameraEvent["cameraType"] = "front" + + + '''2种保存方式: (1) no save subimg, (2) save img''' + ###(1) save images + yrtOut_save = [] + for frdict in yrtOut: + fr_dict = {} + for k, v in frdict.items(): + if k != "imgs": + fr_dict[k]=v + yrtOut_save.append(fr_dict) + CameraEvent["yoloResnetTracker"] = yrtOut_save + + ###(2) no save images + # CameraEvent["yoloResnetTracker"] = yrtOut + + '''================= 4. tracking =================''' + '''(1) 生成用于 tracking 模块的 boxes、feats''' + bboxes = np.empty((0, 6), dtype=np.float64) + trackerboxes = np.empty((0, 9), dtype=np.float64) + trackefeats = {} + for frameDict in yrtOut: + tboxes = frameDict["tboxes"] + ffeats = frameDict["feats"] + + boxes = frameDict["bboxes"] + bboxes = np.concatenate((bboxes, np.array(boxes)), axis=0) + trackerboxes = np.concatenate((trackerboxes, np.array(tboxes)), axis=0) + for i in range(len(tboxes)): + fid, bid = int(tboxes[i, 7]), int(tboxes[i, 8]) + trackefeats.update({f"{fid}_{bid}": ffeats[f"{fid}_{bid}"]}) + + + '''(2) tracking, 后摄''' + if CameraEvent["cameraType"] == "back": + vts = doBackTracks(trackerboxes, trackefeats) + vts.classify() + event_tracks.append(("back", vts)) + + CameraEvent["tracking"] = vts + ShoppingDict["backCamera"] = CameraEvent + + '''====yj callbackdata=======''' + back_cnts = len(vts.Residual) + dict_data['后摄轨迹数'] = back_cnts + print(f"back_cnts: {back_cnts}") + '''==============================''' + + yrtDict["backyrt"] = yrtOut + + '''(2) tracking, 前摄''' + if CameraEvent["cameraType"] == "front": + vts = doFrontTracks(trackerboxes, trackefeats) + vts.classify() + event_tracks.append(("front", vts)) + + CameraEvent["tracking"] = vts + ShoppingDict["frontCamera"] = CameraEvent + + '''====yj callbackdata=======''' + front_cnts = len(vts.Residual) + dict_data['前摄轨迹数'] = front_cnts + print(f"front_cnts: {front_cnts}") + '''==============================''' + + yrtDict["frontyrt"] = yrtOut + + '''========================== 保存模块 =================================''' + # 保存 ShoppingDict + with open(str(pklpath), 'wb') as f: + pickle.dump(ShoppingDict, f) + + # 绘制并保存轨迹图 + show_result(eventpath, event_tracks, yrtDict, savepath_pipe) + return dict_data + +class WriteExcel: + + def is_excel(self, input_excel): + + # 若文件存在，加载工作簿 + wb = load_workbook(input_excel) + sheet_name = wb.sheetnames[0] ##默认回传分析表只有一个sheet + # 获取活动工作表 + ws = wb.active + sheet = wb[sheet_name] + ##确定新增列的位置 + # new_col_index = sheet.max_column + return wb, ws, sheet + + def init_excel(self, input_excel, output_excel, headers, max_col=13): + if os.path.exists(output_excel): + wb, ws, sheet = self.is_excel(output_excel) + return wb, ws, sheet + elif os.path.exists(input_excel): + wb, ws, sheet = self.is_excel(input_excel) + self.add_header(wb, sheet, max_col, headers, output_excel) + return wb, ws, sheet + else: + raise FileNotFoundError(f"文件 '{input_excel}' 不存在") + + '''在已有excel文件中新增列标题''' + def add_header(self, wb, sheet, column, headers, file_name): + + write_data = {} + sub_data = {} + for i, head in enumerate(headers): + k = column + i + 1 + sub_data[k] = head + write_data[1] = sub_data + self.add_data_to_excel(wb, sheet, write_data, file_name) + + def add_data_to_excel(self, wb, sheet, write_data, file_name): + ''' + 示例写入数据，格式为 {行号: {列号: 值}} + write_data = { + 5: {11: 1, 12: 2, 13: 3} + } + ''' + # 写入指定行列的数据 + for row_num, col_data in write_data.items(): + for col_num, value in col_data.items(): + # print('row_num', row_num, 'col_num', col_num, 'value', value) + sheet.cell(row=row_num, column=col_num, value=str(value)) + wb.save(file_name) + + def get_simiTitle_index(self, ws, title_name='事件名'): + '''获取excel文件中标题名称与追加内容相同部分的列索引 + 例如：默认以"事件名"为基准，追加统一事件名下不同组成信息''' + # 获取列标题 + headers = [cell.value for cell in ws[1]] + # # 找到“事件名”所在的列索引 + event_name_index = headers.index(title_name) if title_name in headers else None + if event_name_index is None: + print("未找到标题为'事件名'的列") + return + else: + return event_name_index + + def get_event_row(self, sheet, event_name_index, event_name): + '''获得当前事件名event_name在excel文件中所在的行索引''' + row_index = 0 + for row, content in enumerate(sheet.iter_rows(min_row=2, values_only=True)): + # print('row', row, content[event_name_index]) + if content[event_name_index] == event_name: + row_index = row + 2 ### 默认只有一行标题，若有两行标题则需改为+3 + # print('row_index', row_index) + break + return row_index + def write_simi_add(self, wb, ws, sheet, max_col, evtname, dict_data, headers, file_name): + ''' + 在已有excel文件中追加内容 + 找出事件名所在行索引和原excel最大列索引，在原excel最大列索引后指定行写入新内容，内容顺序与新增headers顺序一致 + ''' + event_index = self.get_simiTitle_index(ws) + if event_index is not None: + print('evtname', evtname) + row_index = self.get_event_row(sheet, event_index, evtname) + if row_index > 0: + sub_dict = {} + print('headers', headers) + for i, header in enumerate(headers): + col_index = max_col + i + 1 + # print('list(dict_data.keys())', list(dict_data.keys())) + if header in list(dict_data.keys()): + sub_dict[col_index] = dict_data[header] + else: + sub_dict[col_index] = '' + write_data = {row_index: sub_dict} + self.add_data_to_excel(wb, sheet, write_data, file_name) + + print("=========save excel===========") + else: + raise Exception(f"未找到事件名:{evtname}") + else: + raise Exception("未找到标题为'事件名'的列") + + +def execute_pipeline(evtdir = r"D:\datasets\ym\后台数据\unzip", + DataType = "raw", # raw, pkl + save_path = r"D:\work\result_pipeline", + kk=1, + source_type = "video", # video, image, + yolo_ver = "V10", # V10, V5 + weight_yolo_v5 = r'./ckpts/best_cls10_0906.pt' , + weight_yolo_v10 = r'./ckpts/best_v10s_width0375_1205.pt', + saveimages = True, + max_col = 12 + ): + ''' + 运行函数 pipeline()，遍历事件文件夹，每个文件夹是一个事件 + ''' + parmDict = {} + parmDict["DataType"] = DataType + parmDict["savepath"] = save_path + parmDict["SourceType"] = source_type + + parmDict["YoloVersion"] = yolo_ver + if parmDict["YoloVersion"] == "V5": + parmDict["weights"] = weight_yolo_v5 + elif parmDict["YoloVersion"] == "V10": + parmDict["weights"] = weight_yolo_v10 + + parmDict["saveimages"] = saveimages + + + evtdir = Path(evtdir) + errEvents = [] + # k = 0 + '''=========change callbackdata==============''' + csv_name = 'data.csv' + xlsx_name = '现场回传数据分析表.xlsx' + output_name = '现场回传数据分析表_all.xlsx' + headers = ['algroStartToEnd', 'one2one', 'one2SN', 'one2n', '前摄轨迹数', '后摄轨迹数'] + + + excelWriter = WriteExcel() ## 实例化excel对象 + + for name in evtdir.iterdir(): ##人名 + for date_file in name.iterdir(): ##2025-01-13 + xlsx_data = os.path.join(date_file, xlsx_name) + csv_data = os.path.join(date_file, csv_name) + excel_name = os.path.join(date_file, output_name) + wb, ws, sheet = excelWriter.init_excel(xlsx_data, excel_name, headers, max_col) + if csv_data == '': + with open('no_datacsv.txt', 'a') as f: + f.write(str(date_file) + '\n') + for item in date_file.iterdir(): + # dict_data = {} + if item.is_dir(): + # item = evtdir/Path("20241212-171505-f0afe929-fdfe-4efa-94d0-2fa748d65fbb_6907992518930") + parmDict["eventpath"] = item + event_name = str(item.name) + dict_data = get_process_csv_data(csv_data, item) + print('dict_data', dict_data) + dict_data_all = pipeline(dict_data, **parmDict) + if dict_data_all is not None: #已保存pickle文件的事件返回为None + # print('dict_data_all', dict_data_all) + excelWriter.write_simi_add(wb, ws, sheet, max_col, event_name, dict_data_all, headers, excel_name) + + # try: + # pipeline(**parmDict) + # except Exception as e: + # errEvents.append(str(item)) + + # k+=1 + # if kk is not None and k==kk: + # break + + errfile = os.path.join(parmDict["savepath"], 'error_events.txt') + with open(errfile, 'w', encoding='utf-8') as f: + for line in errEvents: + f.write(line + '\n') + +if __name__ == "__main__": + # datapath = r'/home/wqg/dataset/test_dataset/base_dataset/single_event/source/' + # savepath = r'/home/wqg/dataset/pipeline/test_result/single_event_V10' + + # datapath = '/home/yujia/yj/gpu_code/callback_data_test/' + datapath = '/home/yujia/yj/gpu_code/callback_data_test_0417/' + savepath = '/home/yujia/yj/gpu_code/result_0417_v10/' + max_col = 12 ##excel表格列索引从0开始，从这列开始写入代码解析内容 + + execute_pipeline(evtdir=datapath, + DataType = "raw", # raw, pkl + kk=1, + source_type = "video", # video, image, + save_path = savepath, + yolo_ver = "V10", # V10, V5 + weight_yolo_v5 = '/home/yujia/yj/gpu_code/ckpts/best_cls10_0906.pt' , + weight_yolo_v10 = '/home/yujia/yj/gpu_code/ckpts/best_v10s_width0375_1205.pt', + saveimages = False, + max_col = max_col + ) + + + + + + \ No newline at end of file diff --git a/practice/6924743915817.txt b/practice/6924743915817.txt new file mode 100644 index 0000000..e937507 --- /dev/null +++ b/practice/6924743915817.txt @@ -0,0 +1,78 @@ +-0.012108 -0.005081 0.040924 0.003206 0.048553 0.078918 -0.036774 -0.049957 -0.034515 -0.037689 -0.058533 -0.006462 -0.106995 0.023758 0.038818 -0.146240 0.001214 -0.077209 0.063110 0.023376 -0.085205 -0.067322 0.021179 -0.015556 0.073242 0.065247 0.014977 0.067261 0.139893 0.000464 0.109802 -0.064697 -0.071045 -0.029648 -0.111511 -0.142334 -0.067627 -0.025436 0.029633 0.066040 0.053192 0.057281 0.001234 -0.017181 -0.051910 0.037964 0.043549 -0.078247 -0.048981 -0.016296 0.035248 -0.083862 -0.072876 0.003597 0.028336 -0.002398 0.107483 0.006378 0.085327 -0.035217 -0.029861 0.000131 -0.062988 0.006710 -0.008194 0.023758 -0.032501 -0.020172 0.024170 0.025360 0.094116 -0.040558 0.105408 -0.062042 0.068726 -0.050781 -0.114441 0.040497 0.053406 0.029068 -0.078308 -0.042084 -0.049103 -0.064392 -0.070618 -0.080261 0.031403 0.029037 -0.032562 -0.031403 0.043365 0.098938 0.028519 -0.054016 -0.072083 -0.052643 0.085449 -0.047882 -0.015900 0.060547 -0.052704 -0.138916 -0.129517 -0.039856 -0.011246 -0.023804 -0.018372 -0.032410 -0.050903 -0.034393 -0.084961 0.021057 0.021820 0.017853 0.013176 -0.117126 -0.067688 0.056610 0.046783 -0.013290 0.103577 -0.041260 0.044434 -0.007912 0.085876 -0.070862 0.111511 -0.028915 0.057495 0.027283 0.020248 0.067871 -0.046204 0.054626 0.028427 -0.036469 -0.026749 -0.139526 0.009445 -0.054443 -0.028671 -0.056305 0.041260 -0.093201 -0.018829 0.037628 -0.005714 -0.018143 0.051910 0.042603 0.027008 -0.056427 0.070496 -0.029129 -0.063782 -0.089355 -0.046783 0.084656 -0.038849 0.090698 0.008919 -0.104309 -0.061005 0.014610 0.047913 -0.006104 -0.095459 -0.017731 0.019455 -0.004402 0.053284 -0.088501 0.057251 -0.054047 -0.017914 0.089661 0.008621 -0.014305 0.015915 0.037170 0.038727 -0.130249 0.015274 0.054504 -0.030716 0.023438 -0.041504 -0.003704 0.158325 0.067566 0.030365 -0.024719 -0.059082 -0.025925 0.050476 -0.016632 0.097839 -0.123474 0.064148 -0.004620 -0.088013 -0.036896 0.013901 0.084900 0.010056 0.102600 -0.083862 -0.117065 -0.007828 0.041779 -0.047638 0.051605 0.071167 -0.017288 -0.054962 -0.060852 0.016983 0.039307 -0.098389 -0.041931 -0.033112 0.116821 0.057831 -0.002653 0.012817 0.169678 0.053131 0.060333 -0.107605 -0.009209 0.014893 0.039185 0.021896 -0.046570 0.000801 -0.055817 0.138062 -0.038574 -0.163208 0.069702 0.098083 -0.007362 0.064392 -0.075867 0.073425 -0.058075 -0.101746 -0.039917 -0.010178 -0.008743 -0.017105 0.078308 0.149292 0.011856 0.003960 -0.000674 +-0.063477 -0.007069 -0.005245 0.043976 0.038483 0.101135 -0.004757 -0.054932 -0.051422 -0.036133 -0.048309 0.027176 -0.103699 -0.004608 0.084229 -0.182983 0.023010 -0.036285 0.033508 0.021790 -0.074341 -0.011391 -0.040161 -0.069519 0.094604 0.004799 -0.030457 0.060059 0.128906 0.026169 0.113586 -0.018616 -0.033478 -0.038574 -0.102966 -0.098999 -0.054718 0.003244 0.031738 0.131714 0.053589 0.042694 0.039795 -0.032562 -0.111450 0.017044 0.081055 -0.019135 -0.044891 -0.014748 0.060181 -0.151001 -0.130371 -0.011879 0.079102 0.053833 0.088257 0.000039 0.068909 -0.086487 -0.038635 0.045715 -0.073547 0.018677 -0.019897 0.001243 -0.059570 0.007343 0.019928 0.029938 0.123413 -0.050232 0.097961 -0.030304 0.034943 -0.046692 -0.151978 0.010780 0.073181 -0.010277 -0.048309 -0.025406 -0.028961 0.009995 -0.071777 -0.092896 -0.025650 0.069519 -0.050781 -0.034546 0.014282 0.098694 0.007473 -0.018600 -0.077515 0.020584 0.070129 -0.044800 -0.061768 0.088379 0.033112 -0.132690 -0.062408 0.024338 -0.002018 0.040039 -0.039001 -0.032806 -0.101868 -0.029999 -0.045227 0.028305 -0.005219 -0.003819 0.008522 -0.106812 -0.018646 0.089233 0.054169 -0.016068 0.053772 -0.081543 -0.048126 -0.014633 0.053986 -0.079407 0.055664 -0.039307 0.091797 0.008270 0.032928 0.002785 -0.072266 0.042084 0.021957 -0.082275 -0.010719 -0.054108 0.020813 -0.013069 -0.023575 -0.006931 0.003746 -0.063049 0.038239 0.017975 0.008720 0.039062 0.032166 0.006096 0.000866 -0.040741 0.063049 -0.025879 -0.061676 -0.102600 -0.004028 0.120789 -0.018600 0.109070 0.042145 -0.088501 -0.085327 0.067139 0.030060 0.057587 -0.105774 -0.033752 -0.003479 -0.071960 0.054749 -0.076416 0.059082 -0.061615 -0.074402 0.011024 0.036255 0.008629 0.022537 0.035675 0.038055 -0.098755 0.017593 0.083496 -0.071899 0.078064 -0.056641 -0.000371 0.142578 0.022171 0.017471 -0.028198 -0.018631 -0.034729 -0.004997 -0.010086 0.029083 -0.144043 0.048889 0.027649 -0.043732 -0.013733 0.032166 0.059937 0.065247 0.086365 -0.082214 -0.118225 0.016159 0.026978 -0.108765 0.075317 0.073730 -0.002377 -0.029755 -0.092407 0.007515 0.040344 -0.076843 -0.001449 -0.085571 0.139893 0.066040 0.032013 0.055267 0.190674 -0.031403 0.079285 -0.110779 -0.014015 0.053802 -0.021652 0.025146 -0.032776 -0.013557 -0.074951 0.108276 -0.014259 -0.162109 0.045227 0.068787 -0.029633 0.033112 -0.031174 0.089355 -0.045166 -0.028091 -0.011375 0.008003 0.033997 -0.005348 -0.004387 0.096558 -0.032715 -0.002901 -0.027359 +-0.077576 -0.053558 0.011093 -0.053497 0.116943 -0.086792 0.075867 -0.045746 -0.000218 -0.057861 -0.091492 0.010445 -0.062073 -0.062866 -0.031174 -0.150024 0.004971 -0.061249 0.024353 0.016251 -0.026306 0.090149 -0.075073 -0.025162 0.028854 -0.065552 0.041901 0.062683 -0.053772 0.017639 0.095154 -0.059906 0.041077 0.026886 -0.126099 -0.036194 -0.080444 -0.029373 -0.018250 0.068970 0.050110 0.062134 0.103149 -0.002596 -0.033783 0.024185 0.058624 0.058197 -0.106445 -0.039825 0.000286 -0.018631 -0.162476 0.019836 0.084473 -0.012520 0.028641 -0.012932 0.034882 -0.086609 0.089172 0.019257 -0.126099 -0.003334 -0.017380 0.016800 -0.025055 -0.046173 -0.045258 -0.032166 0.094666 0.063904 0.086304 -0.051453 0.015327 -0.037659 -0.069519 0.038483 0.021698 0.001744 0.003468 0.001429 0.038605 -0.032288 -0.028336 -0.085144 0.035767 0.028961 -0.001388 -0.052643 0.046967 0.049225 0.057220 -0.093018 -0.072083 0.013733 0.040741 -0.012871 -0.076965 0.140503 0.066833 -0.099792 0.046936 -0.073730 -0.021301 0.040710 -0.017029 -0.049927 -0.005035 0.020096 0.000294 -0.059784 -0.037323 0.005112 0.043823 -0.012642 0.035919 0.067261 0.013954 -0.066895 0.121582 -0.094360 0.035980 -0.079346 0.015457 0.031250 0.072083 0.038910 -0.011421 -0.055542 -0.067932 -0.012093 -0.023499 0.056702 -0.010284 0.040283 0.030228 -0.025574 0.068359 0.013748 -0.087952 0.062195 0.037903 -0.098389 -0.057770 0.079529 -0.028214 -0.088684 -0.014153 0.002840 0.044678 -0.008965 -0.038696 0.114807 -0.026810 0.000520 -0.017410 0.119385 -0.047760 0.092590 0.002943 -0.060791 -0.084351 0.116455 0.001213 -0.007164 -0.016769 -0.080750 -0.012779 -0.105835 -0.004128 0.012833 -0.084106 -0.078552 0.003675 -0.036682 0.045868 0.074219 -0.055664 0.066711 0.042511 -0.059845 -0.023758 0.004200 -0.011436 0.053497 -0.083374 -0.007359 0.106873 -0.058624 -0.051270 -0.001504 0.010979 0.106140 -0.083496 -0.005470 0.084595 -0.136353 0.100037 -0.094055 -0.118530 0.006615 0.097168 0.061401 0.171265 0.074707 -0.022614 -0.033234 0.000768 0.073120 -0.024551 0.022461 0.069702 0.018723 0.005615 0.111572 -0.059998 0.078064 -0.063538 -0.042786 -0.008965 0.087280 0.095154 -0.046509 0.111267 -0.000310 -0.076599 -0.038605 -0.029648 -0.001067 0.084473 0.078613 -0.070251 -0.036621 0.132446 -0.080200 0.064331 -0.073547 -0.057404 -0.019928 -0.001274 -0.003666 0.029327 -0.064880 0.073303 0.033936 -0.085388 -0.101685 -0.072693 -0.088257 0.097290 -0.092285 0.058624 0.083618 -0.001839 0.002214 +0.012100 -0.017914 0.123718 0.020386 -0.031204 0.059601 0.004745 0.045532 -0.051910 0.050903 0.007015 -0.060272 -0.079895 0.067261 0.002798 -0.033691 -0.031464 -0.037476 0.046875 0.075256 -0.049957 -0.001346 -0.024292 0.040894 0.024094 -0.016800 0.035187 0.005093 0.132446 0.115784 0.070129 -0.119995 -0.085449 0.072144 -0.098999 0.043274 0.019623 -0.004848 -0.014832 0.077026 -0.052277 0.055237 0.002892 -0.105774 -0.069458 0.001369 0.058685 -0.028275 -0.055389 -0.015480 0.059052 -0.043640 -0.101868 -0.080688 -0.044342 -0.009109 -0.008865 -0.032959 0.068481 -0.051117 -0.019348 -0.023499 -0.138916 -0.102295 0.007652 -0.011314 -0.051575 -0.036560 0.055298 0.083862 0.101929 0.020950 -0.076294 -0.081787 0.109741 -0.050842 -0.087097 0.006187 0.051941 0.015976 -0.036163 -0.016998 0.029587 0.005726 0.023697 -0.046326 0.079651 0.065063 -0.151611 -0.052094 0.015587 0.095337 0.028885 -0.048798 -0.069641 -0.021301 0.038574 -0.046875 -0.023972 0.031952 -0.006630 -0.082153 -0.101685 0.034210 -0.035095 -0.043854 -0.058197 -0.015915 -0.063721 0.001738 -0.031174 -0.096436 -0.054840 -0.061920 -0.020172 -0.116699 -0.075928 0.029449 0.013954 -0.015602 -0.052979 -0.040497 0.046326 -0.042999 0.162231 -0.049103 0.120300 0.082947 0.103027 -0.014366 0.013855 0.009415 -0.058807 -0.036926 0.044830 0.072815 0.063416 -0.096802 -0.051117 -0.023209 0.076111 -0.051239 0.023956 -0.019775 -0.054108 0.034821 0.028000 0.080566 0.043732 -0.072937 -0.033112 -0.046692 -0.014893 0.013672 -0.068909 -0.118652 -0.046570 0.131836 0.126099 -0.055908 0.015121 -0.140625 -0.007050 0.114441 0.035614 -0.032074 0.054077 -0.035492 -0.003038 -0.006641 0.076782 0.022751 0.037689 -0.002945 0.016556 0.042419 -0.043335 0.028900 0.008049 -0.008308 0.033813 -0.103271 -0.049774 0.082336 -0.068909 -0.073853 -0.021469 -0.044891 0.103394 -0.012192 -0.007942 -0.004391 -0.047180 0.046204 0.131348 -0.021164 0.028107 -0.176880 -0.047241 -0.043976 -0.020782 -0.029831 -0.015900 0.046417 -0.032288 0.097290 -0.137573 -0.046753 -0.063232 0.026077 -0.061981 0.081360 0.011398 0.047455 -0.064758 -0.020905 0.150879 -0.044434 -0.077393 0.055450 -0.063232 0.017776 -0.021881 -0.032196 0.060120 0.076904 0.106079 0.037750 -0.004227 -0.052765 0.031433 -0.076721 -0.005886 -0.052094 0.068359 -0.024261 -0.073181 0.022461 -0.062378 0.032654 -0.013062 0.041748 0.039551 -0.132202 0.002626 -0.028564 -0.096252 0.089661 0.047974 -0.086060 0.008492 0.124695 0.033081 -0.086304 -0.009720 -0.081665 +-0.047424 -0.007607 0.101318 -0.001976 0.117554 -0.018417 -0.014389 0.046661 -0.090576 0.022034 -0.069885 0.004242 -0.119812 -0.069824 0.049042 -0.122742 -0.041412 -0.004848 0.093201 0.120239 -0.058502 0.025162 0.052124 0.064453 0.097229 -0.044189 0.048157 0.035828 0.075500 0.000914 0.000320 -0.031403 -0.004795 0.046814 -0.080994 0.005798 -0.108215 -0.048065 0.012489 0.035736 0.035797 0.002916 0.044556 -0.121277 -0.045837 0.017776 0.095032 -0.018875 -0.017914 -0.092041 -0.031128 0.007225 -0.136108 -0.054810 -0.021439 0.026840 0.051239 -0.031143 0.043793 -0.054443 0.000263 0.015640 -0.146729 -0.053741 -0.033508 -0.018036 -0.056580 -0.063049 -0.085327 0.058502 0.128418 -0.005367 -0.028336 -0.058533 0.126953 -0.028580 -0.074524 0.010513 -0.000984 -0.013832 -0.037323 -0.052795 0.062561 0.056244 0.045380 -0.060303 0.072327 0.044373 -0.099365 -0.080688 0.006454 0.073425 0.060059 -0.069641 -0.092529 -0.048492 0.040070 -0.031586 -0.052032 0.122314 0.001475 -0.107239 -0.066772 -0.017578 -0.058289 -0.086060 -0.102051 -0.068176 -0.052246 -0.002888 -0.000543 -0.078979 -0.070312 -0.022812 0.013718 -0.046539 0.008728 0.011978 0.046112 -0.102539 0.097656 -0.035278 -0.015976 -0.021149 0.111084 0.031128 0.071472 0.076477 0.116333 0.020111 0.001237 -0.041321 -0.054260 0.032471 0.013832 0.005772 -0.059143 -0.116150 -0.047028 -0.039948 -0.041046 -0.007965 0.050507 -0.047516 -0.091125 0.056183 0.013191 -0.006599 -0.020508 -0.024155 -0.065186 -0.068298 -0.074097 0.071960 -0.031082 -0.051300 -0.016998 0.166260 0.100281 -0.021439 0.037872 -0.106567 -0.074341 0.155029 0.023499 -0.009918 -0.002010 0.001662 0.003571 -0.032410 0.074219 -0.019592 -0.067932 -0.036682 0.086426 -0.022354 -0.009445 0.034882 0.007618 -0.028656 0.071655 -0.062195 -0.054230 0.027359 -0.072388 0.016510 -0.014427 -0.024414 0.086121 -0.058350 -0.033936 -0.034302 -0.066101 0.041412 -0.014771 -0.071777 0.081604 -0.134766 0.040802 -0.061432 -0.054962 0.000031 0.010544 0.136475 0.034454 0.123169 -0.086731 -0.038086 -0.058899 0.034698 -0.050049 0.136597 0.031250 -0.026001 -0.096130 0.055176 0.048889 0.003933 -0.036957 0.037384 -0.055939 0.008324 0.055511 -0.054718 0.070557 -0.027756 -0.020584 0.037140 -0.009178 -0.025391 0.077515 -0.107971 -0.034210 0.005085 0.105469 -0.051422 0.022552 -0.028137 -0.075195 0.039276 0.014618 -0.017273 0.039917 -0.107483 0.052277 0.133789 -0.097595 -0.032867 0.043335 -0.053894 0.072083 0.019470 0.102600 -0.076660 -0.050629 -0.030777 +-0.080505 -0.058533 0.011864 -0.001678 0.079285 0.052399 0.012238 -0.104187 -0.009560 -0.064209 -0.072510 0.008438 -0.029907 -0.007111 0.039764 -0.173584 -0.030594 -0.077271 0.026703 0.029831 -0.011559 0.024887 -0.073792 -0.068542 0.084167 -0.060059 -0.056061 0.052063 0.119446 0.026688 0.128418 -0.056915 0.004204 -0.008247 -0.118225 -0.097595 -0.073425 -0.028030 0.060822 0.055725 0.039764 0.027710 0.043304 0.047302 -0.119995 0.019379 0.059143 0.031372 -0.091125 -0.022583 0.038727 -0.126709 -0.134766 0.025482 0.057831 0.068298 0.091980 0.017868 0.033936 -0.018311 -0.019501 0.026367 -0.061829 -0.030289 -0.009972 0.040405 -0.038208 -0.072815 0.049377 0.000505 0.077209 0.013344 0.128540 -0.053497 0.014687 -0.017090 -0.114319 0.016968 0.017120 0.004723 -0.030945 0.010918 0.053375 -0.049744 -0.081604 -0.075989 -0.019226 -0.011200 0.028000 -0.058167 0.043457 0.146606 -0.003473 -0.056763 -0.102661 0.016907 0.088684 -0.003975 -0.008484 0.077271 0.042816 -0.167725 -0.013969 -0.057343 -0.042023 0.051605 -0.007206 -0.023544 -0.055237 0.021118 -0.016296 0.016174 -0.014778 -0.008080 -0.006760 -0.090637 -0.021362 0.085754 -0.004410 -0.043518 0.083923 -0.045410 0.011360 -0.059631 0.055786 -0.065857 0.099609 -0.043365 0.049103 -0.030960 0.079834 0.015091 -0.074524 0.062378 -0.002151 -0.066467 0.008186 -0.060364 0.020996 -0.007622 -0.054474 0.066833 -0.020447 -0.104797 -0.008919 0.059814 0.016541 -0.058502 0.013443 0.067383 0.023605 -0.026932 0.067322 0.038788 -0.054443 -0.084351 -0.013206 0.074707 -0.032166 0.133423 0.013458 -0.081909 -0.090515 0.097107 0.016434 0.008110 -0.103821 -0.057556 0.016388 -0.100098 0.010689 0.008179 0.022156 -0.106812 -0.065125 0.040619 0.076172 -0.019806 0.020126 0.027634 -0.010498 -0.066589 0.001569 0.063721 -0.032288 0.061005 -0.068542 -0.005470 0.133057 -0.012459 0.023788 -0.021194 -0.045990 0.018005 -0.070679 0.002216 0.103149 -0.174072 -0.004009 -0.003956 -0.104187 -0.059967 0.057373 0.048096 0.091614 0.068298 -0.032013 -0.034943 0.038177 0.028671 -0.078003 0.023148 0.051880 -0.016373 0.016083 -0.047852 -0.015106 0.066223 -0.092773 -0.055389 -0.086304 0.176636 0.107788 0.035675 0.112488 0.119019 -0.050842 0.010208 -0.061218 -0.034546 0.059967 0.055756 -0.014908 -0.075439 0.011406 -0.082825 0.138062 -0.003338 -0.110535 -0.005970 0.036713 -0.006996 0.014854 -0.056152 0.054932 -0.083191 -0.064209 -0.054962 -0.034332 -0.010269 0.063171 -0.007107 0.118225 0.044556 0.010048 0.008392 +-0.088867 -0.006584 0.000978 -0.067993 -0.011299 -0.011009 0.091919 -0.084351 0.036530 0.041901 -0.063721 0.012001 -0.107483 -0.014297 0.022476 -0.124573 -0.000116 0.007137 0.017776 0.024994 -0.057831 0.114929 -0.107422 -0.119629 -0.018082 -0.069275 0.029831 0.094910 0.056702 -0.032684 0.101807 -0.029968 0.044250 0.023315 -0.123718 -0.056610 -0.087891 0.049530 0.052582 0.073425 0.061127 0.052246 0.073853 -0.075256 -0.054871 -0.029602 0.077087 0.030472 -0.099854 -0.053833 0.060120 -0.036194 -0.047211 -0.041534 0.003668 -0.029114 -0.004723 -0.045898 0.031342 -0.009613 0.081970 -0.072693 -0.171143 -0.003609 -0.090698 0.020935 0.047028 -0.035736 -0.081055 0.010208 0.024048 0.042877 0.055054 0.009933 0.062866 -0.047363 -0.098572 0.023849 0.138062 -0.007332 -0.095215 0.039154 0.014587 0.045807 -0.023285 -0.072815 0.035950 0.037231 0.002653 -0.076355 -0.052460 0.058777 0.019226 -0.062866 -0.105591 -0.016220 0.041504 -0.006542 -0.126709 0.115234 0.001443 -0.079651 0.064026 0.026627 -0.059448 -0.067078 -0.014763 -0.122192 0.006821 -0.051758 -0.047516 -0.003002 0.004425 -0.054260 0.105713 0.033173 0.099060 0.131104 -0.018097 -0.003410 0.096863 -0.079224 0.019470 -0.086975 0.011299 -0.082825 -0.012100 0.051422 0.013115 -0.035492 -0.059937 0.013115 -0.090332 0.058960 -0.000882 0.038757 -0.038513 -0.004532 0.039673 0.050629 -0.046509 0.003023 0.005344 -0.030563 0.005035 0.064209 -0.028778 0.006279 -0.058899 -0.027649 -0.041077 -0.041351 0.038971 0.049591 -0.023636 -0.045074 0.016525 0.075378 -0.061920 0.112122 0.071411 -0.064880 0.001765 0.072754 -0.046661 0.015541 -0.068176 -0.064514 -0.015358 -0.117432 0.036407 -0.033478 -0.051208 -0.092712 -0.028748 -0.026962 -0.001900 0.014030 0.014725 -0.016388 0.010849 -0.077393 -0.085083 0.049713 -0.052704 0.121094 -0.021759 0.010422 0.097412 -0.024109 -0.140137 -0.008949 0.007927 0.109985 -0.011124 0.024353 0.025482 -0.073242 0.142456 -0.091553 -0.069580 -0.096069 -0.021774 0.060730 0.179810 0.038910 -0.021454 -0.039490 0.057007 0.083679 -0.016129 0.107361 0.080017 0.051025 -0.075317 -0.001435 0.018921 0.044800 -0.044952 0.049469 -0.032196 0.056976 -0.014854 -0.031738 0.077393 0.057098 -0.050568 -0.046539 -0.075928 0.036743 0.049103 0.010155 -0.061493 -0.110291 0.090332 -0.087036 0.046295 -0.108765 -0.011192 -0.040070 -0.048828 -0.023224 0.097900 -0.065552 0.062012 -0.000584 -0.027878 -0.002588 -0.065125 -0.078735 0.083740 -0.138062 0.030319 0.025879 -0.024490 -0.048431 +-0.051300 -0.020859 0.057617 -0.010582 0.022110 0.026962 0.007942 -0.035187 -0.054657 -0.036774 0.011246 0.010773 -0.097595 0.114441 -0.003645 -0.121460 0.017792 -0.015625 0.022110 0.024277 -0.058838 0.069641 -0.045593 -0.020126 0.021149 -0.000751 0.026108 0.135010 0.179443 0.020554 0.078430 -0.009842 -0.005142 -0.000080 -0.073425 0.045380 -0.057129 0.017334 -0.059448 0.048676 -0.035065 0.106262 -0.013092 -0.042725 -0.022888 -0.006790 0.006367 -0.058716 -0.158691 0.033722 0.045197 -0.109619 -0.162354 -0.041290 0.016754 -0.032745 -0.010735 -0.013229 0.205811 -0.041870 0.038025 -0.139648 -0.080872 -0.005981 0.028381 0.085510 -0.015297 -0.015747 0.072754 0.020264 0.124390 -0.015617 0.117004 0.014503 0.050568 -0.003368 -0.117371 0.021469 0.078003 0.013817 -0.025665 0.012680 0.013062 -0.035400 -0.067871 -0.055511 0.106873 0.010971 -0.038879 -0.060150 0.008034 0.161377 -0.002237 0.018600 -0.084839 -0.050323 0.088440 -0.079224 -0.017303 0.040192 -0.023834 -0.163452 -0.040710 -0.018036 -0.100952 -0.044556 0.026993 -0.034271 -0.017792 -0.024857 -0.037750 -0.036255 -0.058258 0.010239 0.001386 -0.077209 -0.012238 0.027695 0.031555 -0.043091 0.095520 -0.041351 0.097778 -0.072327 0.081299 -0.061371 0.050049 0.008827 0.008125 -0.054688 -0.034363 0.035400 -0.059387 0.062622 0.090576 0.055542 0.026047 -0.053192 0.032074 -0.041473 -0.055969 -0.104797 0.082214 -0.038300 0.007065 0.084412 -0.032654 0.013519 0.014771 0.082581 0.032806 -0.030441 -0.031189 0.025406 -0.115051 -0.102478 -0.099670 0.149780 0.008041 0.028137 -0.004635 -0.121460 0.013329 0.004730 0.034149 0.053131 -0.068359 -0.078430 -0.028198 -0.091736 0.034668 -0.002895 -0.025543 -0.116272 -0.095520 -0.004330 0.019272 -0.042755 -0.054260 0.022736 0.036102 -0.087097 -0.008148 0.045898 -0.009041 0.071472 -0.077393 0.042511 0.094360 0.028442 -0.039612 -0.008659 -0.105896 0.052887 0.026489 0.017365 0.109253 -0.134766 0.006134 -0.018097 -0.112793 -0.055603 0.104248 0.024277 0.072449 0.057098 -0.103638 -0.006733 -0.025574 0.020660 -0.076965 0.069153 0.072571 -0.009232 -0.036774 -0.026672 0.037659 0.038025 -0.043610 -0.036255 -0.011955 0.078003 0.078003 0.013298 0.081665 0.115784 -0.032104 -0.024979 -0.013283 -0.042999 0.012604 0.030807 -0.041779 -0.076538 0.077942 -0.095276 0.068420 -0.015778 -0.063171 0.052673 0.025345 0.030396 0.063721 -0.054932 0.001950 -0.026535 -0.125000 -0.000515 -0.070312 -0.020355 0.033813 0.021286 0.008537 0.036682 0.021210 0.006847 +-0.104370 -0.044647 0.030746 0.011848 0.057526 -0.008278 -0.012321 -0.084900 0.013306 -0.008041 -0.011665 0.005608 -0.072449 0.045380 0.052704 -0.225464 -0.021851 -0.055786 -0.028534 -0.033325 0.019592 0.009315 -0.058075 -0.051392 -0.008530 -0.081909 -0.034363 0.078918 0.180542 0.026077 0.092468 -0.055908 0.032135 0.035034 -0.091492 0.007263 -0.044739 0.077881 0.005806 0.027908 0.028259 0.095764 0.095459 -0.047455 -0.086365 0.040619 -0.034576 -0.012230 -0.138672 0.018646 0.058807 -0.078613 -0.145264 0.000744 0.044678 -0.019974 0.025620 0.016449 0.094116 -0.023499 0.011063 -0.116821 -0.093018 -0.079163 -0.033478 0.082214 0.039917 0.005947 -0.034454 0.014687 0.105103 -0.019745 0.149780 -0.017502 0.051941 -0.025269 -0.117737 0.011627 0.097473 -0.019928 -0.035065 -0.002720 -0.009933 -0.001321 -0.135010 -0.019958 0.007996 -0.021515 0.041016 -0.052032 -0.032013 0.105347 -0.003544 -0.019012 -0.118347 -0.036865 0.089050 -0.061798 -0.030060 0.013054 0.009315 -0.126099 0.021194 -0.010216 -0.059998 0.024414 0.025497 -0.119080 -0.012901 -0.043610 -0.049835 0.018463 -0.039795 -0.019989 0.012512 -0.047516 0.040344 0.077881 0.021484 -0.038879 0.143066 0.057037 0.014130 0.008705 0.130249 -0.043915 0.043121 0.047485 0.037292 -0.067200 -0.014229 0.058319 -0.182983 -0.004654 0.027222 -0.017700 -0.024933 -0.033600 -0.003723 -0.018814 -0.044159 -0.006718 0.004150 0.000915 0.005909 0.134277 -0.049316 0.011063 0.017792 0.025909 -0.031799 -0.111511 0.068176 -0.003735 -0.091797 -0.067139 -0.008560 0.115662 -0.003941 0.027985 -0.008255 -0.088806 0.019333 0.007500 0.017883 -0.011841 -0.027328 -0.056152 0.012512 -0.087036 0.081848 -0.014626 0.033234 -0.166748 -0.066833 0.013641 0.020279 -0.022552 -0.005245 0.001663 0.064636 -0.072571 -0.064026 0.027115 0.056152 0.074036 -0.078918 0.010727 0.105347 0.009193 -0.043793 -0.000097 -0.082092 0.019379 0.000548 -0.050171 0.065491 -0.145508 0.002535 -0.064819 -0.101562 -0.062927 0.045013 0.076782 0.113831 0.007042 -0.057007 -0.105835 0.028839 0.020554 -0.074341 0.041046 0.109985 0.019470 -0.026596 -0.069031 -0.014359 0.044800 -0.088257 -0.047028 -0.020508 0.104370 0.013618 0.026505 0.075867 0.107117 0.017227 -0.016815 -0.076355 -0.040466 0.036407 0.053955 0.004261 -0.093384 0.009308 -0.064148 0.042664 -0.062561 -0.035828 0.031494 0.018021 -0.015701 0.112122 -0.114502 -0.024033 -0.028336 -0.092224 0.048737 -0.041534 0.017349 0.008881 -0.074219 0.060486 -0.008347 0.015701 -0.026077 +-0.095947 -0.026047 0.027344 0.035980 0.046448 0.129883 0.020172 -0.056915 -0.045105 -0.038330 0.002014 -0.001935 -0.052185 0.093811 0.022064 -0.136475 -0.049225 0.034637 0.027710 0.038269 -0.049103 0.065002 -0.056946 -0.018112 0.022873 -0.011200 -0.018280 0.084290 0.112793 0.050781 0.059906 -0.030075 -0.001304 0.010452 -0.104248 -0.053345 -0.052673 0.027328 -0.015274 0.077393 -0.003262 0.068054 -0.045074 -0.025360 -0.071289 -0.043030 -0.010139 -0.065796 -0.183838 0.016144 0.036407 -0.138428 -0.114563 -0.108643 -0.009026 0.075745 0.027328 -0.024994 0.111206 -0.014946 -0.013901 -0.061493 -0.060516 0.010551 0.020248 0.074402 -0.028351 -0.078735 0.064270 0.009270 0.059753 -0.037079 0.136719 -0.040497 0.054657 -0.041199 -0.086243 0.060822 0.043182 0.034088 -0.019073 -0.002989 0.005116 -0.018738 -0.123657 -0.032898 0.056366 -0.018906 -0.016174 -0.148438 0.008858 0.132080 0.000813 -0.030853 -0.075317 -0.041046 0.058655 -0.033173 -0.097412 -0.000504 0.018845 -0.106689 -0.062500 0.046875 -0.074463 -0.012138 -0.039978 0.011513 -0.044495 -0.003536 -0.087463 -0.005974 0.012825 -0.054260 -0.017090 -0.107056 -0.083679 0.091797 0.049774 -0.015488 0.029510 -0.032745 0.059174 -0.015213 0.060028 -0.031708 0.087830 -0.000510 0.075195 -0.002844 0.027679 0.042603 -0.133789 0.011833 0.014648 0.020721 -0.047333 -0.118591 -0.008018 -0.050934 -0.000528 -0.060425 0.021545 -0.027740 -0.044586 0.042236 0.034637 -0.058441 0.029419 0.087158 0.060242 -0.032928 0.019379 0.023987 -0.103943 -0.120422 -0.056488 0.115967 0.009727 0.068237 0.020325 -0.129395 0.023407 -0.028778 0.032104 0.056335 -0.065979 -0.075500 -0.028442 -0.086548 0.046143 -0.020660 0.043610 -0.103821 -0.116394 0.068848 0.035919 -0.042480 0.003454 -0.035797 0.048584 -0.114685 0.018066 0.036469 -0.016190 0.095520 -0.009865 0.003464 0.073303 0.039154 0.017899 0.001660 -0.117126 -0.000117 0.054840 0.018219 0.060059 -0.134888 0.041962 -0.028046 -0.152588 -0.041443 0.073730 0.073669 0.061707 0.005127 -0.073181 -0.070007 0.041382 0.110779 -0.088989 0.044067 0.070740 0.012276 -0.035217 -0.096069 0.041901 0.034180 -0.131836 -0.004078 -0.048676 0.093506 0.034943 0.005501 0.058685 0.138672 -0.032196 -0.012367 -0.079590 -0.051605 -0.022095 -0.006977 -0.022263 -0.045441 0.021606 -0.081055 0.053467 0.050232 -0.106323 0.090332 0.005291 -0.009361 0.095703 -0.062988 0.004513 -0.064026 -0.104309 -0.050568 -0.033447 0.005470 0.018951 0.052032 0.073486 -0.012283 0.053772 -0.016800 +-0.031860 -0.047791 0.045197 0.023346 -0.000677 0.045898 0.044037 -0.059296 0.020233 -0.066833 -0.062378 0.015701 -0.057922 0.002020 0.056183 -0.177124 0.024643 -0.071167 0.024078 -0.004017 -0.037689 0.069519 -0.084045 -0.090637 0.013695 -0.056885 0.058044 0.072632 0.085571 0.000285 0.122864 -0.086060 -0.027588 -0.043732 -0.105286 -0.093201 -0.031494 -0.006031 0.039429 0.132812 0.056274 0.099060 0.059418 0.065918 -0.132812 0.068237 0.053192 0.010849 -0.078552 -0.004890 0.047455 -0.099365 -0.153442 -0.001022 0.065186 0.010406 0.017441 0.028320 0.068726 -0.101501 0.025269 0.028992 -0.111511 0.006336 -0.025513 0.007652 -0.020035 -0.049316 -0.016266 -0.004486 -0.014359 0.002268 0.110901 -0.034729 0.017639 -0.023376 -0.133301 0.058838 0.112732 0.009926 -0.045715 0.039032 0.001749 0.038666 -0.130127 -0.144043 0.025848 0.013779 0.018265 -0.029999 0.019424 0.112671 0.006294 -0.083313 -0.065002 0.063232 0.036804 -0.062286 -0.025299 0.074951 0.022781 -0.107117 -0.007671 -0.015015 -0.022781 0.074280 -0.002148 -0.049530 -0.007183 -0.048492 0.006523 -0.021179 0.011612 -0.051544 0.030670 -0.066711 0.070435 0.138672 0.015541 -0.001809 0.083923 -0.100586 -0.007534 -0.070190 0.023270 -0.098145 0.098999 -0.040863 0.059418 -0.044952 0.044586 -0.029282 -0.040985 0.036926 0.012825 0.007034 0.058899 0.013443 0.089233 0.012009 -0.056122 0.049194 -0.005688 -0.097107 0.055145 0.041931 0.011375 -0.001695 -0.015747 0.017181 -0.011345 0.021896 0.044098 -0.006321 -0.000619 -0.109619 -0.029160 0.065674 -0.084229 0.110352 0.005646 -0.083008 -0.113037 0.089966 -0.007980 0.033417 -0.092468 -0.076599 -0.082458 -0.099304 0.025986 -0.080566 0.012039 -0.091736 -0.044342 0.057068 0.040894 -0.100281 0.048981 0.008934 0.035309 -0.088318 -0.042877 0.106079 -0.063416 0.087158 -0.100586 -0.012009 0.112671 -0.004330 -0.014503 -0.033600 0.013237 0.049011 -0.042175 0.007294 0.011345 -0.081055 -0.012764 0.035797 -0.070190 -0.064209 0.099792 0.044342 0.123291 0.077209 -0.048737 -0.036560 -0.088684 0.009758 -0.058044 0.030762 0.060059 -0.017120 -0.048798 -0.057678 -0.025833 0.068970 -0.080811 -0.019363 -0.053497 0.152832 0.071655 0.039093 0.078247 0.127808 -0.023605 -0.035706 -0.061646 -0.066833 0.071777 0.066040 0.020416 -0.059540 -0.007286 -0.087646 0.105530 -0.063293 -0.086548 0.008354 0.029022 0.042572 -0.040436 -0.026779 0.085449 -0.018326 -0.041077 0.017593 -0.042358 -0.046509 0.051147 -0.068542 0.035919 -0.012260 -0.020218 -0.042023 +0.003683 -0.040375 0.045959 -0.005531 0.034393 0.093811 -0.006748 -0.084595 -0.018311 -0.050476 -0.037109 -0.000509 -0.107544 0.006592 0.010948 -0.151733 -0.011551 -0.022964 0.017471 -0.054138 -0.090027 0.007271 0.001972 -0.029480 0.037720 -0.039764 -0.020172 0.074707 0.123474 -0.020996 0.105835 -0.110840 -0.066650 -0.001163 -0.163330 -0.081970 -0.060699 -0.004177 0.023972 0.104736 0.014053 0.032562 0.054413 -0.027374 -0.081543 0.055695 0.035553 -0.090393 -0.110718 -0.011971 0.050842 -0.066406 -0.144409 -0.045166 0.014626 -0.015854 0.058319 0.002174 0.066650 -0.032898 -0.000545 -0.039581 -0.051880 0.024918 -0.039703 0.013618 -0.010017 -0.045837 0.015236 0.000289 0.067017 -0.081421 0.123657 -0.064880 0.057220 -0.080811 -0.088074 0.060547 0.020935 0.019089 -0.040527 -0.057343 -0.026947 0.006191 -0.102722 -0.084045 0.053253 -0.001462 0.008995 -0.062012 0.022095 0.089722 0.039093 -0.095642 -0.113892 -0.015511 0.002720 -0.040955 -0.004417 0.086426 -0.006630 -0.164673 -0.049438 -0.011353 -0.059814 -0.010315 -0.002935 -0.074890 -0.081055 -0.048187 -0.083313 0.012222 0.022583 -0.016068 0.018982 -0.089783 -0.008865 0.079407 0.025314 -0.023987 0.124451 0.000825 0.004566 -0.015068 0.083069 -0.042816 0.095459 -0.010513 0.045746 0.025208 -0.006954 0.035187 -0.062866 0.013298 0.003489 -0.012367 -0.036377 -0.117554 0.010460 -0.036713 -0.065308 -0.005878 0.027969 -0.044922 -0.044067 0.074585 0.000278 -0.084778 0.018478 0.016235 0.018280 -0.045197 0.067566 -0.025940 -0.017914 -0.085022 -0.018265 0.091125 -0.028229 0.058105 0.022385 -0.120911 -0.051758 -0.023163 0.038116 -0.019501 -0.106689 -0.082092 -0.000672 -0.041779 0.070740 -0.068542 0.020950 -0.110840 -0.013809 0.109436 0.041931 -0.003056 0.025116 0.025818 0.065063 -0.093262 -0.008324 0.018311 -0.051270 -0.002619 -0.085327 -0.001694 0.185913 0.078735 0.025497 -0.039276 -0.092407 -0.009804 0.034576 -0.052490 0.044220 -0.120300 0.049438 -0.082336 -0.144043 -0.024139 0.058380 0.131104 0.041687 0.079895 -0.053436 -0.098206 -0.077332 0.048676 -0.052917 0.059998 0.126587 -0.048950 -0.097168 -0.057556 0.012932 0.076416 -0.116577 -0.054443 -0.031586 0.076782 0.029480 -0.022781 0.050568 0.139648 -0.027405 0.030762 -0.095764 -0.045197 0.015251 0.022324 -0.013550 -0.063538 0.014343 -0.030212 0.069824 -0.041199 -0.110779 0.071472 0.039948 -0.008553 0.070251 -0.062164 0.014580 0.002865 -0.116455 0.007370 -0.012520 -0.015114 0.030136 0.054596 0.128906 0.009544 -0.042206 -0.046936 +-0.069946 -0.046143 0.061432 0.004459 0.033783 0.003180 0.025803 -0.036499 -0.056671 -0.008926 -0.099365 0.002981 -0.142700 0.011841 0.006725 -0.138428 -0.034302 -0.034760 0.060303 -0.008133 -0.055664 0.098694 -0.001347 -0.013603 0.047852 -0.076965 0.059235 0.093689 0.057343 -0.041016 0.067566 -0.074890 -0.042511 -0.003345 -0.194824 -0.035339 -0.100342 -0.025818 0.030212 0.098694 -0.013924 0.013809 0.027451 -0.049469 -0.071350 0.031677 0.053040 -0.047333 -0.122620 -0.046753 0.028717 0.010033 -0.179443 -0.069946 0.007759 0.001667 0.023926 -0.048462 0.047241 -0.042969 -0.003458 -0.020813 -0.056641 0.020065 -0.019882 -0.009361 -0.029251 -0.073669 -0.047882 0.007587 0.007053 -0.038025 0.064758 -0.030197 0.073242 -0.087036 -0.046600 0.034424 0.050446 0.017227 -0.022079 -0.007904 0.056763 0.065186 -0.024963 -0.100830 0.098389 0.009987 -0.018906 -0.109985 0.020966 0.071289 0.039825 -0.117981 -0.105774 0.024261 -0.026840 -0.029266 -0.041016 0.141724 0.000244 -0.128174 -0.031738 -0.010307 -0.117432 -0.042145 -0.025726 -0.081604 -0.044800 -0.028748 0.027283 -0.021591 0.025879 -0.011787 0.062134 -0.039734 0.084412 0.094055 0.012749 -0.033844 0.104309 -0.041595 -0.025650 -0.006878 -0.016098 -0.015396 0.062866 -0.027115 0.059113 0.028076 -0.020294 -0.014290 -0.054932 0.043884 0.012466 0.006935 -0.042419 -0.126465 0.048370 -0.064148 -0.075745 0.001100 0.005363 -0.040771 -0.074524 0.063843 0.052429 -0.051331 -0.051849 0.013901 0.019287 0.002291 -0.004154 0.033295 0.007633 -0.091431 -0.022736 0.112061 0.009949 0.039276 0.023102 -0.096252 -0.079041 0.056610 -0.000884 -0.043793 -0.092834 -0.053894 -0.056335 -0.072021 0.056274 -0.043488 -0.094910 -0.087036 0.026169 0.044434 0.025894 -0.028778 0.001543 0.002914 0.077454 -0.107727 -0.022949 0.016464 -0.100830 0.040619 -0.058197 -0.012283 0.116455 0.009964 -0.023285 -0.044189 -0.063049 0.018158 -0.001390 -0.042084 0.036926 -0.067444 0.027557 -0.065186 -0.095032 -0.009460 0.094910 0.147827 0.094666 0.082214 -0.046570 -0.033295 -0.125122 0.065063 0.000936 0.103149 0.108276 -0.107422 -0.102356 0.000134 -0.014381 0.089722 -0.092163 -0.008865 -0.045563 0.029297 0.069824 -0.091736 0.092590 0.029999 -0.106079 -0.054596 -0.028366 -0.036041 0.090332 0.007473 -0.018585 -0.041870 0.066406 -0.099304 0.030273 -0.092102 -0.083313 0.085876 0.015884 -0.001809 0.030472 -0.026962 0.064758 0.090515 -0.106140 -0.008682 -0.068970 -0.098450 0.094543 -0.043030 0.090210 -0.006775 -0.076172 -0.055725 +-0.033020 -0.047211 0.086914 -0.024185 0.076660 -0.006481 -0.014580 -0.011963 -0.074890 -0.034943 -0.060944 -0.031891 -0.118347 -0.027725 0.083130 -0.157227 -0.022842 -0.028839 0.057465 0.046783 -0.072998 0.048798 0.025085 -0.038239 0.050293 -0.075684 0.059479 0.032318 0.100098 -0.065857 0.063354 -0.065491 -0.044098 -0.043976 -0.137207 -0.081665 -0.108826 -0.029785 0.005974 0.079651 0.058624 0.082092 0.101929 -0.030731 -0.091675 0.070496 0.101318 -0.028671 -0.048676 -0.070740 -0.003990 0.010506 -0.163086 -0.017197 0.018311 0.022644 0.051880 0.022369 0.065796 -0.059692 0.004829 -0.021500 -0.117310 -0.019989 -0.023132 0.011238 -0.012711 -0.060974 -0.075928 0.011536 0.022598 -0.018982 0.100647 -0.104858 0.085144 -0.053436 -0.076843 0.044434 0.062622 -0.013496 -0.078491 -0.014091 0.009224 0.033447 -0.014969 -0.113464 0.060425 0.016418 -0.033020 -0.047729 0.036133 0.129028 0.037750 -0.067810 -0.113770 -0.007088 -0.012917 -0.056091 0.001848 0.102417 0.007557 -0.169556 -0.052429 -0.035675 -0.063538 -0.010788 -0.037048 -0.109924 -0.037231 -0.043671 -0.027939 -0.072998 -0.017014 -0.015404 0.026749 -0.076538 0.077148 0.071899 0.025253 -0.065247 0.171021 -0.029816 -0.005688 -0.074890 0.071167 -0.013878 0.095581 0.053864 0.068115 0.018066 0.005634 -0.021606 -0.028275 0.048431 0.023254 0.032013 -0.017166 -0.082764 0.047577 -0.040344 -0.090698 0.026398 0.010330 -0.102539 -0.043915 0.129395 -0.018372 -0.005165 -0.007626 -0.006302 0.011475 -0.031830 0.023941 0.018539 -0.001737 -0.084167 -0.034943 0.103455 0.021912 0.028625 0.039764 -0.076111 -0.084961 0.130371 0.020493 0.001660 -0.062561 -0.047272 -0.055084 -0.046509 0.109558 -0.059113 -0.060760 -0.073425 0.052277 0.012611 0.009064 -0.065247 0.054108 0.006470 0.049744 -0.084961 -0.035034 0.051971 -0.068420 0.050354 -0.075928 0.003685 0.096863 -0.031952 -0.038147 -0.061005 -0.044281 0.019791 -0.037262 -0.064697 0.121704 -0.112854 0.005562 -0.054901 -0.079224 -0.031860 0.086914 0.092224 0.058624 0.101501 -0.067810 -0.032806 -0.093506 0.010292 -0.029755 0.084290 0.073853 -0.054535 -0.105957 0.033539 -0.016205 0.030014 -0.076294 0.002235 -0.040894 0.085693 0.078125 0.002449 0.053894 0.066772 -0.006466 0.002022 -0.041443 -0.043030 0.101257 -0.035431 0.014580 -0.012901 0.076660 -0.012085 0.096191 -0.062561 -0.080994 0.021454 0.030838 -0.003508 0.011597 -0.045502 0.090088 0.072754 -0.063293 -0.005390 -0.031891 -0.068604 0.083679 -0.022125 0.128784 -0.043243 -0.031464 -0.039276 +-0.071960 -0.010109 0.026733 -0.013763 0.130493 -0.041962 0.062500 -0.002930 -0.079590 -0.052002 -0.037842 0.011383 -0.030716 -0.006660 0.030945 -0.160889 0.016815 -0.025757 0.045532 0.057983 -0.042511 0.099548 -0.058655 -0.016708 0.018967 -0.054657 0.044464 0.081238 0.000474 0.007267 0.036591 -0.036530 0.032288 -0.002655 -0.101196 -0.041565 -0.096497 0.012512 -0.030487 0.072571 0.074280 0.098450 0.079712 -0.026245 -0.028397 0.027893 0.084534 0.002537 -0.105713 0.004124 0.014107 -0.043701 -0.172852 -0.044708 0.049194 0.014343 0.022217 0.009819 0.068298 -0.068726 0.079163 -0.014992 -0.097473 -0.014992 0.003750 0.049622 -0.022034 -0.037598 0.000240 -0.011543 0.110352 0.038696 0.152588 -0.061890 0.027649 -0.042877 -0.108826 0.028015 0.047943 0.006035 -0.033997 0.021866 0.008965 -0.053253 -0.076416 -0.059814 0.057739 0.030136 -0.031982 -0.096313 0.008018 0.046570 0.033997 -0.038055 -0.091919 -0.035614 0.063538 -0.040527 -0.092346 0.073425 0.072449 -0.078552 -0.025314 -0.066589 0.015541 0.047729 -0.055023 -0.066406 -0.024887 -0.010658 -0.035919 -0.066345 -0.043762 0.017548 0.030121 -0.052368 -0.012619 0.061035 0.046692 -0.072937 0.123413 -0.055573 0.047424 -0.034821 0.055847 0.056335 0.117004 0.028351 0.020523 -0.046356 -0.044189 -0.016846 -0.092712 0.051605 0.010727 0.055115 0.012657 -0.020889 0.059631 -0.028168 -0.059814 -0.006104 0.032379 -0.078674 -0.046173 0.076172 -0.009941 -0.040283 0.036835 0.033295 0.055908 -0.006760 -0.009346 0.047791 -0.070068 -0.043518 -0.052765 0.140137 0.000720 0.084229 0.025421 -0.079163 -0.038544 0.096130 0.025665 0.067749 -0.036835 -0.044525 -0.058380 -0.124207 0.032715 -0.027237 -0.040558 -0.101807 -0.034302 -0.032654 0.030426 0.006027 -0.045929 0.022736 0.055603 -0.090210 0.002617 0.013718 -0.003084 0.105347 -0.062927 0.005928 0.090942 -0.040009 -0.029129 -0.025345 -0.026413 0.088806 -0.029373 -0.044952 0.115784 -0.133179 0.072510 -0.037354 -0.161743 -0.038452 0.137695 0.130127 0.136597 0.055237 -0.061249 -0.076233 0.027115 0.096191 -0.040314 0.000861 0.086853 0.035400 -0.026260 0.065552 -0.055847 0.066956 -0.082214 -0.037384 0.011841 0.119324 0.099182 -0.019150 0.068115 0.088196 -0.035645 -0.027756 -0.089600 -0.013451 0.057526 0.028732 -0.044220 0.003584 0.113953 -0.096924 0.076416 -0.021881 -0.067505 0.032501 0.045441 0.008781 0.059296 -0.077393 0.112061 0.051819 -0.097168 -0.084473 -0.111084 -0.048737 0.083984 -0.067200 0.071899 0.061371 0.027313 0.019577 +-0.079529 0.000504 -0.010086 0.007622 0.010620 0.083191 -0.005043 -0.026443 0.019272 -0.029617 0.002476 0.063049 -0.135132 0.073303 0.025452 -0.096375 0.058258 0.073425 -0.081421 -0.052979 -0.062988 0.112976 -0.107544 -0.098938 0.029129 -0.033569 0.008575 0.137451 0.139771 0.004406 0.090820 -0.025528 -0.007843 0.026276 -0.100159 -0.023163 -0.083191 0.027557 0.025558 0.093018 -0.062286 0.065247 -0.010727 -0.026474 -0.093445 -0.024399 -0.013863 -0.070923 -0.150879 0.059784 0.093872 -0.136719 -0.126465 -0.110535 0.032684 0.000199 -0.010025 -0.012985 0.134644 -0.027863 0.015442 -0.066223 -0.002583 -0.024261 0.026703 0.062225 -0.031311 0.064087 0.066406 0.009659 0.069153 -0.051331 0.079407 0.074646 0.016968 -0.038879 -0.056671 0.028107 0.064514 0.066895 -0.008865 0.008324 -0.074524 0.063293 -0.096558 -0.101440 0.091309 0.027039 -0.041260 -0.075378 -0.093079 0.045380 0.010269 -0.010284 -0.088684 0.061401 0.073975 -0.005135 -0.047791 0.001044 0.013145 -0.063049 -0.016068 0.078247 -0.046387 -0.053253 0.011703 -0.012413 -0.042145 -0.048218 -0.023483 0.030167 -0.036957 -0.074890 0.044128 -0.040802 0.006348 0.071533 0.085449 0.023560 0.007099 -0.056274 0.001731 0.000491 -0.007584 -0.123291 -0.049133 -0.028992 0.071045 0.000670 0.001268 0.048248 -0.107483 -0.026596 0.090576 -0.003971 0.021408 0.043732 0.018066 -0.019775 -0.064209 -0.063293 0.038971 -0.007889 -0.019424 -0.039795 0.049500 -0.001017 -0.026428 0.064087 0.035126 0.006451 0.023315 0.009567 -0.068420 -0.112427 0.005844 0.106079 0.020996 0.064453 0.021561 -0.149536 0.007706 -0.059448 0.004192 0.045441 -0.058319 -0.073242 -0.050903 -0.113770 0.054443 -0.017990 -0.021790 -0.089844 -0.129395 0.031769 -0.023392 -0.046844 -0.031830 0.018188 0.063721 -0.074829 -0.019226 0.037659 -0.042603 0.125977 -0.028961 0.031189 0.091370 0.007309 -0.065796 0.020187 -0.065979 0.034271 0.054321 -0.022812 -0.053345 -0.029343 0.035492 -0.061523 -0.091675 -0.078491 0.069275 0.024918 0.104126 0.015343 -0.101685 -0.045898 -0.000736 0.069458 -0.045593 0.087036 0.055878 -0.055176 -0.096436 -0.076965 0.055359 0.031494 -0.077576 0.050415 -0.040955 0.010254 0.045319 -0.075562 0.029816 0.142822 -0.105530 -0.000152 -0.111267 0.004009 -0.026749 -0.038391 -0.017105 -0.151001 -0.059906 -0.129883 0.036407 -0.089233 -0.052917 0.091064 0.023361 0.003363 0.090637 0.002390 0.010254 -0.041779 -0.137939 0.010147 -0.095520 0.034332 0.058868 -0.011925 -0.022232 -0.026718 -0.022797 -0.006886 +-0.072693 -0.011787 0.100342 0.029190 0.096741 -0.019684 0.018402 0.037231 -0.064331 0.009186 -0.077515 -0.003145 -0.149170 0.004250 0.038330 -0.127319 -0.080322 0.001558 0.156494 0.048126 -0.017059 0.070435 -0.008087 -0.012787 0.066406 -0.074280 0.047668 0.064941 0.083435 -0.022736 0.011276 -0.090149 -0.010353 0.019760 -0.150513 -0.006905 -0.078674 -0.037964 0.036285 0.051544 0.015617 0.030930 0.026199 -0.106262 -0.060455 0.018219 0.044220 -0.033783 -0.121338 -0.094849 0.042358 -0.006203 -0.165649 -0.084229 -0.022522 0.029800 0.033783 0.014145 0.075684 -0.089539 -0.035156 0.019012 -0.122803 -0.063782 0.001392 0.014618 -0.074280 -0.047272 -0.070312 0.039795 0.094666 -0.013855 -0.021057 -0.027512 0.092041 -0.066589 -0.075439 0.019211 -0.010513 0.003649 -0.046082 0.002186 0.051086 0.064331 -0.012848 -0.091431 0.122009 0.034241 -0.076172 -0.113831 0.004368 0.041870 0.080139 -0.067871 -0.110352 0.002583 -0.037354 0.005005 -0.070068 0.104248 -0.016174 -0.111816 -0.064331 0.016663 -0.043671 -0.065491 -0.073608 -0.035492 -0.039215 -0.015137 0.024582 -0.065674 0.000111 -0.018372 0.024048 -0.049591 0.062561 0.103882 0.069397 -0.050171 0.075073 -0.019592 -0.000917 0.012657 0.006210 -0.007477 0.046173 0.083069 0.132812 0.037933 0.013824 0.014709 -0.068970 -0.030670 0.007851 0.022125 -0.054535 -0.118530 0.009766 -0.050751 -0.001475 0.011261 0.024719 -0.009811 -0.078308 0.065979 0.033386 -0.005829 -0.029327 -0.032776 -0.031342 -0.068420 -0.069214 0.087280 -0.032166 -0.089661 -0.018356 0.136597 0.085327 -0.003134 0.019318 -0.119080 -0.030746 0.100891 0.003513 -0.036255 -0.018600 -0.029480 -0.006737 -0.069763 0.092590 -0.022522 -0.085999 -0.026093 0.037994 0.036346 0.022034 -0.048279 -0.045288 -0.054260 0.094971 -0.103271 -0.036591 0.040100 -0.099182 0.089111 -0.009903 -0.037109 0.070801 -0.016495 -0.000906 -0.038086 -0.072266 0.032196 0.007523 -0.042053 0.077698 -0.043976 0.033417 -0.095215 -0.048279 -0.012512 0.067810 0.123413 0.060181 0.102478 -0.058380 -0.033844 -0.083740 0.068726 -0.036957 0.125488 0.066101 -0.080261 -0.090942 0.000573 0.081970 -0.015747 -0.096252 0.084961 -0.094971 -0.005005 0.021103 -0.058350 0.071594 -0.029114 -0.009407 -0.012665 -0.035980 -0.059021 0.083984 -0.117126 0.011208 -0.037720 0.045197 -0.079163 -0.015533 -0.018127 -0.028595 0.071350 -0.029388 -0.018280 0.027435 -0.080078 0.032562 0.068787 -0.091919 0.013695 0.043823 -0.058716 0.068909 0.007935 0.092773 -0.100098 -0.088745 -0.033356 +-0.007950 -0.035278 0.057831 0.018951 0.040009 0.057861 0.019089 -0.045471 -0.026199 -0.070374 -0.049622 0.026291 -0.040771 0.017334 0.051025 -0.182129 0.035431 -0.081543 0.058807 0.024933 -0.081543 -0.023117 -0.014015 -0.016800 0.025299 0.008194 0.049255 0.041229 0.077942 -0.002451 0.075806 -0.046875 -0.031464 -0.036194 -0.083435 -0.114136 -0.031921 -0.006477 0.022888 0.106445 0.088684 0.072388 0.048492 0.033081 -0.062561 0.078125 0.087036 -0.037048 -0.020081 0.003115 0.030258 -0.103699 -0.108215 0.015900 0.071838 -0.020798 0.079834 0.027252 0.057709 -0.074097 0.012047 0.019257 -0.096436 0.052429 -0.038361 0.001329 -0.029434 -0.046631 -0.014618 0.018448 0.030243 -0.025192 0.168335 -0.088379 0.070740 -0.048981 -0.149780 0.061310 0.101685 -0.002300 -0.043243 0.023529 -0.009438 -0.023346 -0.120361 -0.102051 -0.004658 0.038849 0.002380 -0.024338 0.025665 0.083313 0.008865 -0.060730 -0.055817 0.001698 0.070251 -0.080933 -0.059021 0.053436 0.001541 -0.092163 -0.046600 -0.045563 0.021912 0.058350 -0.028763 -0.058990 -0.036987 -0.063110 -0.061829 -0.027832 -0.005993 0.017380 0.051575 -0.074219 0.005005 0.106628 0.034149 -0.003845 0.116394 -0.068970 -0.014847 -0.035675 0.072754 -0.024902 0.165527 -0.046539 0.062378 -0.013908 0.005917 0.012070 -0.065552 0.059052 0.009010 -0.031052 -0.008057 -0.077026 0.074341 -0.026443 -0.026672 0.000296 0.014328 -0.067383 0.042511 0.028366 -0.027802 -0.002668 0.038971 0.016785 0.016724 -0.013802 0.059692 -0.043030 -0.026474 -0.107239 -0.049988 0.070251 -0.075562 0.099121 0.016769 -0.078369 -0.092834 0.070801 0.035034 0.045410 -0.121155 -0.035400 -0.053223 -0.070068 0.042480 -0.121887 0.055878 -0.075256 -0.000730 0.051697 0.044586 -0.066711 0.017456 0.035095 0.060059 -0.083984 0.011459 0.074646 -0.027832 0.057739 -0.101990 0.004593 0.145142 0.048706 0.018356 -0.048096 -0.018906 -0.001160 -0.027893 -0.030029 0.060669 -0.124695 0.011932 0.069763 -0.093628 -0.042114 0.092407 0.108276 0.072388 0.098572 -0.062408 -0.101440 -0.021042 -0.002533 -0.066101 -0.004780 0.072632 0.016357 -0.023376 -0.052094 -0.019653 0.076721 -0.082336 -0.079590 0.002762 0.137329 0.065735 0.061859 0.024612 0.165894 0.051178 0.026550 -0.091614 -0.044220 0.049561 0.079407 0.036133 0.005314 0.029495 -0.054199 0.127075 -0.020203 -0.164551 0.056854 0.099121 0.046295 0.000232 -0.062988 0.109070 -0.013542 -0.044708 -0.030670 -0.028122 -0.019379 -0.002829 -0.028305 0.127441 -0.012024 0.033508 -0.031250 +-0.036774 0.059906 0.007812 -0.027405 0.027802 0.058563 0.031708 0.019852 -0.058899 -0.040771 0.007118 0.050018 -0.026794 0.039337 0.068176 -0.093628 0.042572 -0.022766 0.023041 0.085815 -0.047882 0.048462 -0.079529 0.003159 0.106384 -0.030624 0.020584 0.071350 0.121277 0.048492 0.021729 -0.000547 -0.001014 0.022263 -0.071899 0.004353 -0.090027 0.009987 0.076843 0.120850 -0.011574 0.002878 -0.028198 -0.043274 -0.122681 -0.010452 0.116882 0.026321 -0.035095 0.026276 0.066223 -0.142700 -0.110718 -0.129028 0.036743 0.092468 0.028946 -0.010063 0.063965 -0.007389 -0.013794 0.029633 -0.019318 -0.097107 0.023788 0.051880 -0.008713 -0.061493 0.045105 0.064148 0.145508 0.015343 0.068848 -0.005005 0.058746 -0.035828 -0.150146 -0.018845 0.108154 0.046631 0.027893 0.075745 0.043640 0.040955 -0.051086 0.006142 0.046204 0.030777 -0.055878 -0.074158 -0.081116 0.028915 0.019135 -0.018143 -0.058716 0.038940 0.121460 -0.087952 -0.074036 0.054962 0.019257 -0.021271 -0.029343 -0.034821 -0.027847 -0.009468 -0.084961 -0.006275 -0.073181 -0.059784 -0.019318 -0.027313 -0.113342 -0.012100 -0.010414 -0.043915 -0.035858 0.075134 0.069458 -0.047394 -0.017044 -0.113342 -0.009277 -0.026855 0.100525 0.005264 0.043335 -0.015671 0.069763 0.025787 0.086182 -0.049744 -0.112976 0.041992 0.081055 -0.014153 -0.055817 -0.042053 -0.008095 -0.052765 -0.008560 0.017288 0.012192 -0.024124 -0.022476 0.016113 0.037781 0.070068 0.021454 -0.025894 0.007637 0.015060 0.012817 0.013016 -0.087769 -0.166626 -0.027710 0.138794 0.077454 0.073425 0.107544 -0.079773 -0.051636 0.136597 -0.005013 0.110535 -0.018265 0.076843 -0.085510 -0.106934 0.052887 -0.015778 0.037201 -0.051636 -0.060486 -0.053040 0.035309 -0.004551 0.028625 -0.018753 0.050964 -0.001712 0.038269 0.086243 -0.074036 0.088501 -0.030411 0.055084 0.105713 -0.083496 -0.026337 -0.014778 -0.037781 0.039856 0.002731 -0.019775 0.012169 -0.116150 -0.026581 0.089050 -0.126953 -0.074463 0.057404 0.060913 0.113647 0.126343 -0.177490 -0.016098 0.121338 0.025635 -0.034912 0.037933 0.054993 0.002819 -0.044312 -0.017319 -0.002031 0.054047 0.002380 0.025864 -0.065430 0.120789 0.033661 0.065063 0.037476 0.164795 0.007702 0.021988 -0.102295 -0.028351 0.027954 -0.056335 -0.024338 -0.043091 -0.024368 -0.092346 0.027893 -0.004936 -0.091309 0.057648 0.029129 0.064697 0.062347 -0.014366 0.105286 -0.029861 -0.035004 -0.012291 -0.043488 -0.002838 0.077637 -0.021118 0.067261 -0.046326 -0.028198 0.019073 +-0.019547 0.017227 0.079468 0.050995 -0.020905 0.054535 0.063477 0.010490 -0.075073 -0.027695 -0.036591 0.005089 -0.050415 -0.009796 0.065369 -0.159302 0.028259 -0.103210 0.031952 0.041595 -0.074524 0.036835 0.003414 0.004292 0.091736 -0.019775 0.019318 0.123596 0.091431 0.084656 0.047760 -0.075256 -0.080139 0.047638 -0.089172 -0.030365 -0.032898 -0.005299 0.034027 0.136230 -0.006153 0.004593 -0.020660 -0.057709 -0.154297 0.028580 0.072815 -0.027557 -0.080261 -0.006535 0.044159 -0.076477 -0.194458 -0.118530 0.001003 0.013329 -0.002794 -0.037262 0.078369 -0.052185 -0.019104 0.067749 -0.052307 -0.044281 -0.019089 -0.037018 -0.019043 -0.112366 -0.004452 0.027481 0.078003 -0.022705 0.033630 -0.048706 0.076233 -0.039429 -0.141479 0.001478 0.106079 0.027679 0.019119 -0.010971 0.076660 0.070923 -0.070129 0.007637 0.054382 0.018478 -0.065247 -0.059662 0.035461 0.056519 0.016312 -0.109497 -0.091309 0.016678 0.058228 -0.058716 -0.019608 0.047974 0.008888 -0.077820 -0.052521 -0.039948 -0.047821 0.062927 -0.074585 -0.025009 -0.042480 -0.041351 0.006721 -0.080444 -0.102600 -0.011734 -0.024414 -0.074036 -0.013779 0.098022 0.035858 -0.045227 0.036591 -0.086182 -0.032288 -0.010651 0.117126 0.044708 0.133179 -0.000095 0.007702 -0.000188 0.034454 -0.052490 -0.103088 0.054626 0.019211 0.024567 0.032745 -0.086304 -0.001982 -0.085938 -0.013824 0.042572 0.031250 -0.121704 -0.041443 0.066345 0.049042 0.030045 -0.000487 -0.053009 -0.018280 0.011147 -0.004734 0.001528 -0.063721 -0.177979 -0.077087 0.121643 0.034363 0.026962 0.038391 -0.059692 -0.070862 0.133667 0.021194 0.040527 0.000845 0.043121 -0.090576 -0.050781 0.028046 -0.032043 0.029404 -0.055267 -0.020523 0.029922 0.021484 -0.019424 0.043457 -0.000768 0.095520 -0.065491 0.007076 0.061493 -0.083496 0.017975 -0.070068 -0.006790 0.103516 -0.037903 0.011345 -0.002974 -0.061798 0.018494 0.024857 -0.005688 0.017654 -0.091797 -0.030640 0.061340 -0.094360 0.028656 0.068726 0.114441 0.042694 0.123596 -0.102112 -0.067688 -0.051117 -0.021927 -0.054047 0.016983 0.085022 -0.055756 -0.088867 -0.021667 0.001245 0.129517 -0.000105 -0.028290 -0.089905 0.091797 0.070679 0.044403 0.107361 0.118713 -0.009193 0.001342 -0.065735 -0.090759 0.062744 -0.050201 -0.011253 -0.028641 -0.019089 -0.081360 0.038177 -0.005924 -0.100769 0.075806 -0.002293 0.025299 0.014084 -0.096252 0.108765 0.042755 -0.044373 0.031555 -0.020432 -0.063660 0.049377 0.059967 0.079712 -0.019913 -0.069519 -0.024734 +-0.099182 -0.005066 0.021225 0.041534 0.022964 0.086365 0.043854 -0.005367 -0.035828 -0.026932 0.039917 0.029739 -0.040344 0.031403 0.073242 -0.131226 0.041107 -0.035278 0.069397 -0.028214 -0.099670 0.088501 -0.085144 -0.116577 0.074890 -0.016815 -0.033081 0.002676 0.087585 0.018158 0.060028 -0.052917 -0.057190 -0.071045 -0.073059 -0.187256 -0.090088 -0.025604 0.024994 0.117737 0.022842 0.034973 -0.064453 -0.025558 -0.175415 0.025665 0.054901 -0.015671 -0.116333 -0.014664 0.101135 -0.145142 -0.055359 -0.048553 0.041168 0.063782 0.061981 0.016357 0.051331 -0.074158 0.000396 0.041718 -0.044952 -0.043640 0.034119 0.048859 0.000998 0.001293 0.014595 0.047272 0.068970 0.003481 0.036926 -0.000822 -0.002008 -0.007492 -0.104431 -0.055817 0.108215 0.069031 -0.047333 0.022583 -0.067383 0.033600 -0.064941 -0.058563 0.044800 0.029953 -0.033630 -0.083862 -0.016327 0.044037 -0.032349 -0.035828 -0.058197 0.022308 0.098083 -0.023361 -0.011063 0.042847 -0.014885 -0.066040 -0.039337 0.057129 -0.029373 -0.044922 0.020065 0.000145 0.013512 -0.030045 -0.062744 0.014297 -0.052612 -0.045868 -0.033386 -0.074707 -0.004780 0.133057 0.078918 -0.004128 0.010666 -0.044159 0.005611 0.006313 0.010712 -0.095337 0.013992 0.051422 0.012039 0.062927 0.044006 -0.047089 -0.063660 0.002865 0.010681 0.040802 -0.003397 0.039734 0.026505 -0.023529 -0.039856 -0.022858 0.052826 -0.099487 -0.001122 0.035339 0.073364 0.030167 -0.028961 0.042938 0.081665 0.006699 0.060913 -0.011246 -0.012276 -0.092712 -0.021683 0.015945 -0.004311 0.099121 0.032410 -0.060486 0.008461 0.063416 -0.004272 0.095154 -0.043365 0.020187 0.018539 -0.083435 0.078674 -0.111816 0.058105 -0.037048 -0.075562 0.071411 -0.007881 -0.094727 0.003986 0.010460 -0.018463 -0.040161 0.013809 0.112366 -0.119629 0.163940 0.004501 0.022598 0.109497 -0.067810 -0.032806 0.011215 -0.023804 0.021713 0.069763 -0.030396 0.063782 -0.068726 0.049561 0.045929 0.006653 -0.064209 -0.001053 -0.026169 0.076294 0.067444 -0.079590 -0.050201 0.036072 0.091858 -0.073486 0.091614 0.091553 -0.017975 -0.025253 -0.041229 -0.034149 -0.005013 -0.031921 0.080383 -0.113281 0.138306 0.010628 -0.000715 0.087891 0.230347 0.031067 0.060028 -0.186768 -0.050842 0.009583 -0.094971 0.042694 -0.068420 -0.054352 -0.093201 0.130371 -0.096924 -0.046082 0.024399 0.033966 -0.073059 0.068176 -0.030472 0.085205 -0.066650 -0.018066 -0.007835 -0.076233 -0.012787 0.037811 -0.001780 0.080505 -0.025040 0.006687 0.001554 +-0.002672 0.004059 0.026733 0.010757 0.015106 0.086914 0.065796 -0.050537 -0.040924 -0.048615 0.022690 0.055389 -0.011513 0.045227 0.033173 -0.125732 0.079285 -0.035553 -0.069458 -0.001429 -0.090271 0.070923 -0.080750 -0.076172 0.032318 -0.036713 -0.007397 0.093140 0.125244 0.035828 0.122253 -0.069458 -0.008095 0.041077 -0.101562 0.009193 -0.057312 0.009148 0.008636 0.118774 -0.010574 0.071045 0.069946 -0.021774 -0.111633 0.034119 0.096008 -0.027496 -0.103882 0.057892 0.067505 -0.157715 -0.170532 -0.086731 0.047028 -0.013161 0.005875 0.026962 0.098511 -0.056976 0.056030 -0.040436 -0.045837 -0.006550 -0.013000 0.021942 0.004185 -0.050049 0.054352 -0.001036 0.066711 -0.023544 0.143799 -0.037781 0.002804 -0.054565 -0.118469 0.071777 0.097595 0.006382 -0.001653 0.035065 -0.008743 -0.016754 -0.103516 -0.054108 0.050659 0.013130 -0.025101 -0.030762 -0.003277 0.105652 -0.010101 -0.058441 -0.089661 0.018799 0.079163 -0.045532 -0.021484 0.046417 0.048798 -0.106201 -0.002993 -0.044739 -0.018768 0.088440 -0.031174 -0.044067 -0.074158 -0.052063 -0.064575 -0.041046 -0.092102 -0.040955 0.045959 -0.056305 -0.005505 0.094849 0.033752 0.009224 0.042328 -0.089050 0.023071 -0.056366 0.105957 -0.021790 0.109131 0.007389 0.033264 -0.050262 -0.002802 0.002913 -0.065247 0.039398 0.060150 0.015266 0.048676 -0.018097 0.017242 -0.038422 -0.031677 0.015205 0.021362 -0.086975 -0.015106 0.045349 -0.014008 0.016617 0.021271 0.014214 -0.007179 0.000177 0.051727 -0.010948 -0.080811 -0.132080 -0.056763 0.114258 -0.016556 0.067444 0.043976 -0.097473 -0.050110 0.037781 0.044250 0.073608 -0.092468 -0.046295 -0.076172 -0.115295 0.041016 -0.032104 0.046783 -0.104492 -0.117310 0.002762 0.021545 -0.027527 0.015839 0.030273 0.060760 -0.074524 0.025879 0.048279 0.001818 0.040070 -0.119873 0.030243 0.137207 0.011017 -0.027191 -0.044159 -0.044098 0.067688 0.014481 -0.005348 -0.002609 -0.142456 0.010223 0.002949 -0.171997 -0.035400 0.100769 0.081604 0.081970 0.059143 -0.123291 -0.072754 -0.009857 0.011314 -0.049072 -0.004211 0.082336 0.008682 -0.069885 -0.027573 0.000644 0.088989 -0.042786 -0.016357 -0.022034 0.107605 0.057037 0.054565 0.092285 0.171631 -0.045654 -0.001109 -0.085327 -0.042786 0.014229 0.013802 -0.041016 -0.080200 0.034760 -0.088806 0.052002 -0.017181 -0.106201 0.052856 0.030502 0.021698 0.059326 -0.036957 0.097900 -0.040710 -0.097534 0.022766 -0.035065 -0.043304 0.049133 0.008873 0.037415 0.034973 -0.000811 -0.023911 +-0.033356 -0.034546 -0.002728 -0.010056 0.047394 0.056671 0.029099 -0.058655 -0.011841 -0.065125 -0.045746 0.014122 -0.076660 -0.031021 0.024994 -0.169678 0.030624 -0.044495 -0.001725 -0.036255 -0.101257 0.080139 -0.048157 -0.075623 0.039581 -0.062866 0.001045 0.083923 0.044678 -0.019852 0.106628 -0.089905 -0.023071 -0.026917 -0.182129 -0.073608 -0.077209 -0.002439 0.049744 0.131470 0.022186 0.041656 0.075195 -0.002686 -0.084961 0.042877 0.093628 -0.039886 -0.089966 0.005455 0.045105 -0.053894 -0.172852 -0.057800 0.061707 0.005562 0.041077 0.003012 0.045776 -0.064636 0.011955 -0.010483 -0.055603 0.030624 -0.040955 -0.002554 -0.006348 -0.032593 -0.009300 -0.040100 0.045074 -0.042480 0.140015 -0.053650 0.026001 -0.093994 -0.088379 0.059540 0.052246 -0.005692 -0.018097 0.005764 -0.009865 -0.022491 -0.081421 -0.091553 0.035004 0.014458 -0.016434 -0.054993 0.022873 0.091614 0.036102 -0.076050 -0.091797 0.046722 0.009140 -0.052368 -0.044617 0.126587 0.048370 -0.133057 -0.008705 -0.047302 -0.049835 0.045441 -0.021759 -0.079041 -0.095947 -0.056152 -0.045135 -0.001440 -0.000105 0.001273 0.044281 -0.042694 0.036774 0.108521 0.006268 -0.019897 0.098999 -0.052521 -0.018921 -0.036499 0.017334 -0.034393 0.091980 -0.027481 0.028458 -0.001714 -0.017715 -0.019150 -0.042480 0.030685 0.010597 -0.005608 -0.005108 -0.054871 0.061890 -0.028534 -0.086487 0.041534 0.016510 -0.054535 -0.017990 0.066528 -0.003099 -0.050201 -0.029160 0.005199 0.038422 0.025894 0.066162 -0.002695 -0.018707 -0.069336 -0.027725 0.089233 -0.053650 0.092407 0.040314 -0.085083 -0.075317 0.048584 0.026321 0.045410 -0.115051 -0.083374 -0.040924 -0.103516 0.055267 -0.074280 -0.021698 -0.116638 -0.051788 0.030289 0.049744 0.005131 -0.006432 0.023773 0.052612 -0.105286 -0.009377 0.040436 -0.077515 0.061005 -0.105042 0.013550 0.160889 0.029907 -0.029160 -0.053619 -0.014496 0.007076 -0.007786 -0.039001 0.028610 -0.123962 0.070007 -0.055634 -0.137451 -0.039246 0.110840 0.134766 0.123657 0.061676 -0.064392 -0.060852 -0.029709 0.063049 -0.045807 0.026840 0.109619 -0.006126 -0.074280 0.000083 -0.064453 0.094727 -0.112854 -0.029205 -0.023346 0.117798 0.074585 -0.022858 0.062134 0.143433 -0.079590 0.000958 -0.085815 -0.018127 0.053772 0.046356 -0.032806 -0.054596 0.056396 -0.064941 0.058990 -0.066284 -0.106201 0.040741 0.034576 -0.010460 0.062805 -0.015701 0.069458 0.016159 -0.072998 -0.021057 -0.067200 -0.071716 0.063721 -0.050934 0.106812 0.044342 -0.018616 -0.041840 +-0.002821 0.025803 0.044159 -0.007591 -0.024857 0.100464 0.018265 -0.035095 -0.049591 -0.051483 0.053223 0.046173 -0.100525 0.140991 0.019089 -0.059235 0.053009 0.067810 0.021881 0.004410 -0.064148 0.106201 -0.068420 -0.088318 -0.002048 0.011604 0.072021 0.107178 0.106323 -0.033020 0.077148 -0.049347 -0.022018 0.043884 -0.082520 -0.040894 -0.057526 0.048553 0.011414 0.103027 -0.017990 0.108459 -0.035797 -0.016418 -0.051697 0.023163 0.007095 -0.065979 -0.154297 0.067993 0.091003 -0.135864 -0.106628 -0.130615 -0.037048 -0.020493 -0.021149 -0.003214 0.139038 -0.022232 0.056915 -0.089417 0.001871 0.043243 0.046478 0.065369 0.028397 -0.019608 0.087036 0.010872 0.025284 -0.052429 0.152222 0.034760 0.020905 -0.041595 -0.078308 0.075989 0.096252 0.080444 -0.035065 0.052490 -0.054169 0.023605 -0.102539 -0.073547 0.130005 -0.001807 0.007305 -0.077881 -0.066772 0.059479 0.007988 -0.028412 -0.092896 -0.011925 0.027695 -0.005627 -0.034302 0.000691 -0.004726 -0.073181 -0.055084 0.019928 -0.072266 -0.013229 0.012375 -0.010880 -0.020554 -0.072815 -0.058380 -0.015511 -0.014206 -0.043518 0.066040 -0.064392 -0.000247 0.099304 0.052490 0.055054 0.046112 -0.056427 0.085571 -0.024567 -0.008087 -0.085571 0.037323 -0.014427 0.026825 -0.016541 0.011932 0.040955 -0.093994 0.043274 0.098633 0.071960 0.004017 -0.027695 0.053528 -0.031250 -0.008774 -0.073975 0.043427 -0.053040 -0.031952 -0.015961 0.043121 -0.045288 -0.002979 0.093811 0.058044 0.015060 0.017899 -0.038239 -0.063110 -0.134277 -0.084229 0.096497 -0.025116 0.036163 0.024384 -0.130493 0.025482 -0.093994 0.017410 0.059753 -0.103088 -0.052063 -0.083374 -0.091187 0.051514 -0.075562 0.025864 -0.063293 -0.094788 0.089661 0.011978 -0.108154 0.006912 0.005478 0.068909 -0.083801 0.034546 0.054291 -0.032684 0.099304 -0.043427 0.046051 0.089722 0.046509 -0.021622 -0.017944 -0.077271 0.085449 0.083252 -0.005154 -0.003345 0.003571 0.014122 -0.010872 -0.154053 -0.054291 0.108093 0.058777 0.062164 0.033569 -0.105652 -0.056183 -0.039368 0.063660 -0.009010 0.036804 0.081543 -0.072205 -0.087402 -0.086365 0.053528 0.036438 -0.058533 0.001586 0.004669 0.008781 0.012077 0.014732 0.028824 0.194580 -0.026169 -0.044373 -0.102966 -0.045837 -0.014503 -0.018097 -0.004013 -0.075745 -0.009956 -0.122498 0.033783 -0.042450 -0.033264 0.083496 0.011345 0.036377 0.034454 -0.009483 0.047577 -0.016571 -0.102356 0.020218 -0.106445 -0.011894 0.062073 0.010345 -0.014198 -0.018204 -0.013016 -0.013954 +-0.051422 -0.053284 0.033264 -0.019348 0.068909 0.092041 -0.007626 -0.049500 -0.018585 -0.018509 -0.024490 0.023331 -0.092041 -0.051697 0.052521 -0.158569 -0.001888 0.013802 -0.018372 0.056274 -0.113647 0.049927 -0.018723 -0.039642 0.066772 -0.038025 0.009285 0.036957 0.110291 -0.014015 0.099365 -0.024445 0.010544 0.026718 -0.131836 -0.059906 -0.123230 -0.000493 0.050476 0.082458 0.036774 0.028610 0.064758 -0.040253 -0.095154 0.006378 0.111267 -0.037750 -0.070679 -0.021194 -0.011139 -0.085022 -0.119202 -0.054291 -0.006268 0.027054 0.079224 -0.026031 0.058014 -0.014084 0.014961 -0.047241 -0.129150 0.021713 -0.073364 -0.006302 -0.001666 -0.072205 -0.045105 0.010315 0.056580 -0.057861 0.106812 -0.061462 0.089111 -0.050049 -0.067810 0.059448 0.066345 -0.023392 -0.041290 -0.026657 0.015091 0.015144 -0.044128 -0.069580 0.012932 0.013809 -0.035828 -0.072083 -0.007751 0.153931 0.008934 -0.084351 -0.102417 -0.019806 0.050262 -0.049683 -0.085144 0.093811 0.042603 -0.149292 -0.028427 0.007362 -0.094604 -0.027130 -0.071899 -0.089783 -0.086975 -0.042114 -0.065063 -0.013649 -0.061584 -0.073181 0.021790 -0.049225 0.014732 0.083130 0.011467 -0.037445 0.103821 -0.039001 -0.027924 -0.068298 0.101074 -0.050781 0.085571 0.016190 0.086304 0.005844 0.006462 -0.003719 -0.077576 0.048737 0.023132 -0.030807 -0.052460 -0.083557 0.000701 -0.020462 -0.086426 0.015518 0.030045 -0.053162 -0.056732 0.058624 -0.009583 -0.033203 -0.043854 0.026459 -0.024567 -0.022308 0.034515 0.014580 -0.033356 -0.088257 -0.021194 0.123474 0.008614 0.056152 0.046875 -0.116333 -0.063660 0.066162 0.046844 0.070007 -0.092346 -0.072937 -0.010422 -0.103455 0.078003 -0.042969 -0.005531 -0.107361 -0.028351 0.019852 0.002773 0.000597 0.063965 -0.000360 0.041931 -0.072998 -0.041748 0.056549 -0.057983 0.047974 -0.065735 0.014565 0.132202 -0.018295 -0.050079 -0.066467 -0.073059 0.010010 -0.011925 -0.054504 0.052734 -0.181274 0.077820 -0.045624 -0.124268 -0.041901 0.023117 0.122131 0.085083 0.061157 -0.073975 -0.050476 0.016479 0.056702 -0.098999 0.088379 0.071533 0.014328 -0.107910 -0.022766 0.011124 0.049835 -0.092468 0.015518 -0.040497 0.075928 0.046265 -0.019745 0.064575 0.108826 -0.066101 0.046539 -0.070496 -0.008858 0.036987 -0.034485 -0.040802 -0.058136 0.095459 -0.040802 0.078247 -0.049438 -0.113708 0.022324 0.021973 -0.035706 0.082458 -0.057648 0.056549 0.057007 -0.092590 -0.034424 -0.006813 -0.047485 0.076721 -0.015038 0.098694 -0.017593 0.005287 -0.057800 +-0.083923 -0.012230 0.036377 0.001880 0.021027 0.098328 0.088074 -0.047119 -0.083435 0.030960 0.012840 0.022110 -0.025635 0.007576 0.040222 -0.104980 0.031799 0.003391 -0.014328 0.080750 -0.052856 0.077148 -0.070862 -0.006203 0.115295 -0.024765 -0.053528 0.059540 0.118042 0.079163 0.098328 -0.004402 -0.044159 0.083862 -0.062927 -0.026306 -0.086182 -0.012077 0.033142 0.084900 -0.015068 -0.005337 -0.015839 -0.082764 -0.130981 -0.024643 0.101074 -0.011688 -0.087646 -0.039490 0.030334 -0.163574 -0.107239 -0.141602 0.025986 0.070068 0.046326 -0.045837 0.022034 -0.036896 0.001183 0.026443 -0.074341 -0.006821 0.000267 -0.016037 -0.049896 -0.116150 0.032227 0.029068 0.074280 -0.000811 0.057343 -0.062927 0.066162 -0.058685 -0.067627 0.012573 0.090881 0.003183 -0.027817 -0.003468 0.041870 0.008095 -0.015152 -0.011559 0.040588 0.029663 -0.109985 -0.122498 0.018463 0.107300 -0.009285 -0.073914 -0.101318 0.005703 0.119202 -0.004696 -0.077576 0.057648 0.032440 -0.077576 -0.057434 0.044891 -0.054169 0.004650 -0.093750 0.040436 -0.063660 0.020065 -0.064148 -0.044128 -0.085144 -0.079407 0.019424 -0.082153 -0.070129 0.058411 0.043793 0.005188 -0.016678 -0.095032 -0.012497 -0.041290 0.122498 0.006672 0.086182 -0.002424 0.070068 -0.000957 0.029709 0.001432 -0.095703 0.081177 0.028687 -0.013474 -0.014427 -0.095276 -0.040283 -0.039459 0.027740 -0.009064 0.014862 -0.117004 -0.057037 -0.005444 0.065369 -0.016647 0.021286 0.017456 0.014336 -0.040070 0.027634 0.029068 -0.074524 -0.130371 -0.008507 0.121033 0.050934 0.060547 0.081299 -0.095215 -0.080566 0.089905 0.025238 0.028152 -0.066772 -0.001808 -0.019608 -0.091431 0.045502 0.032990 0.038818 -0.041168 -0.083618 0.009956 -0.001778 0.008873 0.040405 0.021317 0.054016 -0.076111 0.024612 0.043762 -0.044189 0.048004 -0.010689 -0.006115 0.116516 -0.037201 -0.032440 -0.022400 -0.051666 0.048187 -0.004246 0.000114 0.003225 -0.202393 0.018555 0.010117 -0.122131 0.004211 -0.005424 0.051971 0.013687 0.083557 -0.095276 -0.114319 0.024429 0.051422 -0.080688 0.077942 0.073425 0.004986 -0.061737 -0.074890 0.051819 0.075134 -0.044739 0.034943 -0.110535 0.098267 0.082581 0.013878 0.101440 0.111450 -0.069519 0.039032 -0.114014 -0.040375 0.022415 -0.049500 -0.053467 -0.050598 0.028809 -0.076111 0.046692 0.017807 -0.155518 0.061096 -0.004524 -0.013512 0.040497 -0.059937 0.103699 -0.029770 -0.070801 -0.027710 0.006432 -0.050720 0.047729 0.082214 0.055542 -0.018814 0.014801 -0.007912 +-0.038055 0.019958 0.049561 0.041565 -0.013458 0.048767 0.025894 0.024277 -0.060150 -0.004219 -0.010170 -0.001792 -0.160034 0.095154 0.058044 -0.100464 0.001265 0.027161 0.046783 -0.025162 -0.106812 0.093262 -0.044556 -0.096619 0.016220 -0.045166 0.054718 0.070923 0.123962 -0.019836 0.048279 -0.039490 -0.060577 -0.031555 -0.121948 -0.039825 -0.044128 0.019684 -0.011787 0.158203 0.004612 0.101318 0.011192 -0.075317 -0.130127 0.006428 0.025177 -0.051422 -0.121582 -0.009109 0.119995 -0.091309 -0.114380 -0.091431 0.021194 0.017303 -0.006992 0.022659 0.126465 -0.099365 0.009476 -0.004822 -0.075073 -0.054962 0.034912 0.069763 -0.049225 0.017838 0.015327 0.080994 0.060272 -0.027176 0.003904 -0.002186 0.049194 -0.050659 -0.130371 -0.002192 0.100281 0.084290 -0.059326 0.002989 -0.040375 0.130737 -0.059326 -0.131836 0.087219 0.074768 -0.063599 -0.080322 -0.046356 0.012016 0.013664 -0.023819 -0.105591 0.052124 0.015106 -0.039337 -0.053833 -0.005165 -0.013947 -0.066895 -0.040619 0.096863 -0.030121 -0.054016 0.003914 -0.058014 -0.029083 -0.039673 -0.031174 -0.045319 -0.036102 -0.071655 0.032715 -0.075317 0.040039 0.135498 0.072388 -0.003748 0.016144 -0.075562 -0.011444 -0.019455 0.019958 -0.112732 -0.013557 0.066162 0.090149 0.043732 0.008461 0.006870 -0.085693 -0.037445 0.064697 0.061066 0.021210 0.004478 0.039917 -0.038757 -0.034210 -0.053619 0.005909 -0.031921 -0.039185 0.028717 0.054077 0.104980 0.003412 -0.048187 0.026352 -0.015762 0.012947 0.008896 -0.033752 -0.166016 -0.004364 0.092224 0.071167 0.043793 0.024017 -0.109741 0.025528 0.055420 -0.035034 0.036987 0.007515 -0.043732 -0.076782 -0.078735 0.110657 -0.079407 -0.035950 -0.041412 -0.048218 0.031616 -0.016083 -0.081970 -0.014015 -0.019241 0.048950 -0.077698 -0.032562 0.088989 -0.120178 0.118713 -0.021210 0.001942 0.062225 -0.024872 -0.034668 0.016403 -0.064392 0.011452 0.080566 -0.017426 0.007988 -0.010399 0.001991 0.002514 0.012474 -0.013641 0.063721 0.014748 0.064331 0.069824 -0.111450 -0.060699 -0.041199 0.053741 -0.033447 0.122681 0.055145 -0.037018 -0.078003 -0.044312 0.109619 -0.029160 -0.057770 0.092041 -0.087097 0.020737 0.011780 -0.014877 0.043213 0.157104 0.013802 -0.001750 -0.102783 -0.048615 0.028397 -0.114380 0.045258 -0.077942 -0.051392 -0.072021 0.063110 -0.056976 -0.049683 0.069885 0.005642 0.024658 0.067627 -0.033905 0.082336 -0.031616 -0.055603 0.076233 -0.071289 0.024704 0.048096 -0.029373 0.031235 -0.098694 -0.029358 -0.051636 +-0.004452 -0.003263 0.044342 -0.006626 -0.053192 0.069397 0.042877 -0.056915 -0.046265 0.008614 0.029205 0.022827 -0.115234 0.135010 0.019547 -0.107544 0.045074 -0.041199 0.021286 0.038055 -0.053955 0.044586 -0.040985 -0.052399 0.010002 0.046509 0.083252 0.088867 0.118347 0.006226 0.137695 -0.022202 -0.039490 0.046051 -0.126831 0.006031 -0.063843 0.069519 -0.022369 0.105652 0.003038 0.086365 -0.016495 -0.068909 -0.058624 0.046265 0.041656 -0.025833 -0.147827 0.074951 0.087585 -0.111328 -0.089783 -0.045929 -0.039398 -0.042816 0.020233 -0.039825 0.115112 -0.023148 0.064209 -0.119690 -0.067322 0.015686 -0.012474 0.029480 0.039520 -0.064575 0.069946 0.050079 0.045074 -0.043518 0.098206 0.012657 0.035065 -0.056824 -0.109619 0.025223 0.181030 0.065186 -0.042755 0.002779 -0.012924 0.016998 -0.074890 -0.020874 0.043762 0.030609 -0.012344 -0.064575 -0.027969 0.075317 -0.031830 -0.088074 -0.102173 -0.070312 0.048645 -0.054413 -0.077881 0.036102 0.006519 -0.100037 -0.084290 0.012535 -0.069458 0.001565 -0.011230 -0.068726 -0.033447 -0.057190 -0.034302 -0.009758 -0.048798 -0.059326 0.039276 -0.068848 -0.020523 0.094482 0.011330 0.043030 0.027939 -0.042725 0.054535 -0.037201 0.069885 -0.043793 0.076294 -0.003647 0.001885 -0.037964 -0.023758 0.065857 -0.118225 0.075134 0.091858 0.037872 0.010193 -0.075256 0.036346 -0.028183 -0.009163 -0.073242 0.034363 -0.046265 -0.061066 0.029587 0.015068 0.049042 0.014870 0.028641 0.002771 -0.004677 0.019989 -0.054474 -0.090332 -0.145630 -0.111145 0.144409 0.008011 0.021179 0.024017 -0.122192 0.022675 -0.038422 0.025162 0.049561 -0.044037 -0.019165 -0.047638 -0.083862 0.040771 -0.022903 0.056702 -0.056274 -0.073792 0.062195 -0.005711 -0.024521 0.060150 0.056305 0.059448 -0.111938 0.012901 0.081360 -0.014015 0.033936 -0.054382 0.023087 0.100586 0.006187 -0.010628 -0.044983 -0.104614 0.061401 0.101868 0.008530 0.007843 -0.097961 0.028030 0.008217 -0.117371 0.002672 0.015297 0.066162 0.060944 0.068481 -0.106323 -0.106628 -0.019516 0.086060 -0.014824 0.052307 0.153687 -0.034546 -0.062866 -0.079590 0.061523 0.054749 -0.015961 -0.017670 -0.028564 0.031555 -0.010193 0.001993 0.068970 0.194458 0.015320 -0.025223 -0.097961 -0.022797 0.008041 0.039307 -0.012276 -0.081604 0.062195 -0.112732 0.043121 -0.034241 -0.062866 0.054230 0.003893 -0.008354 0.059448 -0.064819 0.097900 0.029205 -0.097595 0.057068 -0.076172 -0.037506 0.039276 -0.005642 -0.021835 -0.007084 0.002829 -0.035126 +-0.080139 0.004890 -0.001188 0.046844 0.072693 0.085388 0.084534 -0.066772 -0.044617 -0.047699 -0.011375 0.073181 0.018509 0.016113 0.049194 -0.158936 0.030365 -0.027267 -0.007996 0.014519 -0.053772 0.057190 -0.085571 -0.049103 0.083740 -0.027191 -0.049347 0.051361 0.051422 0.041779 0.085693 -0.028198 0.012779 0.012482 -0.092712 -0.095398 -0.074829 -0.011581 0.019470 0.119629 0.044159 0.030579 0.025757 0.000083 -0.146851 0.000938 0.061279 0.013176 -0.099609 0.006954 0.043182 -0.188354 -0.097656 -0.082886 0.081238 0.083069 0.045135 0.009468 0.023773 -0.058685 0.018066 0.069092 -0.034332 0.019608 0.002611 0.055145 -0.013062 -0.083740 -0.012505 0.011131 0.061371 -0.009201 0.141357 -0.026199 -0.012093 -0.035156 -0.107788 0.054169 0.078979 0.015884 0.001720 0.011414 -0.000395 0.002388 -0.108826 -0.038513 -0.011688 0.020126 0.017487 -0.117859 0.001474 0.061646 -0.007038 -0.099609 -0.049377 0.022385 0.109009 -0.014183 -0.091675 0.050903 0.050476 -0.048248 -0.011078 0.019104 -0.001171 0.070435 -0.063477 0.000863 -0.054108 0.003925 -0.076904 0.012589 -0.035004 -0.063782 0.013916 -0.074951 -0.024246 0.123535 0.056976 -0.007507 0.017807 -0.104675 -0.013039 -0.002748 0.068481 -0.010826 0.069519 -0.019684 0.079712 0.008194 0.024170 -0.037933 -0.099243 0.041016 -0.020508 -0.035614 -0.032837 -0.054535 -0.007641 -0.051270 -0.032806 0.027237 -0.019363 -0.102844 -0.036621 0.020004 0.051178 -0.048828 0.022568 0.046082 0.014961 -0.015465 0.061035 0.025742 -0.050659 -0.103210 0.005821 0.090210 -0.042328 0.137695 0.047302 -0.056671 -0.088135 0.026230 0.009293 0.060486 -0.082214 -0.012062 -0.048553 -0.106201 0.036316 -0.059143 0.075867 -0.070923 -0.123840 0.009308 0.043732 -0.019302 0.046173 0.003698 0.048248 -0.068726 0.069641 0.036407 -0.002552 0.116455 -0.049225 -0.005409 0.101929 -0.024002 0.010139 -0.021988 -0.026489 0.043884 -0.013351 0.011971 0.002176 -0.134521 0.053864 0.043152 -0.145874 -0.009880 0.046692 0.069763 0.107300 0.033142 -0.075317 -0.113953 0.057220 0.077209 -0.058929 0.024109 0.078674 0.010574 -0.005505 -0.062317 -0.047791 0.089294 -0.067322 0.031433 -0.073792 0.165283 0.046631 0.052277 0.091003 0.161255 -0.064880 0.014786 -0.149048 -0.061127 0.018341 0.003496 -0.018280 -0.043152 -0.007374 -0.106079 0.107605 0.002144 -0.118347 0.058624 0.047455 -0.022339 0.051453 -0.018509 0.119446 -0.072937 -0.071106 -0.066040 -0.018814 -0.003826 0.042603 -0.009926 0.090393 0.017899 0.019165 -0.000245 +-0.030579 -0.037018 0.093201 0.015778 0.078064 0.032928 0.022293 0.082703 -0.059753 0.003086 -0.045227 0.000596 -0.054291 -0.059052 0.061707 -0.088745 -0.034332 -0.014809 0.086731 0.160278 -0.056366 -0.066589 0.036774 0.089905 0.090210 0.009636 0.026077 -0.027557 0.016968 0.083679 -0.018631 0.008858 0.003799 0.117126 0.026550 0.008095 0.037170 -0.028229 0.033295 0.040161 0.073914 0.005024 0.001634 -0.070068 -0.045959 0.003378 0.119141 -0.002274 0.080994 -0.085571 -0.057495 -0.076599 -0.026367 -0.031464 -0.032349 0.023834 0.024841 -0.012764 0.016953 -0.099731 -0.003805 0.074524 -0.208862 -0.037750 -0.047943 -0.038391 -0.048187 -0.069763 -0.091858 0.089111 0.118225 0.008972 -0.071716 -0.071350 0.172852 0.015701 -0.140259 0.027908 0.032715 -0.045319 -0.041779 -0.024933 0.084473 0.029877 0.050842 -0.041412 0.011452 0.068237 -0.089905 -0.071350 0.063354 0.056488 0.036224 -0.055786 -0.018738 -0.056885 0.116516 -0.026413 -0.086548 0.023834 0.003456 -0.014076 -0.033020 0.016556 -0.023911 -0.034790 -0.151733 -0.000883 0.007282 0.021057 -0.060242 -0.113281 -0.088074 -0.038452 -0.003799 -0.064636 -0.042694 0.023941 0.029373 -0.067871 0.026962 -0.071167 0.017639 -0.045990 0.175659 0.033539 0.085632 0.070190 0.128174 -0.032074 0.009315 -0.019562 -0.051971 0.042542 -0.012505 -0.010437 -0.022537 -0.078613 -0.066284 0.007004 0.042450 -0.038605 0.033142 -0.059052 -0.033142 -0.005440 0.029236 0.005566 0.004143 -0.050781 -0.080627 -0.057678 -0.090698 0.048889 -0.018723 -0.071594 -0.012131 0.097778 0.065002 -0.034515 0.014633 -0.086060 -0.095154 0.182861 0.037933 0.029266 0.042664 -0.020813 0.018372 -0.017059 0.061340 -0.045166 0.019165 0.017838 0.089233 -0.003374 -0.013435 0.050690 -0.005253 -0.067871 0.054901 -0.025330 -0.087952 0.077820 -0.061127 -0.020996 -0.034485 -0.053070 0.055450 -0.042877 -0.023346 0.032867 -0.033447 0.044708 -0.034271 -0.017487 0.050476 -0.171387 0.005592 0.011017 0.029465 0.005554 -0.027390 0.081787 -0.023773 0.083374 -0.061462 -0.067871 0.020706 0.000172 -0.124146 0.077881 -0.054108 0.109253 0.004658 0.000799 0.107300 -0.016739 -0.046753 0.033905 -0.065918 0.052002 0.015900 0.018280 0.033203 -0.024475 0.074829 0.086670 0.011276 -0.063782 0.028366 -0.084595 0.009186 0.057159 0.066040 -0.009193 -0.011086 0.063416 -0.104187 0.005638 0.014366 0.065186 -0.005424 -0.165405 0.008659 0.043091 -0.047668 -0.023590 0.128174 -0.068726 -0.009415 0.021774 0.091858 -0.136597 0.015137 -0.045898 +-0.097900 -0.018066 0.051697 0.019623 0.008308 0.054993 0.025314 0.018570 -0.082092 -0.039581 0.048065 0.013893 -0.109375 0.099854 0.100342 -0.077087 0.026047 0.072449 -0.006367 0.084534 0.055847 0.022552 -0.057312 -0.060486 0.006004 -0.032623 0.081543 0.034210 0.077637 0.053314 0.000253 -0.009506 -0.018860 0.089722 -0.050903 -0.019867 -0.038330 0.086731 0.004475 0.070068 -0.022797 0.084473 -0.011024 -0.071716 -0.089417 0.056183 0.073059 -0.011292 -0.130615 0.056915 0.068665 -0.116150 -0.125244 -0.077576 -0.018539 0.022568 -0.005653 0.009415 0.091187 -0.064270 0.009895 -0.085022 -0.014755 -0.078003 0.048340 0.013161 0.053345 0.062744 0.091125 0.020935 0.079407 -0.019775 0.131348 0.004440 0.037231 -0.047302 -0.085388 -0.055511 0.148926 0.048828 0.012016 0.061859 -0.052124 0.043701 -0.084106 0.016937 0.085449 -0.008255 -0.029602 -0.050293 -0.017563 -0.014091 -0.009750 0.044678 -0.108948 0.004478 0.055115 0.002878 -0.032898 -0.045074 0.082214 0.002533 -0.034515 0.052887 -0.027649 0.064697 -0.076904 0.002522 0.028824 -0.096069 -0.039642 -0.075378 -0.058044 0.027420 0.006630 -0.056244 -0.016632 0.046265 0.115295 0.037964 0.048584 -0.019958 0.029434 0.036774 0.031677 0.102539 0.026459 0.063293 -0.002195 -0.015373 0.055328 0.092224 -0.222046 -0.008385 0.105103 0.042145 -0.004360 0.073853 0.018982 -0.002781 0.087463 -0.057251 0.038391 -0.029236 -0.003014 0.008064 0.056366 0.065063 0.057526 0.026001 0.101929 -0.042145 -0.003368 -0.092712 -0.089111 -0.134155 -0.114685 0.151978 0.124756 -0.077515 0.033600 -0.085449 0.059052 0.060242 0.060303 0.068237 0.011040 0.039124 -0.060333 -0.064514 0.077393 0.022858 0.079651 -0.027756 -0.029053 0.028442 -0.040955 -0.069580 -0.007431 0.016357 0.113892 0.011200 0.002327 0.112061 0.005257 0.063049 -0.024750 0.065796 0.031769 -0.041870 -0.043793 0.047272 -0.039032 0.026520 0.028107 -0.060211 0.005028 -0.050201 -0.088867 -0.031158 -0.087830 -0.014725 0.121155 0.001140 0.015823 0.015404 -0.078125 -0.130859 0.043182 0.061218 -0.070740 -0.057983 0.125732 -0.048492 -0.028580 -0.128052 0.052490 0.029022 -0.032959 0.008850 -0.044250 0.009117 0.011604 0.011948 0.021652 0.154175 0.051300 0.027084 -0.116333 -0.057465 0.009033 -0.081299 0.069641 0.015854 -0.030167 -0.017487 -0.080994 0.013069 -0.023911 0.075012 -0.004845 0.028564 0.012962 -0.094604 0.042114 0.035065 -0.041901 0.061371 -0.045441 0.001947 0.041168 -0.013412 0.010803 -0.132324 -0.024384 0.009148 +-0.050232 -0.029907 0.029968 0.070984 0.040710 0.065491 0.114807 -0.041656 -0.019653 -0.049713 0.020020 0.046844 0.043121 0.021820 0.034637 -0.145630 0.047852 0.023804 -0.006161 -0.040955 -0.043427 0.136475 -0.111938 -0.082581 -0.001426 -0.034973 -0.012955 0.096436 0.057983 0.032257 0.080627 -0.060852 -0.016174 0.016693 -0.043213 -0.080688 -0.024429 0.023132 -0.031769 0.094055 0.025726 0.101501 0.004742 0.028336 -0.125488 0.051392 0.007545 -0.058228 -0.176270 0.028961 0.068970 -0.171387 -0.127075 -0.095520 0.025986 -0.006886 -0.003963 0.056519 0.089661 -0.105347 0.064758 0.019669 -0.063782 0.007202 0.014999 0.034882 -0.034515 -0.042053 0.035858 -0.013237 0.015320 -0.006924 0.108337 -0.011086 -0.015358 -0.022949 -0.088440 0.025055 0.085266 0.061005 -0.041260 0.001339 -0.058899 0.009125 -0.177124 -0.073792 0.059448 0.004276 -0.011566 -0.114624 0.002691 0.027420 -0.015915 -0.088440 -0.077026 0.015884 0.075745 0.013824 -0.046204 -0.033752 0.046173 -0.033112 -0.021545 0.055817 0.058533 0.073975 -0.020203 0.016922 0.015854 -0.007866 -0.065552 -0.027496 -0.030762 -0.083130 -0.013062 -0.079712 -0.051575 0.155884 0.075806 0.015732 0.021255 -0.049866 0.022003 0.001189 0.040924 -0.016296 0.103638 0.013123 0.013039 -0.034515 -0.007328 0.027924 -0.120667 -0.017715 0.004032 0.057098 0.083252 0.068054 0.028336 -0.015518 -0.012276 -0.002171 0.035767 -0.115479 -0.029968 -0.012703 0.080688 -0.033661 0.025665 0.030258 0.041992 0.010574 0.039276 -0.009750 -0.027176 -0.101196 0.001236 0.045654 -0.032471 0.082581 -0.019379 -0.112549 -0.007164 -0.027527 0.015656 0.076782 -0.042267 -0.071228 -0.061951 -0.131348 0.024002 -0.043549 0.032562 -0.092407 -0.122681 0.095642 0.008530 -0.117371 -0.001532 0.015182 0.058105 -0.061249 0.008995 0.044281 -0.008644 0.132935 -0.058197 -0.035675 0.070190 0.010468 0.012047 -0.007393 -0.053864 0.072571 0.029541 -0.009552 -0.011047 -0.050598 0.042328 0.004589 -0.117554 -0.022949 0.094116 0.025574 0.060303 0.010223 -0.022995 -0.119873 -0.039307 0.081360 -0.078552 -0.005035 0.116333 0.007858 -0.052277 -0.078552 0.024384 0.069702 -0.081299 0.028488 -0.078247 0.109619 0.064941 -0.002388 0.070129 0.161743 -0.045288 -0.000926 -0.184326 -0.087952 -0.029404 -0.009354 -0.010010 -0.072388 -0.047394 -0.092651 0.095642 -0.003744 -0.057465 0.071289 0.021484 -0.011467 0.009033 -0.077759 0.086975 -0.024933 -0.098328 -0.000576 -0.064819 -0.001911 0.031372 0.021484 0.007420 0.017944 0.024734 0.006130 +-0.070312 -0.005947 0.027618 -0.002037 0.045868 0.094788 -0.056000 -0.002676 -0.092529 -0.000098 -0.038666 0.000777 -0.112854 -0.026031 0.094971 -0.123108 -0.035004 0.058929 0.032867 0.143921 -0.101318 0.009460 0.048767 0.023743 0.085571 -0.051208 0.009773 0.037415 0.153442 -0.009506 -0.009415 0.047485 -0.008148 -0.026474 -0.073853 -0.024109 -0.130859 -0.002478 0.042175 0.065063 0.014091 -0.019730 -0.001864 -0.110046 -0.086609 -0.030090 0.122681 -0.036438 0.006531 -0.043915 -0.005352 -0.071289 -0.099792 -0.074951 0.020462 0.102295 0.080017 -0.046722 0.071350 -0.001657 -0.061432 -0.028610 -0.097229 -0.024887 -0.031525 0.010147 -0.036255 -0.037689 -0.015274 0.051361 0.073059 -0.043304 0.058685 -0.057983 0.124451 -0.046875 -0.077332 -0.012367 0.040070 -0.007004 -0.012955 -0.023727 0.041443 0.056854 0.003002 -0.048309 0.021881 0.006939 -0.093628 -0.066345 -0.004349 0.089478 0.017868 0.017838 -0.099304 -0.013039 0.048340 -0.071167 -0.080261 0.075989 0.008705 -0.080566 -0.064148 0.042450 -0.083374 -0.087280 -0.110718 -0.050629 -0.102295 -0.045929 -0.040955 -0.021957 -0.051788 -0.016785 0.003160 -0.040283 -0.007442 0.025772 0.054565 -0.064758 0.072998 -0.059113 -0.057770 -0.015366 0.114136 -0.001216 0.047028 0.026245 0.125000 0.076599 0.069824 0.006954 -0.129395 0.022415 0.060455 -0.057617 -0.141724 -0.126831 -0.048492 -0.066589 -0.018692 -0.019989 0.018097 0.004642 -0.044586 0.027374 0.009232 0.081543 0.006332 -0.012268 -0.023163 -0.063416 0.009834 -0.015190 -0.050354 -0.125854 -0.034637 0.161377 0.135254 -0.010040 0.093994 -0.096130 -0.037140 0.148193 0.019608 0.066284 -0.063049 0.018158 -0.025818 -0.031464 0.107483 -0.025482 -0.022354 -0.057739 0.009651 -0.041138 -0.025269 0.002100 0.043091 -0.059875 0.049072 -0.019424 -0.014809 0.080505 -0.093262 0.033081 0.023361 0.031952 0.056000 -0.024506 -0.039490 -0.036987 -0.076477 -0.077393 -0.005325 -0.055328 0.055359 -0.152344 -0.000388 -0.001614 -0.039124 -0.037048 0.022781 0.091675 0.016159 0.081970 -0.097595 -0.041138 0.042297 0.039703 -0.089233 0.108032 0.036102 0.006413 -0.089600 -0.045868 0.068665 -0.008690 -0.049316 0.040558 -0.070251 0.029251 0.045563 0.009613 0.018402 0.071106 -0.039612 0.086121 -0.033844 -0.010750 0.041962 -0.111572 -0.017242 -0.006210 0.042236 0.008698 0.037933 0.017273 -0.159790 0.076294 0.046417 -0.012756 0.083435 -0.056946 0.047729 0.075806 -0.023743 -0.013535 0.029831 0.017609 0.031708 -0.005264 0.139771 -0.110962 0.002268 -0.048065 +-0.035767 -0.028107 0.045929 -0.037933 0.070496 0.030014 -0.012184 -0.032318 -0.044220 -0.037415 -0.006599 -0.011330 -0.102112 -0.027328 0.026840 -0.131714 0.000075 -0.027939 0.042206 0.035828 -0.093262 0.045410 -0.005596 -0.039001 0.031158 -0.007919 0.068787 0.019775 0.079224 -0.026947 0.100586 -0.049438 -0.006439 0.016922 -0.156250 -0.088623 -0.143799 0.002457 0.034729 0.079590 0.053314 0.076050 0.067627 -0.045105 -0.054840 0.045074 0.089355 -0.026901 -0.108093 -0.014633 0.006062 -0.019257 -0.135010 -0.043091 -0.006924 0.009819 0.092285 -0.018723 0.063599 -0.013832 0.025146 -0.062317 -0.085999 0.012665 -0.025925 0.012329 0.038208 -0.059631 -0.033295 -0.005219 0.065918 -0.035797 0.147095 -0.098022 0.075745 -0.080505 -0.058929 0.057678 0.080933 0.002102 -0.039459 0.020325 -0.005451 -0.022675 -0.041809 -0.048492 0.052094 0.000624 -0.019958 -0.042084 -0.002735 0.118835 0.040894 -0.062866 -0.113464 -0.030167 0.006264 -0.055481 -0.066833 0.098206 0.023132 -0.157227 -0.057220 -0.055023 -0.093445 -0.007530 -0.026260 -0.098022 -0.075623 -0.070374 -0.055664 -0.030441 -0.048279 -0.001378 0.035217 -0.053864 0.032135 0.102173 0.018951 -0.026199 0.147583 -0.019958 0.011856 -0.044617 0.048828 -0.011116 0.110779 0.059631 0.029861 0.012886 -0.013382 0.018631 -0.055603 0.059906 0.041870 0.021408 -0.063843 -0.107910 0.060547 -0.055664 -0.073547 0.029037 0.039612 -0.057556 -0.074158 0.116821 -0.035431 -0.027084 -0.019653 0.024796 0.051483 -0.036377 0.030197 0.015320 -0.047394 -0.079163 -0.072998 0.112488 -0.006538 0.038269 0.031555 -0.084351 -0.020676 0.063049 0.054199 0.054077 -0.087952 -0.020065 -0.016617 -0.083801 0.092407 -0.069885 -0.009178 -0.080750 0.012032 0.029785 0.014832 -0.017441 0.030228 0.036194 0.063232 -0.088074 -0.005589 0.055847 -0.049408 0.054474 -0.069763 0.041809 0.150879 -0.020218 -0.037109 -0.076172 -0.051025 0.012947 0.016388 -0.071106 0.109802 -0.121643 0.075317 -0.041870 -0.148315 -0.014671 0.074036 0.124207 0.103149 0.084045 -0.095215 -0.055084 0.019196 0.047821 -0.036011 0.040833 0.103882 -0.032471 -0.097717 0.020737 -0.028336 0.044922 -0.089417 -0.019363 0.011261 0.056000 0.037628 -0.000360 0.052429 0.125732 0.008881 0.008469 -0.083374 -0.002447 0.065125 -0.005718 -0.024139 -0.027908 0.113953 -0.034790 0.071411 -0.088074 -0.095764 0.026947 0.019043 -0.036530 0.084534 -0.050842 0.095459 0.061249 -0.070190 -0.039642 -0.068420 -0.081421 0.086121 -0.024994 0.115662 0.010643 0.001048 -0.042938 +-0.091309 -0.041840 -0.017227 0.037476 0.069885 0.044189 0.057861 -0.029266 -0.035309 -0.055267 -0.106934 0.024063 -0.093384 0.016891 0.045410 -0.146362 0.001634 0.006683 -0.012306 -0.022766 -0.077454 0.142578 -0.045990 -0.073303 -0.001961 -0.089905 0.025497 0.131470 0.034973 -0.025711 0.072876 -0.081116 0.028412 -0.016251 -0.164062 -0.047333 -0.079407 -0.027161 -0.004993 0.115906 -0.000871 0.073303 0.066528 -0.007275 -0.096741 -0.039948 0.037781 -0.021149 -0.162842 0.002285 0.025742 -0.072144 -0.179810 -0.034698 0.064026 0.039551 -0.065430 0.031036 0.084961 -0.096802 0.046753 -0.069153 -0.085754 0.019531 0.001538 0.039551 -0.013947 -0.062134 -0.020050 -0.045135 0.008232 -0.022339 0.072754 -0.022461 0.042664 -0.027893 -0.066223 0.106750 0.055603 0.015106 -0.000353 -0.016129 -0.004581 0.042572 -0.094482 -0.113770 0.063538 -0.018570 0.047882 -0.108887 0.034668 0.083069 0.016022 -0.083069 -0.078003 0.029419 0.018051 -0.007244 -0.045563 0.074158 0.056213 -0.064514 0.036407 -0.005085 -0.079163 0.047119 -0.011284 -0.049530 0.023422 -0.042633 -0.030258 -0.032562 0.023331 -0.043365 0.062622 -0.031860 0.053589 0.091492 0.073608 -0.046844 0.089233 -0.118652 0.055664 -0.030838 0.002535 0.000182 0.031006 0.015556 0.028030 -0.007492 -0.024673 0.002804 -0.065674 -0.035919 -0.008743 0.040161 -0.004658 -0.004826 0.024841 -0.029205 -0.084839 -0.019821 -0.018845 -0.077515 -0.025772 0.092590 0.040924 -0.062927 -0.013405 0.046051 0.015869 0.004486 -0.002451 0.030060 -0.008904 -0.057587 0.004189 0.094482 -0.026001 0.055389 0.022614 -0.095093 -0.021057 0.026230 0.010574 0.017090 -0.046967 -0.122070 -0.079651 -0.090088 0.019547 -0.047516 -0.067566 -0.165405 -0.081055 0.008186 0.000103 0.003269 -0.035248 -0.039276 0.045563 -0.115479 -0.036804 0.019348 -0.029617 0.090759 -0.084106 -0.030762 0.057007 0.000660 -0.056763 0.034943 -0.035400 0.054810 -0.015884 0.019852 0.019379 -0.079041 0.067017 -0.085449 -0.099670 -0.030701 0.124390 0.064148 0.111633 -0.025208 -0.040375 -0.062103 -0.080139 0.138184 -0.017395 0.055206 0.063110 -0.013359 -0.071106 0.022064 -0.030945 0.079041 -0.114380 0.032623 -0.056000 0.107788 0.084167 -0.085693 0.073547 0.055695 -0.135986 -0.074097 -0.056183 -0.044922 0.000926 0.001854 -0.036560 -0.027893 0.038849 -0.101257 0.027390 -0.097290 -0.100708 0.075928 0.016922 0.016983 0.103394 -0.017441 0.029602 0.039764 -0.133789 -0.025330 -0.077148 -0.069824 0.042938 -0.094421 0.061188 0.018356 -0.027145 -0.025879 +-0.061066 -0.021957 0.035431 0.039612 0.062927 0.059570 0.050812 -0.066895 -0.020813 -0.034119 -0.001148 0.043549 -0.057648 0.027618 0.052643 -0.205322 0.026062 -0.054840 0.010536 -0.028885 -0.074707 0.044586 -0.067505 -0.072266 0.040802 -0.040161 -0.032990 0.038513 0.103943 0.008881 0.097107 -0.040070 0.016800 -0.012970 -0.144653 -0.059906 -0.068909 0.024048 -0.019058 0.117981 0.054718 0.063965 0.069275 -0.036316 -0.106079 0.026123 0.047852 -0.004196 -0.139160 0.018509 0.058411 -0.120972 -0.130737 -0.024231 0.047852 0.012466 0.039642 0.025833 0.072998 -0.072571 0.034790 -0.040039 -0.092712 0.003031 -0.043274 0.066650 0.022888 -0.055481 -0.055908 0.037231 0.093384 -0.040070 0.130981 -0.022552 0.026627 -0.030457 -0.127930 0.039185 0.125854 -0.011856 -0.034729 0.004208 -0.013329 0.027222 -0.098328 -0.038208 -0.004276 0.035645 0.026871 -0.069702 -0.033752 0.125366 -0.009605 -0.075562 -0.074463 -0.013626 0.070557 -0.060669 -0.076355 0.072571 0.039673 -0.134766 0.013535 0.013916 -0.059937 0.033691 -0.016647 -0.115356 -0.044189 -0.049225 -0.082092 0.000600 -0.055176 -0.054932 -0.003834 -0.064392 0.029053 0.121155 0.034332 -0.033508 0.079712 -0.024933 -0.009132 -0.041016 0.092529 -0.048798 0.056488 0.038147 0.050446 -0.014854 -0.034546 -0.025528 -0.074890 0.018234 0.001547 -0.002087 -0.014481 -0.023422 0.013733 -0.016922 -0.078247 -0.005035 0.004311 -0.041626 -0.010475 0.121460 -0.014023 -0.003683 -0.015335 0.021606 -0.012581 -0.027542 0.054810 0.007996 -0.063904 -0.077637 -0.014053 0.113525 -0.040710 0.091614 0.022186 -0.059570 -0.020645 0.021362 0.027283 0.066956 -0.056458 -0.046112 -0.014099 -0.104309 0.065491 -0.075256 0.044952 -0.117065 -0.089661 0.004448 0.041565 -0.029251 0.034943 0.019028 0.037933 -0.091064 0.006676 0.051117 -0.005554 0.119019 -0.100830 0.016006 0.124634 -0.021378 -0.024353 -0.039337 -0.066162 0.037781 0.008987 -0.010506 0.054413 -0.155273 0.069824 0.004520 -0.108948 -0.011665 0.039429 0.075989 0.123352 0.037445 -0.085571 -0.080505 0.030151 0.056213 -0.082275 0.067200 0.123779 0.005703 -0.033813 -0.027008 -0.048981 0.056152 -0.060150 0.013588 -0.044891 0.151123 0.009689 0.036743 0.106384 0.175293 -0.015526 0.007092 -0.106384 -0.048767 0.038971 0.006836 0.002825 -0.048950 0.058472 -0.086487 0.096802 -0.052094 -0.065979 0.015602 0.015030 -0.056427 0.105164 -0.034729 0.068726 -0.017593 -0.072266 0.001398 -0.032166 -0.016296 0.043579 -0.044891 0.073181 -0.002176 0.015717 -0.036774 +-0.011597 0.009224 0.061188 0.070740 0.073547 0.052887 -0.026871 0.083252 -0.070923 -0.001356 -0.093811 0.005955 -0.091614 -0.039978 0.073364 -0.068909 -0.029114 -0.033630 0.058228 0.120239 -0.106262 -0.059082 0.057251 0.069885 0.067627 -0.049286 0.014259 -0.031097 0.088196 0.033508 -0.025818 -0.014336 0.009621 0.014389 -0.079468 0.024826 -0.024017 -0.077820 0.016602 0.073914 0.024841 0.001710 0.062378 -0.100647 -0.078003 -0.022888 0.122253 0.000040 0.076477 -0.063660 -0.003466 -0.040161 -0.079285 -0.014557 0.041351 0.068359 0.052734 0.028244 0.029968 -0.084961 -0.066406 0.045776 -0.136475 -0.087402 -0.042419 0.018021 -0.061584 -0.041382 -0.070190 0.104126 0.098328 0.002977 -0.072937 -0.080078 0.143066 -0.029068 -0.132568 0.028336 0.015007 -0.030914 0.022186 -0.002262 0.049469 0.087830 0.045807 -0.076599 0.004013 0.071411 -0.085022 -0.018448 0.013542 0.060883 0.048798 -0.014961 -0.034607 0.030136 0.042328 -0.078186 -0.047882 0.059143 0.006111 -0.029709 -0.021576 0.013359 -0.017532 -0.040680 -0.119812 -0.066467 -0.069275 -0.033325 -0.018875 -0.096008 -0.067505 0.002481 0.014648 -0.038788 0.010139 0.038971 0.049042 -0.079041 0.011505 -0.084534 -0.050446 -0.005039 0.130249 0.012039 0.049469 0.085938 0.175659 0.074280 0.031830 -0.021423 -0.025299 -0.042999 0.006660 -0.051910 -0.071777 -0.079224 -0.076660 -0.045349 0.005753 0.008797 -0.035706 0.030579 -0.033783 0.053070 0.002026 0.114746 0.017044 -0.108704 -0.089905 -0.048492 -0.040924 0.036560 -0.011375 -0.103210 0.014679 0.131836 0.124023 -0.057343 0.045837 -0.080994 -0.054962 0.225952 0.025452 0.000490 0.022919 0.020508 -0.018982 0.003445 0.080078 -0.043701 -0.025253 -0.005783 0.083252 -0.078308 -0.010803 0.050323 -0.010391 -0.081543 0.032043 -0.020203 -0.036926 0.093506 -0.078491 -0.028183 -0.041382 -0.035553 0.061707 -0.043945 -0.007530 -0.010330 -0.018250 -0.056915 -0.019485 -0.027054 0.023788 -0.136719 -0.041107 0.009155 0.041870 -0.008469 0.007008 0.052856 0.003860 0.090637 -0.115417 -0.002768 0.017212 -0.024734 -0.053833 0.091553 -0.062744 0.034027 -0.034973 0.033691 0.096558 -0.058594 -0.018204 0.091980 -0.073120 0.035553 -0.003664 0.016418 0.012451 -0.008110 0.060913 0.102478 0.032410 -0.007519 0.040680 -0.139771 0.026581 0.017151 0.062439 0.025986 -0.003233 0.021759 -0.130249 0.036438 0.039551 0.060944 0.073242 -0.053162 0.029663 0.031250 -0.030518 0.022049 0.114746 -0.025497 -0.012901 -0.002275 0.155518 -0.161133 -0.037170 -0.063354 +-0.047546 -0.023758 0.042542 -0.047363 0.066833 -0.041779 0.041504 -0.040222 -0.007126 -0.023148 0.013535 0.075623 0.037292 0.117981 0.036499 -0.130005 0.065063 -0.047546 -0.057251 0.029495 0.096741 0.032898 -0.152954 0.003876 0.018631 -0.086853 -0.006409 0.061615 0.137817 0.045410 0.065552 -0.032776 0.040253 0.101868 0.046295 0.095825 -0.003689 0.045654 -0.001965 0.002628 0.003094 0.096252 0.092407 -0.000301 -0.030411 0.070801 -0.017792 0.035248 -0.086548 0.040710 0.054749 -0.144531 -0.140625 -0.026474 0.059601 -0.027328 0.003029 0.033813 0.067993 -0.014908 0.105164 -0.083557 -0.056946 -0.053192 0.002748 0.062561 -0.024384 -0.066895 0.061249 0.046051 0.103149 0.039368 0.154785 -0.015549 0.016678 0.009064 -0.130737 0.056152 0.083374 0.003569 -0.022964 0.046509 0.032196 0.001301 -0.147095 -0.020615 0.013458 0.006241 0.022507 -0.056213 -0.086853 0.069214 -0.021530 -0.022369 -0.067627 -0.040405 0.122925 -0.043274 -0.015564 -0.025757 0.032806 -0.054962 0.034637 -0.050781 0.043610 0.078064 0.010925 0.005165 0.043823 -0.012611 0.012711 -0.055298 -0.082886 -0.037628 0.066467 -0.057343 -0.035492 0.005573 0.058411 -0.018677 0.061371 0.014023 0.046448 -0.006062 0.133057 0.051208 0.087708 -0.036682 0.049042 -0.146362 0.006248 0.033600 -0.148071 0.060181 0.053619 -0.022552 0.049866 0.005589 0.027740 0.054840 0.017700 0.008614 -0.005390 -0.048889 0.056854 0.022598 -0.042847 -0.038269 0.069641 0.053619 -0.026962 -0.056732 0.002092 0.011528 -0.084106 -0.072876 0.011353 0.123413 -0.013611 0.046997 0.035767 -0.113037 -0.054047 0.012878 0.018555 -0.016571 -0.016068 -0.046722 -0.034576 -0.131836 -0.015221 0.083557 0.055145 -0.114807 -0.067139 -0.029114 0.051392 -0.038544 0.010284 0.048645 0.067322 -0.032104 0.003948 -0.030884 0.130859 0.050812 -0.085999 0.001678 0.091492 -0.013969 -0.018661 0.000199 -0.078308 0.139160 -0.080933 -0.039093 0.011566 -0.147583 -0.093506 0.002775 -0.182007 -0.064697 0.063904 0.039551 0.084534 0.064941 -0.064941 -0.081909 0.015106 -0.031647 -0.029648 -0.004768 0.049103 -0.009132 -0.000238 -0.073364 0.047211 0.071411 -0.005383 -0.143433 0.010498 0.102234 0.060913 0.069458 0.031860 0.054413 0.024933 -0.060486 -0.056671 -0.021652 0.030045 0.107666 0.000616 -0.051849 0.001122 -0.101013 0.022095 0.000906 -0.044556 0.032837 0.017914 0.101440 -0.040161 -0.076477 0.005505 -0.028046 -0.100037 0.004539 -0.033722 0.070374 0.022491 -0.028687 -0.061249 0.013702 0.033997 0.030701 +0.014473 -0.034119 0.076538 -0.039612 -0.028641 0.061066 0.025070 -0.008263 -0.005573 0.011421 -0.007374 -0.061554 -0.087097 0.022202 0.023483 -0.081116 -0.011429 -0.012222 0.002342 0.027771 -0.042023 0.060852 -0.086670 -0.025589 0.027206 -0.032440 0.048462 0.064453 0.123291 0.051361 0.125854 -0.123230 -0.093323 0.047607 -0.126587 -0.006531 -0.001294 0.017899 0.049347 0.108154 -0.029846 0.091553 0.049713 -0.031235 -0.083130 0.039703 0.069885 -0.061554 -0.088257 -0.031342 0.063354 -0.049011 -0.139160 -0.125122 -0.040863 -0.012276 0.014091 -0.006950 0.072876 -0.041565 -0.003115 -0.011948 -0.124146 -0.067383 -0.012741 -0.031219 -0.061920 -0.048340 0.064026 0.021210 0.057159 -0.006321 0.014450 -0.095825 0.084717 -0.098328 -0.084595 0.047943 0.068665 0.023560 -0.087341 -0.003687 -0.006123 0.002434 -0.054291 -0.087402 0.082031 0.042755 -0.143188 -0.068115 -0.005428 0.108215 0.049103 -0.070618 -0.098938 0.025558 -0.001887 -0.050659 -0.031464 0.047943 0.013100 -0.127686 -0.095276 0.018875 -0.006142 -0.009178 -0.046021 -0.020844 -0.091003 -0.032654 -0.036682 -0.068481 -0.017105 -0.079895 0.011940 -0.114563 -0.054688 0.088318 0.005554 0.003443 0.003407 -0.046753 0.018051 -0.080139 0.103638 -0.066956 0.122498 0.011688 0.087402 -0.040070 0.027115 0.026596 -0.058319 0.000517 0.048065 0.052429 0.082947 -0.065063 0.021484 0.012314 0.025787 0.012779 0.028336 -0.056519 -0.028610 0.026627 0.014771 0.013885 0.026306 -0.073730 0.003214 -0.005219 0.036652 0.012558 -0.054321 -0.135376 -0.006046 0.097778 0.049194 0.029404 0.053009 -0.162231 -0.052979 0.078552 0.017075 -0.005718 -0.014236 -0.096313 -0.058319 -0.086609 0.076782 0.018204 -0.008408 -0.052917 -0.016693 0.073547 -0.001875 -0.023941 0.035248 0.008995 0.055481 -0.131226 -0.072449 0.057892 -0.099121 -0.011238 -0.066101 -0.033051 0.144775 0.017151 -0.028503 -0.044769 -0.038635 0.047211 0.063477 -0.034912 0.005493 -0.143921 -0.007713 -0.073669 -0.118103 -0.075745 0.046417 0.080505 0.037750 0.116882 -0.116699 -0.062347 -0.059784 0.027313 -0.051025 0.061737 0.056671 0.023499 -0.138428 -0.059875 0.115784 0.025925 -0.123718 0.005402 -0.067627 0.060822 0.036591 -0.028976 0.015266 0.117188 0.032471 0.005711 -0.075623 -0.026398 0.034332 -0.024155 -0.017349 -0.085449 0.020905 -0.020660 -0.029572 -0.003895 -0.078003 0.029953 -0.027817 0.064880 0.003700 -0.083801 0.043823 -0.029938 -0.093872 0.063416 -0.011620 -0.064819 0.060669 0.093323 0.026871 -0.031494 -0.012276 -0.063843 +-0.061096 -0.034180 0.016708 -0.004459 0.078674 0.043854 0.034973 -0.013474 -0.020416 -0.079041 -0.052979 0.015152 -0.075928 -0.004086 0.027908 -0.154907 0.022537 -0.027115 0.061340 0.005032 -0.082458 0.102417 -0.106262 -0.059845 0.019073 -0.054169 0.025116 0.042633 0.013664 0.002943 0.083679 -0.076782 0.020721 -0.008904 -0.157715 -0.083618 -0.054688 0.002615 0.030457 0.119812 0.049530 0.081665 0.064941 0.007214 -0.043915 0.024017 0.088196 -0.016129 -0.106934 0.000476 0.054657 -0.065552 -0.165405 -0.052826 0.031311 0.023575 0.041382 0.019058 0.062561 -0.073364 0.031830 -0.015396 -0.106628 0.008072 -0.014641 0.039062 -0.029907 -0.038452 0.003872 0.006195 0.094971 0.007469 0.120117 -0.061401 0.070374 -0.066406 -0.123718 0.063721 0.054291 -0.002232 -0.036285 0.050110 -0.011627 -0.019592 -0.083435 -0.107056 0.039398 0.043976 -0.024170 -0.083496 -0.019394 0.110229 0.052704 -0.044373 -0.066772 0.033447 0.029785 -0.051575 -0.093140 0.103516 0.067261 -0.106812 -0.005936 -0.036285 -0.024399 0.024460 -0.033112 -0.065552 -0.056519 -0.051727 -0.047485 -0.035828 0.000826 0.006443 0.031616 -0.069580 -0.006382 0.119385 0.033997 -0.038025 0.072327 -0.051239 0.028107 -0.038940 0.007168 -0.042236 0.106689 -0.004845 0.066345 -0.024338 -0.018845 -0.015900 -0.054749 0.008995 0.011536 0.016815 0.004055 -0.008705 0.070190 0.020355 -0.052002 0.008987 0.011536 -0.023346 0.009384 0.072876 0.006355 -0.069458 -0.023331 0.021637 0.050232 0.031647 0.025726 0.027863 -0.042084 -0.059723 -0.018326 0.102600 -0.051544 0.081177 0.030106 -0.117065 -0.037018 0.068970 0.037140 0.065125 -0.077576 -0.099731 -0.036926 -0.136108 0.043549 -0.079773 -0.024719 -0.106995 -0.031677 0.022919 0.065308 -0.011642 -0.044739 0.002163 0.029343 -0.116821 -0.016449 0.059479 -0.075195 0.111328 -0.100525 0.004383 0.154663 0.004131 -0.028015 -0.032196 -0.026886 0.048737 -0.012581 -0.037201 0.073486 -0.138306 0.068420 -0.040649 -0.131348 -0.072205 0.120667 0.112854 0.150757 0.066406 -0.085205 -0.036255 0.022202 0.083191 -0.069092 0.037537 0.061035 0.019791 -0.041931 0.009712 -0.025162 0.044495 -0.122681 -0.018677 -0.011780 0.136597 0.057556 -0.032928 0.046082 0.146729 -0.019363 0.002245 -0.083191 -0.004807 0.050751 0.022430 -0.008263 -0.018570 0.085205 -0.088013 0.040436 -0.065125 -0.093872 0.017929 0.023941 0.022186 0.068237 -0.032104 0.045532 0.007732 -0.081238 -0.051208 -0.088257 -0.060059 0.066040 -0.064453 0.093262 0.009872 0.026764 -0.039276 +-0.055603 -0.061920 0.038422 -0.025848 0.053955 0.044678 0.032410 -0.068237 0.017670 -0.039398 -0.052277 0.017654 -0.041870 -0.048950 0.006226 -0.141968 0.011536 -0.020844 0.026443 0.006992 -0.059845 0.092346 -0.109375 -0.057953 0.053650 -0.040527 -0.006413 0.048279 0.083618 -0.001571 0.146118 -0.094421 0.018707 0.051605 -0.086853 -0.092407 -0.055298 -0.045898 0.077576 0.058716 0.029633 0.066467 0.063843 0.022690 -0.066711 0.006302 0.046997 -0.004311 -0.115662 -0.061005 0.006062 -0.081604 -0.133179 -0.049591 -0.009773 -0.013741 0.041199 0.008232 0.063110 -0.042633 0.049469 -0.019806 -0.149658 0.028763 -0.048523 -0.000068 -0.020737 -0.101135 -0.016647 -0.007118 0.062378 -0.001393 0.058441 -0.057922 0.080750 -0.014572 -0.076294 0.102356 0.034515 -0.024353 -0.062042 0.004822 0.025345 -0.024750 -0.057739 -0.088013 0.042908 -0.003452 -0.010773 -0.059357 -0.002190 0.195312 0.040192 -0.098206 -0.080322 -0.003990 0.053528 -0.011665 -0.017410 0.094177 0.028534 -0.157349 0.008324 -0.030838 -0.076965 -0.001004 -0.006222 -0.033081 -0.018646 -0.020279 -0.038116 -0.028366 -0.020126 -0.057648 0.027191 -0.070435 -0.004478 0.100525 -0.002728 -0.039032 0.098145 -0.054993 0.054871 -0.082825 0.053864 -0.095947 0.097534 0.013641 0.043243 -0.067810 0.001536 0.002729 -0.019699 0.060028 -0.022415 0.003891 0.025330 -0.024078 0.036713 0.057190 -0.063354 0.054535 0.026367 -0.128174 0.014526 0.079834 -0.006462 -0.133301 -0.047760 0.044189 -0.013390 -0.003633 0.014923 0.068909 -0.010109 -0.038940 0.031113 0.057648 -0.096313 0.079346 0.023987 -0.125000 -0.088684 0.045746 0.032776 0.016296 -0.081421 -0.131104 0.022369 -0.126709 0.001607 -0.028122 -0.036957 -0.127319 -0.025192 0.048279 0.045624 -0.029633 0.014297 0.009827 0.001327 -0.105896 -0.061401 0.037262 -0.055298 0.063599 -0.100952 -0.036652 0.161377 0.008453 -0.062103 -0.022156 -0.031586 0.092346 -0.052582 -0.010941 0.068359 -0.154907 0.080078 -0.063782 -0.126587 -0.076965 0.032318 0.052856 0.087708 0.072998 -0.037811 -0.019516 -0.027542 0.027939 -0.077637 0.080261 0.032562 -0.006676 -0.088928 -0.017548 0.027084 0.059784 -0.097839 -0.035126 -0.058441 0.135986 0.075928 -0.032806 0.076660 0.075195 -0.051819 0.003353 -0.062561 -0.012054 0.051422 0.019608 -0.047882 -0.066223 0.070740 -0.069336 0.075317 -0.101196 -0.091125 -0.022324 -0.021759 0.014168 0.027542 -0.055725 0.008873 -0.012512 -0.092407 -0.046783 -0.030624 -0.084290 0.065918 0.004498 0.040009 0.049805 -0.000741 -0.034882 +-0.060638 -0.070374 0.009338 0.015732 0.041565 0.058899 0.074402 -0.082520 0.070129 -0.046600 -0.068970 0.046387 -0.047943 -0.062378 0.004658 -0.173706 0.018311 -0.037323 -0.013893 -0.057983 -0.084229 0.062561 -0.109009 -0.099609 0.020355 -0.056549 -0.021133 0.057037 0.040405 0.013985 0.146362 -0.112732 0.021194 0.043304 -0.137451 -0.106018 -0.025864 -0.005577 0.046692 0.102051 0.048157 0.050018 0.086060 0.010963 -0.104248 0.023514 0.063660 -0.015594 -0.115784 -0.026093 0.039185 -0.105042 -0.106873 -0.002371 0.028488 -0.052368 0.026352 0.018463 0.036804 -0.101868 0.062286 -0.004162 -0.159790 0.035095 -0.095093 -0.013168 0.006760 -0.026688 -0.073364 -0.011673 0.044769 -0.021240 0.050720 -0.009956 0.035980 -0.026733 -0.094788 0.063049 0.076843 -0.016647 -0.053009 -0.017960 -0.016739 0.009552 -0.086975 -0.105652 0.009041 0.029953 0.025589 -0.046722 0.017044 0.108521 0.020798 -0.135864 -0.061707 0.016876 0.057404 -0.000764 -0.060303 0.104919 0.041534 -0.115417 0.063782 0.020691 -0.037567 0.025757 -0.011017 -0.075500 -0.008369 -0.030365 -0.066101 0.009224 0.000971 -0.067200 0.028931 -0.042053 0.034882 0.153442 0.008957 -0.010452 0.079773 -0.068176 0.009827 -0.056458 0.041168 -0.088440 0.056549 0.030792 0.023453 -0.038116 -0.049072 0.012306 -0.039520 0.001280 -0.043762 0.006691 0.038025 0.031555 0.029022 0.054901 -0.067688 0.036163 0.030090 -0.085022 0.005413 0.055389 0.002741 -0.067505 -0.063721 -0.010300 -0.031616 -0.008217 0.038391 0.036835 0.016052 -0.031982 0.032227 0.050110 -0.104614 0.097595 -0.009132 -0.100647 -0.064941 0.028381 0.028778 0.019699 -0.059875 -0.132080 0.026428 -0.114929 0.012909 -0.065308 0.002298 -0.118591 -0.045013 0.061707 0.011208 0.007904 -0.000392 0.024292 0.024170 -0.085022 -0.074219 0.060089 -0.051636 0.068481 -0.106750 -0.051636 0.149170 0.027512 -0.056519 -0.006786 -0.003473 0.075562 -0.011917 0.004318 -0.002480 -0.124817 0.132812 -0.087830 -0.069519 -0.049377 0.013985 0.055908 0.119812 0.024948 -0.002237 -0.070862 -0.019821 0.074219 -0.092224 0.062042 0.087341 0.043396 -0.049591 -0.018341 0.003220 0.064697 -0.106079 0.019821 -0.060059 0.128052 0.013252 -0.052917 0.097046 0.100342 -0.050690 0.025925 -0.099487 -0.027206 0.027390 0.031982 -0.034576 -0.093201 0.050415 -0.065369 0.071228 -0.104370 -0.064148 -0.010559 0.001022 -0.033173 0.056854 -0.078613 0.011581 -0.021362 -0.086487 -0.011368 0.001410 -0.093018 0.042023 -0.049072 0.053802 0.035553 -0.020889 -0.060089 +0.023193 0.001149 0.062744 0.003769 0.007133 0.086792 0.035675 0.007191 -0.062805 -0.018066 0.013443 0.092102 0.027893 0.075867 0.041992 -0.098022 0.043182 -0.023315 0.018387 0.077332 -0.068359 -0.022888 -0.045898 0.033783 0.117493 -0.015732 -0.013466 0.049042 0.117188 0.052155 0.031174 -0.058228 -0.023392 0.082825 -0.031097 0.065308 0.006081 -0.002943 0.050476 0.115784 -0.038452 -0.019394 -0.014137 -0.068115 -0.109558 0.029800 0.122192 -0.014748 -0.017899 0.034119 0.056885 -0.188599 -0.111023 -0.129761 0.006203 -0.004696 0.020798 0.011093 0.089844 -0.024490 0.016266 0.022446 -0.070129 -0.019897 -0.047516 -0.013306 -0.024277 -0.096436 0.030365 0.071777 0.135254 -0.033997 0.052582 -0.006252 0.070557 -0.011330 -0.152954 0.038940 0.096375 0.003012 0.033630 0.007126 0.079102 0.022110 -0.059174 0.022232 0.023743 0.047363 -0.059601 -0.083801 -0.051361 0.062378 -0.007526 -0.119080 -0.030197 -0.034607 0.107300 -0.078186 -0.035645 0.050110 0.032471 -0.045105 -0.023514 0.001751 0.003010 0.039948 -0.120972 0.004021 -0.085571 -0.038391 -0.049713 -0.049652 -0.117065 -0.047455 -0.012482 -0.065002 -0.077026 0.059448 0.040222 -0.044250 -0.033234 -0.067139 -0.010063 -0.019150 0.166260 0.049805 0.120300 -0.016617 0.070190 -0.018021 0.037933 -0.045044 -0.138184 0.043823 0.041290 -0.028854 -0.020981 -0.098267 -0.044952 -0.020554 0.029175 0.016891 0.026077 -0.061768 -0.033112 -0.009926 0.029526 0.022690 0.039032 -0.049316 -0.053040 0.019531 -0.011505 -0.028000 -0.035004 -0.158936 -0.028763 0.148315 0.082947 0.013809 0.079773 -0.126343 -0.108093 0.063660 0.050934 0.078003 -0.018356 0.002357 -0.036224 -0.085083 0.035065 0.004337 0.080200 -0.057281 -0.061523 -0.026016 0.031204 0.017181 0.063599 -0.004654 0.074341 -0.000058 0.058197 0.040527 -0.043732 -0.013779 -0.078674 0.001524 0.098877 -0.000624 0.016434 -0.038330 -0.078918 0.055664 0.017487 -0.021576 -0.031494 -0.174194 -0.031586 0.061829 -0.121277 -0.007286 0.013138 0.072876 0.016220 0.107849 -0.132690 -0.058075 0.043945 -0.003456 -0.078918 0.027039 0.079285 0.018265 -0.050232 -0.067200 0.079468 0.074951 -0.004425 -0.002911 -0.091431 0.101624 -0.008171 0.066284 0.049347 0.151611 0.017059 0.068298 -0.083740 -0.076660 0.024567 -0.059509 -0.021866 -0.022949 0.018204 -0.048981 -0.011070 0.066711 -0.094543 0.069580 0.025803 0.054993 -0.010719 -0.074646 0.047302 -0.001543 -0.061493 0.043182 0.057068 -0.007050 0.024445 0.080017 0.041992 -0.056915 -0.034302 -0.030548 +-0.032227 -0.010994 0.012581 0.011612 -0.009598 0.050751 0.035614 -0.057861 0.034912 -0.053345 -0.051361 0.006264 -0.113647 0.020279 0.014130 -0.161377 0.024353 -0.004150 -0.024460 -0.087830 -0.060638 0.078674 -0.105225 -0.079956 -0.003056 -0.030502 0.036102 0.129028 0.093628 0.007591 0.121338 -0.059753 -0.035065 -0.029800 -0.127563 -0.043732 -0.006145 0.038177 0.010666 0.149780 0.017517 0.110535 0.057068 0.022430 -0.078125 0.038849 -0.008675 -0.063904 -0.134033 0.004578 0.078003 -0.080078 -0.161133 -0.085022 0.044830 -0.017853 0.007332 0.000362 0.126587 -0.086182 0.020416 0.006371 -0.076294 0.006870 -0.008270 0.038483 -0.045074 0.005508 0.006691 0.008003 0.051178 -0.032990 0.095520 -0.000670 0.020523 -0.068420 -0.126587 0.049713 0.079590 0.038757 -0.067444 -0.007717 -0.049042 0.064026 -0.151245 -0.145264 0.058167 0.038910 -0.028976 -0.058594 -0.031006 0.066162 0.045654 -0.051422 -0.070618 0.070435 0.016281 -0.066406 -0.070984 0.050598 0.000807 -0.112610 -0.023788 0.039490 -0.006840 0.008278 0.009384 -0.050781 -0.055939 -0.051544 -0.027802 0.013062 0.041687 -0.054810 0.014130 -0.079834 0.012413 0.139771 0.034729 0.004448 0.056213 -0.067810 -0.012077 -0.023544 0.011772 -0.117554 0.035522 -0.061493 0.081299 -0.038971 -0.021088 0.018051 -0.050354 0.001351 0.041473 0.011345 0.050873 0.001576 0.071411 0.015358 -0.059174 -0.005672 0.025665 -0.024551 0.022980 0.025894 0.012794 -0.016144 -0.001170 -0.012070 0.011818 0.014244 0.052368 0.012451 -0.047699 -0.109558 0.025299 0.097290 -0.074585 0.116821 0.006001 -0.113281 -0.065613 -0.017914 -0.015991 0.025360 -0.074402 -0.111694 -0.084229 -0.099609 0.046814 -0.063171 -0.020050 -0.104492 -0.084900 0.069885 0.051514 -0.054138 0.002159 0.018646 0.071777 -0.124573 -0.038147 0.038818 -0.063660 0.093323 -0.086487 -0.004780 0.134277 0.053711 -0.011627 -0.005779 -0.039551 0.012016 0.016525 0.008698 -0.033173 -0.041504 0.056885 -0.026550 -0.110901 -0.035095 0.102051 0.071777 0.135132 0.068054 -0.080383 -0.069397 -0.057648 0.021729 -0.058441 0.065247 0.083069 -0.020020 -0.092041 -0.079529 0.021622 0.067627 -0.112366 -0.030472 -0.025085 0.090698 0.052002 -0.016571 0.022232 0.147827 -0.060120 -0.026093 -0.093140 -0.018387 0.032562 0.060883 0.003983 -0.096313 -0.047974 -0.080200 0.071350 -0.059509 -0.084045 0.049683 0.015381 0.036072 0.035614 -0.019470 0.051239 -0.050354 -0.084839 0.016510 -0.062561 0.022461 0.040619 -0.031982 0.011147 -0.004005 -0.018356 -0.036560 +-0.075134 -0.008278 0.033417 0.027756 0.126221 0.011703 0.049927 -0.051819 -0.031891 -0.074463 -0.035400 0.113342 -0.008476 0.070801 0.052887 -0.204468 0.033569 -0.004620 0.081909 0.032959 -0.017288 0.056580 -0.080688 -0.038116 0.053314 -0.052673 -0.003845 0.121643 0.066467 -0.006992 0.008842 -0.034210 0.065796 -0.027252 -0.073792 -0.019501 -0.077209 0.014946 -0.014816 0.076416 0.056732 0.030182 -0.002415 -0.009209 -0.054169 0.030365 0.054932 0.005394 -0.126465 0.043213 0.056030 -0.157593 -0.148438 -0.024673 0.066833 0.003017 0.021362 0.039185 0.110046 -0.060211 0.056519 -0.022202 -0.060303 0.016266 -0.032410 0.067444 -0.019150 -0.028824 -0.003399 0.009171 0.124207 -0.006424 0.171265 0.048676 -0.000312 0.010818 -0.154175 0.014587 0.066650 0.011513 0.008347 0.032623 0.018051 -0.018341 -0.133545 -0.029755 0.032043 0.022720 0.040802 -0.107300 -0.045685 0.043182 -0.005123 -0.060608 -0.047516 -0.052826 0.104126 -0.031769 -0.084351 0.083069 0.056854 -0.065735 0.014809 -0.023209 0.023392 0.037354 -0.059723 -0.048492 -0.007477 -0.038330 -0.038177 0.003105 -0.032318 0.000036 0.016891 -0.029861 -0.017273 0.080322 0.082703 -0.063904 0.088806 -0.042664 0.020538 0.005650 0.033539 0.037903 0.073853 -0.023315 0.031143 -0.037018 -0.011139 -0.015427 -0.142822 0.030441 0.019379 -0.013016 -0.043701 -0.012146 0.026566 -0.008453 -0.051514 -0.019623 0.047729 -0.035370 0.004627 0.014771 0.008995 -0.040863 0.016357 0.070374 0.003471 -0.000860 -0.012177 -0.006802 -0.054901 -0.063110 -0.049164 0.151001 -0.012665 0.113586 0.032593 -0.098816 -0.053955 0.008766 0.022156 0.094910 -0.071045 -0.021194 -0.026138 -0.129761 0.002737 -0.048431 0.025223 -0.118591 -0.100891 -0.027222 0.065491 -0.041718 -0.019745 0.009544 0.070923 -0.045929 0.060394 0.020767 0.018127 0.149170 -0.059753 0.023682 0.073181 0.002338 0.005871 -0.023178 -0.073120 0.086853 -0.025513 -0.005859 0.060059 -0.098022 0.071228 0.006298 -0.145508 -0.045685 0.102905 0.095825 0.142944 0.041290 -0.049896 -0.063354 0.043335 0.100281 -0.073975 0.015869 0.137451 -0.003206 0.008713 -0.021194 -0.050201 0.078552 -0.054504 -0.023499 -0.036987 0.142456 0.048431 0.024353 0.084473 0.133057 -0.055878 -0.009468 -0.105286 -0.060547 0.036743 0.030609 -0.015823 -0.022720 0.047241 -0.148438 0.088928 0.022034 -0.042389 0.071045 0.052246 -0.020050 0.031586 -0.053009 0.050323 0.014351 -0.090088 -0.053711 -0.032898 0.034149 0.044159 -0.066833 0.048157 0.018021 -0.000889 0.033264 +-0.066040 -0.024734 0.018906 -0.061371 0.065063 0.067810 -0.030350 -0.032257 -0.038818 -0.007263 -0.017715 -0.006317 -0.166016 -0.008316 0.040253 -0.107666 -0.032166 0.048187 -0.007004 0.083679 -0.065430 0.032593 -0.008751 -0.006649 0.075317 -0.009590 0.076965 0.018311 0.076355 -0.016785 0.080139 -0.005001 -0.005802 0.062805 -0.157959 -0.063538 -0.160156 0.021500 0.049408 0.097290 0.016861 0.029221 0.024933 -0.081299 -0.078430 -0.015854 0.077942 -0.012482 -0.106079 -0.026138 -0.005653 -0.053589 -0.075562 -0.079468 -0.032898 0.063660 0.078125 -0.074402 0.044434 0.008598 0.020859 -0.059601 -0.081726 -0.012428 -0.023209 0.021729 0.036957 -0.056519 -0.032867 0.042480 0.089600 -0.055389 0.075317 -0.045166 0.095703 -0.065552 -0.040070 0.032440 0.097168 0.048828 -0.039215 -0.033447 -0.001348 0.056244 -0.008209 -0.025513 0.046936 0.009064 -0.028763 -0.100769 -0.041077 0.074036 0.035278 -0.087830 -0.113281 -0.036133 0.040985 -0.041565 -0.114319 0.091309 0.039001 -0.108215 -0.056854 0.029633 -0.130859 -0.076843 -0.071594 -0.077942 -0.072571 -0.048676 -0.056335 -0.008301 -0.056488 -0.079285 0.015594 -0.052155 -0.006031 0.070984 0.038239 -0.033813 0.084656 -0.054047 0.002674 -0.039185 0.074585 -0.013084 0.015396 0.039459 0.066467 0.047607 0.011658 0.022873 -0.112183 0.055450 0.063782 -0.003454 -0.111023 -0.119324 -0.007423 -0.028503 -0.062561 -0.018509 0.030075 -0.039154 -0.133423 0.063538 0.027222 -0.026001 -0.020828 0.025543 0.035736 -0.039825 -0.000655 0.031174 -0.051208 -0.095093 -0.029678 0.163818 0.072632 0.026154 0.062469 -0.100891 -0.028915 0.050140 0.032196 0.047028 -0.027985 -0.012878 -0.009163 -0.066406 0.095642 -0.008583 -0.013420 -0.060028 0.011673 0.029587 -0.022873 0.043457 0.081116 0.021332 0.072449 -0.050964 -0.029907 0.055817 -0.069214 0.041779 -0.002172 0.033783 0.106201 -0.061493 -0.056610 -0.024475 -0.076172 0.007664 0.039093 -0.055481 0.045258 -0.146606 0.086609 -0.069702 -0.134277 -0.005939 0.004292 0.097839 0.094666 0.076233 -0.111938 -0.073669 0.066345 0.111328 -0.039764 0.107361 0.098267 -0.031616 -0.098999 -0.019775 0.029465 0.026428 -0.075378 0.024750 -0.033752 0.020935 0.001862 -0.056488 0.034485 0.111267 -0.035706 0.023071 -0.105530 0.017334 0.028732 -0.056519 -0.027008 -0.035919 0.072083 -0.035126 0.031372 -0.081787 -0.081604 0.035583 -0.005039 -0.033264 0.116882 -0.052002 0.067322 0.072876 -0.099487 -0.052551 -0.045837 -0.038757 0.109070 -0.025238 0.085815 -0.060486 -0.017059 -0.037506 +-0.031982 -0.021790 0.031555 0.021912 0.060516 0.075806 0.046448 -0.064026 -0.039215 -0.049469 -0.021423 0.049469 -0.033966 0.037079 0.047699 -0.167358 0.011520 -0.014488 0.036743 0.019913 -0.056396 0.066040 -0.031769 -0.033295 0.048401 -0.013565 0.017029 0.106018 0.080444 -0.009750 0.079224 -0.065979 -0.012459 -0.016876 -0.127808 -0.075562 -0.094055 0.012337 -0.004158 0.100586 0.029266 0.046722 0.011780 -0.003387 -0.098206 0.034363 0.048431 -0.048767 -0.140503 0.036713 0.033447 -0.119507 -0.143066 -0.073242 0.019608 0.026260 0.018021 0.008751 0.084900 -0.026169 0.030304 -0.026779 -0.052338 0.041412 -0.017075 0.024506 -0.005310 -0.096802 0.006416 -0.027725 0.030884 -0.050323 0.172607 -0.030350 0.022537 -0.044830 -0.088013 0.071655 0.078186 0.020309 -0.008057 -0.014351 -0.001354 -0.014725 -0.133423 -0.037598 0.033051 -0.005108 0.023010 -0.116638 0.000167 0.079773 -0.002316 -0.123840 -0.068726 -0.042999 0.033783 -0.031494 -0.068604 0.059845 0.032318 -0.095459 -0.063049 -0.008667 -0.017761 0.044617 -0.054596 -0.033600 -0.037292 -0.039703 -0.061676 -0.002571 -0.003704 -0.048035 0.012253 -0.072876 -0.020325 0.099304 0.061096 -0.021072 0.078918 -0.048798 0.014511 -0.022507 0.055786 0.021500 0.116699 -0.025696 0.033691 -0.000589 0.004452 -0.000154 -0.112061 0.030563 -0.000870 0.005127 -0.029694 -0.097046 0.024933 -0.066895 -0.059387 -0.000053 0.017685 -0.082458 -0.050690 0.040497 0.030060 -0.065674 0.024109 0.060089 0.014839 0.002800 0.032532 -0.018768 -0.044434 -0.110352 -0.056061 0.112000 -0.025238 0.095154 0.042145 -0.106506 -0.048981 -0.029465 0.034454 0.062134 -0.090881 -0.037048 -0.065918 -0.096802 0.039948 -0.056274 0.046539 -0.097473 -0.092102 0.058136 0.037354 -0.047485 0.060455 0.013588 0.075562 -0.101257 0.056091 0.018341 -0.002752 0.091370 -0.063538 0.001582 0.097778 0.019562 0.025192 -0.056732 -0.088623 0.048248 0.028030 -0.020157 0.042755 -0.097717 0.059662 -0.008942 -0.182739 -0.021896 0.073730 0.125244 0.083618 0.046875 -0.056824 -0.097351 -0.030396 0.102783 -0.045959 0.019608 0.141113 -0.032471 -0.079346 -0.050262 -0.038116 0.111938 -0.088501 -0.009193 -0.048767 0.113464 0.047760 0.011429 0.060120 0.150879 -0.060730 -0.020203 -0.126953 -0.074097 0.009468 0.018555 -0.027481 -0.040344 0.032959 -0.105713 0.086487 0.000989 -0.094971 0.093506 0.040375 -0.032562 0.034760 -0.040314 0.093750 0.028824 -0.118042 -0.036316 -0.044067 -0.020691 0.048096 0.022095 0.074097 0.031006 -0.009438 -0.003340 +-0.010719 -0.047516 0.054260 -0.028244 0.075684 -0.020187 -0.030701 -0.037933 0.010094 -0.063660 -0.090820 -0.022995 -0.117493 -0.111267 -0.000849 -0.157471 -0.006535 -0.065674 0.036835 0.008980 -0.076416 0.007378 0.016113 0.004253 0.023972 -0.017670 0.074219 0.046722 0.011253 -0.012955 0.108643 -0.105774 -0.006603 0.033295 -0.138306 -0.087891 -0.069824 -0.020798 0.038300 0.071838 0.069458 0.076355 0.117737 -0.009850 -0.032928 0.075317 0.059296 -0.024017 -0.052521 -0.062469 -0.024734 0.052704 -0.158447 -0.004654 0.006565 -0.044983 0.078979 -0.015495 0.048676 -0.052246 0.024277 0.001264 -0.123230 0.003538 -0.064636 -0.023834 0.008736 -0.029434 -0.073120 -0.025101 0.073303 -0.020126 0.080688 -0.092651 0.089844 -0.069824 -0.072205 0.072266 0.013466 -0.014198 -0.039062 -0.043030 -0.006359 -0.016235 -0.037720 -0.090454 0.033691 0.007835 -0.021576 -0.007618 0.058136 0.101746 0.090149 -0.106812 -0.083618 -0.008278 -0.005859 -0.051666 -0.020248 0.143555 0.022537 -0.144775 -0.027939 -0.077271 -0.052826 0.000447 -0.023163 -0.118530 -0.063110 -0.044800 -0.027557 -0.030960 0.006832 0.020248 0.029144 -0.055267 0.021576 0.080505 -0.011185 -0.060150 0.177612 -0.031891 0.019363 -0.032471 0.056915 -0.018036 0.113770 0.036133 0.029388 -0.025772 -0.041138 0.003038 -0.025589 0.041260 -0.003250 0.014160 -0.013855 -0.087463 0.040161 -0.004322 -0.096680 0.065979 0.056793 -0.076355 -0.061096 0.106140 -0.045624 -0.087891 -0.029770 -0.028839 0.004711 -0.031708 0.011780 0.036682 -0.007244 -0.016739 -0.018936 0.106201 -0.062469 0.020096 -0.007771 -0.088623 -0.089783 0.081787 0.038940 -0.027237 -0.050171 -0.076721 -0.007412 -0.039124 0.047363 -0.069641 -0.058502 -0.093079 0.081055 0.034546 0.031311 0.049286 0.010574 0.026428 0.064087 -0.098755 -0.059418 0.021332 -0.051514 -0.029816 -0.098572 -0.021713 0.163940 0.016495 -0.026810 -0.044830 -0.010437 0.013260 -0.007965 -0.056641 0.071228 -0.107910 0.099182 -0.113770 -0.119995 -0.004932 0.077087 0.129150 0.110596 0.095825 -0.053162 -0.053253 -0.061310 0.020645 -0.024933 0.036133 0.062500 -0.018158 -0.099243 0.053955 -0.031708 0.063599 -0.094971 -0.064758 0.010223 0.062134 0.055420 -0.051086 0.042969 0.044373 -0.017044 0.024536 -0.036438 0.011887 0.089172 0.047363 -0.037537 -0.032532 0.110474 -0.010170 0.052917 -0.098694 -0.087402 0.001915 0.023849 0.006237 0.043945 -0.088135 0.029907 0.075989 -0.085022 -0.045990 -0.029343 -0.091431 0.067078 -0.025650 0.126587 0.032471 -0.028030 -0.056061 +-0.026825 -0.001142 0.095154 0.014915 0.076599 -0.012466 0.003685 0.010399 -0.107178 -0.000659 -0.077393 -0.002531 -0.100342 -0.023651 0.034363 -0.146362 -0.026962 -0.043976 0.073914 0.039612 -0.093262 0.040222 0.046600 0.035706 0.071716 -0.087769 0.016632 0.047699 0.067261 -0.026886 0.003588 -0.095642 -0.040344 -0.014626 -0.146484 -0.021500 -0.088440 -0.058685 -0.003698 0.095886 0.014420 0.003836 0.072571 -0.117065 -0.069153 0.027924 0.092773 -0.036438 -0.040619 -0.091248 0.021271 0.026306 -0.177368 -0.085266 0.033417 0.011795 0.036163 -0.006542 0.038300 -0.061523 -0.008820 0.023422 -0.082153 -0.030930 -0.044495 0.000344 -0.046631 -0.069153 -0.091858 0.043823 0.083069 -0.018173 0.020981 -0.083435 0.104614 -0.073608 -0.078796 0.035889 0.015839 -0.006546 -0.024384 -0.038147 0.055878 0.058502 0.014389 -0.058960 0.075867 0.035095 -0.074280 -0.071899 0.016006 0.075439 0.066528 -0.087097 -0.094788 -0.003847 -0.006168 -0.060486 -0.011993 0.150391 0.000656 -0.122070 -0.022919 -0.033752 -0.061127 -0.040100 -0.060242 -0.107910 -0.079529 -0.029907 -0.019394 -0.080322 -0.037598 0.010284 0.034882 -0.052185 0.035095 0.050262 0.033234 -0.094299 0.112000 -0.042816 -0.027237 -0.013023 0.090515 0.038025 0.094788 0.069641 0.085388 0.051483 -0.022247 -0.060974 -0.042023 0.017105 -0.007587 0.028717 -0.054810 -0.149292 -0.024857 -0.060913 -0.052460 0.028793 0.013969 -0.057251 -0.076355 0.114624 -0.010818 -0.000941 -0.009178 -0.066589 -0.014999 -0.044037 -0.009758 0.054108 -0.016388 -0.068604 -0.004456 0.141846 0.069702 -0.000619 0.060913 -0.072876 -0.080688 0.148804 0.013145 -0.039032 -0.028122 -0.011429 -0.035034 -0.011734 0.080017 -0.046448 -0.084106 -0.065979 0.071777 -0.037140 0.015007 0.017715 0.005112 -0.021027 0.070312 -0.083801 -0.008057 0.002850 -0.085449 0.012077 -0.050781 -0.019516 0.098083 -0.010155 -0.026947 -0.046509 -0.044708 0.006699 0.008621 -0.052338 0.067688 -0.125977 0.031006 -0.069336 -0.078247 0.009003 0.070312 0.147339 0.051117 0.117004 -0.103516 -0.035919 -0.085266 0.018402 0.000220 0.108948 0.068298 -0.053955 -0.105408 0.074768 -0.002300 0.033569 -0.048279 0.010574 -0.048676 0.049133 0.045502 -0.030334 0.085266 0.020142 -0.026855 0.014938 -0.016037 -0.034332 0.106628 -0.077209 -0.019073 0.008835 0.096008 -0.026810 0.028778 -0.045227 -0.097229 0.065796 0.022873 -0.010498 0.067627 -0.059692 0.061981 0.084229 -0.062500 -0.007957 -0.001264 -0.080383 0.068237 0.016998 0.151978 -0.034576 -0.055176 -0.057098 +-0.004070 -0.031494 0.024445 0.001722 0.036530 0.064453 0.096924 -0.097107 -0.022339 -0.053741 0.003220 0.037750 0.040558 -0.000585 0.019043 -0.163696 0.021500 -0.009216 0.011467 0.008682 -0.037659 0.088989 -0.055023 -0.046875 0.037292 -0.023849 -0.016541 0.125732 0.062500 0.011032 0.096436 -0.056091 -0.019897 0.012421 -0.068481 -0.054230 -0.048828 0.024414 -0.000864 0.084656 0.031952 0.040253 0.003229 0.034271 -0.093201 0.056824 0.049377 -0.051697 -0.125732 0.028778 0.028763 -0.143311 -0.136597 -0.082703 0.017212 -0.008499 0.006886 0.016556 0.063904 -0.050110 0.055542 0.007164 -0.054413 0.066284 -0.024826 -0.009468 -0.014763 -0.109070 0.029022 -0.060486 0.004425 -0.029022 0.169067 -0.033325 -0.016449 -0.034546 -0.082214 0.053223 0.061462 0.023346 -0.029327 -0.012932 0.018555 -0.052856 -0.143433 -0.034302 0.040497 -0.022659 0.007858 -0.097351 0.041412 0.070190 -0.012215 -0.117065 -0.087585 -0.030899 0.051941 -0.001621 -0.043457 0.042480 0.040070 -0.093994 -0.035736 -0.025787 0.010101 0.088867 -0.042297 0.008209 -0.031372 -0.002741 -0.066833 -0.012344 -0.017181 -0.053131 0.008247 -0.063110 -0.040649 0.106873 0.018707 -0.005123 0.069214 -0.049133 0.017792 -0.052887 0.060089 0.024902 0.144043 -0.046631 -0.025574 -0.047424 0.006733 0.010895 -0.094116 0.068848 0.001453 0.026962 0.036743 -0.049835 0.036133 -0.038269 -0.031830 0.031082 0.040070 -0.134399 -0.042725 -0.003952 0.044739 -0.087219 0.018066 0.054413 0.015381 0.012909 0.041962 -0.013947 -0.030106 -0.097412 -0.052917 0.067322 -0.056580 0.113220 0.019257 -0.089478 -0.070557 -0.023529 0.021576 0.064392 -0.106628 -0.059906 -0.055542 -0.110901 0.004768 -0.025757 0.038422 -0.100830 -0.109558 0.089905 0.051117 -0.062744 0.046021 0.032166 0.070129 -0.067566 0.032349 0.018539 -0.001149 0.072021 -0.076660 -0.013885 0.091492 0.047699 0.029663 -0.049500 -0.068970 0.077026 -0.012909 -0.001268 0.021774 -0.098267 0.051270 -0.000048 -0.182007 -0.008110 0.089172 0.106750 0.053101 0.044159 -0.004147 -0.101990 -0.050293 0.064209 -0.064819 -0.023270 0.145874 0.004982 -0.055511 -0.061829 -0.024124 0.135254 -0.078552 -0.048401 -0.056671 0.119629 0.094360 0.046509 0.083130 0.136597 -0.086853 -0.028305 -0.119568 -0.097595 0.003109 0.062988 -0.059814 -0.049652 0.011078 -0.087952 0.101624 0.044983 -0.084290 0.061066 0.014557 -0.024445 -0.026123 -0.071045 0.096741 0.012817 -0.087646 -0.031586 -0.035828 -0.036011 0.046814 0.038147 0.035736 0.077148 0.010597 0.013542 +-0.089478 -0.001444 0.014336 -0.060852 0.055878 0.036224 -0.029312 -0.086670 -0.066650 -0.050171 0.043976 0.012482 -0.034546 0.020737 0.031891 -0.158936 -0.008736 0.054901 0.033325 0.057861 -0.022369 0.103210 -0.058014 -0.055267 0.081543 -0.035492 -0.037445 0.112671 0.131836 0.001912 0.071106 -0.061615 0.026718 0.017014 -0.010139 -0.044556 -0.117493 -0.002420 0.043213 0.031250 0.003841 0.045776 -0.029953 -0.049530 -0.053009 -0.021027 -0.000838 0.001067 -0.141968 -0.031616 0.030014 -0.099487 -0.140747 -0.104553 0.006149 0.037354 0.021408 -0.022644 0.109924 0.001818 0.050934 -0.064819 -0.042389 -0.003883 0.023865 0.038361 0.001685 -0.041809 0.062622 -0.040436 0.134888 -0.009613 0.107849 -0.007965 0.013290 0.005394 -0.047424 0.039551 -0.022278 0.031708 -0.034515 -0.030197 0.014763 -0.070557 -0.054169 0.012512 0.076538 -0.079773 -0.027557 -0.059998 -0.006908 0.146240 0.032410 0.004662 -0.116333 -0.071167 0.074768 0.000502 0.011696 0.094727 0.023148 -0.162109 -0.016769 -0.055847 -0.107300 -0.023804 0.020004 0.023529 -0.051239 0.000063 -0.046082 0.002375 -0.063110 -0.027664 -0.005657 -0.068970 -0.072266 0.019440 0.035187 -0.076294 0.115723 -0.048798 0.126953 -0.054230 0.075500 -0.038422 0.051056 0.063660 -0.039032 -0.060699 0.050049 0.010971 -0.103271 0.071655 0.051727 0.057098 -0.040863 -0.072632 -0.018005 0.009201 -0.032532 0.026688 0.110413 -0.147461 -0.026855 0.078735 -0.033905 -0.103699 -0.003283 0.101135 0.047089 -0.076965 -0.004360 0.054291 -0.113770 -0.023148 -0.063721 0.117004 0.004364 0.048584 0.078613 -0.103027 -0.013367 0.008011 0.016708 0.058685 -0.048523 -0.056580 0.037842 -0.072998 0.019241 0.012833 -0.022568 -0.140137 -0.083923 0.003632 0.024277 -0.002558 -0.000967 0.001655 0.017136 -0.072144 -0.006947 0.005230 -0.021301 0.085632 0.009605 0.036255 0.058319 0.002375 -0.066162 0.022552 -0.060486 0.076538 0.013504 -0.004963 0.113831 -0.130859 0.094116 -0.092163 -0.165161 -0.070312 0.064392 0.050140 0.064026 0.060059 -0.081848 -0.036682 0.020309 0.078430 -0.050751 0.074524 0.074219 -0.015656 -0.057617 0.001320 -0.003748 0.034241 -0.056183 -0.067200 -0.056732 0.092590 0.085693 0.020050 0.108398 0.110352 -0.080750 -0.020859 -0.069092 -0.029388 0.044891 -0.014008 -0.088745 -0.055084 0.033081 -0.081726 0.059967 -0.034912 -0.068237 0.026260 -0.023468 -0.059692 0.076172 -0.075989 -0.020721 -0.010353 -0.044464 -0.068726 -0.066345 -0.008331 0.061829 0.027527 0.031464 0.080872 0.038513 0.028366 +-0.062408 -0.005272 0.035858 0.024475 0.013756 0.063599 0.076172 -0.012306 -0.046051 -0.010338 0.046051 0.047089 -0.020370 0.125977 0.032806 -0.044434 0.043762 0.031525 -0.032593 -0.036285 -0.016846 0.110229 -0.091736 -0.060394 -0.014778 -0.024841 0.057587 0.054138 0.046539 -0.015022 0.097229 -0.061920 -0.042023 0.043427 -0.091248 -0.060913 -0.045197 0.009727 -0.016266 0.100098 -0.019348 0.131348 0.002016 -0.001991 -0.119385 0.069336 0.021545 -0.038818 -0.149536 0.046021 0.066711 -0.136597 -0.059021 -0.120605 -0.007526 0.016983 -0.029251 0.012665 0.054901 -0.055206 0.084778 -0.060425 -0.007553 -0.017395 0.066528 0.074036 0.067505 -0.029099 0.032806 0.006680 -0.004990 -0.008743 0.113831 0.021347 -0.024155 -0.042786 -0.072388 0.059479 0.149658 0.075378 -0.047089 0.028458 -0.089294 0.038361 -0.113831 -0.077026 0.103210 -0.009605 0.037354 -0.161621 -0.035461 0.005459 -0.025620 -0.107483 -0.056305 -0.006897 0.070251 -0.029190 -0.008888 -0.013275 -0.000651 -0.013863 -0.058502 0.045013 -0.030060 0.001821 -0.009804 -0.007607 0.039581 -0.040863 -0.076416 0.003191 0.018143 -0.084351 0.050446 -0.079956 0.004322 0.112183 0.060577 0.040894 0.027771 -0.041229 0.088013 0.039764 0.024078 -0.018387 0.011353 -0.009911 0.040375 -0.019012 -0.021194 0.000870 -0.094238 0.032318 0.036957 0.085999 0.003948 0.016327 0.025085 -0.032959 -0.012283 -0.080994 0.007828 -0.084900 -0.073181 0.012978 0.102600 -0.067017 0.028152 0.097717 0.066284 0.010513 0.042389 -0.028870 -0.000340 -0.100525 -0.023438 0.055084 -0.036896 0.041748 0.013954 -0.100525 -0.028107 -0.092346 0.005375 0.023224 -0.072510 -0.029663 -0.089783 -0.081360 0.074585 -0.081604 0.076477 -0.070251 -0.090637 0.098511 -0.017960 -0.079651 0.053375 0.010002 0.062042 -0.080994 0.024063 0.038239 -0.007313 0.090820 -0.045624 0.004478 0.088989 -0.018555 -0.023315 0.010490 -0.039551 0.105225 0.082581 -0.026443 -0.044159 -0.026978 0.003798 -0.021225 -0.147827 -0.054535 0.053833 0.026993 0.085815 -0.001345 -0.092773 -0.122253 -0.054199 0.123108 0.026001 0.034058 0.122681 -0.040314 -0.052612 -0.094543 -0.016525 0.056152 -0.062164 0.049469 -0.018387 0.083130 -0.025772 -0.039520 0.033325 0.184326 -0.015305 -0.046356 -0.188232 -0.051666 -0.050781 0.003689 0.007957 -0.068359 -0.035461 -0.109070 0.043793 -0.083435 -0.001154 0.075562 0.038330 0.033325 0.044281 -0.020157 0.088684 -0.042419 -0.149170 0.006863 -0.094788 -0.051361 0.057861 -0.017487 -0.015320 -0.014015 -0.031647 0.013161 +-0.067444 0.000429 -0.001907 -0.031860 0.010582 0.110718 0.027420 -0.031677 -0.004593 -0.075073 0.053162 0.039581 -0.036407 0.003826 0.068542 -0.119995 0.066406 -0.017776 -0.005981 0.014107 -0.057861 0.055878 -0.105591 -0.071838 0.102051 -0.018311 -0.011719 0.038208 0.111877 0.064453 0.075012 -0.036102 -0.049561 -0.016373 -0.062317 -0.128296 -0.097046 0.009689 0.053955 0.136841 -0.011200 0.006046 -0.058594 0.006577 -0.176636 0.029816 0.088440 0.004833 -0.089478 0.027252 0.067810 -0.174316 -0.099182 -0.072021 0.034088 0.078491 0.057098 -0.032867 0.060120 -0.027267 0.001397 0.029449 -0.016098 -0.046661 0.012794 0.018646 0.033539 -0.012070 0.059143 0.026306 0.112244 -0.003887 0.098633 0.007183 -0.000101 -0.002554 -0.121704 -0.058258 0.121521 0.062317 -0.000849 0.032410 -0.032745 0.024734 -0.085327 -0.016525 0.040894 0.010452 -0.027374 -0.055359 -0.028488 0.061249 -0.015854 -0.036316 -0.052795 0.030609 0.120544 -0.057404 -0.020386 0.065247 0.026840 -0.078674 -0.027267 0.012054 -0.051361 -0.012642 -0.018112 0.019577 -0.026978 -0.053802 -0.051086 0.016815 -0.082214 -0.033112 -0.054321 -0.075195 -0.040649 0.087463 0.072021 -0.018967 0.014931 -0.067871 -0.011574 -0.034027 0.054565 -0.044281 0.018768 -0.021057 -0.016037 0.033691 0.082764 -0.055023 -0.084534 0.054352 0.060608 0.011398 -0.009903 0.033142 0.027710 -0.005997 -0.040955 0.008484 0.069580 -0.091736 0.003775 0.011002 0.058746 0.000948 -0.015762 0.041229 0.086487 0.035095 0.051819 -0.031143 -0.033569 -0.122192 -0.046997 0.083740 0.006359 0.104614 0.062164 -0.065369 -0.054413 0.076965 0.015236 0.130127 -0.050812 0.038391 -0.015060 -0.095520 0.045441 -0.046417 0.098206 -0.044281 -0.076965 0.053558 0.017288 -0.034393 0.055969 0.048096 0.023300 0.019592 0.030258 0.113892 -0.100952 0.097839 -0.018906 0.061798 0.152100 -0.074646 -0.019089 0.005634 -0.026855 0.037201 0.027618 -0.033356 0.026581 -0.112183 0.011452 0.051941 -0.090332 -0.072937 0.027786 -0.006287 0.106079 0.103210 -0.111511 -0.028900 0.075806 0.051483 -0.100769 0.031372 0.110229 -0.021194 -0.041199 -0.058167 -0.055511 0.064026 -0.011803 -0.000834 -0.081116 0.149048 0.031891 0.040680 0.069336 0.243896 -0.003000 0.066284 -0.158691 -0.042786 0.006226 -0.039154 0.005371 -0.067383 -0.057678 -0.075317 0.088806 -0.057983 -0.055573 0.016876 0.026413 -0.019943 0.022720 -0.023499 0.067810 -0.040100 -0.020248 -0.024200 -0.057922 -0.002216 0.069153 0.012398 0.050140 -0.017487 -0.019272 0.023651 +-0.043671 -0.044678 0.047729 0.014687 0.095642 0.017212 0.028351 -0.019348 -0.039185 -0.052765 -0.075378 0.017715 -0.024185 -0.091919 0.069336 -0.204102 0.000704 -0.074829 0.043365 0.039551 -0.105713 0.018829 0.015747 -0.014244 0.080627 -0.026230 0.034790 0.042114 0.023254 -0.015869 0.056213 -0.021530 -0.008255 -0.028488 -0.085815 -0.101746 -0.076721 -0.027466 0.044891 0.093445 0.104675 0.023865 0.058289 0.013519 -0.087219 0.059052 0.128906 -0.022736 -0.026108 -0.053192 -0.035187 -0.041473 -0.119812 -0.002947 0.015701 0.007988 0.078186 0.008095 0.038971 -0.070251 0.031006 0.055756 -0.129761 0.027756 -0.076294 -0.029861 -0.011528 -0.088745 -0.096680 0.012352 0.041748 -0.031952 0.092224 -0.080505 0.065857 -0.023895 -0.115112 0.034302 0.072998 -0.026215 -0.054199 -0.024521 0.048065 0.008133 -0.062439 -0.063599 -0.008324 0.011948 -0.007542 -0.055847 0.046173 0.091980 0.005680 -0.109070 -0.060333 -0.025208 0.054749 -0.066406 -0.062103 0.103699 0.022171 -0.121704 -0.021713 -0.056549 -0.030701 0.032104 -0.077759 -0.089905 -0.035126 -0.034241 -0.035645 -0.030441 -0.052338 -0.027451 0.001888 -0.043976 0.026718 0.109802 0.003271 -0.075317 0.140381 -0.053162 -0.034760 -0.043854 0.084839 0.012459 0.131470 -0.003733 0.041870 -0.021561 -0.003759 -0.057495 -0.038940 0.079041 -0.027267 -0.022873 -0.023438 -0.066772 0.039032 -0.034729 -0.098022 0.046783 0.043060 -0.113098 -0.026901 0.084534 -0.014526 -0.029083 -0.031036 -0.002348 -0.023178 0.008476 0.017380 0.016708 -0.003544 -0.066956 -0.030533 0.077820 -0.048370 0.094055 0.025864 -0.041443 -0.117859 0.122559 0.027054 0.082947 -0.064941 -0.033691 -0.034424 -0.093384 0.042725 -0.096802 -0.021591 -0.105835 0.012260 0.014900 0.031143 -0.033997 0.049744 -0.005253 0.045471 -0.062378 -0.039581 0.046661 -0.077271 0.062622 -0.088196 -0.016602 0.105225 -0.030136 -0.017166 -0.046814 -0.038025 0.008324 -0.067078 -0.037140 0.086853 -0.138916 0.072754 0.037872 -0.085999 -0.012009 0.063965 0.144287 0.088135 0.088318 -0.032928 -0.060974 -0.012642 0.022842 -0.082336 0.035309 0.075867 0.026794 -0.060577 0.027206 -0.066284 0.097473 -0.062561 -0.043274 -0.035187 0.140869 0.081055 0.011383 0.075684 0.089417 -0.033630 0.024872 -0.075073 -0.039276 0.076721 0.019577 -0.008881 0.019363 0.065247 -0.045197 0.129272 -0.045105 -0.112122 0.018097 0.053192 -0.012909 0.026047 -0.077332 0.122253 0.078674 -0.042786 -0.065430 -0.018356 -0.059845 0.061127 -0.054840 0.125732 0.017395 0.005493 -0.031677 +0.001164 -0.007889 0.024002 0.004211 0.020706 0.098999 -0.011993 -0.006130 -0.000281 -0.056091 -0.043091 0.040649 -0.030945 -0.003162 0.082581 -0.150024 0.054230 -0.017761 0.032074 0.098511 -0.099121 0.007664 -0.062164 -0.020187 0.060394 -0.003134 0.013832 0.114685 0.145020 0.065369 0.048096 -0.045288 -0.010880 -0.030518 -0.036377 -0.039032 -0.045166 0.025574 0.024597 0.096069 0.025284 0.031250 0.000423 -0.018127 -0.090637 0.010307 0.120972 -0.038330 -0.024521 0.048096 0.052521 -0.144531 -0.133179 -0.032440 0.011749 0.008255 0.044342 -0.001781 0.149414 -0.013947 0.002848 -0.012970 -0.125732 -0.031281 -0.064880 -0.005775 -0.054840 -0.008331 0.082397 0.031189 0.127075 -0.003881 0.091003 -0.008003 0.071472 0.010323 -0.173584 -0.008095 0.071777 0.002977 -0.019073 -0.005424 -0.004250 -0.026947 -0.108032 -0.054199 0.004913 0.057648 -0.093323 -0.037659 -0.016342 0.101318 0.001986 -0.038727 -0.034393 -0.040253 0.101196 -0.083557 -0.059174 0.056152 0.047485 -0.082031 -0.073486 0.003059 0.068542 0.001768 -0.092163 -0.032532 -0.073425 -0.052612 -0.032104 -0.021667 -0.045593 -0.018433 -0.046967 -0.081787 -0.104309 0.048126 0.052216 -0.061798 0.024963 -0.072754 -0.004253 -0.046051 0.124878 -0.017990 0.131592 -0.043213 0.054474 -0.018845 0.047485 -0.013100 -0.130493 0.006847 0.049530 -0.024048 0.025253 -0.003454 -0.002123 0.003275 -0.023209 -0.016083 0.064514 -0.038727 0.044037 -0.022079 -0.005821 0.064636 0.038727 -0.023987 -0.036896 0.035858 0.020767 -0.063782 -0.054962 -0.124207 -0.050812 0.158081 0.044403 0.067749 0.056458 -0.157715 -0.064026 0.095337 0.051270 0.131714 -0.036530 -0.032257 -0.041779 -0.098328 0.008469 -0.018417 0.060974 -0.081848 -0.074951 -0.014259 0.008675 0.005146 0.039886 -0.005363 0.022293 -0.043732 0.014374 0.078369 -0.062195 0.032776 -0.045471 -0.003633 0.111389 0.004295 0.012482 -0.046173 -0.054749 0.021057 0.031616 -0.017715 0.051239 -0.153564 0.032043 0.023361 -0.083801 -0.098083 0.034485 0.057953 0.067566 0.097534 -0.093140 -0.027161 0.048157 0.051392 -0.137329 0.022491 0.060638 0.063721 -0.080627 -0.048859 0.054321 0.046539 -0.043427 0.000618 -0.069824 0.132812 0.045959 0.013954 0.014832 0.185181 0.006157 0.103149 -0.086487 -0.013969 0.010307 -0.037598 -0.021408 -0.067261 0.033234 -0.055634 0.070679 0.045959 -0.106445 0.040070 0.073120 0.032410 0.006523 -0.087036 0.034088 0.017502 -0.062683 0.008438 0.003162 0.041382 0.009300 0.052734 0.055573 -0.020050 -0.002001 -0.015381 +-0.048859 -0.030777 0.000655 -0.051544 0.023270 0.056366 0.023071 -0.088745 -0.016068 -0.035614 -0.055237 0.032227 -0.106079 -0.016068 0.066101 -0.179688 0.008842 -0.006176 0.026794 0.039062 -0.065857 0.037323 -0.050323 -0.081116 0.075989 -0.040466 0.028564 0.074097 0.080505 -0.051025 0.095215 -0.038452 -0.006481 0.002453 -0.139648 -0.099426 -0.112244 0.039093 0.088196 0.119324 0.059906 0.002907 0.052063 -0.033295 -0.086975 0.023544 0.110901 -0.015442 -0.073059 -0.021393 0.044434 -0.084229 -0.117981 -0.047974 0.014778 0.009102 0.073608 -0.031464 0.040588 -0.006077 0.023468 -0.008865 -0.089966 0.050720 -0.086304 -0.030060 0.007065 -0.060455 -0.041718 0.006069 0.046783 -0.054993 0.142700 -0.036896 0.062866 -0.069275 -0.096619 0.036804 0.111389 -0.002861 -0.056702 0.002335 0.029221 0.034332 -0.071167 -0.048676 0.002871 0.024094 0.001997 -0.057465 -0.033630 0.070618 0.022430 -0.095764 -0.110352 -0.015778 0.011658 -0.025558 -0.109131 0.119812 0.043640 -0.127808 -0.008148 -0.002499 -0.041168 0.016281 -0.059509 -0.082703 -0.078064 -0.067932 -0.027939 0.008659 -0.018127 -0.030228 0.052185 -0.031708 0.044861 0.119690 0.023880 -0.013481 0.110901 -0.055542 -0.052338 -0.057281 0.027039 -0.018829 0.076660 -0.021698 0.033508 0.013329 0.016342 0.015656 -0.118286 0.077087 0.017609 -0.044006 -0.067017 -0.089417 0.064514 0.003893 -0.050415 0.049957 0.009140 -0.067505 -0.012100 0.044006 -0.018890 -0.012764 -0.021973 -0.019089 0.001949 -0.008644 0.050140 -0.009819 -0.022095 -0.109863 -0.022659 0.114563 -0.016052 0.109863 0.090576 -0.086609 -0.076294 0.062073 0.017380 0.062561 -0.100708 -0.035461 -0.027695 -0.116821 0.053833 -0.036957 -0.004070 -0.080505 -0.020203 0.021667 0.033691 -0.017731 0.073425 0.039185 0.070740 -0.058105 -0.005924 0.054871 -0.078003 0.090820 -0.049255 0.019379 0.127075 0.011093 -0.035156 -0.062622 -0.042572 0.024872 -0.039734 -0.036255 0.036499 -0.104065 0.085571 -0.024826 -0.130615 -0.039429 0.039368 0.133789 0.107788 0.092712 -0.038300 -0.079895 0.035095 0.053986 -0.055878 0.061432 0.130127 -0.035614 -0.097412 -0.052887 0.002048 0.100159 -0.067505 -0.033875 -0.053253 0.084534 0.033813 0.013580 0.047760 0.135132 -0.054413 0.012276 -0.113403 -0.005337 0.082886 0.002254 -0.012665 -0.043732 0.043274 -0.061127 0.073486 -0.051849 -0.109558 0.038086 0.008926 -0.029297 0.027161 -0.038361 0.105530 0.062561 -0.018860 -0.019562 -0.040131 -0.025650 0.078369 -0.056732 0.089233 -0.006195 -0.039612 -0.047333 +-0.001166 0.010895 0.002741 0.019150 -0.003345 0.100403 0.091431 -0.059937 -0.039917 -0.050903 0.020706 0.076111 0.012131 0.071472 0.063965 -0.100891 0.102600 0.026535 -0.043762 0.036560 -0.066528 0.091980 -0.107422 -0.109680 0.005711 -0.031158 -0.013802 0.112122 0.107910 0.000538 0.100098 -0.034058 0.021286 0.048981 -0.033478 -0.026550 -0.020523 0.047943 -0.000154 0.101013 0.016098 0.103455 0.040955 0.001747 -0.099487 0.018768 0.078674 -0.011574 -0.092285 0.077087 0.076599 -0.209961 -0.106323 -0.084412 0.028778 -0.011780 -0.038696 0.050232 0.112000 -0.062469 0.091125 -0.066895 -0.069031 0.024506 -0.016022 0.052826 0.035217 -0.050659 0.072021 0.006279 0.029785 -0.009750 0.152954 0.014076 0.008110 -0.006203 -0.139160 0.061951 0.137573 0.036865 -0.041595 0.043701 -0.034149 -0.011147 -0.122253 -0.057007 0.042389 0.015793 0.016220 -0.061523 -0.029068 0.079590 -0.034546 -0.043060 -0.083862 -0.017776 0.114746 -0.011864 -0.018906 -0.020691 0.059875 -0.036133 0.005272 0.006901 0.009941 0.081543 -0.032227 -0.028503 0.004406 -0.053314 -0.092163 -0.038422 -0.059418 -0.053711 0.043701 -0.061737 -0.038147 0.097229 0.039795 0.021774 0.030838 -0.091919 0.071533 -0.048126 0.092957 -0.035309 0.080322 -0.012589 -0.002171 -0.063416 -0.000635 0.038055 -0.113403 0.048706 0.058441 0.039246 0.057159 0.067322 0.015236 0.012215 -0.009430 -0.034698 -0.002417 -0.114685 0.007637 -0.003164 0.035309 0.000844 0.036194 0.045624 -0.007515 0.027100 0.063721 -0.058563 -0.048065 -0.126221 -0.032471 0.092163 -0.048218 0.060852 0.033630 -0.104614 -0.016403 -0.011086 0.029343 0.098083 -0.078979 -0.057007 -0.075378 -0.119080 0.013596 -0.033447 0.061615 -0.100525 -0.145386 0.026627 0.000723 -0.060730 0.030823 0.006855 0.025223 -0.052032 0.028183 0.068848 0.023117 0.082275 -0.093201 -0.002077 0.084534 0.020767 -0.035858 -0.008858 -0.043396 0.107910 0.017258 0.022842 -0.013603 -0.092651 0.008057 0.018173 -0.137207 -0.060059 0.070923 0.008690 0.044189 0.004776 -0.064636 -0.094604 0.017929 0.044159 -0.054199 -0.008286 0.076477 0.033630 -0.046692 -0.075317 0.042297 0.069946 -0.022324 0.007179 -0.044067 0.128174 0.048828 0.058136 0.041626 0.208618 -0.034912 0.009781 -0.130737 -0.046173 -0.033386 0.005070 -0.028259 -0.078979 0.000317 -0.091248 0.082214 -0.004185 -0.075623 0.032257 0.030838 0.046143 0.030914 -0.046143 0.080750 -0.037628 -0.089355 0.024200 -0.063660 -0.011574 0.024948 -0.015625 -0.011009 0.017914 0.008499 0.011322 +-0.062378 0.026535 0.004150 0.000620 0.027817 0.069214 0.012505 -0.007492 -0.012291 -0.070618 0.008972 0.037079 -0.111572 0.032379 0.050262 -0.138062 0.059021 0.056854 0.023621 0.005459 -0.100037 0.119141 -0.117737 -0.116638 0.031708 -0.024826 0.045868 0.096619 0.080261 -0.011246 0.055695 -0.026093 0.010155 -0.032104 -0.089722 -0.093018 -0.080078 0.034973 0.042511 0.139771 0.029816 0.087280 -0.007462 -0.012817 -0.073730 -0.019379 0.051361 -0.039764 -0.118774 0.013588 0.088074 -0.117310 -0.130737 -0.114075 0.032928 0.018188 0.025864 0.011612 0.133545 -0.075073 0.022064 0.003918 -0.054230 -0.001420 0.002138 0.055725 -0.032806 0.046234 0.021408 0.026535 0.080261 -0.021820 0.103943 0.021149 0.035248 -0.044922 -0.108032 0.015282 0.085999 0.054596 -0.048340 0.059814 -0.059479 0.035187 -0.088257 -0.097534 0.083496 0.045410 -0.069031 -0.045868 -0.079407 0.059235 0.035522 0.021225 -0.078613 0.057587 0.057373 -0.034454 -0.094604 0.044952 0.035126 -0.067688 -0.011627 0.048035 -0.025146 -0.027130 -0.017731 -0.038727 -0.053741 -0.077148 -0.040192 -0.010078 -0.042328 -0.037750 0.022629 -0.049591 0.004871 0.132812 0.077881 0.001681 0.040619 -0.098755 0.000537 -0.040436 -0.027054 -0.122192 0.013718 0.009972 0.057465 0.010681 0.018082 0.007648 -0.080872 0.002913 0.078003 0.036713 0.012413 0.046600 0.073120 0.001769 -0.047882 -0.018112 0.054962 -0.036255 0.018005 -0.004375 0.012695 0.038361 -0.036255 0.009399 0.061676 0.034393 0.026855 0.015625 -0.070923 -0.112427 -0.020432 0.103943 -0.006359 0.101135 0.043213 -0.108765 0.004704 0.045776 -0.000290 0.129150 -0.054504 -0.052246 -0.064331 -0.135010 0.061157 -0.078918 -0.046021 -0.066650 -0.094055 0.004139 0.005238 -0.087219 -0.038849 -0.001846 0.048615 -0.077271 -0.013664 0.091187 -0.101868 0.182617 -0.027679 0.039856 0.085022 -0.006748 -0.072205 -0.012222 -0.014297 0.028107 0.036560 -0.010910 0.018433 -0.035187 0.080933 -0.000783 -0.074890 -0.076904 0.119446 0.041260 0.133789 0.057617 -0.112183 -0.024323 0.062012 0.060333 -0.070984 0.065369 0.048767 -0.007660 -0.081055 -0.023926 0.027328 0.000690 -0.082214 0.040741 -0.031677 0.074829 0.050171 -0.006275 0.020599 0.188965 -0.038696 0.019943 -0.117798 -0.001084 0.040436 -0.056976 0.006424 -0.069763 -0.013321 -0.101807 0.070984 -0.064453 -0.068726 0.048981 0.013702 0.007206 0.071167 -0.001639 0.069275 -0.028214 -0.033844 -0.021164 -0.106934 0.008148 0.075684 -0.057220 0.024246 -0.030930 0.016113 -0.020020 +-0.065186 -0.059052 0.035492 0.022552 0.102356 -0.026459 0.013931 -0.061310 0.030151 -0.001068 -0.119507 0.046692 -0.110901 -0.092834 0.043762 -0.211060 -0.027374 -0.083801 0.076660 0.024338 -0.056030 0.001520 -0.006508 -0.078918 0.050659 -0.047760 0.012932 0.065979 0.032532 -0.038513 0.077576 -0.057831 0.052887 -0.020432 -0.136230 -0.069397 -0.081970 -0.014381 0.029068 0.044281 0.098206 -0.006912 0.103394 -0.040070 -0.062866 0.036499 0.096130 0.013519 -0.060608 -0.045105 0.003340 -0.014160 -0.128784 0.096863 0.008804 -0.052612 0.061951 0.004765 0.031525 -0.056061 0.034546 -0.000740 -0.164551 0.018723 -0.132568 -0.044678 -0.008385 -0.012566 -0.107117 -0.013695 0.086548 -0.027573 0.048706 0.007782 0.026611 0.009178 -0.111084 0.005573 0.034210 -0.059143 -0.041992 -0.042297 0.047882 0.007759 -0.031372 -0.076050 -0.026779 0.048706 0.018250 -0.018616 0.040070 0.087036 0.003002 -0.109802 -0.077942 -0.063110 0.018112 0.006435 -0.052032 0.166016 0.045563 -0.156372 0.022720 -0.034332 -0.020508 0.012085 -0.048706 -0.139648 -0.022644 -0.017700 0.012711 0.012047 -0.012558 -0.004494 0.037079 0.006783 0.091919 0.075134 -0.004856 -0.080078 0.151611 -0.008293 -0.052856 -0.051514 0.014084 -0.027069 0.056427 0.027802 0.028091 -0.033020 -0.039337 -0.030685 -0.053589 0.036621 -0.060425 -0.047546 -0.004341 -0.012825 0.031219 0.020218 -0.106384 0.043762 0.029831 -0.047302 -0.000170 0.066223 -0.040924 -0.012215 -0.075012 0.000321 -0.091736 -0.034302 -0.012657 0.027466 0.008247 0.007164 -0.019516 0.104675 -0.031174 0.093201 -0.005280 -0.064331 -0.071533 0.095947 0.024597 0.022003 -0.065613 -0.069031 0.055420 -0.089417 0.007980 -0.048981 -0.035492 -0.104370 0.007534 -0.014366 0.035767 0.032684 0.006218 0.015518 0.027771 -0.065125 -0.067383 0.035400 -0.042908 0.060638 -0.079285 -0.025818 0.109802 -0.015175 -0.028564 -0.058014 -0.029236 0.051819 -0.058624 -0.030212 0.084045 -0.104614 0.137085 -0.075073 -0.016144 -0.012451 -0.001327 0.119934 0.131714 0.049347 0.041382 -0.036224 -0.026703 0.062225 -0.082153 0.078857 0.097168 0.000224 -0.043427 0.035492 -0.042572 0.069519 -0.050354 0.009949 -0.060699 0.077393 0.042297 -0.053833 0.123962 0.014854 -0.076111 0.021210 -0.019897 -0.005096 0.094788 -0.014542 -0.023239 -0.056152 0.131348 -0.090393 0.101135 -0.059296 -0.016907 -0.033112 0.014816 -0.094849 0.020081 -0.096558 0.044769 0.101074 -0.058014 -0.022919 0.028015 -0.045319 0.067749 -0.086853 0.085510 0.032898 -0.072021 -0.042542 +-0.044037 -0.017609 -0.003967 0.025787 -0.033356 0.087036 0.052643 -0.085632 -0.028549 -0.049774 -0.004349 0.000102 -0.095947 0.099731 0.009697 -0.097656 -0.008499 0.013268 0.043060 -0.088501 -0.092468 0.114990 -0.066101 -0.149902 -0.021530 -0.033142 0.019958 0.072693 0.088013 -0.057831 0.123535 -0.063171 -0.035126 -0.038910 -0.149902 -0.097900 -0.044312 0.011398 0.022293 0.124634 0.005680 0.098145 -0.008568 0.006710 -0.091675 0.013802 0.002211 -0.047729 -0.178223 0.011108 0.105530 -0.115845 -0.076965 -0.039215 0.019409 -0.009087 -0.002985 0.021515 0.086975 -0.087769 0.043549 -0.063599 -0.037048 0.060913 0.035095 0.061737 0.022415 -0.018860 0.036469 0.003260 -0.022354 -0.040009 0.095032 0.008095 -0.016403 -0.059265 -0.072937 0.050293 0.094299 0.067078 -0.064453 0.036377 -0.049988 0.032196 -0.091492 -0.123779 0.090393 -0.016113 0.058899 -0.062866 0.001644 0.054565 -0.005520 -0.037048 -0.106750 0.038727 -0.013275 0.003876 -0.028824 0.037750 -0.020172 -0.113098 0.000759 0.035950 -0.101257 -0.006878 0.072205 -0.028534 -0.002104 -0.047913 -0.063538 0.027924 0.046906 -0.042145 0.063171 -0.054352 0.076721 0.181030 0.017090 0.055511 0.064148 -0.070129 0.067261 -0.022568 -0.059631 -0.135620 0.003609 0.021454 -0.010277 0.017029 -0.014076 0.065674 -0.027390 0.014503 0.042297 0.055817 -0.006687 -0.014267 0.088379 -0.025696 -0.027466 -0.038239 0.011536 -0.066895 -0.017960 0.059967 0.045441 -0.015358 -0.025345 0.054962 0.061310 -0.010460 0.058777 -0.009186 -0.004234 -0.105408 -0.042542 0.008270 -0.081055 0.079773 0.000951 -0.079224 0.033295 -0.055847 -0.012527 0.022034 -0.119568 -0.088867 -0.029083 -0.091064 0.062805 -0.116089 -0.018814 -0.093872 -0.087097 0.120483 0.012550 -0.102661 -0.026321 0.007473 0.013695 -0.090820 -0.012321 0.084045 -0.096436 0.116211 -0.056946 0.007290 0.090515 0.067871 -0.028885 0.010910 -0.041473 0.023331 0.051575 0.025269 0.018494 0.004665 0.054840 -0.019928 -0.050201 -0.037018 0.083496 0.016907 0.075989 0.006336 -0.025681 -0.065735 -0.056976 0.079712 -0.012032 0.068604 0.104248 -0.046906 -0.030563 -0.073914 0.032532 0.023239 -0.093567 0.017670 -0.044434 0.067505 0.009964 -0.003929 0.073547 0.176025 -0.038116 -0.055908 -0.112488 -0.052582 0.005180 0.021210 0.012222 -0.081421 -0.031799 -0.085693 0.084351 -0.102661 -0.059753 0.056213 0.004128 -0.018127 0.074280 0.004330 0.061523 -0.066772 -0.048248 0.034851 -0.098267 -0.047913 0.030518 -0.076355 0.033478 0.016205 -0.012535 -0.042542 +-0.037567 -0.018387 0.045563 0.016235 -0.037048 0.093018 0.060608 -0.068909 -0.042664 0.020874 0.045776 0.017776 -0.015244 0.056732 0.034210 -0.156616 0.039795 -0.019760 0.015640 -0.002621 -0.077515 0.069153 -0.070496 -0.030380 0.058380 -0.007351 -0.033661 0.065674 0.159668 0.049530 0.098389 -0.056366 -0.074890 -0.031525 -0.083679 -0.053619 -0.056519 0.016769 -0.014328 0.128662 -0.024460 0.031403 -0.019073 -0.055634 -0.140259 0.020050 0.045532 -0.047974 -0.107117 0.004341 0.083923 -0.153198 -0.125610 -0.126953 0.061981 0.016525 0.036987 -0.018875 0.075562 -0.039398 -0.020935 0.009315 -0.060394 0.019669 -0.015366 -0.006897 -0.047546 -0.060730 0.020325 0.006557 0.047150 -0.028259 0.106018 -0.044861 0.034851 -0.060547 -0.087341 0.015587 0.130981 0.002357 -0.020447 -0.000444 -0.003511 0.002573 -0.084412 -0.026291 0.028458 0.050415 -0.100952 -0.088806 -0.023849 0.119751 -0.015503 -0.079163 -0.064453 0.017502 0.065735 -0.074219 -0.054108 0.068176 0.011147 -0.097839 -0.058289 0.070129 -0.009781 0.014503 -0.021255 -0.013092 -0.103210 -0.028290 -0.070007 -0.007164 -0.042236 -0.064026 -0.003733 -0.083435 -0.026459 0.097900 0.038330 0.005661 -0.005699 -0.060455 -0.058838 -0.057495 0.087280 -0.053955 0.103394 -0.010056 0.047211 0.018570 0.013260 -0.045868 -0.099976 0.031738 0.023575 0.012123 0.014061 -0.086975 0.014816 -0.042694 -0.003559 0.003906 0.023346 -0.077332 0.007069 0.016159 0.024155 0.043335 0.024246 -0.017670 0.025314 -0.011322 0.072083 -0.024185 -0.048981 -0.135498 -0.007710 0.093201 0.025772 0.093079 0.071472 -0.101929 -0.062256 0.030899 0.011818 0.055664 -0.091187 -0.024475 -0.044037 -0.084900 0.067444 -0.039703 0.063232 -0.059265 -0.107788 0.006542 0.024261 -0.054504 0.060333 0.041931 0.052368 -0.085388 0.049530 0.052307 -0.061859 0.081726 -0.040619 0.014763 0.124817 0.016571 -0.011169 -0.071350 -0.027237 0.017059 0.054901 -0.015251 -0.004173 -0.157227 0.023834 0.039703 -0.095276 -0.024475 0.012993 0.037994 0.057861 0.085876 -0.116516 -0.084656 -0.021957 0.026901 -0.083740 0.073181 0.130859 0.001221 -0.087341 -0.093811 0.004791 0.072876 -0.061707 0.035431 -0.099731 0.119263 0.035736 0.052429 0.081604 0.201538 -0.024857 0.030411 -0.129395 -0.069580 0.040405 -0.024902 -0.020096 -0.066467 0.014465 -0.062927 0.066772 0.004543 -0.120117 0.066223 0.015350 -0.044128 0.014572 -0.020935 0.091919 -0.053741 -0.022049 0.040131 -0.018784 -0.022980 0.016830 0.079773 0.044586 -0.006500 0.026611 -0.045532 +-0.033173 -0.038910 0.041443 -0.022934 0.004532 0.071899 0.005272 -0.051056 -0.012466 -0.067139 -0.024704 0.005486 -0.195190 0.102722 0.019897 -0.125977 0.006180 0.018005 0.051300 -0.024002 -0.047394 0.067322 -0.076599 -0.101257 0.035339 0.003508 0.048218 0.090454 0.090149 -0.041443 0.104187 -0.070801 -0.029037 0.032104 -0.180176 -0.090149 -0.042603 0.045044 0.031616 0.116028 0.003031 0.087769 0.009178 0.000744 -0.044556 0.004803 -0.005760 -0.071533 -0.172119 0.024673 0.086426 -0.086914 -0.140625 -0.068665 -0.019577 -0.025284 0.015007 0.000917 0.123291 -0.039398 -0.002161 -0.053192 -0.048096 0.044495 0.006908 0.050446 -0.025299 -0.014351 0.031097 0.017014 0.060425 -0.069214 0.126953 0.000859 0.074524 -0.068848 -0.081665 0.068665 0.082214 0.043976 -0.057007 0.026306 -0.024521 0.033630 -0.076904 -0.096252 0.086426 0.052277 -0.001371 -0.072083 -0.042572 0.111206 0.046692 -0.052460 -0.107971 0.022949 -0.008743 0.002626 -0.070923 0.027664 0.023651 -0.143188 -0.039215 0.033325 -0.075684 -0.001597 0.009712 -0.052002 -0.041962 -0.056702 -0.040436 -0.020081 0.016174 -0.018036 0.037323 -0.089050 0.030426 0.112122 0.038788 0.025406 0.064270 -0.031082 0.025269 -0.071472 -0.037781 -0.114807 0.037201 -0.025497 0.049866 0.005981 -0.001042 0.065979 -0.081238 0.030548 0.052246 0.015076 0.031235 -0.040344 0.089111 0.007820 -0.050537 -0.042847 0.001695 -0.039764 -0.010811 0.036743 0.028122 -0.062073 -0.028427 0.041046 0.076111 0.022171 0.019226 0.012497 -0.061218 -0.121277 -0.037750 0.094360 -0.027588 0.067871 0.002193 -0.139038 0.002125 -0.039154 0.028595 0.011581 -0.072571 -0.113281 -0.034851 -0.106384 0.058777 -0.034302 -0.029053 -0.052734 -0.048401 0.092957 0.045807 -0.081787 -0.023315 0.042725 0.062927 -0.126221 -0.010536 0.065491 -0.058807 0.123840 -0.082642 0.020416 0.119202 0.055511 -0.014702 -0.021194 -0.082153 0.043152 0.021591 -0.011375 0.052460 -0.045013 0.038971 -0.057068 -0.113403 -0.043243 0.106201 0.067322 0.081604 0.053070 -0.058807 -0.040985 -0.026306 0.051208 -0.052368 0.066162 0.078247 -0.096619 -0.070007 -0.081665 0.061890 0.044830 -0.117188 -0.007866 -0.039856 0.046722 0.047882 -0.027985 0.026581 0.151733 -0.021057 -0.027252 -0.072510 -0.032440 0.037964 -0.006611 0.034210 -0.071960 0.014946 -0.098389 0.047516 -0.049805 -0.054871 0.052795 -0.015884 0.019424 0.026382 -0.013390 0.022095 -0.020401 -0.101990 0.009636 -0.101013 -0.024231 0.075989 0.009132 0.046875 -0.036804 -0.023529 -0.039276 +0.024765 0.013641 0.069031 0.026260 -0.038086 0.073608 -0.000193 -0.009422 -0.077454 -0.049744 -0.008705 0.017868 -0.154175 0.078308 0.089905 -0.135742 0.047150 -0.103699 0.045074 0.019226 -0.102905 -0.021515 0.020218 -0.096619 0.076660 -0.015213 0.022263 0.052429 0.112915 -0.020950 0.055115 -0.073486 -0.078125 -0.016495 -0.149780 -0.083740 -0.030472 0.006916 0.005970 0.165039 0.010254 0.023239 0.002136 -0.053558 -0.147705 0.051239 0.094055 -0.007942 -0.045044 0.041107 0.102966 -0.073364 -0.137329 -0.000795 -0.006554 -0.015137 -0.002796 0.011002 0.083374 -0.053436 0.004238 -0.010704 -0.024475 -0.047150 -0.031403 0.036774 0.058990 -0.023834 0.015099 0.079529 0.103638 -0.059540 0.036469 0.017517 0.047913 0.005543 -0.182861 -0.025681 0.149658 0.063843 -0.025360 -0.012749 0.016220 0.084717 -0.010193 -0.025177 0.032562 0.063660 0.009819 0.002153 -0.002142 0.082581 -0.020584 -0.060394 -0.102295 -0.011353 0.058533 -0.057617 0.069580 0.058197 0.014580 -0.127441 -0.025650 -0.036469 -0.099670 0.019165 0.001848 -0.131836 -0.029449 -0.074890 -0.029678 -0.042450 -0.098938 0.012848 -0.015549 -0.085205 0.028641 0.060394 0.015060 -0.061005 0.064453 -0.042877 0.001800 -0.065308 0.091064 -0.068604 0.037628 0.019455 -0.026428 0.023270 0.037750 -0.062225 -0.056091 0.060364 0.046600 0.040619 0.054932 0.007812 0.012886 -0.024582 -0.082092 -0.027740 -0.003609 -0.107178 -0.027847 0.099243 0.040680 0.071655 -0.024841 -0.011650 0.012375 0.046326 0.034576 -0.058899 -0.033752 -0.143433 -0.092102 0.109436 0.038300 0.027969 0.021393 -0.040619 -0.021393 0.105591 0.021118 0.057983 0.006222 0.020523 -0.033386 -0.012535 0.061218 -0.071289 0.040558 -0.048431 -0.007534 0.032043 0.015366 -0.028137 0.058105 0.025421 0.008781 -0.041077 -0.005630 0.126343 -0.096497 0.024216 -0.103577 0.039429 0.113159 -0.045288 -0.010056 0.005428 -0.071045 0.032501 0.072632 -0.015289 0.061554 -0.079407 -0.046234 0.044556 0.000374 0.001034 0.040039 0.039856 0.035034 0.098267 -0.109863 -0.017517 -0.030090 -0.008537 -0.025314 0.063232 0.093079 -0.079041 -0.043854 0.003139 -0.010094 0.044525 0.027939 -0.014282 -0.083923 0.106995 0.022217 0.037750 0.098999 0.225342 0.040222 0.031586 -0.046478 -0.064697 0.061401 -0.069336 0.052277 -0.043182 0.003111 -0.073486 0.101379 -0.060059 -0.021957 -0.002312 0.020065 -0.000199 0.044617 -0.056213 0.083435 0.020538 -0.036072 0.078979 -0.077148 -0.040863 0.077271 0.004623 0.075562 -0.042328 -0.085083 -0.030380 +0.001683 0.035309 0.014091 -0.016830 -0.051758 0.094299 0.049835 -0.086304 -0.088623 0.001838 0.031525 0.029373 -0.062378 0.119019 0.043274 -0.061554 0.027847 0.060913 0.009857 0.078369 -0.019348 0.070618 0.007751 -0.043427 -0.002445 0.024277 0.044983 0.168457 0.110962 -0.010437 0.076416 -0.007538 -0.044922 0.021790 -0.036194 0.003620 -0.072937 0.066162 -0.024490 0.071838 0.005402 0.064758 -0.061737 -0.057129 -0.054352 0.030823 0.030914 -0.038940 -0.106201 0.066956 0.067017 -0.128784 -0.073608 -0.108704 -0.027023 0.008163 -0.033203 -0.037262 0.099670 0.011559 0.071228 -0.101196 0.004795 0.039215 0.036346 0.027222 0.035645 -0.069580 0.132568 -0.030014 -0.012436 -0.035858 0.142700 0.025375 -0.016479 -0.040833 -0.080994 0.041870 0.080566 0.085266 -0.034790 -0.029236 -0.017975 -0.009163 -0.095093 -0.019943 0.075378 -0.046112 0.011345 -0.099609 0.024078 0.006474 -0.038025 -0.044189 -0.122864 -0.091187 0.049530 0.007278 -0.022949 0.006344 -0.008057 -0.038116 -0.102478 0.002180 -0.037537 0.007866 -0.029495 0.014854 -0.009705 -0.027756 -0.062317 0.004036 0.010361 -0.051056 0.080383 -0.055206 -0.060638 0.023163 0.025818 0.041382 0.056427 -0.074768 0.106201 -0.006535 0.064758 0.020340 0.057251 -0.039001 -0.022095 -0.048828 0.029251 0.071472 -0.145630 0.087036 0.087891 0.057617 -0.030777 -0.094482 -0.009369 -0.062103 0.020294 -0.082214 0.043518 -0.097473 -0.074524 -0.053589 0.062317 -0.023987 0.072327 0.097534 0.015335 -0.035492 0.032898 -0.085999 -0.084656 -0.129761 -0.114441 0.122986 0.009232 0.035553 0.067200 -0.107117 0.008217 -0.090027 -0.013176 0.027756 -0.096802 -0.003054 -0.094360 -0.022766 0.024231 -0.013062 0.070557 -0.068787 -0.111755 0.074219 -0.007309 -0.019913 0.071716 -0.003485 0.063599 -0.092041 0.043243 0.025131 0.024200 0.007641 0.015129 0.025406 0.037109 0.041077 0.004162 -0.000073 -0.096191 0.074219 0.094727 0.034485 -0.020905 -0.022400 0.011505 -0.029602 -0.164062 -0.012695 0.044708 0.082581 0.000236 0.022232 -0.060547 -0.148560 -0.067383 0.115906 0.025131 0.015236 0.132812 -0.032196 -0.074890 -0.108704 0.047485 0.080383 -0.023422 -0.043091 -0.026459 0.004879 0.038452 0.028244 0.049408 0.155884 -0.080688 -0.058075 -0.091797 -0.049011 -0.045288 0.043091 -0.057129 -0.071289 -0.020828 -0.111877 0.045868 0.049164 -0.078064 0.103577 0.021988 0.008766 0.025848 -0.068115 0.090698 0.010666 -0.097534 0.020340 -0.064087 0.013916 0.014030 0.028152 -0.022598 0.031250 -0.008415 0.029190 +-0.030441 0.005058 0.003736 -0.004818 0.008118 0.096985 0.018341 -0.067200 0.010612 -0.053619 -0.017181 0.037933 -0.060028 0.036072 0.034363 -0.182373 0.043335 -0.019318 0.020920 0.009766 -0.067200 0.060699 -0.105591 -0.051727 0.033691 -0.022385 -0.025391 0.118958 0.124817 0.047699 0.091614 -0.082458 0.000361 -0.031342 -0.111145 -0.042297 -0.029236 0.038452 0.018753 0.126953 -0.002131 0.050598 0.008438 -0.015442 -0.060181 -0.014206 0.049469 -0.029404 -0.091919 0.042816 0.087585 -0.138794 -0.149658 -0.079407 0.062164 0.010269 0.006035 -0.005280 0.105347 -0.048035 0.001196 -0.037537 -0.078674 0.002470 -0.032532 0.033569 -0.037170 -0.007664 0.048492 0.004082 0.120117 -0.017624 0.123047 0.007038 0.052429 -0.041718 -0.149292 0.044525 0.076233 0.003391 -0.020981 0.015266 -0.016769 -0.036926 -0.110229 -0.059296 0.028687 0.039154 -0.043030 -0.059326 -0.048889 0.125366 0.021957 -0.019257 -0.051514 0.013672 0.085815 -0.073242 -0.067078 0.099426 0.035126 -0.103943 -0.011566 -0.005695 -0.020981 0.006992 -0.009209 -0.048584 -0.083679 -0.063538 -0.068848 0.015480 -0.006779 -0.018188 0.011452 -0.073669 -0.050995 0.074524 0.034821 -0.024933 0.024261 -0.074158 0.027603 -0.055420 0.067444 -0.088257 0.079346 -0.049316 0.046387 -0.036987 0.010139 -0.016724 -0.089539 0.004669 0.039642 0.001071 0.017685 -0.034424 0.014992 0.016037 -0.025009 -0.022125 0.036896 -0.012566 0.053253 0.020706 -0.007751 -0.012451 0.009453 0.021393 -0.002148 0.009537 0.059296 -0.013901 -0.087646 -0.083496 -0.011116 0.129272 -0.045410 0.111755 0.054626 -0.125732 -0.035095 0.016220 0.018311 0.075378 -0.085083 -0.075806 -0.038483 -0.097900 0.020401 -0.060760 0.042999 -0.119507 -0.114746 0.004990 0.052826 0.013725 -0.009201 0.002411 0.029800 -0.120300 0.001229 0.054596 -0.044495 0.089600 -0.075684 0.011383 0.147461 0.048065 -0.028366 -0.021164 -0.029358 0.043365 0.049347 0.010307 0.013489 -0.153198 0.065063 -0.019669 -0.122498 -0.100525 0.063416 0.086060 0.126953 0.063354 -0.119690 -0.050720 0.029678 0.063843 -0.088562 0.053070 0.082581 0.054230 -0.053223 -0.046875 -0.015472 0.056396 -0.095215 -0.023987 -0.035675 0.153564 0.037811 0.015076 0.050781 0.203491 -0.036346 0.012558 -0.077026 -0.020294 0.016495 0.039093 -0.032471 -0.068298 0.026428 -0.116089 0.037262 -0.019363 -0.107605 0.041992 0.031982 0.006493 0.079407 -0.032715 0.008957 -0.064087 -0.070190 -0.005875 -0.046356 -0.011818 0.012238 -0.006649 0.042694 0.026016 0.029388 -0.025711 +-0.092346 -0.061859 0.064636 0.016190 -0.011208 0.003059 0.068542 -0.064453 0.013496 -0.043915 -0.048431 0.031158 -0.097961 0.120422 -0.042786 -0.114990 0.011200 -0.024490 0.009338 -0.086914 0.011444 0.110657 -0.107544 -0.040253 -0.025772 -0.068359 0.043457 0.115784 0.059052 0.003843 0.080750 -0.087585 -0.010796 0.095154 -0.152222 -0.011078 -0.019455 0.025848 0.030807 0.066284 -0.049011 0.060211 -0.004284 0.003906 -0.040344 0.035065 -0.052521 -0.053741 -0.217285 0.014046 0.092346 -0.060516 -0.149414 -0.086731 0.000065 -0.074951 -0.017395 -0.015144 0.084839 -0.031204 0.026169 -0.070251 -0.008286 0.018448 0.000338 0.052368 0.001972 -0.043304 0.000965 0.011864 -0.009834 -0.029297 0.119995 0.020721 0.074890 -0.076172 -0.049713 0.046906 0.098816 0.039948 -0.001049 0.071594 0.026825 0.052429 -0.119629 -0.061310 0.115723 -0.007202 0.045471 -0.112915 -0.034149 0.043427 0.029160 -0.096313 -0.090149 0.052307 0.022446 0.001012 -0.052155 0.026550 -0.001058 -0.058899 0.039612 0.019379 -0.092407 -0.004250 0.029648 -0.046906 0.025665 -0.051392 0.001493 -0.017776 0.023880 -0.007988 0.079834 -0.025040 0.068481 0.139160 0.024353 0.046783 0.066895 -0.016922 0.029785 0.022675 -0.041840 -0.034424 0.047394 -0.042328 0.024002 -0.029373 -0.047760 0.074402 -0.123169 0.013611 0.028534 0.021912 0.004181 -0.047729 0.066284 -0.021637 -0.021362 -0.022675 0.002602 0.002033 -0.020706 0.043579 0.047028 -0.091187 -0.038727 0.037994 0.066895 0.009323 0.011887 0.023148 -0.031464 -0.102661 -0.012566 0.066406 -0.057648 0.030640 -0.023041 -0.114258 -0.009094 -0.053162 0.001650 -0.064575 -0.081726 -0.088013 -0.054718 -0.113953 0.014565 -0.014565 -0.060303 -0.105469 -0.030869 0.078125 0.043793 -0.095825 -0.063477 0.011551 0.092163 -0.089539 -0.016647 0.005489 -0.018982 0.088928 -0.089966 -0.001853 0.119141 0.055603 -0.035339 0.009117 -0.070374 0.057343 0.004524 -0.003967 -0.017303 -0.021606 -0.013084 -0.061737 -0.143188 -0.038666 0.128418 0.087830 0.115479 0.020142 -0.037079 -0.045258 -0.066223 0.033051 0.007793 0.020187 0.104736 -0.111084 -0.053650 -0.087402 0.024872 0.106262 -0.098022 -0.046143 -0.006088 0.012596 0.028244 -0.070374 0.074646 0.058868 -0.057922 -0.090454 -0.065430 -0.051697 0.032928 0.086548 0.001606 -0.085693 0.001943 -0.119934 -0.020660 -0.091431 -0.033997 0.102417 -0.011475 0.062500 0.046112 -0.039032 -0.009071 -0.018570 -0.128418 0.020874 -0.115234 -0.080933 0.062439 -0.049377 0.030472 -0.003914 -0.037140 -0.040588 +-0.076355 -0.016800 0.036438 0.028854 0.002865 0.054474 0.033936 -0.019562 -0.085815 -0.018036 0.072449 0.045715 -0.023636 0.152710 0.098694 -0.097290 0.036041 0.045135 0.005322 0.030441 0.025406 0.047180 -0.053864 -0.081421 -0.018188 -0.057922 0.024124 0.051941 0.100037 -0.002913 0.013496 -0.038910 -0.018417 0.018570 -0.050903 -0.023590 -0.018127 0.083008 -0.011833 0.071411 -0.028305 0.101807 0.005505 -0.056732 -0.102722 0.066467 0.056366 -0.026047 -0.124878 0.082947 0.089111 -0.154663 -0.112671 -0.086975 0.020599 0.007843 -0.033539 0.036743 0.097107 -0.039276 0.031586 -0.121643 -0.005344 -0.028152 0.036682 0.052155 0.078613 0.031799 0.070618 -0.007496 0.048370 -0.034149 0.193604 0.023758 -0.008118 -0.028992 -0.109985 -0.006493 0.164917 0.023331 -0.000519 0.056458 -0.052124 -0.003832 -0.124695 0.007481 0.064148 -0.017944 0.028625 -0.100952 -0.030792 0.009163 -0.050995 0.000345 -0.084778 -0.040527 0.061920 -0.038727 0.014633 -0.040192 0.049988 -0.006680 -0.030746 0.038177 -0.015480 0.072632 -0.030975 -0.027222 0.022049 -0.092163 -0.080200 -0.021240 -0.023605 0.016693 0.024307 -0.054413 0.017853 0.058228 0.075562 0.017059 0.073059 0.017578 0.039978 0.049713 0.045837 0.069641 0.042542 0.028198 -0.016846 -0.026123 0.018097 0.029572 -0.223877 0.010040 0.061279 0.043579 -0.025101 0.038605 0.033234 -0.029694 0.044525 -0.067383 0.005478 -0.037628 0.017868 0.032867 0.052155 0.035675 0.063965 0.071533 0.069519 -0.027802 0.058228 -0.122925 -0.040131 -0.121155 -0.086914 0.102844 0.056824 -0.023880 0.039032 -0.076782 0.036743 -0.033325 0.046204 0.080444 -0.067810 0.006123 -0.064453 -0.075562 0.088623 -0.045807 0.119629 -0.091064 -0.106445 0.028641 -0.013626 -0.088074 0.022858 0.001543 0.082764 -0.011200 0.044556 0.067261 0.019608 0.088562 -0.056946 0.054138 0.039703 0.007446 -0.032074 0.008430 -0.046478 0.042725 0.051880 -0.064697 0.018555 -0.060486 -0.072815 0.002707 -0.102783 -0.047546 0.096069 0.016891 0.034485 -0.026123 -0.061096 -0.137939 0.022781 0.075989 -0.053833 -0.043335 0.160645 -0.014679 -0.016495 -0.136353 -0.007889 0.056122 -0.040436 0.007717 -0.036865 0.084290 -0.015213 0.044525 0.021729 0.218018 0.036041 0.001776 -0.151123 -0.088745 -0.012070 -0.044098 0.052856 -0.006870 -0.026077 -0.043610 -0.022552 0.003536 0.000071 0.074646 0.037537 0.012733 0.017776 -0.066772 0.041382 -0.008270 -0.054871 0.074524 -0.066772 0.009590 0.010582 -0.029388 0.010422 -0.059753 -0.018417 0.008064 +-0.061890 -0.039551 0.072510 -0.015511 0.042847 0.019516 -0.046082 0.015228 -0.013947 0.047913 -0.014160 -0.010521 -0.098694 0.078491 0.012909 -0.078613 -0.057953 -0.037445 0.031799 0.107178 0.027527 -0.050232 -0.031921 0.037231 0.100464 0.024445 0.056122 0.016693 0.075134 0.101685 0.070190 -0.120728 -0.038483 0.111023 -0.092773 0.026108 -0.032990 -0.021942 0.050201 0.047913 -0.060089 0.003527 -0.044312 -0.116333 -0.077087 -0.009789 0.050964 0.012131 -0.082458 -0.027512 0.048828 -0.073120 -0.050537 -0.054993 -0.039337 0.006485 0.033813 -0.044403 0.052673 -0.022385 0.003109 -0.012634 -0.098755 -0.123779 -0.000731 -0.027206 -0.019363 0.009132 0.045898 0.053589 0.172363 -0.001333 -0.080750 -0.008438 0.060913 -0.020782 -0.058441 -0.001517 0.061432 0.049561 -0.016602 -0.042450 0.025070 -0.029480 0.029434 0.042084 0.050079 0.020874 -0.114868 -0.091187 0.000793 0.045990 0.020386 -0.110413 -0.042023 -0.076721 0.097351 -0.014015 -0.027191 0.083679 0.023849 -0.059937 -0.091736 0.016815 -0.030197 -0.042236 -0.085510 0.012650 -0.048035 0.009407 -0.012695 -0.024567 -0.070312 -0.084961 -0.022934 -0.080505 -0.109253 0.009560 0.042236 -0.037720 -0.048889 -0.050812 0.082947 0.001262 0.121948 0.025162 0.053375 0.075134 0.063232 -0.030746 0.031708 0.023575 -0.141602 -0.008881 0.046997 0.014778 -0.003922 -0.119751 -0.084534 0.005833 0.062988 -0.020538 0.064087 -0.057983 -0.118713 0.016281 0.029434 0.035400 0.035858 -0.015129 -0.005802 -0.065979 -0.060730 0.037506 -0.082092 -0.061279 -0.049347 0.169922 0.149902 -0.029480 0.040497 -0.144043 -0.046204 0.073425 0.045685 -0.023788 0.088379 -0.002628 0.071533 -0.015152 0.046173 0.070679 0.044800 -0.018021 -0.015213 0.007294 -0.058594 0.105530 0.025711 0.013092 0.059204 -0.084229 -0.042694 0.044952 -0.045288 -0.046875 0.018646 -0.027054 0.055267 -0.051086 -0.028366 0.030670 -0.029022 0.052765 0.124939 -0.012573 0.014542 -0.194092 0.042480 -0.100708 -0.055206 -0.021896 -0.059662 0.031738 0.024445 0.081665 -0.138306 -0.085266 0.040588 0.123474 -0.028107 0.080322 0.064087 0.029160 -0.025192 -0.027786 0.099182 -0.021759 -0.067017 0.055969 -0.109253 0.036011 -0.052612 -0.092957 0.072937 0.065125 0.059021 0.042328 -0.077209 -0.008865 0.030518 -0.072327 -0.016037 -0.051849 0.057068 -0.058441 -0.091431 -0.005932 -0.033173 0.035828 -0.019562 -0.025238 0.066650 -0.149780 -0.008934 -0.024734 -0.121155 0.012817 0.050934 -0.069031 0.045990 0.090759 0.029129 -0.066284 -0.017639 -0.030655 +0.037872 -0.030411 0.065491 -0.066345 0.024139 0.051453 -0.011505 -0.052368 -0.018646 -0.018967 -0.039948 -0.013763 -0.110840 -0.002972 0.040710 -0.152344 -0.006618 -0.011940 0.065369 0.066528 -0.067993 0.008232 -0.003304 -0.007904 0.069824 0.010941 0.038757 0.108032 0.134033 -0.013596 0.095947 -0.077698 -0.061462 -0.002569 -0.114319 -0.054199 -0.062500 0.033600 0.015038 0.089600 0.034149 0.042480 0.027206 -0.033569 -0.054871 0.042603 0.086609 -0.065857 -0.061584 -0.009270 0.024628 -0.060547 -0.132080 -0.053314 -0.036407 -0.022934 0.038116 -0.021759 0.113281 -0.002869 0.009758 -0.031982 -0.122864 0.013100 -0.060303 -0.017746 -0.027649 -0.060638 0.010231 0.004429 0.100769 -0.048828 0.103088 -0.045074 0.088928 -0.035858 -0.114136 0.036469 0.063965 0.004143 -0.080750 -0.056671 0.015053 -0.031677 -0.055664 -0.058228 0.051605 0.042633 -0.072327 -0.074036 0.002657 0.125000 0.036743 -0.105347 -0.089417 -0.078674 0.034515 -0.054657 -0.036621 0.098938 0.008148 -0.171143 -0.105408 -0.016830 -0.016663 -0.029755 -0.060333 -0.072876 -0.084900 -0.035278 -0.057831 -0.020462 -0.012711 -0.040741 -0.015053 -0.100525 -0.048462 0.040039 0.005207 -0.058594 0.099670 -0.020294 0.011292 -0.099731 0.118774 -0.036804 0.120728 -0.033630 0.038757 -0.015930 0.013008 -0.000036 -0.079956 0.065796 0.037048 0.011459 0.015190 -0.112305 0.018326 -0.006958 -0.057892 -0.019684 0.051422 -0.082703 -0.042389 0.031433 -0.001151 -0.032623 0.021286 -0.008308 -0.017868 -0.001355 0.021606 -0.021393 -0.038574 -0.099976 -0.036560 0.143066 0.021103 0.071045 0.039825 -0.143311 -0.088928 0.030731 0.036438 0.037109 -0.061005 -0.069885 -0.020508 -0.058624 0.058197 -0.012260 0.010788 -0.066345 -0.016037 0.055084 0.020248 0.001548 0.077209 0.036072 0.055481 -0.098816 -0.016815 0.036804 -0.059357 0.010712 -0.064697 -0.005898 0.134521 0.030975 0.008690 -0.077087 -0.083557 0.052856 0.035675 -0.033600 0.083252 -0.153809 0.060822 -0.056946 -0.127563 -0.045227 0.021759 0.111877 0.040833 0.119202 -0.075867 -0.058746 -0.049408 0.054718 -0.083435 0.075684 0.118408 0.003222 -0.116394 -0.032379 0.039673 0.064209 -0.088074 -0.017120 -0.059662 0.093079 0.047455 -0.004372 0.028259 0.137329 -0.011383 0.037750 -0.073547 -0.041901 0.039917 -0.003773 -0.031219 -0.058472 0.069214 -0.048431 0.069763 0.015198 -0.076538 0.026321 0.017822 -0.005577 -0.013290 -0.090820 0.050720 0.050995 -0.107361 0.006077 -0.009552 -0.030014 0.065002 0.085327 0.065552 0.002981 -0.033813 -0.028625 +-0.112000 -0.065613 0.082397 0.127197 0.102661 -0.021759 -0.018616 0.000806 -0.097168 -0.024445 -0.131836 -0.011772 -0.074585 0.042389 0.028900 -0.165405 -0.037628 -0.101990 0.115723 0.052216 -0.024811 -0.036499 0.035461 0.026398 -0.021271 -0.016220 0.038788 0.046783 -0.000107 0.009521 0.014137 -0.018539 -0.013023 -0.060944 -0.085144 -0.096497 0.002235 -0.069214 -0.037323 0.013794 0.085083 0.070923 0.020340 0.006546 0.016235 0.058319 0.045502 -0.048981 -0.009430 -0.036346 0.018478 -0.010460 -0.131470 0.083862 0.058990 -0.028320 0.098511 -0.004219 0.037567 -0.082214 -0.033203 0.042023 -0.085693 0.028336 0.002293 0.010948 -0.093262 0.036774 0.012260 0.033905 0.065552 0.027908 0.088074 -0.054565 0.063171 -0.025040 -0.134399 -0.013298 0.005455 -0.010048 -0.062744 0.002703 0.018570 -0.053375 -0.060059 -0.143311 0.012718 0.046722 -0.038757 -0.058380 0.106995 0.052490 0.011658 0.010033 -0.065552 -0.020355 0.081421 -0.035675 -0.017548 0.032013 0.008171 -0.066833 -0.090759 -0.055969 0.042969 0.006832 -0.028152 -0.070923 -0.012566 0.035675 0.025070 -0.033051 0.073303 0.113037 0.043182 -0.100464 -0.025940 0.039398 0.017441 -0.059448 0.125244 -0.013748 0.009254 0.079346 0.009773 -0.012123 0.166260 -0.068298 0.095276 -0.022141 -0.040802 0.041168 -0.064270 0.054077 -0.022949 -0.036469 0.026581 -0.062347 0.052704 -0.036163 0.013634 -0.087769 -0.004974 -0.040863 0.011047 0.013107 0.020416 -0.007298 0.053528 0.078186 0.042328 -0.034332 0.008850 -0.012260 -0.035156 -0.032318 -0.062561 0.077759 -0.013229 0.052216 -0.063599 -0.081848 -0.079102 0.115051 0.049927 -0.043121 -0.110718 -0.041962 -0.002954 -0.017609 -0.002710 -0.095520 -0.025894 -0.061523 0.062866 0.042053 0.029160 -0.041412 -0.107605 0.005695 0.028992 -0.124939 -0.012680 0.046204 -0.041931 0.038971 -0.049103 -0.052094 0.080994 0.074280 0.051361 -0.002745 -0.009933 -0.055603 -0.032593 -0.047699 0.122009 -0.087891 -0.005188 0.027176 0.014091 -0.019974 0.121582 0.121704 0.045197 0.051239 0.006561 -0.096863 -0.064636 0.013847 -0.040802 0.015373 0.022644 -0.018265 0.069580 -0.025055 0.001861 0.037292 -0.105408 -0.105347 0.018784 0.102966 0.115417 -0.070984 0.057861 0.049438 0.004341 0.026627 -0.024338 0.012405 0.085449 0.068054 0.066895 0.031128 0.076477 -0.088562 0.147217 -0.004913 -0.142944 0.081543 0.153931 0.035553 0.015335 -0.113037 0.082947 0.016510 -0.079224 -0.075378 -0.063110 -0.022919 -0.018158 -0.044464 0.152710 0.003454 0.017349 -0.001744 +0.024445 -0.004959 0.021393 -0.007038 0.014244 0.074524 0.087280 -0.074158 -0.004257 -0.042816 0.008919 0.074097 0.011314 0.064880 0.026398 -0.125732 0.051178 -0.037018 0.030655 0.028885 -0.038483 0.010880 -0.094116 -0.034302 0.013962 0.000459 -0.005886 0.045410 0.089294 0.038055 0.110413 -0.046112 0.016556 0.058228 -0.048676 -0.011101 -0.005836 0.056671 -0.025009 0.108276 0.068420 0.079407 0.059448 -0.010284 -0.090515 0.070129 0.059387 0.021088 -0.084473 0.063599 0.085022 -0.198242 -0.077820 -0.024704 0.015488 -0.002390 0.026215 0.020309 0.072449 -0.036804 0.087158 -0.026337 -0.109314 0.017731 -0.037506 0.050171 0.015884 -0.103271 0.043762 0.044067 0.088867 0.003057 0.150513 -0.020325 0.021378 -0.010582 -0.184204 0.064270 0.118286 0.029083 -0.025162 0.012474 -0.005775 0.002539 -0.150269 -0.047638 -0.024475 0.051086 0.048431 -0.082336 -0.033447 0.054291 -0.016647 -0.110657 -0.055267 -0.063782 0.085144 -0.040771 -0.091187 0.017014 0.044647 -0.067322 -0.015297 0.004799 0.058228 0.067993 -0.040314 -0.030045 -0.023438 -0.023712 -0.086121 -0.022034 -0.037598 -0.058044 0.024063 -0.082642 -0.068176 0.117676 0.034210 0.006729 0.025101 -0.037476 0.036438 -0.039001 0.127441 0.000664 0.119690 -0.011345 0.043243 -0.059113 -0.003496 0.019028 -0.125732 0.045990 0.013664 0.003201 -0.000055 -0.036713 0.017227 0.026199 0.005135 0.002310 -0.005512 -0.052521 -0.008881 0.005119 0.001591 -0.036407 0.068420 0.005283 -0.041870 -0.024521 0.038300 -0.035370 -0.042725 -0.119019 -0.025635 0.116638 -0.050873 0.085876 0.031952 -0.117920 -0.044861 -0.026932 0.025986 0.051727 -0.057159 -0.035522 -0.039948 -0.106384 0.017654 -0.041016 0.141357 -0.058533 -0.084656 0.055206 0.072571 0.005478 0.075867 0.033112 0.045471 -0.039124 0.062408 0.047638 0.040558 0.037048 -0.086182 -0.011314 0.137573 0.009262 0.058075 -0.042755 -0.089172 0.111877 0.013428 0.007027 0.003151 -0.131104 0.021729 0.026016 -0.153320 -0.007889 -0.002403 0.057526 0.087036 0.073120 -0.070068 -0.120117 0.033203 0.053223 -0.068420 0.003628 0.113342 0.041046 -0.008690 -0.084595 0.048889 0.066223 -0.025330 -0.031799 -0.041626 0.114441 -0.025009 0.099609 0.052368 0.190796 0.048645 0.016983 -0.124939 -0.060059 -0.013924 0.051483 -0.013954 -0.063416 0.027985 -0.086914 0.078979 0.045837 -0.071411 0.013458 0.019608 0.025742 -0.003618 -0.085388 0.065735 -0.031677 -0.074463 0.016113 0.011948 0.041656 0.004967 0.005054 0.012589 -0.005062 0.017532 -0.011566 +0.029953 -0.013474 0.005219 0.002035 0.036438 0.116882 0.053589 -0.086487 0.025543 -0.071228 -0.028503 0.045746 -0.036041 -0.035278 0.020935 -0.167480 0.026062 -0.015396 0.001961 -0.016571 -0.115295 0.022934 -0.049438 -0.045227 0.035950 -0.010551 -0.011765 0.055298 0.073364 0.010277 0.121826 -0.084473 -0.006413 -0.003103 -0.108093 -0.085815 -0.040924 0.004807 0.010979 0.148926 0.074402 0.053284 0.072937 0.013420 -0.113464 0.037689 0.065918 -0.027008 -0.069885 0.004993 0.047455 -0.155396 -0.089783 -0.035217 0.047302 0.013351 0.044128 0.024063 0.055450 -0.080566 0.057281 0.025635 -0.116211 0.045532 -0.056000 0.022171 -0.012932 -0.075500 -0.013382 0.007359 0.054871 -0.044830 0.113159 -0.052612 0.022385 -0.042480 -0.135498 0.096924 0.050842 0.025711 -0.035339 -0.042694 -0.038208 0.014427 -0.136841 -0.105286 -0.008705 0.026199 0.026550 -0.054749 0.009018 0.073730 0.022873 -0.121338 -0.058472 -0.000524 0.051208 -0.042358 -0.086426 0.083374 0.025757 -0.103516 -0.008835 0.012268 0.010529 0.035583 -0.036377 -0.047058 -0.066467 -0.036163 -0.109863 0.006222 -0.008209 -0.089172 0.022842 -0.073303 -0.032593 0.131104 0.033112 -0.007282 0.062012 -0.097046 0.010574 -0.066833 0.108398 -0.062744 0.093872 -0.002075 0.072998 -0.005943 0.002975 -0.005527 -0.043213 0.004768 -0.013397 -0.006516 -0.014000 -0.061157 0.004002 0.005531 -0.055267 0.027863 0.025208 -0.070923 -0.024551 0.020645 -0.008316 -0.059326 0.021240 -0.007599 -0.040771 -0.024139 0.055939 -0.001364 -0.014488 -0.087219 0.006828 0.093262 -0.082947 0.124084 0.037567 -0.106873 -0.081299 -0.000940 0.010704 0.051056 -0.084412 -0.085449 -0.037170 -0.075928 0.038391 -0.093994 0.069702 -0.090210 -0.069641 0.078491 0.049774 0.012657 0.062744 0.009048 0.042297 -0.074707 0.005501 0.048553 -0.026596 0.038727 -0.088745 -0.021088 0.159058 0.035614 0.025772 -0.036469 -0.047668 0.050629 0.016006 0.010376 -0.012001 -0.133057 0.102722 -0.020599 -0.138794 -0.022644 0.027054 0.093872 0.096008 0.073120 -0.069092 -0.106201 -0.017929 0.072937 -0.075562 0.045990 0.088806 0.055054 -0.065125 -0.041687 0.008331 0.059509 -0.088562 -0.004913 -0.044037 0.125488 0.007214 0.046326 0.055695 0.166382 -0.024643 0.035248 -0.120117 -0.051453 -0.011063 0.034363 -0.034760 -0.066223 0.013374 -0.056030 0.101868 -0.005829 -0.125488 0.024704 0.035126 -0.002274 0.052002 -0.053436 0.063904 -0.032135 -0.082581 -0.023102 0.028015 0.003983 0.021683 0.001993 0.084351 0.007034 0.012581 -0.039917 +-0.087769 0.036560 -0.014854 -0.026093 0.032013 0.087830 -0.023117 -0.024857 -0.040527 -0.076050 0.055786 0.048889 -0.098328 0.078857 0.061951 -0.124268 0.045410 0.056091 -0.023148 -0.006985 -0.083801 0.085632 -0.086304 -0.098145 0.065735 -0.033936 -0.003508 0.066345 0.118591 -0.011742 0.038269 -0.028809 0.005920 -0.010269 -0.085266 -0.078186 -0.106689 0.024307 0.035065 0.133667 -0.021729 0.058624 -0.034790 -0.043762 -0.120056 -0.024948 0.032166 -0.018829 -0.130737 0.033630 0.086975 -0.145508 -0.110291 -0.122253 0.049194 0.072693 0.013710 -0.004509 0.120789 -0.035492 0.021652 -0.049286 0.016449 -0.037872 0.042084 0.098328 0.041840 0.028641 0.025024 0.030563 0.123596 -0.035248 0.113708 0.035736 0.019577 -0.022858 -0.091431 0.013084 0.095642 0.081726 -0.005825 0.030334 -0.060547 0.045776 -0.072571 -0.017334 0.081848 -0.014679 -0.002552 -0.061462 -0.085205 0.064575 0.017456 0.021927 -0.078613 0.030411 0.088013 -0.048950 -0.029282 0.039398 0.013512 -0.074646 0.020996 0.023804 -0.108582 -0.049500 0.018555 -0.020584 -0.031799 -0.078857 -0.075745 0.013847 -0.082397 -0.034393 -0.004452 -0.054504 -0.010002 0.092712 0.093872 -0.025604 0.054260 -0.073547 0.054626 -0.010284 0.032959 -0.068420 -0.044983 0.047974 0.012131 0.038391 0.040283 -0.004875 -0.103577 0.010002 0.095154 0.051270 -0.063538 0.004436 0.007282 -0.019165 -0.056183 -0.030655 0.060638 -0.050812 -0.026428 0.063110 0.013168 0.007820 -0.023697 0.053162 0.095947 -0.009369 0.035278 0.011040 -0.078308 -0.113525 -0.034821 0.108521 0.022324 0.054718 0.066956 -0.073547 0.028915 0.010056 0.007179 0.112854 -0.019485 0.001120 -0.038330 -0.081299 0.088074 -0.061768 0.010460 -0.090271 -0.100586 -0.001813 -0.009277 -0.035065 -0.006092 -0.000404 0.045380 -0.028732 0.018387 0.068726 -0.064453 0.147095 -0.012383 0.081299 0.068420 -0.037445 -0.078491 0.038208 -0.049896 0.021713 0.067261 -0.015533 0.026550 -0.064331 0.056641 -0.019501 -0.101318 -0.057007 0.086731 -0.001513 0.119324 0.042389 -0.153076 -0.031921 0.099487 0.073486 -0.041077 0.077759 0.076721 -0.029739 -0.042633 -0.016678 -0.019562 0.003281 -0.037598 0.026978 -0.038055 0.091553 0.005447 0.023407 0.046204 0.220337 -0.022858 0.016800 -0.130127 -0.016876 0.007256 -0.073120 0.004986 -0.063416 -0.050507 -0.079773 0.054016 -0.091248 -0.055267 0.059845 0.008247 -0.015854 0.132935 0.005527 0.018799 -0.061218 -0.043396 -0.028030 -0.105957 0.017517 0.067932 -0.042023 0.046173 -0.028458 0.005692 0.008682 +-0.033905 -0.010063 0.010475 0.028061 0.000831 0.092346 0.066772 -0.061890 -0.022461 -0.038391 -0.023987 0.047791 -0.089417 0.077515 0.033020 -0.158936 0.044373 -0.005390 0.007298 -0.049622 -0.062805 0.061127 -0.090576 -0.058411 0.037201 0.019714 0.006535 0.118103 0.107910 0.011147 0.113403 -0.028152 -0.020752 0.012268 -0.110291 -0.040619 -0.012619 0.035553 -0.012932 0.143921 0.011864 0.072144 0.002089 -0.005554 -0.071960 0.014870 0.002523 -0.054810 -0.154053 0.034454 0.084351 -0.159912 -0.133545 -0.073669 0.024261 -0.016525 -0.007549 0.012573 0.112671 -0.079712 0.030670 -0.025208 -0.060913 0.046600 -0.009300 0.039520 -0.031311 -0.053864 0.025101 0.031891 0.083862 -0.068115 0.095703 0.016052 0.045746 -0.037903 -0.137329 0.057800 0.126465 0.035156 -0.046997 -0.015930 -0.025940 0.052124 -0.133911 -0.069824 0.032745 0.047363 0.007706 -0.090759 -0.049957 0.089966 0.001418 -0.070557 -0.063416 0.007286 0.063965 -0.045227 -0.076355 0.038208 0.022064 -0.095642 -0.025391 0.036377 -0.030136 0.032959 -0.002844 -0.034363 -0.035461 -0.058136 -0.062866 0.012177 -0.004219 -0.051819 0.014893 -0.086060 -0.031128 0.113525 0.061066 0.003689 0.021027 -0.069519 0.004791 -0.038544 0.037201 -0.074158 0.051666 -0.062927 0.051819 -0.039001 -0.019623 0.018753 -0.079712 0.026642 0.030151 -0.007431 0.034454 -0.016739 0.032898 0.012856 -0.035248 -0.051880 0.009262 -0.056061 0.019958 0.022934 0.042877 -0.031586 0.013885 0.030228 0.005589 0.012245 0.030975 -0.014542 -0.069641 -0.114502 0.013329 0.107056 -0.059784 0.109802 0.022842 -0.117126 -0.031433 -0.059418 0.014366 0.057556 -0.075073 -0.085999 -0.051727 -0.103638 0.021164 -0.058899 0.034485 -0.107117 -0.113586 0.062134 0.058411 -0.043884 0.016235 0.033386 0.058685 -0.120605 0.006039 0.044220 -0.026627 0.105652 -0.097778 0.000656 0.133057 0.043579 0.002386 -0.000733 -0.081299 0.048187 0.020477 0.022964 -0.022614 -0.089478 0.054169 0.027222 -0.121887 -0.010696 0.054169 0.060516 0.083984 0.054321 -0.092896 -0.102966 -0.025711 0.049805 -0.069092 0.072083 0.113281 -0.028076 -0.065979 -0.103699 0.019089 0.085693 -0.073120 -0.026154 -0.057983 0.112488 0.046417 0.002396 0.048737 0.194580 -0.052795 -0.022827 -0.119019 -0.047028 -0.002781 0.034302 0.004517 -0.045349 -0.025986 -0.120300 0.070129 -0.039703 -0.109863 0.063782 0.016815 0.006935 0.052399 -0.022034 0.068359 -0.034088 -0.101135 0.008308 -0.055573 0.016754 0.015465 -0.000374 0.007195 -0.009209 -0.002911 -0.021408 +-0.043091 -0.074707 0.030762 0.044189 0.068420 0.064697 0.070557 -0.066711 0.023422 -0.018509 -0.068115 0.015182 0.015259 -0.031021 0.004971 -0.173340 -0.047028 -0.085815 -0.002056 -0.055786 -0.067444 0.063293 -0.015465 -0.020035 0.030167 -0.042603 -0.023300 0.080322 0.056793 0.006626 0.108521 -0.078430 -0.032959 0.001125 -0.131714 -0.067383 -0.053436 -0.029114 0.023041 0.051941 0.024460 0.018204 0.018951 0.028656 -0.114868 0.057922 0.021469 -0.083435 -0.149658 -0.013062 0.002634 -0.074219 -0.098755 -0.028625 -0.016525 0.002815 0.051331 0.014877 0.036224 -0.028854 0.029007 -0.013840 -0.087097 0.013306 -0.050018 -0.002832 -0.003235 -0.139893 -0.032013 -0.032379 -0.026428 -0.041199 0.093018 -0.061249 0.011299 -0.043488 -0.068909 0.066772 0.039917 0.016357 -0.031982 -0.050903 0.030884 -0.006672 -0.155518 -0.059631 0.021103 -0.055878 0.049042 -0.136841 0.058929 0.076721 -0.021622 -0.182251 -0.069580 -0.034363 0.028671 -0.032654 -0.035553 0.047241 -0.015701 -0.119263 -0.038696 -0.027405 -0.034515 0.028381 -0.032013 -0.031494 0.001493 0.000972 -0.043457 0.012154 0.006317 -0.078064 -0.001413 -0.044128 -0.005516 0.138794 -0.003353 -0.024109 0.081909 0.020981 0.002401 0.013618 0.064697 0.018433 0.130615 -0.029724 0.027985 -0.014122 -0.015076 0.014168 -0.069763 0.021881 -0.057343 -0.018723 -0.013161 -0.102844 0.002810 -0.068542 -0.075195 0.033783 0.019791 -0.076599 -0.082825 0.070618 0.046234 -0.102051 -0.022705 0.054321 -0.025116 -0.003193 0.033783 0.020203 0.003263 -0.080505 -0.028366 0.029434 -0.066040 0.090759 -0.017670 -0.090271 -0.055359 -0.022873 0.026093 0.003563 -0.091675 -0.077026 -0.012459 -0.096680 0.018005 -0.052368 0.018585 -0.139160 -0.061554 0.139038 0.042145 -0.051941 0.045349 -0.013756 0.051697 -0.088867 -0.030624 0.006317 -0.030533 0.047272 -0.082031 -0.050598 0.115112 0.030975 0.045471 -0.038574 -0.118958 0.009911 -0.013443 -0.020813 0.036499 -0.113037 0.054443 -0.031128 -0.144897 -0.019470 0.033966 0.141113 0.071655 0.030624 0.018234 -0.082947 -0.068115 0.085388 -0.046967 0.022537 0.127686 -0.006241 -0.070862 -0.053101 -0.033081 0.125366 -0.101135 -0.025787 -0.065186 0.108337 0.043793 -0.034058 0.087830 0.066711 -0.067688 -0.030396 -0.114014 -0.064880 -0.030090 0.061493 -0.038361 -0.081726 0.008476 -0.071838 0.104797 -0.027725 -0.072205 0.060272 0.018280 -0.041992 0.053253 -0.085815 0.059601 0.029129 -0.146851 -0.035278 -0.010963 -0.057373 0.040222 0.020554 0.108521 0.056427 -0.013786 -0.013420 +-0.039215 -0.067261 0.094055 -0.064087 0.059204 -0.040588 -0.004719 -0.043976 -0.040375 0.006104 -0.084229 -0.029160 -0.164185 0.001306 0.024918 -0.141968 -0.058075 -0.036530 0.089294 0.020020 -0.039429 0.009346 0.022797 0.000039 0.045959 -0.077148 0.057648 0.022858 0.077393 -0.075012 0.054077 -0.072083 -0.042786 0.010666 -0.177368 -0.047852 -0.090759 -0.008438 0.018585 0.064026 0.038452 0.020096 0.077148 -0.079285 -0.039581 0.061005 0.077576 -0.025085 -0.065613 -0.094971 0.011909 0.048462 -0.112488 -0.000763 -0.017792 -0.027252 0.058228 -0.030045 0.029221 -0.021332 0.008820 -0.056274 -0.120544 -0.008987 -0.070496 0.001659 0.014915 -0.063232 -0.118469 0.045441 0.052368 -0.028946 0.052734 -0.067017 0.122498 -0.068542 -0.069641 0.016251 0.082214 -0.012611 -0.082092 -0.028564 0.052734 0.062256 0.025452 -0.066101 0.062866 0.030960 -0.004002 -0.079895 0.003546 0.091309 0.051483 -0.108765 -0.112549 -0.041412 -0.020218 -0.046265 -0.027588 0.148315 -0.014610 -0.169067 -0.013329 -0.013206 -0.105103 -0.082336 -0.026978 -0.146606 -0.031464 -0.043732 -0.024704 -0.042877 0.009636 0.002218 0.044769 -0.041443 0.087524 0.066895 -0.008499 -0.061340 0.165283 0.025574 -0.020477 -0.053375 0.052490 0.004951 0.055481 0.045013 0.047241 0.031494 -0.040131 -0.001564 -0.050476 0.064880 0.008156 0.017136 -0.073730 -0.138916 0.030197 -0.003706 -0.076477 0.003685 0.006142 -0.031616 -0.073914 0.134155 -0.013878 -0.037811 -0.032990 -0.037109 0.002953 -0.051514 0.000969 0.029892 0.021820 -0.051971 -0.006363 0.112793 0.029617 0.013397 0.036743 -0.077454 -0.070007 0.090759 0.009613 -0.065186 -0.046967 -0.044037 0.014175 -0.028412 0.096252 -0.030090 -0.072571 -0.059479 0.109558 0.024170 0.024841 -0.001913 0.044861 0.033081 0.063354 -0.066040 -0.055328 0.026306 -0.077393 0.016174 -0.059357 0.000043 0.123230 -0.007015 -0.043549 -0.053009 -0.068298 0.030396 -0.021835 -0.059265 0.101685 -0.119019 0.038910 -0.106201 -0.064331 -0.011925 0.016785 0.110779 0.070557 0.111938 -0.042145 -0.030884 -0.067566 0.029221 -0.007294 0.128784 0.121887 -0.074097 -0.081421 0.020447 0.013786 0.037659 -0.068787 -0.013481 -0.035614 0.033630 0.006725 -0.046417 0.061432 0.023666 0.006065 -0.009491 -0.026978 -0.022049 0.107727 -0.006538 0.009056 -0.015793 0.110046 -0.008354 0.036865 -0.082458 -0.034393 0.011353 -0.011482 -0.016418 0.030472 -0.069458 0.033051 0.090698 -0.068787 0.010857 -0.011711 -0.089050 0.087891 -0.033722 0.123901 -0.053345 -0.069336 -0.062683 +-0.112854 -0.025482 0.005482 0.020279 0.059692 -0.042267 0.031586 -0.048950 -0.040253 -0.008797 -0.152100 -0.019165 -0.092712 0.010941 0.067322 -0.196411 -0.033844 -0.128418 0.088440 -0.000941 -0.011795 0.060852 -0.057220 -0.080078 0.030167 -0.071960 0.015747 0.114380 0.041992 -0.030792 0.071228 -0.067261 -0.022034 -0.100281 -0.171021 -0.104553 -0.062744 -0.014526 0.041565 0.075012 0.055084 0.038727 0.058868 -0.008957 -0.048981 0.021347 0.071350 -0.002934 -0.083008 -0.048096 0.058441 0.016479 -0.157471 0.041962 0.062988 0.012779 0.034943 0.006897 0.017471 -0.050293 -0.016953 0.004257 -0.076538 -0.017776 -0.031067 -0.000585 -0.045929 -0.025711 -0.016251 -0.004734 0.053497 0.028519 0.076660 -0.033356 0.027313 -0.052063 -0.125488 -0.010010 0.077148 -0.005856 -0.081116 0.015457 0.038208 -0.011597 -0.062988 -0.092346 0.008179 0.033844 -0.002117 -0.068359 0.028885 0.069336 0.012299 -0.041962 -0.082153 0.001546 0.020584 -0.035828 -0.025558 0.138916 0.010719 -0.124268 -0.024048 -0.078857 -0.009933 0.028809 0.010895 -0.094849 -0.002447 -0.027924 0.041931 0.009644 0.069641 0.064941 0.050018 -0.048584 0.053802 0.081238 0.006062 -0.069580 0.112488 -0.035706 -0.007183 -0.000055 -0.043152 -0.015388 0.080627 -0.061707 0.012703 -0.020157 -0.009735 -0.017563 -0.061432 0.052826 -0.030060 -0.027008 0.017593 -0.034180 0.079041 -0.005634 -0.054993 0.007370 -0.028214 -0.054626 0.047760 0.097351 0.009506 0.006466 -0.007553 0.018219 0.018982 0.004433 0.056000 0.005699 -0.029144 -0.038330 -0.012337 0.068604 -0.049377 0.138794 0.038452 -0.048859 -0.057190 0.119629 -0.011131 -0.016388 -0.085999 -0.041748 -0.026672 -0.086365 -0.005589 -0.055542 -0.055328 -0.126465 -0.010002 -0.000075 0.053864 -0.018692 -0.035675 0.012138 0.006001 -0.150146 -0.037598 0.029358 -0.077698 0.111450 -0.063232 -0.025040 0.106018 0.013931 -0.026550 -0.011070 -0.001914 0.011086 -0.045319 -0.014580 0.110291 -0.079956 0.046265 -0.014252 -0.047577 -0.076416 0.076904 0.119873 0.131226 0.067993 -0.004433 -0.053619 -0.027161 0.069031 0.002426 0.054901 0.091003 -0.032990 -0.026718 0.008980 -0.078064 0.097534 -0.083130 -0.061493 -0.047638 0.164429 0.107849 -0.063660 0.074524 0.083435 -0.049347 -0.061462 -0.059631 0.017822 0.106140 0.058350 0.019424 -0.014931 0.043335 -0.115051 0.099854 -0.094849 -0.084900 0.024323 0.053223 -0.014671 0.043610 -0.033905 0.098694 0.012497 -0.041504 -0.034546 -0.106934 -0.053223 0.045807 -0.100769 0.110535 0.067444 -0.041168 -0.007671 +0.017929 -0.018723 -0.028030 -0.025391 -0.024414 0.104248 0.000623 -0.090576 0.012886 -0.088074 -0.033875 0.031174 -0.089233 0.002180 0.017212 -0.142944 0.062256 0.000796 -0.041046 -0.027954 -0.096313 0.043091 -0.066956 -0.069275 0.053711 0.016785 0.011047 0.129639 0.107422 0.004948 0.141479 -0.057739 -0.037903 -0.012688 -0.108948 -0.051483 -0.035034 0.022781 0.030731 0.154053 -0.013000 0.054474 0.030518 0.028152 -0.082092 0.015396 0.034088 -0.048859 -0.096619 0.042938 0.058502 -0.138428 -0.170166 -0.067566 0.056458 -0.012047 0.028152 -0.009102 0.133301 -0.045746 0.004971 -0.000458 -0.037506 0.070374 -0.028275 -0.009819 -0.040436 -0.026962 0.068970 -0.034058 0.070068 -0.070984 0.132202 -0.013420 0.021973 -0.052765 -0.103882 0.072327 0.060364 0.022430 -0.001070 -0.031494 -0.021042 -0.002335 -0.120117 -0.080688 0.014008 0.025497 -0.028107 -0.007694 -0.004406 0.115601 0.030579 -0.062622 -0.068481 0.046753 0.018555 -0.059967 -0.034088 0.076965 0.049377 -0.135620 -0.026871 -0.005478 -0.021561 0.064209 0.002422 -0.012718 -0.115234 -0.056885 -0.036591 0.015289 -0.011833 -0.022507 0.001439 -0.073853 -0.038391 0.088257 0.024506 -0.002724 0.050873 -0.104797 -0.018936 -0.078918 0.032776 -0.081909 0.076416 -0.085388 0.011993 -0.019592 0.017456 0.025284 -0.053833 0.039703 0.056122 -0.031097 0.040771 -0.040741 0.060303 0.008430 -0.068237 0.038300 0.034637 -0.087036 0.021988 0.003504 -0.006550 -0.035339 0.009674 -0.004513 0.029251 0.043396 0.049835 -0.022720 -0.046173 -0.121399 -0.010719 0.112610 -0.057343 0.103699 0.038666 -0.120850 -0.086792 -0.016953 0.029556 0.070862 -0.096436 -0.106140 -0.059906 -0.092346 0.007980 -0.026062 0.008110 -0.095581 -0.107666 0.038788 0.058289 0.006073 0.033997 0.066467 0.055206 -0.080505 0.024368 0.040283 -0.058960 0.023438 -0.107239 0.017838 0.144287 0.062073 0.005550 -0.036774 -0.038910 -0.003952 -0.001529 0.014221 -0.024261 -0.099182 0.055969 -0.010254 -0.140259 -0.009117 0.089600 0.045227 0.080200 0.082581 -0.085815 -0.054718 -0.025909 0.009064 -0.076782 0.021667 0.086670 -0.027649 -0.095337 -0.069458 0.016449 0.102173 -0.077820 -0.062378 -0.053497 0.096436 0.087646 0.025070 0.020370 0.190552 -0.091797 0.025452 -0.078735 -0.017929 0.026031 0.049042 -0.031311 -0.075256 -0.015388 -0.050415 0.075134 -0.027069 -0.144165 0.044617 0.023788 0.020142 0.003881 0.007057 0.047974 -0.027649 -0.059631 -0.002188 -0.046631 0.020798 0.024246 0.027603 0.028336 0.039246 -0.015083 -0.034363 diff --git a/practice/data_.bin b/practice/data_.bin new file mode 100644 index 0000000..7c60451 Binary files /dev/null and b/practice/data_.bin differ diff --git a/practice/json_to_data.py b/practice/json_to_data.py new file mode 100644 index 0000000..fd5d2af --- /dev/null +++ b/practice/json_to_data.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +""" +Created on Wed Dec 18 15:44:33 2024 + +@author: ieemoo-zl003 +""" + +import json +import struct +import numpy as np + +json_path = r"D:\DetectTracking\practice\resv11_test.json" + +def write_binary_file(filename, datas): + with open(filename, 'wb') as f: + # 先写入数据中的key数量（为C++读取提供便利） + key_count = len(datas) + f.write(struct.pack('I', key_count)) # 'I'代表无符号整型（4字节） + + feats_32, feats_16 = [], [] + for data in datas: + key = data['key'] + feats = data['value'] + key_bytes = key.encode('utf-8') + key_len = len(key) + length_byte = struct.pack(' 0: + '''复制事件文件夹到指定路径''' + # save_path_ = os.path.join(save_path, name_, type_) + # os.makedirs(save_path_, exist_ok=True) + # copy_events(date_path, events, save_path_) + '''将事件名称写入txt文件''' + write_events(events, txt_path) + + else: + events = list(set(filter_df['事件名'])) + print('events: ', events) + if len(events) > 0: + '''复制事件文件夹到指定路径''' + # save_path_ = os.path.join(save_path, name_) + # os.makedirs(save_path_, exist_ok=True) + # copy_events(date_path, events, save_path_) + '''将事件名称写入txt文件''' + write_events(events, txt_path) + + except Exception as e: + with open(name_ + 'write_txt_error.txt', 'a') as f: + f.write(f'{date_path}: {e}\n') + + +if __name__ == "__main__": + # file_path = 'finish_data_test' + file_path = '/home/yujia/yj/gpu_code/callback_data_test_0417/' + xlsx_name = '现场回传数据分析表_all.xlsx' + # events_save = os.path.join(file_path, 'events_class') + ''' + title_list = ['barcode', '有无商品操作', '赠品', '散称', '私人物品', '大号购物袋', '遮挡/模糊', '单次/多次事件切分不准确', '复杂操作', '是否购物现场', '备注', + 'algroStartToEnd', 'one2one', 'one2SN', 'one2n', '前摄轨迹数', '后摄轨迹数'] + ''' + + + title_names = ['赠品', '散称', '前摄轨迹数'] ##需筛选的表头标题名称 + ch = '轨迹数' ##筛选出excel表格中前摄轨迹数为0的事件名称，这类事件需重新跑pipline填入轨迹数 + saveClass = False ##是否按照列表内容分类保存，或将列表下非空的内容保存 + for title_name in title_names: + for name in os.listdir(file_path): ##人名 + # name_ = 'cxy' + name_path = os.path.join(file_path, name) + for date in os.listdir(name_path): ##日期 + # date_ = '2025-02-04' + date_path = os.path.join(name_path, date) + data_xlsx = os.path.join(date_path, xlsx_name) + anlay_xlsx_filter_events(data_xlsx, title_name, ch, saveClass) + + # break + + + + + + diff --git a/realtime/__pycache__/event_time_specify.cpython-312.pyc b/realtime/__pycache__/event_time_specify.cpython-312.pyc new file mode 100644 index 0000000..0719985 Binary files /dev/null and b/realtime/__pycache__/event_time_specify.cpython-312.pyc differ diff --git a/realtime/__pycache__/event_time_specify.cpython-39.pyc b/realtime/__pycache__/event_time_specify.cpython-39.pyc new file mode 100644 index 0000000..4035f43 Binary files /dev/null and b/realtime/__pycache__/event_time_specify.cpython-39.pyc differ diff --git a/realtime/__pycache__/intrude_detect.cpython-312.pyc b/realtime/__pycache__/intrude_detect.cpython-312.pyc new file mode 100644 index 0000000..34ada66 Binary files /dev/null and b/realtime/__pycache__/intrude_detect.cpython-312.pyc differ diff --git a/realtime/event_time_specify.py b/realtime/event_time_specify.py new file mode 100644 index 0000000..a33531d --- /dev/null +++ b/realtime/event_time_specify.py @@ -0,0 +1,315 @@ +# -*- coding: utf-8 -*- +""" +Created on Thu Oct 10 11:01:39 2024 + +@author: ym +""" +import os +import numpy as np +# from matplotlib.pylab import mpl +# mpl.use('Qt5Agg') +import matplotlib.pyplot as plt + +import sys +sys.path.append(r"D:\DetectTracking") +from move_detect import MoveDetect +# from tracking.utils.read_data import extract_data, read_deletedBarcode_file, read_tracking_output, read_weight_timeConsuming +from tracking.utils.read_data import read_weight_timeConsuming + +def str_to_float_arr(s): + # 移除字符串末尾的逗号（如果存在） + if s.endswith(','): + s = s[:-1] + + # 使用split()方法分割字符串，然后将每个元素转化为float + float_array = [float(x) for x in s.split(",")] + return float_array + +def find_samebox_in_array(arr, target): + + for i, st in enumerate(arr): + if st[:4] == target[:4]: + return i + return -1 + +def array2frame(bboxes): + frameID = np.sort(np.unique(bboxes[:, 7].astype(int))) + # frame_ids = bboxes[:, frameID].astype(int) + fboxes, ttamps = [], [] + for fid in frameID: + idx = np.where(bboxes[:, 7] == fid)[0] + box = bboxes[idx, :] + + fboxes.append(box) + ttamps.append(int(box[0, 9])) + + frameTstamp = np.concatenate((frameID[:,None], np.array(ttamps)[:,None]), axis=1) + + return fboxes, frameTstamp + + +def extract_data_1(datapath): + ''' + 要求每一帧（包括最后一帧）输出数据后有一空行作为分割行，该分割行为标志行 + ''' + + trackerboxes = np.empty((0, 10), dtype=np.float64) + trackerfeats = np.empty((0, 256), dtype=np.float64) + + boxes, feats, tboxes, tfeats = [], [], [], [] + timestamp = -1 + + newframe = False + with open(datapath, 'r', encoding='utf-8') as lines: + for line in lines: + if line.find("CameraId")>=0: + newframe = True + timestamp, frameId = [int(ln.split(":")[1]) for ln in line.split(",")[1:]] + # boxes, feats, tboxes, tfeats = [], [], [], [] + + + if line.find("box:") >= 0 and line.find("output_box:") < 0: + line = line.strip() + box = line[line.find("box:") + 4:].strip() + # if len(box)==6: + boxes.append(str_to_float_arr(box)) + + if line.find("feat:") >= 0: + line = line.strip() + feat = line[line.find("feat:") + 5:].strip() + # if len(feat)==256: + feats.append(str_to_float_arr(feat)) + + if line.find("output_box:") >= 0: + line = line.strip() + # 确保 boxes 和 feats 一一对应，并可以保证 tboxes 和 tfeats 一一对应 + if len(boxes)==0 or len(boxes)!=len(feats): + continue + + box = str_to_float_arr(line[line.find("output_box:") + 11:].strip()) + + box.append(timestamp) + index = find_samebox_in_array(boxes, box) + if index >= 0: + tboxes.append(box) # 去掉'output_box:'并去除可能的空白字符 + + # feat_f = str_to_float_arr(input_feats[index]) + feat_f = feats[index] + norm_f = np.linalg.norm(feat_f) + feat_f = feat_f / norm_f + tfeats.append(feat_f) + + '''标志行(空行)判断''' + condt = line.find("timestamp")<0 and line.find("box:")<0 and line.find("feat:")<0 + if condt and newframe: + if len(tboxes) and len(tfeats): + trackerboxes = np.concatenate((trackerboxes, np.array(tboxes))) + trackerfeats = np.concatenate((trackerfeats, np.array(tfeats))) + + timestamp = -1 + boxes, feats, tboxes, tfeats = [], [], [], [] + newframe = False + + return trackerboxes, trackerfeats + + +def devide_motion_state(tboxes, width): + '''frameTstamp: 用于标记当前相机视野内用购物车运动状态变化 + + Hand状态： + 0: 不存在 + 1: 手部存在 + 2: 手部存在且处于某种状态（静止） + + ''' + + fboxes, frameTstamp = array2frame(tboxes) + + fnum = len(frameTstamp) + + state = np.zeros((fnum, 2), dtype=np.int64) + frameState = np.concatenate((frameTstamp, state), axis = 1).astype(np.int64) + handState = np.concatenate((frameTstamp, state), axis = 1).astype(np.int64) + + + + if fnum < width: + return frameState, handState + + mtrackFid = {} + handFid = {} + '''frameState 标记由图像判断的购物车状态：0: 静止，1: 运动''' + for idx in range(width, fnum+1): + idx0 = idx-width + + # if idx == 40: + # print("123") + + lboxes = np.concatenate(fboxes[idx0:idx], axis = 0) + md = MoveDetect(lboxes) + md.classify() + + ## track.during 二元素组, 表征在该时间片段内，轨迹 track 的起止时间，数值用 boxes[:, 7] + for track in md.track_motion: + f1, f2 = track.during + # if track.cls == 0: continue + idx1 = set(np.where(frameState[:,0] >= f1)[0]) + idx2 = set(np.where(frameState[:,0] <= f2)[0]) + idx3 = list(idx1.intersection(idx2)) + + if track.tid not in mtrackFid: + mtrackFid[track.tid] = set(idx3) + else: + mtrackFid[track.tid] = mtrackFid[track.tid].union(set(idx3)) + + frameState[idx-1, 3] = 1 + frameState[idx3, 2] = 1 + + for track in md.hand_tracks: + f11, f22 = track.during + idx11 = set(np.where(handState[:,0] >= f11)[0]) + idx22 = set(np.where(handState[:,0] <= f22)[0]) + idx33 = list(idx11.intersection(idx22)) + + '''手部存在标记''' + handState[idx33, 2] = 1 + '''未来改进方向：is_static 可以用手部状态判断的函数代替''' + if track.is_static(70) and len(idx33)>1: + idx11 = set(np.where(handState[:,0] >= f11)[0]) + idx22 = set(np.where(handState[:,0] <= f22)[0]) + idx33 = list(idx11.intersection(idx22)) + + '''手部静止标记''' + handState[idx33, 2] = 2 + + + '''状态变化输出''' + + for tid, fid in mtrackFid.items(): + fstate = np.zeros((fnum, 1), dtype=np.int64) + fstate[list(fid), 0] = tid + + frameState = np.concatenate((frameState, fstate), axis = 1).astype(np.int64) + + return frameState, handState + + +def state_measure(periods, weights, hands, spath=None): + '''两种状态：static、motion, + (t0, t1) + t0: static ----> motion + t1: motion ----> static + ''' + + PrevState = 'static' + CuurState = 'static' + + camtype_0, frstate_0 = periods[0] + camtype_1, frstate_1 = periods[1] + + '''计算总时间区间: tmin, tmax, during''' + tmin_w, tmax_w = np.min(weights[:, 0]), np.max(weights[:, 0]) + tmin_0, tmax_0 = np.min(frstate_0[:, 1]), np.max(frstate_0[:, 1]) + tmin_1, tmax_1 = np.min(frstate_1[:, 1]), np.max(frstate_1[:, 1]) + + tmin = min([tmin_w, tmin_0, tmin_1]) + tmax = max([tmax_w, tmax_0, tmax_1]) + + # for ctype, tboxes, _ in tracker_boxes: + # t_min, t_max = np.min(tboxes[:, 9]), np.max(tboxes[:, 9]) + # if t_mintmax: + # tmax = t_max + # during = tmax - tmin + + + + fig, (ax1, ax2, ax3) = plt.subplots(3, 1) + + ax1.plot(weights[:, 0] - tmin, weights[:, 1], 'bo-', linewidth=1, markersize=4) + # ax1.set_xlim([0, during]) + ax1.set_title('Weight (g)') + + ax2.plot(frstate_0[:, 1] - tmin, frstate_0[:, 2], 'rx-', linewidth=1, markersize=8) + ax2.plot(frstate_0[:, 1] - tmin, frstate_0[:, 3], 'bo-', linewidth=1, markersize=4) + # ax2.set_xlim([0, during]) + ax2.set_title(f'Camera: {int(camtype_0)}') + + ax3.plot(frstate_1[:, 1] - tmin, frstate_1[:, 2], 'rx-', linewidth=1, markersize=8) + ax3.plot(frstate_1[:, 1] - tmin, frstate_1[:, 3], 'bo-', linewidth=1, markersize=4) + ax3.set_title(f'Camera: {int(camtype_1)}') + + if spath: + plt.savefig(spath) + plt.show() + +def read_yolo_weight_data(eventdir): + filepaths = [] + for filename in os.listdir(eventdir): + file, ext = os.path.splitext(filename) + if ext =='.data': + filepath = os.path.join(eventdir, filename) + filepaths.append(filepath) + + if len(filepaths) != 5: + return + + tracker_boxes = [] + WeightDict, SensorDict, ProcessTimeDict = {}, {}, {} + for filepath in filepaths: + filename = os.path.basename(filepath) + + if filename.find('_track.data')>0: + CamerType = filename.split('_')[0] + trackerboxes, trackerfeats = extract_data_1(filepath) + tracker_boxes.append((CamerType, trackerboxes, trackerfeats)) + + if filename.find('process.data')==0: + WeightDict, SensorDict, ProcessTimeDict = read_weight_timeConsuming(filepath) + + '''====================重力信号处理====================''' + weights = [(float(t), w) for t, w in WeightDict.items()] + weights = np.array(weights) + + return tracker_boxes, weights + + +def main(): + eventdir = r"\\192.168.1.28\share\测试_202406\0819\images\20240817-192549-6940120c-634c-481b-97a6-65042729f86b_null" + + tracker_boxes, weights = read_yolo_weight_data(eventdir) + + '''====================图像运动分析====================''' + win_width = 12 + periods, hands = [], [] + for ctype, tboxes, _ in tracker_boxes: + period, handState = devide_motion_state(tboxes, win_width) + + periods.append((ctype, period)) + hands.append((ctype, handState)) + print('done!') + + '''===============重力、图像信息融合===================''' + state_measure(periods, weights, hands) + + +if __name__ == "__main__": + main() + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/realtime/full_realtime.py b/realtime/full_realtime.py new file mode 100644 index 0000000..ec5ef99 --- /dev/null +++ b/realtime/full_realtime.py @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- +""" +Created on Wed Dec 25 09:17:32 2024 + +@author: ym +""" +import os +import sys +import numpy as np +import matplotlib.pyplot as plt +from datetime import datetime + +sys.path.append(r"D:\DetectTracking") + +from contrast.utils.event import ShoppingEvent +from tracking.utils.read_data import read_weight_sensor, extract_data_realtime, read_tracking_output_realtime +from tracking.utils.read_data import read_process + +def time_std2stamp(custom_time): + date_part = custom_time.split('-')[0] + time_part = custom_time.split('-')[1] + ms_part = int(custom_time.split('-')[2]) + standard_time = f"{date_part} {time_part[:2]}:{time_part[2:4]}:{time_part[4:]}" + dt = datetime.strptime(standard_time, "%Y%m%d %H:%M:%S") + timestamp = int(dt.timestamp() * 1000) + ms_part + return timestamp + +def time_stamp2std(timestamp): + if isinstance(timestamp, float) or isinstance(timestamp, str): + timestamp = int(timestamp) + ms = timestamp%1000 + times = timestamp//1000 + std_time = datetime.fromtimestamp(times) + stdtime = std_time.strftime("%Y%m%d-%H%M%S") + '-' +str(ms) + return stdtime + +def get_timeduring_weight(procpath): + eventStart, eventEnd, weightValue = None, None, None + if os.path.isfile(procpath): + timeDict = read_process(procpath) + if "eventStart" in timeDict.keys(): + eventStart = timeDict["eventStart"] + if "eventEnd" in timeDict.keys(): + eventEnd = timeDict["eventEnd"] + if "weightValue" in timeDict.keys(): + weightValue = timeDict["weightValue"] + return eventStart, eventEnd, weightValue + +def event_devide(wpath): + ''' + 基于重力时序数据 _weight.data 进行事件切分 + ''' + # wpath = r'D:\全实时\source_data\2024122416\20241224-162658370_weight.data' + + tpath, _ = os.path.split(wpath) + + wsdata = read_weight_sensor(wpath) + times, weights = wsdata[:, 0], wsdata[:, 1] + Start, End = times[0], times[-1] + + evtpaths, evtTimeWeight = [], [] + for filename in os.listdir(tpath): + filelist = filename.split('_') + custom_time = filelist[0] + evtpath = os.path.join(tpath, filename) + if os.path.isdir(evtpath): + stamp = time_std2stamp(custom_time) + if stamp >= Start and stamp <= End: + evtpaths.append(evtpath) + + for evtpath in evtpaths: + evtname = os.path.basename(evtpath) + event = ShoppingEvent(evtpath, stype = "realtime") + # try: + # event = ShoppingEvent(evtpath, stype = "realtime") + # except Exception as e: + # print(f"Error is: {e}", evtname) + + '''读取事件的起止时间、重力变化值''' + propath = os.path.join(evtpath, "process.data") + evtStart, evtEnd, wgtValue = get_timeduring_weight(propath) + evtTimeWeight.append((evtStart, evtEnd, wgtValue)) + + + '''重力变化曲线、事件起止区间''' + fig, ax1 = plt.subplots(figsize=(16, 9), dpi=100) + ax1.plot(times-Start, weights, 'bo-', linewidth=1, markersize=3) + ax1.set_title('Weight (gram)') + for t0, t1, w in evtTimeWeight: + min_diff = float('inf') + index = None + for i, t in enumerate(times): + diff = abs(t0 - t) + if diff < min_diff: + min_diff = diff + index = i + w0 = weights[index] + w1 = w0 + w + ax1.plot((t0, t0) - Start, (w0, w1), 'r*-', linewidth=1, markersize=6) + ax1.plot((t1, t1) - Start, (w0, w1), 'r*-', linewidth=1, markersize=6) + ax1.plot((t0, t1) - Start, (w1, w1), 'r*-', linewidth=1, markersize=6) + ax1.grid(True) + plt.show() + + return plt + + + + + + + +def main(): + tpath = r"D:\全实时\source_data\2024122416" + rltpath = r"D:\全实时\result" + for filename in os.listdir(tpath): + bname = filename.split("_")[0] + if filename.find("_weight.data") <= 0: + continue + wpath = os.path.join(tpath, filename) + + plt = event_devide(wpath) + + plt.savefig(os.path.join(rltpath, f'{bname}.png' )) # svg, png, pdf + + print(filename) + + + + + + + + print("Done!") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/realtime/intrude_detect.py b/realtime/intrude_detect.py new file mode 100644 index 0000000..d9561b1 --- /dev/null +++ b/realtime/intrude_detect.py @@ -0,0 +1,420 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +Created on Tue Apr 8 10:07:17 2025 + +@author: wqg +""" + +import csv +import os +import platform +import sys +import pickle +import cv2 +import numpy as np +from pathlib import Path +import matplotlib.pyplot as plt +import matplotlib.pyplot as plt +from typing import List, Tuple +from scipy.spatial.distance import cdist +from scipy.spatial import ConvexHull +from shapely.geometry import Point, Polygon + +##################################################### for method: run_yrt() +FILE = Path(__file__).resolve() +ROOT = FILE.parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from track_reid import yolov10_resnet_tracker +from event_time_specify import devide_motion_state + + +def cross(o: Tuple[float, float], a: Tuple[float, float], b: Tuple[float, float]) -> float: + """ 计算向量 OA × OB 的叉积 """ + return (a[0] - o[0]) * (b[1] - o[1]) - (a[1] - o[1]) * (b[0] - o[0]) + +def compute_convex_hull(points: List[Tuple[float, float]]) -> List[Tuple[float, float]]: + """ 使用 Andrew's Monotone Chain 算法求二维点集的凸包 """ + points = sorted(set(points)) # 排序并去重 + if len(points) <= 1: + return points + + lower = [] + for p in points: + while len(lower) >= 2 and cross(lower[-2], lower[-1], p) <= 0: + lower.pop() + lower.append(p) + + upper = [] + for p in reversed(points): + while len(upper) >= 2 and cross(upper[-2], upper[-1], p) <= 0: + upper.pop() + upper.append(p) + + # 去掉重复的连接点 + return lower[:-1] + upper[:-1] + +def is_point_in_convex_hull(point: Tuple[float, float], hull: List[Tuple[float, float]]) -> bool: + """ 判断一个点是否在凸包（含边界）内 """ + n = len(hull) + if n < 3: + # 对于点或线段，直接判断是否共线或在线段上 + if n == 1: + return point == hull[0] + if n == 2: + a, b = hull + return abs(cross(a, b, point)) < 1e-10 and min(a[0], b[0]) <= point[0] <= max(a[0], b[0]) and min(a[1], b[1]) <= point[1] <= max(a[1], b[1]) + return False + + for i in range(n): + a = hull[i] + b = hull[(i + 1) % n] + if cross(a, b, point) < -1e-10: # 必须全部在左边或边上 + return False + return True + + + +def plot_convex_hull(points: List[Tuple[float, float]], hull: List[Tuple[float, float]], test_points: List[Tuple[float, float]] = None): + x_all, y_all = zip(*points) + fig, ax = plt.subplots() + + ax.set_xlim(0, 1024) + ax.set_ylim(1280, 0) + + ax.plot(x_all, y_all, 'o', label='Points') + + # 凸包闭环线 + hull_loop = hull + [hull[0]] + hx, hy = zip(*hull_loop) + ax.plot(hx, hy, 'r-', linewidth=2, label='Convex Hull') + + # 如果有测试点 + if test_points: + for pt in test_points: + color = 'green' if is_point_in_convex_hull(pt, hull) else 'black' + ax.plot(pt[0], pt[1], 's', color=color, markersize=8) + ax.text(pt[0] + 0.05, pt[1], f'{pt}', fontsize=9) + + ax.legend() + ax.grid(True) + plt.title("Convex Hull Visualization") + plt.show() + + +def convex_scipy(): + points = np.array([ + [0, 0], + [2, 0], + [1, 1], + [2, 2], + [0, 2], + [1, 0.5]]) + hull = ConvexHull(points) + + # 凸包顶点的索引 + print("凸包顶点索引:{}".format(hull.vertices)) + print("凸包顶点坐标:") + for i in hull.vertices: + print(points[i]) + + + # 将凸包坐标构造成 Polygon + hull_points = points[hull.vertices] + polygon = Polygon(hull_points) + + # 判断一个点是否在凸包内 + p = Point(1, 1) # 示例点 + print("是否在凸包内:", polygon.contains(p)) # True or False + + +def test_convex(): + # 测试数据 + sample_points = [(0, 0), (1, 1), (2, 2), (2, 0), (0, 2), (1, 0.5)] + convex_hull = compute_convex_hull(sample_points) + + # 测试点在凸包内 + test_point_inside = (1, 1) + test_point_outside = (3, 3) + test_point_on_edge = (1, 0) + + inside = is_point_in_convex_hull(test_point_inside, convex_hull) + outside = is_point_in_convex_hull(test_point_outside, convex_hull) + on_edge = is_point_in_convex_hull(test_point_on_edge, convex_hull) + + convex_hull, inside, outside, on_edge + + # 展示图像 + plot_convex_hull(sample_points, convex_hull, [test_point_inside, test_point_outside, test_point_on_edge]) + +def array2frame(tboxes): + "tboxes: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index]" + idx = np.where(tboxes[:, 6] != 0)[0] + bboxes = tboxes[idx, :] + frameID = np.sort(np.unique(bboxes[:, 7].astype(int))) + fboxes = [] + for fid in frameID: + idx = np.where(bboxes[:, 7] == fid)[0] + box = bboxes[idx, :] + fboxes.append(box) + return fboxes + +def convex_based(tboxes, width, TH=40): + fboxes = array2frame(tboxes) + fnum = len(fboxes) + + fids = np.array([i+1 for i in range(fnum)])[:, np.newaxis] + state = np.zeros((fnum, 1), dtype=np.int64) + frameState = np.concatenate((fids, state), axis = 1).astype(np.int64) + if fnum < width: + return frameState + + for idx1 in range(width, fnum+1): + idx0 = idx1 - width + idx = idx1 - width//2 - 1 + + iboxes = fboxes[:idx] + cboxes = fboxes[idx][:, 0:4] + + cur_xy = np.zeros((len(cboxes), 2)) + cur_xy[:, 0] = (fboxes[idx][:, 0]+fboxes[idx][:, 2])/2 + cur_xy[:, 1] = (fboxes[idx][:, 1]+fboxes[idx][:, 3])/2 + for i in range(width//2): + x1, y1, x2, y2 = iboxes[i][:, 0], iboxes[i][:, 1], iboxes[i][:, 2], iboxes[i][:, 3] + + boxes = np.array([(x1, y1), (x1, y2), (x2, y1), (x2, y2)]).transpose(0, 2, 1).reshape(-1, 2) + box1 = [(x, y) for x, y in boxes] + convex_hull = compute_convex_hull(box1) + + for pt in cur_xy: + inside = is_point_in_convex_hull(pt, convex_hull) + + if not inside: + break + if not inside: + break + + # Based on the distance between the four corners of the current frame boxes + # and adjacent frame boxes + iboxes = fboxes[idx0:idx] + fboxes[idx+1:idx1] + cboxes = fboxes[idx][:, 0:4] + cx1, cy1, cx2, cy2 = cboxes[:, 0], cboxes[:, 1], cboxes[:, 2], cboxes[:, 3] + cxy = np.array([(cx1, cy1), (cx1, cy2), (cx2, cy1), (cx2, cy2)]).transpose(0, 2, 1).reshape(-1, 2) + + iiboxes = np.concatenate(iboxes, axis=0) + ix1, iy1, ix2, iy2 = iiboxes[:, 0], iiboxes[:, 1], iiboxes[:, 2], iiboxes[:, 3] + ixy = np.array([(ix1, iy1), (ix1, iy2), (ix2, iy1), (ix2, iy2)]).transpose(0, 2, 1).reshape(-1, 2) + + Dist = cdist(cxy, ixy).round(2) + max_dist = np.max(np.min(Dist, axis=1)) + if max_dist > TH and not inside: + frameState[idx, 1] = 1 + # plot_convex_hull(boxes, convex_hull, [pt]) + frameState[idx, 1] = 1 + + return frameState + + +def single_point(tboxes, width, TH=60): + """width: window width, >=2""" + + + fboxes = array2frame(tboxes) + fnum = len(fboxes) + + fids = np.array([i+1 for i in range(fnum)])[:, np.newaxis] + state = np.zeros((fnum, 1), dtype=np.int64) + frameState = np.concatenate((fids, state), axis = 1).astype(np.int64) + + + if fnum < width: + return frameState + + for idx1 in range(width, fnum+1): + idx0 = idx1 - width + idx = idx1 - width//2 - 1 + + iboxe1 = fboxes[idx0:idx] + iboxe2 = fboxes[idx+1:idx1] + iboxes = fboxes[idx0:idx] + fboxes[idx+1:idx1] + + cboxes = fboxes[idx][:, 0:4] + cur_xy = np.zeros((len(cboxes), 2)) + cur_xy[:, 0] = (fboxes[idx][:, 0]+fboxes[idx][:, 2])/2 + cur_xy[:, 1] = (fboxes[idx][:, 1]+fboxes[idx][:, 3])/2 + Dist = np.empty((len(cboxes), 0)) + for i in range(width-1): + boxes = iboxes[i][:, 0:4] + + box_xy = np.zeros((len(boxes), 2)) + box_xy[:, 0] = (boxes[:, 0]+boxes[:, 2])/2 + box_xy[:, 1] = (boxes[:, 1]+boxes[:, 3])/2 + dist2 = cdist(cur_xy, box_xy).round(2) + + Dist = np.concatenate((Dist, dist2), axis=1) + + max_dist = np.max(np.min(Dist, axis=1)) + + if max_dist > TH: + frameState[idx, 1] = 1 + + return frameState + + + +def intrude(): + pkpath = Path("/home/wqg/dataset/small-goods/pkfiles") + savepath = Path("/home/wqg/dataset/small-goods/illustration_convex") + + if not savepath.exists(): + savepath.mkdir(parents=True, exist_ok=True) + + err_trail, err_single, err_all = [], [], [] + num = 0 + for pth in pkpath.iterdir(): + # item = r"69042386_20250407-145737_front_returnGood_b82d28427666_15_17700000001.pickle" + # pth = pkpath/item + + with open(str(pth), 'rb') as f: + yrt = pickle.load(f) + + evtname = pth.stem + + bboxes = [] + trackerboxes = np.empty((0, 10), dtype=np.float64) + for frameDict in yrt: + boxes = frameDict["bboxes"] + tboxes = frameDict["tboxes"] + tboxes = np.concatenate((tboxes, tboxes[:,7][:, None]), axis=1) + + bboxes.append(boxes) + + trackerboxes = np.concatenate((trackerboxes, np.array(tboxes)), axis=0) + + '''single-points based for intrusion detection''' + # wd =5 + # fstate1 = single_point(trackerboxes, wd) + + '''convex-based ''' + width = 5 + fstate = convex_based(trackerboxes, width, TH=60) + + # fstate = np.zeros(fstate1.shape) + # fstate[:, 0] = fstate1[:, 0] + # fstate[:, 1] = fstate1[:, 1] * fstate2[:, 1] + + '''trajectory based for intrusion detection + period: 0 1 2 3 + fid timestamp(fid) 基于滑动窗的tid扩展 滑动窗覆盖的运动区间 + ''' + win_width = 12 + period, handState = devide_motion_state(trackerboxes, win_width) + + num += 1 + if np.all(period[:,2:4]==0): + err_trail.append(evtname) + if np.all(fstate[:,1]==0): + err_single.append(evtname) + if np.all(period[:,2:4]==0) and np.all(fstate[:,1]==0): + err_all.append(evtname) + + fig, (ax1, ax2) = plt.subplots(2, 1) + ax1.plot(period[:, 1], period[:, 2], 'bo-', linewidth=1, markersize=4) + ax1.plot(period[:, 1], period[:, 3], 'rx-', linewidth=1, markersize=8) + + ax2.plot(fstate[:, 0], fstate[:, 1], 'rx-', linewidth=1, markersize=8) + plt.savefig(os.path.join(str(savepath), f"{evtname}.png")) + + + plt.close() + # if num==1: + # break + + rate_trail = 1 - len(err_trail)/num + rate_single = 1 - len(err_single)/num + rate_all = 1 - len(err_all)/num + + print(f"rate_trail: {rate_trail}") + print(f"rate_single: {rate_single}") + print(f"rate_all: {rate_all}") + + txtpath = savepath.parents[0] / "error.txt" + with open(str(txtpath), "w") as f: + f.write(f"rate_trail: {rate_trail}" + "\n") + f.write(f"rate_single: {rate_single}" + "\n") + f.write(f"rate_all: {rate_all}" + "\n") + + f.write("\n" + "err_trail" + "\n") + for line in err_trail: + f.write(line + "\n") + + f.write("\n" + "err_single" + "\n") + for line in err_single: + f.write(line + "\n") + + f.write("\n" + "err_all" + "\n") + for line in err_all: + f.write(line + "\n") + print("Done!") + + + + +def run_yrt(): + datapath = Path("/home/wqg/dataset/small-goods/videos/") + savepath = Path("/home/wqg/dataset/small-goods/result/") + pkpath = Path("/home/wqg/dataset/small-goods/pkfiles/") + + if not savepath.exists(): + savepath.mkdir(parents=True, exist_ok=True) + if not pkpath.exists(): + pkpath.mkdir(parents=True, exist_ok=True) + + + optdict = {} + optdict["weights"] = ROOT / 'ckpts/best_v10s_width0375_1205.pt' + optdict["is_save_img"] = False + optdict["is_save_video"] = True + + k = 0 + for pth in datapath.iterdir(): + item = "69042386_20250407-145819_back_returnGood_b82d28427666_15_17700000001.mp4" + pth = pth.parents[0] /item + + optdict["source"] = pth + optdict["save_dir"] = savepath + + # try: + yrtOut = yolov10_resnet_tracker(**optdict) + + pkpath_ = pkpath / f"{Path(pth).stem}.pickle" + with open(str(pkpath_), 'wb') as f: + pickle.dump(yrtOut, f) + + k += 1 + if k==1: + break + # except Exception as e: + # print("abc") + + + +if __name__ == '__main__': + # run_yrt() + + intrude() + + # test_convex() + + + + + + + + + + + diff --git a/realtime/time_devide.py b/realtime/time_devide.py new file mode 100644 index 0000000..d30bb1f --- /dev/null +++ b/realtime/time_devide.py @@ -0,0 +1,544 @@ +# -*- coding: utf-8 -*- +""" +Created on Wed Oct 16 17:37:07 2024 + +@author: ym +""" + +# import csv +import os +# import platform +# import sys +from pathlib import Path +import glob +import numpy as np +import copy +import matplotlib.pyplot as plt +from collections import OrderedDict + +from event_time_specify import devide_motion_state #, state_measure + +import sys + + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[1] # YOLOv5 root directory +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) # add ROOT to PATH + + + + + + +from imgs_inference import run_yolo +from tracking.utils.read_data import read_weight_sensor + +# IMG_FORMATS = 'bmp', 'dng', 'jpeg', 'jpg', 'mpo', 'png', 'tif', 'tiff', 'webp', 'pfm' # include image suffixes +# VID_FORMATS = 'asf', 'avi', 'gif', 'm4v', 'mkv', 'mov', 'mp4', 'mpeg', 'mpg', 'ts', 'wmv' # include video suffixes + +def filesort(p): + ''' + 需将图像文件名标准化 + ''' + + + files = [] + files.extend(sorted(glob.glob(os.path.join(p, '*.jpg')))) + # images = [x for x in files if x.split('.')[-1].lower() in IMG_FORMATS] + + tamps_0, tamps_1 = [], [] + files_0, files_1 = [], [] + for file in files: + basename = os.path.basename(file) + # if basename.find('frameId')<0: continue + + + f, ext = os.path.splitext(basename) + _, camer, tamp = f.split('_') + + if camer == '0': + tamps_0.append(int(tamp)) + files_0.append(file) + + if camer == '1': + tamps_1.append(int(tamp)) + files_1.append(file) + + idx0 = sorted(range(len(tamps_0)), key=lambda k: tamps_0[k]) + files0 = [files_0[i] for i in idx0] + + idx1 = sorted(range(len(tamps_1)), key=lambda k: tamps_1[k]) + files1 = [files_1[i] for i in idx1] + + + return (files0, files1) + + + + + +def rename(filePath, tmin): + """ + 重命名函数fun1 + 输入：文件夹路径 + 功能：对文件夹中的全部文件进行随机命名 + """ + suffix = '.png' # 设置后缀，筛选特定文件以更改名称 + for file in os.listdir(filePath): + if file.endswith(suffix): + name = file.split('.')[0] + + tamp = int(name.split('_')[2])-tmin + + suffix = file.split('.')[1] + + newname = name +f'_{int(tamp)}.'+suffix + + os.rename(os.path.join(filePath, file), os.path.join(filePath, newname)) + + +def rerename(filePath=None): + """ + 重命名函数fun1 + 输入：文件夹路径 + 功能：对文件夹中的全部文件进行随机命名 + """ + + suffix = '.png' # 设置后缀，筛选特定文件以更改名称 + for file in os.listdir(filePath): + if file.endswith(suffix): + name = file.split('.')[0] + names = name.split('_') + if len(names)>=6: + newname = "_".join(names[0:5])+'.png' + os.rename(os.path.join(filePath, file), os.path.join(filePath, newname)) + + +def state_measure(periods, weights, hands, spath=None): + ''' + 数据类型 + 后摄: 0, 前摄: 1, CV综合: 2, 重力: 9 + frstate_0/1: + 帧ID 时间戳 单摄状态1 单摄状态2 活动轨迹标记 + 0 1 2 3 4:end + + + + time_stream = np.concatenate((tstream, weights)) + time_stream: + 序列索引号 数据类型 时间戳 单摄状态1/重力值 单摄状态2/重力(t0, t1) CV状态/重力(t0', t1') 综合数据类型 综合状态 + 0 1 2 3 4 5 6 7 + + 单摄状态1：基于运动轨迹的起止点确定的运动区间 + 单摄状态2: 基于滑动窗口的起止点(窗口终点)确定的运动区间 + 重力(t0, t1): 重力波动的精确时间区间，基于重力波动的起止点，而不是仅依赖重力稳定时间 + 重力(t0', t1'): 根据加退购对重力波动窗口进行扩展，扩展应该涵盖购物事件的发生过程 + 方案： + 前后摄状态进行或运算 + CV与重力状态进行与运算 + ''' + + # BackType = 0 # 后摄数据类型 + # FrontType = 1 # 前摄数据类型 + CameraType = 2 # CV数据综合类型 + HandType = 3 # 手部类型 + WeightType = 9 # 重力数据类型 + WeightStableThresh = 7.5 # 单位：g，重力稳定状态下的最大波动范围 + WeightWinWidth = 10 # 单位：重力数据点数，该值和采样间隔关联，重力稳定时间设定为500ms = WeightWinWidth * 采样间隔 + CameraTimeInterval = 100 # 单位：ms，前后摄状态进行或运算时的时间间隔阈值 + InputFrontNum = 10 # 重力增加（放入）向前延拓的重力数据点数 + InputBackNum = 0 # 重力增加（放入）向后延拓的重力数据点数 + OutputFrontNum = 2 # 重力减少（取出）向前延拓的重力数据点数 + OutputBackNum = 10 # 重力减少（取出）向前延拓的重力数据点数 + CompTimeInterval = 150 # 单位：ms，CV状态和重力状态进行与运算时的时间间隔阈值 + + + '''==================== 1.1 Weight 数据综合并排序 =======================''' + nw = len(weights) + widx = np.array([k for k in range(0, nw)])[:, None] + wtype = WeightType * np.ones((nw, 1)) + wstate = np.zeros((nw, 4)) + weights = np.concatenate((widx, wtype, weights, wstate), axis=1).astype(np.int64) + weights[:, 6] = WeightType + + weights = weights[np.argsort(weights[:, 2]), :] + + '''=================== 1.2 基确Weight的状态切割 =========================''' + w_max = np.max(weights[:, 3]) + # i0=0 + for i2 in range(0, nw): + i1 = max(i2 - WeightWinWidth, 0) + wvalue = weights[i1:i2+1, 3] + wi2 = weights[i2, 3] + wmin = np.min(wvalue) + wmax = np.max(wvalue) + + + '''对重力波动区间进行标记，并标记最新一次重力稳定值的索引和相应重力值''' + if wmax - wmin > WeightStableThresh: + weights[i2, 4] = w_max + + if i2==0: + i0=0 + wi0 = weights[i0, 3] + elif i2>0 and weights[i2-1, 4]==0: + i0 = copy.deepcopy(i2) + wi0 = weights[i0, 3] + + + if i2>0 and weights[i2-1, 4]!=0 and weights[i2, 4]==0: + # 当前稳定状态下的重力值和前一次重力稳定值的差值，确定放入还是取出 + if wi2-wi0 > WeightStableThresh: + i00 = max(i0 - InputFrontNum, 0) + i22 = min(i2 + InputBackNum, nw) + elif wi2-wi0 < -1*WeightStableThresh: + i00 = max(i0 - OutputFrontNum, 0) + i22 = min(i2 + OutputBackNum, nw) + else: + i00 = max(i0 - max(InputFrontNum, OutputFrontNum), 0) + i22 = min(i2 + max(InputBackNum, OutputBackNum), nw) + + weights[i00:i22, 5] = w_max + 100 + + + '''===================== 2.1 CV 数据综合并排序 ==========================''' + BackType, frstate_0 = periods[0] + FrontType, frstate_1 = periods[1] + + + n0, n1 = len(frstate_0), len(frstate_1) + idx0 = np.array([i for i in range(0, n0)], dtype=np.int64)[:, None] + idx1 = np.array([i for i in range(0, n1)], dtype=np.int64)[:, None] + ctype0 = BackType * np.ones((n0, 1), dtype=np.int64) + ctype1 = FrontType * np.ones((n1, 1), dtype=np.int64) + tstamp0 = frstate_0[:,1][:, None] + tstamp1 = frstate_1[:,1][:, None] + state0 = frstate_0[:,2][:, None] + state00 = frstate_0[:,3][:, None] + + state1 = frstate_1[:,2][:, None] + state11 = frstate_1[:,3][:, None] + + + + '''序列索引号， 相机类型，时间戳, 单摄状态1、单摄状态2、CV综合状态、综合数据类型、综合状态 + 0 1 2 3 4 5 6 7 + ''' + tstream0 = np.concatenate((idx0, ctype0, tstamp0, state0, state00), axis=1) + tstream1 = np.concatenate((idx1, ctype1, tstamp1, state1, state11), axis=1) + tstream = np.concatenate((tstream0, tstream1), axis=0) + tstream = np.concatenate((tstream, np.zeros((len(tstream), 3), dtype=np.int64)), axis=1) + tstream[:, 6] = CameraType + tstream = tstream[np.argsort(tstream[:, 2]), :] + + '''=============== 2.2 基于前后摄运动轨迹起止点确定CV综合状态 ============''' + for i in range(0, len(tstream)): + idx, ctype, stamp, state = tstream[i, :4] + if i==0: + tstream[i, 5] = state + if i>0: + j = i-1 + idx0, ctype0, stamp0, state0 = tstream[j, :4] + while stamp-stamp0 < CameraTimeInterval and ctype == ctype0 and j>0: + j -= 1 + idx0, ctype0, stamp0, state0 = tstream[j, :4] + + '''两摄像头状态的或运算. 由于前后摄图像不同时，如何构造或运算，关键在于选择不同摄像头的对齐点 + i时刻摄像头(ctype)状态state，另一摄像头(ctype0 != ctype)距 i 最近最近时刻 j 的状态state0 + ''' + if ctype != ctype0 and state0==1: + tstream[i, 5] = state0 + else: + tstream[i, 5] = state + + + '''================ 3.1 CV、Wweight 数据综合并排序 ======================''' + time_stream = np.concatenate((tstream, weights), axis=0, dtype=np.int64) + time_stream = time_stream[np.argsort(time_stream[:, 2]), :] + tmin = np.min(time_stream[:, 2]) + time_stream[:, 2] = time_stream[:, 2] - tmin + + '''============== 3.2 基于 CV 和 Weight 确定 Cart 的综合状态 ============''' + for i in range(0, len(time_stream)): + idx, _, stamp, value, _, state, ctype = time_stream[i, :7] + state = min(state, 1) + + if i==0: + time_stream[i, 7] = state + + if i>0: + j = i-1 + idx0, _, stamp0, value0, _, state0, ctype0 = time_stream[j, :7] + while stamp-stamp0 < CompTimeInterval and ctype == ctype0 and j>0: + j -= 1 + idx0, _, stamp0, value0, _, state0, ctype0 = time_stream[j, :7] + + '''CV与Weight的与运算. 由于CV与Weight不同时，如何构造与运算，关键在于选择不同数据源的对齐点 + i时数据类型(ctype)状态state，另一数据类型(ctype0 != ctype)距 i 最近最近时刻 j 的状态state0 + ''' + if ctype != ctype0 and state !=0 and state0 !=0: + time_stream[i, 7] = 1 + + MotionSlice, motion_slice = [], [] + t0 = time_stream[0, 7] + for i in range(1, len(time_stream)): + f0 = time_stream[i-1, 7] + f1 = time_stream[i, 7] + if f0==0 and f1==1: + t0 = time_stream[i, 2] + elif f0==1 and f1==0: + t1 = time_stream[i, 2] + if t1-t0>100: #ms + MotionSlice.append((t0+tmin, t1+tmin)) + motion_slice.append((t0, t1)) + else: + print(f"T0: {t0}, T1: {t1}") + + + + + '''===================== 4. Hands数据综合并排序 ==========================''' + BackType, hdstate_0 = hands[0] + FrontType, hdstate_1 = hands[1] + n0, n1 = len(hdstate_0), len(hdstate_1) + idx0 = np.array([i for i in range(0, n0)], dtype=np.int64)[:, None] + idx1 = np.array([i for i in range(0, n1)], dtype=np.int64)[:, None] + ctype0 = BackType * np.ones((n0, 1), dtype=np.int64) + ctype1 = FrontType * np.ones((n1, 1), dtype=np.int64) + hstamp0 = hdstate_0[:,1][:, None] + hstamp1 = hdstate_1[:,1][:, None] + state0 = hdstate_0[:,2][:, None] + state1 = hdstate_1[:,2][:, None] + + '''序列索引号， 相机类型，时间戳, 单摄手部状态、手部综合状态、保留位2、综合数据类型、综合状态 + 0 1 2 3 4 5 6 7 + ''' + hstream0 = np.concatenate((idx0, ctype0, hstamp0, state0), axis=1) + hstream1 = np.concatenate((idx1, ctype1, hstamp1, state1), axis=1) + hstream = np.concatenate((hstream0, hstream1), axis=0) + hstream = np.concatenate((hstream, np.zeros((len(hstream), 4), dtype=np.int64)), axis=1) + hstream[:, 6] = HandType + hstream = hstream[np.argsort(hstream[:, 2]), :] + + for i in range(0, len(hstream)): + idx, ctype, stamp, state = hstream[i, :4] + if i==0: + hstream[i, 4] = state + if i>0: + j = i-1 + idx0, ctype0, stamp0, state0 = hstream[j, :4] + while stamp-stamp0 < CameraTimeInterval and ctype == ctype0 and j>0: + j -= 1 + idx0, ctype0, stamp0, state0 = hstream[j, :4] + + '''两摄像头状态的或运算. 由于前后摄图像不同时，如何构造或运算，关键在于选择不同摄像头的对齐点 + i时刻摄像头(ctype)状态state，另一摄像头(ctype0 != ctype)距 i 最近最近时刻 j 的状态state0 + ''' + if ctype != ctype0 and state0==2: + hstream[i, 4] = state0 + elif ctype != ctype0 and state0==1: + hstream[i, 4] = state0 + else: + hstream[i, 4] = state + + + + + + + '''========================== 5 结果显示 ================================''' + frstate_0[:, 1] = frstate_0[:, 1]-tmin + frstate_1[:, 1] = frstate_1[:, 1]-tmin + tstream[:, 2] = tstream[:, 2]-tmin + + fig, (ax1, ax2, ax3, ax4, ax5, ax6) = plt.subplots(6, 1) + during = np.max(time_stream[:, 2]) + + ax1.plot(weights[:, 2]-tmin, weights[:, 3], 'bo-', linewidth=1, markersize=4) + ax1.plot(weights[:, 2]-tmin, weights[:, 4], 'mx-', linewidth=1, markersize=4) + ax1.plot(weights[:, 2]-tmin, weights[:, 5], 'gx-', linewidth=1, markersize=4) + ax1.set_xlim([0, during]) + ax1.set_title('Weight (gram)') + + ax2.plot(frstate_0[:, 1], frstate_0[:, 3], 'rx-', linewidth=1, markersize=8) + ax2.plot(frstate_0[:, 1], frstate_0[:, 2], 'bo-', linewidth=1, markersize=4) + ax2.set_xlim([0, during]) + ax2.set_title('Back Camera') + + ax3.plot(frstate_1[:, 1], frstate_1[:, 3], 'rx-', linewidth=1, markersize=8) + ax3.plot(frstate_1[:, 1], frstate_1[:, 2], 'bo-', linewidth=1, markersize=4) + ax3.set_xlim([0, during]) + ax3.set_title('Front Camera') + + ax4.plot(tstream[:, 2], tstream[:, 5], 'bx-', linewidth=1, markersize=4) + ax4.set_xlim([0, during]) + ax4.set_title('CV State') + + ax5.plot(time_stream[:, 2], time_stream[:, 7], 'gx-', linewidth=1, markersize=4) + ax5.set_xlim([0, during]) + ax5.set_title('Cart State') + + ax6.plot(hstream[:, 2]-tmin, hstream[:, 4], 'gx-', linewidth=1, markersize=4) + ax6.set_xlim([0, during]) + ax6.set_title('Hand State') + + + plt.show() + if spath: + plt.savefig(spath) + + return tmin, MotionSlice + +def splitevent(imgpath, MotionSlice): + suffix = '.png' + + imgfiles = [f for f in os.listdir(imgpath) if f.endswith(suffix)] + timestamp = np.array([int(f.split('_')[2]) for f in imgfiles]) + + indexes = [] + k = 0 + for t0, t1 in MotionSlice: + idx0 = set(np.where(timestamp >= t0)[0]) + idx1 = set(np.where(timestamp <= t1)[0]) + idx2 = list(idx0.intersection(idx1)) + files = [imgfiles[i] for i in idx2] + + for filename in files: + file, ext = os.path.splitext(filename) + newname = file + f'_{k}.png' + os.rename(os.path.join(imgpath, filename), os.path.join(imgpath, newname)) + k += 1 + + print("Done!") + + + +def runyolo(): + eventdirs = r"\\192.168.1.28\share\个人文件\wqg\realtime\eventdata" + savedir = r"\\192.168.1.28\share\个人文件\wqg\realtime\result" + + k = 0 + for edir in os.listdir(eventdirs): + edir = "1731316835560" + source = os.path.join(eventdirs, edir) + files = filesort(source) + for flist in files: + run_yolo(flist, savedir) + + k += 1 + if k==1: + break + +def run_tracking(trackboxes, MotionSlice): + pass + + +def read_wsensor(filepath): + WeightDict = OrderedDict() + with open(filepath, 'r', encoding='utf-8') as f: + lines = f.readlines() + clean_lines = [line.strip().replace("'", '').replace('"', '') for line in lines] + for i, line in enumerate(clean_lines): + line = line.strip() + + line = line.strip() + + if line.find(':') < 0: continue + # if line.find("Weight") >= 0: + # label = "Weight" + # continue + + + keyword = line.split(':')[0] + value = line.split(':')[1] + + # if label == "Weight": + if len(keyword) and len(value): + vdata = [float(s) for s in value.split(',') if len(s)] + WeightDict[keyword] = vdata[-1] + + + weights = [(float(t), w) for t, w in WeightDict.items()] + weights = np.array(weights).astype(np.int64) + + return weights + + +def show_seri(): + datapath = r"\\192.168.1.28\share\个人文件\wqg\realtime\eventdata\1731316835560" + savedir = r"\\192.168.1.28\share\个人文件\wqg\realtime\1" + + + imgdir = datapath.split('\\')[-2] + "_" + datapath.split('\\')[-1] + imgpath = os.path.join(savedir, imgdir) + + eventname = Path(datapath).stem + + datafiles = sorted(glob.glob(os.path.join(datapath, '*.npy'))) + + periods, trackboxes, hands = [], [], [] + win_width = 12 + for npypath in datafiles: + CameraType = Path(npypath).stem.split('_')[-1] + tkboxes = np.load(npypath) + + trackboxes.append((CameraType, tkboxes)) + + period, handState = devide_motion_state(tkboxes, win_width) + periods.append((int(CameraType), period)) + hands.append((int(CameraType), handState)) + + + + '''===============读取重力信号数据===================''' + seneorfile = os.path.join(datapath, 'sensor.txt') + weights = read_wsensor(seneorfile) + + # weights = [(float(t), w) for t, w in WeightDict.items()] + # weights = np.array(weights) + + + '''===============重力、图像信息融合===================''' + spath = os.path.join(savedir, f"{eventname}.png" ) + tmin, MotionSlice = state_measure(periods, weights, hands, spath) + + # 第一次运行时用于更改图像文件名 + # rerename(imgpath) + # rename(imgpath, tmin) + + # splitevent(imgpath, MotionSlice) + + + + +def main(): + # runyolo() + show_seri() + +if __name__ == '__main__': + main() + + # imgpaths = r"\\192.168.1.28\share\realtime\result\eventdata_1728978106733" + # rerename(imgpaths) + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/realtime/tracker_test.py b/realtime/tracker_test.py new file mode 100644 index 0000000..7a1a4dc --- /dev/null +++ b/realtime/tracker_test.py @@ -0,0 +1,281 @@ +# -*- coding: utf-8 -*- +""" +Created on Sun Mar 2 14:15:57 2025 + +@author: ym +""" +import numpy as np +import cv2 +import os +from pathlib import Path + +import sys +sys.path.append(r"D:\DetectTracking") + +# from tracking.utils.read_data import extract_data_realtime, read_tracking_output_realtime +from tracking.utils.plotting import Annotator, colors +from tracking.utils import Boxes, IterableSimpleNamespace, yaml_load, boxes_add_fid +from tracking.trackers import BOTSORT, BYTETracker +from tracking.utils.showtrack import drawtracks +from hands.hand_inference import hand_pose +from tracking.utils.read_data import read_weight_sensor, extract_data_realtime, read_tracking_output_realtime +from contrast.feat_extract.config import config as conf +from contrast.feat_extract.inference import FeatsInterface +from tracking.utils.drawtracks import drawTrack + + +ReIDEncoder = FeatsInterface(conf) + + +W, H = 1024, 1280 +Mode = 'front' #'back' +ImgFormat = ['.jpg', '.jpeg', '.png', '.bmp'] + + +'''调用tracking()函数，利用本地跟踪算法获取各目标轨迹，可以比较本地跟踪算法与现场跟踪算法的区别。''' +def init_tracker(tracker_yaml = None, bs=1): + """ + Initialize tracker for object tracking during prediction. + """ + TRACKER_MAP = {'bytetrack': BYTETracker, 'botsort': BOTSORT} + cfg = IterableSimpleNamespace(**yaml_load(tracker_yaml)) + + tracker = TRACKER_MAP[cfg.tracker_type](args=cfg, frame_rate=30) + + return tracker + + +def init_trackers(tracker_yaml = None, bs=1): + """ + Initialize trackers for object tracking during prediction. + """ + # tracker_yaml = r"./tracking/trackers/cfg/botsort.yaml" + + TRACKER_MAP = {'bytetrack': BYTETracker, 'botsort': BOTSORT} + + cfg = IterableSimpleNamespace(**yaml_load(tracker_yaml)) + trackers = [] + for _ in range(bs): + tracker = TRACKER_MAP[cfg.tracker_type](args=cfg, frame_rate=30) + trackers.append(tracker) + + return trackers + + +def draw_box(img, tracks): + annotator = Annotator(img.copy(), line_width=2) + # for *xyxy, conf, cls in reversed(tracks): + # name = f'{int(cls)} {conf:.2f}' + # color = colors(int(cls), True) + # annotator.box_label(xyxy, name, color=color) + + for *xyxy, id, conf, cls, fid, bid in reversed(tracks): + name = f'ID:{int(id)} {int(cls)} {conf:.2f}' + color = colors(int(cls), True) + annotator.box_label(xyxy, name, color=color) + + + + im0 = annotator.result() + + return im0 + + + + +def tracking(bboxes, ffeats): + tracker_yaml = "./tracking/trackers/cfg/botsort.yaml" + + tracker = init_tracker(tracker_yaml) + + TrackBoxes = np.empty((0, 9), dtype = np.float32) + TracksDict = {} + + frmIds = [] + '''========================== 执行跟踪处理 =============================''' + # dets 与 feats 应保持严格对应 + k=0 + for dets, feats in zip(bboxes, ffeats): + + frmIds.append(np.unique(dets[:, 6]).astype(np.int64)[0]) + + boxes = dets[:, :6] + det_tracking = Boxes(boxes).cpu().numpy() + tracks, outfeats = tracker.update(det_tracking, features=feats) + '''tracks: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] + 0 1 2 3 4 5 6 7 8 + 这里，frame_index 也可以用视频的 帧ID 代替, box_index 保持不变 + ''' + k += 1 + imgpath = r"D:\全实时\202502\tracker\Yolos_Tracking\tracker\1_1740891284792\1_1740891284792_{}.png".format(int(k)) + img = cv2.imread(imgpath) + + im0 = draw_box(img, tracks) + savepath = r"D:\全实时\202502\tracker\Yolos_Tracking\tracker\1_1740891284792\b\1_1740891284792_{}_b.png".format(k) + cv2.imwrite(savepath, im0) + + + if len(tracks): + TrackBoxes = np.concatenate([TrackBoxes, tracks], axis=0) + +# ============================================================================= +# FeatDict = {} +# for track in tracks: +# tid = int(track[8]) +# FeatDict.update({tid: feats[tid, :]}) +# +# frameID = tracks[0, 7] +# +# # print(f"frameID: {int(frameID)}") +# assert len(tracks) == len(FeatDict), f"Please check the func: tracker.update() at frameID({int(frameID)})" +# +# TracksDict[f"frame_{int(frameID)}"] = {"feats":FeatDict} +# ============================================================================= + + + return TrackBoxes, TracksDict + +def dotrack(): + + datapath = r"D:\全实时\202502\tracker\1_tracker_in.data" + + bboxes, ffeats = extract_data_realtime(datapath) + trackerboxes, tracker_feat_dict = tracking(bboxes, ffeats) + + print("done!") + + +# def plotbox(): + +# fpath = r"\\192.168.1.28\share\测试视频数据以及日志\全实时测试\V12\2025-3-3\20250303-103833-338_6928804010091_6928804010091\1_tracking_output.data" +# imgpath = r"D:\全实时\202502\result\Yolos_Tracking\20250303-103833-338_6928804010091_6928804010091\1_1740969517953" +# trackingboxes, trackingfeats, tracking_outboxes, tracking_outfeats = read_tracking_output_realtime(fpath) + +# for *xyxy, id, conf, cls, fid, bid in tracking_outboxes[0]: +# imgname = f"1_1740969517953_{int(fid)}.png" + +# img_path = os.path.join(imgpath, imgname) +# img = cv2.imread(img_path) +# annotator = Annotator(img.copy(), line_width=2) + +# name = f'ID:{int(id)} {int(cls)} {conf:.2f}' +# color = colors(int(cls), True) +# annotator.box_label(xyxy, name, color=color) +# im0 = annotator.result() +# cv2.imwrite(os.path.join(imgpath, f"1_1740969517953_{int(fid)}_.png"), im0) + +# print(f"1_1740969676295_{int(fid)}_.png") + +# print("done") + +def video2imgs(videopath): + cap = cv2.VideoCapture(str(videopath)) + k = 0 + while True: + ret, frame = cap.read() + if frame is None: + break + k += 1 + imgpath = videopath.parent / f"{videopath.stem}_{k}.png" + cv2.imwrite(str(imgpath), frame) + + + +def extract_evtimgs(evtpath): + vidpaths = [v for v in evtpath.iterdir() if v.suffix == '.mp4'] + for vidpath in vidpaths: + video2imgs(vidpath) + + stamps = [name.stem.split('_')[1] for name in vidpaths] + + if len(set(stamps)==1): + return stamps[0] + return None + +def draw_tracking_boxes(evtpath, stamp): + for datapath in evtpath.iterdir(): + if datapath.name.find('_tracking_output.data')<=0: + continue + + camera = datapath.stem.split('_')[0] + trackingboxes, trackingfeats, tracking_outboxes, tracking_outfeats = read_tracking_output_realtime(str(datapath)) + + ## 该模块先读取轨迹数据，再根据帧ID读取相应图像 + for *xyxy, id, conf, cls, fid, bid in tracking_outboxes[0]: + imgpath = evtpath / f"{camera}_{stamp}_{int(fid)}.png" + + img = cv2.imread(str(imgpath)) + annotator = Annotator(img.copy(), line_width=2) + + name = f'ID:{int(id)} {int(cls)} {conf:.2f}' + color = colors(int(cls), True) + annotator.box_label(xyxy, name, color=color) + im0 = annotator.result() + cv2.imwrite(imgpath, im0) + + print(datapath.name) + +def draw_traj(evtpath): + for datapath in evtpath.iterdir(): + if datapath.name.find('_tracking_output.data')<=0: + continue + + fname = datapath.name + trackingboxes, trackingfeats, tracking_outboxes, tracking_outfeats = read_tracking_output_realtime(datapath) + + CamerType = fname.split('_')[0] + if CamerType == '1': + edgeline = cv2.imread("./CartTemp/board_ftmp_line.png") + if CamerType == '0': + edgeline = cv2.imread("./CartTemp/edgeline.png") + edgeline = drawTrack(tracking_outboxes, edgeline) + + + imgpath = datapath.parent / f"{datapath.stem}.png" + cv2.imwrite(str(imgpath), edgeline) + + +def main(): + + path = r"\\192.168.1.28\share\测试视频数据以及日志\全实时测试\V12\2025-3-3\20250303-104225-381_6920459958674" + evtpaths = [p for p in Path(path).iterdir() if p.is_dir()] + for evtpath in evtpaths: + #1. 从事件的前后摄视频提取图像 + stamp = extract_evtimgs(evtpath) + + #2. 根据 0/1_tracking_output.data 中提取的轨迹在img中绘制box + draw_tracking_boxes(evtpath, stamp) + + #3. 根据 0/1_tracking_output.data 中提取的轨迹在edgeline中绘制box + draw_traj(evtpath) + + + + +if __name__ == '__main__': + # dotrack() + # plotbox() + + vpath = r"D:\datasets\ym\VID_20250307_105606" + extract_evtimgs(Path(vpath)) + + + + + + + + + + + + + + + + + + + + + diff --git a/segment/predict.py b/segment/predict.py new file mode 100644 index 0000000..77e8df7 --- /dev/null +++ b/segment/predict.py @@ -0,0 +1,285 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Run YOLOv5 segmentation inference on images, videos, directories, streams, etc. + +Usage - sources: + $ python segment/predict.py --weights yolov5s-seg.pt --source 0 # webcam + img.jpg # image + vid.mp4 # video + screen # screenshot + path/ # directory + list.txt # list of images + list.streams # list of streams + 'path/*.jpg' # glob + 'https://youtu.be/Zgi9g1ksQHc' # YouTube + 'rtsp://example.com/media.mp4' # RTSP, RTMP, HTTP stream + +Usage - formats: + $ python segment/predict.py --weights yolov5s-seg.pt # PyTorch + yolov5s-seg.torchscript # TorchScript + yolov5s-seg.onnx # ONNX Runtime or OpenCV DNN with --dnn + yolov5s-seg_openvino_model # OpenVINO + yolov5s-seg.engine # TensorRT + yolov5s-seg.mlmodel # CoreML (macOS-only) + yolov5s-seg_saved_model # TensorFlow SavedModel + yolov5s-seg.pb # TensorFlow GraphDef + yolov5s-seg.tflite # TensorFlow Lite + yolov5s-seg_edgetpu.tflite # TensorFlow Edge TPU + yolov5s-seg_paddle_model # PaddlePaddle +""" + +import argparse +import os +import platform +import sys +from pathlib import Path + +import torch + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[1] # YOLOv5 root directory +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) # add ROOT to PATH +ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative + +from ultralytics.utils.plotting import Annotator, colors, save_one_box + +from models.common import DetectMultiBackend +from utils.dataloaders import IMG_FORMATS, VID_FORMATS, LoadImages, LoadScreenshots, LoadStreams +from utils.general import (LOGGER, Profile, check_file, check_img_size, check_imshow, check_requirements, colorstr, cv2, + increment_path, non_max_suppression, print_args, scale_boxes, scale_segments, + strip_optimizer) +from utils.segment.general import masks2segments, process_mask, process_mask_native +from utils.torch_utils import select_device, smart_inference_mode + + +@smart_inference_mode() +def run( + weights=ROOT / 'yolov5s-seg.pt', # model.pt path(s) + source=ROOT / 'data/images', # file/dir/URL/glob/screen/0(webcam) + data=ROOT / 'data/coco128.yaml', # dataset.yaml path + imgsz=(640, 640), # inference size (height, width) + conf_thres=0.25, # confidence threshold + iou_thres=0.45, # NMS IOU threshold + max_det=1000, # maximum detections per image + device='', # cuda device, i.e. 0 or 0,1,2,3 or cpu + view_img=False, # show results + save_txt=False, # save results to *.txt + save_conf=False, # save confidences in --save-txt labels + save_crop=False, # save cropped prediction boxes + nosave=False, # do not save images/videos + classes=None, # filter by class: --class 0, or --class 0 2 3 + agnostic_nms=False, # class-agnostic NMS + augment=False, # augmented inference + visualize=False, # visualize features + update=False, # update all models + project=ROOT / 'runs/predict-seg', # save results to project/name + name='exp', # save results to project/name + exist_ok=False, # existing project/name ok, do not increment + line_thickness=3, # bounding box thickness (pixels) + hide_labels=False, # hide labels + hide_conf=False, # hide confidences + half=False, # use FP16 half-precision inference + dnn=False, # use OpenCV DNN for ONNX inference + vid_stride=1, # video frame-rate stride + retina_masks=False, +): + source = str(source) + save_img = not nosave and not source.endswith('.txt') # save inference images + is_file = Path(source).suffix[1:] in (IMG_FORMATS + VID_FORMATS) + is_url = source.lower().startswith(('rtsp://', 'rtmp://', 'http://', 'https://')) + webcam = source.isnumeric() or source.endswith('.streams') or (is_url and not is_file) + screenshot = source.lower().startswith('screen') + if is_url and is_file: + source = check_file(source) # download + + # Directories + save_dir = increment_path(Path(project) / name, exist_ok=exist_ok) # increment run + (save_dir / 'labels' if save_txt else save_dir).mkdir(parents=True, exist_ok=True) # make dir + + # Load model + device = select_device(device) + model = DetectMultiBackend(weights, device=device, dnn=dnn, data=data, fp16=half) + stride, names, pt = model.stride, model.names, model.pt + imgsz = check_img_size(imgsz, s=stride) # check image size + + # Dataloader + bs = 1 # batch_size + if webcam: + view_img = check_imshow(warn=True) + dataset = LoadStreams(source, img_size=imgsz, stride=stride, auto=pt, vid_stride=vid_stride) + bs = len(dataset) + elif screenshot: + dataset = LoadScreenshots(source, img_size=imgsz, stride=stride, auto=pt) + else: + dataset = LoadImages(source, img_size=imgsz, stride=stride, auto=pt, vid_stride=vid_stride) + vid_path, vid_writer = [None] * bs, [None] * bs + + # Run inference + model.warmup(imgsz=(1 if pt else bs, 3, *imgsz)) # warmup + seen, windows, dt = 0, [], (Profile(), Profile(), Profile()) + for path, im, im0s, vid_cap, s in dataset: + with dt[0]: + im = torch.from_numpy(im).to(model.device) + im = im.half() if model.fp16 else im.float() # uint8 to fp16/32 + im /= 255 # 0 - 255 to 0.0 - 1.0 + if len(im.shape) == 3: + im = im[None] # expand for batch dim + + # Inference + with dt[1]: + visualize = increment_path(save_dir / Path(path).stem, mkdir=True) if visualize else False + pred, proto = model(im, augment=augment, visualize=visualize)[:2] + + # NMS + with dt[2]: + pred = non_max_suppression(pred, conf_thres, iou_thres, classes, agnostic_nms, max_det=max_det, nm=32) + + # Second-stage classifier (optional) + # pred = utils.general.apply_classifier(pred, classifier_model, im, im0s) + + # Process predictions + for i, det in enumerate(pred): # per image + seen += 1 + if webcam: # batch_size >= 1 + p, im0, frame = path[i], im0s[i].copy(), dataset.count + s += f'{i}: ' + else: + p, im0, frame = path, im0s.copy(), getattr(dataset, 'frame', 0) + + p = Path(p) # to Path + save_path = str(save_dir / p.name) # im.jpg + txt_path = str(save_dir / 'labels' / p.stem) + ('' if dataset.mode == 'image' else f'_{frame}') # im.txt + s += '%gx%g ' % im.shape[2:] # print string + imc = im0.copy() if save_crop else im0 # for save_crop + annotator = Annotator(im0, line_width=line_thickness, example=str(names)) + if len(det): + if retina_masks: + # scale bbox first the crop masks + det[:, :4] = scale_boxes(im.shape[2:], det[:, :4], im0.shape).round() # rescale boxes to im0 size + masks = process_mask_native(proto[i], det[:, 6:], det[:, :4], im0.shape[:2]) # HWC + else: + masks = process_mask(proto[i], det[:, 6:], det[:, :4], im.shape[2:], upsample=True) # HWC + det[:, :4] = scale_boxes(im.shape[2:], det[:, :4], im0.shape).round() # rescale boxes to im0 size + + # Segments + if save_txt: + segments = [ + scale_segments(im0.shape if retina_masks else im.shape[2:], x, im0.shape, normalize=True) + for x in reversed(masks2segments(masks))] + + # Print results + for c in det[:, 5].unique(): + n = (det[:, 5] == c).sum() # detections per class + s += f"{n} {names[int(c)]}{'s' * (n > 1)}, " # add to string + + # Mask plotting + annotator.masks( + masks, + colors=[colors(x, True) for x in det[:, 5]], + im_gpu=torch.as_tensor(im0, dtype=torch.float16).to(device).permute(2, 0, 1).flip(0).contiguous() / + 255 if retina_masks else im[i]) + + # Write results + for j, (*xyxy, conf, cls) in enumerate(reversed(det[:, :6])): + if save_txt: # Write to file + seg = segments[j].reshape(-1) # (n,2) to (n*2) + line = (cls, *seg, conf) if save_conf else (cls, *seg) # label format + with open(f'{txt_path}.txt', 'a') as f: + f.write(('%g ' * len(line)).rstrip() % line + '\n') + + if save_img or save_crop or view_img: # Add bbox to image + c = int(cls) # integer class + label = None if hide_labels else (names[c] if hide_conf else f'{names[c]} {conf:.2f}') + annotator.box_label(xyxy, label, color=colors(c, True)) + # annotator.draw.polygon(segments[j], outline=colors(c, True), width=3) + if save_crop: + save_one_box(xyxy, imc, file=save_dir / 'crops' / names[c] / f'{p.stem}.jpg', BGR=True) + + # Stream results + im0 = annotator.result() + if view_img: + if platform.system() == 'Linux' and p not in windows: + windows.append(p) + cv2.namedWindow(str(p), cv2.WINDOW_NORMAL | cv2.WINDOW_KEEPRATIO) # allow window resize (Linux) + cv2.resizeWindow(str(p), im0.shape[1], im0.shape[0]) + cv2.imshow(str(p), im0) + if cv2.waitKey(1) == ord('q'): # 1 millisecond + exit() + + # Save results (image with detections) + if save_img: + if dataset.mode == 'image': + cv2.imwrite(save_path, im0) + else: # 'video' or 'stream' + if vid_path[i] != save_path: # new video + vid_path[i] = save_path + if isinstance(vid_writer[i], cv2.VideoWriter): + vid_writer[i].release() # release previous video writer + if vid_cap: # video + fps = vid_cap.get(cv2.CAP_PROP_FPS) + w = int(vid_cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + h = int(vid_cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + else: # stream + fps, w, h = 30, im0.shape[1], im0.shape[0] + save_path = str(Path(save_path).with_suffix('.mp4')) # force *.mp4 suffix on results videos + vid_writer[i] = cv2.VideoWriter(save_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h)) + vid_writer[i].write(im0) + + # Print time (inference-only) + LOGGER.info(f"{s}{'' if len(det) else '(no detections), '}{dt[1].dt * 1E3:.1f}ms") + + # Print results + t = tuple(x.t / seen * 1E3 for x in dt) # speeds per image + LOGGER.info(f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {(1, 3, *imgsz)}' % t) + if save_txt or save_img: + s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else '' + LOGGER.info(f"Results saved to {colorstr('bold', save_dir)}{s}") + if update: + strip_optimizer(weights[0]) # update model (to fix SourceChangeWarning) + + +def parse_opt(): + parser = argparse.ArgumentParser() + parser.add_argument('--weights', nargs='+', type=str, default=ROOT / 'yolov5s-seg.pt', help='model path(s)') + parser.add_argument('--source', type=str, default=ROOT / 'data/images', help='file/dir/URL/glob/screen/0(webcam)') + parser.add_argument('--data', type=str, default=ROOT / 'data/coco128.yaml', help='(optional) dataset.yaml path') + parser.add_argument('--imgsz', '--img', '--img-size', nargs='+', type=int, default=[640], help='inference size h,w') + parser.add_argument('--conf-thres', type=float, default=0.25, help='confidence threshold') + parser.add_argument('--iou-thres', type=float, default=0.45, help='NMS IoU threshold') + parser.add_argument('--max-det', type=int, default=1000, help='maximum detections per image') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--view-img', action='store_true', help='show results') + parser.add_argument('--save-txt', action='store_true', help='save results to *.txt') + parser.add_argument('--save-conf', action='store_true', help='save confidences in --save-txt labels') + parser.add_argument('--save-crop', action='store_true', help='save cropped prediction boxes') + parser.add_argument('--nosave', action='store_true', help='do not save images/videos') + parser.add_argument('--classes', nargs='+', type=int, help='filter by class: --classes 0, or --classes 0 2 3') + parser.add_argument('--agnostic-nms', action='store_true', help='class-agnostic NMS') + parser.add_argument('--augment', action='store_true', help='augmented inference') + parser.add_argument('--visualize', action='store_true', help='visualize features') + parser.add_argument('--update', action='store_true', help='update all models') + parser.add_argument('--project', default=ROOT / 'runs/predict-seg', help='save results to project/name') + parser.add_argument('--name', default='exp', help='save results to project/name') + parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment') + parser.add_argument('--line-thickness', default=3, type=int, help='bounding box thickness (pixels)') + parser.add_argument('--hide-labels', default=False, action='store_true', help='hide labels') + parser.add_argument('--hide-conf', default=False, action='store_true', help='hide confidences') + parser.add_argument('--half', action='store_true', help='use FP16 half-precision inference') + parser.add_argument('--dnn', action='store_true', help='use OpenCV DNN for ONNX inference') + parser.add_argument('--vid-stride', type=int, default=1, help='video frame-rate stride') + parser.add_argument('--retina-masks', action='store_true', help='whether to plot masks in native resolution') + opt = parser.parse_args() + opt.imgsz *= 2 if len(opt.imgsz) == 1 else 1 # expand + print_args(vars(opt)) + return opt + + +def main(opt): + check_requirements(ROOT / 'requirements.txt', exclude=('tensorboard', 'thop')) + run(**vars(opt)) + + +if __name__ == '__main__': + opt = parse_opt() + main(opt) diff --git a/segment/train.py b/segment/train.py new file mode 100644 index 0000000..2ae09c1 --- /dev/null +++ b/segment/train.py @@ -0,0 +1,666 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Train a YOLOv5 segment model on a segment dataset +Models and datasets download automatically from the latest YOLOv5 release. + +Usage - Single-GPU training: + $ python segment/train.py --data coco128-seg.yaml --weights yolov5s-seg.pt --img 640 # from pretrained (recommended) + $ python segment/train.py --data coco128-seg.yaml --weights '' --cfg yolov5s-seg.yaml --img 640 # from scratch + +Usage - Multi-GPU DDP training: + $ python -m torch.distributed.run --nproc_per_node 4 --master_port 1 segment/train.py --data coco128-seg.yaml --weights yolov5s-seg.pt --img 640 --device 0,1,2,3 + +Models: https://github.com/ultralytics/yolov5/tree/master/models +Datasets: https://github.com/ultralytics/yolov5/tree/master/data +Tutorial: https://docs.ultralytics.com/yolov5/tutorials/train_custom_data +""" + +import argparse +import math +import os +import random +import subprocess +import sys +import time +from copy import deepcopy +from datetime import datetime +from pathlib import Path + +import numpy as np +import torch +import torch.distributed as dist +import torch.nn as nn +import yaml +from torch.optim import lr_scheduler +from tqdm import tqdm + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[1] # YOLOv5 root directory +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) # add ROOT to PATH +ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative + +import segment.val as validate # for end-of-epoch mAP +from models.experimental import attempt_load +from models.yolo import SegmentationModel +from utils.autoanchor import check_anchors +from utils.autobatch import check_train_batch_size +from utils.callbacks import Callbacks +from utils.downloads import attempt_download, is_url +from utils.general import (LOGGER, TQDM_BAR_FORMAT, check_amp, check_dataset, check_file, check_git_info, + check_git_status, check_img_size, check_requirements, check_suffix, check_yaml, colorstr, + get_latest_run, increment_path, init_seeds, intersect_dicts, labels_to_class_weights, + labels_to_image_weights, one_cycle, print_args, print_mutation, strip_optimizer, yaml_save) +from utils.loggers import GenericLogger +from utils.plots import plot_evolve, plot_labels +from utils.segment.dataloaders import create_dataloader +from utils.segment.loss import ComputeLoss +from utils.segment.metrics import KEYS, fitness +from utils.segment.plots import plot_images_and_masks, plot_results_with_masks +from utils.torch_utils import (EarlyStopping, ModelEMA, de_parallel, select_device, smart_DDP, smart_optimizer, + smart_resume, torch_distributed_zero_first) + +LOCAL_RANK = int(os.getenv('LOCAL_RANK', -1)) # https://pytorch.org/docs/stable/elastic/run.html +RANK = int(os.getenv('RANK', -1)) +WORLD_SIZE = int(os.getenv('WORLD_SIZE', 1)) +GIT_INFO = check_git_info() + + +def train(hyp, opt, device, callbacks): # hyp is path/to/hyp.yaml or hyp dictionary + save_dir, epochs, batch_size, weights, single_cls, evolve, data, cfg, resume, noval, nosave, workers, freeze, mask_ratio = \ + Path(opt.save_dir), opt.epochs, opt.batch_size, opt.weights, opt.single_cls, opt.evolve, opt.data, opt.cfg, \ + opt.resume, opt.noval, opt.nosave, opt.workers, opt.freeze, opt.mask_ratio + # callbacks.run('on_pretrain_routine_start') + + # Directories + w = save_dir / 'weights' # weights dir + (w.parent if evolve else w).mkdir(parents=True, exist_ok=True) # make dir + last, best = w / 'last.pt', w / 'best.pt' + + # Hyperparameters + if isinstance(hyp, str): + with open(hyp, errors='ignore') as f: + hyp = yaml.safe_load(f) # load hyps dict + LOGGER.info(colorstr('hyperparameters: ') + ', '.join(f'{k}={v}' for k, v in hyp.items())) + opt.hyp = hyp.copy() # for saving hyps to checkpoints + + # Save run settings + if not evolve: + yaml_save(save_dir / 'hyp.yaml', hyp) + yaml_save(save_dir / 'opt.yaml', vars(opt)) + + # Loggers + data_dict = None + if RANK in {-1, 0}: + logger = GenericLogger(opt=opt, console_logger=LOGGER) + + # Config + plots = not evolve and not opt.noplots # create plots + overlap = not opt.no_overlap + cuda = device.type != 'cpu' + init_seeds(opt.seed + 1 + RANK, deterministic=True) + with torch_distributed_zero_first(LOCAL_RANK): + data_dict = data_dict or check_dataset(data) # check if None + train_path, val_path = data_dict['train'], data_dict['val'] + nc = 1 if single_cls else int(data_dict['nc']) # number of classes + names = {0: 'item'} if single_cls and len(data_dict['names']) != 1 else data_dict['names'] # class names + is_coco = isinstance(val_path, str) and val_path.endswith('coco/val2017.txt') # COCO dataset + + # Model + check_suffix(weights, '.pt') # check weights + pretrained = weights.endswith('.pt') + if pretrained: + with torch_distributed_zero_first(LOCAL_RANK): + weights = attempt_download(weights) # download if not found locally + ckpt = torch.load(weights, map_location='cpu') # load checkpoint to CPU to avoid CUDA memory leak + model = SegmentationModel(cfg or ckpt['model'].yaml, ch=3, nc=nc, anchors=hyp.get('anchors')).to(device) + exclude = ['anchor'] if (cfg or hyp.get('anchors')) and not resume else [] # exclude keys + csd = ckpt['model'].float().state_dict() # checkpoint state_dict as FP32 + csd = intersect_dicts(csd, model.state_dict(), exclude=exclude) # intersect + model.load_state_dict(csd, strict=False) # load + LOGGER.info(f'Transferred {len(csd)}/{len(model.state_dict())} items from {weights}') # report + else: + model = SegmentationModel(cfg, ch=3, nc=nc, anchors=hyp.get('anchors')).to(device) # create + amp = check_amp(model) # check AMP + + # Freeze + freeze = [f'model.{x}.' for x in (freeze if len(freeze) > 1 else range(freeze[0]))] # layers to freeze + for k, v in model.named_parameters(): + v.requires_grad = True # train all layers + # v.register_hook(lambda x: torch.nan_to_num(x)) # NaN to 0 (commented for erratic training results) + if any(x in k for x in freeze): + LOGGER.info(f'freezing {k}') + v.requires_grad = False + + # Image size + gs = max(int(model.stride.max()), 32) # grid size (max stride) + imgsz = check_img_size(opt.imgsz, gs, floor=gs * 2) # verify imgsz is gs-multiple + + # Batch size + if RANK == -1 and batch_size == -1: # single-GPU only, estimate best batch size + batch_size = check_train_batch_size(model, imgsz, amp) + logger.update_params({'batch_size': batch_size}) + # loggers.on_params_update({"batch_size": batch_size}) + + # Optimizer + nbs = 64 # nominal batch size + accumulate = max(round(nbs / batch_size), 1) # accumulate loss before optimizing + hyp['weight_decay'] *= batch_size * accumulate / nbs # scale weight_decay + optimizer = smart_optimizer(model, opt.optimizer, hyp['lr0'], hyp['momentum'], hyp['weight_decay']) + + # Scheduler + if opt.cos_lr: + lf = one_cycle(1, hyp['lrf'], epochs) # cosine 1->hyp['lrf'] + else: + lf = lambda x: (1 - x / epochs) * (1.0 - hyp['lrf']) + hyp['lrf'] # linear + scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lf) # plot_lr_scheduler(optimizer, scheduler, epochs) + + # EMA + ema = ModelEMA(model) if RANK in {-1, 0} else None + + # Resume + best_fitness, start_epoch = 0.0, 0 + if pretrained: + if resume: + best_fitness, start_epoch, epochs = smart_resume(ckpt, optimizer, ema, weights, epochs, resume) + del ckpt, csd + + # DP mode + if cuda and RANK == -1 and torch.cuda.device_count() > 1: + LOGGER.warning( + 'WARNING ⚠️ DP not recommended, use torch.distributed.run for best DDP Multi-GPU results.\n' + 'See Multi-GPU Tutorial at https://docs.ultralytics.com/yolov5/tutorials/multi_gpu_training to get started.' + ) + model = torch.nn.DataParallel(model) + + # SyncBatchNorm + if opt.sync_bn and cuda and RANK != -1: + model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model).to(device) + LOGGER.info('Using SyncBatchNorm()') + + # Trainloader + train_loader, dataset = create_dataloader( + train_path, + imgsz, + batch_size // WORLD_SIZE, + gs, + single_cls, + hyp=hyp, + augment=True, + cache=None if opt.cache == 'val' else opt.cache, + rect=opt.rect, + rank=LOCAL_RANK, + workers=workers, + image_weights=opt.image_weights, + quad=opt.quad, + prefix=colorstr('train: '), + shuffle=True, + mask_downsample_ratio=mask_ratio, + overlap_mask=overlap, + ) + labels = np.concatenate(dataset.labels, 0) + mlc = int(labels[:, 0].max()) # max label class + assert mlc < nc, f'Label class {mlc} exceeds nc={nc} in {data}. Possible class labels are 0-{nc - 1}' + + # Process 0 + if RANK in {-1, 0}: + val_loader = create_dataloader(val_path, + imgsz, + batch_size // WORLD_SIZE * 2, + gs, + single_cls, + hyp=hyp, + cache=None if noval else opt.cache, + rect=True, + rank=-1, + workers=workers * 2, + pad=0.5, + mask_downsample_ratio=mask_ratio, + overlap_mask=overlap, + prefix=colorstr('val: '))[0] + + if not resume: + if not opt.noautoanchor: + check_anchors(dataset, model=model, thr=hyp['anchor_t'], imgsz=imgsz) # run AutoAnchor + model.half().float() # pre-reduce anchor precision + + if plots: + plot_labels(labels, names, save_dir) + # callbacks.run('on_pretrain_routine_end', labels, names) + + # DDP mode + if cuda and RANK != -1: + model = smart_DDP(model) + + # Model attributes + nl = de_parallel(model).model[-1].nl # number of detection layers (to scale hyps) + hyp['box'] *= 3 / nl # scale to layers + hyp['cls'] *= nc / 80 * 3 / nl # scale to classes and layers + hyp['obj'] *= (imgsz / 640) ** 2 * 3 / nl # scale to image size and layers + hyp['label_smoothing'] = opt.label_smoothing + model.nc = nc # attach number of classes to model + model.hyp = hyp # attach hyperparameters to model + model.class_weights = labels_to_class_weights(dataset.labels, nc).to(device) * nc # attach class weights + model.names = names + + # Start training + t0 = time.time() + nb = len(train_loader) # number of batches + nw = max(round(hyp['warmup_epochs'] * nb), 100) # number of warmup iterations, max(3 epochs, 100 iterations) + # nw = min(nw, (epochs - start_epoch) / 2 * nb) # limit warmup to < 1/2 of training + last_opt_step = -1 + maps = np.zeros(nc) # mAP per class + results = (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) # P, R, mAP@.5, mAP@.5-.95, val_loss(box, obj, cls) + scheduler.last_epoch = start_epoch - 1 # do not move + scaler = torch.cuda.amp.GradScaler(enabled=amp) + stopper, stop = EarlyStopping(patience=opt.patience), False + compute_loss = ComputeLoss(model, overlap=overlap) # init loss class + # callbacks.run('on_train_start') + LOGGER.info(f'Image sizes {imgsz} train, {imgsz} val\n' + f'Using {train_loader.num_workers * WORLD_SIZE} dataloader workers\n' + f"Logging results to {colorstr('bold', save_dir)}\n" + f'Starting training for {epochs} epochs...') + for epoch in range(start_epoch, epochs): # epoch ------------------------------------------------------------------ + # callbacks.run('on_train_epoch_start') + model.train() + + # Update image weights (optional, single-GPU only) + if opt.image_weights: + cw = model.class_weights.cpu().numpy() * (1 - maps) ** 2 / nc # class weights + iw = labels_to_image_weights(dataset.labels, nc=nc, class_weights=cw) # image weights + dataset.indices = random.choices(range(dataset.n), weights=iw, k=dataset.n) # rand weighted idx + + # Update mosaic border (optional) + # b = int(random.uniform(0.25 * imgsz, 0.75 * imgsz + gs) // gs * gs) + # dataset.mosaic_border = [b - imgsz, -b] # height, width borders + + mloss = torch.zeros(4, device=device) # mean losses + if RANK != -1: + train_loader.sampler.set_epoch(epoch) + pbar = enumerate(train_loader) + LOGGER.info(('\n' + '%11s' * 8) % + ('Epoch', 'GPU_mem', 'box_loss', 'seg_loss', 'obj_loss', 'cls_loss', 'Instances', 'Size')) + if RANK in {-1, 0}: + pbar = tqdm(pbar, total=nb, bar_format=TQDM_BAR_FORMAT) # progress bar + optimizer.zero_grad() + for i, (imgs, targets, paths, _, masks) in pbar: # batch ------------------------------------------------------ + # callbacks.run('on_train_batch_start') + ni = i + nb * epoch # number integrated batches (since train start) + imgs = imgs.to(device, non_blocking=True).float() / 255 # uint8 to float32, 0-255 to 0.0-1.0 + + # Warmup + if ni <= nw: + xi = [0, nw] # x interp + # compute_loss.gr = np.interp(ni, xi, [0.0, 1.0]) # iou loss ratio (obj_loss = 1.0 or iou) + accumulate = max(1, np.interp(ni, xi, [1, nbs / batch_size]).round()) + for j, x in enumerate(optimizer.param_groups): + # bias lr falls from 0.1 to lr0, all other lrs rise from 0.0 to lr0 + x['lr'] = np.interp(ni, xi, [hyp['warmup_bias_lr'] if j == 0 else 0.0, x['initial_lr'] * lf(epoch)]) + if 'momentum' in x: + x['momentum'] = np.interp(ni, xi, [hyp['warmup_momentum'], hyp['momentum']]) + + # Multi-scale + if opt.multi_scale: + sz = random.randrange(int(imgsz * 0.5), int(imgsz * 1.5) + gs) // gs * gs # size + sf = sz / max(imgs.shape[2:]) # scale factor + if sf != 1: + ns = [math.ceil(x * sf / gs) * gs for x in imgs.shape[2:]] # new shape (stretched to gs-multiple) + imgs = nn.functional.interpolate(imgs, size=ns, mode='bilinear', align_corners=False) + + # Forward + with torch.cuda.amp.autocast(amp): + pred = model(imgs) # forward + loss, loss_items = compute_loss(pred, targets.to(device), masks=masks.to(device).float()) + if RANK != -1: + loss *= WORLD_SIZE # gradient averaged between devices in DDP mode + if opt.quad: + loss *= 4. + + # Backward + scaler.scale(loss).backward() + + # Optimize - https://pytorch.org/docs/master/notes/amp_examples.html + if ni - last_opt_step >= accumulate: + scaler.unscale_(optimizer) # unscale gradients + torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=10.0) # clip gradients + scaler.step(optimizer) # optimizer.step + scaler.update() + optimizer.zero_grad() + if ema: + ema.update(model) + last_opt_step = ni + + # Log + if RANK in {-1, 0}: + mloss = (mloss * i + loss_items) / (i + 1) # update mean losses + mem = f'{torch.cuda.memory_reserved() / 1E9 if torch.cuda.is_available() else 0:.3g}G' # (GB) + pbar.set_description(('%11s' * 2 + '%11.4g' * 6) % + (f'{epoch}/{epochs - 1}', mem, *mloss, targets.shape[0], imgs.shape[-1])) + # callbacks.run('on_train_batch_end', model, ni, imgs, targets, paths) + # if callbacks.stop_training: + # return + + # Mosaic plots + if plots: + if ni < 3: + plot_images_and_masks(imgs, targets, masks, paths, save_dir / f'train_batch{ni}.jpg') + if ni == 10: + files = sorted(save_dir.glob('train*.jpg')) + logger.log_images(files, 'Mosaics', epoch) + # end batch ------------------------------------------------------------------------------------------------ + + # Scheduler + lr = [x['lr'] for x in optimizer.param_groups] # for loggers + scheduler.step() + + if RANK in {-1, 0}: + # mAP + # callbacks.run('on_train_epoch_end', epoch=epoch) + ema.update_attr(model, include=['yaml', 'nc', 'hyp', 'names', 'stride', 'class_weights']) + final_epoch = (epoch + 1 == epochs) or stopper.possible_stop + if not noval or final_epoch: # Calculate mAP + results, maps, _ = validate.run(data_dict, + batch_size=batch_size // WORLD_SIZE * 2, + imgsz=imgsz, + half=amp, + model=ema.ema, + single_cls=single_cls, + dataloader=val_loader, + save_dir=save_dir, + plots=False, + callbacks=callbacks, + compute_loss=compute_loss, + mask_downsample_ratio=mask_ratio, + overlap=overlap) + + # Update best mAP + fi = fitness(np.array(results).reshape(1, -1)) # weighted combination of [P, R, mAP@.5, mAP@.5-.95] + stop = stopper(epoch=epoch, fitness=fi) # early stop check + if fi > best_fitness: + best_fitness = fi + log_vals = list(mloss) + list(results) + lr + # callbacks.run('on_fit_epoch_end', log_vals, epoch, best_fitness, fi) + # Log val metrics and media + metrics_dict = dict(zip(KEYS, log_vals)) + logger.log_metrics(metrics_dict, epoch) + + # Save model + if (not nosave) or (final_epoch and not evolve): # if save + ckpt = { + 'epoch': epoch, + 'best_fitness': best_fitness, + 'model': deepcopy(de_parallel(model)).half(), + 'ema': deepcopy(ema.ema).half(), + 'updates': ema.updates, + 'optimizer': optimizer.state_dict(), + 'opt': vars(opt), + 'git': GIT_INFO, # {remote, branch, commit} if a git repo + 'date': datetime.now().isoformat()} + + # Save last, best and delete + torch.save(ckpt, last) + if best_fitness == fi: + torch.save(ckpt, best) + if opt.save_period > 0 and epoch % opt.save_period == 0: + torch.save(ckpt, w / f'epoch{epoch}.pt') + logger.log_model(w / f'epoch{epoch}.pt') + del ckpt + # callbacks.run('on_model_save', last, epoch, final_epoch, best_fitness, fi) + + # EarlyStopping + if RANK != -1: # if DDP training + broadcast_list = [stop if RANK == 0 else None] + dist.broadcast_object_list(broadcast_list, 0) # broadcast 'stop' to all ranks + if RANK != 0: + stop = broadcast_list[0] + if stop: + break # must break all DDP ranks + + # end epoch ---------------------------------------------------------------------------------------------------- + # end training ----------------------------------------------------------------------------------------------------- + if RANK in {-1, 0}: + LOGGER.info(f'\n{epoch - start_epoch + 1} epochs completed in {(time.time() - t0) / 3600:.3f} hours.') + for f in last, best: + if f.exists(): + strip_optimizer(f) # strip optimizers + if f is best: + LOGGER.info(f'\nValidating {f}...') + results, _, _ = validate.run( + data_dict, + batch_size=batch_size // WORLD_SIZE * 2, + imgsz=imgsz, + model=attempt_load(f, device).half(), + iou_thres=0.65 if is_coco else 0.60, # best pycocotools at iou 0.65 + single_cls=single_cls, + dataloader=val_loader, + save_dir=save_dir, + save_json=is_coco, + verbose=True, + plots=plots, + callbacks=callbacks, + compute_loss=compute_loss, + mask_downsample_ratio=mask_ratio, + overlap=overlap) # val best model with plots + if is_coco: + # callbacks.run('on_fit_epoch_end', list(mloss) + list(results) + lr, epoch, best_fitness, fi) + metrics_dict = dict(zip(KEYS, list(mloss) + list(results) + lr)) + logger.log_metrics(metrics_dict, epoch) + + # callbacks.run('on_train_end', last, best, epoch, results) + # on train end callback using genericLogger + logger.log_metrics(dict(zip(KEYS[4:16], results)), epochs) + if not opt.evolve: + logger.log_model(best, epoch) + if plots: + plot_results_with_masks(file=save_dir / 'results.csv') # save results.png + files = ['results.png', 'confusion_matrix.png', *(f'{x}_curve.png' for x in ('F1', 'PR', 'P', 'R'))] + files = [(save_dir / f) for f in files if (save_dir / f).exists()] # filter + LOGGER.info(f"Results saved to {colorstr('bold', save_dir)}") + logger.log_images(files, 'Results', epoch + 1) + logger.log_images(sorted(save_dir.glob('val*.jpg')), 'Validation', epoch + 1) + torch.cuda.empty_cache() + return results + + +def parse_opt(known=False): + parser = argparse.ArgumentParser() + parser.add_argument('--weights', type=str, default=ROOT / 'yolov5s-seg.pt', help='initial weights path') + parser.add_argument('--cfg', type=str, default='', help='model.yaml path') + parser.add_argument('--data', type=str, default=ROOT / 'data/coco128-seg.yaml', help='dataset.yaml path') + parser.add_argument('--hyp', type=str, default=ROOT / 'data/hyps/hyp.scratch-low.yaml', help='hyperparameters path') + parser.add_argument('--epochs', type=int, default=100, help='total training epochs') + parser.add_argument('--batch-size', type=int, default=16, help='total batch size for all GPUs, -1 for autobatch') + parser.add_argument('--imgsz', '--img', '--img-size', type=int, default=640, help='train, val image size (pixels)') + parser.add_argument('--rect', action='store_true', help='rectangular training') + parser.add_argument('--resume', nargs='?', const=True, default=False, help='resume most recent training') + parser.add_argument('--nosave', action='store_true', help='only save final checkpoint') + parser.add_argument('--noval', action='store_true', help='only validate final epoch') + parser.add_argument('--noautoanchor', action='store_true', help='disable AutoAnchor') + parser.add_argument('--noplots', action='store_true', help='save no plot files') + parser.add_argument('--evolve', type=int, nargs='?', const=300, help='evolve hyperparameters for x generations') + parser.add_argument('--bucket', type=str, default='', help='gsutil bucket') + parser.add_argument('--cache', type=str, nargs='?', const='ram', help='image --cache ram/disk') + parser.add_argument('--image-weights', action='store_true', help='use weighted image selection for training') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--multi-scale', action='store_true', help='vary img-size +/- 50%%') + parser.add_argument('--single-cls', action='store_true', help='train multi-class data as single-class') + parser.add_argument('--optimizer', type=str, choices=['SGD', 'Adam', 'AdamW'], default='SGD', help='optimizer') + parser.add_argument('--sync-bn', action='store_true', help='use SyncBatchNorm, only available in DDP mode') + parser.add_argument('--workers', type=int, default=8, help='max dataloader workers (per RANK in DDP mode)') + parser.add_argument('--project', default=ROOT / 'runs/train-seg', help='save to project/name') + parser.add_argument('--name', default='exp', help='save to project/name') + parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment') + parser.add_argument('--quad', action='store_true', help='quad dataloader') + parser.add_argument('--cos-lr', action='store_true', help='cosine LR scheduler') + parser.add_argument('--label-smoothing', type=float, default=0.0, help='Label smoothing epsilon') + parser.add_argument('--patience', type=int, default=100, help='EarlyStopping patience (epochs without improvement)') + parser.add_argument('--freeze', nargs='+', type=int, default=[0], help='Freeze layers: backbone=10, first3=0 1 2') + parser.add_argument('--save-period', type=int, default=-1, help='Save checkpoint every x epochs (disabled if < 1)') + parser.add_argument('--seed', type=int, default=0, help='Global training seed') + parser.add_argument('--local_rank', type=int, default=-1, help='Automatic DDP Multi-GPU argument, do not modify') + + # Instance Segmentation Args + parser.add_argument('--mask-ratio', type=int, default=4, help='Downsample the truth masks to saving memory') + parser.add_argument('--no-overlap', action='store_true', help='Overlap masks train faster at slightly less mAP') + + return parser.parse_known_args()[0] if known else parser.parse_args() + + +def main(opt, callbacks=Callbacks()): + # Checks + if RANK in {-1, 0}: + print_args(vars(opt)) + check_git_status() + check_requirements(ROOT / 'requirements.txt') + + # Resume + if opt.resume and not opt.evolve: # resume from specified or most recent last.pt + last = Path(check_file(opt.resume) if isinstance(opt.resume, str) else get_latest_run()) + opt_yaml = last.parent.parent / 'opt.yaml' # train options yaml + opt_data = opt.data # original dataset + if opt_yaml.is_file(): + with open(opt_yaml, errors='ignore') as f: + d = yaml.safe_load(f) + else: + d = torch.load(last, map_location='cpu')['opt'] + opt = argparse.Namespace(**d) # replace + opt.cfg, opt.weights, opt.resume = '', str(last), True # reinstate + if is_url(opt_data): + opt.data = check_file(opt_data) # avoid HUB resume auth timeout + else: + opt.data, opt.cfg, opt.hyp, opt.weights, opt.project = \ + check_file(opt.data), check_yaml(opt.cfg), check_yaml(opt.hyp), str(opt.weights), str(opt.project) # checks + assert len(opt.cfg) or len(opt.weights), 'either --cfg or --weights must be specified' + if opt.evolve: + if opt.project == str(ROOT / 'runs/train-seg'): # if default project name, rename to runs/evolve-seg + opt.project = str(ROOT / 'runs/evolve-seg') + opt.exist_ok, opt.resume = opt.resume, False # pass resume to exist_ok and disable resume + if opt.name == 'cfg': + opt.name = Path(opt.cfg).stem # use model.yaml as name + opt.save_dir = str(increment_path(Path(opt.project) / opt.name, exist_ok=opt.exist_ok)) + + # DDP mode + device = select_device(opt.device, batch_size=opt.batch_size) + if LOCAL_RANK != -1: + msg = 'is not compatible with YOLOv5 Multi-GPU DDP training' + assert not opt.image_weights, f'--image-weights {msg}' + assert not opt.evolve, f'--evolve {msg}' + assert opt.batch_size != -1, f'AutoBatch with --batch-size -1 {msg}, please pass a valid --batch-size' + assert opt.batch_size % WORLD_SIZE == 0, f'--batch-size {opt.batch_size} must be multiple of WORLD_SIZE' + assert torch.cuda.device_count() > LOCAL_RANK, 'insufficient CUDA devices for DDP command' + torch.cuda.set_device(LOCAL_RANK) + device = torch.device('cuda', LOCAL_RANK) + dist.init_process_group(backend='nccl' if dist.is_nccl_available() else 'gloo') + + # Train + if not opt.evolve: + train(opt.hyp, opt, device, callbacks) + + # Evolve hyperparameters (optional) + else: + # Hyperparameter evolution metadata (mutation scale 0-1, lower_limit, upper_limit) + meta = { + 'lr0': (1, 1e-5, 1e-1), # initial learning rate (SGD=1E-2, Adam=1E-3) + 'lrf': (1, 0.01, 1.0), # final OneCycleLR learning rate (lr0 * lrf) + 'momentum': (0.3, 0.6, 0.98), # SGD momentum/Adam beta1 + 'weight_decay': (1, 0.0, 0.001), # optimizer weight decay + 'warmup_epochs': (1, 0.0, 5.0), # warmup epochs (fractions ok) + 'warmup_momentum': (1, 0.0, 0.95), # warmup initial momentum + 'warmup_bias_lr': (1, 0.0, 0.2), # warmup initial bias lr + 'box': (1, 0.02, 0.2), # box loss gain + 'cls': (1, 0.2, 4.0), # cls loss gain + 'cls_pw': (1, 0.5, 2.0), # cls BCELoss positive_weight + 'obj': (1, 0.2, 4.0), # obj loss gain (scale with pixels) + 'obj_pw': (1, 0.5, 2.0), # obj BCELoss positive_weight + 'iou_t': (0, 0.1, 0.7), # IoU training threshold + 'anchor_t': (1, 2.0, 8.0), # anchor-multiple threshold + 'anchors': (2, 2.0, 10.0), # anchors per output grid (0 to ignore) + 'fl_gamma': (0, 0.0, 2.0), # focal loss gamma (efficientDet default gamma=1.5) + 'hsv_h': (1, 0.0, 0.1), # image HSV-Hue augmentation (fraction) + 'hsv_s': (1, 0.0, 0.9), # image HSV-Saturation augmentation (fraction) + 'hsv_v': (1, 0.0, 0.9), # image HSV-Value augmentation (fraction) + 'degrees': (1, 0.0, 45.0), # image rotation (+/- deg) + 'translate': (1, 0.0, 0.9), # image translation (+/- fraction) + 'scale': (1, 0.0, 0.9), # image scale (+/- gain) + 'shear': (1, 0.0, 10.0), # image shear (+/- deg) + 'perspective': (0, 0.0, 0.001), # image perspective (+/- fraction), range 0-0.001 + 'flipud': (1, 0.0, 1.0), # image flip up-down (probability) + 'fliplr': (0, 0.0, 1.0), # image flip left-right (probability) + 'mosaic': (1, 0.0, 1.0), # image mixup (probability) + 'mixup': (1, 0.0, 1.0), # image mixup (probability) + 'copy_paste': (1, 0.0, 1.0)} # segment copy-paste (probability) + + with open(opt.hyp, errors='ignore') as f: + hyp = yaml.safe_load(f) # load hyps dict + if 'anchors' not in hyp: # anchors commented in hyp.yaml + hyp['anchors'] = 3 + if opt.noautoanchor: + del hyp['anchors'], meta['anchors'] + opt.noval, opt.nosave, save_dir = True, True, Path(opt.save_dir) # only val/save final epoch + # ei = [isinstance(x, (int, float)) for x in hyp.values()] # evolvable indices + evolve_yaml, evolve_csv = save_dir / 'hyp_evolve.yaml', save_dir / 'evolve.csv' + if opt.bucket: + # download evolve.csv if exists + subprocess.run([ + 'gsutil', + 'cp', + f'gs://{opt.bucket}/evolve.csv', + str(evolve_csv), ]) + + for _ in range(opt.evolve): # generations to evolve + if evolve_csv.exists(): # if evolve.csv exists: select best hyps and mutate + # Select parent(s) + parent = 'single' # parent selection method: 'single' or 'weighted' + x = np.loadtxt(evolve_csv, ndmin=2, delimiter=',', skiprows=1) + n = min(5, len(x)) # number of previous results to consider + x = x[np.argsort(-fitness(x))][:n] # top n mutations + w = fitness(x) - fitness(x).min() + 1E-6 # weights (sum > 0) + if parent == 'single' or len(x) == 1: + # x = x[random.randint(0, n - 1)] # random selection + x = x[random.choices(range(n), weights=w)[0]] # weighted selection + elif parent == 'weighted': + x = (x * w.reshape(n, 1)).sum(0) / w.sum() # weighted combination + + # Mutate + mp, s = 0.8, 0.2 # mutation probability, sigma + npr = np.random + npr.seed(int(time.time())) + g = np.array([meta[k][0] for k in hyp.keys()]) # gains 0-1 + ng = len(meta) + v = np.ones(ng) + while all(v == 1): # mutate until a change occurs (prevent duplicates) + v = (g * (npr.random(ng) < mp) * npr.randn(ng) * npr.random() * s + 1).clip(0.3, 3.0) + for i, k in enumerate(hyp.keys()): # plt.hist(v.ravel(), 300) + hyp[k] = float(x[i + 12] * v[i]) # mutate + + # Constrain to limits + for k, v in meta.items(): + hyp[k] = max(hyp[k], v[1]) # lower limit + hyp[k] = min(hyp[k], v[2]) # upper limit + hyp[k] = round(hyp[k], 5) # significant digits + + # Train mutation + results = train(hyp.copy(), opt, device, callbacks) + callbacks = Callbacks() + # Write mutation results + print_mutation(KEYS[4:16], results, hyp.copy(), save_dir, opt.bucket) + + # Plot results + plot_evolve(evolve_csv) + LOGGER.info(f'Hyperparameter evolution finished {opt.evolve} generations\n' + f"Results saved to {colorstr('bold', save_dir)}\n" + f'Usage example: $ python train.py --hyp {evolve_yaml}') + + +def run(**kwargs): + # Usage: import train; train.run(data='coco128.yaml', imgsz=320, weights='yolov5m.pt') + opt = parse_opt(True) + for k, v in kwargs.items(): + setattr(opt, k, v) + main(opt) + return opt + + +if __name__ == '__main__': + opt = parse_opt() + main(opt) diff --git a/segment/tutorial.ipynb b/segment/tutorial.ipynb new file mode 100644 index 0000000..cd215e7 --- /dev/null +++ b/segment/tutorial.ipynb @@ -0,0 +1,595 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "t6MPjfT5NrKQ" + }, + "source": [ + "
\n", + "\n", + " \n", + " \n", + "\n", + "\n", + "
\n", + " \"Run\n", + " \"Open\n", + " \"Open\n", + "
\n", + "\n", + "This YOLOv5 🚀 notebook by Ultralytics presents simple train, validate and predict examples to help start your AI adventure.
See GitHub for community support or contact us for professional support.\n", + "\n", + "
" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7mGmQbAO5pQb" + }, + "source": [ + "# Setup\n", + "\n", + "Clone GitHub [repository](https://github.com/ultralytics/yolov5), install [dependencies](https://github.com/ultralytics/yolov5/blob/master/requirements.txt) and check PyTorch and GPU." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "wbvMlHd_QwMG", + "outputId": "171b23f0-71b9-4cbf-b666-6fa2ecef70c8" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "YOLOv5 🚀 v7.0-2-gc9d47ae Python-3.7.15 torch-1.12.1+cu113 CUDA:0 (Tesla T4, 15110MiB)\n" + ] + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Setup complete ✅ (2 CPUs, 12.7 GB RAM, 22.6/78.2 GB disk)\n" + ] + } + ], + "source": [ + "!git clone https://github.com/ultralytics/yolov5 # clone\n", + "%cd yolov5\n", + "%pip install -qr requirements.txt comet_ml # install\n", + "\n", + "import torch\n", + "import utils\n", + "display = utils.notebook_init() # checks" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4JnkELT0cIJg" + }, + "source": [ + "# 1. Predict\n", + "\n", + "`segment/predict.py` runs YOLOv5 instance segmentation inference on a variety of sources, downloading models automatically from the [latest YOLOv5 release](https://github.com/ultralytics/yolov5/releases), and saving results to `runs/predict`. Example inference sources are:\n", + "\n", + "```shell\n", + "python segment/predict.py --source 0 # webcam\n", + " img.jpg # image \n", + " vid.mp4 # video\n", + " screen # screenshot\n", + " path/ # directory\n", + " 'path/*.jpg' # glob\n", + " 'https://youtu.be/Zgi9g1ksQHc' # YouTube\n", + " 'rtsp://example.com/media.mp4' # RTSP, RTMP, HTTP stream\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "zR9ZbuQCH7FX", + "outputId": "3f67f1c7-f15e-4fa5-d251-967c3b77eaad" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "\u001b[34m\u001b[1msegment/predict: \u001b[0mweights=['yolov5s-seg.pt'], source=data/images, data=data/coco128.yaml, imgsz=[640, 640], conf_thres=0.25, iou_thres=0.45, max_det=1000, device=, view_img=False, save_txt=False, save_conf=False, save_crop=False, nosave=False, classes=None, agnostic_nms=False, augment=False, visualize=False, update=False, project=runs/predict-seg, name=exp, exist_ok=False, line_thickness=3, hide_labels=False, hide_conf=False, half=False, dnn=False, vid_stride=1, retina_masks=False\n", + "YOLOv5 🚀 v7.0-2-gc9d47ae Python-3.7.15 torch-1.12.1+cu113 CUDA:0 (Tesla T4, 15110MiB)\n", + "\n", + "Downloading https://github.com/ultralytics/yolov5/releases/download/v7.0/yolov5s-seg.pt to yolov5s-seg.pt...\n", + "100% 14.9M/14.9M [00:01<00:00, 12.0MB/s]\n", + "\n", + "Fusing layers... \n", + "YOLOv5s-seg summary: 224 layers, 7611485 parameters, 0 gradients, 26.4 GFLOPs\n", + "image 1/2 /content/yolov5/data/images/bus.jpg: 640x480 4 persons, 1 bus, 18.2ms\n", + "image 2/2 /content/yolov5/data/images/zidane.jpg: 384x640 2 persons, 1 tie, 13.4ms\n", + "Speed: 0.5ms pre-process, 15.8ms inference, 18.5ms NMS per image at shape (1, 3, 640, 640)\n", + "Results saved to \u001b[1mruns/predict-seg/exp\u001b[0m\n" + ] + } + ], + "source": [ + "!python segment/predict.py --weights yolov5s-seg.pt --img 640 --conf 0.25 --source data/images\n", + "#display.Image(filename='runs/predict-seg/exp/zidane.jpg', width=600)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "hkAzDWJ7cWTr" + }, + "source": [ + "        \n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0eq1SMWl6Sfn" + }, + "source": [ + "# 2. Validate\n", + "Validate a model's accuracy on the [COCO](https://cocodataset.org/#home) dataset's `val` or `test` splits. Models are downloaded automatically from the [latest YOLOv5 release](https://github.com/ultralytics/yolov5/releases). To show results by class use the `--verbose` flag." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "WQPtK1QYVaD_", + "outputId": "9d751d8c-bee8-4339-cf30-9854ca530449" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Downloading https://github.com/ultralytics/yolov5/releases/download/v1.0/coco2017labels-segments.zip ...\n", + "Downloading http://images.cocodataset.org/zips/val2017.zip ...\n", + "######################################################################## 100.0%\n", + "######################################################################## 100.0%\n" + ] + } + ], + "source": [ + "# Download COCO val\n", + "!bash data/scripts/get_coco.sh --val --segments # download (780M - 5000 images)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "X58w8JLpMnjH", + "outputId": "a140d67a-02da-479e-9ddb-7d54bf9e407a" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "\u001b[34m\u001b[1msegment/val: \u001b[0mdata=/content/yolov5/data/coco.yaml, weights=['yolov5s-seg.pt'], batch_size=32, imgsz=640, conf_thres=0.001, iou_thres=0.6, max_det=300, task=val, device=, workers=8, single_cls=False, augment=False, verbose=False, save_txt=False, save_hybrid=False, save_conf=False, save_json=False, project=runs/val-seg, name=exp, exist_ok=False, half=True, dnn=False\n", + "YOLOv5 🚀 v7.0-2-gc9d47ae Python-3.7.15 torch-1.12.1+cu113 CUDA:0 (Tesla T4, 15110MiB)\n", + "\n", + "Fusing layers... \n", + "YOLOv5s-seg summary: 224 layers, 7611485 parameters, 0 gradients, 26.4 GFLOPs\n", + "\u001b[34m\u001b[1mval: \u001b[0mScanning /content/datasets/coco/val2017... 4952 images, 48 backgrounds, 0 corrupt: 100% 5000/5000 [00:03<00:00, 1361.31it/s]\n", + "\u001b[34m\u001b[1mval: \u001b[0mNew cache created: /content/datasets/coco/val2017.cache\n", + " Class Images Instances Box(P R mAP50 mAP50-95) Mask(P R mAP50 mAP50-95): 100% 157/157 [01:54<00:00, 1.37it/s]\n", + " all 5000 36335 0.673 0.517 0.566 0.373 0.672 0.49 0.532 0.319\n", + "Speed: 0.6ms pre-process, 4.4ms inference, 2.9ms NMS per image at shape (32, 3, 640, 640)\n", + "Results saved to \u001b[1mruns/val-seg/exp\u001b[0m\n" + ] + } + ], + "source": [ + "# Validate YOLOv5s-seg on COCO val\n", + "!python segment/val.py --weights yolov5s-seg.pt --data coco.yaml --img 640 --half" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZY2VXXXu74w5" + }, + "source": [ + "# 3. Train\n", + "\n", + "

\n", + "Close the active learning loop by sampling images from your inference conditions with the `roboflow` pip package\n", + "

\n", + "\n", + "Train a YOLOv5s-seg model on the [COCO128](https://www.kaggle.com/ultralytics/coco128) dataset with `--data coco128-seg.yaml`, starting from pretrained `--weights yolov5s-seg.pt`, or from randomly initialized `--weights '' --cfg yolov5s-seg.yaml`.\n", + "\n", + "- **Pretrained [Models](https://github.com/ultralytics/yolov5/tree/master/models)** are downloaded\n", + "automatically from the [latest YOLOv5 release](https://github.com/ultralytics/yolov5/releases)\n", + "- **[Datasets](https://github.com/ultralytics/yolov5/tree/master/data)** available for autodownload include: [COCO](https://github.com/ultralytics/yolov5/blob/master/data/coco.yaml), [COCO128](https://github.com/ultralytics/yolov5/blob/master/data/coco128.yaml), [VOC](https://github.com/ultralytics/yolov5/blob/master/data/VOC.yaml), [Argoverse](https://github.com/ultralytics/yolov5/blob/master/data/Argoverse.yaml), [VisDrone](https://github.com/ultralytics/yolov5/blob/master/data/VisDrone.yaml), [GlobalWheat](https://github.com/ultralytics/yolov5/blob/master/data/GlobalWheat2020.yaml), [xView](https://github.com/ultralytics/yolov5/blob/master/data/xView.yaml), [Objects365](https://github.com/ultralytics/yolov5/blob/master/data/Objects365.yaml), [SKU-110K](https://github.com/ultralytics/yolov5/blob/master/data/SKU-110K.yaml).\n", + "- **Training Results** are saved to `runs/train-seg/` with incrementing run directories, i.e. `runs/train-seg/exp2`, `runs/train-seg/exp3` etc.\n", + "

\n", + "\n", + "A **Mosaic Dataloader** is used for training which combines 4 images into 1 mosaic.\n", + "\n", + "## Train on Custom Data with Roboflow 🌟 NEW\n", + "\n", + "[Roboflow](https://roboflow.com/?ref=ultralytics) enables you to easily **organize, label, and prepare** a high quality dataset with your own custom data. Roboflow also makes it easy to establish an active learning pipeline, collaborate with your team on dataset improvement, and integrate directly into your model building workflow with the `roboflow` pip package.\n", + "\n", + "- Custom Training Example: [https://blog.roboflow.com/train-yolov5-instance-segmentation-custom-dataset/](https://blog.roboflow.com/train-yolov5-instance-segmentation-custom-dataset/?ref=ultralytics)\n", + "- Custom Training Notebook: [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1JTz7kpmHsg-5qwVz2d2IH3AaenI1tv0N?usp=sharing)\n", + "
\n", + "\n", + "

Label images lightning fast (including with model-assisted labeling)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "i3oKtE4g-aNn" + }, + "outputs": [], + "source": [ + "#@title Select YOLOv5 🚀 logger {run: 'auto'}\n", + "logger = 'Comet' #@param ['Comet', 'ClearML', 'TensorBoard']\n", + "\n", + "if logger == 'Comet':\n", + " %pip install -q comet_ml\n", + " import comet_ml; comet_ml.init()\n", + "elif logger == 'ClearML':\n", + " %pip install -q clearml\n", + " import clearml; clearml.browser_login()\n", + "elif logger == 'TensorBoard':\n", + " %load_ext tensorboard\n", + " %tensorboard --logdir runs/train" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "1NcFxRcFdJ_O", + "outputId": "3a3e0cf7-e79c-47a5-c8e7-2d26eeeab988" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "\u001b[34m\u001b[1msegment/train: \u001b[0mweights=yolov5s-seg.pt, cfg=, data=coco128-seg.yaml, hyp=data/hyps/hyp.scratch-low.yaml, epochs=3, batch_size=16, imgsz=640, rect=False, resume=False, nosave=False, noval=False, noautoanchor=False, noplots=False, evolve=None, bucket=, cache=ram, image_weights=False, device=, multi_scale=False, single_cls=False, optimizer=SGD, sync_bn=False, workers=8, project=runs/train-seg, name=exp, exist_ok=False, quad=False, cos_lr=False, label_smoothing=0.0, patience=100, freeze=[0], save_period=-1, seed=0, local_rank=-1, mask_ratio=4, no_overlap=False\n", + "\u001b[34m\u001b[1mgithub: \u001b[0mup to date with https://github.com/ultralytics/yolov5 ✅\n", + "YOLOv5 🚀 v7.0-2-gc9d47ae Python-3.7.15 torch-1.12.1+cu113 CUDA:0 (Tesla T4, 15110MiB)\n", + "\n", + "\u001b[34m\u001b[1mhyperparameters: \u001b[0mlr0=0.01, lrf=0.01, momentum=0.937, weight_decay=0.0005, warmup_epochs=3.0, warmup_momentum=0.8, warmup_bias_lr=0.1, box=0.05, cls=0.5, cls_pw=1.0, obj=1.0, obj_pw=1.0, iou_t=0.2, anchor_t=4.0, fl_gamma=0.0, hsv_h=0.015, hsv_s=0.7, hsv_v=0.4, degrees=0.0, translate=0.1, scale=0.5, shear=0.0, perspective=0.0, flipud=0.0, fliplr=0.5, mosaic=1.0, mixup=0.0, copy_paste=0.0\n", + "\u001b[34m\u001b[1mTensorBoard: \u001b[0mStart with 'tensorboard --logdir runs/train-seg', view at http://localhost:6006/\n", + "\n", + "Dataset not found ⚠️, missing paths ['/content/datasets/coco128-seg/images/train2017']\n", + "Downloading https://ultralytics.com/assets/coco128-seg.zip to coco128-seg.zip...\n", + "100% 6.79M/6.79M [00:01<00:00, 6.73MB/s]\n", + "Dataset download success ✅ (1.9s), saved to \u001b[1m/content/datasets\u001b[0m\n", + "\n", + " from n params module arguments \n", + " 0 -1 1 3520 models.common.Conv [3, 32, 6, 2, 2] \n", + " 1 -1 1 18560 models.common.Conv [32, 64, 3, 2] \n", + " 2 -1 1 18816 models.common.C3 [64, 64, 1] \n", + " 3 -1 1 73984 models.common.Conv [64, 128, 3, 2] \n", + " 4 -1 2 115712 models.common.C3 [128, 128, 2] \n", + " 5 -1 1 295424 models.common.Conv [128, 256, 3, 2] \n", + " 6 -1 3 625152 models.common.C3 [256, 256, 3] \n", + " 7 -1 1 1180672 models.common.Conv [256, 512, 3, 2] \n", + " 8 -1 1 1182720 models.common.C3 [512, 512, 1] \n", + " 9 -1 1 656896 models.common.SPPF [512, 512, 5] \n", + " 10 -1 1 131584 models.common.Conv [512, 256, 1, 1] \n", + " 11 -1 1 0 torch.nn.modules.upsampling.Upsample [None, 2, 'nearest'] \n", + " 12 [-1, 6] 1 0 models.common.Concat [1] \n", + " 13 -1 1 361984 models.common.C3 [512, 256, 1, False] \n", + " 14 -1 1 33024 models.common.Conv [256, 128, 1, 1] \n", + " 15 -1 1 0 torch.nn.modules.upsampling.Upsample [None, 2, 'nearest'] \n", + " 16 [-1, 4] 1 0 models.common.Concat [1] \n", + " 17 -1 1 90880 models.common.C3 [256, 128, 1, False] \n", + " 18 -1 1 147712 models.common.Conv [128, 128, 3, 2] \n", + " 19 [-1, 14] 1 0 models.common.Concat [1] \n", + " 20 -1 1 296448 models.common.C3 [256, 256, 1, False] \n", + " 21 -1 1 590336 models.common.Conv [256, 256, 3, 2] \n", + " 22 [-1, 10] 1 0 models.common.Concat [1] \n", + " 23 -1 1 1182720 models.common.C3 [512, 512, 1, False] \n", + " 24 [17, 20, 23] 1 615133 models.yolo.Segment [80, [[10, 13, 16, 30, 33, 23], [30, 61, 62, 45, 59, 119], [116, 90, 156, 198, 373, 326]], 32, 128, [128, 256, 512]]\n", + "Model summary: 225 layers, 7621277 parameters, 7621277 gradients, 26.6 GFLOPs\n", + "\n", + "Transferred 367/367 items from yolov5s-seg.pt\n", + "\u001b[34m\u001b[1mAMP: \u001b[0mchecks passed ✅\n", + "\u001b[34m\u001b[1moptimizer:\u001b[0m SGD(lr=0.01) with parameter groups 60 weight(decay=0.0), 63 weight(decay=0.0005), 63 bias\n", + "\u001b[34m\u001b[1malbumentations: \u001b[0mBlur(p=0.01, blur_limit=(3, 7)), MedianBlur(p=0.01, blur_limit=(3, 7)), ToGray(p=0.01), CLAHE(p=0.01, clip_limit=(1, 4.0), tile_grid_size=(8, 8))\n", + "\u001b[34m\u001b[1mtrain: \u001b[0mScanning /content/datasets/coco128-seg/labels/train2017... 126 images, 2 backgrounds, 0 corrupt: 100% 128/128 [00:00<00:00, 1389.59it/s]\n", + "\u001b[34m\u001b[1mtrain: \u001b[0mNew cache created: /content/datasets/coco128-seg/labels/train2017.cache\n", + "\u001b[34m\u001b[1mtrain: \u001b[0mCaching images (0.1GB ram): 100% 128/128 [00:00<00:00, 238.86it/s]\n", + "\u001b[34m\u001b[1mval: \u001b[0mScanning /content/datasets/coco128-seg/labels/train2017.cache... 126 images, 2 backgrounds, 0 corrupt: 100% 128/128 [00:00 # 2. paste API key\n", + "python train.py --img 640 --epochs 3 --data coco128.yaml --weights yolov5s.pt # 3. train\n", + "```\n", + "To learn more about all of the supported Comet features for this integration, check out the [Comet Tutorial](https://docs.ultralytics.com/yolov5/tutorials/comet_logging_integration). If you'd like to learn more about Comet, head over to our [documentation](https://www.comet.com/docs/v2/?utm_source=yolov5&utm_medium=partner&utm_campaign=partner_yolov5_2022&utm_content=yolov5_colab). Get started by trying out the Comet Colab Notebook:\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1RG0WOQyxlDlo5Km8GogJpIEJlg_5lyYO?usp=sharing)\n", + "\n", + "\n", + "\"Comet" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Lay2WsTjNJzP" + }, + "source": [ + "## ClearML Logging and Automation 🌟 NEW\n", + "\n", + "[ClearML](https://cutt.ly/yolov5-notebook-clearml) is completely integrated into YOLOv5 to track your experimentation, manage dataset versions and even remotely execute training runs. To enable ClearML (check cells above):\n", + "\n", + "- `pip install clearml`\n", + "- run `clearml-init` to connect to a ClearML server (**deploy your own [open-source server](https://github.com/allegroai/clearml-server)**, or use our [free hosted server](https://cutt.ly/yolov5-notebook-clearml))\n", + "\n", + "You'll get all the great expected features from an experiment manager: live updates, model upload, experiment comparison etc. but ClearML also tracks uncommitted changes and installed packages for example. Thanks to that ClearML Tasks (which is what we call experiments) are also reproducible on different machines! With only 1 extra line, we can schedule a YOLOv5 training task on a queue to be executed by any number of ClearML Agents (workers).\n", + "\n", + "You can use ClearML Data to version your dataset and then pass it to YOLOv5 simply using its unique ID. This will help you keep track of your data without adding extra hassle. Explore the [ClearML Tutorial](https://docs.ultralytics.com/yolov5/tutorials/clearml_logging_integration) for details!\n", + "\n", + "\n", + "\"ClearML" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-WPvRbS5Swl6" + }, + "source": [ + "## Local Logging\n", + "\n", + "Training results are automatically logged with [Tensorboard](https://www.tensorflow.org/tensorboard) and [CSV](https://github.com/ultralytics/yolov5/pull/4148) loggers to `runs/train`, with a new experiment directory created for each new training as `runs/train/exp2`, `runs/train/exp3`, etc.\n", + "\n", + "This directory contains train and val statistics, mosaics, labels, predictions and augmentated mosaics, as well as metrics and charts including precision-recall (PR) curves and confusion matrices. \n", + "\n", + "\"Local\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Zelyeqbyt3GD" + }, + "source": [ + "# Environments\n", + "\n", + "YOLOv5 may be run in any of the following up-to-date verified environments (with all dependencies including [CUDA](https://developer.nvidia.com/cuda)/[CUDNN](https://developer.nvidia.com/cudnn), [Python](https://www.python.org/) and [PyTorch](https://pytorch.org/) preinstalled):\n", + "\n", + "- **Notebooks** with free GPU: \"Run \"Open \"Open\n", + "- **Google Cloud** Deep Learning VM. See [GCP Quickstart Guide](https://docs.ultralytics.com/yolov5/environments/google_cloud_quickstart_tutorial/)\n", + "- **Amazon** Deep Learning AMI. See [AWS Quickstart Guide](https://docs.ultralytics.com/yolov5/environments/aws_quickstart_tutorial/)\n", + "- **Docker Image**. See [Docker Quickstart Guide](https://docs.ultralytics.com/yolov5/environments/docker_image_quickstart_tutorial/) \"Docker\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6Qu7Iesl0p54" + }, + "source": [ + "# Status\n", + "\n", + "![YOLOv5 CI](https://github.com/ultralytics/yolov5/actions/workflows/ci-testing.yml/badge.svg)\n", + "\n", + "If this badge is green, all [YOLOv5 GitHub Actions](https://github.com/ultralytics/yolov5/actions) Continuous Integration (CI) tests are currently passing. CI tests verify correct operation of YOLOv5 training ([train.py](https://github.com/ultralytics/yolov5/blob/master/train.py)), testing ([val.py](https://github.com/ultralytics/yolov5/blob/master/val.py)), inference ([detect.py](https://github.com/ultralytics/yolov5/blob/master/detect.py)) and export ([export.py](https://github.com/ultralytics/yolov5/blob/master/export.py)) on macOS, Windows, and Ubuntu every 24 hours and on every commit.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IEijrePND_2I" + }, + "source": [ + "# Appendix\n", + "\n", + "Additional content below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "GMusP4OAxFu6" + }, + "outputs": [], + "source": [ + "# YOLOv5 PyTorch HUB Inference (DetectionModels only)\n", + "import torch\n", + "\n", + "model = torch.hub.load('ultralytics/yolov5', 'yolov5s-seg', force_reload=True, trust_repo=True) # or yolov5n - yolov5x6 or custom\n", + "im = 'https://ultralytics.com/images/zidane.jpg' # file, Path, PIL.Image, OpenCV, nparray, list\n", + "results = model(im) # inference\n", + "results.print() # or .show(), .save(), .crop(), .pandas(), etc." + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "name": "YOLOv5 Segmentation Tutorial", + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.12" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/segment/val.py b/segment/val.py new file mode 100644 index 0000000..dc80818 --- /dev/null +++ b/segment/val.py @@ -0,0 +1,473 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Validate a trained YOLOv5 segment model on a segment dataset + +Usage: + $ bash data/scripts/get_coco.sh --val --segments # download COCO-segments val split (1G, 5000 images) + $ python segment/val.py --weights yolov5s-seg.pt --data coco.yaml --img 640 # validate COCO-segments + +Usage - formats: + $ python segment/val.py --weights yolov5s-seg.pt # PyTorch + yolov5s-seg.torchscript # TorchScript + yolov5s-seg.onnx # ONNX Runtime or OpenCV DNN with --dnn + yolov5s-seg_openvino_label # OpenVINO + yolov5s-seg.engine # TensorRT + yolov5s-seg.mlmodel # CoreML (macOS-only) + yolov5s-seg_saved_model # TensorFlow SavedModel + yolov5s-seg.pb # TensorFlow GraphDef + yolov5s-seg.tflite # TensorFlow Lite + yolov5s-seg_edgetpu.tflite # TensorFlow Edge TPU + yolov5s-seg_paddle_model # PaddlePaddle +""" + +import argparse +import json +import os +import subprocess +import sys +from multiprocessing.pool import ThreadPool +from pathlib import Path + +import numpy as np +import torch +from tqdm import tqdm + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[1] # YOLOv5 root directory +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) # add ROOT to PATH +ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative + +import torch.nn.functional as F + +from models.common import DetectMultiBackend +from models.yolo import SegmentationModel +from utils.callbacks import Callbacks +from utils.general import (LOGGER, NUM_THREADS, TQDM_BAR_FORMAT, Profile, check_dataset, check_img_size, + check_requirements, check_yaml, coco80_to_coco91_class, colorstr, increment_path, + non_max_suppression, print_args, scale_boxes, xywh2xyxy, xyxy2xywh) +from utils.metrics import ConfusionMatrix, box_iou +from utils.plots import output_to_target, plot_val_study +from utils.segment.dataloaders import create_dataloader +from utils.segment.general import mask_iou, process_mask, process_mask_native, scale_image +from utils.segment.metrics import Metrics, ap_per_class_box_and_mask +from utils.segment.plots import plot_images_and_masks +from utils.torch_utils import de_parallel, select_device, smart_inference_mode + + +def save_one_txt(predn, save_conf, shape, file): + # Save one txt result + gn = torch.tensor(shape)[[1, 0, 1, 0]] # normalization gain whwh + for *xyxy, conf, cls in predn.tolist(): + xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh + line = (cls, *xywh, conf) if save_conf else (cls, *xywh) # label format + with open(file, 'a') as f: + f.write(('%g ' * len(line)).rstrip() % line + '\n') + + +def save_one_json(predn, jdict, path, class_map, pred_masks): + # Save one JSON result {"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236} + from pycocotools.mask import encode + + def single_encode(x): + rle = encode(np.asarray(x[:, :, None], order='F', dtype='uint8'))[0] + rle['counts'] = rle['counts'].decode('utf-8') + return rle + + image_id = int(path.stem) if path.stem.isnumeric() else path.stem + box = xyxy2xywh(predn[:, :4]) # xywh + box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner + pred_masks = np.transpose(pred_masks, (2, 0, 1)) + with ThreadPool(NUM_THREADS) as pool: + rles = pool.map(single_encode, pred_masks) + for i, (p, b) in enumerate(zip(predn.tolist(), box.tolist())): + jdict.append({ + 'image_id': image_id, + 'category_id': class_map[int(p[5])], + 'bbox': [round(x, 3) for x in b], + 'score': round(p[4], 5), + 'segmentation': rles[i]}) + + +def process_batch(detections, labels, iouv, pred_masks=None, gt_masks=None, overlap=False, masks=False): + """ + Return correct prediction matrix + Arguments: + detections (array[N, 6]), x1, y1, x2, y2, conf, class + labels (array[M, 5]), class, x1, y1, x2, y2 + Returns: + correct (array[N, 10]), for 10 IoU levels + """ + if masks: + if overlap: + nl = len(labels) + index = torch.arange(nl, device=gt_masks.device).view(nl, 1, 1) + 1 + gt_masks = gt_masks.repeat(nl, 1, 1) # shape(1,640,640) -> (n,640,640) + gt_masks = torch.where(gt_masks == index, 1.0, 0.0) + if gt_masks.shape[1:] != pred_masks.shape[1:]: + gt_masks = F.interpolate(gt_masks[None], pred_masks.shape[1:], mode='bilinear', align_corners=False)[0] + gt_masks = gt_masks.gt_(0.5) + iou = mask_iou(gt_masks.view(gt_masks.shape[0], -1), pred_masks.view(pred_masks.shape[0], -1)) + else: # boxes + iou = box_iou(labels[:, 1:], detections[:, :4]) + + correct = np.zeros((detections.shape[0], iouv.shape[0])).astype(bool) + correct_class = labels[:, 0:1] == detections[:, 5] + for i in range(len(iouv)): + x = torch.where((iou >= iouv[i]) & correct_class) # IoU > threshold and classes match + if x[0].shape[0]: + matches = torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1).cpu().numpy() # [label, detect, iou] + if x[0].shape[0] > 1: + matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 1], return_index=True)[1]] + # matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 0], return_index=True)[1]] + correct[matches[:, 1].astype(int), i] = True + return torch.tensor(correct, dtype=torch.bool, device=iouv.device) + + +@smart_inference_mode() +def run( + data, + weights=None, # model.pt path(s) + batch_size=32, # batch size + imgsz=640, # inference size (pixels) + conf_thres=0.001, # confidence threshold + iou_thres=0.6, # NMS IoU threshold + max_det=300, # maximum detections per image + task='val', # train, val, test, speed or study + device='', # cuda device, i.e. 0 or 0,1,2,3 or cpu + workers=8, # max dataloader workers (per RANK in DDP mode) + single_cls=False, # treat as single-class dataset + augment=False, # augmented inference + verbose=False, # verbose output + save_txt=False, # save results to *.txt + save_hybrid=False, # save label+prediction hybrid results to *.txt + save_conf=False, # save confidences in --save-txt labels + save_json=False, # save a COCO-JSON results file + project=ROOT / 'runs/val-seg', # save to project/name + name='exp', # save to project/name + exist_ok=False, # existing project/name ok, do not increment + half=True, # use FP16 half-precision inference + dnn=False, # use OpenCV DNN for ONNX inference + model=None, + dataloader=None, + save_dir=Path(''), + plots=True, + overlap=False, + mask_downsample_ratio=1, + compute_loss=None, + callbacks=Callbacks(), +): + if save_json: + check_requirements('pycocotools>=2.0.6') + process = process_mask_native # more accurate + else: + process = process_mask # faster + + # Initialize/load model and set device + training = model is not None + if training: # called by train.py + device, pt, jit, engine = next(model.parameters()).device, True, False, False # get model device, PyTorch model + half &= device.type != 'cpu' # half precision only supported on CUDA + model.half() if half else model.float() + nm = de_parallel(model).model[-1].nm # number of masks + else: # called directly + device = select_device(device, batch_size=batch_size) + + # Directories + save_dir = increment_path(Path(project) / name, exist_ok=exist_ok) # increment run + (save_dir / 'labels' if save_txt else save_dir).mkdir(parents=True, exist_ok=True) # make dir + + # Load model + model = DetectMultiBackend(weights, device=device, dnn=dnn, data=data, fp16=half) + stride, pt, jit, engine = model.stride, model.pt, model.jit, model.engine + imgsz = check_img_size(imgsz, s=stride) # check image size + half = model.fp16 # FP16 supported on limited backends with CUDA + nm = de_parallel(model).model.model[-1].nm if isinstance(model, SegmentationModel) else 32 # number of masks + if engine: + batch_size = model.batch_size + else: + device = model.device + if not (pt or jit): + batch_size = 1 # export.py models default to batch-size 1 + LOGGER.info(f'Forcing --batch-size 1 square inference (1,3,{imgsz},{imgsz}) for non-PyTorch models') + + # Data + data = check_dataset(data) # check + + # Configure + model.eval() + cuda = device.type != 'cpu' + is_coco = isinstance(data.get('val'), str) and data['val'].endswith(f'coco{os.sep}val2017.txt') # COCO dataset + nc = 1 if single_cls else int(data['nc']) # number of classes + iouv = torch.linspace(0.5, 0.95, 10, device=device) # iou vector for mAP@0.5:0.95 + niou = iouv.numel() + + # Dataloader + if not training: + if pt and not single_cls: # check --weights are trained on --data + ncm = model.model.nc + assert ncm == nc, f'{weights} ({ncm} classes) trained on different --data than what you passed ({nc} ' \ + f'classes). Pass correct combination of --weights and --data that are trained together.' + model.warmup(imgsz=(1 if pt else batch_size, 3, imgsz, imgsz)) # warmup + pad, rect = (0.0, False) if task == 'speed' else (0.5, pt) # square inference for benchmarks + task = task if task in ('train', 'val', 'test') else 'val' # path to train/val/test images + dataloader = create_dataloader(data[task], + imgsz, + batch_size, + stride, + single_cls, + pad=pad, + rect=rect, + workers=workers, + prefix=colorstr(f'{task}: '), + overlap_mask=overlap, + mask_downsample_ratio=mask_downsample_ratio)[0] + + seen = 0 + confusion_matrix = ConfusionMatrix(nc=nc) + names = model.names if hasattr(model, 'names') else model.module.names # get class names + if isinstance(names, (list, tuple)): # old format + names = dict(enumerate(names)) + class_map = coco80_to_coco91_class() if is_coco else list(range(1000)) + s = ('%22s' + '%11s' * 10) % ('Class', 'Images', 'Instances', 'Box(P', 'R', 'mAP50', 'mAP50-95)', 'Mask(P', 'R', + 'mAP50', 'mAP50-95)') + dt = Profile(), Profile(), Profile() + metrics = Metrics() + loss = torch.zeros(4, device=device) + jdict, stats = [], [] + # callbacks.run('on_val_start') + pbar = tqdm(dataloader, desc=s, bar_format=TQDM_BAR_FORMAT) # progress bar + for batch_i, (im, targets, paths, shapes, masks) in enumerate(pbar): + # callbacks.run('on_val_batch_start') + with dt[0]: + if cuda: + im = im.to(device, non_blocking=True) + targets = targets.to(device) + masks = masks.to(device) + masks = masks.float() + im = im.half() if half else im.float() # uint8 to fp16/32 + im /= 255 # 0 - 255 to 0.0 - 1.0 + nb, _, height, width = im.shape # batch size, channels, height, width + + # Inference + with dt[1]: + preds, protos, train_out = model(im) if compute_loss else (*model(im, augment=augment)[:2], None) + + # Loss + if compute_loss: + loss += compute_loss((train_out, protos), targets, masks)[1] # box, obj, cls + + # NMS + targets[:, 2:] *= torch.tensor((width, height, width, height), device=device) # to pixels + lb = [targets[targets[:, 0] == i, 1:] for i in range(nb)] if save_hybrid else [] # for autolabelling + with dt[2]: + preds = non_max_suppression(preds, + conf_thres, + iou_thres, + labels=lb, + multi_label=True, + agnostic=single_cls, + max_det=max_det, + nm=nm) + + # Metrics + plot_masks = [] # masks for plotting + for si, (pred, proto) in enumerate(zip(preds, protos)): + labels = targets[targets[:, 0] == si, 1:] + nl, npr = labels.shape[0], pred.shape[0] # number of labels, predictions + path, shape = Path(paths[si]), shapes[si][0] + correct_masks = torch.zeros(npr, niou, dtype=torch.bool, device=device) # init + correct_bboxes = torch.zeros(npr, niou, dtype=torch.bool, device=device) # init + seen += 1 + + if npr == 0: + if nl: + stats.append((correct_masks, correct_bboxes, *torch.zeros((2, 0), device=device), labels[:, 0])) + if plots: + confusion_matrix.process_batch(detections=None, labels=labels[:, 0]) + continue + + # Masks + midx = [si] if overlap else targets[:, 0] == si + gt_masks = masks[midx] + pred_masks = process(proto, pred[:, 6:], pred[:, :4], shape=im[si].shape[1:]) + + # Predictions + if single_cls: + pred[:, 5] = 0 + predn = pred.clone() + scale_boxes(im[si].shape[1:], predn[:, :4], shape, shapes[si][1]) # native-space pred + + # Evaluate + if nl: + tbox = xywh2xyxy(labels[:, 1:5]) # target boxes + scale_boxes(im[si].shape[1:], tbox, shape, shapes[si][1]) # native-space labels + labelsn = torch.cat((labels[:, 0:1], tbox), 1) # native-space labels + correct_bboxes = process_batch(predn, labelsn, iouv) + correct_masks = process_batch(predn, labelsn, iouv, pred_masks, gt_masks, overlap=overlap, masks=True) + if plots: + confusion_matrix.process_batch(predn, labelsn) + stats.append((correct_masks, correct_bboxes, pred[:, 4], pred[:, 5], labels[:, 0])) # (conf, pcls, tcls) + + pred_masks = torch.as_tensor(pred_masks, dtype=torch.uint8) + if plots and batch_i < 3: + plot_masks.append(pred_masks[:15]) # filter top 15 to plot + + # Save/log + if save_txt: + save_one_txt(predn, save_conf, shape, file=save_dir / 'labels' / f'{path.stem}.txt') + if save_json: + pred_masks = scale_image(im[si].shape[1:], + pred_masks.permute(1, 2, 0).contiguous().cpu().numpy(), shape, shapes[si][1]) + save_one_json(predn, jdict, path, class_map, pred_masks) # append to COCO-JSON dictionary + # callbacks.run('on_val_image_end', pred, predn, path, names, im[si]) + + # Plot images + if plots and batch_i < 3: + if len(plot_masks): + plot_masks = torch.cat(plot_masks, dim=0) + plot_images_and_masks(im, targets, masks, paths, save_dir / f'val_batch{batch_i}_labels.jpg', names) + plot_images_and_masks(im, output_to_target(preds, max_det=15), plot_masks, paths, + save_dir / f'val_batch{batch_i}_pred.jpg', names) # pred + + # callbacks.run('on_val_batch_end') + + # Compute metrics + stats = [torch.cat(x, 0).cpu().numpy() for x in zip(*stats)] # to numpy + if len(stats) and stats[0].any(): + results = ap_per_class_box_and_mask(*stats, plot=plots, save_dir=save_dir, names=names) + metrics.update(results) + nt = np.bincount(stats[4].astype(int), minlength=nc) # number of targets per class + + # Print results + pf = '%22s' + '%11i' * 2 + '%11.3g' * 8 # print format + LOGGER.info(pf % ('all', seen, nt.sum(), *metrics.mean_results())) + if nt.sum() == 0: + LOGGER.warning(f'WARNING ⚠️ no labels found in {task} set, can not compute metrics without labels') + + # Print results per class + if (verbose or (nc < 50 and not training)) and nc > 1 and len(stats): + for i, c in enumerate(metrics.ap_class_index): + LOGGER.info(pf % (names[c], seen, nt[c], *metrics.class_result(i))) + + # Print speeds + t = tuple(x.t / seen * 1E3 for x in dt) # speeds per image + if not training: + shape = (batch_size, 3, imgsz, imgsz) + LOGGER.info(f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {shape}' % t) + + # Plots + if plots: + confusion_matrix.plot(save_dir=save_dir, names=list(names.values())) + # callbacks.run('on_val_end') + + mp_bbox, mr_bbox, map50_bbox, map_bbox, mp_mask, mr_mask, map50_mask, map_mask = metrics.mean_results() + + # Save JSON + if save_json and len(jdict): + w = Path(weights[0] if isinstance(weights, list) else weights).stem if weights is not None else '' # weights + anno_json = str(Path('../datasets/coco/annotations/instances_val2017.json')) # annotations + pred_json = str(save_dir / f'{w}_predictions.json') # predictions + LOGGER.info(f'\nEvaluating pycocotools mAP... saving {pred_json}...') + with open(pred_json, 'w') as f: + json.dump(jdict, f) + + try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb + from pycocotools.coco import COCO + from pycocotools.cocoeval import COCOeval + + anno = COCO(anno_json) # init annotations api + pred = anno.loadRes(pred_json) # init predictions api + results = [] + for eval in COCOeval(anno, pred, 'bbox'), COCOeval(anno, pred, 'segm'): + if is_coco: + eval.params.imgIds = [int(Path(x).stem) for x in dataloader.dataset.im_files] # img ID to evaluate + eval.evaluate() + eval.accumulate() + eval.summarize() + results.extend(eval.stats[:2]) # update results (mAP@0.5:0.95, mAP@0.5) + map_bbox, map50_bbox, map_mask, map50_mask = results + except Exception as e: + LOGGER.info(f'pycocotools unable to run: {e}') + + # Return results + model.float() # for training + if not training: + s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else '' + LOGGER.info(f"Results saved to {colorstr('bold', save_dir)}{s}") + final_metric = mp_bbox, mr_bbox, map50_bbox, map_bbox, mp_mask, mr_mask, map50_mask, map_mask + return (*final_metric, *(loss.cpu() / len(dataloader)).tolist()), metrics.get_maps(nc), t + + +def parse_opt(): + parser = argparse.ArgumentParser() + parser.add_argument('--data', type=str, default=ROOT / 'data/coco128-seg.yaml', help='dataset.yaml path') + parser.add_argument('--weights', nargs='+', type=str, default=ROOT / 'yolov5s-seg.pt', help='model path(s)') + parser.add_argument('--batch-size', type=int, default=32, help='batch size') + parser.add_argument('--imgsz', '--img', '--img-size', type=int, default=640, help='inference size (pixels)') + parser.add_argument('--conf-thres', type=float, default=0.001, help='confidence threshold') + parser.add_argument('--iou-thres', type=float, default=0.6, help='NMS IoU threshold') + parser.add_argument('--max-det', type=int, default=300, help='maximum detections per image') + parser.add_argument('--task', default='val', help='train, val, test, speed or study') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--workers', type=int, default=8, help='max dataloader workers (per RANK in DDP mode)') + parser.add_argument('--single-cls', action='store_true', help='treat as single-class dataset') + parser.add_argument('--augment', action='store_true', help='augmented inference') + parser.add_argument('--verbose', action='store_true', help='report mAP by class') + parser.add_argument('--save-txt', action='store_true', help='save results to *.txt') + parser.add_argument('--save-hybrid', action='store_true', help='save label+prediction hybrid results to *.txt') + parser.add_argument('--save-conf', action='store_true', help='save confidences in --save-txt labels') + parser.add_argument('--save-json', action='store_true', help='save a COCO-JSON results file') + parser.add_argument('--project', default=ROOT / 'runs/val-seg', help='save results to project/name') + parser.add_argument('--name', default='exp', help='save to project/name') + parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment') + parser.add_argument('--half', action='store_true', help='use FP16 half-precision inference') + parser.add_argument('--dnn', action='store_true', help='use OpenCV DNN for ONNX inference') + opt = parser.parse_args() + opt.data = check_yaml(opt.data) # check YAML + # opt.save_json |= opt.data.endswith('coco.yaml') + opt.save_txt |= opt.save_hybrid + print_args(vars(opt)) + return opt + + +def main(opt): + check_requirements(ROOT / 'requirements.txt', exclude=('tensorboard', 'thop')) + + if opt.task in ('train', 'val', 'test'): # run normally + if opt.conf_thres > 0.001: # https://github.com/ultralytics/yolov5/issues/1466 + LOGGER.warning(f'WARNING ⚠️ confidence threshold {opt.conf_thres} > 0.001 produces invalid results') + if opt.save_hybrid: + LOGGER.warning('WARNING ⚠️ --save-hybrid returns high mAP from hybrid labels, not from predictions alone') + run(**vars(opt)) + + else: + weights = opt.weights if isinstance(opt.weights, list) else [opt.weights] + opt.half = torch.cuda.is_available() and opt.device != 'cpu' # FP16 for fastest results + if opt.task == 'speed': # speed benchmarks + # python val.py --task speed --data coco.yaml --batch 1 --weights yolov5n.pt yolov5s.pt... + opt.conf_thres, opt.iou_thres, opt.save_json = 0.25, 0.45, False + for opt.weights in weights: + run(**vars(opt), plots=False) + + elif opt.task == 'study': # speed vs mAP benchmarks + # python val.py --task study --data coco.yaml --iou 0.7 --weights yolov5n.pt yolov5s.pt... + for opt.weights in weights: + f = f'study_{Path(opt.data).stem}_{Path(opt.weights).stem}.txt' # filename to save to + x, y = list(range(256, 1536 + 128, 128)), [] # x axis (image sizes), y axis + for opt.imgsz in x: # img-size + LOGGER.info(f'\nRunning {f} --imgsz {opt.imgsz}...') + r, _, t = run(**vars(opt), plots=False) + y.append(r + t) # results and times + np.savetxt(f, y, fmt='%10.4g') # save + subprocess.run(['zip', '-r', 'study.zip', 'study_*.txt']) + plot_val_study(x=x) # plot + else: + raise NotImplementedError(f'--task {opt.task} not in ("train", "val", "test", "speed", "study")') + + +if __name__ == '__main__': + opt = parse_opt() + main(opt) diff --git a/shopper.py b/shopper.py new file mode 100644 index 0000000..7e8bdd4 --- /dev/null +++ b/shopper.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +""" +Created on Tue Dec 17 10:45:10 2024 + +@author: ym +""" +import os +import numpy as np +import pandas as pd +from pathlib import Path + + + + + +xpath = r"\\192.168.1.28\share\模型\部署相关\永辉金源广场店商品信息.xlsx" +xroot, xfilename = os.path.split(xpath) +xfile, ext = os.path.splitext(xfilename) +spath = os.path.join(xroot, xfile+'_diff.xlsx') + + +df = pd.read_excel(xpath) +barcodes = df["商品条码"].tolist() +names_caojq = df["商品名称"].tolist() + +stdpath = r"\\192.168.1.28\share\数据\已完成数据\比对数据\barcode\all_totalBarocde\totalBarcode" + +stdpath = Path(stdpath) +stdBarcodes = [int(f.stem) for f in stdpath.iterdir() if f.is_dir() and f.stem.isdigit() and len(f.stem)>=8] + + +barcodes_s = set(barcodes) +stdBarcodes_s = set(stdBarcodes) + +A = barcodes_s - stdBarcodes_s + +record_bcd, record_name = [], [] +for bcd in A: + if np.isnan(bcd): continue + try: + index = barcodes.index(bcd) + name = names_caojq[index] + + record_bcd.append(bcd) + record_name.append(name) + except ValueError: + print(f"元素 {bcd} 不在列表中") + +df_save = pd.DataFrame({ + '商品条码': record_bcd, + '商品名称': record_name +}) + +df.to_excel(spath, index=False) + +print("Done") + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/track_.py b/track_.py new file mode 100644 index 0000000..342ebe8 --- /dev/null +++ b/track_.py @@ -0,0 +1,402 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Run YOLOv5 detection inference on images, videos, directories, globs, YouTube, webcam, streams, etc. + +Usage - sources: + $ python detect.py --weights yolov5s.pt --source 0 # webcam + img.jpg # image + vid.mp4 # video + screen # screenshot + path/ # directory + list.txt # list of images + list.streams # list of streams + 'path/*.jpg' # glob + 'https://youtu.be/Zgi9g1ksQHc' # YouTube + 'rtsp://example.com/media.mp4' # RTSP, RTMP, HTTP stream + +Usage - formats: + $ python detect.py --weights yolov5s.pt # PyTorch + yolov5s.torchscript # TorchScript + yolov5s.onnx # ONNX Runtime or OpenCV DNN with --dnn + yolov5s_openvino_model # OpenVINO + yolov5s.engine # TensorRT + yolov5s.mlmodel # CoreML (macOS-only) + yolov5s_saved_model # TensorFlow SavedModel + yolov5s.pb # TensorFlow GraphDef + yolov5s.tflite # TensorFlow Lite + yolov5s_edgetpu.tflite # TensorFlow Edge TPU + yolov5s_paddle_model # PaddlePaddle +""" + +import argparse +import csv +import os +import platform +import sys +from pathlib import Path +import glob +import numpy as np +import pickle +import torch + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[0] # YOLOv5 root directory +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) # add ROOT to PATH +ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative + +from models.common import DetectMultiBackend +from utils.dataloaders import IMG_FORMATS, VID_FORMATS, LoadImages, LoadScreenshots, LoadStreams +from utils.general import (LOGGER, Profile, check_file, check_img_size, check_imshow, check_requirements, colorstr, cv2, + increment_path, non_max_suppression, print_args, scale_boxes, strip_optimizer, xyxy2xywh) +from utils.torch_utils import select_device, smart_inference_mode + +'''集成跟踪模块，输出跟踪结果文件 .npy''' +# from ultralytics.engine.results import Boxes # Results +# from ultralytics.utils import IterableSimpleNamespace, yaml_load +from tracking.utils.plotting import Annotator, colors +from tracking.utils import Boxes, IterableSimpleNamespace, yaml_load, boxes_add_fid +from tracking.trackers import BOTSORT, BYTETracker +from tracking.utils.showtrack import drawtracks + + +def init_trackers(tracker_yaml = None, bs=1): + """ + Initialize trackers for object tracking during prediction. + """ + # 需要将配置文件中的cmc_method改为gmc_method + # tracker_yaml = r"D:\DeepLearning\ultralytics\ultralytics\tracker\cfg\botsort.yaml" + + tracker_yaml = r"./tracking/trackers/cfg/botsort.yaml" + + TRACKER_MAP = {'bytetrack': BYTETracker, 'botsort': BOTSORT} + + cfg = IterableSimpleNamespace(**yaml_load(tracker_yaml)) + trackers = [] + for _ in range(bs): + tracker = TRACKER_MAP[cfg.tracker_type](args=cfg, frame_rate=30) + trackers.append(tracker) + + return trackers + + +@smart_inference_mode() +def run( + weights=ROOT / 'yolov5s.pt', # model path or triton URL + source=ROOT / 'data/images', # file/dir/URL/glob/screen/0(webcam) + data=ROOT / 'data/coco128.yaml', # dataset.yaml path + imgsz=(640, 640), # inference size (height, width) + conf_thres=0.25, # confidence threshold + iou_thres=0.45, # NMS IOU threshold + max_det=1000, # maximum detections per image + device='', # cuda device, i.e. 0 or 0,1,2,3 or cpu + view_img=False, # show results + save_txt=False, # save results to *.txt + save_csv=False, # save results in CSV format + save_conf=False, # save confidences in --save-txt labels + save_crop=False, # save cropped prediction boxes + nosave=False, # do not save images/videos + classes=None, # filter by class: --class 0, or --class 0 2 3 + agnostic_nms=False, # class-agnostic NMS + augment=False, # augmented inference + visualize=False, # visualize features + update=False, # update all models + project=ROOT / 'runs/detect', # save results to project/name + name='exp', # save results to project/name + exist_ok=False, # existing project/name ok, do not increment + line_thickness=3, # bounding box thickness (pixels) + hide_labels=False, # hide labels + hide_conf=False, # hide confidencesL + half=False, # use FP16 half-precision inference + dnn=False, # use OpenCV DNN for ONNX inference + vid_stride=1, # video frame-rate stride +): + source = str(source) + save_img = not nosave and not source.endswith('.txt') # save inference images + is_file = Path(source).suffix[1:] in (IMG_FORMATS + VID_FORMATS) + is_url = source.lower().startswith(('rtsp://', 'rtmp://', 'http://', 'https://')) + webcam = source.isnumeric() or source.endswith('.streams') or (is_url and not is_file) + screenshot = source.lower().startswith('screen') + if is_url and is_file: + source = check_file(source) # download + + save_dir = Path(project) / Path(source).stem + if save_dir.exists(): + print(Path(source).stem) + # return + + save_dir = increment_path(Path(project) / name, exist_ok=exist_ok) # increment run + (save_dir / 'labels' if save_txt else save_dir).mkdir(parents=True, exist_ok=True) # make dir + else: + save_dir.mkdir(parents=True, exist_ok=True) + + # Load model + device = select_device(device) + model = DetectMultiBackend(weights, device=device, dnn=dnn, data=data, fp16=half) + stride, names, pt = model.stride, model.names, model.pt + imgsz = check_img_size(imgsz, s=stride) # check image size + + # Dataloader + bs = 1 # batch_size + + dataset = LoadImages(source, img_size=imgsz, stride=stride, auto=pt, vid_stride=vid_stride) + vid_path, vid_writer = [None] * bs, [None] * bs + + + # Run inference + model.warmup(imgsz=(1 if pt or model.triton else bs, 3, *imgsz)) # warmup + seen, windows, dt = 0, [], (Profile(), Profile(), Profile()) + + + ## ================================================================================== writed by WQG + tracker = init_trackers(bs)[0] + vid_path_track, vid_writer_track = [None] * bs, [None] * bs + tboxes = [] + vboxes = [] + + f_i = 1 + for path, im, im0s, vid_cap, s in dataset: + if f_i == 1: + f_i == 0 + imgshow = im0s.copy() + + ## ============================= tracking 功能只处理视频，writed by WQG + if dataset.mode == 'image': + continue + + with dt[0]: + im = torch.from_numpy(im).to(model.device) + im = im.half() if model.fp16 else im.float() # uint8 to fp16/32 + im /= 255 # 0 - 255 to 0.0 - 1.0 + if len(im.shape) == 3: + im = im[None] # expand for batch dim + + # Inference + with dt[1]: + visualize = increment_path(save_dir / Path(path).stem, mkdir=True) if visualize else False + pred = model(im, augment=augment, visualize=visualize) + + # NMS + with dt[2]: + pred = non_max_suppression(pred, conf_thres, iou_thres, classes, agnostic_nms, max_det=max_det) + + # Second-stage classifier (optional) + # pred = utils.general.apply_classifier(pred, classifier_model, im, im0s) + + # Define the path for the CSV file + csv_path = save_dir / 'predictions.csv' + + # Create or append to the CSV file + def write_to_csv(image_name, prediction, confidence): + data = {'Image Name': image_name, 'Prediction': prediction, 'Confidence': confidence} + with open(csv_path, mode='a', newline='') as f: + writer = csv.DictWriter(f, fieldnames=data.keys()) + if not csv_path.is_file(): + writer.writeheader() + writer.writerow(data) + + # Process predictions + for i, det in enumerate(pred): # per image + seen += 1 + if webcam: # batch_size >= 1 + p, im0, frame = path[i], im0s[i].copy(), dataset.count + s += f'{i}: ' + else: + p, im0, frame = path, im0s.copy(), getattr(dataset, 'frame', 0) + + p = Path(p) # to Path + save_path = str(save_dir / p.name) # im.jpg + + s += '%gx%g ' % im.shape[2:] # print string + + annotator = Annotator(im0, line_width=line_thickness, example=str(names)) + if len(det): + # Rescale boxes from img_size to im0 size + det[:, :4] = scale_boxes(im.shape[2:], det[:, :4], im0.shape).round() + + vboxes.append((det.cpu().numpy(), im0, frame)) + ## ================================================================ writed by WQG + det_tracking = Boxes(det, im0.shape).cpu().numpy() + tracks = tracker.update(det_tracking, im0) + + if len(tracks) > 0: + det = torch.as_tensor(tracks[:, :-2]) + tboxes.append((det, frame)) + else: + idmark = -1 * np.ones([det.shape[0], 1]) + det = np.concatenate([det[:,:4], idmark, det[:, 4:]], axis=1) + + for *xyxy, id, conf, cls in reversed(det): + name = ('' if id==-1 else f'id:{int(id)} ') + names[int(cls)] + label = None if hide_labels else (name if hide_conf else f'{name} {conf:.2f}') + + if id >=0 and cls==0: + color = colors(int(cls), True) + elif id >=0 and cls!=0: + color = colors(int(id), True) + else: + color = colors(19, True) # 19为调色板的最后一个元素 + + annotator.box_label(xyxy, label, color=color) + + # Save results (image and video with tracking) + im0 = annotator.result() + if save_img: + save_path_img, ext = os.path.splitext(save_path) + imgpath = save_path_img + f"_{dataset.frame}.png" + cv2.imwrite(Path(imgpath), im0) + + if vid_path[i] != save_path: # new video + vid_path[i] = save_path + if isinstance(vid_writer[i], cv2.VideoWriter): + vid_writer[i].release() # release previous video writer + if vid_cap: # video + fps = vid_cap.get(cv2.CAP_PROP_FPS) + w = int(vid_cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + h = int(vid_cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + else: # stream + fps, w, h = 30, im0.shape[1], im0.shape[0] + save_path = str(Path(save_path).with_suffix('.mp4')) # force *.mp4 suffix on results videos + vid_writer[i] = cv2.VideoWriter(save_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h)) + vid_writer[i].write(im0) + + # Print time (inference-only) + LOGGER.info(f"{s}{'' if len(det) else '(no detections), '}{dt[1].dt * 1E3:.1f}ms") + + + ## ======================================================================== written by WQG + ## tboxes: list, [(det, frame_index)] + ## bboxes: Array, [x, y, w, h, track_id, score, cls, frame_index] + + filename = os.path.split(save_path_img)[-1] + file, ext = os.path.splitext(filename) + + bboxes = boxes_add_fid(tboxes) + + imgshow = drawtracks(bboxes, file=filename) + + showpath_1 = save_path_img + "_show.png" + cv2.imwrite(Path(showpath_1), imgshow) + + ##================================================== save .npy + with open(f'./tracking/vboxes/{file}.pkl', 'wb') as file: + pickle.dump(vboxes, file) + + + boxes_dir = Path('./runs/boxes/') + if not boxes_dir.exists(): + boxes_dir.mkdir(parents=True, exist_ok=True) + bboxes_path = boxes_dir.joinpath(filename + ".npy") + np.save(bboxes_path, bboxes) + + + # Print results + t = tuple(x.t / seen * 1E3 for x in dt) # speeds per image + LOGGER.info(f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {(1, 3, *imgsz)}' % t) + if save_txt or save_img: + s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else '' + LOGGER.info(f"Results saved to {colorstr('bold', save_dir)}{s}") + if update: + strip_optimizer(weights[0]) # update model (to fix SourceChangeWarning) + + +def parse_opt(): + modelpath = ROOT / 'ckpts/best_158734_cls11_noaug10.pt' # 'ckpts/best_15000_0908.pt', 'ckpts/yolov5s.pt', 'ckpts/best_20000_cls30.pt' + + # datapath = r"D:/datasets/ym/videos/标记视频/" # ROOT/'data/videos', ROOT/'data/images' images + datapath = r"D:\datasets\ym\highvalue\videos" + + # datapath = r"D:/dcheng/videos/" + # modelpath = ROOT / 'ckpts/yolov5s.pt' + + parser = argparse.ArgumentParser() + parser.add_argument('--weights', nargs='+', type=str, default=modelpath, help='model path or triton URL') # 'yolov5s.pt', best_15000_0908.pt + parser.add_argument('--source', type=str, default=datapath, help='file/dir/URL/glob/screen/0(webcam)') # images, videos + parser.add_argument('--data', type=str, default=ROOT / 'data/coco128.yaml', help='(optional) dataset.yaml path') + parser.add_argument('--imgsz', '--img', '--img-size', nargs='+', type=int, default=[640], help='inference size h,w') + parser.add_argument('--conf-thres', type=float, default=0.25, help='confidence threshold') + parser.add_argument('--iou-thres', type=float, default=0.45, help='NMS IoU threshold') + parser.add_argument('--max-det', type=int, default=1000, help='maximum detections per image') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--view-img', action='store_true', help='show results') + parser.add_argument('--save-txt', action='store_true', help='save results to *.txt') + parser.add_argument('--save-csv', action='store_true', help='save results in CSV format') + parser.add_argument('--save-conf', action='store_true', help='save confidences in --save-txt labels') + parser.add_argument('--save-crop', action='store_true', help='save cropped prediction boxes') + parser.add_argument('--nosave', action='store_true', help='do not save images/videos') + parser.add_argument('--classes', nargs='+', type=int, help='filter by class: --classes 0, or --classes 0 2 3') + parser.add_argument('--agnostic-nms', action='store_true', help='class-agnostic NMS') + parser.add_argument('--augment', action='store_true', help='augmented inference') + parser.add_argument('--visualize', action='store_true', help='visualize features') + parser.add_argument('--update', action='store_true', help='update all models') + parser.add_argument('--project', default=ROOT / 'runs/detect', help='save results to project/name') + parser.add_argument('--name', default='exp', help='save results to project/name') + parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment') + parser.add_argument('--line-thickness', default=3, type=int, help='bounding box thickness (pixels)') + parser.add_argument('--hide-labels', default=False, action='store_true', help='hide labels') + parser.add_argument('--hide-conf', default=False, action='store_true', help='hide confidences') + parser.add_argument('--half', action='store_true', help='use FP16 half-precision inference') + parser.add_argument('--dnn', action='store_true', help='use OpenCV DNN for ONNX inference') + parser.add_argument('--vid-stride', type=int, default=1, help='video frame-rate stride') + opt = parser.parse_args() + opt.imgsz *= 2 if len(opt.imgsz) == 1 else 1 # expand + print_args(vars(opt)) + return opt + +def main_loop_folders(opt): + path1 = r"D:\datasets\ym\videos\标记视频" + path2 = r"D:\datasets\ym\永辉双摄视频\退购_前摄\videos" + path3 = r"D:\datasets\ym\永辉双摄视频\退购_后摄\videos" + path4 = r"D:\datasets\ym\永辉双摄视频\加购_前摄\videos" + path5 = r"D:\datasets\ym\永辉双摄视频\加购_后摄\videos" + paths = [path1] # [path1, path2, path3, path4, path5] + + check_requirements(ROOT / 'requirements.txt', exclude=('tensorboard', 'thop')) + optdict = vars(opt) + k = 0 + for p in paths: + files = [] + if os.path.isdir(p): + files.extend(sorted(glob.glob(os.path.join(p, '*.*')))) + for file in files: + # file = r"D:\datasets\ym\videos\标记视频\加购_100.mp4" + + optdict["source"] = file + run(**optdict) + + # k += 1 + # if k == 100: + # break + elif os.path.isfile(p): + run(**optdict) + + + + +def main(opt): + check_requirements(ROOT / 'requirements.txt', exclude=('tensorboard', 'thop')) + # files.extend(sorted(glob.glob(os.path.join(p, '*.*')))) + + optdict = vars(opt) + p = optdict["source"] + + files = [] + k = 0 + if os.path.isdir(p): + files.extend(sorted(glob.glob(os.path.join(p, '*.*')))) + + for file in files: + optdict["source"] = file + run(**optdict) + + k += 1 + if k == 100: + break + + elif os.path.isfile(p): + run(**vars(opt)) + + +if __name__ == '__main__': + opt = parse_opt() + main_loop_folders(opt) diff --git a/track_reid.py b/track_reid.py new file mode 100644 index 0000000..9dd39ee --- /dev/null +++ b/track_reid.py @@ -0,0 +1,959 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Run YOLOv5 detection inference on images, videos, directories, globs, YouTube, webcam, streams, etc. + +Usage - sources: + $ python detect.py --weights yolov5s.pt --source 0 # webcam + img.jpg # image + vid.mp4 # video + screen # screenshot + path/ # directory + list.txt # list of images + list.streams # list of streams + 'path/*.jpg' # glob + 'https://youtu.be/Zgi9g1ksQHc' # YouTube + 'rtsp://example.com/media.mp4' # RTSP, RTMP, HTTP stream + +Usage - formats: + $ python detect.py --weights yolov5s.pt # PyTorch + yolov5s.torchscript # TorchScript + yolov5s.onnx # ONNX Runtime or OpenCV DNN with --dnn + yolov5s_openvino_model # OpenVINO + yolov5s.engine # TensorRT + yolov5s.mlmodel # CoreML (macOS-only) + yolov5s_saved_model # TensorFlow SavedModel + yolov5s.pb # TensorFlow GraphDef + yolov5s.tflite # TensorFlow Lite + yolov5s_edgetpu.tflite # TensorFlow Edge TPU + yolov5s_paddle_model # PaddlePaddle +""" + +import argparse +import csv +import os +import platform +import sys +from pathlib import Path +import glob +import numpy as np +import pickle +import torch +from scipy.spatial.distance import cdist + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[0] # YOLOv5 root directory +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) # add ROOT to PATH +ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative + +from models.common import DetectMultiBackend +from utils.dataloaders import IMG_FORMATS, VID_FORMATS, LoadImages, LoadScreenshots, LoadStreams +from utils.general import (LOGGER, Profile, check_file, check_img_size, check_imshow, check_requirements, colorstr, cv2, + increment_path, non_max_suppression, print_args, scale_boxes, strip_optimizer, xyxy2xywh) +from utils.torch_utils import select_device, smart_inference_mode + +'''集成跟踪模块，输出跟踪结果文件 .npy''' +# from ultralytics.engine.results import Boxes # Results +# from ultralytics.utils import IterableSimpleNamespace, yaml_load +from tracking.utils.plotting import Annotator, colors +from tracking.utils import Boxes, IterableSimpleNamespace, yaml_load, boxes_add_fid +from tracking.trackers import BOTSORT, BYTETracker +from tracking.utils.showtrack import drawtracks +from hands.hand_inference import hand_pose + +from contrast.feat_extract.config import config as conf +from contrast.feat_extract.inference import FeatsInterface + +from ultralytics import YOLOv10 + +ReIDEncoder = FeatsInterface(conf) +print(f'load model {conf.testbackbone} in {Path(__file__).stem}') + +IMG_FORMATS = '.bmp', '.dng', '.jpeg', '.jpg', '.mpo', '.png', '.tif', '.tiff', '.webp', '.pfm' # include image suffixes +VID_FORMATS = '.asf', '.avi', '.gif', '.m4v', '.mkv', '.mov', '.mp4', '.mpeg', '.mpg', '.ts', '.wmv' # include video suffixes + +# from tracking.trackers.reid.reid_interface import ReIDInterface +# from tracking.trackers.reid.config import config as ReIDConfig +# ReIDEncoder = ReIDInterface(ReIDConfig) + +# tracker_yaml = r"./tracking/trackers/cfg/botsort.yaml" + +# def inference_image(image, detections): +# H, W, _ = np.shape(image) +# imgs = [] +# batch_patches = [] +# patches = [] +# for d in range(np.size(detections, 0)): +# tlbr = detections[d, :4].astype(np.int_) +# tlbr[0] = max(0, tlbr[0]) +# tlbr[1] = max(0, tlbr[1]) +# tlbr[2] = min(W - 1, tlbr[2]) +# tlbr[3] = min(H - 1, tlbr[3]) +# img1 = image[tlbr[1]:tlbr[3], tlbr[0]:tlbr[2], :] + +# img = img1[:, :, ::-1].copy() # the model expects RGB inputs +# patch = ReIDEncoder.transform(img) + +# imgs.append(img1) +# # patch = patch.to(device=self.device).half() +# if str(ReIDEncoder.device) != "cpu": +# patch = patch.to(device=ReIDEncoder.device).half() +# else: +# patch = patch.to(device=ReIDEncoder.device) + +# patches.append(patch) +# if (d + 1) % ReIDEncoder.batch_size == 0: +# patches = torch.stack(patches, dim=0) +# batch_patches.append(patches) +# patches = [] + +# if len(patches): +# patches = torch.stack(patches, dim=0) +# batch_patches.append(patches) + +# features = np.zeros((0, ReIDEncoder.embedding_size)) +# for patches in batch_patches: +# pred = ReIDEncoder.model(patches) +# pred[torch.isinf(pred)] = 1.0 +# feat = pred.cpu().data.numpy() +# features = np.vstack((features, feat)) + +# return imgs, features + + + +def init_trackers(tracker_yaml = None, bs=1): + """ + Initialize trackers for object tracking during prediction. + """ + # tracker_yaml = r"./tracking/trackers/cfg/botsort.yaml" + + tracker_yaml = str(tracker_yaml) + + TRACKER_MAP = {'bytetrack': BYTETracker, 'botsort': BOTSORT} + + cfg = IterableSimpleNamespace(**yaml_load(tracker_yaml)) + trackers = [] + for _ in range(bs): + tracker = TRACKER_MAP[cfg.tracker_type](args=cfg, frame_rate=30) + if cfg.with_reid: + tracker.encoder = ReIDEncoder + + trackers.append(tracker) + + return trackers + +'''=============== used in pipeline.py for Yolov10 ==================''' +def yolov10_resnet_tracker( + weights = ROOT / 'ckpts/best_v10s_width0375_1205.pt', # model path or triton URL + source = '', # file/dir/URL/glob/screen/0(webcam) + save_dir = '', + is_save_img = True, + is_save_video = True, + + tracker_yaml = ROOT / "tracking/trackers/cfg/botsort.yaml", + line_thickness=3, # bounding box thickness (pixels) + hide_labels=False, # hide labels + ): + + ## load a custom model + model = YOLOv10(weights) + + custom = {"conf": 0.1, "batch": 1, "save": False, "mode": "predict"} + kwargs = {"save": True, "imgsz": 640, "conf": 0.1} + args = {**model.overrides, **custom, **kwargs} + predictor = model.task_map[model.task]["predictor"](overrides=args, _callbacks=model.callbacks) + + vid_path, vid_writer = None, None + tracker = init_trackers(tracker_yaml)[0] + yoloResnetTracker = [] + for i, result in enumerate(predictor.stream_inference(source)): + datamode = predictor.dataset.mode + + det = result.boxes.data.cpu().numpy() + im0 = result.orig_img + names = result.names + path = result.path + im_array = result.plot() + + + ## to do tracker.update() + det_tracking = Boxes(det, im0.shape) + tracks, outfeats = tracker.update(det_tracking, im0) + + + + if datamode == "video": + frameId = predictor.dataset.frame + elif datamode == "image": + frameId = predictor.dataset.count + annotator = Annotator(im0.copy(), line_width=line_thickness, example=str(names)) + + simdict, simdict1 = {}, {} + for fid, bid, mfeat, cfeat, features in outfeats: + if mfeat is not None and cfeat is not None: + simi = 1 - np.maximum(0.0, cdist(mfeat[None, :], cfeat[None, :], "cosine"))[0][0] + simdict.update({f"{int(frameId)}_{int(bid)}":simi}) + + if cfeat is not None and len(features)>=2: + mfeat = features[-2] + simi = 1 - np.maximum(0.0, cdist(mfeat[None, :], cfeat[None, :], "cosine"))[0][0] + simdict1.update({f"{int(frameId)}_{int(bid)}":simi}) + + + if len(tracks) > 0: + tracks[:, 7] = frameId + # trackerBoxes = np.concatenate([trackerBoxes, tracks], axis=0) + '''================== 1. 存储 dets/subimgs/features Dict =============''' + imgs, features = ReIDEncoder.inference(im0, tracks) + imgdict, featdict = {}, {} + for ii, bid in enumerate(tracks[:, 8]): + featdict.update({f"{int(frameId)}_{int(bid)}": features[ii, :]}) # [f"feat_{int(bid)}"] = features[i, :] + imgdict.update({f"{int(frameId)}_{int(bid)}": imgs[ii]}) + + frameDict = {"path": path, + "fid": int(frameId), + "bboxes": det, + "tboxes": tracks, + "imgs": imgdict, + "feats": featdict, + "featsimi": simdict, # 当前 box 特征和该轨迹 smooth_feat 特征的相似度 + "featsimi1": simdict1 # 当前 box 特征和该轨迹前一个 box 特征的相似度 + } + yoloResnetTracker.append(frameDict) + + # imgs, features = inference_image(im0, tracks) + # TrackerFeats = np.concatenate([TrackerFeats, features], axis=0) + + '''================== 2. 提取手势位置 ===================''' + for *xyxy, id, conf, cls, fid, bid in reversed(tracks): + name = ('' if id==-1 else f'id:{int(id)} ') + names[int(cls)] + if f"{int(frameId)}_{int(bid)}" in simdict.keys(): + sim = simdict[f"{int(frameId)}_{int(bid)}"] + label = f"{name} {sim:.2f}" + else: + label = None if hide_labels else name + + + # label = None if hide_labels else (name if hide_conf else f'{name} {conf:.1f}') + + if id >=0 and cls==0: + color = colors(int(cls), True) + elif id >=0 and cls!=0: + color = colors(int(id), True) + else: + color = colors(19, True) # 19为调色板的最后一个元素 + annotator.box_label(xyxy, label, color=color) + + '''====== Save results (image and video) ======''' + # save_path = str(save_dir / Path(path).name) # 带有后缀名 + im0 = annotator.result() + if is_save_img: + save_path_img = str(save_dir / Path(path).stem) + if datamode == 'image': + imgpath = save_path_img + ".png" + if datamode == 'video' : + imgpath = save_path_img + f"_{frameId}.png" + cv2.imwrite(Path(imgpath), im0) + + # if dataset.mode == 'video' and is_save_video: + + if is_save_video: + if datamode == 'video': + video_path = str(save_dir / Path(path).stem) + '.mp4' # 带有后缀名 + else: + videoname = str(Path(path).stem).split('_')[0] + '.mp4' + video_path = str(save_dir / videoname) + + if vid_path != video_path: # new video + vid_path = video_path + vid_cap = predictor.dataset.cap + + if isinstance(vid_writer, cv2.VideoWriter): + vid_writer.release() # release previous video writer + if vid_cap: # video + fps = vid_cap.get(cv2.CAP_PROP_FPS) + w = int(vid_cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + h = int(vid_cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + else: # stream + fps, w, h = 25, im0.shape[1], im0.shape[0] + ## for image rotating in dataloader.LoadImages.__next__() + w, h = im0.shape[1], im0.shape[0] + + video_path = str(Path(video_path).with_suffix('.mp4')) # force *.mp4 suffix on results videos + vid_writer = cv2.VideoWriter(video_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h)) + vid_writer.write(im0) + + return yoloResnetTracker + + +'''=============== used in pipeline.py for Yolov5 ==================''' +@smart_inference_mode() +def yolo_resnet_tracker( + weights=ROOT / 'yolov5s.pt', # model path or triton URL + source=ROOT / 'data/images', # file/dir/URL/glob/screen/0(webcam) + save_dir = '', + is_save_img = True, + is_save_video = True, + + tracker_yaml = ROOT / "tracking/trackers/cfg/botsort.yaml", + imgsz=(640, 640), # inference size (height, width) + conf_thres=0.25, # confidence threshold + iou_thres=0.45, # NMS IOU threshold + max_det=1000, # maximum detections per image + device='', # cuda device, i.e. 0 or 0,1,2,3 or cpu + + classes=None, # filter by class: --class 0, or --class 0 2 3 + agnostic_nms=False, # class-agnostic NMS + augment=False, # augmented inference + + line_thickness=3, # bounding box thickness (pixels) + hide_labels=False, # hide labels + hide_conf=False, # hide confidencesL + half=False, # use FP16 half-precision inference + dnn=False, # use OpenCV DNN for ONNX inference + vid_stride=1, # video frame-rate stride + data=ROOT / 'data/coco128.yaml', # dataset.yaml path +): + # source = str(source) + # Load model + device = select_device(device) + model = DetectMultiBackend(weights, device=device, dnn=dnn, data=data, fp16=half) + stride, names, pt = model.stride, model.names, model.pt + imgsz = check_img_size(imgsz, s=stride) # check image size + + + + # Dataloader + bs = 1 # batch_size + + dataset = LoadImages(source, img_size=imgsz, stride=stride, auto=pt, vid_stride=vid_stride) + vid_path, vid_writer = [None] * bs, [None] * bs + + # Run inference + model.warmup(imgsz=(1 if pt or model.triton else bs, 3, *imgsz)) # warmup + tracker = init_trackers(tracker_yaml, bs)[0] + + dt = (Profile(), Profile(), Profile()) + # trackerBoxes = np.empty((0, 9), dtype = np.float32) + yoloResnetTracker = [] + for path, im, im0s, vid_cap, s in dataset: + with dt[0]: + im = torch.from_numpy(im).to(model.device) + im = im.half() if model.fp16 else im.float() # uint8 to fp16/32 + im /= 255 # 0 - 255 to 0.0 - 1.0 + if len(im.shape) == 3: + im = im[None] # expand for batch dim + + # Inference + with dt[1]: + # visualize = increment_path(project / Path(path).stem, mkdir=True) if visualize else False + pred = model(im, augment=augment, visualize=False) + + # NMS + with dt[2]: + pred = non_max_suppression(pred, conf_thres, iou_thres, classes, agnostic_nms, max_det=max_det) + + if dataset.mode == "video": + frameId = dataset.frame + else: + frameId = dataset.count + # Process predictions + for i, det in enumerate(pred): # per image + im0 = im0s.copy() + + annotator = Annotator(im0.copy(), line_width=line_thickness, example=str(names)) + s += '%gx%g ' % im.shape[2:] # print string + if len(det): + # Rescale boxes from img_size to im0 size + det[:, :4] = scale_boxes(im.shape[2:], det[:, :4], im0.shape).round() + + det = det.cpu().numpy() + ## ================================================================ writed by WQG + '''tracks: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] + 0 1 2 3 4 5 6 7 8 + 这里，frame_index 也可以用视频的 帧ID 代替, box_index 保持不变 + ''' + det_tracking = Boxes(det, im0.shape).cpu().numpy() + tracks, outfeats = tracker.update(det_tracking, im0) + + simdict, simdict1 = {}, {} + for fid, bid, mfeat, cfeat, features in outfeats: + if mfeat is not None and cfeat is not None: + simi = 1 - np.maximum(0.0, cdist(mfeat[None, :], cfeat[None, :], "cosine"))[0][0] + simdict.update({f"{int(frameId)}_{int(bid)}":simi}) + + if cfeat is not None and len(features)>=2: + mfeat = features[-2] + simi = 1 - np.maximum(0.0, cdist(mfeat[None, :], cfeat[None, :], "cosine"))[0][0] + simdict1.update({f"{int(frameId)}_{int(bid)}":simi}) + + + if len(tracks) > 0: + tracks[:, 7] = frameId + # trackerBoxes = np.concatenate([trackerBoxes, tracks], axis=0) + '''================== 1. 存储 dets/subimgs/features Dict =============''' + imgs, features = ReIDEncoder.inference(im0, tracks) + imgdict, featdict = {}, {} + for ii, bid in enumerate(tracks[:, 8]): + featdict.update({f"{int(frameId)}_{int(bid)}": features[ii, :]}) # [f"feat_{int(bid)}"] = features[i, :] + imgdict.update({f"{int(frameId)}_{int(bid)}": imgs[ii]}) + + frameDict = {"path": path, + "fid": int(frameId), + "bboxes": det, + "tboxes": tracks, + "imgs": imgdict, + "feats": featdict, + "featsimi": simdict, # 当前 box 特征和该轨迹 smooth_feat 特征的相似度 + "featsimi1": simdict1 # 当前 box 特征和该轨迹前一个 box 特征的相似度 + } + yoloResnetTracker.append(frameDict) + + # imgs, features = inference_image(im0, tracks) + # TrackerFeats = np.concatenate([TrackerFeats, features], axis=0) + + '''================== 2. 提取手势位置 ===================''' + for *xyxy, id, conf, cls, fid, bid in reversed(tracks): + name = ('' if id==-1 else f'id:{int(id)} ') + names[int(cls)] + if f"{int(frameId)}_{int(bid)}" in simdict.keys(): + sim = simdict[f"{int(frameId)}_{int(bid)}"] + label = f"{name} {sim:.2f}" + else: + label = None if hide_labels else name + + + # label = None if hide_labels else (name if hide_conf else f'{name} {conf:.1f}') + + if id >=0 and cls==0: + color = colors(int(cls), True) + elif id >=0 and cls!=0: + color = colors(int(id), True) + else: + color = colors(19, True) # 19为调色板的最后一个元素 + annotator.box_label(xyxy, label, color=color) + + '''====== Save results (image and video) ======''' + # save_path = str(save_dir / Path(path).name) # 带有后缀名 + im0 = annotator.result() + if is_save_img: + save_path_img = str(save_dir / Path(path).stem) + if dataset.mode == 'image': + imgpath = save_path_img + ".png" + else: + imgpath = save_path_img + f"_{frameId}.png" + cv2.imwrite(Path(imgpath), im0) + + # if dataset.mode == 'video' and is_save_video: + + if is_save_video: + if dataset.mode == 'video': + vdieo_path = str(save_dir / Path(path).stem) + '.mp4' # 带有后缀名 + else: + videoname = str(Path(path).stem).split('_')[0] + '.mp4' + vdieo_path = str(save_dir / videoname) + + if vid_path[i] != vdieo_path: # new video + vid_path[i] = vdieo_path + if isinstance(vid_writer[i], cv2.VideoWriter): + vid_writer[i].release() # release previous video writer + if vid_cap: # video + fps = vid_cap.get(cv2.CAP_PROP_FPS) + w = int(vid_cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + h = int(vid_cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + else: # stream + fps, w, h = 25, im0.shape[1], im0.shape[0] + ## for image rotating in dataloader.LoadImages.__next__() + + w, h = im0.shape[1], im0.shape[0] + + vdieo_path = str(Path(vdieo_path).with_suffix('.mp4')) # force *.mp4 suffix on results videos + vid_writer[i] = cv2.VideoWriter(vdieo_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h)) + vid_writer[i].write(im0) + + # Print time (inference-only) + LOGGER.info(f"{s}{'' if len(det) else '(no detections), '}{dt[1].dt * 1E3:.1f}ms") + + + return yoloResnetTracker + + + + +@smart_inference_mode() +def run( + weights=ROOT / 'yolov5s.pt', # model path or triton URL + source=ROOT / 'data/images', # file/dir/URL/glob/screen/0(webcam) + + project=ROOT / 'runs/detect', # save results to project/name + name='exp', # save results to project/name + + tracker_yaml = ROOT / "tracking/trackers/cfg/botsort.yaml", + imgsz=(640, 640), # inference size (height, width) + conf_thres=0.25, # confidence threshold + iou_thres=0.45, # NMS IOU threshold + max_det=1000, # maximum detections per image + device='', # cuda device, i.e. 0 or 0,1,2,3 or cpu + view_img=False, # show results + save_txt=False, # save results to *.txt + save_csv=False, # save results in CSV format + save_conf=False, # save confidences in --save-txt labels + save_crop=False, # save cropped prediction boxes + nosave=False, # do not save images/videos + classes=None, # filter by class: --class 0, or --class 0 2 3 + agnostic_nms=False, # class-agnostic NMS + augment=False, # augmented inference + visualize=False, # visualize features + update=False, # update all models + exist_ok=False, # existing project/name ok, do not increment + line_thickness=3, # bounding box thickness (pixels) + hide_labels=False, # hide labels + hide_conf=False, # hide confidencesL + half=False, # use FP16 half-precision inference + dnn=False, # use OpenCV DNN for ONNX inference + vid_stride=1, # video frame-rate stride + data=ROOT / 'data/coco128.yaml', # dataset.yaml path +): + ''' + source: 视频文件或图像列表 + ''' + source = str(source) + # filename = os.path.split(source)[-1] + + save_img = not nosave and not source.endswith('.txt') # save inference images + is_file = Path(source).suffix[1:] in (IMG_FORMATS + VID_FORMATS) + is_url = source.lower().startswith(('rtsp://', 'rtmp://', 'http://', 'https://')) + webcam = source.isnumeric() or source.endswith('.streams') or (is_url and not is_file) + screenshot = source.lower().startswith('screen') + if is_url and is_file: + source = check_file(source) # download + + + + + + # spth = source.split('\\')[-2] + "_" + Path(source).stem + save_dir = Path(project) / Path(source.split('\\')[-2] + "_" + str(Path(source).stem)) + # save_dir = Path(project) / Path(source).stem + if save_dir.exists(): + print(Path(source).stem) + # return + + save_dir = increment_path(Path(project) / name, exist_ok=exist_ok) # increment run + (save_dir / 'labels' if save_txt else save_dir).mkdir(parents=True, exist_ok=True) # make dir + else: + save_dir.mkdir(parents=True, exist_ok=True) + + # Load model + device = select_device(device) + model = DetectMultiBackend(weights, device=device, dnn=dnn, data=data, fp16=half) + stride, names, pt = model.stride, model.names, model.pt + imgsz = check_img_size(imgsz, s=stride) # check image size + + # Dataloader + bs = 1 # batch_size + + dataset = LoadImages(source, img_size=imgsz, stride=stride, auto=pt, vid_stride=vid_stride) + vid_path, vid_writer = [None] * bs, [None] * bs + + + # Run inference + model.warmup(imgsz=(1 if pt or model.triton else bs, 3, *imgsz)) # warmup + seen, dt = 0, (Profile(), Profile(), Profile()) + + tracker = init_trackers(tracker_yaml, bs)[0] + + handpose = hand_pose() + handlocals_dict = {} + + boxes_and_imgs = [] + + BoxesFeats = [] + + + + track_boxes = np.empty((0, 9), dtype = np.float32) + det_boxes = np.empty((0, 9), dtype = np.float32) + + DetBoxes = np.empty((0, 6), dtype = np.float32) + TrackerBoxes = np.empty((0, 9), dtype = np.float32) + TrackerFeats = np.empty((0, 256), dtype = np.float32) + + features_dict = {} + TracksDict = {} + for path, im, im0s, vid_cap, s in dataset: + if save_img and 'imgshow' not in locals().keys(): + imgshow = im0s.copy() + + ## ============================= tracking 功能只处理视频，writed by WQG + # if dataset.mode == 'image': + # continue + + with dt[0]: + im = torch.from_numpy(im).to(model.device) + im = im.half() if model.fp16 else im.float() # uint8 to fp16/32 + im /= 255 # 0 - 255 to 0.0 - 1.0 + if len(im.shape) == 3: + im = im[None] # expand for batch dim + + # Inference + with dt[1]: + visualize = increment_path(save_dir / Path(path).stem, mkdir=True) if visualize else False + pred = model(im, augment=augment, visualize=visualize) + + # NMS + with dt[2]: + pred = non_max_suppression(pred, conf_thres, iou_thres, classes, agnostic_nms, max_det=max_det) + + # Process predictions + for i, det in enumerate(pred): # per image + seen += 1 + if webcam: # batch_size >= 1 + p, im0, frame = path[i], im0s[i].copy(), dataset.count + s += f'{i}: ' + else: + p, im0, frame = path, im0s.copy(), getattr(dataset, 'frame', 0) + + + + + s += '%gx%g ' % im.shape[2:] # print string + + # im0_ant = im0.copy() + annotator = Annotator(im0.copy(), line_width=line_thickness, example=str(names)) + + nd = len(det) + if nd: + # Rescale boxes from img_size to im0 size + det[:, :4] = scale_boxes(im.shape[2:], det[:, :4], im0.shape).round() + + # det = det.cpu().numpy() + ## ============================================================ 前后帧相同 boxes 的特征赋值 + # def static_estimate(box1, box2, TH1=8, TH2=12): + # dij_abs = max(np.abs(box1 - box2)) + # dij_euc = max([np.linalg.norm((box1[:2] - box2[:2])), + # np.linalg.norm((box1[2:4] - box2[2:4])) + # ]) + # if dij_abs < TH1 and dij_euc < TH2: + # return True + # else: + # return False + + # nw = 3 # 向前递推检查的窗口大小 + # nf = len(BoxesFeats) # 已经检测+特征提取的帧数 + # feat_curr = [None] * nd # nd: 当前帧检测出的boxes数 + # for ii in range(nd): + # box = det[ii, :4] + + # kk=1 + # feat = None + # while kk <= nw and nf>=kk: + # ki = -1 * kk + # boxes_ = BoxesFeats[ki][0] + # feats_ = BoxesFeats[ki][1] + + # flag = [jj for jj in range(len(boxes_)) if static_estimate(box, boxes_[jj, :4])] + # if len(flag) == 1: + # feat = feats_[flag[0]] + # break + # kk += 1 + # if feat is not None: + # feat_curr[ii] = feat + + + ## ================================================================ writed by WQG + '''tracks: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] + 0 1 2 3 4 5 6 7 8 + 这里，frame_index 也可以用视频的 帧ID 代替, box_index 保持不变 + ''' + + det_tracking = Boxes(det, im0.shape).cpu().numpy() + tracks, outfeats = tracker.update(det_tracking, im0) + if len(tracks) == 0: + continue + + if dataset.mode == "video": + frameId = dataset.frame + else: + frameId = dataset.count + tracks[:, 7] = frameId + + tracks[:, 7] = frameId + + '''================== 1. 存储 dets/subimgs/features Dict =============''' + # imgs, features = inference_image(im0, tracks) + imgs, features = ReIDEncoder.inference(im0, tracks) + + TrackerFeats = np.concatenate([TrackerFeats, features], axis=0) + + imgdict = {} + boxdict = {} + featdict = {} + for ii, bid in enumerate(tracks[:, 8]): + imgdict.update({int(bid): imgs[ii]}) # [f"img_{int(bid)}"] = imgs[i] + boxdict.update({int(bid): tracks[ii, :]}) # [f"box_{int(bid)}"] = tracks[i, :] + featdict.update({int(bid): features[ii, :]}) # [f"feat_{int(bid)}"] = features[i, :] + TracksDict[f"frame_{int(frameId)}"] = {"imgs":imgdict, "boxes":boxdict, "feats":featdict} + + track_boxes = np.concatenate([track_boxes, tracks], axis=0) + + '''================== 2. 提取手势位置 ===================''' + # idx_0 = tracks[:, 6].astype(np.int_) == 0 + # hn = 0 + # for j, index in enumerate(idx_0): + # if index: + # track = tracks[j, :] + # hand_local, imgshow = handpose.get_hand_local(track, im0) + # handlocals_dict.update({int(track[7]): {int(track[8]): hand_local}}) + + # # '''yoloV5和手势检测的召回率并不一直，用hand_local代替tracks中手部的(x1, y1, x2, y2)，会使得两种坐标方式混淆''' + # # if hand_local: tracks[j, :4] = hand_local + + # hn += 1 + # cv2.imwrite(f"D:\DeepLearning\yolov5\hands\images\{Path(source).stem}_{int(track[7])}_{hn}.png", imgshow) + + for *xyxy, id, conf, cls, fid, bid in reversed(tracks): + name = ('' if id==-1 else f'id:{int(id)} ') + names[int(cls)] + label = None if hide_labels else (name if hide_conf else f'{name} {conf:.2f}') + + if id >=0 and cls==0: + color = colors(int(cls), True) + elif id >=0 and cls!=0: + color = colors(int(id), True) + else: + color = colors(19, True) # 19为调色板的最后一个元素 + + annotator.box_label(xyxy, label, color=color) + + # Save results (image and video with tracking) + im0 = annotator.result() + + p = Path(p) # to Path + save_path = str(save_dir / p.name) # im.jpg + if save_img: + save_path_img, ext = os.path.splitext(save_path) + if dataset.mode == 'image': + imgpath = save_path_img + ".png" + else: + imgpath = save_path_img + f"_{frameId}.png" + cv2.imwrite(Path(imgpath), im0) + + if dataset.mode == 'video': + + if vid_path[i] != save_path: # new video + vid_path[i] = save_path + if isinstance(vid_writer[i], cv2.VideoWriter): + vid_writer[i].release() # release previous video writer + if vid_cap: # video + fps = vid_cap.get(cv2.CAP_PROP_FPS) + w = int(vid_cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + h = int(vid_cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + else: # stream + fps, w, h = 30, im0.shape[1], im0.shape[0] + + ## for image rotating in dataloader.LoadImages.__next__() + w, h = im0.shape[1], im0.shape[0] + save_path = str(Path(save_path).with_suffix('.mp4')) # force *.mp4 suffix on results videos + vid_writer[i] = cv2.VideoWriter(save_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h)) + vid_writer[i].write(im0) + + + # Print time (inference-only) + LOGGER.info(f"{s}{'' if len(det) else '(no detections), '}{dt[1].dt * 1E3:.1f}ms") + + if track_boxes.size == 0: + return + + ## ======================================================================== written by WQG + ## track_boxes: Array, [x1, y1, x2, y2, track_id, score, cls, frame_index, box_id] + TracksDict.update({"TrackBoxes": track_boxes}) + + + '''上面保存了检测结果是视频和图像，以下还保存五种类型的数据''' + filename = os.path.split(save_path_img)[-1] + + '''======================== 1. save in './run/detect/' ====================''' + if source.find("front") >= 0 or Path(source).stem.split('_')[0] == '1': + carttemp = cv2.imread("./tracking/shopcart/cart_tempt/board_ftmp_line.png") + else: + carttemp = cv2.imread("./tracking/shopcart/cart_tempt/edgeline.png") + imgshow = drawtracks(track_boxes, carttemp) + showpath_1 = save_path_img + "_show.png" + cv2.imwrite(Path(showpath_1), imgshow) + + '''======================== 2. save dets/subimgs/features Dict ==================''' + trackdicts_dir = Path('./tracking/data/trackdicts/') + if not trackdicts_dir.exists(): + trackdicts_dir.mkdir(parents=True, exist_ok=True) + trackdicts_dir = trackdicts_dir.joinpath(f'{filename}.pkl') + with open(trackdicts_dir, 'wb') as file: + pickle.dump(TracksDict, file) + + # np.save(f'{filename}.npy', DetBoxes) + + + '''======================== 3. save hand_local data ==================''' + # handlocal_dir = Path('./tracking/data/handlocal/') + # if not handlocal_dir.exists(): + # handlocal_dir.mkdir(parents=True, exist_ok=True) + # handlocal_path = handlocal_dir.joinpath(f'{filename}.pkl') + # with open(handlocal_path, 'wb') as file: + # pickle.dump(handlocals_dict, file) + + + # Print results + t = tuple(x.t / seen * 1E3 for x in dt) # speeds per image + LOGGER.info(f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {(1, 3, *imgsz)}' % t) + if save_txt or save_img: + s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else '' + LOGGER.info(f"Results saved to {colorstr('bold', save_dir)}{s}") + if update: + strip_optimizer(weights[0]) # update model (to fix SourceChangeWarning) + + +def parse_opt(): + modelpath = ROOT / 'ckpts/best_cls10_0906.pt' # 'ckpts/best_15000_0908.pt', 'ckpts/yolov5s.pt', 'ckpts/best_20000_cls30.pt, best_yolov5m_250000' + '''datapath为视频文件目录或视频文件''' + datapath = r"D:/datasets/ym/videos/标记视频/" # ROOT/'data/videos', ROOT/'data/images' images + # datapath = r"D:\datasets\ym\highvalue\videos" + # datapath = r"D:/dcheng/videos/" + # modelpath = ROOT / 'ckpts/yolov5s.pt' + + parser = argparse.ArgumentParser() + parser.add_argument('--weights', nargs='+', type=str, default=modelpath, help='model path or triton URL') # 'yolov5s.pt', best_15000_0908.pt + parser.add_argument('--source', type=str, default=datapath, help='file/dir/URL/glob/screen/0(webcam)') # images, videos + parser.add_argument('--data', type=str, default=ROOT / 'data/coco128.yaml', help='(optional) dataset.yaml path') + parser.add_argument('--imgsz', '--img', '--img-size', nargs='+', type=int, default=[640], help='inference size h,w') + parser.add_argument('--conf-thres', type=float, default=0.25, help='confidence threshold') + parser.add_argument('--iou-thres', type=float, default=0.45, help='NMS IoU threshold') + parser.add_argument('--max-det', type=int, default=1000, help='maximum detections per image') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--view-img', action='store_true', help='show results') + parser.add_argument('--save-txt', action='store_true', help='save results to *.txt') + parser.add_argument('--save-csv', action='store_true', help='save results in CSV format') + parser.add_argument('--save-conf', action='store_true', help='save confidences in --save-txt labels') + parser.add_argument('--save-crop', action='store_true', help='save cropped prediction boxes') + parser.add_argument('--nosave', action='store_true', help='do not save images/videos') + parser.add_argument('--classes', nargs='+', type=int, help='filter by class: --classes 0, or --classes 0 2 3') + parser.add_argument('--agnostic-nms', action='store_true', help='class-agnostic NMS') + parser.add_argument('--augment', action='store_true', help='augmented inference') + parser.add_argument('--visualize', action='store_true', help='visualize features') + parser.add_argument('--update', action='store_true', help='update all models') + parser.add_argument('--project', default=ROOT / 'runs/detect', help='save results to project/name') + parser.add_argument('--name', default='exp', help='save results to project/name') + parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment') + parser.add_argument('--line-thickness', default=3, type=int, help='bounding box thickness (pixels)') + parser.add_argument('--hide-labels', default=False, action='store_true', help='hide labels') + parser.add_argument('--hide-conf', default=False, action='store_true', help='hide confidences') + parser.add_argument('--half', action='store_true', help='use FP16 half-precision inference') + parser.add_argument('--dnn', action='store_true', help='use OpenCV DNN for ONNX inference') + parser.add_argument('--vid-stride', type=int, default=1, help='video frame-rate stride') + opt = parser.parse_args() + opt.imgsz *= 2 if len(opt.imgsz) == 1 else 1 # expand + print_args(vars(opt)) + return opt + +def find_video_imgs(root_dir): + all_files = [] + extensions = ['.mp4'] + for dirpath, dirnames, filenames in os.walk(root_dir): + for filename in filenames: + file, ext = os.path.splitext(filename) + if ext in IMG_FORMATS + VID_FORMATS: + all_files.append(os.path.join(dirpath, filename)) + return all_files + + + +def main_v5(): + ''' + run(): 单张图像或单个视频文件的推理，不支持图像序列， + ''' + + check_requirements(ROOT / 'requirements.txt', exclude=('tensorboard', 'thop')) + opt = parse_opt() + optdict = vars(opt) + + # p = r"D:\datasets\ym\永辉测试数据_比对" + # p = r"D:\datasets\ym\广告板遮挡测试\8" + # p = r"D:\datasets\ym\videos\标记视频" + # p = r"D:\datasets\ym\实验室测试" + # p = r"D:\datasets\ym\永辉双摄视频\新建文件夹" + # p = r"\\192.168.1.28\share\测试_202406\0723\0723_2\20240723-112522_" + # p = r"D:\datasets\ym\联华中环" + # p = r"D:\exhibition\images\153112511_0_seek_105.mp4" + # p = r"D:\exhibition\images\image" + + p = r"D:\datasets\ym\后台数据\unzip\20250310-175352-741" + optdict["project"] = r"D:\work\result" + + optdict["weights"] = ROOT / 'ckpts/best_cls10_0906.pt' + if os.path.isdir(p): + files = find_video_imgs(p) + k = 0 + for file in files: + optdict["source"] = file + run(**optdict) + + k += 1 + if k == 2: + break + elif os.path.isfile(p): + optdict["source"] = p + run(**optdict) + + +def main_v10(): + datapath = r'D:\datasets\ym\后台数据\unzip\20250310-175352-741\0.mp4' + savepath = r'D:\work\result' + savepath = savepath / Path(str(Path(datapath).stem)) + if not savepath.exists(): + savepath.mkdir(parents=True, exist_ok=True) + + weightpath = ROOT / 'ckpts/best_v10s_width0375_1205.pt' + + optdict = {} + optdict["weights"] = weightpath + optdict["source"] = datapath + optdict["save_dir"] = savepath + optdict["is_save_img"] = True + optdict["is_save_video"] = True + + yrtOut = yolov10_resnet_tracker(**optdict) + + + + +if __name__ == '__main__': + # main_v5() + + + main_v10() + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tracking/__init__.py b/tracking/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tracking/__pycache__/__init__.cpython-312.pyc b/tracking/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..e1f826c Binary files /dev/null and b/tracking/__pycache__/__init__.cpython-312.pyc differ diff --git a/tracking/__pycache__/__init__.cpython-39.pyc b/tracking/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..d1bcf8a Binary files /dev/null and b/tracking/__pycache__/__init__.cpython-39.pyc differ diff --git a/tracking/__pycache__/contrast_analysis.cpython-39.pyc b/tracking/__pycache__/contrast_analysis.cpython-39.pyc new file mode 100644 index 0000000..80898f0 Binary files /dev/null and b/tracking/__pycache__/contrast_analysis.cpython-39.pyc differ diff --git a/tracking/data/说明文档.txt b/tracking/data/说明文档.txt new file mode 100644 index 0000000..983d60b --- /dev/null +++ b/tracking/data/说明文档.txt @@ -0,0 +1 @@ +文件夹 trackdicts_20240608 和 trackdicts_1 下的数据为和手部关联前的跟踪结果数据 \ No newline at end of file diff --git a/tracking/deprecated/contrast_one2one.py b/tracking/deprecated/contrast_one2one.py new file mode 100644 index 0000000..37b10fe --- /dev/null +++ b/tracking/deprecated/contrast_one2one.py @@ -0,0 +1,337 @@ +# -*- coding: utf-8 -*- +""" +Created on Fri Aug 30 17:53:03 2024 + have Deprecated! + +1. 确认在相同CamerType下，track.data 中 CamerID 项数量 = 图像数 = 帧ID数 = 最大帧ID + +2. 读取0/1_tracking_output.data 中数据，boxes、feats，len(boxes)=len(feats) + 帧ID约束 + +3. 优先选择前摄 + +4. 保存图像数据 + +5. 一次购物事件类型 + shopEvent: {barcode: + type: getout, input + front_traj:[{imgpath: str, + box: arrar(1, 9), + feat: array(1, 256) + }] + back_traj: [{imgpath: str, + box: arrar(1, 9), + feat: array(1, 256) + }] + } + + + +@author: ym + +""" +import numpy as np +import cv2 +import os +import sys +import json +import pickle +sys.path.append(r"D:\DetectTracking") +from tracking.utils.read_data import extract_data, read_tracking_output, read_deletedBarcode_file + +IMG_FORMAT = ['.bmp', '.jpg', '.jpeg', '.png'] + +def creat_shopping_event(basepath): + eventList = [] + + '''一、构造放入商品事件列表''' + k = 0 + for filename in os.listdir(basepath): + # filename = "20240723-155413_6904406215720" + + '''filename下为一次购物事件''' + filepath = os.path.join(basepath, filename) + + '''================ 0. 检查 filename 及 filepath 正确性和有效性 ================''' + nmlist = filename.split('_') + if filename.find('2024')<0 or len(nmlist)!=2 or len(nmlist[0])!=15 or len(nmlist[1])<11: + continue + if not os.path.isdir(filepath): continue + print(f"Event name: {filename}") + + '''================ 1. 构造事件描述字典，暂定 9 items ===============''' + event = {} + event['barcode'] = nmlist[1] + event['type'] = 'input' + event['filepath'] = filepath + event['back_imgpaths'] = [] + event['front_imgpaths'] = [] + event['back_boxes'] = np.empty((0, 9), dtype=np.float64) + event['front_boxes'] = np.empty((0, 9), dtype=np.float64) + event['back_feats'] = np.empty((0, 256), dtype=np.float64) + event['front_feats'] = np.empty((0, 256), dtype=np.float64) + # event['feats_compose'] = np.empty((0, 256), dtype=np.float64) + # event['feats_select'] = np.empty((0, 256), dtype=np.float64) + + + '''================= 1. 读取 data 文件 =============================''' + for dataname in os.listdir(filepath): + # filename = '1_track.data' + datapath = os.path.join(filepath, dataname) + if not os.path.isfile(datapath): continue + + CamerType = dataname.split('_')[0] + ''' 3.1 读取 0/1_track.data 中数据，暂不考虑''' + # if dataname.find("_track.data")>0: + # bboxes, ffeats, trackerboxes, tracker_feat_dict, trackingboxes, tracking_feat_dict = extract_data(datapath) + + ''' 3.2 读取 0/1_tracking_output.data 中数据''' + if dataname.find("_tracking_output.data")>0: + tracking_output_boxes, tracking_output_feats = read_tracking_output(datapath) + if len(tracking_output_boxes) != len(tracking_output_feats): continue + if CamerType == '0': + event['back_boxes'] = tracking_output_boxes + event['back_feats'] = tracking_output_feats + elif CamerType == '1': + event['front_boxes'] = tracking_output_boxes + event['front_feats'] = tracking_output_feats + + # '''1.1 事件的特征表征方式选择''' + # bk_feats = event['back_feats'] + # ft_feats = event['front_feats'] + + # feats_compose = np.empty((0, 256), dtype=np.float64) + # if len(ft_feats): + # feats_compose = np.concatenate((feats_compose, ft_feats), axis=0) + # if len(bk_feats): + # feats_compose = np.concatenate((feats_compose, bk_feats), axis=0) + # event['feats_compose'] = feats_compose + + # '''3. 构造前摄特征''' + # if len(ft_feats): + # event['feats_select'] = ft_feats + + + + '''================ 2. 读取图像文件地址，并按照帧ID排序 =============''' + frontImgs, frontFid = [], [] + backImgs, backFid = [], [] + for imgname in os.listdir(filepath): + name, ext = os.path.splitext(imgname) + if ext not in IMG_FORMAT or name.find('frameId')<0: continue + + CamerType = name.split('_')[0] + frameId = int(name.split('_')[3]) + imgpath = os.path.join(filepath, imgname) + if CamerType == '0': + backImgs.append(imgpath) + backFid.append(frameId) + if CamerType == '1': + frontImgs.append(imgpath) + frontFid.append(frameId) + + frontIdx = np.argsort(np.array(frontFid)) + backIdx = np.argsort(np.array(backFid)) + + '''2.1 生成依据帧 ID 排序的前后摄图像地址列表''' + frontImgs = [frontImgs[i] for i in frontIdx] + backImgs = [backImgs[i] for i in backIdx] + + '''2.2 将前、后摄图像路径添加至事件字典''' + bfid = event['back_boxes'][:, 7].astype(np.int64) + ffid = event['front_boxes'][:, 7].astype(np.int64) + if len(bfid) and max(bfid) <= len(backImgs): + event['back_imgpaths'] = [backImgs[i-1] for i in bfid] + if len(ffid) and max(ffid) <= len(frontImgs): + event['front_imgpaths'] = [frontImgs[i-1] for i in ffid] + + + '''================ 3. 判断当前事件有效性，并添加至事件列表 ==========''' + condt1 = len(event['back_imgpaths'])==0 or len(event['front_imgpaths'])==0 + condt2 = len(event['front_feats'])==0 and len(event['back_feats'])==0 + + if condt1 or condt2: + print(f" Error, condt1: {condt1}, condt2: {condt2}") + continue + + eventList.append(event) + + # k += 1 + # if k==1: + # continue + + '''一、构造放入商品事件列表，暂不处理''' + # delepath = os.path.join(basepath, 'deletedBarcode.txt') + # bcdList = read_deletedBarcode_file(delepath) + # for slist in bcdList: + # getoutFold = slist['SeqDir'].strip() + # getoutPath = os.path.join(basepath, getoutFold) + + # '''取出事件文件夹不存在，跳出循环''' + # if not os.path.exists(getoutPath) and not os.path.isdir(getoutPath): + # continue + + # ''' 生成取出事件字典 ''' + # event = {} + # event['barcode'] = slist['Deleted'].strip() + # event['type'] = 'getout' + # event['basepath'] = getoutPath + + + return eventList + +def get_std_barcodeDict(bcdpath): + stdBlist = [] + for filename in os.listdir(bcdpath): + filepath = os.path.join(bcdpath, filename) + if not os.path.isdir(filepath) or not filename.isdigit(): continue + + stdBlist.append(filename) + + + bcdpaths = [(barcode, os.path.join(bcdpath, barcode)) for barcode in stdBlist] + + stdBarcodeDict = {} + for barcode, bpath in bcdpaths: + stdBarcodeDict[barcode] = [] + for root, dirs, files in os.walk(bpath): + + imgpaths = [] + if "base" in dirs: + broot = os.path.join(root, "base") + for imgname in os.listdir(broot): + imgpath = os.path.join(broot, imgname) + _, ext = os.path.splitext(imgpath) + if ext not in IMG_FORMAT: continue + imgpaths.append(imgpath) + + stdBarcodeDict[barcode].extend(imgpaths) + break + + else: + for imgname in files: + imgpath = os.path.join(root, imgname) + _, ext = os.path.splitext(imgpath) + if ext not in IMG_FORMAT: continue + imgpaths.append(imgpath) + stdBarcodeDict[barcode].extend(imgpaths) + + jsonpath = os.path.join(r'\\192.168.1.28\share\测试_202406\contrast\barcodes', f"{barcode}.pickle") + with open(jsonpath, 'wb') as f: + pickle.dump(stdBarcodeDict, f) + + print(f"Barcode: {barcode}") + + + + return stdBarcodeDict + + +def one2one_test(filepath): + + savepath = r'\\192.168.1.28\share\测试_202406\contrast' + + '''获得 Barcode 列表''' + bcdpath = r'\\192.168.1.28\share\已标注数据备份\对比数据\barcode\barcode_1771' + stdBarcodeDict = get_std_barcodeDict(bcdpath) + + + eventList = creat_shopping_event(filepath) + print("=========== eventList have generated! ===========") + barcodeDict = {} + for event in eventList: + '''9 items: barcode, type, filepath, back_imgpaths, front_imgpaths, + back_boxes, front_boxes, back_feats, front_feats + ''' + + barcode = event['barcode'] + if barcode not in stdBarcodeDict.keys(): + continue + + + if len(event['feats_select']): + event_feats = event['feats_select'] + elif len(event['back_feats']): + event_feats = event['back_feats'] + else: + continue + + std_bcdpath = os.path.join(bcdpath, barcode) + + + + for root, dirs, files in os.walk(std_bcdpath): + if "base" in files: + std_bcdpath = os.path.join(root, "base") + break + + + + + + + + + + + + + + '''保存一次购物事件的轨迹子图''' + basename = os.path.basename(event['filepath']) + spath = os.path.join(savepath, basename) + if not os.path.exists(spath): + os.makedirs(spath) + cameras = ('front', 'back') + for camera in cameras: + if camera == 'front': + boxes = event['front_boxes'] + imgpaths = event['front_imgpaths'] + else: + boxes = event['back_boxes'] + imgpaths = event['back_imgpaths'] + + for i, box in enumerate(boxes): + x1, y1, x2, y2, tid, score, cls, fid, bid = box + + imgpath = imgpaths[i] + image = cv2.imread(imgpath) + subimg = image[int(y1/2):int(y2/2), int(x1/2):int(x2/2), :] + + camerType, timeTamp, _, frameID = os.path.basename(imgpath).split('.')[0].split('_') + subimgName = f"{camerType}_{tid}_fid({fid}, {frameID}).png" + subimgPath = os.path.join(spath, subimgName) + + cv2.imwrite(subimgPath, subimg) + print(f"Image saved: {basename}") + + + + + + + + +def main(): + fplist = [r'\\192.168.1.28\share\测试_202406\0723\0723_1', + r'\\192.168.1.28\share\测试_202406\0723\0723_2', + # r'\\192.168.1.28\share\测试_202406\0723\0723_3', + r'\\192.168.1.28\share\测试_202406\0722\0722_01', + r'\\192.168.1.28\share\测试_202406\0722\0722_02' + ] + + + + for filepath in fplist: + one2one_test(filepath) + + # for filepath in fplist: + # try: + # one2one_test(filepath) + + # except Exception as e: + # print(f'{filepath}, Error: {e}') + +if __name__ == '__main__': + + main() \ No newline at end of file diff --git a/tracking/deprecated/eventsmatch.py b/tracking/deprecated/eventsmatch.py new file mode 100644 index 0000000..decd9b2 --- /dev/null +++ b/tracking/deprecated/eventsmatch.py @@ -0,0 +1,806 @@ +# -*- coding: utf-8 -*- +""" +Created on Tue Apr 16 11:51:07 2024 + +@author: ym +""" +import cv2 +import os +import numpy as np +# import time +import pickle +import json +# import matplotlib.pyplot as plt +import pandas as pd +import shutil +import random +import math +import sys +from scipy.spatial.distance import cdist +import matplotlib.pyplot as plt + +from pathlib import Path +from utils.gen import Profile + +sys.path.append(r"D:\DetectTracking\tracking") + +from dotrack.dotracks_back import doBackTracks +from dotrack.dotracks_front import doFrontTracks +from utils.drawtracks import plot_frameID_y2, draw_all_trajectories + +# from utils.drawtracks import draw5points, drawTrack, drawtracefeat, drawFeatures +# from datetime import datetime + +from utils.mergetrack import readDict + +import csv + + +def read_csv_file(): + + file_path = r'D:\DeepLearning\yolov5_track\tracking\matching\featdata\Similarity.csv' + with open(file_path, mode='r', newline='') as file: + data = list(csv.reader(file)) + + matrix = [] + for i in range(1, len(data)): + matrix.append(data[i][1:]) + + matrix = np.array(matrix, dtype = np.float32) + + simil = 1 + (matrix-1)/2 + + print("done!!!") + + + + +def get_img_filename(imgpath = r'./matching/images/' ): + imgexts = ['.png', '.jpg', '.jpeg'] + ImgFileList = [] + for root, dirs, files in os.walk(imgpath): + ImgList = [] + for file in files: + _, ext = os.path.splitext(file) + if ext in imgexts: + ImgFileList.append(os.path.join(root, file)) + + return ImgFileList + + +def calculate_similarity_track(similarmode = 'other'): + ''' + similarmode: + 'mean' + 'other' + + ''' + + ContrastDict = np.load('./matching/featdata/imgs_feats_data_refined.pkl', allow_pickle=True) + print(f"The Num of imgsample: {len(ContrastDict)}") + + ''''================= 构造空字典 MatchObjDict =================''' + def splitkey(key): + videoname = key.split('_') + BarCode = videoname[0] + SampleTime = videoname[1].split('-')[1] + CameraType = videoname[2] + ActionType = videoname[3] + TrackID = '_'.join(key.split('_')[7:]) + + return BarCode, SampleTime, CameraType, ActionType, TrackID + + MatchObjList = [] + CameraList = [] + for key in ContrastDict.keys(): + BarCode, SampleTime, CameraType, ActionType, FeatureID = splitkey(key) + + MatchObjList.append('_'.join([BarCode, SampleTime, ActionType])) + CameraList.append(CameraType) + + # MatchObjSet = set(MatchObjList) + # CameraSet = set(CameraList) + + + objects = list(set(MatchObjList)) + cameras = list(set(CameraList)) + assert len(cameras) == 2, "The num of cameras is error!" + + MatchObjDict = {} + for obj in objects: + CameraDict = {} + for camera in cameras: + CameraDict[camera] = {} + + MatchObjDict[obj] = CameraDict + + + + for key, value in ContrastDict.items(): + BarCode, SampleTime, CameraType, ActionType, FeatureID = splitkey(key) + + MatchObj = '_'.join([BarCode, SampleTime, ActionType]) + + vdict = {} + if FeatureID not in MatchObjDict[MatchObj][CameraType]: + vdict[FeatureID] = value['feature'] + MatchObjDict[MatchObj][CameraType].update(vdict) + + print(f"The Num of MatchObjDict: {len(MatchObjDict)}") + + # MatchKeys = [key for key in MatchObjDict.keys()] + num = len(objects) + GtMatrix = np.zeros((num, num), dtype=np.float32) + Similarity = np.zeros((num, num), dtype=np.float32) + InterMatrix = np.zeros((num, num), dtype=np.float32) # 类间 + IntraMatrix = np.zeros((num, num), dtype=np.float32) # 类内 + + '''生成GT矩阵: GtMatrix, IntraMatrix, InterMatrix''' + for i, obi in enumerate(objects): + barcode_i = obi.split('_')[0] + for j, obj in enumerate(objects): + barcode_j = obj.split('_')[0] + if barcode_i == barcode_j: + GtMatrix[i, j] = 1 + if i!=j: IntraMatrix[i, j] = 1 + else: + GtMatrix[i, j] = 0 + InterMatrix[i, j] = 1 + + + '''生成相似度矩阵: Similarity ''' + + ObjFeatList = [] + for i, obi in enumerate(objects): + obidict = MatchObjDict[obi] + camlist = [] + for camera in obidict.keys(): + featlist = [] + for fid in obidict[camera].keys(): + featlist.append(MatchObjDict[obi][camera][fid]) + + camlist.append(featlist) + + ObjFeatList.append(camlist) + + Similarity_1 = Similarity.copy() + for i in range(len(objects)): + obi = ObjFeatList[i] + for j in range(len(objects)): + obj = ObjFeatList[j] + simival = [] + for ii in range(len(obi)): + if len(obi[ii])==0: continue + feat_ii = np.asarray(obi[ii]) + + for jj in range(len(obj)): + if len(obj[jj])==0: continue + feat_jj = np.asarray(obj[jj]) + + if similarmode == 'mean': + featii = np.mean(feat_ii, axis=0) + featjj = np.mean(feat_jj, axis=0) + try: + matrix = 1- np.maximum(0.0, cdist(featii[None, :], featjj[None, :], 'cosine')) + except Exception as e: + print(f'error is {e.__class__.__name__}') + + else: + matrix = 1- np.maximum(0.0, cdist(feat_ii, feat_jj, 'cosine')) + simival.append(np.max(matrix)) + if len(simival)==0: continue + + Similarity[i, j] = max(simival) + + # feat_i = np.empty((0, 256), dtype = np.float32) + # feat_j = np.empty((0, 256), dtype = np.float32) + # for ii in range(len(obi)): + # feat_ii = np.asarray(obi[ii]) + # feat_i = np.concatenate((feat_i, feat_ii), axis=0) + # for jj in range(len(obj)): + # feat_jj = np.asarray(obi[jj]) + # feat_j = np.concatenate((feat_j, feat_jj), axis=0) + + # if similarmode == 'mean': + # feati = np.mean(feat_i, axis=0) + # featj = np.mean(feat_j, axis=0) + # matrix = 1- np.maximum(0.0, cdist(feati[None, :], featj[None, :], 'cosine')) + # else: + # matrix = 1- np.maximum(0.0, cdist(feat_i, feat_j, 'cosine')) + + # Similarity_1[i, j] = np.max(matrix) + + + SimiDict = {'keys': objects, 'GtMatrix': GtMatrix, 'Similarity': Similarity, + 'IntraMatrix':IntraMatrix, 'InterMatrix':InterMatrix} + with open(r"./matching/featdata/MatchDict_track.pkl", "wb") as f: + pickle.dump(SimiDict, f) + + # SimiDict_1 = {'keys': objects, 'GtMatrix': GtMatrix, 'Similarity':Similarity_1, + # 'IntraMatrix':IntraMatrix, 'InterMatrix':InterMatrix} + # with open(r"./matching/featdata/MatchDict_track_1.pkl", "wb") as f: + # pickle.dump(SimiDict_1, f) + + df_GtMatrix = pd.DataFrame(data=GtMatrix, columns = objects, index = objects) + df_GtMatrix.to_csv('./matching/featdata/GtMatrix_track.csv',index=True) + + df_similarity = pd.DataFrame(data=Similarity, columns = objects, index = objects) + df_similarity.to_csv('./matching/featdata/Similarity_track.csv',index=True) + + # df_similarity_1 = pd.DataFrame(data=Similarity_1, columns = objects, index = objects) + # df_similarity_1.to_csv('./matching/featdata/Similarity_track_1.csv',index=True) + + print("Done!!!!") + +# SimilarMode = ['mean', 'max'] +def calculate_similarity(similarmode = 'mean'): + ContrastDict = np.load('./matching/featdata/imgs_feats_data_noplane.pkl', allow_pickle=True) + print(f"The Num of imgsample: {len(ContrastDict)}") + + FrontBackMerged = True + TracKeys = {} + for key, value in ContrastDict.items(): + feature = value['feature'] + videoname = key.split('_')[:7] + BarCode = videoname[0] + SampleTime = videoname[1].split('-')[1] + CameraType = videoname[2] + ActionType = videoname[3] + TrackID = key.split('_')[7] + + if FrontBackMerged: + TracKey = '_'.join([BarCode, SampleTime, ActionType]) + else: + TracKey = '_'.join([BarCode, SampleTime, CameraType, ActionType]) + + if TracKey in TracKeys: + TracKeys[TracKey].append(feature) + else: + TracKeys[TracKey] = [] + TracKeys[TracKey].append(feature) + + '''===== 生成GT矩阵: Similarity、GtMatrix、IntraMatrix、InterMatrix =====''' + num = len(TracKeys) + keys = [key for key in TracKeys.keys()] + + GtMatrix = np.zeros((num, num), dtype=np.float32) + Similarity = np.zeros((num, num), dtype=np.float32) + + InterMatrix = np.zeros((num, num), dtype=np.float32) # 类间 + IntraMatrix = np.zeros((num, num), dtype=np.float32) # 类内 + + for i, key_i in enumerate(keys): + barcode_i = key_i.split('_')[0] + feat_i = np.asarray(TracKeys[key_i], dtype=np.float32) + for j, key_j in enumerate(keys): + barcode_j = key_j.split('_')[0] + feat_j = np.asarray(TracKeys[key_j], dtype=np.float32) + + if similarmode == 'mean': + feati = np.mean(feat_i, axis=0) + featj = np.mean(feat_j, axis=0) + matrix = 1- np.maximum(0.0, cdist(feati[None, :], featj[None, :], 'cosine')) + else: + matrix = 1- np.maximum(0.0, cdist(feat_i, feat_j, 'cosine')) + Similarity[i, j] = np.max(matrix) + + if barcode_i == barcode_j: + GtMatrix[i, j] = 1 + if i!=j: IntraMatrix[i, j] = 1 + else: + GtMatrix[i, j] = 0 + InterMatrix[i, j] = 1 + + + +# ============================================================================= +# '''生成相似度矩阵: Similarity ''' +# for i, key_i in enumerate(keys): +# feat_i = np.asarray(TracKeys[key_i], dtype=np.float32) +# for j, key_j in enumerate(keys): +# feat_j = np.asarray(TracKeys[key_j], dtype=np.float32) +# +# if similarmode == 'mean': +# feati = np.mean(feat_i, axis=0) +# featj = np.mean(feat_j, axis=0) +# matrix = 1- np.maximum(0.0, cdist(feati[None, :], featj[None, :], 'cosine')) +# else: +# matrix = 1- np.maximum(0.0, cdist(feat_i, feat_j, 'cosine')) +# Similarity[i, j] = np.max(matrix) +# ============================================================================= + + MatchDict = {'keys': keys, 'GtMatrix':GtMatrix, 'Similarity':Similarity, + 'IntraMatrix':IntraMatrix, 'InterMatrix':InterMatrix} + with open(r"./matching/featdata/MatchDict_noplane.pkl", "wb") as f: + pickle.dump(MatchDict, f) + + df_GtMatrix = pd.DataFrame(data=GtMatrix, columns = keys, index = keys) + df_GtMatrix.to_csv('./matching/featdata/GtMatrix_noplane.csv',index=True) + + df_similarity = pd.DataFrame(data=Similarity, columns = keys, index = keys) + df_similarity.to_csv('./matching/featdata/Similarity_noplane.csv',index=True) + + +def sortN_matching(filename = r'./matching/featdata/MatchDict.pkl'): + SimilarDict = np.load(filename, allow_pickle=True) + + '''********** keys的顺序与Similarity中行列值索引一一对应 **********''' + keys = SimilarDict['keys'] + Similarity = SimilarDict['Similarity'] + + '''1. 将时间根据 Barcode 归并，并确保每个 Barcode 下至少两个事件''' + BarcodeDict1 = {} + for i, key in enumerate(keys): + barcode = key.split('_')[0] + + if barcode not in BarcodeDict1.keys(): + BarcodeDict1[barcode] = [] + BarcodeDict1[barcode].append(i) + + + BarcodeDict = {} + BarcodeList = [] + for barcode, value in BarcodeDict1.items(): + if len(value) < 2: continue + BarcodeDict[barcode] = value + BarcodeList.append(barcode) + + BarcodeList = list(set(BarcodeList)) + + '''实验参数设定 + N: 任意选取的 Barcode 数 + R：重复实验次数，每次从同一 Barcode 下随机选取2个事件，分别归入加购、退购集合 + Thresh：相似度阈值 + + ''' + N = 10 + if N > len(BarcodeList): + N = math.ceil(len(BarcodeList)/2) + R = 20 + Thresh = np.linspace(0.1, 1, 100) + # Thresh = np.linspace(0.601, 0.7, 100) + + Recall, Precision = [], [] + for th in Thresh: + recall = np.zeros((1, R), dtype=np.float32) + precision = np.zeros((1, R), dtype=np.float32) + + for rep in range(R): + BarcodeSelect = random.sample(BarcodeList, N) + + AddDict = {} + TakeoutDict = {} + for barcode in BarcodeSelect: + barlist = BarcodeDict[barcode] + if len(barlist) < 2:continue + + selected = random.sample(barlist, 2) + + AddDict[barcode] = selected[0] + TakeoutDict[barcode] = selected[1] + + OrderMatrix = np.zeros((N, N), dtype=np.float32) + GTMatrix = np.zeros((N, N), dtype=np.float32) + + MatchMatrix_1 = np.zeros((N, N), dtype=np.float32) + MatchMatrix_2 = np.zeros((N, N), dtype=np.float32) + + i = 0 + for keyi in BarcodeSelect: + ii = TakeoutDict[keyi] + j = 0 + for keyj in BarcodeSelect: + jj = AddDict[keyj] + + OrderMatrix[i, j] = Similarity[int(ii), int(jj)] + + if keyi == keyj: + GTMatrix[i, j] = 1 + + j += 1 + i += 1 + + max_indices = np.argmax(OrderMatrix, axis = 1) + for i in range(N): + MatchMatrix_1[i, max_indices[i]] = 1 + + similar = OrderMatrix[i, max_indices[i]] + if similar > th: + MatchMatrix_2[i, max_indices[i]] = 1 + + + GT_indices = np.where(GTMatrix == 1) + + FNTP = MatchMatrix_2[GT_indices] + pred_indices = np.where(MatchMatrix_2 == 1) + + + + TP = np.sum(FNTP==1) + FN = np.sum(FNTP==0) + FPTP = GTMatrix[pred_indices] + + FP = np.sum(FPTP == 0) + # assert TP == np.sum(FPTP == 0), "Please Check Errors!!!" + + recall[0, rep] = TP/(TP+FN) + precision[0, rep] = TP/(TP+FP+1e-3) # 阈值太大时可能TP、FP都为0， + + Recall.append(recall) + Precision.append(precision) + + + + Recall = np.asarray(Recall).reshape([len(Thresh),-1]) + Precision = np.asarray(Precision).reshape([len(Thresh),-1]) + + reclmean = np.sum(Recall, axis=1) / (np.count_nonzero(Recall, axis=1) + 1e-3) + precmean = np.sum(Precision, axis=1) / (np.count_nonzero(Precision, axis=1) + 1e-3) + + print("Done!!!!!") + + # th1, recl = [c[0] for c in Recall], [c[1] for c in Recall] + # th2, prep = [c[0] for c in Precision], [c[1] for c in Precision] + + recl = [r for r in reclmean] + prep = [p for p in precmean] + + + '''================= Precision & Recall =================''' + fig, ax = plt.subplots() + ax.plot(Thresh, recl, 'g', label='Recall = TP/(TP+FN)') + ax.plot(Thresh, prep, 'r', label='PrecisePos = TP/(TP+FP)') + # ax.set_xlim([0, 1]) + # ax.set_ylim([0, 1]) + ax.grid(True) + ax.set_title('Precision & Recall') + ax.legend() + plt.show() + + + +def match_evaluate(filename = r'./matching/featdata/MatchDict.pkl'): + SimiDict = np.load(filename, allow_pickle=True) + + keys = SimiDict['keys'] + GtMatrix = SimiDict['GtMatrix'] + Similarity = SimiDict['Similarity'] + + IntraMatrix = SimiDict['IntraMatrix'] + InterMatrix = SimiDict['InterMatrix'] + + + BarcodeList = [] + for key in keys: + BarcodeList.append(key.split('_')[0]) + BarcodeList = list(set(BarcodeList)) + + + IntraRows, IntraCols = np.nonzero(IntraMatrix) + InterRows, InterCols = np.nonzero(InterMatrix) + IntraN, InterN = len(IntraRows), len(InterRows) + assert IntraN <= InterN, "类内大于类间数，样本不平衡" + + InterNSelect = IntraN + Thresh = np.linspace(0.1, 1, 100) + # Thresh = np.linspace(0.2, 0.4, 11) + Correct = [] + PrecisePos = [] + PreciseNeg = [] + Recall = [] + CorrectMatries = [] + for th in Thresh: + MatchMatrix = Similarity > th + CorrectMatrix = MatchMatrix == GtMatrix + + CorrectMatries.append(CorrectMatrix) + + nn = np.random.permutation(np.arange(InterN))[:InterNSelect] + InterRowsSelect, InterColsSelect = InterRows[nn], InterCols[nn] + + IntraCorrMatrix = CorrectMatrix[IntraRows, IntraCols] + InterCorrMatrix = CorrectMatrix[InterRowsSelect, InterColsSelect] + + TP = np.sum(IntraCorrMatrix) + TN = np.sum(InterCorrMatrix) + FN = IntraN - TP + FP = InterNSelect - TN + + if TP+FP > 0: + PrecisePos.append((th, TP/(TP+FP))) + if TN+FN > 0: + PreciseNeg.append((th, TN/(TN+FN))) + if TP+FN > 0: + Recall.append((th, TP/(TP+FN))) + + if TP+TN+FP+FN > 0: + Correct.append((th, (TP+TN)/(TP+TN+FP+FN))) + # print(f'Th: {th}') + # print(f'TP:{TP}, FP:{FP}, TN:{TN}, FN:{FN}') + + + CorrectMatries = np.asarray(CorrectMatries) + + '''====================== 分析错误原因 ==========================''' + ''' + keys两种构成方式，其中的元素来自于：MatchingDict + BarCode, SampleTime, ActionType #不考虑摄像头类型（前后摄） + BarCode, SampleTime, CameraType, ActionType# 考虑摄像头类型（前后摄） + 为了便于显示，在图像文件名中，将 ActionType 进行了缩写，匹配时取 [:3] + "addGood" --------> "add" + "returnGood" --------> "return" + ''' + ##============= 获取图像存储位置,可以通过 keys 检索到对应的图像文件 + imgpath = r'./matching/images/' + ImgFileList = get_img_filename(imgpath) + + rowx, colx = np.where(CorrectMatries[66,:,:] == False) + rows, cols = [], [] + for i in range(len(rowx)): + ri, ci = rowx[i], colx[i] + if ci > ri: + rows.append(ri) + cols.append(ci) + + + KeysError = [(keys[rows[i]], keys[cols[i]]) for i in range(len(rows))] + SimiScore = [Similarity[rows[i], cols[i]] for i in range(len(rows))] + + + for i, keykey in enumerate(KeysError): + key1, key2 = keykey + sscore = SimiScore[i] + + kt1, kt2 = key1.split('_'), key2.split('_') + + if len(kt1)==3 and len(kt2)==3: + file1 = [f for f in ImgFileList if kt1[0] in f and kt1[1] in f and kt1[2][:3] in f] + file2 = [f for f in ImgFileList if kt2[0] in f and kt2[1] in f and kt2[2][:3] in f] + + elif len(kt1)==4 and len(kt1)==4: + file1 = [f for f in ImgFileList if kt1[0] in f and kt1[1] in f and kt1[2] in f and kt1[3][:3] in f] + file2 = [f for f in ImgFileList if kt2[0] in f and kt2[1] in f and kt2[2] in f and kt2[3][:3] in f] + else: + pass + + if len(file1)==0 or len(file2)==0: + continue + + if kt1[0] == kt2[0]: + gt = "same" + else: + gt = "diff" + + path = Path(f'./matching/results/{i}_{gt}_{sscore:.2f}') + if path.exists() and path.is_dir(): + shutil.rmtree(path) + path1, path2 = path.joinpath(key1), path.joinpath(key2) + + path1.mkdir(parents=True, exist_ok=True) + path2.mkdir(parents=True, exist_ok=True) + for file in file1: + shutil.copy2(file, path1) + for file in file2: + shutil.copy2(file, path2) + + if i==99: + break + + th1, corr = [c[0] for c in Correct], [c[1] for c in Correct] + th2, recl = [c[0] for c in Recall], [c[1] for c in Recall] + th3, prep = [c[0] for c in PrecisePos], [c[1] for c in PrecisePos] + th4, pren = [c[0] for c in PreciseNeg], [c[1] for c in PreciseNeg] + + '''================= Correct ===================''' + fig, ax = plt.subplots() + ax.plot(th1, corr, 'b', label='Correct = (TP+TN)/(TP+TN+FP+FN)') + max_corr = max(corr) + max_index = corr.index(max_corr) + max_thresh = th1[max_index] + ax.plot([0, max_thresh], [max_corr, max_corr], 'r--') + ax.plot([max_thresh, max_thresh], [0, max_corr], 'r--') + ax.set_xlim([0, 1]) + ax.set_ylim([0, 1]) + ax.grid(True) + ax.set_title('Correct') + ax.legend() + plt.show() + + + '''================= PrecisePos & PreciseNeg & Recall =================''' + fig, ax = plt.subplots() + ax.plot(th2, recl, 'g', label='Recall = TP/(TP+FN)') + ax.plot(th3, prep, 'c', label='PrecisePos = TP/(TP+FP)') + ax.plot(th4, pren, 'm', label='PreciseNeg = TN/(TN+FN)') + + ax.set_xlim([0, 1]) + ax.set_ylim([0, 1]) + ax.grid(True) + ax.set_title('PrecisePos & PreciseNeg') + ax.legend() + plt.show() + + +def have_tracked(): + trackdir = r"./data/tracks" + +# ============================================================================= +# FileList = [] +# with open(r'./matching/视频分类/单.txt', 'r') as file: +# lines = file.readlines() +# for line in lines: +# file = line.split('.')[0] +# FileList.append(file) +# FileList = list(set(FileList)) +# ============================================================================= + + MatchingDict = {} + k, gt = 0, Profile() + for filename in os.listdir(trackdir): + file, ext = os.path.splitext(filename) + + # if file not in FileList: continue + if file.find('20240508')<0: continue + filepath = os.path.join(trackdir, filename) + + + tracksDict = np.load(filepath, allow_pickle=True) + bboxes = tracksDict['TrackBoxes'] + with gt: + if filename.find("front") >= 0: + vts = doFrontTracks(bboxes, tracksDict) + vts.classify() + + elif filename.find("back") >= 0: + vts = doBackTracks(bboxes, tracksDict) + vts.classify() + + print(file+f" need time: {gt.dt:.2f}s") + + elements = file.split('_') + assert len(elements) == 7, f"{filename} fields num: {len(elements)}" + BarCode = elements[0] + + ## ====================================== 只用于在images文件夹下保存图片 + SampleTime = elements[1].split('-')[1] + CameraType = elements[2] + if elements[3]=="addGood": + ActionType = "add" + elif elements[3]=="returnGood": + ActionType = "return" + else: + ActionType = "x" + + subimg_dir = Path(f'./matching/images/{BarCode}_{SampleTime}_{ActionType}/') + if not subimg_dir.exists(): + subimg_dir.mkdir(parents=True, exist_ok=True) + + # boxes: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] + # 0, 1, 2, 3, 4, 5, 6, 7, 8 + for track in vts.Residual: + boxes = track.boxes + for i in range(boxes.shape[0]): + box = boxes[i, :] + tid, fid, bid = int(box[4]), int(box[7]), int(box[8]) + + feat_dict = tracksDict[fid] + feature = feat_dict[bid] + img = feat_dict[f'{bid}_img'] + + sub_img_file = subimg_dir.joinpath(f"{BarCode}_{SampleTime}_{CameraType}_{ActionType}_{tid}_{fid}_{bid}.png") + cv2.imwrite(str(sub_img_file), img) + + condict = {f"{file}_{tid}_{fid}_{bid}": {'img': img, 'feature': feature}} + + MatchingDict.update(condict) + # k += 1 + # if k == 100: + # break + + featpath = Path('./matching/featdata/') + if not featpath.exists(): + featpath.mkdir(parents=True, exist_ok=True) + + featdata = featpath.joinpath('imgs_feats_data_noplane.pkl') + with open(featdata, 'wb') as file: + pickle.dump(MatchingDict, file) + +def imgsample_cleaning(): + ContrastDict = np.load('./matching/featdata/imgs_feats_data.pkl', allow_pickle=True) + print(f"The Num of imgsample: {len(ContrastDict)}") + + + MatchingDict_refined = {} + for filename, value in ContrastDict.items(): + elements = filename.split('_') + + + tid = elements[7] + fid = elements[8] + bid = elements[9] + + BarCode = elements[0] + SampleTime = elements[1].split('-')[1] + CameraType = elements[2] + if elements[3]=="addGood": + ActionType = "add" + elif elements[3]=="returnGood": + ActionType = "return" + else: + ActionType = "x" + refimgdir = f'.\matching\images_refined\{BarCode}_{SampleTime}_{ActionType}' + + file = '_'.join(elements[0:7]) + if os.path.exists(refimgdir) and os.path.isdir(refimgdir): + imgpath = os.path.join(refimgdir, f"{BarCode}_{SampleTime}_{CameraType}_{ActionType}_{tid}_{fid}_{bid}.png") + if os.path.isfile(imgpath): + condict = {f"{file}_{tid}_{fid}_{bid}": value} + + MatchingDict_refined.update(condict) + + featpath = Path('./matching/featdata/') + if not featpath.exists(): + featpath.mkdir(parents=True, exist_ok=True) + + featdata = featpath.joinpath('imgs_feats_data_refined.pkl') + with open(featdata, 'wb') as file: + pickle.dump(MatchingDict_refined, file) + + print(f"The Num of ContrastDict: {len(ContrastDict)}") + print(f"The Num of MatchingDict_refined: {len(MatchingDict_refined)}") + print(f"The Num of cleaned img: {len(ContrastDict)} - {len(MatchingDict_refined)}") + + +def main(): + '''1. 提取运动商品轨迹''' + # have_tracked() + + '''2. 清除一次事件中包含多件商品的事件''' + # imgsample_cleaning() + + '''3.1 计算事件间相似度: 将 front、back 的所有 track 特征合并''' + calculate_similarity() + + '''3.2 计算事件间相似度: 考虑前后摄的不同组合，或 track 间的不同组合''' + # calculate_similarity_track() + + + '''4.1 事件间匹配的总体性能评估''' + filename = r'./matching/featdata/MatchDict_plane.pkl' + match_evaluate(filename) + + filename = r'./matching/featdata/MatchDict_noplane.pkl' + match_evaluate(filename) + + '''4.2 模拟实际场景，任选N件作为一组作为加购，取出其中一件时的性能评估''' + # filename = r'./matching/featdata/MatchDict_refined.pkl' + # sortN_matching(filename) + + +if __name__ == "__main__": + # save_dir = Path(f'./result/') + # read_csv_file() + + main() + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tracking/dotrack/__init__.py b/tracking/dotrack/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tracking/dotrack/__pycache__/__init__.cpython-312.pyc b/tracking/dotrack/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..ab597cf Binary files /dev/null and b/tracking/dotrack/__pycache__/__init__.cpython-312.pyc differ diff --git a/tracking/dotrack/__pycache__/__init__.cpython-39.pyc b/tracking/dotrack/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..e71c618 Binary files /dev/null and b/tracking/dotrack/__pycache__/__init__.cpython-39.pyc differ diff --git a/tracking/dotrack/__pycache__/dotracks.cpython-312.pyc b/tracking/dotrack/__pycache__/dotracks.cpython-312.pyc new file mode 100644 index 0000000..e26f6df Binary files /dev/null and b/tracking/dotrack/__pycache__/dotracks.cpython-312.pyc differ diff --git a/tracking/dotrack/__pycache__/dotracks.cpython-39.pyc b/tracking/dotrack/__pycache__/dotracks.cpython-39.pyc new file mode 100644 index 0000000..31724ce Binary files /dev/null and b/tracking/dotrack/__pycache__/dotracks.cpython-39.pyc differ diff --git a/tracking/dotrack/__pycache__/dotracks_back.cpython-312.pyc b/tracking/dotrack/__pycache__/dotracks_back.cpython-312.pyc new file mode 100644 index 0000000..25d448a Binary files /dev/null and b/tracking/dotrack/__pycache__/dotracks_back.cpython-312.pyc differ diff --git a/tracking/dotrack/__pycache__/dotracks_back.cpython-39.pyc b/tracking/dotrack/__pycache__/dotracks_back.cpython-39.pyc new file mode 100644 index 0000000..9483b06 Binary files /dev/null and b/tracking/dotrack/__pycache__/dotracks_back.cpython-39.pyc differ diff --git a/tracking/dotrack/__pycache__/dotracks_front.cpython-312.pyc b/tracking/dotrack/__pycache__/dotracks_front.cpython-312.pyc new file mode 100644 index 0000000..3b4f270 Binary files /dev/null and b/tracking/dotrack/__pycache__/dotracks_front.cpython-312.pyc differ diff --git a/tracking/dotrack/__pycache__/dotracks_front.cpython-39.pyc b/tracking/dotrack/__pycache__/dotracks_front.cpython-39.pyc new file mode 100644 index 0000000..eaa7a57 Binary files /dev/null and b/tracking/dotrack/__pycache__/dotracks_front.cpython-39.pyc differ diff --git a/tracking/dotrack/__pycache__/track_back.cpython-312.pyc b/tracking/dotrack/__pycache__/track_back.cpython-312.pyc new file mode 100644 index 0000000..7f5870f Binary files /dev/null and b/tracking/dotrack/__pycache__/track_back.cpython-312.pyc differ diff --git a/tracking/dotrack/__pycache__/track_back.cpython-39.pyc b/tracking/dotrack/__pycache__/track_back.cpython-39.pyc new file mode 100644 index 0000000..c2923a2 Binary files /dev/null and b/tracking/dotrack/__pycache__/track_back.cpython-39.pyc differ diff --git a/tracking/dotrack/__pycache__/track_front.cpython-312.pyc b/tracking/dotrack/__pycache__/track_front.cpython-312.pyc new file mode 100644 index 0000000..4b2a912 Binary files /dev/null and b/tracking/dotrack/__pycache__/track_front.cpython-312.pyc differ diff --git a/tracking/dotrack/__pycache__/track_front.cpython-39.pyc b/tracking/dotrack/__pycache__/track_front.cpython-39.pyc new file mode 100644 index 0000000..523c566 Binary files /dev/null and b/tracking/dotrack/__pycache__/track_front.cpython-39.pyc differ diff --git a/tracking/dotrack/dotracks.py b/tracking/dotrack/dotracks.py new file mode 100644 index 0000000..4e53b08 --- /dev/null +++ b/tracking/dotrack/dotracks.py @@ -0,0 +1,721 @@ +# -*- coding: utf-8 -*- +""" +Created on Mon Mar 4 18:16:01 2024 + +@author: ym +""" +import numpy as np +import cv2 +from pathlib import Path +from scipy.spatial.distance import cdist +from tracking.utils.mergetrack import track_equal_track, readDict +curpath = Path(__file__).resolve().parents[0] + +curpath = Path(curpath) +parpath = curpath.parent + +class MoveState: + """商品运动状态标志""" + Static = 0 + DownWard = 1 + UpWard = 2 + FreeMove = 3 + Unknown = -1 + +def bbox_ioa(box1, box2, iou=False, eps=1e-7): + """ + Calculate the intersection over box2 area given box1 and box2. Boxes are in x1y1x2y2 format. + + Args: + box1 (np.array): A numpy array of shape (n, 4) representing n bounding boxes. + box2 (np.array): A numpy array of shape (m, 4) representing m bounding boxes. + iou (bool): Calculate the standard iou if True else return inter_area/box2_area. + eps (float, optional): A small value to avoid division by zero. Defaults to 1e-7. + + Returns: + (np.array): A numpy array of shape (n, m) representing the intersection over box2 area. + """ + + # Get the coordinates of bounding boxes + b1_x1, b1_y1, b1_x2, b1_y2 = box1.T + b2_x1, b2_y1, b2_x2, b2_y2 = box2.T + + # Intersection area + inter_area = (np.minimum(b1_x2[:, None], b2_x2) - np.maximum(b1_x1[:, None], b2_x1)).clip(0) * \ + (np.minimum(b1_y2[:, None], b2_y2) - np.maximum(b1_y1[:, None], b2_y1)).clip(0) + + # box2 area + area = (b2_x2 - b2_x1) * (b2_y2 - b2_y1) + if iou: + box1_area = (b1_x2 - b1_x1) * (b1_y2 - b1_y1) + area = area + box1_area[:, None] - inter_area + + # Intersection over box2 area + return inter_area / (area + eps) + +class ShoppingCart: + + def __init__(self, bboxes): + self.bboxes = bboxes + self.loadrate = self.load_rate() + + def load_rate(self): + bboxes = self.bboxes + + fid = min(bboxes[:, 7]) + idx = bboxes[:, 7] == fid + boxes = bboxes[idx] + + temp = np.zeros(self.incart.shape, np.uint8) + for i in range(boxes.shape[0]): + x1, y1, x2, y2, tid = boxes[i, 0:5] + cv2.rectangle(temp, (int(x1), int(y1)), (int(x2), int(y2)), 255, cv2.FILLED) + + '''1. and 滤除购物车边框外的干扰''' + loadstate = cv2.bitwise_and(self.incart, temp) + + '''2. xor 得到购物车内内被填充的区域''' + # loadstate = cv2.bitwise_xor(self.incart, temp1) + + num_loadstate = cv2.countNonZero(loadstate) + num_incart = cv2.countNonZero(self.incart) + loadrate = num_loadstate / (num_incart+0.01) + + # edgeline = cv2.imread("./shopcart/cart_tempt/edgeline.png", cv2.IMREAD_GRAYSCALE) + # cv2.imwrite(f"./test/temp.png", cv2.add(temp, edgeline)) + # cv2.imwrite(f"./test/incart.png", cv2.add(self.incart, edgeline)) + # cv2.imwrite(f"./test/loadstate.png", cv2.add(loadstate, edgeline)) + + return loadrate + + @property + def incart(self): + img = cv2.imread(str(parpath/'shopcart/cart_tempt/incart.png'), cv2.IMREAD_GRAYSCALE) + ret, binary = cv2.threshold(img, 250, 255, cv2.THRESH_BINARY) + + return binary + + @property + def outcart(self): + img = cv2.imread(str(parpath/'shopcart/cart_tempt/outcart.png'), cv2.IMREAD_GRAYSCALE) + ret, binary = cv2.threshold(img, 250, 255, cv2.THRESH_BINARY) + + return binary + + @property + def cartedge(self): + img = cv2.imread(str(parpath/'shopcart/cart_tempt/cartedge.png'), cv2.IMREAD_GRAYSCALE) + ret, binary = cv2.threshold(img, 250, 255, cv2.THRESH_BINARY) + + return binary + +class Track: + '''抽象基类，不能实例化对象''' + def __init__(self, boxes, features=None, imgshape=(1024, 1280)): + ''' + boxes: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] + 0 1 2 3 4 5 6 7 8 + ''' + # assert len(set(boxes[:, 4].astype(int))) == 1, "For a Track, track_id more than 1" + # assert len(set(boxes[:, 6].astype(int))) == 1, "For a Track, class number more than 1" + + self.boxes = boxes + self.features = features + self.slt_boxes = self.select_boxes() + + self.tid = int(boxes[0, 4]) + self.cls = int(boxes[0, 6]) + self.frnum = boxes.shape[0] + + self.isCornpoint = False + self.imgshape = imgshape + # self.isBorder = False + # self.state = MoveState.Unknown + + '''轨迹开始帧、结束帧 ID''' + # self.start_fid = int(np.min(boxes[:, 7])) + # self.end_fid = int(np.max(boxes[:, 7])) + + '''''' + self.Hands = [] + + self.HandsIou = [] + + self.Goods = [] + self.GoodsIou = [] + + + '''5个关键点（中心点、左上点、右上点、左下点、右下点 ）坐标''' + self.compute_cornpoints() + + '''5个关键点轨迹特征，可以在子类中实现，降低顺序处理时的计算量 + （中心点、左上点、右上点、左下点、右下点 ）轨迹特征''' + self.compute_cornpts_feats() + + '''应计算各个角点面积、平均面积''' + mw, mh = np.mean(boxes[:, 2]-boxes[:, 0]), np.mean((boxes[:, 3]-boxes[:, 1])) + self.mwh = np.mean((mw, mh)) + self.Area = mw * mh + + ''' + 最后一帧与第一帧间的位移： + vshift: 正值为向下，负值为向上 + hshift: 负值为向购物车边框两边移动，正值为物品向中心移动 + ''' + self.vshift = self.cornpoints[-1, 1] - self.cornpoints[0, 1] # 纵向位移 + self.hshift = abs(self.cornpoints[0, 0] - self.imgshape[0]/2) - \ + abs(self.cornpoints[-1, 0] - self.imgshape[0]/2) + + '''手部状态分析''' + self.HAND_STATIC_THRESH = 100 + if self.cls == 0: + self.extract_hand_features() + + def select_boxes(self): + + slt_boxes = [] + idx = np.argsort(self.boxes[:, 7]) + boxes = self.boxes[idx] + features = self.features[idx] + + for i in range(len(boxes)): + simi = None + box, tid, fid, bid = boxes[i, :4], int(boxes[i, 4]), int(boxes[i, 7]), int(boxes[i, 8]) + + if i == 0: + slt_boxes.append(boxes[i, :]) + continue + + if len(boxes)!=len(features): + print("check!") + continue + + box0, tid0, fid0, bid0 = boxes[i-1, :4], int(boxes[i-1, 4]), int(boxes[i-1, 7]), int(boxes[i-1, 8]) + + # 当前 box 和轨迹上一个 box 的iou + iou = bbox_ioa(box[None, :], box0[None, :]) + + # 当前 box 和轨迹上一个 box 的 feat similarity + feat0 = features[i, :][None, :] + feat1 = features[i-1, :][None, :] + simi = 1 - np.maximum(0.0, cdist(feat0, feat1, "cosine"))[0][0] + + if iou > 0.85 and simi>0.85: + continue + + slt_boxes.append(boxes[i, :]) + + + return np.array(slt_boxes) + + + def compute_cornpoints(self): + ''' + cornpoints 共10项，分别是个点的坐标值（x, y） + (center, top_left, top_right, bottom_left, bottom_right) + ''' + boxes = self.boxes + cornpoints = np.zeros((self.frnum, 10)) + cornpoints[:,0] = (boxes[:, 0] + boxes[:, 2]) / 2 + cornpoints[:,1] = (boxes[:, 1] + boxes[:, 3]) / 2 + cornpoints[:,2], cornpoints[:,3] = boxes[:, 0], boxes[:, 1] + cornpoints[:,4], cornpoints[:,5] = boxes[:, 2], boxes[:, 1] + cornpoints[:,6], cornpoints[:,7] = boxes[:, 0], boxes[:, 3] + cornpoints[:,8], cornpoints[:,9] = boxes[:, 2], boxes[:, 3] + + self.cornpoints = cornpoints + def compute_cornpts_feats(self): + ''' + ''' + # print(f"TrackID: {self.tid}") + trajectory = [] + trajlens = [] + trajdist = [] + trajrects = [] + trajrects_wh = [] + for k in range(5): + # diff_xy2 = np.power(np.diff(self.cornpoints[:, 2*k:2*(k+1)], axis = 0), 2) + # trajlen = np.sum(np.sqrt(np.sum(diff_xy2, axis = 1))) + + X = self.cornpoints[:, 2*k:2*(k+1)] + + traj = np.linalg.norm(np.diff(X, axis=0), axis=1) + trajectory.append(traj) + + trajlen = np.sum(traj) + trajlens.append(trajlen) + + ptdist = np.max(cdist(X, X)) + trajdist.append(ptdist) + + '''最小外接矩形： + rect[0]: 中心(x, y) + rect[1]: (w, h) + rect[0]: 旋转角度 (-90°, 0] + ''' + rect = cv2.minAreaRect(X.astype(np.int64)) + rect_wh = max(rect[1]) + + + trajrects_wh.append(rect_wh) + trajrects.append(rect) + + self.trajectory = trajectory + self.trajlens = trajlens + self.trajdist = trajdist + self.trajrects = trajrects + self.trajrects_wh = trajrects_wh + + + + def trajfeature(self): + ''' + 分两种情况计算轨迹特征（检测框边界不在图像边界范围内，在图像边界范围内）： + -最小长度轨迹：trajmin + -最小轨迹长度：trajlen_min + -最小轨迹欧氏距离：trajdist_max + ''' + + # idx1 = self.trajlens.index(max(self.trajlens)) + idx1 = self.trajrects_wh.index(max(self.trajrects_wh)) + + trajmax = self.trajectory[idx1] + trajlen_max = self.trajlens[idx1] + trajdist_max = self.trajdist[idx1] + if not self.isCornpoint: + # idx2 = self.trajlens.index(min(self.trajlens)) + idx2 = self.trajrects_wh.index(min(self.trajrects_wh)) + + trajmin = self.trajectory[idx2] + trajlen_min = self.trajlens[idx2] + trajdist_min = self.trajdist[idx2] + else: + trajmin = self.trajectory[0] + trajlen_min = self.trajlens[0] + trajdist_min = self.trajdist[0] + + + '''最小轨迹长度/最大轨迹长度，越小，代表运动幅度越小''' + trajlen_rate = trajlen_min/(trajlen_max+0.0001) + + '''最小轨迹欧氏距离/目标框尺度均值''' + trajdist_rate = trajdist_min/(self.mwh+0.0001) + + + + self.trajmin = trajmin + self.trajmax = trajmax + self.TrajFeat = [trajlen_min, trajlen_max, + trajdist_min, trajdist_max, + trajlen_rate, trajdist_rate] + + def pt_state_fids(self, det_y, STATIC_THRESH = 8): + ''' + 前摄时，y一般选择为 box 的 y1 坐标，且需限定商品在购物车内。 + inputs： + y：1D array， + parameters： + STATIC_THRESH：轨迹处于静止状态的阈值。 + outputs： + 输出为差分值小于 STATIC_THRESH 的y中元素的（start, end）索引 + ranges = [(x1, y1), + (x1, y1), + ...] + ''' + # print(f"The ID is: {self.tid}") + + # det_y = np.diff(y, axis=0) + ranges, rangex = [], [] + + static_indices = np.where(np.abs(det_y) < STATIC_THRESH)[0] + + if len(static_indices) == 0: + rangex.append((0, len(det_y))) + return ranges, rangex + + start_index = static_indices[0] + + for i in range(1, len(static_indices)): + if static_indices[i] != static_indices[i-1] + 1: + ranges.append((start_index, static_indices[i-1] + 1)) + start_index = static_indices[i] + ranges.append((start_index, static_indices[-1] + 1)) + + if len(ranges) == 0: + rangex.append((0, len(det_y))) + return ranges, rangex + + idx1, idx2 = ranges[0][0], ranges[-1][1] + + if idx1 != 0: + rangex.append((0, idx1)) + + # 轨迹的最后阶段是运动状态 + for k in range(1, len(ranges)): + index1 = ranges[k-1][1] + index2 = ranges[k][0] + rangex.append((index1, index2)) + + if idx2 != len(det_y): + rangex.append((idx2, len(det_y))) + + return ranges, rangex + + def PositionState(self, camerType="back"): + ''' + camerType: back, 后置摄像头 + front, 前置摄像头 + ''' + if camerType=="back": + incart = cv2.imread(str(parpath/'shopcart/cart_tempt/incart.png'), cv2.IMREAD_GRAYSCALE) + outcart = cv2.imread(str(parpath/'shopcart/cart_tempt/outcart.png'), cv2.IMREAD_GRAYSCALE) + else: + incart = cv2.imread(str(parpath/'shopcart/cart_tempt/incart_ftmp.png'), cv2.IMREAD_GRAYSCALE) + outcart = cv2.imread(str(parpath/'shopcart/cart_tempt/outcart_ftmp.png'), cv2.IMREAD_GRAYSCALE) + + # incart = cv2.imread('./cart_tempt/incart_ftmp.png', cv2.IMREAD_GRAYSCALE) + # outcart = cv2.imread('./cart_tempt/outcart_ftmp.png', cv2.IMREAD_GRAYSCALE) + + + xc, yc = self.cornpoints[:,0].clip(0,self.imgshape[0]-1).astype(np.int64), self.cornpoints[:,1].clip(0,self.imgshape[1]-1).astype(np.int64) + x1, y1 = self.cornpoints[:,6].clip(0,self.imgshape[0]-1).astype(np.int64), self.cornpoints[:,7].clip(0,self.imgshape[1]-1).astype(np.int64) + x2, y2 = self.cornpoints[:,8].clip(0,self.imgshape[0]-1).astype(np.int64), self.cornpoints[:,9].clip(0,self.imgshape[1]-1).astype(np.int64) + + # print(self.tid) + Cent_inCartnum = np.count_nonzero(incart[(yc, xc)]) + LB_inCartnum = np.count_nonzero(incart[(y1, x1)]) + RB_inCartnum = np.count_nonzero(incart[(y2, x2)]) + + Cent_outCartnum = np.count_nonzero(outcart[(yc, xc)]) + LB_outCartnum = np.count_nonzero(outcart[(y1, x1)]) + RB_outCartnum = np.count_nonzero(outcart[(y2, x2)]) + + '''Track完全在车内：左下角点、右下角点与 outcart 的交集为 0''' + self.isWholeInCart = False + if LB_outCartnum + RB_outCartnum == 0: + self.isWholeInCart = True + + '''Track完全在车外：左下角点、中心点与 incart 的交集为 0 + 右下角点、中心点与 incart 的交集为 0 + ''' + self.isWholeOutCart = False + if Cent_inCartnum + LB_inCartnum == 0 or Cent_inCartnum + RB_inCartnum == 0: + self.isWholeOutCart = True + + + self.Cent_isIncart = False + self.LB_isIncart = False + self.RB_isIncart = False + if Cent_inCartnum: self.Cent_isIncart = True + if LB_inCartnum: self.LB_isIncart = True + if RB_inCartnum: self.RB_isIncart = True + + self.posState = self.Cent_isIncart+self.LB_isIncart+self.RB_isIncart + + + def is_freemove(self): + # if self.tid==4: + # print(f"track ID: {self.tid}") + # boxes = self.boxes + # features = self.features + # similars = 1 - np.maximum(0.0, cdist(self.features, self.features, metric = 'cosine')) + + box1 = self.boxes[0, :4] + box2 = self.boxes[-1, :4] + + ''' 第1帧、最后一帧subimg的相似度 ''' + feat1 = self.features[0, :][None, :] + feat2 = self.features[-1, :][None, :] + similar = 1 - np.maximum(0.0, cdist(feat1, feat2, metric = 'cosine')) + condta = similar > 0.8 + + ''' 第1帧、最后一帧 boxes 四个角点间的距离 ''' + ptd = box2 - box1 + ptd1 = np.linalg.norm((ptd[0], ptd[1])) + ptd2 = np.linalg.norm((ptd[2], ptd[1])) + ptd3 = np.linalg.norm((ptd[0], ptd[3])) + ptd4 = np.linalg.norm((ptd[2], ptd[3])) + condtb = ptd1<50 and ptd2<50 and ptd3<50 and ptd4<50 + + condt = condta and condtb + return condt + + + def extract_hand_features(self): + assert self.cls == 0, "The class of traj must be HAND!" + + self.isHandStatic = False + + x0 = (self.boxes[:, 0] + self.boxes[:, 2]) / 2 + y0 = (self.boxes[:, 1] + self.boxes[:, 3]) / 2 + + handXY = np.stack((x0, y0), axis=-1) + # handMaxY0 = np.max(y0) + + handCenter = np.array([(max(x0)+min(x0))/2, (max(y0)+min(y0))/2]) + + handMaxDist = np.max(np.linalg.norm(handXY - handCenter)) + + if handMaxDist < self.HAND_STATIC_THRESH: + self.isHandStatic = True + + return + + +class doTracks: + def __init__(self, bboxes, trackefeats): + '''fundamental property + trackefeats: dict, key 格式 "fid_bid" + ''' + self.bboxes = bboxes + # self.TracksDict = TracksDict + self.frameID = np.unique(bboxes[:, 7].astype(int)) + self.trackID = np.unique(bboxes[:, 4].astype(int)) + + self.lboxes = self.array2list() + self.lfeats = self.getfeats(trackefeats) + + '''对 self.tracks 中的元素进行分类，将 track 归入相应列表中''' + self.Hands = [] + self.Kids = [] + self.Static = [] + self.Residual = [] + self.Confirmed = [] + self.DownWard = [] # subset of self.Residual + self.UpWard = [] # subset of self.Residual + self.FreeMove = [] # subset of self.Residual + + + + + def array2list(self): + ''' + 将 bboxes 变换为 track 列表 + bboxes: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] + Return： + lboxes：列表，列表中元素具有同一 track_id，x1y1x2y2 格式 + [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] + ''' + track_ids = self.bboxes[:, 4].astype(int) + lboxes = [] + for t_id in self.trackID: + # print(f"The ID is: {t_id}") + idx = np.where(track_ids == t_id)[0] + box = self.bboxes[idx, :] + + assert len(set(box[:, 7])) == len(box), "Please check!!!" + + lboxes.append(box) + + return lboxes + + def getfeats(self, trackefeats): + lboxes = self.lboxes + lfeats = [] + for boxes in lboxes: + feats = [] + for i in range(boxes.shape[0]): + fid, bid = int(boxes[i, 7]), int(boxes[i, 8]) + key = f"{int(fid)}_{int(bid)}" + if key in trackefeats: + feats.append(trackefeats[key]) + feats = np.asarray(feats, dtype=np.float32) + lfeats.append(feats) + + return lfeats + + + + def similarity(self): + nt = len(self.tracks) + similar_dict = {} + if nt >= 2: + for i in range(nt): + for j in range(i, nt): + tracka = self.tracks[i] + trackb = self.tracks[j] + similar = self.feat_similarity(tracka, trackb) + similar_dict.update({(tracka.tid, trackb.tid): similar}) + return similar_dict + + + def feat_similarity(self, tracka, trackb, metric='cosine'): + boxes_a, boxes_b = tracka.boxes, trackb.boxes + na, nb = tracka.boxes.shape[0], trackb.boxes.shape[0] + feata, featb = [], [] + for i in range(na): + fid, bid = tracka.boxes[i, 7:9] + feata.append(self.features_dict[fid][bid]) + for i in range(nb): + fid, bid = trackb.boxes[i, 7:9] + featb.append(self.features_dict[fid][bid]) + + feata = np.asarray(feata, dtype=np.float32) + featb = np.asarray(featb, dtype=np.float32) + similarity_matrix = 1-np.maximum(0.0, cdist(feata, featb, metric)) + + feata_m = np.mean(feata, axis =0)[None,:] + featb_m = np.mean(featb, axis =0)[None,:] + simi_ab = 1 - cdist(feata_m, featb_m, metric) + print(f'tid {int(boxes_a[0, 4])} vs {int(boxes_b[0, 4])}: {simi_ab[0][0]}') + + # return np.max(similarity_matrix) + return simi_ab + + def merge_tracks_loop(self, alist): + na, nb = len(alist), 0 + while na!=nb: + na = len(alist) + alist = self.merge_tracks(alist) #func is from subclass + nb = len(alist) + return alist + + def base_merge_tracks(self, Residual): + """ + 对不同id，但可能是同一商品的目标进行归并 + """ + mergedTracks = [] + alist = [t for t in Residual] + while alist: + atrack = alist[0] + cur_list = [] + cur_list.append(atrack) + alist.pop(0) + + blist = [b for b in alist] + alist = [] + for btrack in blist: + # afids = [] + # for track in cur_list: + # afids.extend(list(track.boxes[:, 7].astype(np.int_))) + # bfids = btrack.boxes[:, 7].astype(np.int_) + # interfid = set(afids).intersection(set(bfids)) + # if len(interfid): + # print("wait!!!") + # if track_equal_track(atrack, btrack) and len(interfid)==0: + if track_equal_track(atrack, btrack): + cur_list.append(btrack) + else: + alist.append(btrack) + + mergedTracks.append(cur_list) + + return mergedTracks + + @staticmethod + def join_tracks(tlista, tlistb): + """Combine two lists of stracks into a single one.""" + exists = {} + res = [] + for t in tlista: + exists[t.tid] = 1 + res.append(t) + for t in tlistb: + tid = t.tid + if not exists.get(tid, 0): + exists[tid] = 1 + res.append(t) + return res + + @staticmethod + def sub_tracks(tlista, tlistb): + track_ids_b = {t.tid for t in tlistb} + return [t for t in tlista if t.tid not in track_ids_b] + + + + def array2frame(self, bboxes): + frameID = np.sort(np.unique(bboxes[:, 7].astype(int))) + fboxes = [] + for fid in frameID: + idx = np.where(bboxes[:, 7] == fid)[0] + box = bboxes[idx, :] + fboxes.append(box) + return fboxes + + + def isintrude(self): + ''' + boxes: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] + 0 1 2 3 4 5 6 7 8 + ''' + OverlapNum = 3 + bboxes = self.bboxes.astype(np.int64) + fboxes = self.array2frame(bboxes) + + incart = cv2.bitwise_not(self.incart) + sum_incart = np.zeros(incart.shape, dtype=np.int64) + for fid, boxes in enumerate(fboxes): + for i in range(len(boxes)): + x1, y1, x2, y2 = boxes[i, 0:4] + sum_incart[y1:y2, x1:x2] += 1 + + sumincart = np.zeros(sum_incart.shape, dtype=np.uint8) + idx255 = np.where(sum_incart >= OverlapNum) + sumincart[idx255] = 255 + + idxnzr = np.where(sum_incart!=0) + base = np.zeros(sum_incart.shape, dtype=np.uint8) + base[idxnzr] = 255 + + contours_sum, _ = cv2.findContours(sumincart, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + contours_base, _ = cv2.findContours(base, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + + have_existed, invasion = [], [] + for k, ct_temp in enumerate(contours_base): + tmp1 = np.zeros(sum_incart.shape, dtype=np.uint8) + cv2.drawContours(tmp1, [ct_temp], -1, 255, cv2.FILLED) + + # 确定轮廓的包含关系 + for ct_sum in contours_sum: + tmp2 = np.zeros(sum_incart.shape, dtype=np.uint8) + cv2.drawContours(tmp2, [ct_sum], -1, 255, cv2.FILLED) + tmp = cv2.bitwise_and(tmp1, tmp2) + if np.count_nonzero(tmp) == np.count_nonzero(tmp2): + have_existed.append(k) + + inIdx = [i for i in range(len(contours_base)) if i not in have_existed] + invasion = np.zeros(sum_incart.shape, dtype=np.uint8) + + for i in inIdx: + cv2.drawContours(invasion, [contours_base[i]], -1, 255, cv2.FILLED) + cv2.imwrite("./result/intrude/invasion.png", invasion) + + + Intrude = True if len(inIdx)>=1 else False + print(f"is intruded: {Intrude}") + + return Intrude + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tracking/dotrack/dotracks_back.py b/tracking/dotrack/dotracks_back.py new file mode 100644 index 0000000..81bc54d --- /dev/null +++ b/tracking/dotrack/dotracks_back.py @@ -0,0 +1,277 @@ +# -*- coding: utf-8 -*- +""" +Created on Mon Mar 4 18:36:31 2024 + +@author: ym +""" +import numpy as np +import cv2 +import copy +import sys +from pathlib import Path + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[2] # YOLOv5 root directory +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) + +from tracking.utils.mergetrack import track_equal_track + + +from scipy.spatial.distance import cdist + +curpath = Path(__file__).resolve().parents[0] +curpath = Path(curpath) +parpath = curpath.parent + + +from .dotracks import doTracks, ShoppingCart +from .track_back import backTrack + + +class doBackTracks(doTracks): + def __init__(self, bboxes, trackefeats): + + super().__init__(bboxes, trackefeats) + + self.tracks = [backTrack(b, f) for b, f in zip(self.lboxes, self.lfeats)] + + # self.similar_dict = self.similarity() + # self.shopcart = ShoppingCart(bboxes) + + self.incart = self.getincart() + + + def getincart(self): + img1 = cv2.imread(str(parpath/'shopcart/cart_tempt/incart.png'), cv2.IMREAD_GRAYSCALE) + img2 = cv2.imread(str(parpath/'shopcart/cart_tempt/cartedge.png'), cv2.IMREAD_GRAYSCALE) + + + ret, binary1 = cv2.threshold(img1, 250, 255, cv2.THRESH_BINARY) + ret, binary2 = cv2.threshold(img2, 250, 255, cv2.THRESH_BINARY) + + binary = cv2.bitwise_or(binary1, binary2) + + + return binary + + + + def classify(self): + '''功能：对 tracks 中元素分类 ''' + + tracks = self.tracks + # 提取手的frame_id，并和动目标的frame_id 进行关联 + hand_tracks = [t for t in tracks if t.cls==0] + self.Hands.extend(hand_tracks) + + tracks = self.sub_tracks(tracks, hand_tracks) + + # 提取小孩的track，并计算状态：left, right, incart + kid_tracks = [t for t in tracks if t.cls==9] + kid_states = [self.kid_state(t) for t in kid_tracks] + self.Kids = [x for x in zip(kid_tracks, kid_states)] + + tracks = self.sub_tracks(tracks, kid_tracks) + + out_trcak = [t for t in tracks if t.isWholeOutCart] + tracks = self.sub_tracks(tracks, out_trcak) + + static_tracks = [t for t in tracks if t.frnum>1 and t.is_static()] + self.Static.extend(static_tracks) + + '''剔除静止目标后的 tracks''' + tracks = self.sub_tracks(tracks, static_tracks) + + + tracks_free = [t for t in tracks if t.frnum>1 and t.is_freemove()] + self.FreeMove.extend(tracks_free) + tracks = self.sub_tracks(tracks, tracks_free) + + + + # '''购物框边界外具有运动状态的干扰目标''' + # out_trcak = [t for t in tracks if t.is_OutTrack()] + # tracks = self.sub_tracks(tracks, out_trcak) + + + + '''轨迹循环归并''' + # merged_tracks = self.merge_tracks(tracks) + merged_tracks = self.merge_tracks_loop(tracks) + + [self.associate_with_hand(htrack, gtrack) for htrack in hand_tracks for gtrack in tracks] + + + tracks = [t for t in merged_tracks if t.frnum > 1] + + self.merged_tracks = merged_tracks + + static_tracks = [t for t in tracks if t.frnum>1 and t.is_static()] + self.Static.extend(static_tracks) + + tracks = self.sub_tracks(tracks, static_tracks) + + # for gtrack in tracks: + # for htrack in hand_tracks: + # hand_ious = self.associate_with_hand(htrack, gtrack) + # if len(hand_ious): + # gtrack.Hands.append(htrack) + # gtrack.HandsIou.append(hand_ious) + # htrack.Goods.append((gtrack, hand_ious)) + + # for htrack in hand_tracks: + # self.merge_based_hands(htrack) + + self.Residual = tracks + self.Confirmed = self.confirm_track() + + def confirm_track(self): + Confirmed = None + mindist = 0 + for track in self.Residual: + md = min(track.trajrects_wh) + if md > mindist: + mindist = copy.deepcopy(md) + Confirmed = copy.deepcopy(track) + + if Confirmed is not None: + return [Confirmed] + + return [] + + + # def merge_based_hands(self, htrack): + # gtracks = htrack.Goods + + # if len(gtracks) >= 2: + # atrack, afious = gtracks[0] + # btrack, bfious = gtracks[1] + + + + + def associate_with_hand(self, htrack, gtrack): + ''' + 迁移至基类： + 手部 Track、商品 Track 建立关联的依据： + a. 运动帧的帧索引有交集 + b. 帧索引交集部分iou均大于0 + ''' + + assert htrack.cls==0 and gtrack.cls!=0 and gtrack.cls!=9, 'Track cls is Error!' + + hand_ious = [] + + hboxes = np.empty(shape=(0, 9), dtype = np.float64) + gboxes = np.empty(shape=(0, 9), dtype = np.float64) + + + # start, end 为索引值，需要 start:(end+1) + for start, end in htrack.moving_index: + hboxes = np.concatenate((hboxes, htrack.boxes[start:end+1, :]), axis=0) + for start, end in gtrack.moving_index: + gboxes = np.concatenate((gboxes, gtrack.boxes[start:end+1, :]), axis=0) + + hfids, gfids = hboxes[:, 7], gboxes[:, 7] + fids = sorted(set(hfids).intersection(set(gfids))) + + + if len(fids)==0: + return None + + # print(f"Goods ID: {gtrack.tid}, Hand ID: {htrack.tid}") + + for f in fids: + h = np.where(hboxes[:,7] == f)[0][0] + g = np.where(gboxes[:,7] == f)[0][0] + + x11, y11, x12, y12 = hboxes[h, 0:4] + x21, y21, x22, y22 = gboxes[g, 0:4] + + x1, y1 = max((x11, x21)), max((y11, y21)) + x2, y2 = min((x12, x22)), min((y12, y22)) + + union = (x2 - x1).clip(0) * (y2 - y1).clip(0) + area1 = (x12 - x11) * (y12 - y11) + area2 = (x22 - x21) * (y22 - y21) + + iou = union / (area1 + area2 - union + 1e-6) + + if iou >= 0.01: + gtrack.Hands.append((htrack.tid, f, iou)) + + + return gtrack.Hands + + def merge_tracks(self, Residual): + """ + 对不同id，但可能是同一商品的目标进行归并 + 和 dotrack_front.py中函数相同，可以合并，可以合并至基类 + """ + mergedTracks = self.base_merge_tracks(Residual) + + oldtracks, newtracks = [], [] + for tracklist in mergedTracks: + if len(tracklist) > 1: + boxes = np.empty((0, 9), dtype=np.float32) + feats = np.empty((0, 256), dtype=np.float32) + for i, track in enumerate(tracklist): + if i==0: ntid, ncls=track.boxes[0, 4], track.boxes[0, 6] + iboxes = track.boxes.copy() + ifeats = track.features.copy() + + # iboxes[:, 4], iboxes[:, 6] = ntid, ncls + + boxes = np.concatenate((boxes, iboxes), axis=0) + feats = np.concatenate((feats, ifeats), axis=0) + + oldtracks.append(track) + + fid_indices = np.argsort(boxes[:, 7]) + + boxes_fid = boxes[fid_indices] + feats_fid = feats[fid_indices] + + + + newtracks.append(backTrack(boxes_fid, feats_fid)) + elif len(tracklist) == 1: + oldtracks.append(tracklist[0]) + newtracks.append(tracklist[0]) + + + redu = self.sub_tracks(Residual, oldtracks) + merged = self.join_tracks(redu, newtracks) + + return merged + + def kid_state(self, track): + + left_dist = track.cornpoints[:, 2] + right_dist = 1024 - track.cornpoints[:, 4] + + if np.sum(left_dist<30)/track.frnum>0.8 and np.sum(right_dist>512)/track.frnum>0.7: + kidstate = "left" + elif np.sum(left_dist>512)/track.frnum>0.7 and np.sum(right_dist<30)/track.frnum>0.8: + kidstate = "right" + else: + kidstate = "incart" + + return kidstate + + + def isuptrack(self, track): + Flag = False + + return Flag + + def isdowntrack(self, track): + Flag = False + + return Flag + + def isfreetrack(self, track): + Flag = False + + return Flag \ No newline at end of file diff --git a/tracking/dotrack/dotracks_front.py b/tracking/dotrack/dotracks_front.py new file mode 100644 index 0000000..cfa899a --- /dev/null +++ b/tracking/dotrack/dotracks_front.py @@ -0,0 +1,193 @@ +# -*- coding: utf-8 -*- +""" +Created on Mon Mar 4 18:38:20 2024 + +@author: ym +""" +import cv2 +import copy +import numpy as np +from pathlib import Path + +curpath = Path(__file__).resolve().parents[0] +curpath = Path(curpath) +parpath = curpath.parent +# from tracking.utils.mergetrack import track_equal_track +from .dotracks import doTracks +from .track_front import frontTrack + +class doFrontTracks(doTracks): + def __init__(self, bboxes, frameDictList): + super().__init__(bboxes, frameDictList) + + # self.tracks = [frontTrack(b) for b in self.lboxes] + self.tracks = [frontTrack(b, f) for b, f in zip(self.lboxes, self.lfeats)] + + self.incart = self.getincart() + + def getincart(self): + img = cv2.imread(str(parpath/'shopcart/cart_tempt/incart_ftmp.png'), cv2.IMREAD_GRAYSCALE) + ret, binary = cv2.threshold(img, 250, 255, cv2.THRESH_BINARY) + + return binary + + + def classify(self): + '''功能：对 tracks 中元素分类 ''' + + tracks = self.tracks + + '''提取手的 tracks''' + hand_tracks = [t for t in tracks if t.cls==0] + + + self.Hands.extend(hand_tracks) + tracks = self.sub_tracks(tracks, hand_tracks) + + + + '''提取小孩的 tracks''' + kid_tracks = [t for t in tracks if t.cls==9] + tracks = self.sub_tracks(tracks, kid_tracks) + + out_trcak = [t for t in tracks if t.isWholeOutCart] + tracks = self.sub_tracks(tracks, out_trcak) + + '''静态 tracks''' + static_tracks = [t for t in tracks if t.frnum>1 and t.is_static()] + + + '''剔除静止目标后的 tracks''' + tracks = self.sub_tracks(tracks, static_tracks) + + tracks_free = [t for t in tracks if t.frnum>1 and t.is_freemove()] + self.FreeMove.extend(tracks_free) + tracks = self.sub_tracks(tracks, tracks_free) + + # [self.associate_with_hand(htrack, gtrack) for htrack in hand_tracks for gtrack in tracks] + '''轨迹循环归并''' + merged_tracks = self.merge_tracks_loop(tracks) + + [self.associate_with_hand(htrack, gtrack) for htrack in hand_tracks for gtrack in merged_tracks] + + tracks = [t for t in merged_tracks if t.frnum > 1] + + # for gtrack in tracks: + # # print(f"Goods ID:{gtrack.tid}") + # for htrack in hand_tracks: + # hand_ious = self.associate_with_hand(htrack, gtrack) + # if len(hand_ious): + # gtrack.Hands.append(htrack) + # gtrack.HandsIou.append(hand_ious) + + '''静止 tracks 判断与剔除静止 tracks''' + static_tracks = [t for t in tracks if t.frnum>1 and t.is_static()] + tracks = self.sub_tracks(tracks, static_tracks) + + freemoved_tracks = [t for t in tracks if t.is_free_move()] + tracks = self.sub_tracks(tracks, freemoved_tracks) + + self.Residual = tracks + self.Confirmed = self.confirm_track() + + def confirm_track(self): + Confirmed = None + mindist = 0 + for track in self.Residual: + md = min(track.trajrects_wh) + if md > mindist: + mindist = copy.deepcopy(md) + Confirmed = copy.deepcopy(track) + + if Confirmed is not None: + return [Confirmed] + + return [] + + def associate_with_hand(self, htrack, gtrack): + ''' + 迁移至基类： + 手部 Track、商品 Track 建立关联的依据： + a. 运动帧的帧索引有交集 + b. 帧索引交集部分iou均大于0 + ''' + assert htrack.cls==0 and gtrack.cls!=0 and gtrack.cls!=9, 'Track cls is Error!' + + hboxes = np.empty(shape=(0, 9), dtype = np.float64) + gboxes = np.empty(shape=(0, 9), dtype = np.float64) + + # start, end 为索引值，需要 start:(end+1) + for start, end in htrack.dynamic_y2: + hboxes = np.concatenate((hboxes, htrack.boxes[start:end+1, :]), axis=0) + for start, end in gtrack.dynamic_y1: + gboxes = np.concatenate((gboxes, gtrack.boxes[start:end+1, :]), axis=0) + + hfids, gfids = hboxes[:, 7], gboxes[:, 7] + fids = sorted(set(hfids).intersection(set(gfids))) + + if len(fids)==0: + return None + + # print(f"Goods ID: {gtrack.tid}, Hand ID: {htrack.tid}") + for f in fids: + h = np.where(hfids==f)[0][0] + g = np.where(gfids==f)[0][0] + + x11, y11, x12, y12 = hboxes[h, 0:4] + x21, y21, x22, y22 = gboxes[g, 0:4] + + x1, y1 = max((x11, x21)), max((y11, y21)) + x2, y2 = min((x12, x22)), min((y12, y22)) + + union = (x2 - x1).clip(0) * (y2 - y1).clip(0) + area1 = (x12 - x11) * (y12 - y11) + area2 = (x22 - x21) * (y22 - y21) + + iou = union / (area1 + area2 - union + 1e-6) + + if iou >= 0.01: + gtrack.Hands.append((htrack.tid, f, iou)) + + return gtrack.Hands + + + + def merge_tracks(self, Residual): + """ + 对不同id，但可能是同一商品的目标进行归并 + 和 dotrack_back.py中函数相同，可以合并至基类 + """ + mergedTracks = self.base_merge_tracks(Residual) + + oldtracks, newtracks = [], [] + for tracklist in mergedTracks: + if len(tracklist) > 1: + boxes = np.empty((0, 9), dtype=np.float32) + feats = np.empty((0, 256), dtype=np.float32) + for i, track in enumerate(tracklist): + if i==0: ntid, ncls=track.boxes[0, 4], track.boxes[0, 6] + iboxes = track.boxes.copy() + ifeats = track.features.copy() + + # iboxes[:, 4], iboxes[:, 6] = ntid, ncls + + boxes = np.concatenate((boxes, iboxes), axis=0) + feats = np.concatenate((feats, ifeats), axis=0) + + oldtracks.append(track) + + fid_indices = np.argsort(boxes[:, 7]) + boxes_fid = boxes[fid_indices] + feats_fid = feats[fid_indices] + + newtracks.append(frontTrack(boxes_fid, feats_fid)) + elif len(tracklist) == 1: + oldtracks.append(tracklist[0]) + newtracks.append(tracklist[0]) + + + redu = self.sub_tracks(Residual, oldtracks) + merged = self.join_tracks(redu, newtracks) + + return merged + diff --git a/tracking/dotrack/track_back.py b/tracking/dotrack/track_back.py new file mode 100644 index 0000000..6f40ea4 --- /dev/null +++ b/tracking/dotrack/track_back.py @@ -0,0 +1,241 @@ +# -*- coding: utf-8 -*- +""" +Created on Mon Mar 4 18:28:47 2024 + +@author: ym +""" +import cv2 +import numpy as np +from scipy.spatial.distance import cdist +from sklearn.decomposition import PCA +from .dotracks import MoveState, Track + +from pathlib import Path +curpath = Path(__file__).resolve().parents[0] +curpath = Path(curpath) +parpath = curpath.parent + +class backTrack(Track): + # boxes: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] + # 0, 1, 2, 3, 4, 5, 6, 7, 8 + def __init__(self, boxes, features, imgshape=(1024, 1280)): + + super().__init__(boxes, features, imgshape) + + '''该函数依赖项: self.cornpoints + MarginState: list, seven elements, 表示轨迹中boxes出现在图像的 + [左上，右上，左中，右中，左下，右下底部] + ''' + self.isCornpoint, self.MarginState = self.isimgborder() + + '''该函数依赖项: self.isCornpoint，不能在父类中初始化''' + self.trajfeature() + + + '''静止点帧索引''' + # self.static_index = self.compute_static_fids() + + '''运动点帧索引(运动帧两端的静止帧索引)''' + # self.moving_index = self.compute_dynamic_fids() + + self.static_index, self.moving_index = self.compute_static_dynamic_fids() + + '''该函数依赖项: self.cornpoints，定义 4 个商品位置变量： + self.Cent_isIncart, self.LB_isIncart, self.RB_isIncart + self.posState = self.Cent_isIncart+self.LB_isIncart+self.RB_isIncart''' + self.PositionState(camerType="back") + + '''self.feature_ious = (incart_iou, outcart_iou, cartboarder_iou, maxbox_iou, minbox_iou) + self.incartrates = incartrates''' + self.compute_ious_feat() + + + + + def isimgborder(self, BoundPixel=10, BoundThresh=0.3): + + x1, y1 = self.cornpoints[:,2], self.cornpoints[:,3], + x2, y2 = self.cornpoints[:,8], self.cornpoints[:,9] + + condt1 = sum(abs(x1) BoundThresh + condt2 = sum(abs(y1) BoundThresh + condt3 = sum(abs(x2-self.imgshape[0]) BoundThresh + condt4 = sum(abs(y2-self.imgshape[1]) BoundThresh + + condt = condt1 or condt2 or condt3 or condt4 + isCornpoint = False + if condt: + isCornpoint = True + + condtA = condt1 and condt2 + condtB = condt3 and condt2 + condtC = condt1 and not condt2 and not condt4 + condtD = condt3 and not condt2 and not condt4 + condtE = condt1 and condt4 + condtF = condt3 and condt4 + condtG = condt4 and not condt1 and not condt3 + MarginState = [condtA, condtB, condtC, condtD, condtE, condtF, condtG] + + return isCornpoint, MarginState + + def PCA(self): + self.pca = PCA() + + X = self.cornpoints[:, 0:2] + self.pca.fit(X) + + + def compute_ious_feat(self): + '''输出： + self.feature_ious = (incart_iou, outcart_iou, cartboarder_iou, maxbox_iou, minbox_iou) + self.incartrates = incartrates， + 其中： + boxes流：track中所有boxes形成的轨迹图，可分为三部分：incart, outcart, cartboarder + incart_iou, outcart_iou, cartboarder_iou：各部分和 boxes流的 iou。 + incart_iou = 0，track在购物车外， + outcart_iou = 0，track在购物车内，也可能是通过左下角、右下角置入购物车， + maxbox_iou, minbox_iou：track中最大、最小 box 和boxes流的iou，二者差值越小，越接近 1，表明track的运动型越小。 + incartrates: 各box和incart的iou时序，由小变大，反应的是置入过程，由大变小，反应的是取出过程 + ''' + incart = cv2.imread(str(parpath/"shopcart/cart_tempt/incart.png"), cv2.IMREAD_GRAYSCALE) + outcart = cv2.imread(str(parpath/"shopcart/cart_tempt/outcart.png"), cv2.IMREAD_GRAYSCALE) + cartboarder = cv2.imread(str(parpath/"shopcart/cart_tempt/cartboarder.png"), cv2.IMREAD_GRAYSCALE) + + incartrates = [] + temp = np.zeros(incart.shape, np.uint8) + maxarea, minarea = 0, self.imgshape[0]*self.imgshape[1] + for i in range(self.frnum): + # x, y, w, h = self.boxes[i, 0:4] + + x = (self.boxes[i, 2] + self.boxes[i, 0]) / 2 + w = (self.boxes[i, 2] - self.boxes[i, 0]) / 2 + y = (self.boxes[i, 3] + self.boxes[i, 1]) / 2 + h = (self.boxes[i, 3] - self.boxes[i, 1]) / 2 + + + if w*h > maxarea: maxarea = w*h + if w*h < minarea: minarea = w*h + cv2.rectangle(temp, (int(x-w/2), int(y-h/2)), (int(x+w/2), int(y+h/2)), 255, cv2.FILLED) + + temp1 = np.zeros(incart.shape, np.uint8) + cv2.rectangle(temp1, (int(x-w/2), int(y-h/2)), (int(x+w/2), int(y+h/2)), 255, cv2.FILLED) + temp2 = cv2.bitwise_and(incart, temp1) + inrate = cv2.countNonZero(temp1)/(w*h) + incartrates.append(inrate) + + isincart = cv2.bitwise_and(incart, temp) + isoutcart = cv2.bitwise_and(outcart, temp) + iscartboarder = cv2.bitwise_and(cartboarder, temp) + + num_temp = cv2.countNonZero(temp) + num_incart = cv2.countNonZero(isincart) + num_outcart = cv2.countNonZero(isoutcart) + num_cartboarder = cv2.countNonZero(iscartboarder) + + incart_iou = num_incart/(num_temp+1e-6) + outcart_iou = num_outcart/(num_temp+1e-6) + cartboarder_iou = num_cartboarder/(num_temp+1e-6) + maxbox_iou = maxarea/(num_temp+1e-6) + minbox_iou = minarea/(num_temp+1e-6) + + self.feature_ious = (incart_iou, outcart_iou, cartboarder_iou, maxbox_iou, minbox_iou) + self.incartrates = incartrates + + + def compute_static_dynamic_fids(self): + + if self.MarginState[0] or self.MarginState[2]: + idx1 = 4 + elif self.MarginState[1] or self.MarginState[3]: + idx1 = 3 + elif self.MarginState[4]: + idx1 = 2 + elif self.MarginState[5]: + idx1 = 1 + elif self.MarginState[6]: + if self.trajlens[1] < self.trajlens[2]: + idx1 = 1 + else: + idx1 = 2 + else: + idx1 = self.trajlens.index(min(self.trajlens)) + + # idx1 = self.trajlens.index(min(self.trajlens)) + trajmin = self.trajectory[idx1] + + static, dynamic = self.pt_state_fids(trajmin) + + static = np.array(static) + dynamic = np.array(dynamic) + + if static.size: + indx = np.argsort(static[:, 0]) + static = static[indx] + if dynamic.size: + indx = np.argsort(dynamic[:, 0]) + dynamic = dynamic[indx] + + return static, dynamic + + + def is_static(self): + '''静态情况 1: 目标关键点最小相对运动轨迹 < 0.2, 指标值偏大 + TrajFeat = [trajlen_min, trajlen_max, + trajdist_min, trajdist_max, + trajlen_rate, trajdist_rate] + ''' + # print(f"TrackID: {self.tid}") + + boxes = self.boxes + + '''静态情况 1: ''' + condt1 = self.TrajFeat[5] < 0.2 or self.TrajFeat[3] < 120 + + '''静态情况 2: 目标初始状态为静止，适当放宽关键点最小相对运动轨迹 < 0.5''' + condt2 = self.static_index.size > 0 \ + and self.static_index[0, 0] <= 2 \ + and self.static_index[0, 1] >= 5 \ + and self.TrajFeat[5] < 0.5 \ + and self.TrajFeat[1] < 240 \ + and self.isWholeInCart + # and self.posState >= 2 + # and self.TrajFeat[0] < 240 \ + + '''静态情况 3: 目标初始状态和最终状态均为静止''' + condt3 = self.static_index.shape[0] >= 2 \ + and self.static_index[0, 0] <= 2 \ + and self.static_index[0, 1] >= 5 \ + and self.static_index[-1, 1] >= self.frnum-3 \ + and self.TrajFeat[1] < 240 \ + and self.isWholeInCart + # and self.posState >= 2 + # and self.TrajFeat[0] < 240 \ + + condt4 = self.static_index.shape[0] >= 2 \ + and self.static_index[0, 0] <= 2 \ + and self.static_index[0, 1] >= 6 \ + and self.static_index[-1, 0] <= self.frnum-5 \ + and self.static_index[-1, 1] >= self.frnum-2 + + condt = condt1 or condt2 or condt3 or condt4 + + return condt + + + + + def is_OutTrack(self): + if self.posState <= 1: + isout = True + else: + isout = False + return isout + + def compute_distance(self): + pass + + def move_start_fid(self): + pass + + def move_end_fid(self): + pass \ No newline at end of file diff --git a/tracking/dotrack/track_front.py b/tracking/dotrack/track_front.py new file mode 100644 index 0000000..1b28685 --- /dev/null +++ b/tracking/dotrack/track_front.py @@ -0,0 +1,194 @@ +# -*- coding: utf-8 -*- +""" +Created on Mon Mar 4 18:33:01 2024 + +@author: ym +""" +import numpy as np +import cv2 +# from sklearn.cluster import KMeans +from .dotracks import MoveState, Track + +from pathlib import Path +curpath = Path(__file__).resolve().parents[0] +curpath = Path(curpath) +parpath = curpath.parent + + +class frontTrack(Track): + # boxes: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] + # 0, 1, 2, 3, 4, 5, 6, 7, 8 + def __init__(self, boxes, features, imgshape=(1024, 1280)): + + super().__init__(boxes, features, imgshape) + + self.CART_HIGH_THRESH1 = imgshape[1]/2.98 + + '''y1、y2静止状态区间，值是 boxes 中对 axis=0 的索引，不是帧索引''' + det_y1 = np.diff(boxes[:, 1], axis=0) + det_y2 = np.diff(boxes[:, 3], axis=0) + self.static_y1, self.dynamic_y1 = self.pt_state_fids(det_y1) + self.static_y2, self.dynamic_y2 = self.pt_state_fids(det_y2) + + self.isCornpoint = self.is_left_or_right_cornpoint() + self.isBotmpoint = self.is_bottom_cornpoint() + + '''该函数依赖项: self.isCornpoint，不能在父类中初始化''' + self.trajfeature() + + self.PositionState(camerType="front") + + '''手部状态分析''' + self.HAND_STATIC_THRESH = 100 + self.CART_POSIT_0 = 430 + self.CART_POSIT_1 = 620 + + def is_left_or_right_cornpoint(self): + ''' 基于 all(boxes)， + boxes左下角点和图像左下角点重叠 或 + boxes右下角点和图像左下角点重叠 + ''' + x1, y1 = self.boxes[:, 0], self.boxes[:, 1] + x2, y2 = self.boxes[:, 2], self.boxes[:, 3] + + # Left-Bottom cornpoint + condt1 = all(x1 < 5) and all(y2 > self.imgshape[1]-5) + + # Right-Bottom cornpoint + condt2 = all(x2 > self.imgshape[0]-5) and all(y2 > self.imgshape[1]-5) + + condt = condt1 or condt2 + + return condt + + def is_edge_cornpoint(self): + '''基于 all(boxes)，boxes是否和图像左右边缘重叠''' + x1, x2 = self.boxes[:, 0], self.boxes[:, 2] + condt = all(x1 < 3) or all(x2 > self.imgshape[0]-3) + + return condt + + def is_bottom_cornpoint(self): + '''基于 all(boxes)，boxes是否和图像下边缘重叠''' + condt = all(self.boxes[:, 3] > self.imgshape[1]-20) + + return condt + + + def is_static(self): + assert self.frnum > 1, "boxes number must greater than 1" + # print(f"The ID is: {self.tid}") + + # 手部和小孩目标不考虑 + if self.cls == 0 or self.cls == 9: + return False + + # boxes 全部 y2=1280 + if self.isBotmpoint: + return True + + boxes = self.boxes + y0 = (boxes[:, 1]+boxes[:, 3])/2 + + ## 纵轴矢量和 + sum_y0 = y0[-1] - y0[0] + sum_y1 = boxes[-1, 1]-boxes[0, 1] + sum_y2 = boxes[-1, 3]-boxes[0, 3] + + # 一些需要考虑的特殊情况 + isbottom = max(boxes[:, 3]) > 1280-3 + istop = min(boxes[:, 1]) < 3 + isincart = min(y0) > self.CART_HIGH_THRESH1 + uncert = abs(sum_y1)<100 and abs(sum_y2)<100 + + '''初始条件：商品中心点始终在购物车内、''' + condt0 = max((boxes[:, 1]+boxes[:, 3])/2) > self.CART_HIGH_THRESH1 + + '''条件1：轨迹运动纵向和（y1 或 y2）描述商品轨迹长度，存在情况： + (1). 检测框可能与图像上下边缘重合， + (2). 上边或下边存在跳动 + ''' + if isbottom and istop: + condt1 = abs(sum_y0) < 300 + elif isbottom: # y2在底部，用y1表征运动 + condt1 = sum_y1 > -120 and abs(sum_y0)<80 # 有底部点，方向向上阈值小于100 + elif istop: # y1在顶部，用y2表征运动 + condt1 = abs(sum_y2) < 100 + else: + condt1 = (abs(sum_y1) < 30 or abs(sum_y2)<30) + + '''条件2：轨迹的开始和结束阶段均处于静止状态, 利用静止状态区间判断，用 y1 + a. 商品在购物车内， + b. 检测框的起始阶段和结束阶段均为静止状态 + c. 静止帧长度 > 3''' + + condt2 = False + if len(self.static_y1)>=2: + condt_s0 = self.static_y1[0][0]==0 and self.static_y1[0][1] - self.static_y1[0][0] >= 3 + condt_s1 = self.static_y1[-1][1]==self.frnum-1 and self.static_y1[-1][1] - self.static_y1[-1][0] >= 3 + condt2 = condt_s0 and condt_s1 and isincart + + + condt = condt0 and (condt1 or condt2) + + return condt + + + def is_upward(self): + '''判断商品是否取出，''' + print(f"The ID is: {self.tid}") + + def is_free_move(self): + if self.frnum == 1: + return True + # print(f"The ID is: {self.tid}") + + + y0 = (self.boxes[:, 1] + self.boxes[:, 3]) / 2 + det_y0 = np.diff(y0, axis=0) + sum_y0 = y0[-1] - y0[0] + + '''情况1：中心点向下 ''' + ## 初始条件：商品第一次检测到在购物车内 + condt0 = y0[0] > self.CART_HIGH_THRESH1 + + condt_a = False + ## 条件1：商品初始为静止状态，静止条件应严格一些 + condt11, condt12 = False, False + if len(self.static_y1)>0: + condt11 = self.static_y1[0][0]==0 and self.static_y1[0][1] - self.static_y1[0][0] >= 5 + if len(self.static_y2)>0: + condt12 = self.static_y2[0][0]==0 and self.static_y2[0][1] - self.static_y2[0][0] >= 5 + + # 条件2：商品中心发生向下移动 + condt2 = y0[-1] > y0[0] + + # 综合判断a + condt_a = condt0 and (condt11 or condt12) and condt2 + + '''情况2：中心点向上 ''' + ## 商品中心点向上移动，但没有关联的Hand轨迹，也不是左右边界点 + condt_b = condt0 and len(self.Hands)==0 and y0[-1] < y0[0] and (not self.is_edge_cornpoint()) and min(y0)>self.CART_HIGH_THRESH1 + + + '''情况3: 商品在购物车内，但运动方向无序''' + ## 中心点在购物车内，纵向轨迹和小于轨迹差中绝对值最大的两个值的和，说明运动没有主方向 + condt_c = False + if self.frnum > 3: + condt_c = all(y0>self.CART_HIGH_THRESH1) and \ + (abs(sum_y0) < sum(np.sort(np.abs(det_y0))[::-1][:2])-1) + + condt = (condt_a or condt_b or condt_c) and self.cls!=0 + + return condt + + + + + + + + + + + \ No newline at end of file diff --git a/tracking/dotrack/track_select.py b/tracking/dotrack/track_select.py new file mode 100644 index 0000000..d27fd1a --- /dev/null +++ b/tracking/dotrack/track_select.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +""" +Created on Mon Jul 29 10:28:21 2024 + 未来需将这一部分和轨迹分析代码集成 + +@author: ym +""" +import numpy as np +import cv2 +from scipy.spatial.distance import cdist + +class TProp: + def __init__(self, boxes): + + self.boxes = boxes + + + + + +class TProp: + '''抽象基类，不能实例化对象''' + def __init__(self, boxes): + ''' + boxes: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] + 0 1 2 3 4 5 6 7 8 + ''' + # assert len(set(boxes[:, 4].astype(int))) == 1, "For a Track, track_id more than 1" + # assert len(set(boxes[:, 6].astype(int))) == 1, "For a Track, class number more than 1" + + self.boxes = boxes + + '''5个关键点（中心点、左上点、右上点、左下点、右下点 ）坐标''' + self.compute_cornpoints() + + '''5个关键点轨迹特征，可以在子类中实现，降低顺序处理时的计算量 + （中心点、左上点、右上点、左下点、右下点 ）轨迹特征''' + self.compute_cornpts_feats() + + self.distmax = max(self.trajdist) + + + def compute_cornpoints(self): + ''' + cornpoints 共10项，分别是个点的坐标值（x, y） + (center, top_left, top_right, bottom_left, bottom_right) + ''' + boxes = self.boxes + cornpoints = np.zeros((self.frnum, 10)) + cornpoints[:,0] = (boxes[:, 0] + boxes[:, 2]) / 2 + cornpoints[:,1] = (boxes[:, 1] + boxes[:, 3]) / 2 + cornpoints[:,2], cornpoints[:,3] = boxes[:, 0], boxes[:, 1] + cornpoints[:,4], cornpoints[:,5] = boxes[:, 2], boxes[:, 1] + cornpoints[:,6], cornpoints[:,7] = boxes[:, 0], boxes[:, 3] + cornpoints[:,8], cornpoints[:,9] = boxes[:, 2], boxes[:, 3] + + self.cornpoints = cornpoints + def compute_cornpts_feats(self): + ''' + ''' + trajectory = [] + trajlens = [] + trajdist = [] + trajrects = [] + for k in range(5): + # diff_xy2 = np.power(np.diff(self.cornpoints[:, 2*k:2*(k+1)], axis = 0), 2) + # trajlen = np.sum(np.sqrt(np.sum(diff_xy2, axis = 1))) + + X = self.cornpoints[:, 2*k:2*(k+1)] + + traj = np.linalg.norm(np.diff(X, axis=0), axis=1) + trajectory.append(traj) + + trajlen = np.sum(traj) + trajlens.append(trajlen) + + ptdist = np.max(cdist(X, X)) + trajdist.append(ptdist) + + '''最小外接矩形： + rect[0]: 中心(x, y) + rect[1]: (w, h) + rect[0]: 旋转角度 (-90°, 0] + ''' + rect = cv2.minAreaRect(X.astype(np.int64)) + trajrects.append(rect) + + self.trajectory = trajectory + self.trajlens = trajlens + self.trajdist = trajdist + self.trajrects = trajrects \ No newline at end of file diff --git a/tracking/matching/视频分类/单.txt b/tracking/matching/视频分类/单.txt new file mode 100644 index 0000000..69ab04b --- /dev/null +++ b/tracking/matching/视频分类/单.txt @@ -0,0 +1,807 @@ +230537101280010007_20240411-144918_back_addGood_70f75407b7ae_570_17788571404.mp4 +230537101280010007_20240411-144918_front_addGood_70f75407b7ae_570_17788571404.mp4 +230537101280010007_20240411-144945_back_returnGood_70f75407b7ae_565_17788571404.mp4 +230537101280010007_20240411-144945_front_returnGood_70f75407b7ae_565_17788571404.mp4 +230538001280010009_20240411-144924_back_addGood_70f754088050_550_17327712807.mp4 +230538001280010009_20240411-144924_front_addGood_70f754088050_550_17327712807.mp4 +230538001280010009_20240411-144934_back_returnGood_70f754088050_550_17327712807.mp4 +230538001280010009_20240411-144934_front_returnGood_70f754088050_550_17327712807.mp4 +2500456001326_20240411-145321_back_addGood_70f75407b7ae_155_17788571404.mp4 +2500456001326_20240411-145321_front_addGood_70f75407b7ae_155_17788571404.mp4 +2500456001326_20240411-145327_back_returnGood_70f75407b7ae_155_17788571404.mp4 +2500456001326_20240411-145327_front_returnGood_70f75407b7ae_155_17788571404.mp4 +2500456001326_20240411-145330_back_addGood_70f754088050_155_17327712807.mp4 +2500456001326_20240411-145330_front_addGood_70f754088050_155_17327712807.mp4 +2500456001326_20240411-145338_back_returnGood_70f754088050_155_17327712807.mp4 +2500456001326_20240411-145338_front_returnGood_70f754088050_155_17327712807.mp4 +2500458675341_20240411-144658_back_addGood_70f75407b7ae_140_17788571404.mp4 +2500458675341_20240411-144658_front_addGood_70f75407b7ae_140_17788571404.mp4 +2500458675341_20240411-144707_back_returnGood_70f75407b7ae_140_17788571404.mp4 +2500458675341_20240411-144707_front_returnGood_70f75407b7ae_140_17788571404.mp4 +2500458675341_20240411-144711_back_addGood_70f754088050_135_17327712807.mp4 +2500458675341_20240411-144711_front_addGood_70f754088050_135_17327712807.mp4 +2500458675341_20240411-144718_back_returnGood_70f754088050_135_17327712807.mp4 +2500458675341_20240411-144718_front_returnGood_70f754088050_135_17327712807.mp4 +2500463464671_20240411-145041_back_addGood_70f75407b7ae_805_17788571404.mp4 +2500463464671_20240411-145041_front_addGood_70f75407b7ae_805_17788571404.mp4 +2500463464671_20240411-145042_back_addGood_70f754088050_815_17327712807.mp4 +2500463464671_20240411-145042_front_addGood_70f754088050_815_17327712807.mp4 +2500463464671_20240411-145049_back_returnGood_70f754088050_815_17327712807.mp4 +2500463464671_20240411-145049_front_returnGood_70f754088050_815_17327712807.mp4 +2500463464671_20240411-145053_back_returnGood_70f75407b7ae_810_17788571404.mp4 +2500463464671_20240411-145053_front_returnGood_70f75407b7ae_810_17788571404.mp4 +6901070613142_20240411-142722_back_addGood_70f754088050_240_17327712807.mp4 +6901070613142_20240411-142722_front_addGood_70f754088050_240_17327712807.mp4 +6901070613142_20240411-142725_back_addGood_70f75407b7ae_240_17788571404.mp4 +6901070613142_20240411-142725_front_addGood_70f75407b7ae_240_17788571404.mp4 +6901070613142_20240411-142730_back_returnGood_70f754088050_240_17327712807.mp4 +6901070613142_20240411-142730_front_returnGood_70f754088050_240_17327712807.mp4 +6901070613142_20240411-142734_back_returnGood_70f75407b7ae_240_17788571404.mp4 +6901070613142_20240411-142734_front_returnGood_70f75407b7ae_240_17788571404.mp4 +6901668053893_20240411-143608_back_addGood_70f75407b7ae_70_17788571404.mp4 +6901668053893_20240411-143608_back_addGood_70f754088050_70_17327712807.mp4 +6901668053893_20240411-143608_front_addGood_70f75407b7ae_70_17788571404.mp4 +6901668053893_20240411-143608_front_addGood_70f754088050_70_17327712807.mp4 +6901668053893_20240411-143616_back_returnGood_70f754088050_70_17327712807.mp4 +6901668053893_20240411-143616_front_returnGood_70f754088050_70_17327712807.mp4 +6901668053893_20240411-143617_back_returnGood_70f75407b7ae_70_17788571404.mp4 +6901668053893_20240411-143617_front_returnGood_70f75407b7ae_70_17788571404.mp4 +6902007010249_20240411-142528_back_addGood_70f75407b7ae_755_17788571404.mp4 +6902007010249_20240411-142528_back_addGood_70f754088050_755_17327712807.mp4 +6902007010249_20240411-142528_front_addGood_70f75407b7ae_755_17788571404.mp4 +6902007010249_20240411-142528_front_addGood_70f754088050_755_17327712807.mp4 +6902007010249_20240411-142535_back_returnGood_70f75407b7ae_755_17788571404.mp4 +6902007010249_20240411-142535_front_returnGood_70f75407b7ae_755_17788571404.mp4 +6902007010249_20240411-142541_back_returnGood_70f754088050_755_17327712807.mp4 +6902007010249_20240411-142541_front_returnGood_70f754088050_755_17327712807.mp4 +6902022135514_20240411-142819_back_addGood_70f75407b7ae_3180_17788571404.mp4 +6902022135514_20240411-142819_front_addGood_70f75407b7ae_3180_17788571404.mp4 +6902022135514_20240411-142828_back_addGood_70f754088050_3185_17327712807.mp4 +6902022135514_20240411-142828_front_addGood_70f754088050_3185_17327712807.mp4 +6902022135514_20240411-142830_back_returnGood_70f75407b7ae_3180_17788571404.mp4 +6902022135514_20240411-142830_front_returnGood_70f75407b7ae_3180_17788571404.mp4 +6902022135514_20240411-142840_back_returnGood_70f754088050_3185_17327712807.mp4 +6902022135514_20240411-142840_front_returnGood_70f754088050_3185_17327712807.mp4 +6902265114369_20240411-142331_back_addGood_70f75407b7ae_715_17788571404.mp4 +6902265114369_20240411-142331_front_addGood_70f75407b7ae_715_17788571404.mp4 +6902265114369_20240411-142338_back_returnGood_70f75407b7ae_720_17788571404.mp4 +6902265114369_20240411-142338_front_returnGood_70f75407b7ae_720_17788571404.mp4 +6902265114369_20240411-142355_back_addGood_70f754088050_720_17327712807.mp4 +6902265114369_20240411-142355_front_addGood_70f754088050_720_17327712807.mp4 +6902265114369_20240411-142403_back_returnGood_70f754088050_715_17327712807.mp4 +6902265114369_20240411-142403_front_returnGood_70f754088050_715_17327712807.mp4 +6902265908012_20240411-142446_back_addGood_70f75407b7ae_1150_17788571404.mp4 +6902265908012_20240411-142446_front_addGood_70f75407b7ae_1150_17788571404.mp4 +6902265908012_20240411-142447_back_addGood_70f754088050_1150_17327712807.mp4 +6902265908012_20240411-142447_front_addGood_70f754088050_1150_17327712807.mp4 +6902265908012_20240411-142456_back_returnGood_70f75407b7ae_1150_17788571404.mp4 +6902265908012_20240411-142456_front_returnGood_70f75407b7ae_1150_17788571404.mp4 +6902265908012_20240411-142459_back_returnGood_70f754088050_1150_17327712807.mp4 +6902265908012_20240411-142459_front_returnGood_70f754088050_1150_17327712807.mp4 +69025143_20240411-163325_back_addGood_70f75407b7ae_3385_17788571404.mp4 +69025143_20240411-163325_front_addGood_70f75407b7ae_3385_17788571404.mp4 +69025143_20240411-163352_back_returnGood_70f75407b7ae_3390_17788571404.mp4 +69025143_20240411-163352_front_returnGood_70f75407b7ae_3390_17788571404.mp4 +69025143_20240411-163417_back_addGood_70f754088050_3380_17327712807.mp4 +69025143_20240411-163417_front_addGood_70f754088050_3380_17327712807.mp4 +69025143_20240411-163428_back_returnGood_70f754088050_3380_17327712807.mp4 +69025143_20240411-163428_front_returnGood_70f754088050_3380_17327712807.mp4 +6902538007367_20240411-144030_back_addGood_70f754088050_660_17327712807.mp4 +6902538007367_20240411-144030_front_addGood_70f754088050_660_17327712807.mp4 +6902538007367_20240411-144031_back_addGood_70f75407b7ae_660_17788571404.mp4 +6902538007367_20240411-144031_front_addGood_70f75407b7ae_660_17788571404.mp4 +6902538007367_20240411-144037_back_returnGood_70f75407b7ae_660_17788571404.mp4 +6902538007367_20240411-144037_front_returnGood_70f75407b7ae_660_17788571404.mp4 +6902538007367_20240411-144040_back_returnGood_70f754088050_660_17327712807.mp4 +6902538007367_20240411-144040_front_returnGood_70f754088050_660_17327712807.mp4 +69028571_20240411-163728_back_addGood_70f75407b7ae_1845_17788571404.mp4 +69028571_20240411-163728_front_addGood_70f75407b7ae_1845_17788571404.mp4 +69028571_20240411-163739_back_returnGood_70f75407b7ae_1845_17788571404.mp4 +69028571_20240411-163739_front_returnGood_70f75407b7ae_1845_17788571404.mp4 +69028571_20240411-163853_back_addGood_70f754088050_1840_17327712807.mp4 +69028571_20240411-163853_front_addGood_70f754088050_1840_17327712807.mp4 +69028571_20240411-163904_back_returnGood_70f754088050_1840_17327712807.mp4 +69028571_20240411-163904_front_returnGood_70f754088050_1840_17327712807.mp4 +6907992103952_20240411-142013_back_addGood_70f75407b7ae_190_17788571404.mp4 +6907992103952_20240411-142013_front_addGood_70f75407b7ae_190_17788571404.mp4 +6907992103952_20240411-142021_back_returnGood_70f75407b7ae_190_17788571404.mp4 +6907992103952_20240411-142021_front_returnGood_70f75407b7ae_190_17788571404.mp4 +6907992103952_20240411-142029_back_addGood_70f754088050_190_17327712807.mp4 +6907992103952_20240411-142029_front_addGood_70f754088050_190_17327712807.mp4 +6907992103952_20240411-142036_back_returnGood_70f754088050_190_17327712807.mp4 +6907992103952_20240411-142036_front_returnGood_70f754088050_190_17327712807.mp4 +6907992104157_20240411-141633_back_addGood_70f754088050_1120_17327712807.mp4 +6907992104157_20240411-141633_front_addGood_70f754088050_1120_17327712807.mp4 +6907992104157_20240411-141637_back_addGood_70f75407b7ae_1120_17788571404.mp4 +6907992104157_20240411-141637_front_addGood_70f75407b7ae_1120_17788571404.mp4 +6907992104157_20240411-141646_back_returnGood_70f75407b7ae_1120_17788571404.mp4 +6907992104157_20240411-141646_back_returnGood_70f754088050_1120_17327712807.mp4 +6907992104157_20240411-141646_front_returnGood_70f75407b7ae_1120_17788571404.mp4 +6907992104157_20240411-141646_front_returnGood_70f754088050_1120_17327712807.mp4 +6907992105765_20240411-142214_back_addGood_70f754088050_1860_17327712807.mp4 +6907992105765_20240411-142214_front_addGood_70f754088050_1860_17327712807.mp4 +6907992105765_20240411-142220_back_addGood_70f75407b7ae_1855_17788571404.mp4 +6907992105765_20240411-142220_front_addGood_70f75407b7ae_1855_17788571404.mp4 +6907992105765_20240411-142228_back_returnGood_70f754088050_1860_17327712807.mp4 +6907992105765_20240411-142228_front_returnGood_70f754088050_1860_17327712807.mp4 +6907992105765_20240411-142230_back_returnGood_70f75407b7ae_1855_17788571404.mp4 +6907992105765_20240411-142230_front_returnGood_70f75407b7ae_1855_17788571404.mp4 +6907992106113_20240411-142100_back_addGood_70f754088050_3085_17327712807.mp4 +6907992106113_20240411-142100_front_addGood_70f754088050_3085_17327712807.mp4 +6907992106113_20240411-142133_back_returnGood_70f754088050_3085_17327712807.mp4 +6907992106113_20240411-142133_front_returnGood_70f754088050_3085_17327712807.mp4 +6907992106113_20240411-142149_back_addGood_70f75407b7ae_3085_17788571404.mp4 +6907992106113_20240411-142149_front_addGood_70f75407b7ae_3085_17788571404.mp4 +6907992106113_20240411-142157_back_returnGood_70f75407b7ae_3080_17788571404.mp4 +6907992106113_20240411-142157_front_returnGood_70f75407b7ae_3080_17788571404.mp4 +6907992106205_20240411-141741_back_addGood_70f75407b7ae_795_17788571404.mp4 +6907992106205_20240411-141741_front_addGood_70f75407b7ae_795_17788571404.mp4 +6907992106205_20240411-141750_back_returnGood_70f75407b7ae_795_17788571404.mp4 +6907992106205_20240411-141750_front_returnGood_70f75407b7ae_795_17788571404.mp4 +6907992106205_20240411-141806_back_addGood_70f754088050_795_17327712807.mp4 +6907992106205_20240411-141806_front_addGood_70f754088050_795_17327712807.mp4 +6907992106205_20240411-141815_back_returnGood_70f754088050_795_17327712807.mp4 +6907992106205_20240411-141815_front_returnGood_70f754088050_795_17327712807.mp4 +6907992106311_20240411-141711_back_addGood_70f754088050_880_17327712807.mp4 +6907992106311_20240411-141711_front_addGood_70f754088050_880_17327712807.mp4 +6907992106311_20240411-141713_back_addGood_70f75407b7ae_885_17788571404.mp4 +6907992106311_20240411-141713_front_addGood_70f75407b7ae_885_17788571404.mp4 +6907992106311_20240411-141721_back_returnGood_70f75407b7ae_885_17788571404.mp4 +6907992106311_20240411-141721_front_returnGood_70f75407b7ae_885_17788571404.mp4 +6907992106311_20240411-141726_back_returnGood_70f754088050_880_17327712807.mp4 +6907992106311_20240411-141726_front_returnGood_70f754088050_880_17327712807.mp4 +6914973602908_20240411-162105_back_returnGood_70f75407b7ae_5_17788571404.mp4 +6914973602908_20240411-162105_front_returnGood_70f75407b7ae_5_17788571404.mp4 +6914973602908_20240411-162113_back_returnGood_70f75407b7ae_740_17788571404.mp4 +6914973602908_20240411-162113_front_returnGood_70f75407b7ae_740_17788571404.mp4 +6914973602908_20240411-162420_back_addGood_70f754088050_740_17327712807.mp4 +6914973602908_20240411-162420_front_addGood_70f754088050_740_17327712807.mp4 +6914973602908_20240411-162434_back_returnGood_70f754088050_740_17327712807.mp4 +6914973602908_20240411-162434_front_returnGood_70f754088050_740_17327712807.mp4 +6914973606340_20240411-161927_back_addGood_70f75407b7ae_350_17788571404.mp4 +6914973606340_20240411-161927_front_addGood_70f75407b7ae_350_17788571404.mp4 +6914973606340_20240411-161936_back_returnGood_70f75407b7ae_355_17788571404.mp4 +6914973606340_20240411-161936_front_returnGood_70f75407b7ae_355_17788571404.mp4 +6914973606340_20240411-162001_back_addGood_70f754088050_5_17327712807.mp4 +6914973606340_20240411-162001_front_addGood_70f754088050_5_17327712807.mp4 +6914973606340_20240411-162009_back_returnGood_70f754088050_355_17327712807.mp4 +6914973606340_20240411-162009_front_returnGood_70f754088050_355_17327712807.mp4 +6920152400630_20240411-144222_back_addGood_70f75407b7ae_580_17788571404.mp4 +6920152400630_20240411-144222_front_addGood_70f75407b7ae_580_17788571404.mp4 +6920152400630_20240411-144223_back_addGood_70f754088050_570_17327712807.mp4 +6920152400630_20240411-144223_front_addGood_70f754088050_570_17327712807.mp4 +6920152400630_20240411-144228_back_returnGood_70f75407b7ae_575_17788571404.mp4 +6920152400630_20240411-144228_front_returnGood_70f75407b7ae_575_17788571404.mp4 +6920152400630_20240411-144236_back_returnGood_70f754088050_570_17327712807.mp4 +6920152400630_20240411-144236_front_returnGood_70f754088050_570_17327712807.mp4 +6920174757101_20240411-143201_back_addGood_70f75407b7ae_1305_17788571404.mp4 +6920174757101_20240411-143201_front_addGood_70f75407b7ae_1305_17788571404.mp4 +6920174757101_20240411-143202_back_addGood_70f754088050_1305_17327712807.mp4 +6920174757101_20240411-143202_front_addGood_70f754088050_1305_17327712807.mp4 +6920174757101_20240411-143208_back_returnGood_70f75407b7ae_1305_17788571404.mp4 +6920174757101_20240411-143208_front_returnGood_70f75407b7ae_1305_17788571404.mp4 +6920174757101_20240411-143211_back_returnGood_70f754088050_1305_17327712807.mp4 +6920174757101_20240411-143211_front_returnGood_70f754088050_1305_17327712807.mp4 +6920459905012_20240411-143922_back_addGood_70f75407b7ae_550_17788571404.mp4 +6920459905012_20240411-143922_front_addGood_70f75407b7ae_550_17788571404.mp4 +6920459905012_20240411-143930_back_returnGood_70f75407b7ae_555_17788571404.mp4 +6920459905012_20240411-143930_front_returnGood_70f75407b7ae_555_17788571404.mp4 +6920459905012_20240411-143947_back_addGood_70f754088050_550_17327712807.mp4 +6920459905012_20240411-143947_front_addGood_70f754088050_550_17327712807.mp4 +6920459905012_20240411-143956_back_returnGood_70f754088050_550_17327712807.mp4 +6920459905012_20240411-143956_front_returnGood_70f754088050_550_17327712807.mp4 +6920907810707_20240411-143338_back_addGood_70f754088050_75_17327712807.mp4 +6920907810707_20240411-143338_front_addGood_70f754088050_75_17327712807.mp4 +6920907810707_20240411-143339_back_addGood_70f75407b7ae_80_17788571404.mp4 +6920907810707_20240411-143339_front_addGood_70f75407b7ae_80_17788571404.mp4 +6920907810707_20240411-143347_back_returnGood_70f754088050_80_17327712807.mp4 +6920907810707_20240411-143347_front_returnGood_70f754088050_80_17327712807.mp4 +6920907810707_20240411-143356_back_returnGood_70f75407b7ae_80_17788571404.mp4 +6920907810707_20240411-143356_front_returnGood_70f75407b7ae_80_17788571404.mp4 +6922130119213_20240411-142631_back_addGood_70f754088050_1020_17327712807.mp4 +6922130119213_20240411-142631_front_addGood_70f754088050_1020_17327712807.mp4 +6922130119213_20240411-142636_back_addGood_70f75407b7ae_1020_17788571404.mp4 +6922130119213_20240411-142636_front_addGood_70f75407b7ae_1020_17788571404.mp4 +6922130119213_20240411-142641_back_returnGood_70f754088050_1020_17327712807.mp4 +6922130119213_20240411-142641_front_returnGood_70f754088050_1020_17327712807.mp4 +6922130119213_20240411-142647_back_returnGood_70f75407b7ae_1020_17788571404.mp4 +6922130119213_20240411-142647_front_returnGood_70f75407b7ae_1020_17788571404.mp4 +6922577700968_20240411-141822_back_addGood_70f75407b7ae_1040_17788571404.mp4 +6922577700968_20240411-141822_front_addGood_70f75407b7ae_1040_17788571404.mp4 +6922577700968_20240411-141834_back_addGood_70f754088050_1045_17327712807.mp4 +6922577700968_20240411-141834_front_addGood_70f754088050_1045_17327712807.mp4 +6922577700968_20240411-141839_back_returnGood_70f75407b7ae_1045_17788571404.mp4 +6922577700968_20240411-141839_front_returnGood_70f75407b7ae_1045_17788571404.mp4 +6922577700968_20240411-141844_back_returnGood_70f754088050_1045_17327712807.mp4 +6922577700968_20240411-141844_front_returnGood_70f754088050_1045_17327712807.mp4 +6922868291168_20240411-142913_back_addGood_70f754088050_1160_17327712807.mp4 +6922868291168_20240411-142913_front_addGood_70f754088050_1160_17327712807.mp4 +6922868291168_20240411-142921_back_addGood_70f75407b7ae_1150_17788571404.mp4 +6922868291168_20240411-142921_front_addGood_70f75407b7ae_1150_17788571404.mp4 +6922868291168_20240411-142929_back_returnGood_70f754088050_1160_17327712807.mp4 +6922868291168_20240411-142929_front_returnGood_70f754088050_1160_17327712807.mp4 +6922868291168_20240411-142933_back_returnGood_70f75407b7ae_1155_17788571404.mp4 +6922868291168_20240411-142933_front_returnGood_70f75407b7ae_1155_17788571404.mp4 +6923450601549_20240411-162014_back_addGood_70f75407b7ae_600_17788571404.mp4 +6923450601549_20240411-162014_front_addGood_70f75407b7ae_600_17788571404.mp4 +6923450601549_20240411-162024_back_returnGood_70f75407b7ae_600_17788571404.mp4 +6923450601549_20240411-162024_front_returnGood_70f75407b7ae_600_17788571404.mp4 +6923450601549_20240411-162216_back_addGood_70f754088050_595_17327712807.mp4 +6923450601549_20240411-162216_front_addGood_70f754088050_595_17327712807.mp4 +6923450601549_20240411-162227_back_returnGood_70f754088050_595_17327712807.mp4 +6923450601549_20240411-162227_front_returnGood_70f754088050_595_17327712807.mp4 +6923450603574_20240411-163042_back_addGood_70f75407b7ae_870_17788571404.mp4 +6923450603574_20240411-163042_front_addGood_70f75407b7ae_870_17788571404.mp4 +6923450603574_20240411-163049_back_returnGood_70f75407b7ae_870_17788571404.mp4 +6923450603574_20240411-163049_front_returnGood_70f75407b7ae_870_17788571404.mp4 +6923450603574_20240411-163104_back_addGood_70f754088050_865_17327712807.mp4 +6923450603574_20240411-163104_front_addGood_70f754088050_865_17327712807.mp4 +6923450603574_20240411-163114_back_returnGood_70f754088050_865_17327712807.mp4 +6923450603574_20240411-163114_front_returnGood_70f754088050_865_17327712807.mp4 +6923450605288_20240411-161704_back_addGood_70f75407b7ae_250_17788571404.mp4 +6923450605288_20240411-161704_front_addGood_70f75407b7ae_250_17788571404.mp4 +6923450605288_20240411-161715_back_returnGood_70f75407b7ae_250_17788571404.mp4 +6923450605288_20240411-161715_front_returnGood_70f75407b7ae_250_17788571404.mp4 +6923450605288_20240411-161748_back_addGood_70f754088050_245_17327712807.mp4 +6923450605288_20240411-161748_front_addGood_70f754088050_245_17327712807.mp4 +6923450605288_20240411-161800_back_returnGood_70f754088050_245_17327712807.mp4 +6923450605288_20240411-161800_front_returnGood_70f754088050_245_17327712807.mp4 +6923450610428_20240411-161209_back_addGood_70f75407b7ae_275_17788571404.mp4 +6923450610428_20240411-161209_front_addGood_70f75407b7ae_275_17788571404.mp4 +6923450610428_20240411-161217_back_returnGood_70f75407b7ae_275_17788571404.mp4 +6923450610428_20240411-161217_front_returnGood_70f75407b7ae_275_17788571404.mp4 +6923450610428_20240411-161235_back_addGood_70f754088050_270_17327712807.mp4 +6923450610428_20240411-161235_front_addGood_70f754088050_270_17327712807.mp4 +6923450610428_20240411-161249_back_returnGood_70f754088050_275_17327712807.mp4 +6923450610428_20240411-161249_front_returnGood_70f754088050_275_17327712807.mp4 +6923450610459_20240411-162814_back_addGood_70f75407b7ae_445_17788571404.mp4 +6923450610459_20240411-162814_front_addGood_70f75407b7ae_445_17788571404.mp4 +6923450610459_20240411-162822_back_returnGood_70f75407b7ae_450_17788571404.mp4 +6923450610459_20240411-162822_front_returnGood_70f75407b7ae_450_17788571404.mp4 +6923450610459_20240411-162854_back_addGood_70f754088050_445_17327712807.mp4 +6923450610459_20240411-162854_front_addGood_70f754088050_445_17327712807.mp4 +6923450610459_20240411-162906_back_returnGood_70f754088050_445_17327712807.mp4 +6923450610459_20240411-162906_front_returnGood_70f754088050_445_17327712807.mp4 +6923450611067_20240411-162639_back_addGood_70f75407b7ae_625_17788571404.mp4 +6923450611067_20240411-162639_front_addGood_70f75407b7ae_625_17788571404.mp4 +6923450611067_20240411-162648_back_returnGood_70f75407b7ae_625_17788571404.mp4 +6923450611067_20240411-162648_front_returnGood_70f75407b7ae_625_17788571404.mp4 +6923450611067_20240411-162752_back_addGood_70f754088050_620_17327712807.mp4 +6923450611067_20240411-162752_front_addGood_70f754088050_620_17327712807.mp4 +6923450611067_20240411-162808_back_returnGood_70f754088050_625_17327712807.mp4 +6923450611067_20240411-162808_front_returnGood_70f754088050_625_17327712807.mp4 +6923450612415_20240411-160636_back_addGood_70f75407b7ae_870_17788571404.mp4 +6923450612415_20240411-160636_front_addGood_70f75407b7ae_870_17788571404.mp4 +6923450612415_20240411-160724_back_addGood_70f754088050_865_17327712807.mp4 +6923450612415_20240411-160724_front_addGood_70f754088050_865_17327712807.mp4 +6923450612415_20240411-160735_back_returnGood_70f754088050_865_17327712807.mp4 +6923450612415_20240411-160735_front_returnGood_70f754088050_865_17327712807.mp4 +6923450612415_20240411-161013_back_returnGood_70f75407b7ae_865_17788571404.mp4 +6923450612415_20240411-161013_front_returnGood_70f75407b7ae_865_17788571404.mp4 +6923450612484_20240411-161509_back_addGood_70f75407b7ae_445_17788571404.mp4 +6923450612484_20240411-161509_front_addGood_70f75407b7ae_445_17788571404.mp4 +6923450612484_20240411-161517_back_returnGood_70f75407b7ae_445_17788571404.mp4 +6923450612484_20240411-161517_front_returnGood_70f75407b7ae_445_17788571404.mp4 +6923450612484_20240411-161535_back_addGood_70f754088050_450_17327712807.mp4 +6923450612484_20240411-161535_front_addGood_70f754088050_450_17327712807.mp4 +6923450612484_20240411-161546_back_returnGood_70f754088050_450_17327712807.mp4 +6923450612484_20240411-161546_front_returnGood_70f754088050_450_17327712807.mp4 +6923450657829_20240411-163522_back_addGood_70f75407b7ae_985_17788571404.mp4 +6923450657829_20240411-163522_front_addGood_70f75407b7ae_985_17788571404.mp4 +6923450657829_20240411-163532_back_returnGood_70f75407b7ae_985_17788571404.mp4 +6923450657829_20240411-163532_front_returnGood_70f75407b7ae_985_17788571404.mp4 +6923450657829_20240411-163554_back_addGood_70f754088050_990_17327712807.mp4 +6923450657829_20240411-163554_front_addGood_70f754088050_990_17327712807.mp4 +6923450657829_20240411-163606_back_returnGood_70f754088050_990_17327712807.mp4 +6923450657829_20240411-163606_front_returnGood_70f754088050_990_17327712807.mp4 +6923450659441_20240411-163152_back_addGood_70f75407b7ae_1695_17788571404.mp4 +6923450659441_20240411-163152_front_addGood_70f75407b7ae_1695_17788571404.mp4 +6923450659441_20240411-163204_back_returnGood_70f75407b7ae_1695_17788571404.mp4 +6923450659441_20240411-163204_front_returnGood_70f75407b7ae_1695_17788571404.mp4 +6923450659441_20240411-163217_back_addGood_70f754088050_1690_17327712807.mp4 +6923450659441_20240411-163217_front_addGood_70f754088050_1690_17327712807.mp4 +6923450659441_20240411-163231_back_returnGood_70f754088050_1690_17327712807.mp4 +6923450659441_20240411-163231_front_returnGood_70f754088050_1690_17327712807.mp4 +6923450666838_20240411-162541_back_addGood_70f75407b7ae_285_17788571404.mp4 +6923450666838_20240411-162541_front_addGood_70f75407b7ae_285_17788571404.mp4 +6923450666838_20240411-162549_back_returnGood_70f75407b7ae_285_17788571404.mp4 +6923450666838_20240411-162549_front_returnGood_70f75407b7ae_285_17788571404.mp4 +6923450666838_20240411-162637_back_addGood_70f754088050_285_17327712807.mp4 +6923450666838_20240411-162637_front_addGood_70f754088050_285_17327712807.mp4 +6923450666838_20240411-162704_back_returnGood_70f754088050_285_17327712807.mp4 +6923450666838_20240411-162704_front_returnGood_70f754088050_285_17327712807.mp4 +6923450668207_20240411-161832_back_addGood_70f75407b7ae_350_17788571404.mp4 +6923450668207_20240411-161832_front_addGood_70f75407b7ae_350_17788571404.mp4 +6923450668207_20240411-161839_back_returnGood_70f75407b7ae_350_17788571404.mp4 +6923450668207_20240411-161839_front_returnGood_70f75407b7ae_350_17788571404.mp4 +6923450668207_20240411-161904_back_addGood_70f754088050_350_17327712807.mp4 +6923450668207_20240411-161904_front_addGood_70f754088050_350_17327712807.mp4 +6923450668207_20240411-161913_back_returnGood_70f754088050_355_17327712807.mp4 +6923450668207_20240411-161913_front_returnGood_70f754088050_355_17327712807.mp4 +6923450677858_20240411-163609_back_addGood_70f75407b7ae_5_17788571404.mp4 +6923450677858_20240411-163609_front_addGood_70f75407b7ae_5_17788571404.mp4 +6923450677858_20240411-163618_back_returnGood_70f75407b7ae_1020_17788571404.mp4 +6923450677858_20240411-163618_front_returnGood_70f75407b7ae_1020_17788571404.mp4 +6923450677858_20240411-163726_back_addGood_70f754088050_1025_17327712807.mp4 +6923450677858_20240411-163726_front_addGood_70f754088050_1025_17327712807.mp4 +6923450677858_20240411-163737_back_returnGood_70f754088050_1025_17327712807.mp4 +6923450677858_20240411-163737_front_returnGood_70f754088050_1025_17327712807.mp4 +6923644286293_20240411-141557_back_addGood_70f754088050_795_17327712807.mp4 +6923644286293_20240411-141557_front_addGood_70f754088050_795_17327712807.mp4 +6923644286293_20240411-141601_back_addGood_70f75407b7ae_800_17788571404.mp4 +6923644286293_20240411-141601_front_addGood_70f75407b7ae_800_17788571404.mp4 +6923644286293_20240411-141605_back_returnGood_70f754088050_795_17327712807.mp4 +6923644286293_20240411-141605_front_returnGood_70f754088050_795_17327712807.mp4 +6923644286293_20240411-141609_back_returnGood_70f75407b7ae_800_17788571404.mp4 +6923644286293_20240411-141609_front_returnGood_70f75407b7ae_800_17788571404.mp4 +6923644298760_20240411-141459_back_addGood_70f75407b7ae_1020_17788571404.mp4 +6923644298760_20240411-141459_front_addGood_70f75407b7ae_1020_17788571404.mp4 +6923644298760_20240411-141500_back_addGood_70f754088050_1020_17327712807.mp4 +6923644298760_20240411-141500_front_addGood_70f754088050_1020_17327712807.mp4 +6923644298760_20240411-141511_back_returnGood_70f754088050_1020_17327712807.mp4 +6923644298760_20240411-141511_front_returnGood_70f754088050_1020_17327712807.mp4 +6923644298760_20240411-141516_back_returnGood_70f75407b7ae_1020_17788571404.mp4 +6923644298760_20240411-141516_front_returnGood_70f75407b7ae_1020_17788571404.mp4 +6924743915824_20240411-143302_back_addGood_70f75407b7ae_150_17788571404.mp4 +6924743915824_20240411-143302_back_addGood_70f754088050_150_17327712807.mp4 +6924743915824_20240411-143302_front_addGood_70f75407b7ae_150_17788571404.mp4 +6924743915824_20240411-143302_front_addGood_70f754088050_150_17327712807.mp4 +6924743915824_20240411-143310_back_returnGood_70f75407b7ae_155_17788571404.mp4 +6924743915824_20240411-143310_back_returnGood_70f754088050_150_17327712807.mp4 +6924743915824_20240411-143310_front_returnGood_70f75407b7ae_155_17788571404.mp4 +6924743915824_20240411-143310_front_returnGood_70f754088050_150_17327712807.mp4 +6924882497106_20240411-143858_back_addGood_70f75407b7ae_360_17788571404.mp4 +6924882497106_20240411-143858_front_addGood_70f75407b7ae_360_17788571404.mp4 +6924882497106_20240411-143904_back_returnGood_70f75407b7ae_360_17788571404.mp4 +6924882497106_20240411-143904_front_returnGood_70f75407b7ae_360_17788571404.mp4 +6924882497106_20240411-143915_back_addGood_70f754088050_355_17327712807.mp4 +6924882497106_20240411-143915_front_addGood_70f754088050_355_17327712807.mp4 +6924882497106_20240411-143924_back_returnGood_70f754088050_355_17327712807.mp4 +6924882497106_20240411-143924_front_returnGood_70f754088050_355_17327712807.mp4 +6925307305525_20240411-141034_back_addGood_70f75407b7ae_1670_17788571404.mp4 +6925307305525_20240411-141034_front_addGood_70f75407b7ae_1670_17788571404.mp4 +6925307305525_20240411-141044_back_returnGood_70f75407b7ae_1670_17788571404.mp4 +6925307305525_20240411-141044_front_returnGood_70f75407b7ae_1670_17788571404.mp4 +6925307305525_20240411-141055_back_addGood_70f754088050_1670_17327712807.mp4 +6925307305525_20240411-141055_front_addGood_70f754088050_1670_17327712807.mp4 +6925307305525_20240411-141106_back_returnGood_70f754088050_1670_17327712807.mp4 +6925307305525_20240411-141106_front_returnGood_70f754088050_1670_17327712807.mp4 +6928804011173_20240411-143817_back_addGood_70f75407b7ae_555_17788571404.mp4 +6928804011173_20240411-143817_front_addGood_70f75407b7ae_555_17788571404.mp4 +6928804011173_20240411-143825_back_returnGood_70f75407b7ae_550_17788571404.mp4 +6928804011173_20240411-143825_front_returnGood_70f75407b7ae_550_17788571404.mp4 +6928804011173_20240411-143833_back_addGood_70f754088050_545_17327712807.mp4 +6928804011173_20240411-143833_front_addGood_70f754088050_545_17327712807.mp4 +6928804011173_20240411-143841_back_returnGood_70f754088050_550_17327712807.mp4 +6928804011173_20240411-143841_front_returnGood_70f754088050_550_17327712807.mp4 +6931925828032_20240411-162501_back_addGood_70f75407b7ae_405_17788571404.mp4 +6931925828032_20240411-162501_front_addGood_70f75407b7ae_405_17788571404.mp4 +6931925828032_20240411-162511_back_returnGood_70f75407b7ae_405_17788571404.mp4 +6931925828032_20240411-162511_front_returnGood_70f75407b7ae_405_17788571404.mp4 +6931925828032_20240411-162532_back_addGood_70f754088050_405_17327712807.mp4 +6931925828032_20240411-162532_front_addGood_70f754088050_405_17327712807.mp4 +6931925828032_20240411-162540_back_returnGood_70f754088050_405_17327712807.mp4 +6931925828032_20240411-162540_front_returnGood_70f754088050_405_17327712807.mp4 +6933620900051_20240411-150018_back_addGood_70f75407b7ae_5_17788571404.mp4 +6933620900051_20240411-150018_front_addGood_70f75407b7ae_5_17788571404.mp4 +6933620900051_20240411-150035_back_returnGood_70f75407b7ae_380_17788571404.mp4 +6933620900051_20240411-150035_front_returnGood_70f75407b7ae_380_17788571404.mp4 +6933620900051_20240411-150103_back_addGood_70f754088050_455_17327712807.mp4 +6933620900051_20240411-150103_front_addGood_70f754088050_455_17327712807.mp4 +6933620900051_20240411-150116_back_returnGood_70f754088050_455_17327712807.mp4 +6933620900051_20240411-150116_front_returnGood_70f754088050_455_17327712807.mp4 +6934665095108_20240411-141931_back_addGood_70f754088050_360_17327712807.mp4 +6934665095108_20240411-141931_front_addGood_70f754088050_360_17327712807.mp4 +6934665095108_20240411-141936_back_addGood_70f75407b7ae_360_17788571404.mp4 +6934665095108_20240411-141936_front_addGood_70f75407b7ae_360_17788571404.mp4 +6934665095108_20240411-141940_back_returnGood_70f754088050_365_17327712807.mp4 +6934665095108_20240411-141940_front_returnGood_70f754088050_365_17327712807.mp4 +6934665095108_20240411-141945_back_returnGood_70f75407b7ae_360_17788571404.mp4 +6934665095108_20240411-141945_front_returnGood_70f75407b7ae_360_17788571404.mp4 +6935270642121_20240411-144253_back_addGood_70f75407b7ae_155_17788571404.mp4 +6935270642121_20240411-144253_front_addGood_70f75407b7ae_155_17788571404.mp4 +6935270642121_20240411-144255_back_addGood_70f754088050_155_17327712807.mp4 +6935270642121_20240411-144255_front_addGood_70f754088050_155_17327712807.mp4 +6935270642121_20240411-144300_back_returnGood_70f75407b7ae_155_17788571404.mp4 +6935270642121_20240411-144300_front_returnGood_70f75407b7ae_155_17788571404.mp4 +6935270642121_20240411-144304_back_returnGood_70f754088050_155_17327712807.mp4 +6935270642121_20240411-144304_front_returnGood_70f754088050_155_17327712807.mp4 +6935284417326_20240411-143504_back_addGood_70f754088050_420_17327712807.mp4 +6935284417326_20240411-143504_front_addGood_70f754088050_420_17327712807.mp4 +6935284417326_20240411-143508_back_addGood_70f75407b7ae_415_17788571404.mp4 +6935284417326_20240411-143508_front_addGood_70f75407b7ae_415_17788571404.mp4 +6935284417326_20240411-143517_back_returnGood_70f754088050_420_17327712807.mp4 +6935284417326_20240411-143517_front_returnGood_70f754088050_420_17327712807.mp4 +6935284417326_20240411-143521_back_returnGood_70f75407b7ae_415_17788571404.mp4 +6935284417326_20240411-143521_front_returnGood_70f75407b7ae_415_17788571404.mp4 +6941025140798_20240411-135647_back_addGood_70f75407b7ae_1760_17788571404.mp4 +6941025140798_20240411-135647_front_addGood_70f75407b7ae_1760_17788571404.mp4 +6941025140798_20240411-135655_back_returnGood_70f75407b7ae_1760_17788571404.mp4 +6941025140798_20240411-135655_front_returnGood_70f75407b7ae_1760_17788571404.mp4 +6941025140798_20240411-140026_back_addGood_70f754088050_1765_17327712807.mp4 +6941025140798_20240411-140026_front_addGood_70f754088050_1765_17327712807.mp4 +6941025140798_20240411-140105_back_returnGood_70f754088050_1760_17327712807.mp4 +6941025140798_20240411-140105_front_returnGood_70f754088050_1760_17327712807.mp4 +6952074634794_20240411-143701_back_addGood_70f75407b7ae_275_17788571404.mp4 +6952074634794_20240411-143701_front_addGood_70f75407b7ae_275_17788571404.mp4 +6952074634794_20240411-143707_back_returnGood_70f75407b7ae_275_17788571404.mp4 +6952074634794_20240411-143709_back_returnGood_70f754088050_5_17327712807.mp4 +6952074634794_20240411-143709_front_returnGood_70f754088050_5_17327712807.mp4 +6952074634794_20240411-143720_back_returnGood_70f754088050_265_17327712807.mp4 +6952074634794_20240411-143720_front_returnGood_70f754088050_265_17327712807.mp4 +6954432711307_20240411-161612_back_addGood_70f75407b7ae_460_17788571404.mp4 +6954432711307_20240411-161612_front_addGood_70f75407b7ae_460_17788571404.mp4 +6954432711307_20240411-161622_back_returnGood_70f75407b7ae_460_17788571404.mp4 +6954432711307_20240411-161622_front_returnGood_70f75407b7ae_460_17788571404.mp4 +6954432711307_20240411-161647_back_addGood_70f754088050_460_17327712807.mp4 +6954432711307_20240411-161647_front_addGood_70f754088050_460_17327712807.mp4 +6954432711307_20240411-161700_back_returnGood_70f754088050_460_17327712807.mp4 +6954432711307_20240411-161700_front_returnGood_70f754088050_460_17327712807.mp4 +6959546100993_20240411-135257_back_addGood_70f75407b7ae_295_17788571404.mp4 +6959546100993_20240411-135257_front_addGood_70f75407b7ae_295_17788571404.mp4 +6959546100993_20240411-135339_back_returnGood_70f75407b7ae_295_17788571404.mp4 +6959546100993_20240411-135339_front_returnGood_70f75407b7ae_295_17788571404.mp4 +6959546100993_20240411-141255_back_addGood_70f754088050_295_17327712807.mp4 +6959546100993_20240411-141255_front_addGood_70f754088050_295_17327712807.mp4 +6959546100993_20240411-141309_back_returnGood_70f754088050_295_17327712807.mp4 +6959546100993_20240411-141309_front_returnGood_70f754088050_295_17327712807.mp4 +6971075127463_20240411-135002_back_addGood_70f75407b7ae_210_17788571404.mp4 +6971075127463_20240411-135002_front_addGood_70f75407b7ae_210_17788571404.mp4 +6971075127463_20240411-135058_back_returnGood_70f75407b7ae_210_17788571404.mp4 +6971075127463_20240411-135058_front_returnGood_70f75407b7ae_210_17788571404.mp4 +6971075127463_20240411-141154_back_addGood_70f754088050_215_17327712807.mp4 +6971075127463_20240411-141154_front_addGood_70f754088050_215_17327712807.mp4 +6971075127463_20240411-141214_back_returnGood_70f754088050_210_17327712807.mp4 +6971075127463_20240411-141214_front_returnGood_70f754088050_210_17327712807.mp4 +6971328580533_20240411-135748_back_addGood_70f75407b7ae_405_17788571404.mp4 +6971328580533_20240411-135748_front_addGood_70f75407b7ae_405_17788571404.mp4 +6971328580533_20240411-135757_back_returnGood_70f75407b7ae_405_17788571404.mp4 +6971328580533_20240411-135757_front_returnGood_70f75407b7ae_405_17788571404.mp4 +6971328580533_20240411-140231_back_addGood_70f754088050_405_17327712807.mp4 +6971328580533_20240411-140231_front_addGood_70f754088050_405_17327712807.mp4 +6971328580533_20240411-140529_back_addGood_70f754088050_410_17327712807.mp4 +6971328580533_20240411-140529_front_addGood_70f754088050_410_17327712807.mp4 +6971328580533_20240411-140745_back_returnGood_70f754088050_405_17327712807.mp4 +6971328580533_20240411-140745_front_returnGood_70f754088050_405_17327712807.mp4 +6971738655333_20240411-144616_back_addGood_70f75407b7ae_270_17788571404.mp4 +6971738655333_20240411-144616_back_addGood_70f754088050_260_17327712807.mp4 +6971738655333_20240411-144616_front_addGood_70f75407b7ae_270_17788571404.mp4 +6971738655333_20240411-144616_front_addGood_70f754088050_260_17327712807.mp4 +6971738655333_20240411-144633_back_returnGood_70f75407b7ae_270_17788571404.mp4 +6971738655333_20240411-144633_front_returnGood_70f75407b7ae_270_17788571404.mp4 +6971738655333_20240411-144635_back_returnGood_70f754088050_260_17327712807.mp4 +6971738655333_20240411-144635_front_returnGood_70f754088050_260_17327712807.mp4 +6972378998200_20240411-142603_back_addGood_70f754088050_410_17327712807.mp4 +6972378998200_20240411-142603_front_addGood_70f754088050_410_17327712807.mp4 +6972378998200_20240411-142604_back_addGood_70f75407b7ae_410_17788571404.mp4 +6972378998200_20240411-142604_front_addGood_70f75407b7ae_410_17788571404.mp4 +6972378998200_20240411-142613_back_returnGood_70f75407b7ae_410_17788571404.mp4 +6972378998200_20240411-142613_back_returnGood_70f754088050_410_17327712807.mp4 +6972378998200_20240411-142613_front_returnGood_70f75407b7ae_410_17788571404.mp4 +6972378998200_20240411-142613_front_returnGood_70f754088050_410_17327712807.mp4 +6972790052733_20240411-162926_back_addGood_70f75407b7ae_690_17788571404.mp4 +6972790052733_20240411-162926_front_addGood_70f75407b7ae_690_17788571404.mp4 +6972790052733_20240411-162933_back_returnGood_70f75407b7ae_690_17788571404.mp4 +6972790052733_20240411-162933_front_returnGood_70f75407b7ae_690_17788571404.mp4 +6972790052733_20240411-162948_back_addGood_70f754088050_690_17327712807.mp4 +6972790052733_20240411-162948_front_addGood_70f754088050_690_17327712807.mp4 +6972790052733_20240411-163001_back_returnGood_70f754088050_690_17327712807.mp4 +6972790052733_20240411-163001_front_returnGood_70f754088050_690_17327712807.mp4 +6974627182033_20240411-150747_back_addGood_70f75407b7ae_485_17788571404.mp4 +6974627182033_20240411-150747_front_addGood_70f75407b7ae_485_17788571404.mp4 +6974627182033_20240411-150756_back_returnGood_70f75407b7ae_480_17788571404.mp4 +6974627182033_20240411-150756_front_returnGood_70f75407b7ae_480_17788571404.mp4 +6974627182033_20240411-150825_back_addGood_70f754088050_495_17327712807.mp4 +6974627182033_20240411-150825_front_addGood_70f754088050_495_17327712807.mp4 +6974627182033_20240411-150835_back_returnGood_70f754088050_495_17327712807.mp4 +6974627182033_20240411-150835_front_returnGood_70f754088050_495_17327712807.mp4 +6974913231612_20240411-135543_back_addGood_70f75407b7ae_870_17788571404.mp4 +6974913231612_20240411-135543_front_addGood_70f75407b7ae_870_17788571404.mp4 +6974913231612_20240411-135552_back_returnGood_70f75407b7ae_865_17788571404.mp4 +6974913231612_20240411-135552_front_returnGood_70f75407b7ae_865_17788571404.mp4 +6974913231612_20240411-140906_back_addGood_70f754088050_755_17327712807.mp4 +6974913231612_20240411-140906_front_addGood_70f754088050_755_17327712807.mp4 +6974913231612_20240411-140917_back_returnGood_70f754088050_755_17327712807.mp4 +6974913231612_20240411-140917_front_returnGood_70f754088050_755_17327712807.mp4 +6974995172711_20240411-135456_back_addGood_70f75407b7ae_1455_17788571404.mp4 +6974995172711_20240411-135456_front_addGood_70f75407b7ae_1455_17788571404.mp4 +6974995172711_20240411-135506_back_returnGood_70f75407b7ae_1460_17788571404.mp4 +6974995172711_20240411-135506_front_returnGood_70f75407b7ae_1460_17788571404.mp4 +6974995172711_20240411-140831_back_addGood_70f754088050_1460_17327712807.mp4 +6974995172711_20240411-140831_front_addGood_70f754088050_1460_17327712807.mp4 +6974995172711_20240411-140844_back_returnGood_70f754088050_1460_17327712807.mp4 +6974995172711_20240411-140844_front_returnGood_70f754088050_1460_17327712807.mp4 +6976090230303_20240411-144457_back_addGood_70f75407b7ae_390_17788571404.mp4 +6976090230303_20240411-144457_front_addGood_70f75407b7ae_390_17788571404.mp4 +6976090230303_20240411-144458_back_addGood_70f754088050_390_17327712807.mp4 +6976090230303_20240411-144458_front_addGood_70f754088050_390_17327712807.mp4 +6976090230303_20240411-144508_back_returnGood_70f754088050_390_17327712807.mp4 +6976090230303_20240411-144508_front_returnGood_70f754088050_390_17327712807.mp4 +6976090230303_20240411-144509_back_returnGood_70f75407b7ae_390_17788571404.mp4 +6976090230303_20240411-144509_front_returnGood_70f75407b7ae_390_17788571404.mp4 +6976371220276_20240411-150456_back_addGood_70f75407b7ae_390_17788571404.mp4 +6976371220276_20240411-150456_front_addGood_70f75407b7ae_390_17788571404.mp4 +6976371220276_20240411-150505_back_returnGood_70f75407b7ae_390_17788571404.mp4 +6976371220276_20240411-150505_front_returnGood_70f75407b7ae_390_17788571404.mp4 +6976371220276_20240411-150559_back_addGood_70f754088050_405_17327712807.mp4 +6976371220276_20240411-150559_front_addGood_70f754088050_405_17327712807.mp4 +6976371220276_20240411-150610_back_returnGood_70f754088050_405_17327712807.mp4 +6976371220276_20240411-150610_front_returnGood_70f754088050_405_17327712807.mp4 +230537101280010007_20240412-140824_back_addGood_70f754088050_565_13725988807.mp4 +230537101280010007_20240412-140824_front_addGood_70f754088050_565_13725988807.mp4 +230537101280010007_20240412-140835_back_returnGood_70f754088050_565_13725988807.mp4 +230537101280010007_20240412-140835_front_returnGood_70f754088050_565_13725988807.mp4 +2500456001326_20240412-140949_back_addGood_70f754088050_150_13725988807.mp4 +2500456001326_20240412-140949_front_addGood_70f754088050_150_13725988807.mp4 +2500456001326_20240412-140958_back_returnGood_70f754088050_155_13725988807.mp4 +2500456001326_20240412-140958_front_returnGood_70f754088050_155_13725988807.mp4 +2500458675341_20240412-140915_back_addGood_70f754088050_130_13725988807.mp4 +2500458675341_20240412-140915_front_addGood_70f754088050_130_13725988807.mp4 +2500458675341_20240412-140924_back_returnGood_70f754088050_130_13725988807.mp4 +2500458675341_20240412-140924_front_returnGood_70f754088050_130_13725988807.mp4 +2500463464671_20240412-140718_back_addGood_70f754088050_810_13725988807.mp4 +2500463464671_20240412-140718_front_addGood_70f754088050_810_13725988807.mp4 +2500463464671_20240412-140731_back_returnGood_70f754088050_810_13725988807.mp4 +2500463464671_20240412-140731_front_returnGood_70f754088050_810_13725988807.mp4 +6901070613142_20240412-144631_back_addGood_70f754088050_240_13725988807.mp4 +6901070613142_20240412-144631_front_addGood_70f754088050_240_13725988807.mp4 +6901070613142_20240412-144643_back_returnGood_70f754088050_240_13725988807.mp4 +6901070613142_20240412-144643_front_returnGood_70f754088050_240_13725988807.mp4 +6901586000993_20240412-152404_back_addGood_70f754088050_890_13725988807.mp4 +6901586000993_20240412-152404_front_addGood_70f754088050_890_13725988807.mp4 +6901586000993_20240412-152440_back_returnGood_70f754088050_885_13725988807.mp4 +6901586000993_20240412-152440_front_returnGood_70f754088050_885_13725988807.mp4 +6901668053893_20240412-150115_back_addGood_70f754088050_70_13725988807.mp4 +6901668053893_20240412-150115_front_addGood_70f754088050_70_13725988807.mp4 +6901668053893_20240412-150125_back_returnGood_70f754088050_70_13725988807.mp4 +6901668053893_20240412-150125_front_returnGood_70f754088050_70_13725988807.mp4 +6902007010249_20240412-144518_back_addGood_70f754088050_755_13725988807.mp4 +6902007010249_20240412-144518_front_addGood_70f754088050_755_13725988807.mp4 +6902007010249_20240412-144529_back_returnGood_70f754088050_755_13725988807.mp4 +6902007010249_20240412-144529_front_returnGood_70f754088050_755_13725988807.mp4 +6902022135514_20240412-144740_back_addGood_70f754088050_3180_13725988807.mp4 +6902022135514_20240412-144740_front_addGood_70f754088050_3180_13725988807.mp4 +6902022135514_20240412-144750_back_returnGood_70f754088050_3180_13725988807.mp4 +6902022135514_20240412-144750_front_returnGood_70f754088050_3180_13725988807.mp4 +6902265114369_20240412-144022_back_addGood_70f754088050_720_13725988807.mp4 +6902265114369_20240412-144022_front_addGood_70f754088050_720_13725988807.mp4 +6902265114369_20240412-144034_back_returnGood_70f754088050_720_13725988807.mp4 +6902265114369_20240412-144034_front_returnGood_70f754088050_720_13725988807.mp4 +6902265908012_20240412-144141_back_addGood_70f754088050_1145_13725988807.mp4 +6902265908012_20240412-144141_front_addGood_70f754088050_1145_13725988807.mp4 +6902265908012_20240412-144149_back_returnGood_70f754088050_1145_13725988807.mp4 +6902265908012_20240412-144149_front_returnGood_70f754088050_1145_13725988807.mp4 +69025143_20240412-135423_back_addGood_70f754088050_3015_13725988807.mp4 +69025143_20240412-135423_front_addGood_70f754088050_3015_13725988807.mp4 +69025143_20240412-135439_back_returnGood_70f754088050_3015_13725988807.mp4 +69025143_20240412-135439_front_returnGood_70f754088050_3015_13725988807.mp4 +6907992103952_20240412-141542_back_addGood_70f754088050_190_13725988807.mp4 +6907992103952_20240412-141542_front_addGood_70f754088050_190_13725988807.mp4 +6907992103952_20240412-141554_back_returnGood_70f754088050_190_13725988807.mp4 +6907992103952_20240412-141554_front_returnGood_70f754088050_190_13725988807.mp4 +6907992104157_20240412-141255_back_addGood_70f754088050_1120_13725988807.mp4 +6907992104157_20240412-141255_front_addGood_70f754088050_1120_13725988807.mp4 +6907992104157_20240412-141307_back_returnGood_70f754088050_1120_13725988807.mp4 +6907992104157_20240412-141307_front_returnGood_70f754088050_1120_13725988807.mp4 +6907992105260_20240412-142144_back_addGood_70f754088050_2775_13725988807.mp4 +6907992105260_20240412-142144_front_addGood_70f754088050_2775_13725988807.mp4 +6907992105260_20240412-142156_back_returnGood_70f754088050_2775_13725988807.mp4 +6907992105260_20240412-142156_front_returnGood_70f754088050_2775_13725988807.mp4 +6907992105260_20240412-143214_back_addGood_70f754088050_2770_13725988807.mp4 +6907992105260_20240412-143214_front_addGood_70f754088050_2770_13725988807.mp4 +6907992105260_20240412-143225_back_returnGood_70f754088050_2775_13725988807.mp4 +6907992105260_20240412-143225_front_returnGood_70f754088050_2775_13725988807.mp4 +6907992105765_20240412-141505_back_addGood_70f754088050_2085_13725988807.mp4 +6907992105765_20240412-141505_front_addGood_70f754088050_2085_13725988807.mp4 +6907992105765_20240412-141523_back_returnGood_70f754088050_2085_13725988807.mp4 +6907992105765_20240412-141523_front_returnGood_70f754088050_2085_13725988807.mp4 +6907992106205_20240412-141729_back_addGood_70f754088050_790_13725988807.mp4 +6907992106205_20240412-141729_front_addGood_70f754088050_790_13725988807.mp4 +6907992106205_20240412-141741_back_returnGood_70f754088050_790_13725988807.mp4 +6907992106205_20240412-141741_front_returnGood_70f754088050_790_13725988807.mp4 +6907992106311_20240412-141326_back_addGood_70f754088050_890_13725988807.mp4 +6907992106311_20240412-141326_front_addGood_70f754088050_890_13725988807.mp4 +6907992106311_20240412-141341_back_returnGood_70f754088050_890_13725988807.mp4 +6907992106311_20240412-141341_front_returnGood_70f754088050_890_13725988807.mp4 +6914973602908_20240412-134547_back_addGood_70f754088050_505_13725988807.mp4 +6914973602908_20240412-134547_front_addGood_70f754088050_505_13725988807.mp4 +6914973602908_20240412-134600_back_returnGood_70f754088050_505_13725988807.mp4 +6914973602908_20240412-134600_front_returnGood_70f754088050_505_13725988807.mp4 +6914973604223_20240412-143609_back_addGood_70f754088050_230_13725988807.mp4 +6914973604223_20240412-143609_front_addGood_70f754088050_230_13725988807.mp4 +6914973604223_20240412-143619_back_returnGood_70f754088050_230_13725988807.mp4 +6914973604223_20240412-143619_front_returnGood_70f754088050_230_13725988807.mp4 +6914973606340_20240412-134352_back_addGood_70f754088050_325_13725988807.mp4 +6914973606340_20240412-134352_front_addGood_70f754088050_325_13725988807.mp4 +6914973606340_20240412-134406_back_returnGood_70f754088050_330_13725988807.mp4 +6914973606340_20240412-134406_front_returnGood_70f754088050_330_13725988807.mp4 +6919188092377_20240412-141841_back_addGood_70f754088050_515_13725988807.mp4 +6919188092377_20240412-141841_front_addGood_70f754088050_515_13725988807.mp4 +6919188092377_20240412-141850_back_returnGood_70f754088050_515_13725988807.mp4 +6919188092377_20240412-141850_front_returnGood_70f754088050_515_13725988807.mp4 +6920152400630_20240412-151036_back_returnGood_70f754088050_580_13725988807.mp4 +6920174757101_20240412-144934_back_addGood_70f754088050_1300_13725988807.mp4 +6920174757101_20240412-144934_front_addGood_70f754088050_1300_13725988807.mp4 +6920174757101_20240412-144946_back_returnGood_70f754088050_1300_13725988807.mp4 +6920174757101_20240412-144946_front_returnGood_70f754088050_1300_13725988807.mp4 +6920459905012_20240412-150826_back_returnGood_70f754088050_550_13725988807.mp4 +6920907810707_20240412-145330_back_addGood_70f754088050_75_13725988807.mp4 +6920907810707_20240412-145330_front_addGood_70f754088050_75_13725988807.mp4 +6920907810707_20240412-145340_back_returnGood_70f754088050_75_13725988807.mp4 +6920907810707_20240412-145340_front_returnGood_70f754088050_75_13725988807.mp4 +6922130119213_20240412-144416_back_addGood_70f754088050_1020_13725988807.mp4 +6922130119213_20240412-144416_front_addGood_70f754088050_1020_13725988807.mp4 +6922130119213_20240412-144426_back_returnGood_70f754088050_1020_13725988807.mp4 +6922130119213_20240412-144426_front_returnGood_70f754088050_1020_13725988807.mp4 +6922577700968_20240412-141700_back_addGood_70f754088050_1040_13725988807.mp4 +6922577700968_20240412-141700_front_addGood_70f754088050_1040_13725988807.mp4 +6922577700968_20240412-141710_back_returnGood_70f754088050_1040_13725988807.mp4 +6922577700968_20240412-141710_front_returnGood_70f754088050_1040_13725988807.mp4 +6922868291168_20240412-145057_back_addGood_70f754088050_1160_13725988807.mp4 +6922868291168_20240412-145057_front_addGood_70f754088050_1160_13725988807.mp4 +6922868291168_20240412-145113_back_returnGood_70f754088050_1160_13725988807.mp4 +6922868291168_20240412-145113_front_returnGood_70f754088050_1160_13725988807.mp4 +6923450601549_20240412-134454_back_addGood_70f754088050_640_13725988807.mp4 +6923450601549_20240412-134454_front_addGood_70f754088050_640_13725988807.mp4 +6923450601549_20240412-134504_back_returnGood_70f754088050_635_13725988807.mp4 +6923450601549_20240412-134504_front_returnGood_70f754088050_635_13725988807.mp4 +6923450603574_20240412-135226_back_addGood_70f754088050_970_13725988807.mp4 +6923450603574_20240412-135226_front_addGood_70f754088050_970_13725988807.mp4 +6923450603574_20240412-135240_back_returnGood_70f754088050_965_13725988807.mp4 +6923450603574_20240412-135240_front_returnGood_70f754088050_965_13725988807.mp4 +6923450605288_20240412-134133_back_addGood_70f754088050_470_13725988807.mp4 +6923450605288_20240412-134133_front_addGood_70f754088050_470_13725988807.mp4 +6923450605288_20240412-134143_back_returnGood_70f754088050_470_13725988807.mp4 +6923450605288_20240412-134143_front_returnGood_70f754088050_470_13725988807.mp4 +6923450605332_20240412-135742_back_addGood_70f754088050_1785_13725988807.mp4 +6923450605332_20240412-135742_front_addGood_70f754088050_1785_13725988807.mp4 +6923450605332_20240412-135756_back_returnGood_70f754088050_1785_13725988807.mp4 +6923450605332_20240412-135756_front_returnGood_70f754088050_1785_13725988807.mp4 +6923450610428_20240412-133618_back_addGood_70f754088050_320_13725988807.mp4 +6923450610428_20240412-133618_front_addGood_70f754088050_320_13725988807.mp4 +6923450610428_20240412-133632_back_returnGood_70f754088050_320_13725988807.mp4 +6923450610428_20240412-133632_front_returnGood_70f754088050_320_13725988807.mp4 +6923450610459_20240412-135040_back_addGood_70f754088050_555_13725988807.mp4 +6923450610459_20240412-135040_front_addGood_70f754088050_555_13725988807.mp4 +6923450610459_20240412-135057_back_returnGood_70f754088050_555_13725988807.mp4 +6923450610459_20240412-135057_front_returnGood_70f754088050_555_13725988807.mp4 +6923450611067_20240412-134949_back_addGood_70f754088050_800_13725988807.mp4 +6923450611067_20240412-134949_front_addGood_70f754088050_800_13725988807.mp4 +6923450611067_20240412-135002_back_returnGood_70f754088050_800_13725988807.mp4 +6923450611067_20240412-135002_front_returnGood_70f754088050_800_13725988807.mp4 +6923450612415_20240412-133515_back_addGood_70f754088050_820_13725988807.mp4 +6923450612415_20240412-133515_front_addGood_70f754088050_820_13725988807.mp4 +6923450612415_20240412-133532_back_returnGood_70f754088050_820_13725988807.mp4 +6923450612415_20240412-133532_front_returnGood_70f754088050_820_13725988807.mp4 +6923450612484_20240412-133746_back_addGood_70f754088050_255_13725988807.mp4 +6923450612484_20240412-133746_front_addGood_70f754088050_255_13725988807.mp4 +6923450612484_20240412-133758_back_returnGood_70f754088050_255_13725988807.mp4 +6923450612484_20240412-133758_front_returnGood_70f754088050_255_13725988807.mp4 +6923450657829_20240412-135628_back_addGood_70f754088050_965_13725988807.mp4 +6923450657829_20240412-135628_front_addGood_70f754088050_965_13725988807.mp4 +6923450657829_20240412-135640_back_returnGood_70f754088050_965_13725988807.mp4 +6923450657829_20240412-135640_front_returnGood_70f754088050_965_13725988807.mp4 +6923450659441_20240412-135319_back_addGood_70f754088050_1825_13725988807.mp4 +6923450659441_20240412-135319_front_addGood_70f754088050_1825_13725988807.mp4 +6923450659441_20240412-135334_back_returnGood_70f754088050_1825_13725988807.mp4 +6923450659441_20240412-135334_front_returnGood_70f754088050_1825_13725988807.mp4 +6923450666838_20240412-134807_back_addGood_70f754088050_760_13725988807.mp4 +6923450666838_20240412-134807_front_addGood_70f754088050_760_13725988807.mp4 +6923450666838_20240412-134818_back_returnGood_70f754088050_760_13725988807.mp4 +6923450666838_20240412-134818_front_returnGood_70f754088050_760_13725988807.mp4 +6923450668207_20240412-134250_back_addGood_70f754088050_700_13725988807.mp4 +6923450668207_20240412-134250_front_addGood_70f754088050_700_13725988807.mp4 +6923450668207_20240412-134302_back_returnGood_70f754088050_700_13725988807.mp4 +6923450668207_20240412-134302_front_returnGood_70f754088050_700_13725988807.mp4 +6923450677858_20240412-135523_back_addGood_70f754088050_910_13725988807.mp4 +6923450677858_20240412-135523_front_addGood_70f754088050_910_13725988807.mp4 +6923450677858_20240412-135537_back_returnGood_70f754088050_910_13725988807.mp4 +6923450677858_20240412-135537_front_returnGood_70f754088050_910_13725988807.mp4 +6923644286293_20240412-141148_back_addGood_70f754088050_810_13725988807.mp4 +6923644286293_20240412-141148_front_addGood_70f754088050_810_13725988807.mp4 +6923644286293_20240412-141157_back_returnGood_70f754088050_805_13725988807.mp4 +6923644286293_20240412-141157_front_returnGood_70f754088050_805_13725988807.mp4 +6923644298760_20240412-141219_back_addGood_70f754088050_1020_13725988807.mp4 +6923644298760_20240412-141219_front_addGood_70f754088050_1020_13725988807.mp4 +6923644298760_20240412-141227_back_returnGood_70f754088050_1020_13725988807.mp4 +6923644298760_20240412-141227_front_returnGood_70f754088050_1020_13725988807.mp4 +6924743915824_20240412-145245_back_addGood_70f754088050_155_13725988807.mp4 +6924743915824_20240412-145245_front_addGood_70f754088050_155_13725988807.mp4 +6924743915824_20240412-145254_back_returnGood_70f754088050_155_13725988807.mp4 +6924743915824_20240412-145254_front_returnGood_70f754088050_155_13725988807.mp4 +6924882497106_20240412-150553_back_addGood_70f754088050_355_13725988807.mp4 +6924882497106_20240412-150553_front_addGood_70f754088050_355_13725988807.mp4 +6924882497106_20240412-150604_back_returnGood_70f754088050_355_13725988807.mp4 +6924882497106_20240412-150604_front_returnGood_70f754088050_355_13725988807.mp4 +6925307305525_20240412-152608_back_addGood_70f754088050_590_13725988807.mp4 +6925307305525_20240412-152608_front_addGood_70f754088050_590_13725988807.mp4 +6925307305525_20240412-152627_back_returnGood_70f754088050_590_13725988807.mp4 +6925307305525_20240412-152627_front_returnGood_70f754088050_590_13725988807.mp4 +6928033404968_20240412-143452_back_addGood_70f754088050_400_13725988807.mp4 +6928033404968_20240412-143452_front_addGood_70f754088050_400_13725988807.mp4 +6928033404968_20240412-143502_back_returnGood_70f754088050_400_13725988807.mp4 +6928033404968_20240412-143502_front_returnGood_70f754088050_400_13725988807.mp4 +6928804011173_20240412-152735_back_addGood_70f754088050_545_13725988807.mp4 +6928804011173_20240412-152735_front_addGood_70f754088050_545_13725988807.mp4 +6928804011173_20240412-152746_back_returnGood_70f754088050_545_13725988807.mp4 +6928804011173_20240412-152746_front_returnGood_70f754088050_545_13725988807.mp4 +6931925828032_20240412-134659_back_addGood_70f754088050_415_13725988807.mp4 +6931925828032_20240412-134659_front_addGood_70f754088050_415_13725988807.mp4 +6931925828032_20240412-134711_back_returnGood_70f754088050_415_13725988807.mp4 +6931925828032_20240412-134711_front_returnGood_70f754088050_415_13725988807.mp4 +6933620900051_20240412-140306_back_addGood_70f754088050_385_13725988807.mp4 +6933620900051_20240412-140306_front_addGood_70f754088050_385_13725988807.mp4 +6933620900051_20240412-140326_back_returnGood_70f754088050_390_13725988807.mp4 +6933620900051_20240412-140326_front_returnGood_70f754088050_390_13725988807.mp4 +6934665095108_20240412-141758_back_addGood_70f754088050_355_13725988807.mp4 +6934665095108_20240412-141758_front_addGood_70f754088050_355_13725988807.mp4 +6934665095108_20240412-141807_back_returnGood_70f754088050_355_13725988807.mp4 +6934665095108_20240412-141807_front_returnGood_70f754088050_355_13725988807.mp4 +6935270642121_20240412-151112_back_addGood_70f754088050_165_13725988807.mp4 +6935270642121_20240412-151112_front_addGood_70f754088050_165_13725988807.mp4 +6935270642121_20240412-151124_back_returnGood_70f754088050_165_13725988807.mp4 +6935270642121_20240412-151124_front_returnGood_70f754088050_165_13725988807.mp4 +6935284417326_20240412-145558_back_addGood_70f754088050_410_13725988807.mp4 +6935284417326_20240412-145558_front_addGood_70f754088050_410_13725988807.mp4 +6935284417326_20240412-145610_back_returnGood_70f754088050_410_13725988807.mp4 +6935284417326_20240412-145610_front_returnGood_70f754088050_410_13725988807.mp4 +6941025140798_20240412-152222_back_addGood_70f754088050_1160_13725988807.mp4 +6941025140798_20240412-152222_front_addGood_70f754088050_1160_13725988807.mp4 +6941025140798_20240412-152235_back_returnGood_70f754088050_1155_13725988807.mp4 +6941025140798_20240412-152235_front_returnGood_70f754088050_1155_13725988807.mp4 +6952074634794_20240412-153052_back_addGood_70f754088050_265_13725988807.mp4 +6952074634794_20240412-153052_front_addGood_70f754088050_265_13725988807.mp4 +6952074634794_20240412-153105_back_returnGood_70f754088050_265_13725988807.mp4 +6952074634794_20240412-153105_front_returnGood_70f754088050_265_13725988807.mp4 +6952074634794_20240412-153233_back_addGood_70f754088050_265_13725988807.mp4 +6952074634794_20240412-153233_front_addGood_70f754088050_265_13725988807.mp4 +6952074634794_20240412-153245_back_returnGood_70f754088050_265_13725988807.mp4 +6952074634794_20240412-153245_front_returnGood_70f754088050_265_13725988807.mp4 +6954432711307_20240412-134028_back_addGood_70f754088050_355_13725988807.mp4 +6954432711307_20240412-134028_front_addGood_70f754088050_355_13725988807.mp4 +6954432711307_20240412-134040_back_returnGood_70f754088050_350_13725988807.mp4 +6954432711307_20240412-134040_front_returnGood_70f754088050_350_13725988807.mp4 +6959546100993_20240412-142438_back_addGood_70f754088050_295_13725988807.mp4 +6959546100993_20240412-142438_front_addGood_70f754088050_295_13725988807.mp4 +6959546100993_20240412-142455_back_returnGood_70f754088050_295_13725988807.mp4 +6959546100993_20240412-142455_front_returnGood_70f754088050_295_13725988807.mp4 +6971075127463_20240412-142330_back_addGood_70f754088050_210_13725988807.mp4 +6971075127463_20240412-142330_front_addGood_70f754088050_210_13725988807.mp4 +6971075127463_20240412-142347_back_returnGood_70f754088050_215_13725988807.mp4 +6971075127463_20240412-142347_front_returnGood_70f754088050_215_13725988807.mp4 +6971075127470_20240412-142650_back_addGood_70f754088050_215_13725988807.mp4 +6971075127470_20240412-142650_front_addGood_70f754088050_215_13725988807.mp4 +6971075127470_20240412-142700_back_returnGood_70f754088050_215_13725988807.mp4 +6971075127470_20240412-142700_front_returnGood_70f754088050_215_13725988807.mp4 +6971328580533_20240412-152303_back_addGood_70f754088050_655_13725988807.mp4 +6971328580533_20240412-152303_front_addGood_70f754088050_655_13725988807.mp4 +6971328580533_20240412-152322_back_returnGood_70f754088050_650_13725988807.mp4 +6971328580533_20240412-152322_front_returnGood_70f754088050_650_13725988807.mp4 +6971738655333_20240412-141033_back_addGood_70f754088050_270_13725988807.mp4 +6971738655333_20240412-141033_front_addGood_70f754088050_270_13725988807.mp4 +6971738655333_20240412-141042_back_returnGood_70f754088050_270_13725988807.mp4 +6971738655333_20240412-141042_front_returnGood_70f754088050_270_13725988807.mp4 +6972378998200_20240412-144314_back_addGood_70f754088050_410_13725988807.mp4 +6972378998200_20240412-144314_front_addGood_70f754088050_410_13725988807.mp4 +6972378998200_20240412-144326_back_returnGood_70f754088050_410_13725988807.mp4 +6972378998200_20240412-144326_front_returnGood_70f754088050_410_13725988807.mp4 +6972790052733_20240412-135134_back_addGood_70f754088050_525_13725988807.mp4 +6972790052733_20240412-135134_front_addGood_70f754088050_525_13725988807.mp4 +6972790052733_20240412-135145_back_returnGood_70f754088050_525_13725988807.mp4 +6972790052733_20240412-135145_front_returnGood_70f754088050_525_13725988807.mp4 +6974913231612_20240412-142848_back_addGood_70f754088050_500_13725988807.mp4 +6974913231612_20240412-142848_front_addGood_70f754088050_500_13725988807.mp4 +6974913231612_20240412-142900_back_returnGood_70f754088050_495_13725988807.mp4 +6974913231612_20240412-142900_front_returnGood_70f754088050_495_13725988807.mp4 +6974995172711_20240412-152143_back_addGood_70f754088050_455_13725988807.mp4 +6974995172711_20240412-152143_front_addGood_70f754088050_455_13725988807.mp4 +6974995172711_20240412-152158_back_returnGood_70f754088050_455_13725988807.mp4 +6974995172711_20240412-152158_front_returnGood_70f754088050_455_13725988807.mp4 +6976371220276_20240412-140448_back_addGood_70f754088050_295_13725988807.mp4 +6976371220276_20240412-140448_front_addGood_70f754088050_295_13725988807.mp4 +6976371220276_20240412-140459_back_returnGood_70f754088050_295_13725988807.mp4 +6976371220276_20240412-140459_front_returnGood_70f754088050_295_13725988807.mp4 +850009021632_20240412-152522_back_addGood_70f754088050_470_13725988807.mp4 +850009021632_20240412-152522_front_addGood_70f754088050_470_13725988807.mp4 +850009021632_20240412-152543_back_returnGood_70f754088050_470_13725988807.mp4 +850009021632_20240412-152543_front_returnGood_70f754088050_470_13725988807.mp4 \ No newline at end of file diff --git a/tracking/matching/视频分类/双.txt b/tracking/matching/视频分类/双.txt new file mode 100644 index 0000000..22d2abe --- /dev/null +++ b/tracking/matching/视频分类/双.txt @@ -0,0 +1,208 @@ +6901070613142_20240411-170415_back_addGood_70f75407b7ae_430_17788571404.mp4 +6901070613142_20240411-170415_front_addGood_70f75407b7ae_430_17788571404.mp4 +6901070613142_20240411-170424_back_returnGood_70f75407b7ae_430_17788571404.mp4 +6901070613142_20240411-170424_front_returnGood_70f75407b7ae_430_17788571404.mp4 +6901070613142_20240411-170441_back_addGood_70f754088050_430_17327712807.mp4 +6901070613142_20240411-170441_front_addGood_70f754088050_430_17327712807.mp4 +6901070613142_20240411-170450_back_returnGood_70f754088050_430_17327712807.mp4 +6901070613142_20240411-170450_front_returnGood_70f754088050_430_17327712807.mp4 +6902538007367_20240411-165931_back_addGood_70f75407b7ae_995_17788571404.mp4 +6902538007367_20240411-165931_front_addGood_70f75407b7ae_995_17788571404.mp4 +6902538007367_20240411-165942_back_returnGood_70f75407b7ae_995_17788571404.mp4 +6902538007367_20240411-165942_front_returnGood_70f75407b7ae_995_17788571404.mp4 +6902538007367_20240411-165954_back_addGood_70f754088050_1000_17327712807.mp4 +6902538007367_20240411-165954_front_addGood_70f754088050_1000_17327712807.mp4 +6902538007367_20240411-170005_back_returnGood_70f754088050_1000_17327712807.mp4 +6902538007367_20240411-170005_front_returnGood_70f754088050_1000_17327712807.mp4 +6920152400630_20240411-165136_back_addGood_70f75407b7ae_720_17788571404.mp4 +6920152400630_20240411-165136_front_addGood_70f75407b7ae_720_17788571404.mp4 +6920152400630_20240411-165151_back_returnGood_70f75407b7ae_715_17788571404.mp4 +6920152400630_20240411-165151_front_returnGood_70f75407b7ae_715_17788571404.mp4 +6920152400630_20240411-165213_back_addGood_70f754088050_720_17327712807.mp4 +6920152400630_20240411-165213_front_addGood_70f754088050_720_17327712807.mp4 +6920152400630_20240411-165224_back_returnGood_70f754088050_720_17327712807.mp4 +6920152400630_20240411-165224_front_returnGood_70f754088050_720_17327712807.mp4 +6920907810707_20240411-165615_back_addGood_70f75407b7ae_225_17788571404.mp4 +6920907810707_20240411-165615_front_addGood_70f75407b7ae_225_17788571404.mp4 +6920907810707_20240411-165625_back_returnGood_70f75407b7ae_225_17788571404.mp4 +6920907810707_20240411-165625_front_returnGood_70f75407b7ae_225_17788571404.mp4 +6920907810707_20240411-165635_back_addGood_70f754088050_225_17327712807.mp4 +6920907810707_20240411-165635_front_addGood_70f754088050_225_17327712807.mp4 +6920907810707_20240411-165646_back_returnGood_70f754088050_225_17327712807.mp4 +6920907810707_20240411-165646_front_returnGood_70f754088050_225_17327712807.mp4 +6923450605288_20240411-172045_back_addGood_70f75407b7ae_750_17788571404.mp4 +6923450605288_20240411-172045_front_addGood_70f75407b7ae_750_17788571404.mp4 +6923450605288_20240411-172058_back_returnGood_70f75407b7ae_750_17788571404.mp4 +6923450605288_20240411-172058_front_returnGood_70f75407b7ae_750_17788571404.mp4 +6923450605288_20240411-172134_back_addGood_70f754088050_750_17327712807.mp4 +6923450605288_20240411-172134_front_addGood_70f754088050_750_17327712807.mp4 +6923450605288_20240411-172147_back_returnGood_70f754088050_750_17327712807.mp4 +6923450605288_20240411-172147_front_returnGood_70f754088050_750_17327712807.mp4 +6923450610428_20240411-171842_back_addGood_70f75407b7ae_270_17788571404.mp4 +6923450610428_20240411-171842_front_addGood_70f75407b7ae_270_17788571404.mp4 +6923450610428_20240411-171900_back_returnGood_70f75407b7ae_755_17788571404.mp4 +6923450610428_20240411-171900_front_returnGood_70f75407b7ae_755_17788571404.mp4 +6923450610428_20240411-171918_back_addGood_70f754088050_755_17327712807.mp4 +6923450610428_20240411-171918_front_addGood_70f754088050_755_17327712807.mp4 +6923450610428_20240411-172000_back_returnGood_70f754088050_755_17327712807.mp4 +6923450610428_20240411-172000_front_returnGood_70f754088050_755_17327712807.mp4 +6923450659441_20240411-171417_back_addGood_70f75407b7ae_2340_17788571404.mp4 +6923450659441_20240411-171417_front_addGood_70f75407b7ae_2340_17788571404.mp4 +6923450659441_20240411-171430_back_returnGood_70f75407b7ae_2335_17788571404.mp4 +6923450659441_20240411-171430_front_returnGood_70f75407b7ae_2335_17788571404.mp4 +6923450659441_20240411-171445_back_addGood_70f754088050_2340_17327712807.mp4 +6923450659441_20240411-171445_front_addGood_70f754088050_2340_17327712807.mp4 +6923450659441_20240411-171456_back_returnGood_70f754088050_2340_17327712807.mp4 +6923450659441_20240411-171456_front_returnGood_70f754088050_2340_17327712807.mp4 +6923450677858_20240411-171658_back_addGood_70f75407b7ae_1310_17788571404.mp4 +6923450677858_20240411-171658_front_addGood_70f75407b7ae_1310_17788571404.mp4 +6923450677858_20240411-171709_back_returnGood_70f75407b7ae_1310_17788571404.mp4 +6923450677858_20240411-171709_front_returnGood_70f75407b7ae_1310_17788571404.mp4 +6923450677858_20240411-171720_back_addGood_70f754088050_1310_17327712807.mp4 +6923450677858_20240411-171720_front_addGood_70f754088050_1310_17327712807.mp4 +6923450677858_20240411-171730_back_returnGood_70f754088050_1315_17327712807.mp4 +6923450677858_20240411-171730_front_returnGood_70f754088050_1315_17327712807.mp4 +6928804011173_20240411-165756_back_addGood_70f75407b7ae_615_17788571404.mp4 +6928804011173_20240411-165756_front_addGood_70f75407b7ae_615_17788571404.mp4 +6928804011173_20240411-165808_back_returnGood_70f75407b7ae_620_17788571404.mp4 +6928804011173_20240411-165808_front_returnGood_70f75407b7ae_620_17788571404.mp4 +6928804011173_20240411-165822_back_addGood_70f754088050_620_17327712807.mp4 +6928804011173_20240411-165822_front_addGood_70f754088050_620_17327712807.mp4 +6928804011173_20240411-165830_back_returnGood_70f754088050_620_17327712807.mp4 +6928804011173_20240411-165830_front_returnGood_70f754088050_620_17327712807.mp4 +6976371220276_20240411-170159_back_addGood_70f75407b7ae_745_17788571404.mp4 +6976371220276_20240411-170159_front_addGood_70f75407b7ae_745_17788571404.mp4 +6976371220276_20240411-170211_back_returnGood_70f75407b7ae_745_17788571404.mp4 +6976371220276_20240411-170211_front_returnGood_70f75407b7ae_745_17788571404.mp4 +6976371220276_20240411-170230_back_addGood_70f754088050_745_17327712807.mp4 +6976371220276_20240411-170230_front_addGood_70f754088050_745_17327712807.mp4 +6976371220276_20240411-170240_back_returnGood_70f754088050_745_17327712807.mp4 +6976371220276_20240411-170240_front_returnGood_70f754088050_745_17327712807.mp4 +230537101280010007_20240412-114205_back_addGood_70f75407b7ae_720_17327712807.mp4 +230537101280010007_20240412-114205_front_addGood_70f75407b7ae_720_17327712807.mp4 +230537101280010007_20240412-114214_back_returnGood_70f75407b7ae_720_17327712807.mp4 +230537101280010007_20240412-114214_front_returnGood_70f75407b7ae_720_17327712807.mp4 +2500456001326_20240412-110503_back_addGood_70f75407b7ae_1085_17327712807.mp4 +2500456001326_20240412-110503_front_addGood_70f75407b7ae_1085_17327712807.mp4 +2500456001326_20240412-110513_back_returnGood_70f75407b7ae_1085_17327712807.mp4 +2500456001326_20240412-110513_front_returnGood_70f75407b7ae_1085_17327712807.mp4 +6901070613142_20240412-110200_back_addGood_70f754088050_1180_17788571404.mp4 +6901070613142_20240412-110200_front_addGood_70f754088050_1180_17788571404.mp4 +6901070613142_20240412-110207_back_returnGood_70f754088050_1180_17788571404.mp4 +6901070613142_20240412-110207_front_returnGood_70f754088050_1180_17788571404.mp4 +6901070613142_20240412-112959_back_addGood_70f75407b7ae_655_17327712807.mp4 +6901070613142_20240412-112959_front_addGood_70f75407b7ae_655_17327712807.mp4 +6901070613142_20240412-113011_back_returnGood_70f75407b7ae_655_17327712807.mp4 +6901070613142_20240412-113011_front_returnGood_70f75407b7ae_655_17327712807.mp4 +6901668053893_20240412-113635_back_addGood_70f75407b7ae_565_17327712807.mp4 +6901668053893_20240412-113635_front_addGood_70f75407b7ae_565_17327712807.mp4 +6901668053893_20240412-113645_back_returnGood_70f75407b7ae_640_17327712807.mp4 +6901668053893_20240412-113645_front_returnGood_70f75407b7ae_640_17327712807.mp4 +6902007010249_20240412-113050_back_addGood_70f75407b7ae_1460_17327712807.mp4 +6902007010249_20240412-113050_front_addGood_70f75407b7ae_1460_17327712807.mp4 +6902007010249_20240412-113103_back_returnGood_70f75407b7ae_1460_17327712807.mp4 +6902007010249_20240412-113103_front_returnGood_70f75407b7ae_1460_17327712807.mp4 +6902022135514_20240412-112914_back_addGood_70f75407b7ae_3640_17327712807.mp4 +6902022135514_20240412-112914_front_addGood_70f75407b7ae_3640_17327712807.mp4 +6902022135514_20240412-112926_back_returnGood_70f75407b7ae_3640_17327712807.mp4 +6902022135514_20240412-112926_front_returnGood_70f75407b7ae_3640_17327712807.mp4 +6902265114369_20240412-110609_back_addGood_70f75407b7ae_990_17327712807.mp4 +6902265114369_20240412-110609_front_addGood_70f75407b7ae_990_17327712807.mp4 +6902265114369_20240412-110618_back_returnGood_70f75407b7ae_990_17327712807.mp4 +6902265114369_20240412-110618_front_returnGood_70f75407b7ae_990_17327712807.mp4 +6902265908012_20240412-111958_back_addGood_70f75407b7ae_1520_17327712807.mp4 +6902265908012_20240412-111958_front_addGood_70f75407b7ae_1520_17327712807.mp4 +6902265908012_20240412-112012_back_returnGood_70f75407b7ae_1520_17327712807.mp4 +6902265908012_20240412-112012_front_returnGood_70f75407b7ae_1520_17327712807.mp4 +6902538007367_20240412-105046_back_addGood_70f754088050_930_17788571404.mp4 +6902538007367_20240412-105046_front_addGood_70f754088050_930_17788571404.mp4 +6902538007367_20240412-105057_back_returnGood_70f754088050_930_17788571404.mp4 +6902538007367_20240412-105057_front_returnGood_70f754088050_930_17788571404.mp4 +6902538007367_20240412-105130_back_addGood_70f754088050_930_17788571404.mp4 +6902538007367_20240412-105130_front_addGood_70f754088050_930_17788571404.mp4 +6902538007367_20240412-105143_back_returnGood_70f754088050_935_17788571404.mp4 +6902538007367_20240412-105143_front_returnGood_70f754088050_935_17788571404.mp4 +6907992103952_20240412-114032_back_addGood_70f75407b7ae_545_17327712807.mp4 +6907992103952_20240412-114032_front_addGood_70f75407b7ae_545_17327712807.mp4 +6907992103952_20240412-114043_back_returnGood_70f75407b7ae_540_17327712807.mp4 +6907992103952_20240412-114043_front_returnGood_70f75407b7ae_540_17327712807.mp4 +6907992106311_20240412-110328_back_addGood_70f75407b7ae_2565_17327712807.mp4 +6907992106311_20240412-110328_front_addGood_70f75407b7ae_2565_17327712807.mp4 +6907992106311_20240412-110342_back_returnGood_70f75407b7ae_2570_17327712807.mp4 +6907992106311_20240412-110342_front_returnGood_70f75407b7ae_2570_17327712807.mp4 +6920152400630_20240412-112256_back_addGood_70f75407b7ae_1230_17327712807.mp4 +6920152400630_20240412-112256_front_addGood_70f75407b7ae_1230_17327712807.mp4 +6920152400630_20240412-112308_back_returnGood_70f75407b7ae_1230_17327712807.mp4 +6920152400630_20240412-112308_front_returnGood_70f75407b7ae_1230_17327712807.mp4 +6920174757101_20240412-112806_back_addGood_70f75407b7ae_2120_17327712807.mp4 +6920174757101_20240412-112806_front_addGood_70f75407b7ae_2120_17327712807.mp4 +6920174757101_20240412-112823_back_returnGood_70f75407b7ae_2125_17327712807.mp4 +6920174757101_20240412-112823_front_returnGood_70f75407b7ae_2125_17327712807.mp4 +6920459905012_20240412-104811_back_addGood_70f754088050_840_17788571404.mp4 +6920459905012_20240412-104811_front_addGood_70f754088050_840_17788571404.mp4 +6920459905012_20240412-104906_back_returnGood_70f754088050_840_17788571404.mp4 +6920459905012_20240412-104906_front_returnGood_70f754088050_840_17788571404.mp4 +6920459905012_20240412-105345_back_addGood_70f754088050_830_17788571404.mp4 +6920459905012_20240412-105345_front_addGood_70f754088050_830_17788571404.mp4 +6920459905012_20240412-105356_back_returnGood_70f754088050_830_17788571404.mp4 +6920459905012_20240412-105356_front_returnGood_70f754088050_830_17788571404.mp4 +6920459905012_20240412-113755_back_addGood_70f75407b7ae_970_17327712807.mp4 +6920459905012_20240412-113755_front_addGood_70f75407b7ae_970_17327712807.mp4 +6920459905012_20240412-113808_back_returnGood_70f75407b7ae_970_17327712807.mp4 +6920459905012_20240412-113808_front_returnGood_70f75407b7ae_970_17327712807.mp4 +6920907810707_20240412-105728_back_addGood_70f754088050_150_17788571404.mp4 +6920907810707_20240412-105728_front_addGood_70f754088050_150_17788571404.mp4 +6920907810707_20240412-105739_back_returnGood_70f754088050_145_17788571404.mp4 +6920907810707_20240412-105739_front_returnGood_70f754088050_145_17788571404.mp4 +6920907810707_20240412-113710_back_addGood_70f75407b7ae_900_17327712807.mp4 +6920907810707_20240412-113710_front_addGood_70f75407b7ae_900_17327712807.mp4 +6920907810707_20240412-113720_back_returnGood_70f75407b7ae_900_17327712807.mp4 +6920907810707_20240412-113720_front_returnGood_70f75407b7ae_900_17327712807.mp4 +6922130119213_20240412-111854_back_addGood_70f75407b7ae_1310_17327712807.mp4 +6922130119213_20240412-111854_front_addGood_70f75407b7ae_1310_17327712807.mp4 +6922130119213_20240412-111904_back_returnGood_70f75407b7ae_1310_17327712807.mp4 +6922130119213_20240412-111904_front_returnGood_70f75407b7ae_1310_17327712807.mp4 +6922577700968_20240412-114323_back_addGood_70f75407b7ae_1045_17327712807.mp4 +6922577700968_20240412-114323_front_addGood_70f75407b7ae_1045_17327712807.mp4 +6922577700968_20240412-114333_back_returnGood_70f75407b7ae_1310_17327712807.mp4 +6922577700968_20240412-114333_front_returnGood_70f75407b7ae_1310_17327712807.mp4 +6922868291168_20240412-112724_back_addGood_70f75407b7ae_4540_17327712807.mp4 +6922868291168_20240412-112724_front_addGood_70f75407b7ae_4540_17327712807.mp4 +6922868291168_20240412-112736_back_returnGood_70f75407b7ae_4540_17327712807.mp4 +6922868291168_20240412-112736_front_returnGood_70f75407b7ae_4540_17327712807.mp4 +6923644286293_20240412-114002_back_addGood_70f75407b7ae_1535_17327712807.mp4 +6923644286293_20240412-114002_front_addGood_70f75407b7ae_1535_17327712807.mp4 +6923644286293_20240412-114012_back_returnGood_70f75407b7ae_1535_17327712807.mp4 +6923644286293_20240412-114012_front_returnGood_70f75407b7ae_1535_17327712807.mp4 +6924743915824_20240412-104437_back_addGood_70f754088050_455_17788571404.mp4 +6924743915824_20240412-104437_front_addGood_70f754088050_455_17788571404.mp4 +6924743915824_20240412-104448_back_returnGood_70f754088050_455_17788571404.mp4 +6924743915824_20240412-104448_front_returnGood_70f754088050_455_17788571404.mp4 +6924743915824_20240412-113553_back_addGood_70f75407b7ae_920_17327712807.mp4 +6924743915824_20240412-113553_front_addGood_70f75407b7ae_920_17327712807.mp4 +6924743915824_20240412-113603_back_returnGood_70f75407b7ae_920_17327712807.mp4 +6924743915824_20240412-113603_front_returnGood_70f75407b7ae_920_17327712807.mp4 +6928804011173_20240412-105808_back_addGood_70f754088050_915_17788571404.mp4 +6928804011173_20240412-105808_front_addGood_70f754088050_915_17788571404.mp4 +6928804011173_20240412-105818_back_returnGood_70f754088050_910_17788571404.mp4 +6928804011173_20240412-105818_front_returnGood_70f754088050_910_17788571404.mp4 +6934665095108_20240412-114106_back_addGood_70f75407b7ae_885_17327712807.mp4 +6934665095108_20240412-114106_front_addGood_70f75407b7ae_885_17327712807.mp4 +6934665095108_20240412-114116_back_returnGood_70f75407b7ae_885_17327712807.mp4 +6934665095108_20240412-114116_front_returnGood_70f75407b7ae_885_17327712807.mp4 +6935270642121_20240412-111602_back_addGood_70f75407b7ae_540_17327712807.mp4 +6935270642121_20240412-111602_front_addGood_70f75407b7ae_540_17327712807.mp4 +6935270642121_20240412-111614_back_returnGood_70f75407b7ae_540_17327712807.mp4 +6935270642121_20240412-111614_front_returnGood_70f75407b7ae_540_17327712807.mp4 +6952074634794_20240412-104633_back_addGood_70f754088050_855_17788571404.mp4 +6952074634794_20240412-104633_front_addGood_70f754088050_855_17788571404.mp4 +6952074634794_20240412-104700_back_returnGood_70f754088050_855_17788571404.mp4 +6952074634794_20240412-104700_front_returnGood_70f754088050_855_17788571404.mp4 +6952074634794_20240412-113515_back_addGood_70f75407b7ae_595_17327712807.mp4 +6952074634794_20240412-113515_front_addGood_70f75407b7ae_595_17327712807.mp4 +6952074634794_20240412-113524_back_returnGood_70f75407b7ae_595_17327712807.mp4 +6952074634794_20240412-113524_front_returnGood_70f75407b7ae_595_17327712807.mp4 +6972378998200_20240412-111721_back_addGood_70f75407b7ae_870_17327712807.mp4 +6972378998200_20240412-111721_front_addGood_70f75407b7ae_870_17327712807.mp4 +6972378998200_20240412-111741_back_returnGood_70f75407b7ae_865_17327712807.mp4 +6972378998200_20240412-111741_front_returnGood_70f75407b7ae_865_17327712807.mp4 \ No newline at end of file diff --git a/tracking/matching/视频分类/视频文件命名规则.txt b/tracking/matching/视频分类/视频文件命名规则.txt new file mode 100644 index 0000000..886c4a3 --- /dev/null +++ b/tracking/matching/视频分类/视频文件命名规则.txt @@ -0,0 +1,12 @@ +采集文件名字段规则： +230537101280010007_20240411-144945_back_returnGood_70f75407b7ae_565_17788571404.mp4 + +String targetName = + barCode + "_" （条形码字段） + + recordFileName + "_" （文件名字段：时间格式精确到秒） + + "back/front"+ "_" （后/前摄字段） + + "addGood/returnGood"+ "_"（加/退购字段） + + macId + "_" （mac地址字段：去除中间冒号） + + Math.abs(goodsWeight) + "_" （商品重量字段：变化的绝对值） + + user.phone （采集人手机号字段） + + ".mp4"; \ No newline at end of file diff --git a/tracking/matching/视频分类/较暗.txt b/tracking/matching/视频分类/较暗.txt new file mode 100644 index 0000000..cfa2835 --- /dev/null +++ b/tracking/matching/视频分类/较暗.txt @@ -0,0 +1,6 @@ +6920152400630_20240412-151024_back_addGood_70f754088050_580_13725988807.mp4 +6920152400630_20240412-151024_front_addGood_70f754088050_580_13725988807.mp4 +6920152400630_20240412-151036_front_returnGood_70f754088050_580_13725988807.mp4 +6920459905012_20240412-150815_back_addGood_70f754088050_550_13725988807.mp4 +6920459905012_20240412-150815_front_addGood_70f754088050_550_13725988807.mp4 +6920459905012_20240412-150826_front_returnGood_70f754088050_550_13725988807.mp4 diff --git a/tracking/merge_track_test.py b/tracking/merge_track_test.py new file mode 100644 index 0000000..54df7b7 --- /dev/null +++ b/tracking/merge_track_test.py @@ -0,0 +1,173 @@ +# -*- coding: utf-8 -*- +""" +Created on Fri Feb 23 11:04:48 2024 + +@author: ym +""" +import numpy as np +import cv2 +from scipy.spatial.distance import cdist +# from trackers.utils import matching + +def readDict(boxes, feat_dicts): + feat = [] + for i in range(boxes.shape[0]): + tid, fid, bid = int(boxes[i, 4]), int(boxes[i, 7]), int(boxes[i, 8]) + feat.append(feat_dicts[fid][bid]) + + # img = feat_dicts[fid][f'{bid}_img'] + # cv2.imwrite(f'./result/imgs/{tid}_{fid}_{bid}.png', img) + + return np.asarray(feat, dtype=np.float32) + + + +def track_equal_track(atrack, btrack, feat_dicts): + # boxes: [x, y, w, h, track_id, score, cls, frame_index, box_index] + aboxes = atrack.boxes + bboxes = btrack.boxes + + ''' 1. 判断轨迹在时序上是否有交集 ''' + afids = aboxes[:, 7].astype(np.int_) + bfids = bboxes[:, 7].astype(np.int_) + # 帧索引交集 + interfid = set(afids).intersection(set(bfids)) + + # 或者直接判断帧索引是否有交集，返回 Ture or False + # interfid = set(afids).isdisjoint(set(bfids)) + + + ''' 2. 轨迹空间iou''' + alabel = np.array([0] * afids.size, dtype=np.int_) + blabel = np.array([1] * bfids.size, dtype=np.int_) + + label = np.concatenate((alabel, blabel), axis=0) + fids = np.concatenate((afids, bfids), axis=0) + indices = np.argsort(fids) + idx_pair = [] + for i in range(len(indices)-1): + idx1, idx2 = indices[i], indices[i+1] + if label[idx1] != label[idx2] and fids[idx2] - fids[idx1] == 1: + if label[idx1] == 0: + a_idx = idx1 + b_idx = idx2-alabel.size + else: + a_idx = idx2 + b_idx = idx1-alabel.size + + idx_pair.append((a_idx, b_idx)) + + ious = [] + for a, b in idx_pair: + abox, bbox = aboxes[a, :], bboxes[b, :] + + xa1, ya1 = abox[0] - abox[2]/2, abox[1] - abox[3]/2 + xa2, ya2 = abox[0] + abox[2]/2, abox[1] + abox[3]/2 + + xb1, yb1 = bbox[0] - bbox[2]/2, bbox[1] - bbox[3]/2 + xb2, yb2 = bbox[0] + bbox[2]/2, bbox[1] + bbox[3]/2 + + + inter = (np.minimum(xb2, xa2) - np.maximum(xb1, xa1)).clip(0) * \ + (np.minimum(yb2, ya2) - np.maximum(yb1, ya1)).clip(0) + + # Union Area + box1_area = abox[2] * abox[3] + box2_area = bbox[2] * bbox[3] + union = box1_area + box2_area - inter + 1e-6 + + ious.append(inter/union) + + ''' 3. 轨迹特征相似度判断''' + afeat = readDict(aboxes, feat_dicts) + bfeat = readDict(bboxes, feat_dicts) + feat = np.concatenate((afeat, bfeat), axis=0) + + emb_simil = 1-np.maximum(0.0, cdist(feat, feat, 'cosine')) + emb_ = 1-cdist(np.mean(afeat, axis=0)[None, :], np.mean(bfeat, axis=0)[None, :], 'cosine') + + cont1 = False if len(interfid) else True + cont2 = all(iou>0.5 for iou in ious) + cont3 = emb_[0, 0]>0.75 + + cont = cont1 and cont2 and cont3 + + + + + + + + return cont + + + +def track_equal_str(atrack, btrack): + if atrack == btrack: + return True + else: + return False + + +def merge_track(Residual): + out_list = [] + alist = [t for t in Residual] + while alist: + atrack = alist[0] + cur_list = [] + cur_list.append(atrack) + alist.pop(0) + + blist = [b for b in alist] + alist = [] + for btrack in blist: + if track_equal_str(atrack, btrack): + cur_list.append(btrack) + else: + alist.append(btrack) + + out_list.append(cur_list) + return out_list + +def main(): + Residual = ['a', 'b', 'c', 'd', 'a', 'b', 'c', 'b', 'c', 'd'] + out_list = merge_track(Residual) + + print(Residual) + print(out_list) + +if __name__ == "__main__": + + main() + + + + +# ============================================================================= +# for i, atrack in enumerate(input_list): +# cur_list = [] +# cur_list.append(atrack) +# del input_list[i] +# +# for j, btrack in enumerate(input_list): +# if track_equal(atrack, btrack): +# cur_list.append(btrack) +# del input_list[j] +# +# out_list.append(cur_list) +# ============================================================================= + + + + + + + + + + + + + + + diff --git a/tracking/module_analysis.py b/tracking/module_analysis.py new file mode 100644 index 0000000..1752100 --- /dev/null +++ b/tracking/module_analysis.py @@ -0,0 +1,470 @@ +# -*- coding: utf-8 -*- +""" +Created on Thu May 30 14:03:03 2024 + +轨迹分析现场测试性能分析: +(1) 读取 data 文件中的轨迹数据，绘制轨迹图 +(2) 读取本地运行 Yolo+Rsenet+Tracker+Tracking 的数据，绘制轨迹图 + + +@author: ym +""" +import os +import cv2 +import numpy as np +from pathlib import Path +import warnings +import sys +sys.path.append(r"D:\DetectTracking") + +from tracking.utils.read_data import extract_data_realtime, read_tracking_output_realtime + +from tracking.utils.plotting import Annotator, colors, draw_tracking_boxes +from tracking.utils import Boxes, IterableSimpleNamespace, yaml_load +from tracking.trackers import BOTSORT, BYTETracker +from tracking.dotrack.dotracks_back import doBackTracks +from tracking.dotrack.dotracks_front import doFrontTracks +from tracking.utils.drawtracks import plot_frameID_y2, draw_all_trajectories + +from tracking.utils.read_data import extract_data, read_deletedBarcode_file, read_tracking_output, read_returnGoods_file + +from contrast.one2n_contrast import get_contrast_paths, one2n_return +from tracking.utils.annotator import TrackAnnotator + +W, H = 1024, 1280 +Mode = 'front' #'back' +ImgFormat = ['.jpg', '.jpeg', '.png', '.bmp'] + + + +'''调用tracking()函数，利用本地跟踪算法获取各目标轨迹，可以比较本地跟踪算法与现场跟踪算法的区别。''' +def init_tracker(tracker_yaml = None, bs=1): + """ + Initialize tracker for object tracking during prediction. + """ + TRACKER_MAP = {'bytetrack': BYTETracker, 'botsort': BOTSORT} + cfg = IterableSimpleNamespace(**yaml_load(tracker_yaml)) + + tracker = TRACKER_MAP[cfg.tracker_type](args=cfg, frame_rate=30) + + return tracker + +def tracking(bboxes, ffeats): + tracker_yaml = r"./trackers/cfg/botsort.yaml" + tracker = init_tracker(tracker_yaml) + + TrackBoxes = np.empty((0, 9), dtype = np.float32) + TracksDict = {} + + '''========================== 执行跟踪处理 =============================''' + # dets 与 feats 应保持严格对应 + for dets, feats in zip(bboxes, ffeats): + det_tracking = Boxes(dets).cpu().numpy() + tracks = tracker.update(det_tracking, features=feats) + + + '''tracks: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] + 0 1 2 3 4 5 6 7 8 + 这里，frame_index 也可以用视频的 帧ID 代替, box_index 保持不变 + ''' + + if len(tracks): + TrackBoxes = np.concatenate([TrackBoxes, tracks], axis=0) + + FeatDict = {} + for track in tracks: + tid = int(track[8]) + FeatDict.update({tid: feats[tid, :]}) + + frameID = tracks[0, 7] + + # print(f"frameID: {int(frameID)}") + assert len(tracks) == len(FeatDict), f"Please check the func: tracker.update() at frameID({int(frameID)})" + + TracksDict[f"frame_{int(frameID)}"] = {"feats":FeatDict} + + + return TrackBoxes, TracksDict + + + + + + + +def read_imgs(imgspath, CamerType): + ''' + inputs: + imgspath；序列图像地址 + CamerType：相机类型，0：后摄，1：前摄 + outputs： + imgs：图像序列 + 功能： + 根据CamerType类型读取imgspath文件夹中的图像，并根据帧索引进行排序。 + do_tracking()中调用该函数，实现（1）读取imgs并绘制各目标轨迹框；（2）获取subimgs + ''' + imgs, frmIDs = [], [] + for filename in os.listdir(imgspath): + file, ext = os.path.splitext(filename) + flist = file.split('_') + if len(flist)==4 and ext in ImgFormat: + camID, frmID = flist[0], int(flist[-1]) + if camID==CamerType: + img = cv2.imread(os.path.join(imgspath, filename)) + imgs.append(img) + frmIDs.append(frmID) + if len(frmIDs): + indice = np.argsort(np.array(frmIDs)) + imgs = [imgs[i] for i in indice] + + return imgs + +def do_tracking(fpath, savedir, event_name='images'): + ''' + args: + fpath: 算法各模块输出的data文件地址，匹配； + savedir: 对 fpath 各模块输出的复现； + 分析具体视频时，需指定 fpath 和 savedir + outputs: + img_tracking：目标跟踪轨迹、本地轨迹分析算法的轨迹对比图 + abimg：现场轨迹分析算法、轨迹选择输出的对比图 + ''' + # fpath = r'D:\contrast\dataset\1_to_n\709\20240709-102758_6971558612189\1_track.data' + # savedir = r'D:\contrast\dataset\result\20240709-102843_6958770005357_6971558612189\error_6971558612189' + + imgpath, dfname = os.path.split(fpath) + CamerType = dfname.split('_')[0] + + + '''1.1 构造 0/1_tracking_output.data 文件地址，读取文件数据''' + tracking_output_path = os.path.join(imgpath, CamerType + '_tracking_output.data') + + basename = os.path.basename(imgpath) + if not os.path.isfile(fpath): + print(f"{basename}: Can't find {dfname} file!") + return None, None + if not os.path.isfile(tracking_output_path): + print(f"{basename}: Can't find {CamerType}_tracking_output.data file!") + return None, None + + bboxes, ffeats, trackerboxes, tracker_feat_dict, trackingboxes, tracking_feat_dict = extract_data(fpath) + tracking_output_boxes, _ = read_tracking_output(tracking_output_path) + + '''1.2 利用本地跟踪算法生成各商品轨迹''' + # trackerboxes, tracker_feat_dict = tracking(bboxes, ffeats) + + '''1.3 分别构造 2 个文件夹，(1) 存储画框后的图像; (2) 运动轨迹对应的 boxes子图''' + save_dir = os.path.join(savedir, event_name + '_images') + subimg_dir = os.path.join(savedir, event_name + '_subimgs') + if not os.path.exists(save_dir): + os.makedirs(save_dir) + if not os.path.exists(subimg_dir): + os.makedirs(subimg_dir) + + + '''2. 执行轨迹分析, 保存轨迹分析前后的对比图示''' + traj_graphic = event_name + '_' + CamerType + if CamerType == '1': + vts = doFrontTracks(trackerboxes, tracker_feat_dict) + vts.classify() + + plt = plot_frameID_y2(vts) + # ftpath = os.path.join(savedir, f"{traj_graphic}_front_y2.png") + # plt.savefig(ftpath) + plt.close() + + edgeline = cv2.imread("./shopcart/cart_tempt/board_ftmp_line.png") + img_tracking = draw_all_trajectories(vts, edgeline, savedir, CamerType, draw5p=True) + + + elif CamerType == '0': + vts = doBackTracks(trackerboxes, tracker_feat_dict) + vts.classify() + + edgeline = cv2.imread("./shopcart/cart_tempt/edgeline.png") + img_tracking = draw_all_trajectories(vts, edgeline, savedir, CamerType, draw5p=True) + + # imgpth = os.path.join(savedir, f"{traj_graphic}_.png") + # cv2.imwrite(str(imgpth), img) + else: + print("Please check data file!") + + + '''3 tracking() 算法输出后多轨迹选择问题分析''' + if CamerType == '1': + aline = cv2.imread("./shopcart/cart_tempt/board_ftmp_line.png") + elif CamerType == '0': + aline = cv2.imread("./shopcart/cart_tempt/edgeline.png") + else: + print("Please check data file!") + + bline = aline.copy() + + annotator = TrackAnnotator(aline, line_width=2) + for track in trackingboxes: + annotator.plotting_track(track) + aline = annotator.result() + + annotator = TrackAnnotator(bline, line_width=2) + if not isinstance(tracking_output_boxes, list): + tracking_output_boxes = [tracking_output_boxes] + + for track in tracking_output_boxes: + annotator.plotting_track(track) + bline = annotator.result() + + abimg = np.concatenate((aline, bline), axis = 1) + abH, abW = abimg.shape[:2] + cv2.line(abimg, (int(abW/2), 0), (int(abW/2), abH), (128, 255, 128), 2) + + # algpath = os.path.join(savedir, f"{traj_graphic}_alg.png") + # cv2.imwrite(str(algpath), abimg) + + '''4. 画框后的图像和子图保存，若imgs数与tracker中fid数不匹配，只保存原图，不保存子图''' + '''4.0 读取 fpath 中对应的图像 imgs ''' + imgs = read_imgs(imgpath, CamerType) + + '''4.1 imgs数 < trackerboxes 的 max(fid)，返回原图''' + if len(imgs) < np.max(trackerboxes[:,7]): + for i in range(len(imgs)): + img_savepath = os.path.join(save_dir, CamerType + "_" + f"{i}.png") + cv2.imwrite(img_savepath, imgs[i]) + print(f"{basename}: len(imgs) = {len(imgs)} < Tracker max(fid) = {int(np.max(trackerboxes[:,7]))}, 无法匹配画框") + return img_tracking, abimg + + '''4.2 在 imgs 上画框并保存''' + imgs_dw = draw_tracking_boxes(imgs, trackerboxes) + for fid, img in imgs_dw: + img_savepath = os.path.join(save_dir, CamerType + "_fid_" + f"{int(fid)}.png") + cv2.imwrite(img_savepath, img) + + '''4.3.2 保存轨迹选择对应的子图''' + # for track in tracking_output_boxes: + for track in vts.Residual: + for *xyxy, tid, conf, cls, fid, bid in track.boxes: + img = imgs[int(fid-1)] + x1, y1, x2, y2 = int(xyxy[0]/2), int(xyxy[1]/2), int(xyxy[2]/2), int(xyxy[3]/2) + subimg = img[y1:y2, x1:x2] + + subimg_path = os.path.join(subimg_dir, f'{CamerType}_tid{int(tid)}_{int(fid)}_{int(bid)}.png' ) + cv2.imwrite(subimg_path, subimg) + + for track in tracking_output_boxes: + for *xyxy, tid, conf, cls, fid, bid in track: + img = imgs[int(fid-1)] + x1, y1, x2, y2 = int(xyxy[0]/2), int(xyxy[1]/2), int(xyxy[2]/2), int(xyxy[3]/2) + subimg = img[y1:y2, x1:x2] + + subimg_path = os.path.join(subimg_dir, f'x_{CamerType}_tid{int(tid)}_{int(fid)}_{int(bid)}.png' ) + cv2.imwrite(subimg_path, subimg) + + + return img_tracking, abimg + + +def tracking_simulate(eventpath, savepath): + '''args: + eventpath: 事件文件夹 + savepath: 存储文件夹 + 遍历eventpath + ''' + +# ============================================================================= +# '''1. 获取事件名''' +# event_names = os.path.basename(eventpath).strip().split('_') +# if len(event_names)==2 and len(event_names[1])>=8: +# enent_name = event_names[1] +# elif len(event_names)==2 and len(event_names[1])==0: +# enent_name = event_names[0] +# else: +# return +# ============================================================================= + enent_name = os.path.basename(eventpath) + + ## only for simplify the filename + idx = enent_name.find('2024') + if idx>=0: + enent_name = enent_name[idx:(idx+15)] + + + '''2. 依次读取 0/1_track.data 中数据，进行仿真''' + illu_tracking, illu_select = [], [] + for filename in os.listdir(eventpath): + # filename = '1_track.data' + if filename.find("track.data") < 0: continue + + fpath = os.path.join(eventpath, filename) + if not os.path.isfile(fpath): continue + + img_tracking, img_select = do_tracking(fpath, savepath, enent_name) + + if img_select is not None: + illu_select.append(img_select) + if img_tracking is not None: + illu_tracking.append(img_tracking) + + '''3. 共幅8图，上下子图显示的是前后摄，每一行4个子图，分别为： + (1) tracker输出原始轨迹; (2)本地tracking输出; (3)现场算法轨迹选择前轨迹; (4)现场算法轨迹选择后的轨迹 + ''' + if len(illu_select)==2: + Img_s = np.concatenate((illu_select[0], illu_select[1]), axis = 0) + H, W = Img_s.shape[:2] + cv2.line(Img_s, (0, int(H/2)), (int(W), int(H/2)), (128, 255, 128), 2) + elif len(illu_select)==1: + Img_s = illu_select[0] + else: + Img_s = None + + if len(illu_tracking)==2: + Img_t = np.concatenate((illu_tracking[0], illu_tracking[1]), axis = 0) + H, W = Img_t.shape[:2] + cv2.line(Img_t, (0, int(H/2)), (int(W), int(H/2)), (128, 255, 128), 2) + elif len(illu_tracking)==1: + Img_t = illu_tracking[0] + else: + Img_t = None + + + '''3.1 保存输出轨迹图，若tracking、select的shape相同，则合并输出，否则单独输出''' + imgpath_tracking = os.path.join(savepath, enent_name + '_tracking.png') + imgpath_select = os.path.join(savepath, enent_name + '_select.png') + imgpath_ts = os.path.join(savepath, enent_name + '_tracking_select.png') + + if Img_t is not None and Img_s is not None and np.all(Img_s.shape==Img_t.shape): + Img_ts = np.concatenate((Img_t, Img_s), axis = 1) + H, W = Img_ts.shape[:2] + cv2.line(Img_ts, (int(W/2), 0), (int(W/2), int(H)), (0, 0, 255), 4) + cv2.imwrite(imgpath_ts, Img_ts) + + else: + if Img_s: cv2.imwrite(imgpath_select, Img_s) # 不会执行到该处 + if Img_t: cv2.imwrite(imgpath_tracking, Img_t) # 不会执行到该处 + Img_ts = None + + '''3.2 单独另存保存完好的 8 轨迹图''' + if Img_ts is not None: + basepath, _ = os.path.split(savepath) + trajpath = os.path.join(basepath, 'trajs') + if not os.path.exists(trajpath): + os.makedirs(trajpath) + traj_path = os.path.join(trajpath, enent_name+'.png') + cv2.imwrite(traj_path, Img_ts) + + + return Img_ts + + + + +# warnings.simplefilter("error", category=np.VisibleDeprecationWarning) + +def main_loop(): + + + del_barcode_file = r'\\192.168.1.28\share\测试_202406\0723\0723_3\deletedBarcode.txt' + basepath = r'\\192.168.1.28\share\测试_202406\0723\0723_3' # 测试数据文件夹地址 + + # del_barcode_file = r'\\192.168.1.28\share\测试_202406\1030\images\returnGoods.txt' + # basepath = r'\\192.168.1.28\share\测试_202406\1030\images' # 测试数据文件夹地址 + + '''获取性能测试数据相关路径''' + SavePath = r'D:\contrast\dataset\resultx' # 结果保存地址 + saveimgs = True + + if os.path.basename(del_barcode_file).find('deletedBarcode'): + relative_paths = get_contrast_paths(del_barcode_file, basepath, SavePath, saveimgs) + elif os.path.basename(del_barcode_file).find('returnGoods'): + blist = read_returnGoods_file(del_barcode_file) + errpairs, corrpairs, err_similarity, correct_similarity = one2n_return(blist) + relative_paths = [] + for getoutevent, inputevent, errevent in errpairs: + relative_paths.append(os.path.join(basepath, getoutevent)) + relative_paths.append(os.path.join(basepath, inputevent)) + relative_paths.append(os.path.join(basepath, errevent)) + + # prefix = ["getout_", "input_", "error_"] + '''开始循环执行每次测试过任务''' + k = 0 + for tuple_paths in relative_paths: + + '''1. 生成存储结果图像的文件夹''' + namedirs = [] + for data_path in tuple_paths: + base_name = os.path.basename(data_path).strip().split('_') + if len(base_name[-1]): + name = base_name[-1] + else: + name = base_name[0] + namedirs.append(name) + sdir = "_".join(namedirs) + savepath = os.path.join(SavePath, sdir) + + # if os.path.exists(savepath): + # continue + if not os.path.exists(savepath): + os.makedirs(savepath) + + '''2. 循环执行操作事件：取出、放入、错误匹配''' + for eventpath in tuple_paths: + try: + tracking_simulate(eventpath, savepath) + except Exception as e: + print(f'Error! {eventpath}, {e}') + + # k +=1 + # if k==1: + # break + + +def main(): + ''' + eventPaths: data文件地址，该 data 文件包括 Pipeline 各模块输出 + SavePath: 包含二级目录，一级目录为轨迹图像;二级目录为与data文件对应的序列图像存储地址。 + ''' + # eventPaths = r'\\192.168.1.28\share\测试_202406\0723\0723_3' + eventPaths = r'\\192.168.1.28\share\测试视频数据以及日志\各模块测试记录\展厅测试\1120_展厅模型v801测试\扫A放A' + savePath = r'D:\exhibition\result' + + k=0 + for pathname in os.listdir(eventPaths): + pathname = "20241121-144901-fdba61c6-aefa-4b50-876d-5e05998befdc_6920459905012_6920459905012" + + eventpath = os.path.join(eventPaths, pathname) + savepath = os.path.join(savePath, pathname) + if not os.path.exists(savepath): + os.makedirs(savepath) + + tracking_simulate(eventpath, savepath) + + # try: + # tracking_simulate(eventpath, savepath) + # except Exception as e: + # print(f'Error! {eventpath}, {e}') + + k += 1 + if k==1: + break + + +if __name__ == "__main__": + + # main_loop() + main() + # try: + # main_loop() + # except Exception as e: + # print(f'Error: {e}') + + + + + + + + + + + + + + diff --git a/tracking/shopcart/cart_program/新建文本文档.txt b/tracking/shopcart/cart_program/新建文本文档.txt new file mode 100644 index 0000000..c2d5f92 --- /dev/null +++ b/tracking/shopcart/cart_program/新建文本文档.txt @@ -0,0 +1 @@ +求取购物车轮廓，判断任一点是否在购物车内、是否在购物车边框附近 \ No newline at end of file diff --git a/tracking/shopcart/cart_tempt/说明.txt b/tracking/shopcart/cart_tempt/说明.txt new file mode 100644 index 0000000..671b906 --- /dev/null +++ b/tracking/shopcart/cart_tempt/说明.txt @@ -0,0 +1,6 @@ +5幅图： +incart.png +outcart.png +incart_ftmp.png +outcart_ftmp.png +cartboarder.png \ No newline at end of file diff --git a/tracking/shopcart/carttempt.py b/tracking/shopcart/carttempt.py new file mode 100644 index 0000000..e8046c1 --- /dev/null +++ b/tracking/shopcart/carttempt.py @@ -0,0 +1,151 @@ +# -*- coding: utf-8 -*- +""" +Created on Tue Sep 19 18:17:55 2023 + +@author: ym +""" + +import cv2 +import os +import numpy as np + + +def tempt_add_adjc(): + + temp = cv2.imread("img.png") + + path = r"D:\DeepLearning\yolov5\runs\trajectory" + patr = r"D:\DeepLearning\yolov5\tracking\result" + for filename in os.listdir(path): + imgpath = os.path.join(path, filename) + + img = cv2.imread(imgpath) + + img1 = cv2.add(img, temp) + + img1path = os.path.join(patr, filename) + cv2.imwrite(img1path, img1) + +def temp_add_boarder(): + temp = cv2.imread("cartedge.png") + temp[640:, 0:20, :] = 255 + temp[640:, -20:, :] = 255 + temp[-20:, :, :] = 255 + + cv2.imwrite("cartboarder.png", temp) + + +def create_front_temp(): + image = cv2.imread("./iCart4/b.png") + Height, Width = image.shape[:2] + gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + thresh, binary = cv2.threshold(gray, 128, 255, cv2.THRESH_BINARY_INV) + board = cv2.bitwise_not(binary) + contours, _ = cv2.findContours(board, cv2.RETR_LIST, cv2.CHAIN_APPROX_NONE) + + k = 0 + for cnt in contours: + img = np.zeros((Height, Width), dtype=np.uint8) + cv2.drawContours(img, [cnt], -1, 255, 3) + k += 1 + cv2.imwrite(f"./iCart4/back{k}.png", img) + + imgshow = cv2.drawContours(image, contours, -1, (0,255,0), 3) + cv2.imwrite("./iCart4/board_back_line.png", imgshow) + + # cv2.imwrite("./iCart4/4.png", board) + # cv2.imwrite("1.png", gray) + # cv2.imwrite("2.png", binary) + + + + +def create_back_temp(): + ''' + image1.png：从中获取轮廓的初始图像 + image2.png：主要用于显示效果 + Return：img.png + ''' + + image = cv2.imread("image1.png") + + Height, Width = image.shape[:2] + + gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + + gray[:405, :] = 0 + thresh, binary = cv2.threshold(gray, 254, 255, cv2.THRESH_BINARY) + cv2.imwrite("shopcart.png", binary) + + imgshow = cv2.cvtColor(binary, cv2.COLOR_GRAY2BGR) + + + + contours, _ = cv2.findContours(binary, cv2.RETR_LIST, cv2.CHAIN_APPROX_NONE) + + imgshow = cv2.drawContours(imgshow, contours, -1, (0,255,0), 1) + cv2.imwrite("imgshow.png", imgshow) + + + image2 = cv2.imread("image2.png") + image2 = cv2.drawContours(image2, contours, -1, (0,255,0), 3) + + + for cnt in contours: + _, start, _, num = cv2.boundingRect(cnt) + + x1 = (cnt[:, 0, 0] != 0) + x2 = (cnt[:, 0, 0] != Width-1) + x3 = (cnt[:, 0, 1] != Height-1) + x = (x1 & x2) & x3 + idx = np.where(x) + cntx = cnt[idx, :, :][0] + + cnt1 = cntx[:,0,:].copy() + + cntx[:, 0, 1] -= 60 + cnt2 = cntx[:,0,:].copy() + + + cv2.drawContours(image2,[cntx], 0, (0,0,255), 2) + + + + img = np.zeros(gray.shape, np.uint8) + for i in range(len(cnt1)): + x1, y1 = cnt1[i] + x2, y2 = cnt2[i] + cv2.rectangle(img, (x1-1, y1-1), (x1+1, y1+1), 255, 1) + cv2.rectangle(img, (x2-1, y2-1), (x2+1, y2+1), 255, 1) + + + + cv2.imwrite("img.png", img) + + +if __name__ == "__main__": + # create_back_temp() + # temp_add_boarder() + # tempt_add_adjc() + + create_front_temp() + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tracking/shopcart/iCart4.zip b/tracking/shopcart/iCart4.zip new file mode 100644 index 0000000..401c719 Binary files /dev/null and b/tracking/shopcart/iCart4.zip differ diff --git a/tracking/time_test.py b/tracking/time_test.py new file mode 100644 index 0000000..cab8c8e --- /dev/null +++ b/tracking/time_test.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- +""" +Created on Tue Aug 13 09:39:42 2024 + +@author: ym +""" +import os +import time +import datetime +import numpy as np +import sys +sys.path.append(r"D:\DetectTracking") +from tracking.utils.read_data import extract_data, read_weight_timeConsuming + + + +def main(): + directory = r"\\192.168.1.28\share\测试_202406\0821\images" + + TimeConsuming = [] + DayHMS = [] + for root, dirs, files in os.walk(directory): + if root.find('20240821') == -1: continue + for name in files: + if name.find('process.data') == -1: continue + datename = os.path.basename(root)[:15] + + fpath = os.path.join(root, name) + WeightDict, SensorDict, ProcessTimeDict = read_weight_timeConsuming(fpath) + try: + t1 = ProcessTimeDict['algroDoStart'] # 算法处理的第一帧图像时间 + t2 = ProcessTimeDict['breakinFirst'] # 第一次入侵时间 + t3 = ProcessTimeDict['algroLastFrame'] # 算法处理的最后一帧图像时间 + t4 = ProcessTimeDict['breakinLast'] # 最后一次入侵时间 + t5 = ProcessTimeDict['weightStablityTime'] # 重力稳定时间 + wv = ProcessTimeDict['weightValue'] # 重力值 + t6 = ProcessTimeDict['YoloResnetTrackerEnd'] # Yolo、Resnet、tracker执行结束时间 + t7 = ProcessTimeDict['trackingEnd'] # 轨迹分析结束时间 + t8 = ProcessTimeDict['contrastEnd'] # 比对结束时间 + t9 = ProcessTimeDict['algroStartToEnd'] # 算法总执行时间 + t10 = ProcessTimeDict['weightstablityToEnd'] # 重力稳定至算法结束时间 + t11 = ProcessTimeDict['frameEndToEnd'] # 最后一帧图像至算法结束时间 + + TimeConsuming.append((t1, t2, t3, t4, t5, wv, t6, t7, t8, t9, t10, t11)) + DayHMS.append(datename) + except Exception as e: + print(f'Error! {datename}, {e}') + + TimeConsuming = np.array(TimeConsuming, dtype = np.int64) + + TimeTotal = np.concatenate((TimeConsuming, + TimeConsuming[:,4][:, None] - TimeConsuming[:,0][:, None], + TimeConsuming[:,4][:, None] - TimeConsuming[:,2][:, None]), axis=1) + + tt = TimeTotal[:, 3]==0 + + TimeTotal0 = TimeTotal[tt] + DayHMS0 = [DayHMS[ti] for i, ti in enumerate(tt) if ti] + + TimeTotalMinus = TimeTotal[TimeTotal[:, 5]<0] + TimeTotalAdd = TimeTotal[TimeTotal[:, 5]>=0] + + TimeTotalAdd0 = TimeTotalAdd[TimeTotalAdd[:,3] == 0] + TimeTotalAdd1 = TimeTotalAdd[TimeTotalAdd[:,3] != 0] + + TimeTotalMinus0 = TimeTotalMinus[TimeTotalMinus[:,3] == 0] + TimeTotalMinus1 = TimeTotalMinus[TimeTotalMinus[:,3] != 0] + + print(f"Total number is {len(TimeConsuming)}") + + +if __name__ == "__main__": + main() + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tracking/trackers/README.md b/tracking/trackers/README.md new file mode 100644 index 0000000..a6505e0 --- /dev/null +++ b/tracking/trackers/README.md @@ -0,0 +1,94 @@ +# Tracker + +## Supported Trackers + +- [x] ByteTracker +- [x] BoT-SORT + +## Usage + +### python interface: + +You can use the Python interface to track objects using the YOLO model. + +```python +from ultralytics import YOLO + +model = YOLO("yolov8n.pt") # or a segmentation model .i.e yolov8n-seg.pt +model.track( + source="video/streams", + stream=True, + tracker="botsort.yaml", # or 'bytetrack.yaml' + show=True, +) +``` + +You can get the IDs of the tracked objects using the following code: + +```python +from ultralytics import YOLO + +model = YOLO("yolov8n.pt") + +for result in model.track(source="video.mp4"): + print( + result.boxes.id.cpu().numpy().astype(int) + ) # this will print the IDs of the tracked objects in the frame +``` + +If you want to use the tracker with a folder of images or when you loop on the video frames, you should use the `persist` parameter to tell the model that these frames are related to each other so the IDs will be fixed for the same objects. Otherwise, the IDs will be different in each frame because in each loop, the model creates a new object for tracking, but the `persist` parameter makes it use the same object for tracking. + +```python +import cv2 +from ultralytics import YOLO + +cap = cv2.VideoCapture("video.mp4") +model = YOLO("yolov8n.pt") +while True: + ret, frame = cap.read() + if not ret: + break + results = model.track(frame, persist=True) + boxes = results[0].boxes.xyxy.cpu().numpy().astype(int) + ids = results[0].boxes.id.cpu().numpy().astype(int) + for box, id in zip(boxes, ids): + cv2.rectangle(frame, (box[0], box[1]), (box[2], box[3]), (0, 255, 0), 2) + cv2.putText( + frame, + f"Id {id}", + (box[0], box[1]), + cv2.FONT_HERSHEY_SIMPLEX, + 1, + (0, 0, 255), + 2, + ) + cv2.imshow("frame", frame) + if cv2.waitKey(1) & 0xFF == ord("q"): + break +``` + +## Change tracker parameters + +You can change the tracker parameters by editing the `tracker.yaml` file which is located in the ultralytics/cfg/trackers folder. + +## Command Line Interface (CLI) + +You can also use the command line interface to track objects using the YOLO model. + +```bash +yolo detect track source=... tracker=... +yolo segment track source=... tracker=... +yolo pose track source=... tracker=... +``` + +By default, trackers will use the configuration in `ultralytics/cfg/trackers`. We also support using a modified tracker config file. Please refer to the tracker config files in `ultralytics/cfg/trackers`. + +## Contribute to Our Trackers Section + +Are you proficient in multi-object tracking and have successfully implemented or adapted a tracking algorithm with Ultralytics YOLO? We invite you to contribute to our Trackers section! Your real-world applications and solutions could be invaluable for users working on tracking tasks. + +By contributing to this section, you help expand the scope of tracking solutions available within the Ultralytics YOLO framework, adding another layer of functionality and utility for the community. + +To initiate your contribution, please refer to our [Contributing Guide](https://docs.ultralytics.com/help/contributing) for comprehensive instructions on submitting a Pull Request (PR) 🛠️. We are excited to see what you bring to the table! + +Together, let's enhance the tracking capabilities of the Ultralytics YOLO ecosystem 🙏! diff --git a/tracking/trackers/__init__.py b/tracking/trackers/__init__.py new file mode 100644 index 0000000..d4af88b --- /dev/null +++ b/tracking/trackers/__init__.py @@ -0,0 +1,10 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from .bot_sort import BOTSORT +from .byte_tracker import BYTETracker +from .track import register_tracker + + + +__all__ = 'register_tracker', 'BOTSORT', 'BYTETracker' # allow simpler import + diff --git a/tracking/trackers/__pycache__/__init__.cpython-312.pyc b/tracking/trackers/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..b32cfce Binary files /dev/null and b/tracking/trackers/__pycache__/__init__.cpython-312.pyc differ diff --git a/tracking/trackers/__pycache__/__init__.cpython-39.pyc b/tracking/trackers/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..145d091 Binary files /dev/null and b/tracking/trackers/__pycache__/__init__.cpython-39.pyc differ diff --git a/tracking/trackers/__pycache__/basetrack.cpython-312.pyc b/tracking/trackers/__pycache__/basetrack.cpython-312.pyc new file mode 100644 index 0000000..6233e5e Binary files /dev/null and b/tracking/trackers/__pycache__/basetrack.cpython-312.pyc differ diff --git a/tracking/trackers/__pycache__/basetrack.cpython-39.pyc b/tracking/trackers/__pycache__/basetrack.cpython-39.pyc new file mode 100644 index 0000000..2e97c78 Binary files /dev/null and b/tracking/trackers/__pycache__/basetrack.cpython-39.pyc differ diff --git a/tracking/trackers/__pycache__/bot_sort.cpython-312.pyc b/tracking/trackers/__pycache__/bot_sort.cpython-312.pyc new file mode 100644 index 0000000..0d2472c Binary files /dev/null and b/tracking/trackers/__pycache__/bot_sort.cpython-312.pyc differ diff --git a/tracking/trackers/__pycache__/bot_sort.cpython-39.pyc b/tracking/trackers/__pycache__/bot_sort.cpython-39.pyc new file mode 100644 index 0000000..7dd0e3c Binary files /dev/null and b/tracking/trackers/__pycache__/bot_sort.cpython-39.pyc differ diff --git a/tracking/trackers/__pycache__/byte_tracker.cpython-312.pyc b/tracking/trackers/__pycache__/byte_tracker.cpython-312.pyc new file mode 100644 index 0000000..730b2d5 Binary files /dev/null and b/tracking/trackers/__pycache__/byte_tracker.cpython-312.pyc differ diff --git a/tracking/trackers/__pycache__/byte_tracker.cpython-39.pyc b/tracking/trackers/__pycache__/byte_tracker.cpython-39.pyc new file mode 100644 index 0000000..33d3ddd Binary files /dev/null and b/tracking/trackers/__pycache__/byte_tracker.cpython-39.pyc differ diff --git a/tracking/trackers/__pycache__/track.cpython-312.pyc b/tracking/trackers/__pycache__/track.cpython-312.pyc new file mode 100644 index 0000000..496345c Binary files /dev/null and b/tracking/trackers/__pycache__/track.cpython-312.pyc differ diff --git a/tracking/trackers/__pycache__/track.cpython-39.pyc b/tracking/trackers/__pycache__/track.cpython-39.pyc new file mode 100644 index 0000000..32cb52c Binary files /dev/null and b/tracking/trackers/__pycache__/track.cpython-39.pyc differ diff --git a/tracking/trackers/basetrack.py b/tracking/trackers/basetrack.py new file mode 100644 index 0000000..3c7b0f7 --- /dev/null +++ b/tracking/trackers/basetrack.py @@ -0,0 +1,71 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from collections import OrderedDict + +import numpy as np + + +class TrackState: + """Enumeration of possible object tracking states.""" + + New = 0 + Tracked = 1 + Lost = 2 + Removed = 3 + + +class BaseTrack: + """Base class for object tracking, handling basic track attributes and operations.""" + + _count = 0 + + track_id = 0 + is_activated = False + state = TrackState.New + + history = OrderedDict() + features = [] + curr_feature = None + score = 0 + start_frame = 0 + frame_id = 0 + time_since_update = 0 + + # Multi-camera + location = (np.inf, np.inf) + + @property + def end_frame(self): + """Return the last frame ID of the track.""" + return self.frame_id + + @staticmethod + def next_id(): + """Increment and return the global track ID counter.""" + BaseTrack._count += 1 + return BaseTrack._count + + def activate(self, *args): + """Activate the track with the provided arguments.""" + raise NotImplementedError + + def predict(self): + """Predict the next state of the track.""" + raise NotImplementedError + + def update(self, *args, **kwargs): + """Update the track with new observations.""" + raise NotImplementedError + + def mark_lost(self): + """Mark the track as lost.""" + self.state = TrackState.Lost + + def mark_removed(self): + """Mark the track as removed.""" + self.state = TrackState.Removed + + @staticmethod + def reset_id(): + """Reset the global track ID counter.""" + BaseTrack._count = 0 diff --git a/tracking/trackers/bot_sort.py b/tracking/trackers/bot_sort.py new file mode 100644 index 0000000..fb9ddba --- /dev/null +++ b/tracking/trackers/bot_sort.py @@ -0,0 +1,211 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from collections import deque + +import numpy as np + +from .basetrack import TrackState +from .byte_tracker import BYTETracker, STrack +from .utils import matching +# from .utils.gmc import GMC +from .utils.kalman_filter import KalmanFilterXYWH + +# from .reid.reid_interface import ReIDInterface +# from .reid.config import config + +from contrast.feat_extract.inference import FeatsInterface +from contrast.feat_extract.config import config as conf + + + + + +class BOTrack(STrack): + shared_kalman = KalmanFilterXYWH() + + def __init__(self, tlwh, score, cls, feat=None, feat_history=50): + """Initialize YOLOv8 object with temporal parameters, such as feature history, alpha and current features.""" + super().__init__(tlwh, score, cls) + + self.smooth_feat = None + self.curr_feat = None + if feat is not None: + self.update_features(feat) + self.features = deque([], maxlen=feat_history) + self.alpha = 0.9 + + def update_features(self, feat): + """Update features vector and smooth it using exponential moving average.""" + feat /= np.linalg.norm(feat) + self.curr_feat = feat + if self.smooth_feat is None: + self.smooth_feat = feat + else: + self.smooth_feat = self.alpha * self.smooth_feat + (1 - self.alpha) * feat + self.features.append(feat) + self.smooth_feat /= np.linalg.norm(self.smooth_feat) + + def predict(self): + """Predicts the mean and covariance using Kalman filter.""" + mean_state = self.mean.copy() + if self.state != TrackState.Tracked: + mean_state[6] = 0 + mean_state[7] = 0 + + self.mean, self.covariance = self.kalman_filter.predict(mean_state, self.covariance) + + def re_activate(self, new_track, frame_id, new_id=False): + """Reactivates a track with updated features and optionally assigns a new ID.""" + if new_track.curr_feat is not None: + self.update_features(new_track.curr_feat) + super().re_activate(new_track, frame_id, new_id) + + def update(self, new_track, frame_id): + """Update the YOLOv8 instance with new track and frame ID.""" + if new_track.curr_feat is not None: + self.update_features(new_track.curr_feat) + super().update(new_track, frame_id) + + @property + def tlwh(self): + """Get current position in bounding box format `(top left x, top left y, + width, height)`. + """ + if self.mean is None: + return self._tlwh.copy() + ret = self.mean[:4].copy() + ret[:2] -= ret[2:] / 2 + return ret + + @staticmethod + def multi_predict(stracks): + """Predicts the mean and covariance of multiple object tracks using shared Kalman filter.""" + if len(stracks) <= 0: + return + multi_mean = np.asarray([st.mean.copy() for st in stracks]) + multi_covariance = np.asarray([st.covariance for st in stracks]) + for i, st in enumerate(stracks): + if st.state != TrackState.Tracked: + multi_mean[i][6] = 0 + multi_mean[i][7] = 0 + multi_mean, multi_covariance = BOTrack.shared_kalman.multi_predict(multi_mean, multi_covariance) + for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)): + stracks[i].mean = mean + stracks[i].covariance = cov + + def convert_coords(self, tlwh): + """Converts Top-Left-Width-Height bounding box coordinates to X-Y-Width-Height format.""" + return self.tlwh_to_xywh(tlwh) + + @staticmethod + def tlwh_to_xywh(tlwh): + """Convert bounding box to format `(center x, center y, width, + height)`. + """ + ret = np.asarray(tlwh).copy() + ret[:2] += ret[2:] / 2 + return ret + + +class BOTSORT(BYTETracker): + + def __init__(self, args, frame_rate=30): + """Initialize YOLOv8 object with ReID module and GMC algorithm.""" + super().__init__(args, frame_rate) + # ReID module + self.proximity_thresh = args.proximity_thresh + self.appearance_thresh = args.appearance_thresh + + # if args.with_reid: + # # Haven't supported BoT-SORT(reid) yet + # # self.encoder = ReIDInterface(config) + + # self.encoder = FeatsInterface(conf) + + # print('load model {} in BOTSORT'.format(conf.testbackbone)) + + # self.gmc = GMC(method=args.gmc_method) # commented by WQG + + def get_kalmanfilter(self): + """Returns an instance of KalmanFilterXYWH for object tracking.""" + return KalmanFilterXYWH() + + def init_track(self, dets, scores, cls, image, features_keep): + """Initialize track with detections, scores, and classes.""" + if len(dets) == 0: + return [] + if self.args.with_reid and self.encoder is not None: + if features_keep is None: + imgs, features_keep = self.encoder.inference(image, dets) + + return [BOTrack(xyxy, s, c, f) for (xyxy, s, c, f) in zip(dets, scores, cls, features_keep)] # detections + else: + return [BOTrack(xyxy, s, c) for (xyxy, s, c) in zip(dets, scores, cls)] # detections + + def get_dists(self, tracks, detections): + """Get distances between tracks and detections using IoU and (optionally) ReID embeddings.""" + dists = matching.iou_distance(tracks, detections) + # proximity_thresh 应该设较大的值，表示只有两个boxes离得较远时，不考虑reid特征 + dists_mask = (dists > self.proximity_thresh) + + # TODO: mot20 + # if not self.args.mot20: + dists = matching.fuse_score(dists, detections) + + if self.args.with_reid and self.encoder is not None: + emb_dists = matching.embedding_distance(tracks, detections) / 2.0 + emb_dists[emb_dists > self.appearance_thresh] = 1.0 + emb_dists[dists_mask] = 1.0 + dists = np.minimum(dists, emb_dists) + + return dists + + def get_dists_1(self, tracks, detections): + """Get distances between tracks and detections using IoU and (optionally) ReID embeddings.""" + iou_dists = matching.iou_distance(tracks, detections) + iou_dists_mask = (iou_dists>0.9) + + iou_dists = matching.fuse_score(iou_dists, detections) + weight = 0.4 + if self.args.with_reid and self.encoder is not None: + emb_dists = matching.embedding_distance(tracks, detections) + + '''============ iou_dists 和 emb_dists 融合有两种策略 ===========''' + '''1. reid 相似度阈值，低于该值的两 boxes 图像不可能是同一对象，需要确定一个合理的可信阈值 + 2. iou 的约束为若约束，故 iou_dists 应设置为较大的值 + ''' + emb_dists_mask = (emb_dists > 0.8) + iou_dists[emb_dists_mask] = 1 + emb_dists[iou_dists_mask] = 1 + + dists = np.minimum(iou_dists, emb_dists) + '''2. embed 阈值''' + # dists = (1-weight)*iou_dists + weight*emb_dists + else: + + dists = iou_dists.copy() + + return dists + + + + + + + + def multi_predict(self, tracks): + """Predict and track multiple objects with YOLOv8 model.""" + BOTrack.multi_predict(tracks) + + + def get_result(self): + '''written by WQG''' + activate_tracks = np.asarray([x.tlbr.tolist() + [x.track_id, x.score, x.cls, x.idx] + for x in self.tracked_stracks if x.is_activated], dtype=np.float32) + + track_features = [] + if self.args.with_reid and self.encoder is not None: + track_features = np.asarray([x.curr_feat for x in self.tracked_stracks if x.is_activated], dtype=np.float32) + + + return (activate_tracks, track_features) diff --git a/tracking/trackers/byte_tracker.py b/tracking/trackers/byte_tracker.py new file mode 100644 index 0000000..bca8aca --- /dev/null +++ b/tracking/trackers/byte_tracker.py @@ -0,0 +1,496 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import numpy as np + +from .basetrack import BaseTrack, TrackState +from .utils import matching +from .utils.kalman_filter import KalmanFilterXYAH + + +def dists_update(dists, strack_pool, detections): + '''written by WQG''' + + if len(strack_pool) and len(detections): + # alabel = np.array([int(stack.cls) if int(stack.cls)==0 or int(stack.cls)==9 else -1 for stack in strack_pool]) + # blabel = np.array([int(stack.cls) if int(stack.cls)==0 or int(stack.cls)==9 else -1 for stack in detections]) + + alabel = np.array([int(stack.cls) for stack in strack_pool]) + blabel = np.array([int(stack.cls) for stack in detections]) + amlabel = np.expand_dims(alabel, axis=1).repeat(len(detections),axis=1) + bmlabel = np.expand_dims(blabel, axis=0).repeat(len(strack_pool),axis=0) + + mlabel = bmlabel == amlabel + iou_dist = matching.iou_distance(strack_pool, detections) > 0.1 #boxes iou>0.9时，可以不考虑类别 + dist_label = (1 - mlabel) & iou_dist # 不同类，且不是严格重叠，需考虑类别距离 + + dist_label = 1 - mlabel + dists = np.where(dists > dist_label, dists, dist_label) + return dists + + +class STrack(BaseTrack): + shared_kalman = KalmanFilterXYAH() + + def __init__(self, tlwh, score, cls): + """wait activate.""" + self._tlwh = np.asarray(self.tlbr_to_tlwh(tlwh[:-1]), dtype=np.float32) + self.kalman_filter = None + self.mean, self.covariance = None, None + self.is_activated = False + + self.first_find = False ### + + self.score = score + self.tracklet_len = 0 + self.cls = cls + self.idx = tlwh[-1] + + def predict(self): + """Predicts mean and covariance using Kalman filter.""" + mean_state = self.mean.copy() + if self.state != TrackState.Tracked: + mean_state[7] = 0 + self.mean, self.covariance = self.kalman_filter.predict(mean_state, self.covariance) + + @staticmethod + def multi_predict(stracks): + """Perform multi-object predictive tracking using Kalman filter for given stracks.""" + if len(stracks) <= 0: + return + multi_mean = np.asarray([st.mean.copy() for st in stracks]) + multi_covariance = np.asarray([st.covariance for st in stracks]) + for i, st in enumerate(stracks): + if st.state != TrackState.Tracked: + multi_mean[i][7] = 0 + multi_mean, multi_covariance = STrack.shared_kalman.multi_predict(multi_mean, multi_covariance) + for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)): + stracks[i].mean = mean + stracks[i].covariance = cov + + @staticmethod + def multi_gmc(stracks, H=np.eye(2, 3)): + """Update state tracks positions and covariances using a homography matrix.""" + if len(stracks) > 0: + multi_mean = np.asarray([st.mean.copy() for st in stracks]) + multi_covariance = np.asarray([st.covariance for st in stracks]) + + R = H[:2, :2] + R8x8 = np.kron(np.eye(4, dtype=float), R) + t = H[:2, 2] + + for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)): + mean = R8x8.dot(mean) + mean[:2] += t + cov = R8x8.dot(cov).dot(R8x8.transpose()) + + stracks[i].mean = mean + stracks[i].covariance = cov + + def activate(self, kalman_filter, frame_id): + """Start a new tracklet.""" + self.kalman_filter = kalman_filter + self.track_id = self.next_id() + self.mean, self.covariance = self.kalman_filter.initiate(self.convert_coords(self._tlwh)) + + self.tracklet_len = 0 + self.state = TrackState.Tracked + if frame_id == 1: + self.is_activated = True + else: + self.first_find = True ### Add by WQG + self.frame_id = frame_id + self.start_frame = frame_id + + def re_activate(self, new_track, frame_id, new_id=False): + """Reactivates a previously lost track with a new detection.""" + self.mean, self.covariance = self.kalman_filter.update(self.mean, self.covariance, + self.convert_coords(new_track.tlwh)) + self.tracklet_len = 0 + self.state = TrackState.Tracked + self.is_activated = True + self.first_find = False + self.frame_id = frame_id + if new_id: + self.track_id = self.next_id() + self.score = new_track.score + self.cls = new_track.cls + self.idx = new_track.idx + + self._tlwh = new_track._tlwh + + def update(self, new_track, frame_id): + """ + Update a matched track + :type new_track: STrack + :type frame_id: int + :return: + """ + self.frame_id = frame_id + self.tracklet_len += 1 + + new_tlwh = new_track.tlwh + self.mean, self.covariance = self.kalman_filter.update(self.mean, self.covariance, + self.convert_coords(new_tlwh)) + self.state = TrackState.Tracked + self.is_activated = True + self.first_find = False + + self.score = new_track.score + self.cls = new_track.cls + self.idx = new_track.idx + + self._tlwh = new_track._tlwh + + + def convert_coords(self, tlwh): + """Convert a bounding box's top-left-width-height format to its x-y-angle-height equivalent.""" + return self.tlwh_to_xyah(tlwh) + + @property + def tlwh(self): + """Get current position in bounding box format `(top left x, top left y, + width, height)`. + """ + if self.mean is None: + return self._tlwh.copy() + ret = self.mean[:4].copy() + ret[2] *= ret[3] + ret[:2] -= ret[2:] / 2 + return ret + + @property + def tlbr(self): + """Convert bounding box to format `(min x, min y, max x, max y)`, i.e., + `(top left, bottom right)`. + """ + ret = self.tlwh.copy() + ret[2:] += ret[:2] + return ret + + @staticmethod + def tlwh_to_xyah(tlwh): + """Convert bounding box to format `(center x, center y, aspect ratio, + height)`, where the aspect ratio is `width / height`. + """ + ret = np.asarray(tlwh).copy() + ret[:2] += ret[2:] / 2 + ret[2] /= ret[3] + return ret + + @staticmethod + def tlbr_to_tlwh(tlbr): + """Converts top-left bottom-right format to top-left width height format.""" + ret = np.asarray(tlbr).copy() + ret[2:] -= ret[:2] + return ret + + @staticmethod + def tlwh_to_tlbr(tlwh): + """Converts tlwh bounding box format to tlbr format.""" + ret = np.asarray(tlwh).copy() + ret[2:] += ret[:2] + return ret + + def __repr__(self): + """Return a string representation of the BYTETracker object with start and end frames and track ID.""" + return f'OT_{self.track_id}_({self.start_frame}-{self.end_frame})' + + +class BYTETracker: + + def __init__(self, args, frame_rate=30): + """Initialize a YOLOv8 object to track objects with given arguments and frame rate.""" + self.tracked_stracks = [] # type: list[STrack] + self.lost_stracks = [] # type: list[STrack] + self.removed_stracks = [] # type: list[STrack] + + self.frame_id = 0 + self.args = args + self.max_time_lost = int(frame_rate / 30.0 * args.track_buffer) + self.kalman_filter = self.get_kalmanfilter() + self.reset_id() + + # Add by WQG + self.args.new_track_thresh = 0.5 + + + def update(self, results, img=None, features=None): + """Updates object tracker with new detections and returns tracked object bounding boxes.""" + self.frame_id += 1 + activated_stracks = [] + refind_stracks = [] + lost_stracks = [] + removed_stracks = [] + + first_finded = [] + + scores = results.conf + cls = results.cls + +# ============================================================================= +# # get xyxy and add index +# bboxes = results.xyxy +# bboxes = np.concatenate([bboxes, np.arange(len(bboxes)).reshape(-1, 1)], axis=-1) +# ============================================================================= + bboxes = results.xyxyb + + + remain_inds = scores > self.args.track_high_thresh + inds_low = scores > self.args.track_low_thresh + inds_high = scores < self.args.track_high_thresh + + inds_second = np.logical_and(inds_low, inds_high) + dets_second = bboxes[inds_second] + dets = bboxes[remain_inds] + scores_keep = scores[remain_inds] + scores_second = scores[inds_second] + cls_keep = cls[remain_inds] + cls_second = cls[inds_second] + + detections = self.init_track(dets, scores_keep, cls_keep, img, features) + + # Add newly detected tracklets to tracked_stracks + unconfirmed = [] + tracked_stracks = [] # type: list[STrack] + for track in self.tracked_stracks: + if not track.is_activated: + unconfirmed.append(track) + else: + tracked_stracks.append(track) + + + # Step 2: First association, with high score detection boxes + strack_pool = self.joint_stracks(tracked_stracks, self.lost_stracks) + # Predict the current location with KF + self.multi_predict(strack_pool) + +# ============================================================= 没必要gmc，WQG +# if hasattr(self, 'gmc') and img is not None: +# warp = self.gmc.apply(img, dets) +# STrack.multi_gmc(strack_pool, warp) +# STrack.multi_gmc(unconfirmed, warp) +# ============================================================================= + + dists = self.get_dists_1(strack_pool, detections) + + '''written by WQG for different class''' + dists = dists_update(dists, strack_pool, detections) + + matches, u_track, u_detection = matching.linear_assignment(dists, thresh=self.args.match_thresh) + for itracked, idet in matches: + track = strack_pool[itracked] + det = detections[idet] + if track.state == TrackState.Tracked: + track.update(det, self.frame_id) + activated_stracks.append(track) + else: + track.re_activate(det, self.frame_id, new_id=False) + refind_stracks.append(track) + + + # Step 3: Second association, with low score detection boxes + # association the untrack to the low score detections + detections_second = self.init_track(dets_second, scores_second, cls_second, img, features) + r_tracked_stracks = [strack_pool[i] for i in u_track if strack_pool[i].state == TrackState.Tracked] + + # TODO + dists = matching.iou_distance(r_tracked_stracks, detections_second) + '''written by WQG for different class''' + dists = dists_update(dists, r_tracked_stracks, detections_second) + + matches, u_track, u_detection_second = matching.linear_assignment(dists, thresh=0.5) + for itracked, idet in matches: + track = r_tracked_stracks[itracked] + det = detections_second[idet] + if track.state == TrackState.Tracked: + track.update(det, self.frame_id) + activated_stracks.append(track) + else: + track.re_activate(det, self.frame_id, new_id=False) + refind_stracks.append(track) + + for it in u_track: + track = r_tracked_stracks[it] + if track.state != TrackState.Lost: + track.mark_lost() + lost_stracks.append(track) + + # Deal with unconfirmed tracks, usually tracks with only one beginning frame + detections = [detections[i] for i in u_detection] + dists = self.get_dists_1(unconfirmed, detections) + '''written by WQG for different class''' + dists = dists_update(dists, unconfirmed, detections) + + matches, u_unconfirmed, u_detection = matching.linear_assignment(dists, thresh=0.7) + for itracked, idet in matches: + unconfirmed[itracked].update(detections[idet], self.frame_id) + activated_stracks.append(unconfirmed[itracked]) + for it in u_unconfirmed: + track = unconfirmed[it] + if self.frame_id - track.end_frame > 2: # Add by WQG + track.mark_removed() + removed_stracks.append(track) + # Step 4: Init new stracks + for inew in u_detection: + track = detections[inew] + if track.score < self.args.new_track_thresh: + continue + track.activate(self.kalman_filter, self.frame_id) + activated_stracks.append(track) + + first_finded.append(track) + + # Step 5: Update state + for track in self.lost_stracks: + if self.frame_id - track.end_frame > self.max_time_lost: + track.mark_removed() + removed_stracks.append(track) + + self.tracked_stracks = [t for t in self.tracked_stracks if t.state == TrackState.Tracked] + self.tracked_stracks = self.joint_stracks(self.tracked_stracks, activated_stracks) + self.tracked_stracks = self.joint_stracks(self.tracked_stracks, refind_stracks) + self.lost_stracks = self.sub_stracks(self.lost_stracks, self.tracked_stracks) + self.lost_stracks.extend(lost_stracks) + self.lost_stracks = self.sub_stracks(self.lost_stracks, self.removed_stracks) + self.tracked_stracks, self.lost_stracks = self.remove_duplicate_stracks(self.tracked_stracks, self.lost_stracks) + self.removed_stracks.extend(removed_stracks) + if len(self.removed_stracks) > 1000: + self.removed_stracks = self.removed_stracks[-999:] # clip remove stracks to 1000 maximum + + '''x.tlbr have update by function: + @property + def tlwh(self): + ''' + + ##================ 原算法输出 + # output = np.asarray([x.tlbr.tolist() + [x.track_id, x.score, x.cls, x.frame_id, x.idx] + # for x in self.tracked_stracks if x.is_activated], dtype=np.float32) + + ## ===== write by WQG + output1 = [x.tlwh_to_tlbr(x._tlwh).tolist() + [x.track_id, x.score, x.cls, x.frame_id, x.idx] + for x in self.tracked_stracks if x.is_activated] + + output2 = [x.tlwh_to_tlbr(x._tlwh).tolist() + [x.track_id, x.score, x.cls, x.frame_id, x.idx] + for x in first_finded if x.first_find] + + output = np.asarray(output1 + output2, dtype=np.float32) + + + out_feat1 = [(x.frame_id, x.idx, x.smooth_feat, x.curr_feat, x.features) for x in self.tracked_stracks if x.is_activated] + out_feat2 = [(x.frame_id, x.idx, x.smooth_feat, x.curr_feat, x.features) for x in first_finded if x.first_find] + + + return output, out_feat1 + out_feat2 + + + def get_result(self): + '''written by WQG''' +# ============================================================================= +# activate_tracks = np.asarray([x.tlbr.tolist() + [x.track_id, x.score, x.cls, x.idx] +# for x in self.tracked_stracks if x.is_activated], dtype=np.float32) +# +# track_features = [] +# ============================================================================= + tracks = [] + feats = [] + for t in self.tracked_stracks: + if t.is_activated or t.first_find: + track = t.tlbr.tolist() + [t.track_id, t.score, t.cls, t.idx] + feat = t.curr_feature + + tracks.append(track) + feats.append(feat) + + tracks = np.asarray(tracks, dtype=np.float32) + + return (tracks, feats) + + + def get_kalmanfilter(self): + """Returns a Kalman filter object for tracking bounding boxes.""" + return KalmanFilterXYAH() + + def init_track(self, dets, scores, cls, img=None, feats=None): + """Initialize object tracking with detections and scores using STrack algorithm.""" + return [STrack(xyxy, s, c) for (xyxy, s, c) in zip(dets, scores, cls)] if len(dets) else [] # detections + + def get_dists(self, tracks, detections): + """Calculates the distance between tracks and detections using IOU and fuses scores.""" + dists = matching.iou_distance(tracks, detections) + # TODO: mot20 + # if not self.args.mot20: + dists = matching.fuse_score(dists, detections) + return dists + def get_dists_1(self, tracks, detections): + """Calculates the distance between tracks and detections using IOU and fuses scores.""" + + pass + + + def multi_predict(self, tracks): + """Returns the predicted tracks using the YOLOv8 network.""" + STrack.multi_predict(tracks) + + def reset_id(self): + """Resets the ID counter of STrack.""" + STrack.reset_id() + + @staticmethod + def joint_stracks(tlista, tlistb): + """Combine two lists of stracks into a single one.""" + exists = {} + res = [] + for t in tlista: + exists[t.track_id] = 1 + res.append(t) + for t in tlistb: + tid = t.track_id + if not exists.get(tid, 0): + exists[tid] = 1 + res.append(t) + return res + + @staticmethod + def sub_stracks(tlista, tlistb): + """DEPRECATED CODE in https://github.com/ultralytics/ultralytics/pull/1890/ + stracks = {t.track_id: t for t in tlista} + for t in tlistb: + tid = t.track_id + if stracks.get(tid, 0): + del stracks[tid] + return list(stracks.values()) + """ + track_ids_b = {t.track_id for t in tlistb} + return [t for t in tlista if t.track_id not in track_ids_b] + + @staticmethod + def remove_duplicate_stracks(stracksa, stracksb): + """Remove duplicate stracks with non-maximum IOU distance.""" + pdist = matching.iou_distance(stracksa, stracksb) + + #### ===================================== written by WQG + mlabel = [] + if len(stracksa) and len(stracksb): + alabel = np.array([int(stack.cls) for stack in stracksa]) + blabel = np.array([int(stack.cls) for stack in stracksb]) + amlabel = np.expand_dims(alabel, axis=1).repeat(len(stracksb),axis=1) + bmlabel = np.expand_dims(blabel, axis=0).repeat(len(stracksa),axis=0) + mlabel = bmlabel == amlabel + if len(mlabel): + condt = (pdist<0.15) & mlabel # 需满足iou足够小，且类别相同，才予以排除 + else: + condt = pdist<0.15 + + + pairs = np.where(condt) + dupa, dupb = [], [] + for p, q in zip(*pairs): + timep = stracksa[p].frame_id - stracksa[p].start_frame + timeq = stracksb[q].frame_id - stracksb[q].start_frame + if timep > timeq: + dupb.append(q) + else: + dupa.append(p) + resa = [t for i, t in enumerate(stracksa) if i not in dupa] + resb = [t for i, t in enumerate(stracksb) if i not in dupb] + return resa, resb diff --git a/tracking/trackers/cfg/botsort.yaml b/tracking/trackers/cfg/botsort.yaml new file mode 100644 index 0000000..b32591a --- /dev/null +++ b/tracking/trackers/cfg/botsort.yaml @@ -0,0 +1,18 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# Default YOLO tracker settings for BoT-SORT tracker https://github.com/NirAharon/BoT-SORT + +tracker_type: botsort # tracker type, ['botsort', 'bytetrack'] +track_high_thresh: 0.5 # threshold for the first association +track_low_thresh: 0.1 # threshold for the second association +new_track_thresh: 0.6 # threshold for init new track if the detection does not match any tracks +track_buffer: 30 # buffer to calculate the time when to remove tracks +match_thresh: 0.8 # threshold for matching tracks +# min_box_area: 10 # threshold for min box areas(for tracker evaluation, not used for now) +# mot20: False # for tracker evaluation(not used for now) + +# BoT-SORT settings +gmc_method: sparseOptFlow # method of global motion compensation +# ReID model related thresh (not supported yet) +proximity_thresh: 0.5 +appearance_thresh: 0.25 +with_reid: True diff --git a/tracking/trackers/cfg/bytetrack.yaml b/tracking/trackers/cfg/bytetrack.yaml new file mode 100644 index 0000000..5060f92 --- /dev/null +++ b/tracking/trackers/cfg/bytetrack.yaml @@ -0,0 +1,11 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# Default YOLO tracker settings for ByteTrack tracker https://github.com/ifzhang/ByteTrack + +tracker_type: bytetrack # tracker type, ['botsort', 'bytetrack'] +track_high_thresh: 0.5 # threshold for the first association +track_low_thresh: 0.1 # threshold for the second association +new_track_thresh: 0.6 # threshold for init new track if the detection does not match any tracks +track_buffer: 30 # buffer to calculate the time when to remove tracks +match_thresh: 0.8 # threshold for matching tracks +# min_box_area: 10 # threshold for min box areas(for tracker evaluation, not used for now) +# mot20: False # for tracker evaluation(not used for now) diff --git a/tracking/trackers/reid/__init__.py b/tracking/trackers/reid/__init__.py new file mode 100644 index 0000000..94c5ca1 --- /dev/null +++ b/tracking/trackers/reid/__init__.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Created on Fri Jan 19 16:15:35 2024 + +@author: ym +""" + diff --git a/tracking/trackers/reid/__pycache__/__init__.cpython-39.pyc b/tracking/trackers/reid/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..af967f0 Binary files /dev/null and b/tracking/trackers/reid/__pycache__/__init__.cpython-39.pyc differ diff --git a/tracking/trackers/reid/__pycache__/config.cpython-39.pyc b/tracking/trackers/reid/__pycache__/config.cpython-39.pyc new file mode 100644 index 0000000..504c63b Binary files /dev/null and b/tracking/trackers/reid/__pycache__/config.cpython-39.pyc differ diff --git a/tracking/trackers/reid/__pycache__/reid_interface.cpython-39.pyc b/tracking/trackers/reid/__pycache__/reid_interface.cpython-39.pyc new file mode 100644 index 0000000..3bae23d Binary files /dev/null and b/tracking/trackers/reid/__pycache__/reid_interface.cpython-39.pyc differ diff --git a/tracking/trackers/reid/config.py b/tracking/trackers/reid/config.py new file mode 100644 index 0000000..7419dd0 --- /dev/null +++ b/tracking/trackers/reid/config.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +""" +Created on Fri Jan 19 14:01:46 2024 + +@author: ym +""" + +import torch +import os +# import torchvision.transforms as T +class Config: + # network settings + backbone = 'resnet18' # [resnet18, mobilevit_s, mobilenet_v2, mobilenetv3] + batch_size = 8 + embedding_size = 256 + img_size = 224 + + ckpt_path = r"ckpts\resnet18_1220\best.pth" + ckpt_path = r"ckpts\best_resnet18_1887_0311.pth" + + current_path = os.path.dirname(os.path.abspath(__file__)) + model_path = os.path.join(current_path, ckpt_path) + + # model_path = "./trackers/reid/ckpts/resnet18_1220/best.pth" + device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + +# ============================================================================= +# metric = 'arcface' # [cosface, arcface] +# drop_ratio = 0.5 +# +# # training settings +# checkpoints = "checkpoints/Mobilev3Large_1225" # [resnet18, mobilevit_s, mobilenet_v2, mobilenetv3] +# restore = False +# +# test_model = "./checkpoints/resnet18_1220/best.pth" +# +# +# +# +# device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') +# pin_memory = True # if memory is large, set it True to speed up a bit +# num_workers = 4 # dataloader +# ============================================================================= + +config = Config() \ No newline at end of file diff --git a/tracking/trackers/reid/model/BAM.py b/tracking/trackers/reid/model/BAM.py new file mode 100644 index 0000000..c3fb591 --- /dev/null +++ b/tracking/trackers/reid/model/BAM.py @@ -0,0 +1,83 @@ +import torch.nn as nn +import torchvision +from torch.nn import init + +class Flatten(nn.Module): + def forward(self, x): + return x.view(x.shape[0], -1) + +class ChannelAttention(nn.Module): + def __int__(self,channel,reduction, num_layers): + super(ChannelAttention,self).__init__() + self.avgpool = nn.AdaptiveAvgPool2d(1) + gate_channels = [channel] + gate_channels += [len(channel)//reduction]*num_layers + gate_channels += [channel] + + self.ca = nn.Sequential() + self.ca.add_module('flatten', Flatten()) + for i in range(len(gate_channels)-2): + self.ca.add_module('',nn.Linear(gate_channels[i], gate_channels[i+1])) + self.ca.add_module('',nn.BatchNorm1d(gate_channels[i+1])) + self.ca.add_module('',nn.ReLU()) + self.ca.add_module('',nn.Linear(gate_channels[-2], gate_channels[-1])) + + def forward(self, x): + res = self.avgpool(x) + res = self.ca(res) + res = res.unsqueeze(-1).unsqueeze(-1).expand_as(x) + return res + +class SpatialAttention(nn.Module): + def __int__(self, channel,reduction=16,num_lay=3,dilation=2): + super(SpatialAttention).__init__() + self.sa = nn.Sequential() + self.sa.add_module('', nn.Conv2d(kernel_size=1, in_channels=channel, out_channels=(channel//reduction)*3)) + self.sa.add_module('',nn.BatchNorm2d(num_features=(channel//reduction))) + self.sa.add_module('',nn.ReLU()) + for i in range(num_lay): + self.sa.add_module('', nn.Conv2d(kernel_size=3, + in_channels=(channel//reduction), + out_channels=(channel//reduction), + padding=1, + dilation= 2)) + self.sa.add_module('',nn.BatchNorm2d(channel//reduction)) + self.sa.add_module('',nn.ReLU()) + self.sa.add_module('',nn.Conv2d(channel//reduction, 1, kernel_size=1)) + def forward(self,x): + res = self.sa(x) + res = res.expand_as(x) + return res + +class BAMblock(nn.Module): + def __init__(self,channel=512, reduction=16, dia_val=2): + super(BAMblock, self).__init__() + self.ca = ChannelAttention(channel, reduction) + self.sa = SpatialAttention(channel,reduction,dia_val) + self.sigmoid = nn.Sigmoid() + + def init_weights(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + init.kaiming_normal(m.weight, mode='fan_out') + if m.bais is not None: + init.constant_(m.bias, 0) + elif isinstance(m, nn.BatchNorm2d): + init.constant_(m.weight, 1) + init.constant_(m.bias, 0) + elif isinstance(m, nn.Linear): + init.normal_(m.weight, std=0.001) + if m.bias is not None: + init.constant_(m.bias, 0) + + def forward(self,x): + b, c, _, _ = x.size() + sa_out=self.sa(x) + ca_out=self.ca(x) + weight=self.sigmoid(sa_out+ca_out) + out=(1+weight)*x + return out + +if __name__ =="__main__": + + print(512//14) \ No newline at end of file diff --git a/tracking/trackers/reid/model/CBAM.py b/tracking/trackers/reid/model/CBAM.py new file mode 100644 index 0000000..b0b0057 --- /dev/null +++ b/tracking/trackers/reid/model/CBAM.py @@ -0,0 +1,68 @@ +import torch +import torch.nn as nn +import torch.nn.init as init + +class channelAttention(nn.Module): + def __init__(self, channel, reduction=16): + super(channelAttention, self).__init__() + self.Maxpooling = nn.AdaptiveMaxPool2d(1) + self.Avepooling = nn.AdaptiveAvgPool2d(1) + self.ca = nn.Sequential() + self.ca.add_module('conv1',nn.Conv2d(channel, channel//reduction, 1, bias=False)) + self.ca.add_module('Relu', nn.ReLU()) + self.ca.add_module('conv2',nn.Conv2d(channel//reduction, channel, 1, bias=False)) + self.sigmod = nn.Sigmoid() + + def forward(self, x): + M_out = self.Maxpooling(x) + A_out = self.Avepooling(x) + M_out = self.ca(M_out) + A_out = self.ca(A_out) + out = self.sigmod(M_out+A_out) + return out + +class SpatialAttention(nn.Module): + def __init__(self, kernel_size=7): + super().__init__() + self.conv = nn.Conv2d(in_channels=2, out_channels=1, kernel_size=kernel_size, padding=kernel_size // 2) + self.sigmoid = nn.Sigmoid() + + def forward(self, x): + max_result, _ = torch.max(x, dim=1, keepdim=True) + avg_result = torch.mean(x, dim=1, keepdim=True) + result = torch.cat([max_result, avg_result], dim=1) + output = self.conv(result) + output = self.sigmoid(output) + return output +class CBAM(nn.Module): + def __init__(self, channel=512, reduction=16, kernel_size=7): + super().__init__() + self.ca = channelAttention(channel, reduction) + self.sa = SpatialAttention(kernel_size) + + def init_weights(self): + for m in self.modules():#权重初始化 + if isinstance(m, nn.Conv2d): + init.kaiming_normal_(m.weight, mode='fan_out') + if m.bias is not None: + init.constant_(m.bias, 0) + elif isinstance(m, nn.BatchNorm2d): + init.constant_(m.weight, 1) + init.constant_(m.bias, 0) + elif isinstance(m, nn.Linear): + init.normal_(m.weight, std=0.001) + if m.bias is not None: + init.constant_(m.bias, 0) + + def forward(self, x): + # b,c_,_ = x.size() + # residual = x + out = x*self.ca(x) + out = out*self.sa(out) + return out +if __name__ == '__main__': + input=torch.randn(50,512,7,7) + kernel_size=input.shape[2] + cbam = CBAM(channel=512,reduction=16,kernel_size=kernel_size) + output=cbam(input) + print(output.shape) diff --git a/tracking/trackers/reid/model/Tool.py b/tracking/trackers/reid/model/Tool.py new file mode 100644 index 0000000..378c11a --- /dev/null +++ b/tracking/trackers/reid/model/Tool.py @@ -0,0 +1,33 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +class GeM(nn.Module): + def __init__(self, p=3, eps=1e-6): + super(GeM, self).__init__() + self.p = nn.Parameter(torch.ones(1) * p) + self.eps = eps + + def forward(self, x): + return self.gem(x, p=self.p, eps=self.eps, stride = 2) + + def gem(self, x, p=3, eps=1e-6, stride = 2): + return F.avg_pool2d(x.clamp(min=eps).pow(p), (x.size(-2), x.size(-1)), stride=2).pow(1. / p) + + def __repr__(self): + return self.__class__.__name__ + \ + '(' + 'p=' + '{:.4f}'.format(self.p.data.tolist()[0]) + \ + ', ' + 'eps=' + str(self.eps) + ')' + +class TripletLoss(nn.Module): + def __init__(self, margin): + super(TripletLoss, self).__init__() + self.margin = margin + + def forward(self, anchor, positive, negative, size_average = True): + distance_positive = (anchor-positive).pow(2).sum(1) + distance_negative = (anchor-negative).pow(2).sum(1) + losses = F.relu(distance_negative-distance_positive+self.margin) + return losses.mean() if size_average else losses.sum() + +if __name__ == '__main__': + print('') diff --git a/tracking/trackers/reid/model/__init__.py b/tracking/trackers/reid/model/__init__.py new file mode 100644 index 0000000..b1ec69d --- /dev/null +++ b/tracking/trackers/reid/model/__init__.py @@ -0,0 +1,9 @@ +from .fmobilenet import FaceMobileNet +from .resnet_face import ResIRSE +from .mobilevit import mobilevit_s +from .metric import ArcFace, CosFace +from .loss import FocalLoss +from .resbam import resnet +from .resnet_pre import resnet18, resnet34, resnet50 +from .mobilenet_v2 import mobilenet_v2 +from .mobilenet_v3 import MobileNetV3_Small, MobileNetV3_Large \ No newline at end of file diff --git a/tracking/trackers/reid/model/__pycache__/CBAM.cpython-39.pyc b/tracking/trackers/reid/model/__pycache__/CBAM.cpython-39.pyc new file mode 100644 index 0000000..8cc2d54 Binary files /dev/null and b/tracking/trackers/reid/model/__pycache__/CBAM.cpython-39.pyc differ diff --git a/tracking/trackers/reid/model/__pycache__/Tool.cpython-39.pyc b/tracking/trackers/reid/model/__pycache__/Tool.cpython-39.pyc new file mode 100644 index 0000000..1bff1f8 Binary files /dev/null and b/tracking/trackers/reid/model/__pycache__/Tool.cpython-39.pyc differ diff --git a/tracking/trackers/reid/model/__pycache__/__init__.cpython-39.pyc b/tracking/trackers/reid/model/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..69869c1 Binary files /dev/null and b/tracking/trackers/reid/model/__pycache__/__init__.cpython-39.pyc differ diff --git a/tracking/trackers/reid/model/__pycache__/fmobilenet.cpython-39.pyc b/tracking/trackers/reid/model/__pycache__/fmobilenet.cpython-39.pyc new file mode 100644 index 0000000..169e3e3 Binary files /dev/null and b/tracking/trackers/reid/model/__pycache__/fmobilenet.cpython-39.pyc differ diff --git a/tracking/trackers/reid/model/__pycache__/loss.cpython-39.pyc b/tracking/trackers/reid/model/__pycache__/loss.cpython-39.pyc new file mode 100644 index 0000000..1c283e8 Binary files /dev/null and b/tracking/trackers/reid/model/__pycache__/loss.cpython-39.pyc differ diff --git a/tracking/trackers/reid/model/__pycache__/metric.cpython-39.pyc b/tracking/trackers/reid/model/__pycache__/metric.cpython-39.pyc new file mode 100644 index 0000000..6265652 Binary files /dev/null and b/tracking/trackers/reid/model/__pycache__/metric.cpython-39.pyc differ diff --git a/tracking/trackers/reid/model/__pycache__/mobilenet_v2.cpython-39.pyc b/tracking/trackers/reid/model/__pycache__/mobilenet_v2.cpython-39.pyc new file mode 100644 index 0000000..5bc976b Binary files /dev/null and b/tracking/trackers/reid/model/__pycache__/mobilenet_v2.cpython-39.pyc differ diff --git a/tracking/trackers/reid/model/__pycache__/mobilenet_v3.cpython-39.pyc b/tracking/trackers/reid/model/__pycache__/mobilenet_v3.cpython-39.pyc new file mode 100644 index 0000000..deac9cc Binary files /dev/null and b/tracking/trackers/reid/model/__pycache__/mobilenet_v3.cpython-39.pyc differ diff --git a/tracking/trackers/reid/model/__pycache__/mobilevit.cpython-39.pyc b/tracking/trackers/reid/model/__pycache__/mobilevit.cpython-39.pyc new file mode 100644 index 0000000..c2a98b7 Binary files /dev/null and b/tracking/trackers/reid/model/__pycache__/mobilevit.cpython-39.pyc differ diff --git a/tracking/trackers/reid/model/__pycache__/resbam.cpython-39.pyc b/tracking/trackers/reid/model/__pycache__/resbam.cpython-39.pyc new file mode 100644 index 0000000..989ae8c Binary files /dev/null and b/tracking/trackers/reid/model/__pycache__/resbam.cpython-39.pyc differ diff --git a/tracking/trackers/reid/model/__pycache__/resnet_face.cpython-39.pyc b/tracking/trackers/reid/model/__pycache__/resnet_face.cpython-39.pyc new file mode 100644 index 0000000..b38798f Binary files /dev/null and b/tracking/trackers/reid/model/__pycache__/resnet_face.cpython-39.pyc differ diff --git a/tracking/trackers/reid/model/__pycache__/resnet_pre.cpython-39.pyc b/tracking/trackers/reid/model/__pycache__/resnet_pre.cpython-39.pyc new file mode 100644 index 0000000..26fb649 Binary files /dev/null and b/tracking/trackers/reid/model/__pycache__/resnet_pre.cpython-39.pyc differ diff --git a/tracking/trackers/reid/model/__pycache__/utils.cpython-39.pyc b/tracking/trackers/reid/model/__pycache__/utils.cpython-39.pyc new file mode 100644 index 0000000..d1beae7 Binary files /dev/null and b/tracking/trackers/reid/model/__pycache__/utils.cpython-39.pyc differ diff --git a/tracking/trackers/reid/model/fmobilenet.py b/tracking/trackers/reid/model/fmobilenet.py new file mode 100644 index 0000000..2e38a44 --- /dev/null +++ b/tracking/trackers/reid/model/fmobilenet.py @@ -0,0 +1,124 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + + + +class Flatten(nn.Module): + def forward(self, x): + return x.view(x.shape[0], -1) + +class ConvBn(nn.Module): + + def __init__(self, in_c, out_c, kernel=(1, 1), stride=1, padding=0, groups=1): + super().__init__() + self.net = nn.Sequential( + nn.Conv2d(in_c, out_c, kernel, stride, padding, groups=groups, bias=False), + nn.BatchNorm2d(out_c) + ) + + def forward(self, x): + return self.net(x) + + +class ConvBnPrelu(nn.Module): + + def __init__(self, in_c, out_c, kernel=(1, 1), stride=1, padding=0, groups=1): + super().__init__() + self.net = nn.Sequential( + ConvBn(in_c, out_c, kernel, stride, padding, groups), + nn.PReLU(out_c) + ) + + def forward(self, x): + return self.net(x) + + +class DepthWise(nn.Module): + + def __init__(self, in_c, out_c, kernel=(3, 3), stride=2, padding=1, groups=1): + super().__init__() + self.net = nn.Sequential( + ConvBnPrelu(in_c, groups, kernel=(1, 1), stride=1, padding=0), + ConvBnPrelu(groups, groups, kernel=kernel, stride=stride, padding=padding, groups=groups), + ConvBn(groups, out_c, kernel=(1, 1), stride=1, padding=0), + ) + + def forward(self, x): + return self.net(x) + + +class DepthWiseRes(nn.Module): + """DepthWise with Residual""" + + def __init__(self, in_c, out_c, kernel=(3, 3), stride=2, padding=1, groups=1): + super().__init__() + self.net = DepthWise(in_c, out_c, kernel, stride, padding, groups) + + def forward(self, x): + return self.net(x) + x + + +class MultiDepthWiseRes(nn.Module): + + def __init__(self, num_block, channels, kernel=(3, 3), stride=1, padding=1, groups=1): + super().__init__() + + self.net = nn.Sequential(*[ + DepthWiseRes(channels, channels, kernel, stride, padding, groups) + for _ in range(num_block) + ]) + + def forward(self, x): + return self.net(x) + + +class FaceMobileNet(nn.Module): + + def __init__(self, embedding_size): + super().__init__() + self.conv1 = ConvBnPrelu(1, 64, kernel=(3, 3), stride=2, padding=1) + self.conv2 = ConvBn(64, 64, kernel=(3, 3), stride=1, padding=1, groups=64) + self.conv3 = DepthWise(64, 64, kernel=(3, 3), stride=2, padding=1, groups=128) + self.conv4 = MultiDepthWiseRes(num_block=4, channels=64, kernel=3, stride=1, padding=1, groups=128) + self.conv5 = DepthWise(64, 128, kernel=(3, 3), stride=2, padding=1, groups=256) + self.conv6 = MultiDepthWiseRes(num_block=6, channels=128, kernel=(3, 3), stride=1, padding=1, groups=256) + self.conv7 = DepthWise(128, 128, kernel=(3, 3), stride=2, padding=1, groups=512) + self.conv8 = MultiDepthWiseRes(num_block=2, channels=128, kernel=(3, 3), stride=1, padding=1, groups=256) + self.conv9 = ConvBnPrelu(128, 512, kernel=(1, 1)) + self.conv10 = ConvBn(512, 512, groups=512, kernel=(7, 7)) + self.flatten = Flatten() + self.linear = nn.Linear(2048, embedding_size, bias=False) + self.bn = nn.BatchNorm1d(embedding_size) + + def forward(self, x): + #print('x',x.shape) + out = self.conv1(x) + out = self.conv2(out) + out = self.conv3(out) + out = self.conv4(out) + out = self.conv5(out) + out = self.conv6(out) + out = self.conv7(out) + out = self.conv8(out) + out = self.conv9(out) + out = self.conv10(out) + out = self.flatten(out) + out = self.linear(out) + out = self.bn(out) + return out + +if __name__ == "__main__": + from PIL import Image + import numpy as np + + x = Image.open("../samples/009.jpg").convert('L') + x = x.resize((128, 128)) + x = np.asarray(x, dtype=np.float32) + x = x[None, None, ...] + x = torch.from_numpy(x) + net = FaceMobileNet(512) + net.eval() + with torch.no_grad(): + out = net(x) + print(out.shape) diff --git a/tracking/trackers/reid/model/loss.py b/tracking/trackers/reid/model/loss.py new file mode 100644 index 0000000..8f40c5c --- /dev/null +++ b/tracking/trackers/reid/model/loss.py @@ -0,0 +1,18 @@ +import torch +import torch.nn as nn + + +class FocalLoss(nn.Module): + + def __init__(self, gamma=2): + super().__init__() + self.gamma = gamma + self.ce = torch.nn.CrossEntropyLoss() + + def forward(self, input, target): + + #print(f'theta {input.shape, input[0]}, target {target.shape, target}') + logp = self.ce(input, target) + p = torch.exp(-logp) + loss = (1 - p) ** self.gamma * logp + return loss.mean() \ No newline at end of file diff --git a/tracking/trackers/reid/model/metric.py b/tracking/trackers/reid/model/metric.py new file mode 100644 index 0000000..b5b0189 --- /dev/null +++ b/tracking/trackers/reid/model/metric.py @@ -0,0 +1,83 @@ +# Definition of ArcFace loss and CosFace loss + +import math +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class ArcFace(nn.Module): + + def __init__(self, embedding_size, class_num, s=30.0, m=0.50): + """ArcFace formula: + cos(m + theta) = cos(m)cos(theta) - sin(m)sin(theta) + Note that: + 0 <= m + theta <= Pi + So if (m + theta) >= Pi, then theta >= Pi - m. In [0, Pi] + we have: + cos(theta) < cos(Pi - m) + So we can use cos(Pi - m) as threshold to check whether + (m + theta) go out of [0, Pi] + + Args: + embedding_size: usually 128, 256, 512 ... + class_num: num of people when training + s: scale, see normface https://arxiv.org/abs/1704.06369 + m: margin, see SphereFace, CosFace, and ArcFace paper + """ + super().__init__() + self.in_features = embedding_size + self.out_features = class_num + self.s = s + self.m = m + self.weight = nn.Parameter(torch.FloatTensor(class_num, embedding_size)) + nn.init.xavier_uniform_(self.weight) + + self.cos_m = math.cos(m) + self.sin_m = math.sin(m) + self.th = math.cos(math.pi - m) + self.mm = math.sin(math.pi - m) * m + + def forward(self, input, label): + #print(f"embding {self.in_features}, class_num {self.out_features}, input {len(input)}, label {len(label)}") + cosine = F.linear(F.normalize(input), F.normalize(self.weight)) + # print('F.normalize(input)',input.shape) + # print('F.normalize(self.weight)',F.normalize(self.weight).shape) + sine = ((1.0 - cosine.pow(2)).clamp(0, 1)).sqrt() + phi = cosine * self.cos_m - sine * self.sin_m + phi = torch.where(cosine > self.th, phi, cosine - self.mm) # drop to CosFace + #print(f'consine {cosine.shape, cosine}, sine {sine.shape, sine}, phi {phi.shape, phi}') + # update y_i by phi in cosine + output = cosine * 1.0 # make backward works + batch_size = len(output) + output[range(batch_size), label] = phi[range(batch_size), label] + # print(f'output {(output * self.s).shape}') + # print(f'phi[range(batch_size), label] {phi[range(batch_size), label]}') + return output * self.s + + +class CosFace(nn.Module): + + def __init__(self, in_features, out_features, s=30.0, m=0.40): + """ + Args: + embedding_size: usually 128, 256, 512 ... + class_num: num of people when training + s: scale, see normface https://arxiv.org/abs/1704.06369 + m: margin, see SphereFace, CosFace, and ArcFace paper + """ + super().__init__() + self.in_features = in_features + self.out_features = out_features + self.s = s + self.m = m + self.weight = nn.Parameter(torch.FloatTensor(out_features, in_features)) + nn.init.xavier_uniform_(self.weight) + + def forward(self, input, label): + cosine = F.linear(F.normalize(input), F.normalize(self.weight)) + phi = cosine - self.m + output = cosine * 1.0 # make backward works + batch_size = len(output) + output[range(batch_size), label] = phi[range(batch_size), label] + return output * self.s \ No newline at end of file diff --git a/tracking/trackers/reid/model/mobilenet_v2.py b/tracking/trackers/reid/model/mobilenet_v2.py new file mode 100644 index 0000000..eec44d4 --- /dev/null +++ b/tracking/trackers/reid/model/mobilenet_v2.py @@ -0,0 +1,200 @@ +from torch import nn +from .utils import load_state_dict_from_url +from ..config import config as conf + +__all__ = ['MobileNetV2', 'mobilenet_v2'] + + +model_urls = { + 'mobilenet_v2': 'https://download.pytorch.org/models/mobilenet_v2-b0353104.pth', +} + + +def _make_divisible(v, divisor, min_value=None): + """ + This function is taken from the original tf repo. + It ensures that all layers have a channel number that is divisible by 8 + It can be seen here: + https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py + :param v: + :param divisor: + :param min_value: + :return: + """ + if min_value is None: + min_value = divisor + new_v = max(min_value, int(v + divisor / 2) // divisor * divisor) + # Make sure that round down does not go down by more than 10%. + if new_v < 0.9 * v: + new_v += divisor + return new_v + + +class ConvBNReLU(nn.Sequential): + def __init__(self, in_planes, out_planes, kernel_size=3, stride=1, groups=1, norm_layer=None): + padding = (kernel_size - 1) // 2 + if norm_layer is None: + norm_layer = nn.BatchNorm2d + super(ConvBNReLU, self).__init__( + nn.Conv2d(in_planes, out_planes, kernel_size, stride, padding, groups=groups, bias=False), + norm_layer(out_planes), + nn.ReLU6(inplace=True) + ) + + +class InvertedResidual(nn.Module): + def __init__(self, inp, oup, stride, expand_ratio, norm_layer=None): + super(InvertedResidual, self).__init__() + self.stride = stride + assert stride in [1, 2] + + if norm_layer is None: + norm_layer = nn.BatchNorm2d + + hidden_dim = int(round(inp * expand_ratio)) + self.use_res_connect = self.stride == 1 and inp == oup + + layers = [] + if expand_ratio != 1: + # pw + layers.append(ConvBNReLU(inp, hidden_dim, kernel_size=1, norm_layer=norm_layer)) + layers.extend([ + # dw + ConvBNReLU(hidden_dim, hidden_dim, stride=stride, groups=hidden_dim, norm_layer=norm_layer), + # pw-linear + nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False), + norm_layer(oup), + ]) + self.conv = nn.Sequential(*layers) + + def forward(self, x): + if self.use_res_connect: + return x + self.conv(x) + else: + return self.conv(x) + + +class MobileNetV2(nn.Module): + def __init__(self, + num_classes=conf.embedding_size, + width_mult=1.0, + inverted_residual_setting=None, + round_nearest=8, + block=None, + norm_layer=None): + """ + MobileNet V2 main class + + Args: + num_classes (int): Number of classes + width_mult (float): Width multiplier - adjusts number of channels in each layer by this amount + inverted_residual_setting: Network structure + round_nearest (int): Round the number of channels in each layer to be a multiple of this number + Set to 1 to turn off rounding + block: Module specifying inverted residual building block for mobilenet + norm_layer: Module specifying the normalization layer to use + + """ + super(MobileNetV2, self).__init__() + + if block is None: + block = InvertedResidual + + if norm_layer is None: + norm_layer = nn.BatchNorm2d + + input_channel = 32 + last_channel = 1280 + + if inverted_residual_setting is None: + inverted_residual_setting = [ + # t, c, n, s + [1, 16, 1, 1], + [6, 24, 2, 2], + [6, 32, 3, 2], + [6, 64, 4, 2], + [6, 96, 3, 1], + [6, 160, 3, 2], + [6, 320, 1, 1], + ] + + # only check the first element, assuming user knows t,c,n,s are required + if len(inverted_residual_setting) == 0 or len(inverted_residual_setting[0]) != 4: + raise ValueError("inverted_residual_setting should be non-empty " + "or a 4-element list, got {}".format(inverted_residual_setting)) + + # building first layer + input_channel = _make_divisible(input_channel * width_mult, round_nearest) + self.last_channel = _make_divisible(last_channel * max(1.0, width_mult), round_nearest) + features = [ConvBNReLU(3, input_channel, stride=2, norm_layer=norm_layer)] + # building inverted residual blocks + for t, c, n, s in inverted_residual_setting: + output_channel = _make_divisible(c * width_mult, round_nearest) + for i in range(n): + stride = s if i == 0 else 1 + features.append(block(input_channel, output_channel, stride, expand_ratio=t, norm_layer=norm_layer)) + input_channel = output_channel + # building last several layers + features.append(ConvBNReLU(input_channel, self.last_channel, kernel_size=1, norm_layer=norm_layer)) + # make it nn.Sequential + self.features = nn.Sequential(*features) + + # building classifier + self.classifier = nn.Sequential( + nn.Dropout(0.2), + nn.Linear(self.last_channel, num_classes), + ) + + # weight initialization + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out') + if m.bias is not None: + nn.init.zeros_(m.bias) + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.ones_(m.weight) + nn.init.zeros_(m.bias) + elif isinstance(m, nn.Linear): + nn.init.normal_(m.weight, 0, 0.01) + nn.init.zeros_(m.bias) + + def _forward_impl(self, x): + # This exists since TorchScript doesn't support inheritance, so the superclass method + # (this one) needs to have a name other than `forward` that can be accessed in a subclass + x = self.features(x) + # Cannot use "squeeze" as batch-size can be 1 => must use reshape with x.shape[0] + x = nn.functional.adaptive_avg_pool2d(x, 1).reshape(x.shape[0], -1) + x = self.classifier(x) + return x + + def forward(self, x): + return self._forward_impl(x) + + +def mobilenet_v2(pretrained=True, progress=True, **kwargs): + """ + Constructs a MobileNetV2 architecture from + `"MobileNetV2: Inverted Residuals and Linear Bottlenecks" `_. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + model = MobileNetV2(**kwargs) + if pretrained: + state_dict = load_state_dict_from_url(model_urls['mobilenet_v2'], + progress=progress) + src_state_dict = state_dict + target_state_dict = model.state_dict() + skip_keys = [] + # skip mismatch size tensors in case of pretraining + for k in src_state_dict.keys(): + if k not in target_state_dict: + continue + if src_state_dict[k].size() != target_state_dict[k].size(): + skip_keys.append(k) + for k in skip_keys: + del src_state_dict[k] + missing_keys, unexpected_keys = model.load_state_dict(src_state_dict, strict=False) + #.load_state_dict(state_dict) + return model diff --git a/tracking/trackers/reid/model/mobilenet_v3.py b/tracking/trackers/reid/model/mobilenet_v3.py new file mode 100644 index 0000000..953db51 --- /dev/null +++ b/tracking/trackers/reid/model/mobilenet_v3.py @@ -0,0 +1,200 @@ +'''MobileNetV3 in PyTorch. + +See the paper "Inverted Residuals and Linear Bottlenecks: +Mobile Networks for Classification, Detection and Segmentation" for more details. +''' +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn import init +from ..config import config as conf + + +class hswish(nn.Module): + def forward(self, x): + out = x * F.relu6(x + 3, inplace=True) / 6 + return out + + +class hsigmoid(nn.Module): + def forward(self, x): + out = F.relu6(x + 3, inplace=True) / 6 + return out + + +class SeModule(nn.Module): + def __init__(self, in_size, reduction=4): + super(SeModule, self).__init__() + self.se = nn.Sequential( + nn.AdaptiveAvgPool2d(1), + nn.Conv2d(in_size, in_size // reduction, kernel_size=1, stride=1, padding=0, bias=False), + nn.BatchNorm2d(in_size // reduction), + nn.ReLU(inplace=True), + nn.Conv2d(in_size // reduction, in_size, kernel_size=1, stride=1, padding=0, bias=False), + nn.BatchNorm2d(in_size), + hsigmoid() + ) + + def forward(self, x): + return x * self.se(x) + + +class Block(nn.Module): + '''expand + depthwise + pointwise''' + def __init__(self, kernel_size, in_size, expand_size, out_size, nolinear, semodule, stride): + super(Block, self).__init__() + self.stride = stride + self.se = semodule + + self.conv1 = nn.Conv2d(in_size, expand_size, kernel_size=1, stride=1, padding=0, bias=False) + self.bn1 = nn.BatchNorm2d(expand_size) + self.nolinear1 = nolinear + self.conv2 = nn.Conv2d(expand_size, expand_size, kernel_size=kernel_size, stride=stride, padding=kernel_size//2, groups=expand_size, bias=False) + self.bn2 = nn.BatchNorm2d(expand_size) + self.nolinear2 = nolinear + self.conv3 = nn.Conv2d(expand_size, out_size, kernel_size=1, stride=1, padding=0, bias=False) + self.bn3 = nn.BatchNorm2d(out_size) + + self.shortcut = nn.Sequential() + if stride == 1 and in_size != out_size: + self.shortcut = nn.Sequential( + nn.Conv2d(in_size, out_size, kernel_size=1, stride=1, padding=0, bias=False), + nn.BatchNorm2d(out_size), + ) + + def forward(self, x): + out = self.nolinear1(self.bn1(self.conv1(x))) + out = self.nolinear2(self.bn2(self.conv2(out))) + out = self.bn3(self.conv3(out)) + if self.se != None: + out = self.se(out) + out = out + self.shortcut(x) if self.stride==1 else out + return out + + +class MobileNetV3_Large(nn.Module): + def __init__(self, num_classes=conf.embedding_size): + super(MobileNetV3_Large, self).__init__() + self.conv1 = nn.Conv2d(3, 16, kernel_size=3, stride=2, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(16) + self.hs1 = hswish() + + self.bneck = nn.Sequential( + Block(3, 16, 16, 16, nn.ReLU(inplace=True), None, 1), + Block(3, 16, 64, 24, nn.ReLU(inplace=True), None, 2), + Block(3, 24, 72, 24, nn.ReLU(inplace=True), None, 1), + Block(5, 24, 72, 40, nn.ReLU(inplace=True), SeModule(40), 2), + Block(5, 40, 120, 40, nn.ReLU(inplace=True), SeModule(40), 1), + Block(5, 40, 120, 40, nn.ReLU(inplace=True), SeModule(40), 1), + Block(3, 40, 240, 80, hswish(), None, 2), + Block(3, 80, 200, 80, hswish(), None, 1), + Block(3, 80, 184, 80, hswish(), None, 1), + Block(3, 80, 184, 80, hswish(), None, 1), + Block(3, 80, 480, 112, hswish(), SeModule(112), 1), + Block(3, 112, 672, 112, hswish(), SeModule(112), 1), + Block(5, 112, 672, 160, hswish(), SeModule(160), 1), + Block(5, 160, 672, 160, hswish(), SeModule(160), 2), + Block(5, 160, 960, 160, hswish(), SeModule(160), 1), + ) + + + self.conv2 = nn.Conv2d(160, 960, kernel_size=1, stride=1, padding=0, bias=False) + self.bn2 = nn.BatchNorm2d(960) + self.hs2 = hswish() + self.linear3 = nn.Linear(960, 1280) + self.bn3 = nn.BatchNorm1d(1280) + self.hs3 = hswish() + self.linear4 = nn.Linear(1280, num_classes) + self.init_params() + + def init_params(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + init.kaiming_normal_(m.weight, mode='fan_out') + if m.bias is not None: + init.constant_(m.bias, 0) + elif isinstance(m, nn.BatchNorm2d): + init.constant_(m.weight, 1) + init.constant_(m.bias, 0) + elif isinstance(m, nn.Linear): + init.normal_(m.weight, std=0.001) + if m.bias is not None: + init.constant_(m.bias, 0) + + def forward(self, x): + out = self.hs1(self.bn1(self.conv1(x))) + out = self.bneck(out) + out = self.hs2(self.bn2(self.conv2(out))) + out = F.avg_pool2d(out, conf.img_size // 32) + out = out.view(out.size(0), -1) + out = self.hs3(self.bn3(self.linear3(out))) + out = self.linear4(out) + return out + + + +class MobileNetV3_Small(nn.Module): + def __init__(self, num_classes=conf.embedding_size): + super(MobileNetV3_Small, self).__init__() + self.conv1 = nn.Conv2d(3, 16, kernel_size=3, stride=2, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(16) + self.hs1 = hswish() + + self.bneck = nn.Sequential( + Block(3, 16, 16, 16, nn.ReLU(inplace=True), SeModule(16), 2), + Block(3, 16, 72, 24, nn.ReLU(inplace=True), None, 2), + Block(3, 24, 88, 24, nn.ReLU(inplace=True), None, 1), + Block(5, 24, 96, 40, hswish(), SeModule(40), 2), + Block(5, 40, 240, 40, hswish(), SeModule(40), 1), + Block(5, 40, 240, 40, hswish(), SeModule(40), 1), + Block(5, 40, 120, 48, hswish(), SeModule(48), 1), + Block(5, 48, 144, 48, hswish(), SeModule(48), 1), + Block(5, 48, 288, 96, hswish(), SeModule(96), 2), + Block(5, 96, 576, 96, hswish(), SeModule(96), 1), + Block(5, 96, 576, 96, hswish(), SeModule(96), 1), + ) + + + self.conv2 = nn.Conv2d(96, 576, kernel_size=1, stride=1, padding=0, bias=False) + self.bn2 = nn.BatchNorm2d(576) + self.hs2 = hswish() + self.linear3 = nn.Linear(576, 1280) + self.bn3 = nn.BatchNorm1d(1280) + self.hs3 = hswish() + self.linear4 = nn.Linear(1280, num_classes) + self.init_params() + + def init_params(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + init.kaiming_normal_(m.weight, mode='fan_out') + if m.bias is not None: + init.constant_(m.bias, 0) + elif isinstance(m, nn.BatchNorm2d): + init.constant_(m.weight, 1) + init.constant_(m.bias, 0) + elif isinstance(m, nn.Linear): + init.normal_(m.weight, std=0.001) + if m.bias is not None: + init.constant_(m.bias, 0) + + def forward(self, x): + out = self.hs1(self.bn1(self.conv1(x))) + out = self.bneck(out) + out = self.hs2(self.bn2(self.conv2(out))) + out = F.avg_pool2d(out, conf.img_size // 32) + out = out.view(out.size(0), -1) + + out = self.hs3(self.bn3(self.linear3(out))) + out = self.linear4(out) + return out + + + +def test(): + net = MobileNetV3_Small() + x = torch.randn(2,3,224,224) + y = net(x) + print(y.size()) + +# test() \ No newline at end of file diff --git a/tracking/trackers/reid/model/mobilevit.py b/tracking/trackers/reid/model/mobilevit.py new file mode 100644 index 0000000..59ad7df --- /dev/null +++ b/tracking/trackers/reid/model/mobilevit.py @@ -0,0 +1,265 @@ +import torch +import torch.nn as nn + +from einops import rearrange +from ..config import config as conf + + +def conv_1x1_bn(inp, oup): + return nn.Sequential( + nn.Conv2d(inp, oup, 1, 1, 0, bias=False), + nn.BatchNorm2d(oup), + nn.SiLU() + ) + + +def conv_nxn_bn(inp, oup, kernal_size=3, stride=1): + return nn.Sequential( + nn.Conv2d(inp, oup, kernal_size, stride, 1, bias=False), + nn.BatchNorm2d(oup), + nn.SiLU() + ) + + +class PreNorm(nn.Module): + def __init__(self, dim, fn): + super().__init__() + self.norm = nn.LayerNorm(dim) + self.fn = fn + + def forward(self, x, **kwargs): + return self.fn(self.norm(x), **kwargs) + + +class FeedForward(nn.Module): + def __init__(self, dim, hidden_dim, dropout=0.): + super().__init__() + self.net = nn.Sequential( + nn.Linear(dim, hidden_dim), + nn.SiLU(), + nn.Dropout(dropout), + nn.Linear(hidden_dim, dim), + nn.Dropout(dropout) + ) + + def forward(self, x): + return self.net(x) + + +class Attention(nn.Module): + def __init__(self, dim, heads=8, dim_head=64, dropout=0.): + super().__init__() + inner_dim = dim_head * heads + project_out = not (heads == 1 and dim_head == dim) + + self.heads = heads + self.scale = dim_head ** -0.5 + + self.attend = nn.Softmax(dim=-1) + self.to_qkv = nn.Linear(dim, inner_dim * 3, bias=False) + + self.to_out = nn.Sequential( + nn.Linear(inner_dim, dim), + nn.Dropout(dropout) + ) if project_out else nn.Identity() + + def forward(self, x): + qkv = self.to_qkv(x).chunk(3, dim=-1) + q, k, v = map(lambda t: rearrange(t, 'b p n (h d) -> b p h n d', h=self.heads), qkv) + + dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale + attn = self.attend(dots) + out = torch.matmul(attn, v) + out = rearrange(out, 'b p h n d -> b p n (h d)') + return self.to_out(out) + + +class Transformer(nn.Module): + def __init__(self, dim, depth, heads, dim_head, mlp_dim, dropout=0.): + super().__init__() + self.layers = nn.ModuleList([]) + for _ in range(depth): + self.layers.append(nn.ModuleList([ + PreNorm(dim, Attention(dim, heads, dim_head, dropout)), + PreNorm(dim, FeedForward(dim, mlp_dim, dropout)) + ])) + + def forward(self, x): + for attn, ff in self.layers: + x = attn(x) + x + x = ff(x) + x + return x + + +class MV2Block(nn.Module): + def __init__(self, inp, oup, stride=1, expansion=4): + super().__init__() + self.stride = stride + assert stride in [1, 2] + + hidden_dim = int(inp * expansion) + self.use_res_connect = self.stride == 1 and inp == oup + + if expansion == 1: + self.conv = nn.Sequential( + # dw + nn.Conv2d(hidden_dim, hidden_dim, 3, stride, 1, groups=hidden_dim, bias=False), + nn.BatchNorm2d(hidden_dim), + nn.SiLU(), + # pw-linear + nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False), + nn.BatchNorm2d(oup), + ) + else: + self.conv = nn.Sequential( + # pw + nn.Conv2d(inp, hidden_dim, 1, 1, 0, bias=False), + nn.BatchNorm2d(hidden_dim), + nn.SiLU(), + # dw + nn.Conv2d(hidden_dim, hidden_dim, 3, stride, 1, groups=hidden_dim, bias=False), + nn.BatchNorm2d(hidden_dim), + nn.SiLU(), + # pw-linear + nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False), + nn.BatchNorm2d(oup), + ) + + def forward(self, x): + if self.use_res_connect: + return x + self.conv(x) + else: + return self.conv(x) + + +class MobileViTBlock(nn.Module): + def __init__(self, dim, depth, channel, kernel_size, patch_size, mlp_dim, dropout=0.): + super().__init__() + self.ph, self.pw = patch_size + + self.conv1 = conv_nxn_bn(channel, channel, kernel_size) + self.conv2 = conv_1x1_bn(channel, dim) + + self.transformer = Transformer(dim, depth, 4, 8, mlp_dim, dropout) + + self.conv3 = conv_1x1_bn(dim, channel) + self.conv4 = conv_nxn_bn(2 * channel, channel, kernel_size) + + def forward(self, x): + y = x.clone() + + # Local representations + x = self.conv1(x) + x = self.conv2(x) + + # Global representations + _, _, h, w = x.shape + x = rearrange(x, 'b d (h ph) (w pw) -> b (ph pw) (h w) d', ph=self.ph, pw=self.pw) + x = self.transformer(x) + x = rearrange(x, 'b (ph pw) (h w) d -> b d (h ph) (w pw)', h=h // self.ph, w=w // self.pw, ph=self.ph, + pw=self.pw) + + # Fusion + x = self.conv3(x) + x = torch.cat((x, y), 1) + x = self.conv4(x) + return x + + +class MobileViT(nn.Module): + def __init__(self, image_size, dims, channels, num_classes, expansion=4, kernel_size=3, patch_size=(2, 2)): + super().__init__() + ih, iw = image_size + ph, pw = patch_size + assert ih % ph == 0 and iw % pw == 0 + + L = [2, 4, 3] + + self.conv1 = conv_nxn_bn(3, channels[0], stride=2) + + self.mv2 = nn.ModuleList([]) + self.mv2.append(MV2Block(channels[0], channels[1], 1, expansion)) + self.mv2.append(MV2Block(channels[1], channels[2], 2, expansion)) + self.mv2.append(MV2Block(channels[2], channels[3], 1, expansion)) + self.mv2.append(MV2Block(channels[2], channels[3], 1, expansion)) # Repeat + self.mv2.append(MV2Block(channels[3], channels[4], 2, expansion)) + self.mv2.append(MV2Block(channels[5], channels[6], 2, expansion)) + self.mv2.append(MV2Block(channels[7], channels[8], 2, expansion)) + + self.mvit = nn.ModuleList([]) + self.mvit.append(MobileViTBlock(dims[0], L[0], channels[5], kernel_size, patch_size, int(dims[0] * 2))) + self.mvit.append(MobileViTBlock(dims[1], L[1], channels[7], kernel_size, patch_size, int(dims[1] * 4))) + self.mvit.append(MobileViTBlock(dims[2], L[2], channels[9], kernel_size, patch_size, int(dims[2] * 4))) + + self.conv2 = conv_1x1_bn(channels[-2], channels[-1]) + + self.pool = nn.AvgPool2d(ih // 32, 1) + self.fc = nn.Linear(channels[-1], num_classes, bias=False) + + def forward(self, x): + #print('x',x.shape) + x = self.conv1(x) + x = self.mv2[0](x) + + x = self.mv2[1](x) + x = self.mv2[2](x) + x = self.mv2[3](x) # Repeat + + x = self.mv2[4](x) + x = self.mvit[0](x) + + x = self.mv2[5](x) + x = self.mvit[1](x) + + x = self.mv2[6](x) + x = self.mvit[2](x) + x = self.conv2(x) + + + #print('pool_before',x.shape) + x = self.pool(x).view(-1, x.shape[1]) + #print('self_pool',self.pool) + #print('pool_after',x.shape) + x = self.fc(x) + return x + + +def mobilevit_xxs(): + dims = [64, 80, 96] + channels = [16, 16, 24, 24, 48, 48, 64, 64, 80, 80, 320] + return MobileViT((256, 256), dims, channels, num_classes=1000, expansion=2) + + +def mobilevit_xs(): + dims = [96, 120, 144] + channels = [16, 32, 48, 48, 64, 64, 80, 80, 96, 96, 384] + return MobileViT((256, 256), dims, channels, num_classes=1000) + + +def mobilevit_s(): + dims = [144, 192, 240] + channels = [16, 32, 64, 64, 96, 96, 128, 128, 160, 160, 640] + return MobileViT((conf.img_size, conf.img_size), dims, channels, num_classes=conf.embedding_size) + + +def count_parameters(model): + return sum(p.numel() for p in model.parameters() if p.requires_grad) + + +if __name__ == '__main__': + img = torch.randn(5, 3, 256, 256) + + vit = mobilevit_xxs() + out = vit(img) + print(out.shape) + print(count_parameters(vit)) + + vit = mobilevit_xs() + out = vit(img) + print(out.shape) + print(count_parameters(vit)) + + vit = mobilevit_s() + out = vit(img) + print(out.shape) + print(count_parameters(vit)) diff --git a/tracking/trackers/reid/model/resbam.py b/tracking/trackers/reid/model/resbam.py new file mode 100644 index 0000000..a407f93 --- /dev/null +++ b/tracking/trackers/reid/model/resbam.py @@ -0,0 +1,134 @@ +from .CBAM import CBAM +import torch +import torch.nn as nn +from .Tool import GeM as gem + +class Bottleneck(nn.Module): + expansion = 4 + def __init__(self, inchannel, outchannel,stride =1,dowsample=None): + # super(Bottleneck, self).__init__() + super().__init__() + self.conv1 = nn.Conv2d(in_channels=inchannel,out_channels=outchannel, kernel_size=1, stride=1, bias=False) + self.bn1 = nn.BatchNorm2d(outchannel) + self.conv2 = nn.Conv2d(in_channels=outchannel, out_channels=outchannel,kernel_size=3,bias=False, stride=stride,padding=1) + self.bn2 = nn.BatchNorm2d(outchannel) + self.conv3 =nn.Conv2d(in_channels=outchannel, out_channels=outchannel*self.expansion,stride=1,bias=False,kernel_size=1) + self.bn3 = nn.BatchNorm2d(outchannel*self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = dowsample + + def forward(self, x): + self.identity = x + # print('>>>>>>>>',type(x)) + if self.downsample is not None: + # print('>>>>downsample>>>>', type(self.downsample)) + self.identity = self.downsample(x) + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + out = self.conv3(out) + out = self.bn3(out) + # print('>>>>out>>>identity',out.size(),self.identity.size()) + out = out+self.identity + out = self.relu(out) + return out + +class resnet(nn.Module): + def __init__(self,block=Bottleneck, block_num=[3,4,6,3], num_class=1000): + super().__init__() + self.in_channel = 64 + self.conv1 = nn.Conv2d(in_channels=3, + out_channels=self.in_channel, + stride=2, + kernel_size=7, + padding=3, + bias=False) + self.bn1 = nn.BatchNorm2d(self.in_channel) + self.relu = nn.ReLU(inplace=True) + self.cbam = CBAM(self.in_channel) + self.cbam1 = CBAM(2048) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, 64, block_num[0],stride=1) + self.layer2 = self._make_layer(block, 128, block_num[1],stride=2) + self.layer3 = self._make_layer(block, 256, block_num[2],stride=2) + self.layer4 = self._make_layer(block, 512, block_num[3],stride=2) + self.avgpool = nn.AdaptiveAvgPool2d((1,1)) + self.gem = gem() + self.fc = nn.Linear(512*block.expansion, num_class) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal(m.weight,mode = 'fan_out', + nonlinearity='relu') + if isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1.0) + nn.init.constant_(m.bias, 1.0) + + def _make_layer(self,block ,channel, block_num, stride=1): + downsample = None + if stride !=1 or self.in_channel != channel*block.expansion: + downsample = nn.Sequential( + nn.Conv2d(self.in_channel, channel*block.expansion,kernel_size=1,stride=stride,bias=False), + nn.BatchNorm2d(channel*block.expansion)) + layer = [] + layer.append(block(self.in_channel, channel, stride, downsample)) + self.in_channel = channel*block.expansion + for _ in range(1, block_num): + layer.append(block(self.in_channel, channel)) + return nn.Sequential(*layer) + + def forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + x = self.cbam(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.cbam1(x) + # x = self.avgpool(x) + x = self.gem(x) + x = torch.flatten(x, 1) + x = self.fc(x) + return x + +class TripletNet(nn.Module): + def __init__(self, num_class, flag=True): + super(TripletNet, self).__init__() + self.initnet = rescbam(num_class) + self.flag = flag + + def forward(self, x1, x2=None, x3=None): + if self.flag: + output1 = self.initnet(x1) + output2 = self.initnet(x2) + output3 = self.initnet(x3) + return output1, output2, output3 + else: + output = self.initnet(x1) + return output + +def rescbam(num_class): + return resnet(block=Bottleneck, block_num=[3,4,6,3],num_class=num_class) + +if __name__ =='__main__': + input1 = torch.randn(4,3,640,640) + input2 = torch.randn(4,3,640,640) + input3 = torch.randn(4,3,640,640) + + #rescbam测试 + # Resnet50 = rescbam(512) + # output = Resnet50.forward(input1) + # print(Resnet50) + + #trnet测试 + trnet = TripletNet(512) + output = trnet(input1, input2, input3) + print(output) diff --git a/tracking/trackers/reid/model/resnet.py b/tracking/trackers/reid/model/resnet.py new file mode 100644 index 0000000..39e5024 --- /dev/null +++ b/tracking/trackers/reid/model/resnet.py @@ -0,0 +1,182 @@ +"""resnet in pytorch + + + +[1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun. + + Deep Residual Learning for Image Recognition + https://arxiv.org/abs/1512.03385v1 +""" + +import torch +import torch.nn as nn +from config import config as conf + +class BasicBlock(nn.Module): + """Basic Block for resnet 18 and resnet 34 + + """ + + #BasicBlock and BottleNeck block + #have different output size + #we use class attribute expansion + #to distinct + expansion = 1 + + def __init__(self, in_channels, out_channels, stride=1): + super().__init__() + + #residual function + self.residual_function = nn.Sequential( + nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1, bias=False), + nn.BatchNorm2d(out_channels), + nn.ReLU(inplace=True), + nn.Conv2d(out_channels, out_channels * BasicBlock.expansion, kernel_size=3, padding=1, bias=False), + nn.BatchNorm2d(out_channels * BasicBlock.expansion) + ) + + #shortcut + self.shortcut = nn.Sequential() + + #the shortcut output dimension is not the same with residual function + #use 1*1 convolution to match the dimension + if stride != 1 or in_channels != BasicBlock.expansion * out_channels: + self.shortcut = nn.Sequential( + nn.Conv2d(in_channels, out_channels * BasicBlock.expansion, kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(out_channels * BasicBlock.expansion) + ) + + def forward(self, x): + return nn.ReLU(inplace=True)(self.residual_function(x) + self.shortcut(x)) + +class BottleNeck(nn.Module): + """Residual block for resnet over 50 layers + + """ + expansion = 4 + def __init__(self, in_channels, out_channels, stride=1): + super().__init__() + self.residual_function = nn.Sequential( + nn.Conv2d(in_channels, out_channels, kernel_size=1, bias=False), + nn.BatchNorm2d(out_channels), + nn.ReLU(inplace=True), + nn.Conv2d(out_channels, out_channels, stride=stride, kernel_size=3, padding=1, bias=False), + nn.BatchNorm2d(out_channels), + nn.ReLU(inplace=True), + nn.Conv2d(out_channels, out_channels * BottleNeck.expansion, kernel_size=1, bias=False), + nn.BatchNorm2d(out_channels * BottleNeck.expansion), + ) + + self.shortcut = nn.Sequential() + + if stride != 1 or in_channels != out_channels * BottleNeck.expansion: + self.shortcut = nn.Sequential( + nn.Conv2d(in_channels, out_channels * BottleNeck.expansion, stride=stride, kernel_size=1, bias=False), + nn.BatchNorm2d(out_channels * BottleNeck.expansion) + ) + + def forward(self, x): + return nn.ReLU(inplace=True)(self.residual_function(x) + self.shortcut(x)) + +class ResNet(nn.Module): + + def __init__(self, block, num_block, num_classes=conf.embedding_size): + super().__init__() + + self.in_channels = 64 + + # self.conv1 = nn.Sequential( + # nn.Conv2d(3, 64, kernel_size=3, padding=1, bias=False), + # nn.BatchNorm2d(64), + # nn.ReLU(inplace=True)) + + self.conv1 = nn.Sequential( + nn.Conv2d(3, 64,stride=2,kernel_size=7,padding=3,bias=False), + nn.BatchNorm2d(64), + nn.ReLU(inplace=True), + nn.MaxPool2d(kernel_size=3, stride=2, padding=1)) + + + #we use a different inputsize than the original paper + #so conv2_x's stride is 1 + self.conv2_x = self._make_layer(block, 64, num_block[0], 1) + self.conv3_x = self._make_layer(block, 128, num_block[1], 2) + self.conv4_x = self._make_layer(block, 256, num_block[2], 2) + self.conv5_x = self._make_layer(block, 512, num_block[3], 2) + self.avg_pool = nn.AdaptiveAvgPool2d((1, 1)) + self.fc = nn.Linear(512 * block.expansion, num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal(m.weight,mode = 'fan_out', + nonlinearity='relu') + if isinstance(m, (nn.BatchNorm2d)): + nn.init.constant_(m.weight, 1.0) + nn.init.constant_(m.bias, 1.0) + + def _make_layer(self, block, out_channels, num_blocks, stride): + """make resnet layers(by layer i didnt mean this 'layer' was the + same as a neuron netowork layer, ex. conv layer), one layer may + contain more than one residual block + + Args: + block: block type, basic block or bottle neck block + out_channels: output depth channel number of this layer + num_blocks: how many blocks per layer + stride: the stride of the first block of this layer + + Return: + return a resnet layer + """ + + # we have num_block blocks per layer, the first block + # could be 1 or 2, other blocks would always be 1 + strides = [stride] + [1] * (num_blocks - 1) + layers = [] + for stride in strides: + layers.append(block(self.in_channels, out_channels, stride)) + self.in_channels = out_channels * block.expansion + + return nn.Sequential(*layers) + + def forward(self, x): + output = self.conv1(x) + output = self.conv2_x(output) + output = self.conv3_x(output) + output = self.conv4_x(output) + output = self.conv5_x(output) + print('pollBefore',output.shape) + output = self.avg_pool(output) + print('poolAfter',output.shape) + output = output.view(output.size(0), -1) + print('fcBefore',output.shape) + output = self.fc(output) + + return output + +def resnet18(): + """ return a ResNet 18 object + """ + return ResNet(BasicBlock, [2, 2, 2, 2]) + +def resnet34(): + """ return a ResNet 34 object + """ + return ResNet(BasicBlock, [3, 4, 6, 3]) + +def resnet50(): + """ return a ResNet 50 object + """ + return ResNet(BottleNeck, [3, 4, 6, 3]) + +def resnet101(): + """ return a ResNet 101 object + """ + return ResNet(BottleNeck, [3, 4, 23, 3]) + +def resnet152(): + """ return a ResNet 152 object + """ + return ResNet(BottleNeck, [3, 8, 36, 3]) + + diff --git a/tracking/trackers/reid/model/resnet_face.py b/tracking/trackers/reid/model/resnet_face.py new file mode 100644 index 0000000..e605d25 --- /dev/null +++ b/tracking/trackers/reid/model/resnet_face.py @@ -0,0 +1,120 @@ +""" Resnet_IR_SE in ArcFace """ + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class Flatten(nn.Module): + def forward(self, x): + return x.reshape(x.shape[0], -1) + + +class SEConv(nn.Module): + """Use Convolution instead of FullyConnection in SE""" + + def __init__(self, channels, reduction): + super().__init__() + self.net = nn.Sequential( + nn.AdaptiveAvgPool2d(1), + nn.Conv2d(channels, channels // reduction, kernel_size=1, bias=False), + nn.ReLU(inplace=True), + nn.Conv2d(channels // reduction, channels, kernel_size=1, bias=False), + nn.Sigmoid(), + ) + + def forward(self, x): + return self.net(x) * x + + +class SE(nn.Module): + + def __init__(self, channels, reduction): + super().__init__() + self.net = nn.Sequential( + nn.AdaptiveAvgPool2d(1), + nn.Linear(channels, channels // reduction), + nn.ReLU(inplace=True), + nn.Linear(channels // reduction, channels), + nn.Sigmoid(), + ) + + def forward(self, x): + return self.net(x) * x + + +class IRSE(nn.Module): + + def __init__(self, channels, depth, stride): + super().__init__() + if channels == depth: + self.shortcut = nn.MaxPool2d(kernel_size=1, stride=stride) + else: + self.shortcut = nn.Sequential( + nn.Conv2d(channels, depth, (1, 1), stride, bias=False), + nn.BatchNorm2d(depth), + ) + self.residual = nn.Sequential( + nn.BatchNorm2d(channels), + nn.Conv2d(channels, depth, (3, 3), 1, 1, bias=False), + nn.PReLU(depth), + nn.Conv2d(depth, depth, (3, 3), stride, 1, bias=False), + nn.BatchNorm2d(depth), + SEConv(depth, 16), + ) + + def forward(self, x): + return self.shortcut(x) + self.residual(x) + +class ResIRSE(nn.Module): + """Resnet50-IRSE backbone""" + + def __init__(self, ih,embedding_size, drop_ratio): + super().__init__() + ih_last = ih // 16 + self.input_layer = nn.Sequential( + nn.Conv2d(3, 64, (3, 3), 1, 1, bias=False), + nn.BatchNorm2d(64), + nn.PReLU(64), + ) + self.output_layer = nn.Sequential( + nn.BatchNorm2d(512), + nn.Dropout(drop_ratio), + Flatten(), + nn.Linear(512 * ih_last * ih_last, embedding_size), + nn.BatchNorm1d(embedding_size), + ) + + # ["channels", "depth", "stride"], + self.res50_arch = [ + [64, 64, 2], [64, 64, 1], [64, 64, 1], + [64, 128, 2], [128, 128, 1], [128, 128, 1], [128, 128, 1], + [128, 256, 2], [256, 256, 1], [256, 256, 1], [256, 256, 1], [256, 256, 1], + [256, 256, 1], [256, 256, 1], [256, 256, 1], [256, 256, 1], [256, 256, 1], + [256, 256, 1], [256, 256, 1], [256, 256, 1], [256, 256, 1], + [256, 512, 2], [512, 512, 1], [512, 512, 1], + ] + + self.body = nn.Sequential(*[ IRSE(a,b,c) for (a,b,c) in self.res50_arch ]) + + def forward(self, x): + x = self.input_layer(x) + x = self.body(x) + x = self.output_layer(x) + return x + + +if __name__ == "__main__": + from PIL import Image + import numpy as np + + x = Image.open("../samples/009.jpg").convert('L') + x = x.resize((128, 128)) + x = np.asarray(x, dtype=np.float32) + x = x[None, None, ...] + x = torch.from_numpy(x) + net = ResIRSE(512, 0.6) + net.eval() + with torch.no_grad(): + out = net(x) + print(out.shape) diff --git a/tracking/trackers/reid/model/resnet_pre.py b/tracking/trackers/reid/model/resnet_pre.py new file mode 100644 index 0000000..8b1594d --- /dev/null +++ b/tracking/trackers/reid/model/resnet_pre.py @@ -0,0 +1,384 @@ +import torch +import torch.nn as nn +# from config import config as conf +from ..config import config as conf + +try: + from torch.hub import load_state_dict_from_url +except ImportError: + from torch.utils.model_zoo import load_url as load_state_dict_from_url +#from .utils import load_state_dict_from_url + + +__all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', + 'resnet152', 'resnext50_32x4d', 'resnext101_32x8d', + 'wide_resnet50_2', 'wide_resnet101_2'] + + +model_urls = { + 'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth', + 'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth', + 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth', + 'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth', + 'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth', + 'resnext50_32x4d': 'https://download.pytorch.org/models/resnext50_32x4d-7cdf4587.pth', + 'resnext101_32x8d': 'https://download.pytorch.org/models/resnext101_32x8d-8ba56ff5.pth', + 'wide_resnet50_2': 'https://download.pytorch.org/models/wide_resnet50_2-95faca4d.pth', + 'wide_resnet101_2': 'https://download.pytorch.org/models/wide_resnet101_2-32ee1156.pth', +} + + +def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1): + """3x3 convolution with padding""" + return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=dilation, groups=groups, bias=False, dilation=dilation) + + +def conv1x1(in_planes, out_planes, stride=1): + """1x1 convolution""" + return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1, + base_width=64, dilation=1, norm_layer=None): + super(BasicBlock, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + if groups != 1 or base_width != 64: + raise ValueError('BasicBlock only supports groups=1 and base_width=64') + if dilation > 1: + raise NotImplementedError("Dilation > 1 not supported in BasicBlock") + # Both self.conv1 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = norm_layer(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = norm_layer(planes) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2) + # while original implementation places the stride at the first 1x1 convolution(self.conv1) + # according to "Deep residual learning for image recognition"https://arxiv.org/abs/1512.03385. + # This variant is also known as ResNet V1.5 and improves accuracy according to + # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch. + + expansion = 4 + + def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1, + base_width=64, dilation=1, norm_layer=None): + super(Bottleneck, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + width = int(planes * (base_width / 64.)) * groups + # Both self.conv2 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv1x1(inplanes, width) + self.bn1 = norm_layer(width) + self.conv2 = conv3x3(width, width, stride, groups, dilation) + self.bn2 = norm_layer(width) + self.conv3 = conv1x1(width, planes * self.expansion) + self.bn3 = norm_layer(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class ResNet(nn.Module): + + def __init__(self, block, layers, num_classes=conf.embedding_size, zero_init_residual=False, + groups=1, width_per_group=64, replace_stride_with_dilation=None, + norm_layer=None): + super(ResNet, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + self._norm_layer = norm_layer + + self.inplanes = 64 + self.dilation = 1 + if replace_stride_with_dilation is None: + # each element in the tuple indicates if we should replace + # the 2x2 stride with a dilated convolution instead + replace_stride_with_dilation = [False, False, False] + if len(replace_stride_with_dilation) != 3: + raise ValueError("replace_stride_with_dilation should be None " + "or a 3-element tuple, got {}".format(replace_stride_with_dilation)) + self.groups = groups + self.base_width = width_per_group + self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3, + bias=False) + self.bn1 = norm_layer(self.inplanes) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer(block, 128, layers[1], stride=2, + dilate=replace_stride_with_dilation[0]) + self.layer3 = self._make_layer(block, 256, layers[2], stride=2, + dilate=replace_stride_with_dilation[1]) + self.layer4 = self._make_layer(block, 512, layers[3], stride=2, + dilate=replace_stride_with_dilation[2]) + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + self.fc = nn.Linear(512 * block.expansion, num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + # Zero-initialize the last BN in each residual branch, + # so that the residual branch starts with zeros, and each residual block behaves like an identity. + # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 + if zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + nn.init.constant_(m.bn3.weight, 0) + elif isinstance(m, BasicBlock): + nn.init.constant_(m.bn2.weight, 0) + + def _make_layer(self, block, planes, blocks, stride=1, dilate=False): + norm_layer = self._norm_layer + downsample = None + previous_dilation = self.dilation + if dilate: + self.dilation *= stride + stride = 1 + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1(self.inplanes, planes * block.expansion, stride), + norm_layer(planes * block.expansion), + ) + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample, self.groups, + self.base_width, previous_dilation, norm_layer)) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append(block(self.inplanes, planes, groups=self.groups, + base_width=self.base_width, dilation=self.dilation, + norm_layer=norm_layer)) + + return nn.Sequential(*layers) + + def _forward_impl(self, x): + # See note [TorchScript super()] + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + #print('poolBefore', x.shape) + x = self.avgpool(x) + #print('poolAfter', x.shape) + x = torch.flatten(x, 1) + #print('fcBefore',x.shape) + x = self.fc(x) + # print('fcAfter',x.shape) + + return x + + def forward(self, x): + return self._forward_impl(x) + + +# def _resnet(arch, block, layers, pretrained, progress, **kwargs): +# model = ResNet(block, layers, **kwargs) +# if pretrained: +# state_dict = load_state_dict_from_url(model_urls[arch], +# progress=progress) +# model.load_state_dict(state_dict, strict=False) +# return model +def _resnet(arch, block, layers, pretrained, progress, **kwargs): + model = ResNet(block, layers, **kwargs) + if pretrained: + state_dict = load_state_dict_from_url(model_urls[arch], + progress=progress) + #print('state_dict',state_dict) + src_state_dict = state_dict + target_state_dict = model.state_dict() + skip_keys = [] + # skip mismatch size tensors in case of pretraining + for k in src_state_dict.keys(): + if k not in target_state_dict: + continue + if src_state_dict[k].size() != target_state_dict[k].size(): + skip_keys.append(k) + for k in skip_keys: + del src_state_dict[k] + missing_keys, unexpected_keys = model.load_state_dict(src_state_dict, strict=False) + + return model + + +def resnet18(pretrained=True, progress=True, **kwargs): + r"""ResNet-18 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet18', BasicBlock, [2, 2, 2, 2], pretrained, progress, + **kwargs) + + +def resnet34(pretrained=False, progress=True, **kwargs): + r"""ResNet-34 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet34', BasicBlock, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet50(pretrained=False, progress=True, **kwargs): + r"""ResNet-50 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet50', Bottleneck, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet101(pretrained=False, progress=True, **kwargs): + r"""ResNet-101 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet101', Bottleneck, [3, 4, 23, 3], pretrained, progress, + **kwargs) + + +def resnet152(pretrained=False, progress=True, **kwargs): + r"""ResNet-152 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet152', Bottleneck, [3, 8, 36, 3], pretrained, progress, + **kwargs) + + +def resnext50_32x4d(pretrained=False, progress=True, **kwargs): + r"""ResNeXt-50 32x4d model from + `"Aggregated Residual Transformation for Deep Neural Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 4 + return _resnet('resnext50_32x4d', Bottleneck, [3, 4, 6, 3], + pretrained, progress, **kwargs) + + +def resnext101_32x8d(pretrained=False, progress=True, **kwargs): + r"""ResNeXt-101 32x8d model from + `"Aggregated Residual Transformation for Deep Neural Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 8 + return _resnet('resnext101_32x8d', Bottleneck, [3, 4, 23, 3], + pretrained, progress, **kwargs) + + +def wide_resnet50_2(pretrained=False, progress=True, **kwargs): + r"""Wide ResNet-50-2 model from + `"Wide Residual Networks" `_ + + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['width_per_group'] = 64 * 2 + return _resnet('wide_resnet50_2', Bottleneck, [3, 4, 6, 3], + pretrained, progress, **kwargs) + + +def wide_resnet101_2(pretrained=False, progress=True, **kwargs): + r"""Wide ResNet-101-2 model from + `"Wide Residual Networks" `_ + + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['width_per_group'] = 64 * 2 + return _resnet('wide_resnet101_2', Bottleneck, [3, 4, 23, 3], + pretrained, progress, **kwargs) diff --git a/tracking/trackers/reid/model/utils.py b/tracking/trackers/reid/model/utils.py new file mode 100644 index 0000000..638ef07 --- /dev/null +++ b/tracking/trackers/reid/model/utils.py @@ -0,0 +1,4 @@ +try: + from torch.hub import load_state_dict_from_url +except ImportError: + from torch.utils.model_zoo import load_url as load_state_dict_from_url diff --git a/tracking/trackers/reid/reid_interface.py b/tracking/trackers/reid/reid_interface.py new file mode 100644 index 0000000..06aeca9 --- /dev/null +++ b/tracking/trackers/reid/reid_interface.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- +""" +Created on Thu Jan 18 17:21:01 2024 + +@author: ym +""" +import numpy as np +import torch +import cv2 +import torch.nn as nn +import torchvision.transforms as T +from .model import mobilevit_s, resnet18, resnet34, resnet50, mobilenet_v2, MobileNetV3_Small +from .config import config as conf + + +class ReIDInterface: + def __init__(self, config): + self.device = conf.device + if conf.backbone == 'resnet18': + # model = ResIRSE(img_size, embedding_size, conf.drop_ratio).to(device) + model = resnet18().to(self.device) + elif conf.backbone == 'resnet34': + model = resnet34().to(self.device) + elif conf.backbone == 'resnet50': + model = resnet50().to(self.device) + elif conf.backbone == 'mobilevit_s': + model = mobilevit_s().to(self.device) + elif conf.backbone == 'mobilenetv3': + model = MobileNetV3_Small().to(self.device) + else: + model = mobilenet_v2().to(self.device) + + self.batch_size = conf.batch_size + self.embedding_size = conf.embedding_size + self.img_size = conf.img_size + + self.model_path = conf.model_path + + # 原输入为PIL + self.transform = T.Compose([ + T.ToTensor(), + T.Resize((self.img_size, self.img_size)), + T.ConvertImageDtype(torch.float32), + T.Normalize(mean=[0.5], std=[0.5]), + ]) + + + # self.model = nn.DataParallel(model).to(self.device) + self.model = model + self.model.load_state_dict(torch.load(self.model_path, map_location=self.device)) + self.model.eval() + + def inference(self, images, detections): + if isinstance(images, np.ndarray): + features = self.inference_image(images, detections) + return features + + batch_patches = [] + patches = [] + for i, img in enumerate(images): + img = img.copy() + patch = self.transform(img) + if str(self.device) != "cpu": + patch = patch.to(device=self.device).half() + else: + patch = patch.to(device=self.device) + + patches.append(patch) + if (i + 1) % self.batch_size == 0: + patches = torch.stack(patches, dim=0) + batch_patches.append(patches) + patches = [] + + if len(patches): + patches = torch.stack(patches, dim=0) + batch_patches.append(patches) + + features = np.zeros((0, self.embedding_size)) + for patches in batch_patches: + pred=self.model(patches) + pred[torch.isinf(pred)] = 1.0 + feat = pred.cpu().data.numpy() + features = np.vstack((features, feat)) + return features + + def inference_image(self, image, detections): + H, W, _ = np.shape(image) + + batch_patches = [] + patches = [] + for d in range(np.size(detections, 0)): + tlbr = detections[d, :4].astype(np.int_) + tlbr[0] = max(0, tlbr[0]) + tlbr[1] = max(0, tlbr[1]) + tlbr[2] = min(W - 1, tlbr[2]) + tlbr[3] = min(H - 1, tlbr[3]) + img = image[tlbr[1]:tlbr[3], tlbr[0]:tlbr[2], :] + + img = img[:, :, ::-1].copy() # the model expects RGB inputs + patch = self.transform(img) + + # patch = patch.to(device=self.device).half() + if str(self.device) != "cpu": + patch = patch.to(device=self.device).half() + else: + patch = patch.to(device=self.device) + + patches.append(patch) + if (d + 1) % self.batch_size == 0: + patches = torch.stack(patches, dim=0) + batch_patches.append(patches) + patches = [] + + if len(patches): + patches = torch.stack(patches, dim=0) + batch_patches.append(patches) + + features = np.zeros((0, self.embedding_size)) + for patches in batch_patches: + pred = self.model(patches) + pred[torch.isinf(pred)] = 1.0 + feat = pred.cpu().data.numpy() + features = np.vstack((features, feat)) + + return features + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tracking/trackers/reid/resnet_pre_lc.py b/tracking/trackers/reid/resnet_pre_lc.py new file mode 100644 index 0000000..83b3068 --- /dev/null +++ b/tracking/trackers/reid/resnet_pre_lc.py @@ -0,0 +1,462 @@ +import torch +import torch.nn as nn +from tools.config import config as conf + +try: + from torch.hub import load_state_dict_from_url +except ImportError: + from torch.utils.model_zoo import load_url as load_state_dict_from_url +# from .utils import load_state_dict_from_url + +__all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', + 'resnet152', 'resnext50_32x4d', 'resnext101_32x8d', + 'wide_resnet50_2', 'wide_resnet101_2'] + +model_urls = { + 'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth', + 'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth', + 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth', + 'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth', + 'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth', + 'resnext50_32x4d': 'https://download.pytorch.org/models/resnext50_32x4d-7cdf4587.pth', + 'resnext101_32x8d': 'https://download.pytorch.org/models/resnext101_32x8d-8ba56ff5.pth', + 'wide_resnet50_2': 'https://download.pytorch.org/models/wide_resnet50_2-95faca4d.pth', + 'wide_resnet101_2': 'https://download.pytorch.org/models/wide_resnet101_2-32ee1156.pth', +} + + +def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1): + """3x3 convolution with padding""" + return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=dilation, groups=groups, bias=False, dilation=dilation) + +def conv1x1(in_planes, out_planes, stride=1): + """1x1 convolution""" + return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +class SpatialAttention(nn.Module): + def __init__(self, kernel_size=7): + super(SpatialAttention, self).__init__() + + assert kernel_size in (3, 7), 'kernel size must be 3 or 7' + padding = 3 if kernel_size == 7 else 1 + + self.conv1 = nn.Conv2d(2, 1, kernel_size, padding=padding, bias=False) + self.sigmoid = nn.Sigmoid() + + def forward(self, x): + avg_out = torch.mean(x, dim=1, keepdim=True) + max_out, _ = torch.max(x, dim=1, keepdim=True) + x = torch.cat([avg_out, max_out], dim=1) + x = self.conv1(x) + return self.sigmoid(x) + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1, + base_width=64, dilation=1, norm_layer=None, cam=False, bam=False): + super(BasicBlock, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + if groups != 1 or base_width != 64: + raise ValueError('BasicBlock only supports groups=1 and base_width=64') + if dilation > 1: + raise NotImplementedError("Dilation > 1 not supported in BasicBlock") + self.cam = cam + self.bam = bam + # Both self.conv1 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = norm_layer(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = norm_layer(planes) + self.downsample = downsample + self.stride = stride + if self.cam: + if planes == 64: + self.globalAvgPool = nn.AvgPool2d(56, stride=1) + elif planes == 128: + self.globalAvgPool = nn.AvgPool2d(28, stride=1) + elif planes == 256: + self.globalAvgPool = nn.AvgPool2d(14, stride=1) + elif planes == 512: + self.globalAvgPool = nn.AvgPool2d(7, stride=1) + + self.fc1 = nn.Linear(in_features=planes, out_features=round(planes / 16)) + self.fc2 = nn.Linear(in_features=round(planes / 16), out_features=planes) + self.sigmod = nn.Sigmoid() + if self.bam: + self.bam = SpatialAttention() + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + identity = self.downsample(x) + + if self.cam: + ori_out = self.globalAvgPool(out) + out = out.view(out.size(0), -1) + out = self.fc1(out) + out = self.relu(out) + out = self.fc2(out) + out = self.sigmod(out) + out = out.view(out.size(0), out.size(-1), 1, 1) + out = out * ori_out + + if self.bam: + out = out*self.bam(out) + + out += identity + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2) + # while original implementation places the stride at the first 1x1 convolution(self.conv1) + # according to "Deep residual learning for image recognition"https://arxiv.org/abs/1512.03385. + # This variant is also known as ResNet V1.5 and improves accuracy according to + # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch. + + expansion = 4 + + def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1, + base_width=64, dilation=1, norm_layer=None, cam=False, bam=False): + super(Bottleneck, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + width = int(planes * (base_width / 64.)) * groups + self.cam = cam + self.bam = bam + # Both self.conv2 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv1x1(inplanes, width) + self.bn1 = norm_layer(width) + self.conv2 = conv3x3(width, width, stride, groups, dilation) + self.bn2 = norm_layer(width) + self.conv3 = conv1x1(width, planes * self.expansion) + self.bn3 = norm_layer(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + if self.cam: + if planes == 64: + self.globalAvgPool = nn.AvgPool2d(56, stride=1) + elif planes == 128: + self.globalAvgPool = nn.AvgPool2d(28, stride=1) + elif planes == 256: + self.globalAvgPool = nn.AvgPool2d(14, stride=1) + elif planes == 512: + self.globalAvgPool = nn.AvgPool2d(7, stride=1) + + self.fc1 = nn.Linear(planes * self.expansion, round(planes / 4)) + self.fc2 = nn.Linear(round(planes / 4), planes * self.expansion) + self.sigmod = nn.Sigmoid() + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + if self.cam: + ori_out = self.globalAvgPool(out) + out = out.view(out.size(0), -1) + out = self.fc1(out) + out = self.relu(out) + out = self.fc2(out) + out = self.sigmod(out) + out = out.view(out.size(0), out.size(-1), 1, 1) + out = out * ori_out + out += identity + out = self.relu(out) + return out + + +class ResNet(nn.Module): + + def __init__(self, block, layers, num_classes=conf.embedding_size, zero_init_residual=False, + groups=1, width_per_group=64, replace_stride_with_dilation=None, + norm_layer=None, scale=0.75): + super(ResNet, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + self._norm_layer = norm_layer + + self.inplanes = 64 + self.dilation = 1 + if replace_stride_with_dilation is None: + # each element in the tuple indicates if we should replace + # the 2x2 stride with a dilated convolution instead + replace_stride_with_dilation = [False, False, False] + if len(replace_stride_with_dilation) != 3: + raise ValueError("replace_stride_with_dilation should be None " + "or a 3-element tuple, got {}".format(replace_stride_with_dilation)) + self.groups = groups + self.base_width = width_per_group + self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3, + bias=False) + self.bn1 = norm_layer(self.inplanes) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, int(64*scale), layers[0]) + self.layer2 = self._make_layer(block, int(128*scale), layers[1], stride=2, + dilate=replace_stride_with_dilation[0]) + self.layer3 = self._make_layer(block, int(256*scale), layers[2], stride=2, + dilate=replace_stride_with_dilation[1]) + self.layer4 = self._make_layer(block, int(512*scale), layers[3], stride=2, + dilate=replace_stride_with_dilation[2]) + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + self.fc = nn.Linear(int(512 * block.expansion*scale), num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + # Zero-initialize the last BN in each residual branch, + # so that the residual branch starts with zeros, and each residual block behaves like an identity. + # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 + if zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + nn.init.constant_(m.bn3.weight, 0) + elif isinstance(m, BasicBlock): + nn.init.constant_(m.bn2.weight, 0) + + def _make_layer(self, block, planes, blocks, stride=1, dilate=False): + norm_layer = self._norm_layer + downsample = None + previous_dilation = self.dilation + if dilate: + self.dilation *= stride + stride = 1 + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1(self.inplanes, planes * block.expansion, stride), + norm_layer(planes * block.expansion), + ) + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample, self.groups, + self.base_width, previous_dilation, norm_layer)) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append(block(self.inplanes, planes, groups=self.groups, + base_width=self.base_width, dilation=self.dilation, + norm_layer=norm_layer)) + return nn.Sequential(*layers) + + def _forward_impl(self, x): + # See note [TorchScript super()] + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + # print('poolBefore', x.shape) + x = self.avgpool(x) + # print('poolAfter', x.shape) + x = torch.flatten(x, 1) + # print('fcBefore',x.shape) + x = self.fc(x) + + # print('fcAfter',x.shape) + + return x + + def forward(self, x): + return self._forward_impl(x) + + +# def _resnet(arch, block, layers, pretrained, progress, **kwargs): +# model = ResNet(block, layers, **kwargs) +# if pretrained: +# state_dict = load_state_dict_from_url(model_urls[arch], +# progress=progress) +# model.load_state_dict(state_dict, strict=False) +# return model +def _resnet(arch, block, layers, pretrained, progress, **kwargs): + model = ResNet(block, layers, **kwargs) + if pretrained: + state_dict = load_state_dict_from_url(model_urls[arch], + progress=progress) + + src_state_dict = state_dict + target_state_dict = model.state_dict() + skip_keys = [] + # skip mismatch size tensors in case of pretraining + for k in src_state_dict.keys(): + if k not in target_state_dict: + continue + if src_state_dict[k].size() != target_state_dict[k].size(): + skip_keys.append(k) + for k in skip_keys: + del src_state_dict[k] + missing_keys, unexpected_keys = model.load_state_dict(src_state_dict, strict=False) + + return model + + +def resnet14(pretrained=True, progress=True, **kwargs): + r"""ResNet-14 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet18', BasicBlock, [2, 1, 1, 2], pretrained, progress, + **kwargs) + + +def resnet18(pretrained=True, progress=True, **kwargs): + r"""ResNet-18 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet18', BasicBlock, [2, 2, 2, 2], pretrained, progress, + **kwargs) + + +def resnet34(pretrained=False, progress=True, **kwargs): + r"""ResNet-34 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet34', BasicBlock, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet50(pretrained=False, progress=True, **kwargs): + r"""ResNet-50 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet50', Bottleneck, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet101(pretrained=False, progress=True, **kwargs): + r"""ResNet-101 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet101', Bottleneck, [3, 4, 23, 3], pretrained, progress, + **kwargs) + + +def resnet152(pretrained=False, progress=True, **kwargs): + r"""ResNet-152 model from + `"Deep Residual Learning for Image Recognition" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet152', Bottleneck, [3, 8, 36, 3], pretrained, progress, + **kwargs) + + +def resnext50_32x4d(pretrained=False, progress=True, **kwargs): + r"""ResNeXt-50 32x4d model from + `"Aggregated Residual Transformation for Deep Neural Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 4 + return _resnet('resnext50_32x4d', Bottleneck, [3, 4, 6, 3], + pretrained, progress, **kwargs) + + +def resnext101_32x8d(pretrained=False, progress=True, **kwargs): + r"""ResNeXt-101 32x8d model from + `"Aggregated Residual Transformation for Deep Neural Networks" `_ + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 8 + return _resnet('resnext101_32x8d', Bottleneck, [3, 4, 23, 3], + pretrained, progress, **kwargs) + + +def wide_resnet50_2(pretrained=False, progress=True, **kwargs): + r"""Wide ResNet-50-2 model from + `"Wide Residual Networks" `_ + + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['width_per_group'] = 64 * 2 + return _resnet('wide_resnet50_2', Bottleneck, [3, 4, 6, 3], + pretrained, progress, **kwargs) + + +def wide_resnet101_2(pretrained=False, progress=True, **kwargs): + r"""Wide ResNet-101-2 model from + `"Wide Residual Networks" `_ + + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['width_per_group'] = 64 * 2 + return _resnet('wide_resnet101_2', Bottleneck, [3, 4, 23, 3], + pretrained, progress, **kwargs) diff --git a/tracking/trackers/reid/test.py b/tracking/trackers/reid/test.py new file mode 100644 index 0000000..a2b7c4d --- /dev/null +++ b/tracking/trackers/reid/test.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +""" +Created on Fri Jan 19 16:10:39 2024 + +@author: ym +""" +import torch +from model.resnet_pre import resnet18 + + + +def main(): + model_path = "best.pth" + device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + model = resnet18().to(device) + model.load_state_dict(torch.load(model_path, map_location=device)) + + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tracking/trackers/track.py b/tracking/trackers/track.py new file mode 100644 index 0000000..cfb4b08 --- /dev/null +++ b/tracking/trackers/track.py @@ -0,0 +1,66 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from functools import partial + +import torch + +from ultralytics.utils import IterableSimpleNamespace, yaml_load +from ultralytics.utils.checks import check_yaml + +from .bot_sort import BOTSORT +from .byte_tracker import BYTETracker + +TRACKER_MAP = {'bytetrack': BYTETracker, 'botsort': BOTSORT} + + +def on_predict_start(predictor, persist=False): + """ + Initialize trackers for object tracking during prediction. + + Args: + predictor (object): The predictor object to initialize trackers for. + persist (bool, optional): Whether to persist the trackers if they already exist. Defaults to False. + + Raises: + AssertionError: If the tracker_type is not 'bytetrack' or 'botsort'. + """ + if hasattr(predictor, 'trackers') and persist: + return + tracker = check_yaml(predictor.args.tracker) + cfg = IterableSimpleNamespace(**yaml_load(tracker)) + assert cfg.tracker_type in ['bytetrack', 'botsort'], \ + f"Only support 'bytetrack' and 'botsort' for now, but got '{cfg.tracker_type}'" + trackers = [] + for _ in range(predictor.dataset.bs): + tracker = TRACKER_MAP[cfg.tracker_type](args=cfg, frame_rate=30) + trackers.append(tracker) + predictor.trackers = trackers + + +def on_predict_postprocess_end(predictor): + """Postprocess detected boxes and update with object tracking.""" + bs = predictor.dataset.bs + im0s = predictor.batch[1] + for i in range(bs): + det = predictor.results[i].boxes.cpu().numpy() + if len(det) == 0: + continue + tracks = predictor.trackers[i].update(det, im0s[i]) + if len(tracks) == 0: + continue + idx = tracks[:, -1].astype(int) + predictor.results[i] = predictor.results[i][idx] + predictor.results[i].update(boxes=torch.as_tensor(tracks[:, :-1])) + + +def register_tracker(model, persist): + """ + Register tracking callbacks to the model for object tracking during prediction. + + Args: + model (object): The model object to register tracking callbacks for. + persist (bool): Whether to persist the trackers if they already exist. + + """ + model.add_callback('on_predict_start', partial(on_predict_start, persist=persist)) + model.add_callback('on_predict_postprocess_end', on_predict_postprocess_end) diff --git a/tracking/trackers/utils/__init__.py b/tracking/trackers/utils/__init__.py new file mode 100644 index 0000000..0419cea --- /dev/null +++ b/tracking/trackers/utils/__init__.py @@ -0,0 +1,3 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + + diff --git a/tracking/trackers/utils/__pycache__/__init__.cpython-312.pyc b/tracking/trackers/utils/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..e25dfd0 Binary files /dev/null and b/tracking/trackers/utils/__pycache__/__init__.cpython-312.pyc differ diff --git a/tracking/trackers/utils/__pycache__/__init__.cpython-39.pyc b/tracking/trackers/utils/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..c3d3a94 Binary files /dev/null and b/tracking/trackers/utils/__pycache__/__init__.cpython-39.pyc differ diff --git a/tracking/trackers/utils/__pycache__/gmc.cpython-39.pyc b/tracking/trackers/utils/__pycache__/gmc.cpython-39.pyc new file mode 100644 index 0000000..61deff8 Binary files /dev/null and b/tracking/trackers/utils/__pycache__/gmc.cpython-39.pyc differ diff --git a/tracking/trackers/utils/__pycache__/kalman_filter.cpython-312.pyc b/tracking/trackers/utils/__pycache__/kalman_filter.cpython-312.pyc new file mode 100644 index 0000000..da37c2f Binary files /dev/null and b/tracking/trackers/utils/__pycache__/kalman_filter.cpython-312.pyc differ diff --git a/tracking/trackers/utils/__pycache__/kalman_filter.cpython-39.pyc b/tracking/trackers/utils/__pycache__/kalman_filter.cpython-39.pyc new file mode 100644 index 0000000..da580d1 Binary files /dev/null and b/tracking/trackers/utils/__pycache__/kalman_filter.cpython-39.pyc differ diff --git a/tracking/trackers/utils/__pycache__/matching.cpython-312.pyc b/tracking/trackers/utils/__pycache__/matching.cpython-312.pyc new file mode 100644 index 0000000..4eb6ef5 Binary files /dev/null and b/tracking/trackers/utils/__pycache__/matching.cpython-312.pyc differ diff --git a/tracking/trackers/utils/__pycache__/matching.cpython-39.pyc b/tracking/trackers/utils/__pycache__/matching.cpython-39.pyc new file mode 100644 index 0000000..82bea82 Binary files /dev/null and b/tracking/trackers/utils/__pycache__/matching.cpython-39.pyc differ diff --git a/tracking/trackers/utils/__pycache__/results.cpython-39.pyc b/tracking/trackers/utils/__pycache__/results.cpython-39.pyc new file mode 100644 index 0000000..05876d8 Binary files /dev/null and b/tracking/trackers/utils/__pycache__/results.cpython-39.pyc differ diff --git a/tracking/trackers/utils/gmc.py b/tracking/trackers/utils/gmc.py new file mode 100644 index 0000000..4d91df4 --- /dev/null +++ b/tracking/trackers/utils/gmc.py @@ -0,0 +1,279 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import copy + +import cv2 +import numpy as np + +from ultralytics.utils import LOGGER + + +class GMC: + + def __init__(self, method='sparseOptFlow', downscale=2): + """Initialize a video tracker with specified parameters.""" + super().__init__() + + self.method = method + self.downscale = max(1, int(downscale)) + + if self.method == 'orb': + self.detector = cv2.FastFeatureDetector_create(20) + self.extractor = cv2.ORB_create() + self.matcher = cv2.BFMatcher(cv2.NORM_HAMMING) + + elif self.method == 'sift': + self.detector = cv2.SIFT_create(nOctaveLayers=3, contrastThreshold=0.02, edgeThreshold=20) + self.extractor = cv2.SIFT_create(nOctaveLayers=3, contrastThreshold=0.02, edgeThreshold=20) + self.matcher = cv2.BFMatcher(cv2.NORM_L2) + + elif self.method == 'ecc': + number_of_iterations = 5000 + termination_eps = 1e-6 + self.warp_mode = cv2.MOTION_EUCLIDEAN + self.criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, number_of_iterations, termination_eps) + + elif self.method == 'sparseOptFlow': + self.feature_params = dict(maxCorners=1000, + qualityLevel=0.01, + minDistance=1, + blockSize=3, + useHarrisDetector=False, + k=0.04) + + elif self.method in ['none', 'None', None]: + self.method = None + else: + raise ValueError(f'Error: Unknown GMC method:{method}') + + self.prevFrame = None + self.prevKeyPoints = None + self.prevDescriptors = None + + self.initializedFirstFrame = False + + def apply(self, raw_frame, detections=None): + """Apply object detection on a raw frame using specified method.""" + if self.method in ['orb', 'sift']: + return self.applyFeatures(raw_frame, detections) + elif self.method == 'ecc': + return self.applyEcc(raw_frame, detections) + elif self.method == 'sparseOptFlow': + return self.applySparseOptFlow(raw_frame, detections) + else: + return np.eye(2, 3) + + def applyEcc(self, raw_frame, detections=None): + """Initialize.""" + height, width, _ = raw_frame.shape + frame = cv2.cvtColor(raw_frame, cv2.COLOR_BGR2GRAY) + H = np.eye(2, 3, dtype=np.float32) + + # Downscale image (TODO: consider using pyramids) + if self.downscale > 1.0: + frame = cv2.GaussianBlur(frame, (3, 3), 1.5) + frame = cv2.resize(frame, (width // self.downscale, height // self.downscale)) + width = width // self.downscale + height = height // self.downscale + + # Handle first frame + if not self.initializedFirstFrame: + # Initialize data + self.prevFrame = frame.copy() + + # Initialization done + self.initializedFirstFrame = True + + return H + + # Run the ECC algorithm. The results are stored in warp_matrix. + # (cc, H) = cv2.findTransformECC(self.prevFrame, frame, H, self.warp_mode, self.criteria) + try: + (cc, H) = cv2.findTransformECC(self.prevFrame, frame, H, self.warp_mode, self.criteria, None, 1) + except Exception as e: + LOGGER.warning(f'WARNING: find transform failed. Set warp as identity {e}') + + return H + + def applyFeatures(self, raw_frame, detections=None): + """Initialize.""" + height, width, _ = raw_frame.shape + frame = cv2.cvtColor(raw_frame, cv2.COLOR_BGR2GRAY) + H = np.eye(2, 3) + + # Downscale image (TODO: consider using pyramids) + if self.downscale > 1.0: + # frame = cv2.GaussianBlur(frame, (3, 3), 1.5) + frame = cv2.resize(frame, (width // self.downscale, height // self.downscale)) + width = width // self.downscale + height = height // self.downscale + + # Find the keypoints + mask = np.zeros_like(frame) + # mask[int(0.05 * height): int(0.95 * height), int(0.05 * width): int(0.95 * width)] = 255 + mask[int(0.02 * height):int(0.98 * height), int(0.02 * width):int(0.98 * width)] = 255 + if detections is not None: + for det in detections: + tlbr = (det[:4] / self.downscale).astype(np.int_) + mask[tlbr[1]:tlbr[3], tlbr[0]:tlbr[2]] = 0 + + keypoints = self.detector.detect(frame, mask) + + # Compute the descriptors + keypoints, descriptors = self.extractor.compute(frame, keypoints) + + # Handle first frame + if not self.initializedFirstFrame: + # Initialize data + self.prevFrame = frame.copy() + self.prevKeyPoints = copy.copy(keypoints) + self.prevDescriptors = copy.copy(descriptors) + + # Initialization done + self.initializedFirstFrame = True + + return H + + # Match descriptors. + knnMatches = self.matcher.knnMatch(self.prevDescriptors, descriptors, 2) + + # Filtered matches based on smallest spatial distance + matches = [] + spatialDistances = [] + + maxSpatialDistance = 0.25 * np.array([width, height]) + + # Handle empty matches case + if len(knnMatches) == 0: + # Store to next iteration + self.prevFrame = frame.copy() + self.prevKeyPoints = copy.copy(keypoints) + self.prevDescriptors = copy.copy(descriptors) + + return H + + for m, n in knnMatches: + if m.distance < 0.9 * n.distance: + prevKeyPointLocation = self.prevKeyPoints[m.queryIdx].pt + currKeyPointLocation = keypoints[m.trainIdx].pt + + spatialDistance = (prevKeyPointLocation[0] - currKeyPointLocation[0], + prevKeyPointLocation[1] - currKeyPointLocation[1]) + + if (np.abs(spatialDistance[0]) < maxSpatialDistance[0]) and \ + (np.abs(spatialDistance[1]) < maxSpatialDistance[1]): + spatialDistances.append(spatialDistance) + matches.append(m) + + meanSpatialDistances = np.mean(spatialDistances, 0) + stdSpatialDistances = np.std(spatialDistances, 0) + + inliers = (spatialDistances - meanSpatialDistances) < 2.5 * stdSpatialDistances + + goodMatches = [] + prevPoints = [] + currPoints = [] + for i in range(len(matches)): + if inliers[i, 0] and inliers[i, 1]: + goodMatches.append(matches[i]) + prevPoints.append(self.prevKeyPoints[matches[i].queryIdx].pt) + currPoints.append(keypoints[matches[i].trainIdx].pt) + + prevPoints = np.array(prevPoints) + currPoints = np.array(currPoints) + + # Draw the keypoint matches on the output image + # if False: + # import matplotlib.pyplot as plt + # matches_img = np.hstack((self.prevFrame, frame)) + # matches_img = cv2.cvtColor(matches_img, cv2.COLOR_GRAY2BGR) + # W = np.size(self.prevFrame, 1) + # for m in goodMatches: + # prev_pt = np.array(self.prevKeyPoints[m.queryIdx].pt, dtype=np.int_) + # curr_pt = np.array(keypoints[m.trainIdx].pt, dtype=np.int_) + # curr_pt[0] += W + # color = np.random.randint(0, 255, 3) + # color = (int(color[0]), int(color[1]), int(color[2])) + # + # matches_img = cv2.line(matches_img, prev_pt, curr_pt, tuple(color), 1, cv2.LINE_AA) + # matches_img = cv2.circle(matches_img, prev_pt, 2, tuple(color), -1) + # matches_img = cv2.circle(matches_img, curr_pt, 2, tuple(color), -1) + # + # plt.figure() + # plt.imshow(matches_img) + # plt.show() + + # Find rigid matrix + if (np.size(prevPoints, 0) > 4) and (np.size(prevPoints, 0) == np.size(prevPoints, 0)): + H, inliers = cv2.estimateAffinePartial2D(prevPoints, currPoints, cv2.RANSAC) + + # Handle downscale + if self.downscale > 1.0: + H[0, 2] *= self.downscale + H[1, 2] *= self.downscale + else: + LOGGER.warning('WARNING: not enough matching points') + + # Store to next iteration + self.prevFrame = frame.copy() + self.prevKeyPoints = copy.copy(keypoints) + self.prevDescriptors = copy.copy(descriptors) + + return H + + def applySparseOptFlow(self, raw_frame, detections=None): + """Initialize.""" + height, width, _ = raw_frame.shape + frame = cv2.cvtColor(raw_frame, cv2.COLOR_BGR2GRAY) + H = np.eye(2, 3) + + # Downscale image + if self.downscale > 1.0: + # frame = cv2.GaussianBlur(frame, (3, 3), 1.5) + frame = cv2.resize(frame, (width // self.downscale, height // self.downscale)) + + # Find the keypoints + keypoints = cv2.goodFeaturesToTrack(frame, mask=None, **self.feature_params) + + # Handle first frame + if not self.initializedFirstFrame: + # Initialize data + self.prevFrame = frame.copy() + self.prevKeyPoints = copy.copy(keypoints) + + # Initialization done + self.initializedFirstFrame = True + + return H + + # Find correspondences + matchedKeypoints, status, err = cv2.calcOpticalFlowPyrLK(self.prevFrame, frame, self.prevKeyPoints, None) + + # Leave good correspondences only + prevPoints = [] + currPoints = [] + + for i in range(len(status)): + if status[i]: + prevPoints.append(self.prevKeyPoints[i]) + currPoints.append(matchedKeypoints[i]) + + prevPoints = np.array(prevPoints) + currPoints = np.array(currPoints) + + # Find rigid matrix + if (np.size(prevPoints, 0) > 4) and (np.size(prevPoints, 0) == np.size(prevPoints, 0)): + H, inliers = cv2.estimateAffinePartial2D(prevPoints, currPoints, cv2.RANSAC) + + # Handle downscale + if self.downscale > 1.0: + H[0, 2] *= self.downscale + H[1, 2] *= self.downscale + else: + LOGGER.warning('WARNING: not enough matching points') + + # Store to next iteration + self.prevFrame = frame.copy() + self.prevKeyPoints = copy.copy(keypoints) + + return H diff --git a/tracking/trackers/utils/kalman_filter.py b/tracking/trackers/utils/kalman_filter.py new file mode 100644 index 0000000..9527ede --- /dev/null +++ b/tracking/trackers/utils/kalman_filter.py @@ -0,0 +1,368 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import numpy as np +import scipy.linalg + + +class KalmanFilterXYAH: + """ + For bytetrack. A simple Kalman filter for tracking bounding boxes in image space. + + The 8-dimensional state space (x, y, a, h, vx, vy, va, vh) contains the bounding box center position (x, y), + aspect ratio a, height h, and their respective velocities. + + Object motion follows a constant velocity model. The bounding box location (x, y, a, h) is taken as direct + observation of the state space (linear observation model). + """ + + def __init__(self): + """Initialize Kalman filter model matrices with motion and observation uncertainty weights.""" + ndim, dt = 4, 1. + + # Create Kalman filter model matrices. + self._motion_mat = np.eye(2 * ndim, 2 * ndim) + for i in range(ndim): + self._motion_mat[i, ndim + i] = dt + self._update_mat = np.eye(ndim, 2 * ndim) + + # Motion and observation uncertainty are chosen relative to the current state estimate. These weights control + # the amount of uncertainty in the model. This is a bit hacky. + self._std_weight_position = 1. / 20 + self._std_weight_velocity = 1. / 160 + + def initiate(self, measurement): + """ + Create track from unassociated measurement. + + Parameters + ---------- + measurement : ndarray + Bounding box coordinates (x, y, a, h) with center position (x, y), + aspect ratio a, and height h. + + Returns + ------- + (ndarray, ndarray) + Returns the mean vector (8 dimensional) and covariance matrix (8x8 + dimensional) of the new track. Unobserved velocities are initialized + to 0 mean. + """ + mean_pos = measurement + mean_vel = np.zeros_like(mean_pos) + mean = np.r_[mean_pos, mean_vel] + + std = [ + 2 * self._std_weight_position * measurement[3], 2 * self._std_weight_position * measurement[3], 1e-2, + 2 * self._std_weight_position * measurement[3], 10 * self._std_weight_velocity * measurement[3], + 10 * self._std_weight_velocity * measurement[3], 1e-5, 10 * self._std_weight_velocity * measurement[3]] + covariance = np.diag(np.square(std)) + return mean, covariance + + def predict(self, mean, covariance): + """ + Run Kalman filter prediction step. + + Parameters + ---------- + mean : ndarray + The 8 dimensional mean vector of the object state at the previous time step. + covariance : ndarray + The 8x8 dimensional covariance matrix of the object state at the previous time step. + + Returns + ------- + (ndarray, ndarray) + Returns the mean vector and covariance matrix of the predicted state. Unobserved velocities are + initialized to 0 mean. + """ + std_pos = [ + self._std_weight_position * mean[3], self._std_weight_position * mean[3], 1e-2, + self._std_weight_position * mean[3]] + std_vel = [ + self._std_weight_velocity * mean[3], self._std_weight_velocity * mean[3], 1e-5, + self._std_weight_velocity * mean[3]] + motion_cov = np.diag(np.square(np.r_[std_pos, std_vel])) + + # mean = np.dot(self._motion_mat, mean) + mean = np.dot(mean, self._motion_mat.T) + covariance = np.linalg.multi_dot((self._motion_mat, covariance, self._motion_mat.T)) + motion_cov + + return mean, covariance + + def project(self, mean, covariance): + """ + Project state distribution to measurement space. + + Parameters + ---------- + mean : ndarray + The state's mean vector (8 dimensional array). + covariance : ndarray + The state's covariance matrix (8x8 dimensional). + + Returns + ------- + (ndarray, ndarray) + Returns the projected mean and covariance matrix of the given state estimate. + """ + std = [ + self._std_weight_position * mean[3], self._std_weight_position * mean[3], 1e-1, + self._std_weight_position * mean[3]] + innovation_cov = np.diag(np.square(std)) + + mean = np.dot(self._update_mat, mean) + covariance = np.linalg.multi_dot((self._update_mat, covariance, self._update_mat.T)) + return mean, covariance + innovation_cov + + def multi_predict(self, mean, covariance): + """ + Run Kalman filter prediction step (Vectorized version). + + Parameters + ---------- + mean : ndarray + The Nx8 dimensional mean matrix of the object states at the previous time step. + covariance : ndarray + The Nx8x8 dimensional covariance matrix of the object states at the previous time step. + + Returns + ------- + (ndarray, ndarray) + Returns the mean vector and covariance matrix of the predicted state. Unobserved velocities are + initialized to 0 mean. + """ + std_pos = [ + self._std_weight_position * mean[:, 3], self._std_weight_position * mean[:, 3], + 1e-2 * np.ones_like(mean[:, 3]), self._std_weight_position * mean[:, 3]] + std_vel = [ + self._std_weight_velocity * mean[:, 3], self._std_weight_velocity * mean[:, 3], + 1e-5 * np.ones_like(mean[:, 3]), self._std_weight_velocity * mean[:, 3]] + sqr = np.square(np.r_[std_pos, std_vel]).T + + motion_cov = [np.diag(sqr[i]) for i in range(len(mean))] + motion_cov = np.asarray(motion_cov) + + mean = np.dot(mean, self._motion_mat.T) + left = np.dot(self._motion_mat, covariance).transpose((1, 0, 2)) + covariance = np.dot(left, self._motion_mat.T) + motion_cov + + return mean, covariance + + def update(self, mean, covariance, measurement): + """ + Run Kalman filter correction step. + + Parameters + ---------- + mean : ndarray + The predicted state's mean vector (8 dimensional). + covariance : ndarray + The state's covariance matrix (8x8 dimensional). + measurement : ndarray + The 4 dimensional measurement vector (x, y, a, h), where (x, y) is the center position, a the aspect + ratio, and h the height of the bounding box. + + Returns + ------- + (ndarray, ndarray) + Returns the measurement-corrected state distribution. + """ + projected_mean, projected_cov = self.project(mean, covariance) + + chol_factor, lower = scipy.linalg.cho_factor(projected_cov, lower=True, check_finite=False) + kalman_gain = scipy.linalg.cho_solve((chol_factor, lower), + np.dot(covariance, self._update_mat.T).T, + check_finite=False).T + innovation = measurement - projected_mean + + new_mean = mean + np.dot(innovation, kalman_gain.T) + new_covariance = covariance - np.linalg.multi_dot((kalman_gain, projected_cov, kalman_gain.T)) + return new_mean, new_covariance + + def gating_distance(self, mean, covariance, measurements, only_position=False, metric='maha'): + """ + Compute gating distance between state distribution and measurements. A suitable distance threshold can be + obtained from `chi2inv95`. If `only_position` is False, the chi-square distribution has 4 degrees of + freedom, otherwise 2. + + Parameters + ---------- + mean : ndarray + Mean vector over the state distribution (8 dimensional). + covariance : ndarray + Covariance of the state distribution (8x8 dimensional). + measurements : ndarray + An Nx4 dimensional matrix of N measurements, each in format (x, y, a, h) where (x, y) is the bounding box + center position, a the aspect ratio, and h the height. + only_position : Optional[bool] + If True, distance computation is done with respect to the bounding box center position only. + + Returns + ------- + ndarray + Returns an array of length N, where the i-th element contains the squared Mahalanobis distance between + (mean, covariance) and `measurements[i]`. + """ + mean, covariance = self.project(mean, covariance) + if only_position: + mean, covariance = mean[:2], covariance[:2, :2] + measurements = measurements[:, :2] + + d = measurements - mean + if metric == 'gaussian': + return np.sum(d * d, axis=1) + elif metric == 'maha': + cholesky_factor = np.linalg.cholesky(covariance) + z = scipy.linalg.solve_triangular(cholesky_factor, d.T, lower=True, check_finite=False, overwrite_b=True) + return np.sum(z * z, axis=0) # square maha + else: + raise ValueError('invalid distance metric') + + +class KalmanFilterXYWH(KalmanFilterXYAH): + """ + For BoT-SORT. A simple Kalman filter for tracking bounding boxes in image space. + + The 8-dimensional state space (x, y, w, h, vx, vy, vw, vh) contains the bounding box center position (x, y), + width w, height h, and their respective velocities. + + Object motion follows a constant velocity model. The bounding box location (x, y, w, h) is taken as direct + observation of the state space (linear observation model). + """ + + def initiate(self, measurement): + """ + Create track from unassociated measurement. + + Parameters + ---------- + measurement : ndarray + Bounding box coordinates (x, y, w, h) with center position (x, y), width w, and height h. + + Returns + ------- + (ndarray, ndarray) + Returns the mean vector (8 dimensional) and covariance matrix (8x8 dimensional) of the new track. + Unobserved velocities are initialized to 0 mean. + """ + mean_pos = measurement + mean_vel = np.zeros_like(mean_pos) + mean = np.r_[mean_pos, mean_vel] + + std = [ + 2 * self._std_weight_position * measurement[2], 2 * self._std_weight_position * measurement[3], + 2 * self._std_weight_position * measurement[2], 2 * self._std_weight_position * measurement[3], + 10 * self._std_weight_velocity * measurement[2], 10 * self._std_weight_velocity * measurement[3], + 10 * self._std_weight_velocity * measurement[2], 10 * self._std_weight_velocity * measurement[3]] + covariance = np.diag(np.square(std)) + return mean, covariance + + def predict(self, mean, covariance): + """ + Run Kalman filter prediction step. + + Parameters + ---------- + mean : ndarray + The 8 dimensional mean vector of the object state at the previous time step. + covariance : ndarray + The 8x8 dimensional covariance matrix of the object state at the previous time step. + + Returns + ------- + (ndarray, ndarray) + Returns the mean vector and covariance matrix of the predicted state. Unobserved velocities are + initialized to 0 mean. + """ + std_pos = [ + self._std_weight_position * mean[2], self._std_weight_position * mean[3], + self._std_weight_position * mean[2], self._std_weight_position * mean[3]] + std_vel = [ + self._std_weight_velocity * mean[2], self._std_weight_velocity * mean[3], + self._std_weight_velocity * mean[2], self._std_weight_velocity * mean[3]] + motion_cov = np.diag(np.square(np.r_[std_pos, std_vel])) + + mean = np.dot(mean, self._motion_mat.T) + covariance = np.linalg.multi_dot((self._motion_mat, covariance, self._motion_mat.T)) + motion_cov + + return mean, covariance + + def project(self, mean, covariance): + """ + Project state distribution to measurement space. + + Parameters + ---------- + mean : ndarray + The state's mean vector (8 dimensional array). + covariance : ndarray + The state's covariance matrix (8x8 dimensional). + + Returns + ------- + (ndarray, ndarray) + Returns the projected mean and covariance matrix of the given state estimate. + """ + std = [ + self._std_weight_position * mean[2], self._std_weight_position * mean[3], + self._std_weight_position * mean[2], self._std_weight_position * mean[3]] + innovation_cov = np.diag(np.square(std)) + + mean = np.dot(self._update_mat, mean) + covariance = np.linalg.multi_dot((self._update_mat, covariance, self._update_mat.T)) + return mean, covariance + innovation_cov + + def multi_predict(self, mean, covariance): + """ + Run Kalman filter prediction step (Vectorized version). + + Parameters + ---------- + mean : ndarray + The Nx8 dimensional mean matrix of the object states at the previous time step. + covariance : ndarray + The Nx8x8 dimensional covariance matrix of the object states at the previous time step. + + Returns + ------- + (ndarray, ndarray) + Returns the mean vector and covariance matrix of the predicted state. Unobserved velocities are + initialized to 0 mean. + """ + std_pos = [ + self._std_weight_position * mean[:, 2], self._std_weight_position * mean[:, 3], + self._std_weight_position * mean[:, 2], self._std_weight_position * mean[:, 3]] + std_vel = [ + self._std_weight_velocity * mean[:, 2], self._std_weight_velocity * mean[:, 3], + self._std_weight_velocity * mean[:, 2], self._std_weight_velocity * mean[:, 3]] + sqr = np.square(np.r_[std_pos, std_vel]).T + + motion_cov = [np.diag(sqr[i]) for i in range(len(mean))] + motion_cov = np.asarray(motion_cov) + + mean = np.dot(mean, self._motion_mat.T) + left = np.dot(self._motion_mat, covariance).transpose((1, 0, 2)) + covariance = np.dot(left, self._motion_mat.T) + motion_cov + + return mean, covariance + + def update(self, mean, covariance, measurement): + """ + Run Kalman filter correction step. + + Parameters + ---------- + mean : ndarray + The predicted state's mean vector (8 dimensional). + covariance : ndarray + The state's covariance matrix (8x8 dimensional). + measurement : ndarray + The 4 dimensional measurement vector (x, y, w, h), where (x, y) is the center position, w the width, + and h the height of the bounding box. + + Returns + ------- + (ndarray, ndarray) + Returns the measurement-corrected state distribution. + """ + return super().update(mean, covariance, measurement) diff --git a/tracking/trackers/utils/matching.py b/tracking/trackers/utils/matching.py new file mode 100644 index 0000000..116e95c --- /dev/null +++ b/tracking/trackers/utils/matching.py @@ -0,0 +1,215 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import numpy as np +import math +import torch +import scipy +from scipy.spatial.distance import cdist + +# from ultralytics.utils.metrics import bbox_ioa + +try: + import lap # for linear_assignment + + assert lap.__version__ # verify package is not directory +except (ImportError, AssertionError, AttributeError): + from ultralytics.utils.checks import check_requirements + + check_requirements('lapx>=0.5.2') # update to lap package from https://github.com/rathaROG/lapx + import lap + + +def bbox_iou(box1, box2, GIoU=False, DIoU=False, CIoU=False, eps=1e-7): + '''由根目录下 utils.metrics.metrics.bbox_iou 更改而来''' + # Returns Intersection over Union (IoU) of box1(1,4) to box2(n,4) + + # Get the coordinates of bounding boxes + # x1, y1, x2, y2 = box1 + # box1 = torch.tensor(box1) + # box2 = torch.tensor(box2) + + b1_x1, b1_y1, b1_x2, b1_y2 = box1.T + b2_x1, b2_y1, b2_x2, b2_y2 = box2.T + w1, h1 = b1_x2 - b1_x1, (b1_y2 - b1_y1).clip(eps) + w2, h2 = b2_x2 - b2_x1, (b2_y2 - b2_y1).clip(eps) + + # Intersection area + # inter = (b1_x2.minimum(b2_x2) - b1_x1.maximum(b2_x1)).clamp(0) * \ + # (b1_y2.minimum(b2_y2) - b1_y1.maximum(b2_y1)).clamp(0) + + inter = (np.minimum(b1_x2[:, None], b2_x2) - np.maximum(b1_x1[:, None], b2_x1)).clip(0) * \ + (np.minimum(b1_y2[:, None], b2_y2) - np.maximum(b1_y1[:, None], b2_y1)).clip(0) + + # Union Area + box1_area = w1 * h1 + box2_area = w2 * h2 + + union = box1_area[:, None] + box2_area - inter + eps + + # IoU + iou = inter / union + if CIoU or DIoU or GIoU: + cw = np.maximum(b1_x2[:, None], b2_x2) - np.minimum(b1_x1[:, None], b2_x1) # convex (smallest enclosing box) width + ch = np.maximum(b1_y2[:, None], b2_y2) - np.minimum(b1_y1[:, None], b2_y1) # convex height + if CIoU or DIoU: # Distance or Complete IoU https://arxiv.org/abs/1911.08287v1 + c2 = cw ** 2 + ch ** 2 + eps # convex diagonal squared + '''center dist ** 2''' + rho2 = ((b1_x1[:, None] + b1_x2[:, None] - b2_x1 - b2_x2) ** 2 + \ + (b1_y1[:, None] + b1_y2[:, None] - b2_y1 - b2_y2) ** 2) / 4 + if CIoU: # https://github.com/Zzh-tju/DIoU-SSD-pytorch/blob/master/utils/box/box_utils.py#L47 + v = (4 / math.pi ** 2) * (np.arctan(w1 / h1)[:, None] - np.arctan(w2 / h2))**2 + with torch.no_grad(): + alpha = v / (v - iou + (1 + eps)) + return iou - (rho2 / c2 + v * alpha) # CIoU + return iou - rho2 / c2 # DIoU + c_area = cw * ch + eps # convex area + return iou - (c_area - union) / c_area # GIoU https://arxiv.org/pdf/1902.09630.pdf + return iou # IoU + + +def bbox_ioa(box1, box2, iou=False, eps=1e-7): + """ + Calculate the intersection over box2 area given box1 and box2. Boxes are in x1y1x2y2 format. + + Args: + box1 (np.array): A numpy array of shape (n, 4) representing n bounding boxes. + box2 (np.array): A numpy array of shape (m, 4) representing m bounding boxes. + iou (bool): Calculate the standard iou if True else return inter_area/box2_area. + eps (float, optional): A small value to avoid division by zero. Defaults to 1e-7. + + Returns: + (np.array): A numpy array of shape (n, m) representing the intersection over box2 area. + """ + + # Get the coordinates of bounding boxes + b1_x1, b1_y1, b1_x2, b1_y2 = box1.T + b2_x1, b2_y1, b2_x2, b2_y2 = box2.T + + # Intersection area + inter_area = (np.minimum(b1_x2[:, None], b2_x2) - np.maximum(b1_x1[:, None], b2_x1)).clip(0) * \ + (np.minimum(b1_y2[:, None], b2_y2) - np.maximum(b1_y1[:, None], b2_y1)).clip(0) + + # box2 area + area = (b2_x2 - b2_x1) * (b2_y2 - b2_y1) + if iou: + box1_area = (b1_x2 - b1_x1) * (b1_y2 - b1_y1) + area = area + box1_area[:, None] - inter_area + + # Intersection over box2 area + return inter_area / (area + eps) + + +# def linear_assignment(cost_matrix, thresh, use_lap=True): +def linear_assignment(cost_matrix: np.ndarray, thresh: float, use_lap: bool = True) -> tuple: + """ + Perform linear assignment using scipy or lap.lapjv. + + Args: + cost_matrix (np.ndarray): The matrix containing cost values for assignments. + thresh (float): Threshold for considering an assignment valid. + use_lap (bool, optional): Whether to use lap.lapjv. Defaults to True. + + Returns: + (tuple): Tuple containing matched indices, unmatched indices from 'a', and unmatched indices from 'b'. + """ + + if cost_matrix.size == 0: + return np.empty((0, 2), dtype=int), tuple(range(cost_matrix.shape[0])), tuple(range(cost_matrix.shape[1])) + + if use_lap: + # https://github.com/gatagat/lap + _, x, y = lap.lapjv(cost_matrix, extend_cost=True, cost_limit=thresh) + matches = [[ix, mx] for ix, mx in enumerate(x) if mx >= 0] + unmatched_a = np.where(x < 0)[0] + unmatched_b = np.where(y < 0)[0] + else: + # https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.linear_sum_assignment.html + x, y = scipy.optimize.linear_sum_assignment(cost_matrix) # row x, col y + matches = np.asarray([[x[i], y[i]] for i in range(len(x)) if cost_matrix[x[i], y[i]] <= thresh]) + if len(matches) == 0: + unmatched_a = list(np.arange(cost_matrix.shape[0])) + unmatched_b = list(np.arange(cost_matrix.shape[1])) + else: + unmatched_a = list(set(np.arange(cost_matrix.shape[0])) - set(matches[:, 0])) + unmatched_b = list(set(np.arange(cost_matrix.shape[1])) - set(matches[:, 1])) + + return matches, unmatched_a, unmatched_b + + +# def iou_distance(atracks, btracks): +def iou_distance(atracks: list, btracks: list) -> np.ndarray: + """ + Compute cost based on Intersection over Union (IoU) between tracks. + + Args: + atracks (list[STrack] | list[np.ndarray]): List of tracks 'a' or bounding boxes. + btracks (list[STrack] | list[np.ndarray]): List of tracks 'b' or bounding boxes. + + Returns: + (np.ndarray): Cost matrix computed based on IoU. + """ + + if (len(atracks) > 0 and isinstance(atracks[0], np.ndarray)) \ + or (len(btracks) > 0 and isinstance(btracks[0], np.ndarray)): + atlbrs = atracks + btlbrs = btracks + else: + atlbrs = [track.tlbr for track in atracks] + btlbrs = [track.tlbr for track in btracks] + + ious = np.zeros((len(atlbrs), len(btlbrs)), dtype=np.float32) + if len(atlbrs) and len(btlbrs): + box1 = np.ascontiguousarray(atlbrs, dtype=np.float32) + box2 = np.ascontiguousarray(btlbrs, dtype=np.float32) + + ious = bbox_ioa(box1, box2, iou=True) + ious_g = bbox_iou(box1, box2, GIoU=True).clip(-1.0, 1.0) + ious_d = bbox_iou(box1, box2, DIoU=True).clip(-1.0, 1.0) + ious_c = bbox_iou(box1, box2, CIoU=True).clip(-1.0, 1.0) + + return 1 - ious # cost matrix + + +def embedding_distance(tracks, detections, metric='cosine'): + """ + Compute distance between tracks and detections based on embeddings. + + Args: + tracks (list[STrack]): List of tracks. + detections (list[BaseTrack]): List of detections. + metric (str, optional): Metric for distance computation. Defaults to 'cosine'. + + Returns: + (np.ndarray): Cost matrix computed based on embeddings. + """ + + cost_matrix = np.zeros((len(tracks), len(detections)), dtype=np.float32) + if cost_matrix.size == 0: + return cost_matrix + det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float32) + # for i, track in enumerate(tracks): + # cost_matrix[i, :] = np.maximum(0.0, cdist(track.smooth_feat.reshape(1,-1), det_features, metric)) + track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float32) + cost_matrix = np.maximum(0.0, cdist(track_features, det_features, metric)) # Normalized features + return cost_matrix + + +def fuse_score(cost_matrix, detections): + """ + Fuses cost matrix with detection scores to produce a single similarity matrix. + + Args: + cost_matrix (np.ndarray): The matrix containing cost values for assignments. + detections (list[BaseTrack]): List of detections with scores. + + Returns: + (np.ndarray): Fused similarity matrix. + """ + + if cost_matrix.size == 0: + return cost_matrix + iou_sim = 1 - cost_matrix + det_scores = np.array([det.score for det in detections]) + det_scores = np.expand_dims(det_scores, axis=0).repeat(cost_matrix.shape[0], axis=0) + fuse_sim = iou_sim * det_scores + return 1 - fuse_sim # fuse_cost diff --git a/tracking/tracking_pipeline.py b/tracking/tracking_pipeline.py new file mode 100644 index 0000000..54354b2 --- /dev/null +++ b/tracking/tracking_pipeline.py @@ -0,0 +1,232 @@ +# -*- coding: utf-8 -*- +""" +Created on Thu Mar 27 16:09:07 2025 + +@author: ym +""" +import os +import sys +import cv2 +import pickle +import numpy as np +from pathlib import Path +from scipy.spatial.distance import cdist +import copy + +from dotrack.dotracks_back import doBackTracks +from dotrack.dotracks_front import doFrontTracks +from utils.drawtracks import plot_frameID_y2, draw_all_trajectories +from utils.read_data import read_similar + + +def get_trail(ShoppingDict, ppath): + + evtname = ShoppingDict["eventName"] + + back_yrt = ShoppingDict["backCamera"]["yoloResnetTracker"] + front_yrt = ShoppingDict["frontCamera"]["yoloResnetTracker"] + + back_vts = ShoppingDict["frontCamera"]["tracking"] + front_vts = ShoppingDict["backCamera"]["tracking"] + + + event_tracks = [("back", back_yrt, back_vts), ("front", front_yrt, front_vts)] + + + savepath = ppath / "alltrail" + if not savepath.exists(): + savepath.mkdir() + + savepath = str(savepath) + evtime = evtname[:15] + + illus = [None, None] + for camera_type, yrtOut, vts in event_tracks: + if len(vts.Residual)==1: continue + + if camera_type == 'front': + edgeline = cv2.imread("./shopcart/cart_tempt/board_ftmp_line.png") + + img_tracking = draw_all_trajectories(vts, edgeline, savepath, camera_type, draw5p=False) + illus[0] = img_tracking + + plt = plot_frameID_y2(vts) + plt.savefig(os.path.join(savepath, f"{evtime}_front.png")) + + if camera_type == 'back': + edgeline = cv2.imread("./shopcart/cart_tempt/edgeline.png") + + img_tracking = draw_all_trajectories(vts, edgeline, savepath, camera_type, draw5p=False) + illus[1] = img_tracking + + illus = [im for im in illus if im is not None] + if len(illus): + img_cat = np.concatenate(illus, axis = 1) + if len(illus)==2: + H, W = img_cat.shape[:2] + cv2.line(img_cat, (int(W/2), 0), (int(W/2), int(H)), (128, 128, 255), 3) + + trajpath = os.path.join(savepath, f"{evtime}.png") + cv2.imwrite(trajpath, img_cat) + + return evtime + + return None + + + + + + + + + + +def track_opt(ShoppingDict, ppath): + ''' + 将一个对象读取，修改其中一个属性 + + ''' + evtname = ShoppingDict["eventName"] + + shopping = copy.deepcopy(ShoppingDict) + + ## only need to init item: tracking for each Camera + shopping["frontCamera"]["tracking"] = [] + shopping["backCamera"]["tracking"] = [] + + back_camera = ShoppingDict["backCamera"]["cameraType"] + back_yrt = ShoppingDict["backCamera"]["yoloResnetTracker"] + front_camera = ShoppingDict["frontCamera"]["cameraType"] + front_yrt = ShoppingDict["frontCamera"]["yoloResnetTracker"] + yrts = [(back_camera, back_yrt), (front_camera, front_yrt)] + + event_tracks = [] + errtrail = '' + for camera_type, yrtOut in yrts: + '''================= 1. tracking =================''' + '''(1) 生成用于 tracking 模块的 boxes、feats''' + # bboxes = np.empty((0, 6), dtype=np.float64) + trackerboxes = np.empty((0, 9), dtype=np.float64) + trackefeats = {} + for frameDict in yrtOut: + tboxes = frameDict["tboxes"] + ffeats = frameDict["feats"] + + # boxes = frameDict["bboxes"] + # bboxes = np.concatenate((bboxes, np.array(boxes)), axis=0) + trackerboxes = np.concatenate((trackerboxes, np.array(tboxes)), axis=0) + for i in range(len(tboxes)): + fid, bid = int(tboxes[i, 7]), int(tboxes[i, 8]) + trackefeats.update({f"{fid}_{bid}": ffeats[f"{fid}_{bid}"]}) + + '''(2) tracking, 后摄''' + if camera_type == "back": + vts = doBackTracks(trackerboxes, trackefeats) + vts.classify() + shopping["backCamera"]["tracking"] = vts + + if len(vts.Residual)!=1: + errtrail = evtname + + '''(3) tracking, 前摄''' + if camera_type == "front": + vts = doFrontTracks(trackerboxes, trackefeats) + vts.classify() + shopping["frontCamera"]["tracking"] = vts + + if len(vts.Residual)!=1: + errtrail = evtname + + event_tracks.append((camera_type, yrtOut, vts)) + + + pckpath = ppath / "track_optim" + if not pckpath.exists(): + pckpath.mkdir() + + fpath = pckpath / "{}_new.pickle".format(evtname) + with open(str(fpath), 'wb') as f: + pickle.dump(shopping, f) + + + savepath = ppath / "yolos_tracking" / evtname + + illus = [None, None] + for camera_type, yrtOut, vts in event_tracks: + if len(vts.tracks)==0: continue + + if camera_type == 'front': + edgeline = cv2.imread("./shopcart/cart_tempt/board_ftmp_line.png") + + img_tracking = draw_all_trajectories(vts, edgeline, savepath, camera_type, draw5p=False) + illus[0] = img_tracking + + plt = plot_frameID_y2(vts) + plt.savefig(os.path.join(savepath, "front_y2_new.png")) + + if camera_type == 'back': + edgeline = cv2.imread("./shopcart/cart_tempt/edgeline.png") + + img_tracking = draw_all_trajectories(vts, edgeline, savepath, camera_type, draw5p=False) + illus[1] = img_tracking + + illus = [im for im in illus if im is not None] + if len(illus): + img_cat = np.concatenate(illus, axis = 1) + if len(illus)==2: + H, W = img_cat.shape[:2] + cv2.line(img_cat, (int(W/2), 0), (int(W/2), int(H)), (128, 128, 255), 3) + + trajpath = os.path.join(savepath, "trajectory_new.png") + cv2.imwrite(trajpath, img_cat) + + return errtrail + + +def main(): + # evttypes = ["single_event_V10", "single_event_V5", "performence_V10", "performence_V5"] + evttypes = ["single_event_V10"] + + k = 0 + error_trail = [] + for evttype in evttypes: + ppath = Path("/home/wqg/dataset/pipeline/yrt/{}".format(evttype)) + + pkpath = ppath / "shopping_pkl" + for fp in pkpath.iterdir(): + # fp = pkpath / "{}.pickle".format("20250305-152917-635_6970209860221_6970209860221") + print(fp) + + if fp.suffix != '.pickle': continue + with open(str(fp), 'rb') as f: + ShoppingDict = pickle.load(f) + + # errtrail = track_opt(ShoppingDict, ppath) + # error_trail.append(errtrail) + + errtrail = get_trail(ShoppingDict, ppath) + if errtrail is not None: + error_trail.append(errtrail) + + # k+=1 + # if k==100: + # break + + errfile = ppath / 'error_trail.txt' + with open(errfile, 'w', encoding='utf-8') as f: + for line in error_trail: + f.write(line + '\n') + + + + +if __name__ == "__main__": + main() + + + + + + + \ No newline at end of file diff --git a/tracking/tracking_test.py b/tracking/tracking_test.py new file mode 100644 index 0000000..a55b0b3 --- /dev/null +++ b/tracking/tracking_test.py @@ -0,0 +1,176 @@ +# -*- coding: utf-8 -*- +""" +Created on Wed Sep 20 17:33:00 2023 + +@author: ym +""" + +import cv2 +import os +import numpy as np +import time +import pickle +import matplotlib.pyplot as plt +import pandas as pd +from scipy.spatial.distance import cdist + +from pathlib import Path +# ================= using for import ultralytics +import sys +sys.path.append(r"D:\DetectTracking") +from utils.gen import Profile + +from dotrack.dotracks_back import doBackTracks +from dotrack.dotracks_front import doFrontTracks +from utils.drawtracks import draw5points, drawTrack, drawtracefeat, plot_frameID_y2, drawFeatures, draw_all_trajectories + +from utils.read_data import extract_data_realtime, read_tracking_output_realtime + +# from datetime import datetime +# from utils.proBoxes import boxes_add_fid +# from utils.plotting import boxing_img #, Annotator, colors, +# from utils import Boxes, IterableSimpleNamespace, yaml_load +# from trackers import BOTSORT, BYTETracker +# from utils.mergetrack import track_equal_track +# from utils.basetrack import MoveState, ShoppingCart, doTracks + + +def detect_start_end(bboxes, features_dict, filename): + + boxes = np.empty(shape=(0, 9), dtype = np.float) + if filename.find("back") >= 0: + vts = doBackTracks(bboxes, features_dict) + vtx = [t for t in vts if t.cls != 0] + for track in vtx: + if track.moving_index.size: + boxes = np.concatenate((boxes, track.moving_index), axis=0) + + elif filename.find("front") >= 0: + vts = doFrontTracks(bboxes, features_dict) + vtx = [t for t in vts if t.cls != 0] + + for track in vtx: + for start, end in track.dynamic_y2: + boxes = np.concatenate((boxes, track.boxes[start:end+1, :]), axis=0) + for start, end in track.dynamic_y1: + boxes = np.concatenate((boxes, track.boxes[start:end+1, :]), axis=0) + + start = np.min(boxes[:, 0]) + end = np.max(boxes[:, 1]) + + if start > 5: + start = start - 5 + else: + start = 0 + + return start, end + +def save_subimgs(vts, file, TracksDict): + imgdir = Path(f'./result/imgs/{file}') + if not imgdir.exists(): + imgdir.mkdir(parents=True, exist_ok=True) + + for i, track in enumerate(vts.Residual): + boxes = track.boxes + for ii in range(len(boxes)): + tid, fid, bid = int(boxes[ii, 4]), int(boxes[ii, 7]), int(boxes[ii, 8]) + + img = TracksDict[f"frame_{fid}"]["imgs"][bid] + # feat = TracksDict[f"frame_{fid}"]["feats"][bid] + # box = TracksDict[f"frame_{fid}"]["boxes"][bid] + + cv2.imwrite(str(imgdir) + f"/{tid}_{fid}_{bid}.png", img) + +def have_tracked(): + # trackdict = r'./data/trackdicts' + trackdict = r'D:\全实时\202502\20250228-152846-438_6901668934727_6901668934727' + + bboxes, alltracks = [], [] + k = 0 + gt = Profile() + for filename in os.listdir(trackdict): + # filename = '153112511_0_seek_105.pkl' + + file, ext = os.path.splitext(filename) + filepath = os.path.join(trackdict, filename) + + if file.split('_')[0]=='0' or file.find("back") >= 0: + CamerType = "back" + if file.split('_')[0]=='1' or file.find("front") >= 0: + CamerType = "front" + + ## 1. 加载 tracker 输出的 pickle 文件 + if ext in ['.pkl', '.pickle']: + filepath = os.path.join(trackdict, filename) + TracksDict = np.load(filepath, allow_pickle=True) + bboxes = TracksDict['TrackBoxes'] + + ## 2. 加载 data 文件 + if filename.find('tracker.data')>0: + bboxes, TracksDict = extract_data_realtime(filepath) + + if len(bboxes)==0: + continue + + with gt: + if CamerType == "front": + vts = doFrontTracks(bboxes, TracksDict) + + Intrude = vts.isintrude() + + vts.classify() + + save_subimgs(vts, file, TracksDict) + + plt = plot_frameID_y2(vts) + + savedir = save_dir.joinpath(f'{file}_y2.png') + + plt.savefig(savedir) + plt.close() + + edgeline = cv2.imread("./shopcart/cart_tempt/board_ftmp_line.png") + img_tracking = draw_all_trajectories(vts, edgeline, save_dir, file, draw5p=True) + + if CamerType == "back": + vts = doBackTracks(bboxes, TracksDict) + + Intrude = vts.isintrude() + + vts.classify() + alltracks.append(vts) + + save_subimgs(vts, file, TracksDict) + edgeline = cv2.imread("./shopcart/cart_tempt/edgeline.png") + img_tracking = draw_all_trajectories(vts, edgeline, save_dir, file) + + trackpath = save_dir.joinpath(f'{file}.png') + cv2.imwrite(str(trackpath), img_tracking) + print(file+f" need time: {gt.dt:.2f}s") + + k += 1 + if k==1: + break + + if len(alltracks): + drawFeatures(alltracks, save_dir) + + + +if __name__ == "__main__": + # now = datetime.now() + # time_string = now.strftime("%Y%m%d%H%M%S")[:8] + + save_dir = Path('./result/tracks') + if not save_dir.exists(): + save_dir.mkdir(parents=True, exist_ok=True) + + have_tracked() + + + + + + + + \ No newline at end of file diff --git a/tracking/utils/__init__.py b/tracking/utils/__init__.py new file mode 100644 index 0000000..adc8bc3 --- /dev/null +++ b/tracking/utils/__init__.py @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- +""" +Created on Sun Dec 31 17:06:34 2023 + +@author: ym +""" +from .proBoxes import Boxes, boxes_add_fid +from .iterYaml import IterableSimpleNamespace, yaml_load + + +__all__ = "IterableSimpleNamespace", "yaml_load", "Boxes", "boxes_add_fid" \ No newline at end of file diff --git a/tracking/utils/__pycache__/__init__.cpython-312.pyc b/tracking/utils/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..4a505fa Binary files /dev/null and b/tracking/utils/__pycache__/__init__.cpython-312.pyc differ diff --git a/tracking/utils/__pycache__/__init__.cpython-39.pyc b/tracking/utils/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..68c05ec Binary files /dev/null and b/tracking/utils/__pycache__/__init__.cpython-39.pyc differ diff --git a/tracking/utils/__pycache__/annotator.cpython-312.pyc b/tracking/utils/__pycache__/annotator.cpython-312.pyc new file mode 100644 index 0000000..eb7fb91 Binary files /dev/null and b/tracking/utils/__pycache__/annotator.cpython-312.pyc differ diff --git a/tracking/utils/__pycache__/annotator.cpython-39.pyc b/tracking/utils/__pycache__/annotator.cpython-39.pyc new file mode 100644 index 0000000..182245b Binary files /dev/null and b/tracking/utils/__pycache__/annotator.cpython-39.pyc differ diff --git a/tracking/utils/__pycache__/drawtracks.cpython-312.pyc b/tracking/utils/__pycache__/drawtracks.cpython-312.pyc new file mode 100644 index 0000000..47a77d0 Binary files /dev/null and b/tracking/utils/__pycache__/drawtracks.cpython-312.pyc differ diff --git a/tracking/utils/__pycache__/drawtracks.cpython-39.pyc b/tracking/utils/__pycache__/drawtracks.cpython-39.pyc new file mode 100644 index 0000000..90325be Binary files /dev/null and b/tracking/utils/__pycache__/drawtracks.cpython-39.pyc differ diff --git a/tracking/utils/__pycache__/gen.cpython-39.pyc b/tracking/utils/__pycache__/gen.cpython-39.pyc new file mode 100644 index 0000000..e084f9b Binary files /dev/null and b/tracking/utils/__pycache__/gen.cpython-39.pyc differ diff --git a/tracking/utils/__pycache__/iterYaml.cpython-312.pyc b/tracking/utils/__pycache__/iterYaml.cpython-312.pyc new file mode 100644 index 0000000..7e78e0e Binary files /dev/null and b/tracking/utils/__pycache__/iterYaml.cpython-312.pyc differ diff --git a/tracking/utils/__pycache__/iterYaml.cpython-39.pyc b/tracking/utils/__pycache__/iterYaml.cpython-39.pyc new file mode 100644 index 0000000..81c51d6 Binary files /dev/null and b/tracking/utils/__pycache__/iterYaml.cpython-39.pyc differ diff --git a/tracking/utils/__pycache__/mergetrack.cpython-312.pyc b/tracking/utils/__pycache__/mergetrack.cpython-312.pyc new file mode 100644 index 0000000..487dc6f Binary files /dev/null and b/tracking/utils/__pycache__/mergetrack.cpython-312.pyc differ diff --git a/tracking/utils/__pycache__/mergetrack.cpython-39.pyc b/tracking/utils/__pycache__/mergetrack.cpython-39.pyc new file mode 100644 index 0000000..3b6f40a Binary files /dev/null and b/tracking/utils/__pycache__/mergetrack.cpython-39.pyc differ diff --git a/tracking/utils/__pycache__/plotting.cpython-312.pyc b/tracking/utils/__pycache__/plotting.cpython-312.pyc new file mode 100644 index 0000000..7809473 Binary files /dev/null and b/tracking/utils/__pycache__/plotting.cpython-312.pyc differ diff --git a/tracking/utils/__pycache__/plotting.cpython-39.pyc b/tracking/utils/__pycache__/plotting.cpython-39.pyc new file mode 100644 index 0000000..4de0e4e Binary files /dev/null and b/tracking/utils/__pycache__/plotting.cpython-39.pyc differ diff --git a/tracking/utils/__pycache__/proBoxes.cpython-312.pyc b/tracking/utils/__pycache__/proBoxes.cpython-312.pyc new file mode 100644 index 0000000..bf7ff47 Binary files /dev/null and b/tracking/utils/__pycache__/proBoxes.cpython-312.pyc differ diff --git a/tracking/utils/__pycache__/proBoxes.cpython-39.pyc b/tracking/utils/__pycache__/proBoxes.cpython-39.pyc new file mode 100644 index 0000000..9ef37a7 Binary files /dev/null and b/tracking/utils/__pycache__/proBoxes.cpython-39.pyc differ diff --git a/tracking/utils/__pycache__/processboxes.cpython-39.pyc b/tracking/utils/__pycache__/processboxes.cpython-39.pyc new file mode 100644 index 0000000..dd4ebe2 Binary files /dev/null and b/tracking/utils/__pycache__/processboxes.cpython-39.pyc differ diff --git a/tracking/utils/__pycache__/readData.cpython-39.pyc b/tracking/utils/__pycache__/readData.cpython-39.pyc new file mode 100644 index 0000000..ab579b5 Binary files /dev/null and b/tracking/utils/__pycache__/readData.cpython-39.pyc differ diff --git a/tracking/utils/__pycache__/read_data.cpython-312.pyc b/tracking/utils/__pycache__/read_data.cpython-312.pyc new file mode 100644 index 0000000..ddd90b6 Binary files /dev/null and b/tracking/utils/__pycache__/read_data.cpython-312.pyc differ diff --git a/tracking/utils/__pycache__/read_data.cpython-39.pyc b/tracking/utils/__pycache__/read_data.cpython-39.pyc new file mode 100644 index 0000000..9748378 Binary files /dev/null and b/tracking/utils/__pycache__/read_data.cpython-39.pyc differ diff --git a/tracking/utils/__pycache__/read_pipeline_data.cpython-39.pyc b/tracking/utils/__pycache__/read_pipeline_data.cpython-39.pyc new file mode 100644 index 0000000..3379b2a Binary files /dev/null and b/tracking/utils/__pycache__/read_pipeline_data.cpython-39.pyc differ diff --git a/tracking/utils/__pycache__/showtrack.cpython-312.pyc b/tracking/utils/__pycache__/showtrack.cpython-312.pyc new file mode 100644 index 0000000..85a1fbf Binary files /dev/null and b/tracking/utils/__pycache__/showtrack.cpython-312.pyc differ diff --git a/tracking/utils/__pycache__/showtrack.cpython-39.pyc b/tracking/utils/__pycache__/showtrack.cpython-39.pyc new file mode 100644 index 0000000..5c446a4 Binary files /dev/null and b/tracking/utils/__pycache__/showtrack.cpython-39.pyc differ diff --git a/tracking/utils/annotator.py b/tracking/utils/annotator.py new file mode 100644 index 0000000..eb04bfd --- /dev/null +++ b/tracking/utils/annotator.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +""" +Created on Wed Sep 20 14:21:13 2023 + +@author: ym +""" +import cv2 + +# import sys +# sys.path.append(r"D:\DeepLearning\yolov5") +# from ultralytics.utils.plotting import Annotator, colors + +from .plotting import Annotator, colors + +class TrackAnnotator(Annotator): + + def plotting_track(self, track, names='abc'): + """ + track：[x, y, w, h, track_id, score, cls, frame_index] + + boxes: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] + 0 1 2 3 4 5 6 7 8 + id：跟踪id，从 1 开始计数， + frame_index: 帧索引，从 1 开始计数 + cls：类别编号，从 0 开始计数，用作 names 的 key 值 + """ + + if track.size==0: return + + id, cls = track[0, 4], track[0, 6] + if id >=0 and cls==0: + color = colors(int(cls), True) + elif id >=0 and cls!=0: + color = colors(int(id), True) # 不存在 id = 0，不会和上面产生冲突 + else: + color = colors(19, True) # 19为调色板的最后一个元素 + + nb = track.shape[0] + for i in range(nb): + if i == 0: + # label = f'{int(track[i, 4])}:({int(track[i, 7])})' + label = f'ID_{int(track[i, 4])}' + elif i == nb-1: + label = '' + # label = f'{int(track[i, 4])}:({int(track[i, 7])})&{int(nb)}' + else: + label = '' + + self.circle_label(track[i, :], label, color=color) + + + def circle_label(self, track, label='', color=(128, 128, 128), txt_color=(255, 255, 255)): + """ + 绘制选定 track 的轨迹 + """ + if track.size==0: return + + x, y = int((track[0]+track[2])/2), int((track[1]+track[3])/2) + + cv2.circle(self.im, (x, y), 6, color, 2) + # txt_color = (0,0,0) + + if label: + tf = max(self.lw - 1, 1) # font thickness + w, h = cv2.getTextSize(label, 0, fontScale=self.lw / 3, thickness=tf)[0] # text width, height + outside = x + w <= self.im.shape[1]-3 + # p2 = x + w, y - h - 3 if outside else y + h + 3 + # cv2.rectangle(self.im, p1, p2, color, -1, cv2.LINE_AA) # filled + + cv2.putText(self.im, + label, (x-10 if outside else x-w+2, y-20), + 0, + # self.lw / 3, + self.lw/2, + txt_color, + thickness=tf, + lineType=cv2.LINE_AA) + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tracking/utils/drawtracks.py b/tracking/utils/drawtracks.py new file mode 100644 index 0000000..0b668e1 --- /dev/null +++ b/tracking/utils/drawtracks.py @@ -0,0 +1,387 @@ +# -*- coding: utf-8 -*- +""" +Created on Mon Jan 15 15:26:38 2024 + +@author: ym +""" +import numpy as np +import cv2 +import os +from pathlib import Path +import matplotlib.pyplot as plt +from sklearn.decomposition import PCA + +from tracking.utils.annotator import TrackAnnotator +from tracking.utils.plotting import colors + + + +def plot_frameID_y2(vts): + # boxes: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] + # 0, 1, 2, 3, 4, 5, 6, 7, 8 + markers = ['o', 'v', '^', '<', '>', 's', 'p', 'P','*', '+', 'x', 'X', 'd', 'D', 'H'] + colors = ['b', 'g', 'c', 'm', 'y', ] + + bboxes = vts.bboxes + maxfid = max(vts.bboxes[:, 7]) + + CART_HIGH_THRESH1 = 430 + TRACK_STATIC_THRESH = 8 + + fig = plt.figure(figsize=(16, 12)) + gs = fig.add_gridspec(2, 1, left=0.1, right=0.9, bottom=0.1, top=0.9, + wspace=0.05, hspace=0.15) + # ax1, ax2 = axs + + ax1 = fig.add_subplot(gs[0,0]) + ax2 = fig.add_subplot(gs[1,0]) + ax1.plot((0, maxfid+5), (1280-CART_HIGH_THRESH1, 1280-CART_HIGH_THRESH1), 'b--', linewidth=2 ) + ax2.plot((0, maxfid+5), (1280-CART_HIGH_THRESH1, 1280-CART_HIGH_THRESH1), 'b--', linewidth=2 ) + + + hands = [t for t in vts.Hands if not t.isHandStatic] + + tracks = vts.join_tracks(vts.Residual, hands) + + for i, track in enumerate(vts.tracks): + boxes = track.boxes + cls, tid = track.cls, track.tid + + y2, fids = boxes[:, 3], boxes[:, 7] + if cls==0: + ax1.scatter(fids, 1280-y2, marker='4', s=50, color=colors[tid%len(colors)], label = f"ID_{tid}") + else: + ax1.scatter(fids, 1280-y2, marker=markers[tid%len(markers)], color=colors[tid%len(colors)], + s=50, label = f"ID_{tid}") + # hist, bins = np.histogram(1280-y2, bins='auto') + ax1.set_ylim([-50, 1350]) + + for i, track in enumerate(tracks): + boxes = track.boxes + cls, tid = track.cls, track.tid + + y2, fids = boxes[:, 3], boxes[:, 7] + if cls==0: + ax2.scatter(fids, 1280-y2, marker='4', s=50, color=colors[tid%len(colors)], label = f"ID_{tid}") + else: + ax2.scatter(fids, 1280-y2, marker=markers[tid%len(markers)], color=colors[tid%len(colors)], + s=50, label = f"ID_{tid}") + # hist, bins = np.histogram(1280-y2, bins='auto') + ax2.set_ylim([-50, 1350]) + + + ax1.grid(True), ax1.set_xlim(0, maxfid+5), ax1.set_title('y2') + ax1.legend() + ax2.grid(True), ax2.set_xlim(0, maxfid+5), ax2.set_title('y2') + ax2.legend() + # plt.show() + + + + + return plt + + +def draw_all_trajectories(vts, edgeline, save_dir, file, draw5p=False): + '''显示四种类型结果''' + # file, ext = os.path.splitext(filename) + # edgeline = cv2.imread("./shopcart/cart_tempt/edgeline.png") + # edgeline2 = edgeline1.copy() + # edgeline = np.concatenate((edgeline1, edgeline2), exis = 1) + + if not isinstance(save_dir, Path): save_dir = Path(save_dir) + + ''' all tracks 中心轨迹''' + img1, img2 = edgeline.copy(), edgeline.copy() + img1 = drawTrack(vts.tracks, img1) + img2 = drawTrack(vts.Residual, img2) + + imgcat = np.concatenate((img1, img2), axis = 1) + H, W = imgcat.shape[:2] + cv2.line(imgcat, (int(W/2), 0), (int(W/2), H), (128, 255, 128), 2) + + + # imgpth = save_dir.joinpath(f"{file}_show.png") + # cv2.imwrite(str(imgpth), img) + + if not draw5p: + return imgcat + ## yi write piclke change + ''' tracks 5点轨迹''' + # trackpth = save_dir / Path("trajectory") + # if not trackpth.exists(): + # trackpth.mkdir(parents=True, exist_ok=True) + # for track in vts.tracks: + # # if track.cls != 0: + # img = edgeline.copy() + # img = draw5points(track, img) + # + # pth = trackpth.joinpath(f"{file}_{track.tid}_.png") + # cv2.imwrite(str(pth), img) + + # for track in vts.Residual: + # # if track.cls != 0: + # img = edgeline.copy() + # img = draw5points(track, img) + + # pth = trackpth.joinpath(f"{file}_{track.tid}_.png") + # cv2.imwrite(str(pth), img) + + return imgcat + + +# ============================================================================= +# '''3. moving tracks 中心轨迹''' +# filename2 = f"{file}_show_r.png" +# img = edgeline.copy() +# img = drawTrack(vts.Residual, img) +# pth = save_dir.joinpath(filename2) +# cv2.imwrite(pth, img) +# ============================================================================= + + + + '''5. tracks 时序：trajmin、trajmax、arearate、incartrate''' + # plt = drawtracefeat(vts) + # pth = save_dir.joinpath(f"{file}_x.png") + # plt.savefig(pth) + # plt.close('all') + + + + +def drawFeatures(allvts, save_dir): + # [trajlen_min, trajdist_max, trajlen_rate, trajist_rate]] + + feats = [track.TrajFeat for vts in allvts for track in vts.tracks] + feats = np.array(feats) + fig, ax = plt.subplots() + ax.scatter(feats[:,3], feats[:, 1], s=10) + # ax.set_xlim(0, 2) + # ax.set_ylim(0, 100) + ax.grid(True) + + plt.show() + + + pth = save_dir.joinpath("scatter.png") + plt.savefig(pth) + plt.close('all') + + + +def drawtracefeat(vts): + ''' + 需要对曲线进行特征提取和分类 + boxes: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] + 0 1 2 3 4 5 6 7 8 + ''' + # matplotlib.use('Agg') + fid = vts.frameid + fid1, fid2 = min(fid), max(fid) + + fig, axs = plt.subplots(2, 2,figsize=(18, 8)) + kernel = [0.15, 0.7, 0.15] + for i, track in enumerate(vts.tracks): + boxes = track.boxes + tid = int(track.tid) + cls = int(track.cls) + posState = track.posState + + if track.frnum>=5: + x1 = boxes[1:, 7] + y1 = track.trajmin + x11 = [i for i in range(int(min(x1)), int(max(x1)+1))] + y11 = np.interp(x11, x1, y1) + y11[1:-1] = np.convolve(y11, kernel, 'valid') + + x3 = boxes[1:, 7] + y3 = track.trajmax + x33 = [i for i in range(int(min(x3)), int(max(x3)+1))] + y33 = np.interp(x33, x3, y3) + y33[1:-1] = np.convolve(y33, kernel, 'valid') + + + x2 = boxes[:, 7] + # y2 = track.Area/max(track.Area) - min(track.Area/max(track.Area)) + y2 = track.Area/max(track.Area) + x22 = [i for i in range(int(min(x2)), int(max(x2)+1))] + y22 = np.interp(x22, x2, y2) + y22[1:-1] = np.convolve(y22, kernel, 'valid') + + x4 = boxes[:, 7] + y4 = track.incartrates + x44 = [i for i in range(int(min(x4)), int(max(x4)+1))] + y44 = np.interp(x44, x4, y4) + y44[1:-1] = np.convolve(y44, kernel, 'valid') + + + elif track.frnum>=2: + x11 = boxes[1:, 7] + y11 = track.trajmin + x33 = boxes[1:, 7] + y33 = track.trajmax + + x22 = boxes[:, 7] + # y22 = track.Area/max(track.Area) - min(track.Area/max(track.Area)) + y22 = track.Area/max(track.Area) + x44 = boxes[:, 7] + y44 = track.incartrates + else: + continue + + # cls!=0, max(y)>20 + if cls!=0 and cls!=9 and posState>=2 and max(y11)>10 and max(y33)>10 and max(y22>0.1): + axs[0, 0].plot(x11, y11, label=f"ID_{tid}") + axs[0, 0].legend() + # axs[0].set_ylim(0, 100) + + axs[0, 1].plot(x22, y22, label=f"ID_{tid}") + axs[0, 1].legend() + + axs[1, 0].plot(x33, y33, label=f"ID_{tid}") + axs[1, 0].legend() + + axs[1, 1].plot(x44, y44, label=f"ID_{tid}") + axs[1, 1].legend() + + + axs[0, 0].grid(True), axs[0, 0].set_xlim(fid1, fid2+10), axs[0, 0].set_title('trajmin') + axs[0, 1].grid(True), axs[0, 1].set_xlim(fid1, fid2+10), axs[0, 1].set_title('arearate') + axs[1, 0].grid(True), axs[1, 0].set_xlim(fid1, fid2+10), axs[1, 0].set_title('trajmax') + axs[1, 1].grid(True), axs[1, 1].set_xlim(fid1, fid2+10), axs[1, 1].set_ylim(-0.1, 1.1) + axs[1, 1].set_title('incartrate') + + # pth = save_dir.joinpath(f"{file}_show_x.png") + # plt.savefig(pth) + + # plt.savefig(f"./result/cls11_80212_time/{file}_show_x.png") + # plt.show() + + return plt + +def draw5points(track, img): + """ + 显示中心点、4角点的轨迹，以及轨迹 features + """ + colorx = np.array([[255, 255, 255], [255, 255, 255], [255, 255, 255], [255, 255, 255], [255, 255, 255], + [0, 0, 255], [0, 255, 0], [255, 51, 255], [102, 178, 255], [51, 153, 255],[255, 153, 153], + [255, 102, 102], [255, 51, 51], [153, 255, 153], [102, 255, 102], [51, 255, 51], + [255, 102, 255], [153, 204, 255], [255, 0, 0], [255, 255, 255]], dtype=np.uint8) + color = ((0, 0, 255), (255, 128, 0)) + + # img = cv2.imread("./shopcart/cart_tempt/edgeline.png") + boxes = track.boxes + cornpoints = track.cornpoints + + trajlens = [int(t) for t in track.trajlens] + trajdist = [int(t) for t in track.trajdist] + + if len(track.trajmin): + trajstd = np.std(track.trajmin) + else: + trajstd = 0 + + + trajlen_min, trajlen_max, trajdist_min, trajdist_max, trajlen_rate, trajdist_rate = track.TrajFeat + + for i in range(boxes.shape[0]): + cv2.circle(img, (int(cornpoints[i, 0]), int(cornpoints[i, 1])), 6, (255, 255, 255), 2) + cv2.circle(img, (int(cornpoints[i, 2]), int(cornpoints[i, 3])), 6, (255, 0, 255), 2) + cv2.circle(img, (int(cornpoints[i, 4]), int(cornpoints[i, 5])), 6, (0, 255, 0), 2) + cv2.circle(img, (int(cornpoints[i, 6]), int(cornpoints[i, 7])), 6, (64, 128, 255), 2) + cv2.circle(img, (int(cornpoints[i, 8]), int(cornpoints[i, 9])), 6, (255, 128, 64), 2) + + + + label_0 = f"ID: {track.tid}, Class: {track.cls}" + label_1 = f"trajlens: {trajlens}, trajlen_min: {int(trajlen_min)}" + label_2 = f"trajdist: {trajdist}: trajdist_max: {int(trajdist_max)}" + label_3 = "trajlen_min/trajlen_max: {:.2f}/{:.2f} = {:.2f}".format(trajlen_min, trajlen_max, trajlen_rate) + label_4 = "trajdist_min/mwh : {:.2f}/{:.2f} = {:.2f}".format(trajdist_min, track.mwh, trajdist_rate) + label_5 = "std(trajmin) : {:.2f}".format(trajstd) + label_6 = "PCA(variance_ratio) : " + label_7 = "Rect W&H&Ratio : " + + label_8 = "" + # label_8 = "IOU of incart/maxbox/minbox: {:.2f}, {:.2f}, {:.2f}".format( + # track.feature_ious[0], track.feature_ious[3], track.feature_ious[4]) + + + '''=============== 最小轨迹长度索引 ====================''' + trajlens = [int(t) for t in track.trajrects_wh] + if track.isCornpoint: + idx = 0 + else: + idx = trajlens.index(min(trajlens)) + + + '''=============== PCA ====================''' + if trajlens[idx] > 12: + X = cornpoints[:, 2*idx:2*(idx+1)] + + pca = PCA() + pca.fit(X) + label_6 = "PCA(variance_ratio): {:.2f}".format(pca.explained_variance_ratio_[0]) + # if sum(np.isnan(pca.explained_variance_ratio_)) == 0: + for i, (comp, var) in enumerate(zip(pca.components_, pca.explained_variance_ratio_)): + pt1 = (pca.mean_ - comp*var*200).astype(np.int64) + pt2 = (pca.mean_ + comp*var*200).astype(np.int64) + cv2.line(img, pt1, pt2, color=color[i], thickness=2) + + '''=============== RECT ====================''' + rect = track.trajrects[idx] + box = cv2.boxPoints(rect) + box = np.int0(box) + cv2.drawContours(img, [box], 0, (0, 255, 0), 2) + label_7 = "Rect W&H&Ratio: {}, {}, {:.2f}".format(int(rect[1][0]), int(rect[1][1]), min(rect[1])/(max(rect[1])+0.001)) + + + '''=============== 显示文字 ====================''' + # label = [label_0, label_1, label_2, label_3, label_4, label_5, label_6, label_7, label_8] + # w, h = cv2.getTextSize('abc', 0, fontScale=2, thickness=1)[0] + # for i in range(len(label)): + # cv2.putText(img, label[i], (20, int((i+1)*1.1*h)), 0, 1, + # [int(x) for x in colorx[i]], 2, lineType=cv2.LINE_AA) + + # pth = save_dir.joinpath(f"{file}_{track.tid}.png") + # cv2.imwrite(pth, img) + + '''撰写专利需要，生成黑白图像''' + # imgbt = cv2.bitwise_not(img) + # for i in range(box.shape[0]): + # cv2.circle(imgbt, (int(cornpoints[i, 0]), int(cornpoints[i, 1])), 14, (0, 0, 0), 2) + # cv2.drawMarker(imgbt, (int(cornpoints[i, 2]), int(cornpoints[i, 3])), color= (0, 0, 0), markerType=3, markerSize = 30, thickness=2) + # cv2.drawMarker(imgbt, (int(cornpoints[i, 4]), int(cornpoints[i, 5])), color= (0, 0, 0), markerType=4, markerSize = 30, thickness=2) + # cv2.drawMarker(imgbt, (int(cornpoints[i, 6]), int(cornpoints[i, 7])), color= (0, 0, 0), markerType=5, markerSize = 30, thickness=2) + # cv2.drawMarker(imgbt, (int(cornpoints[i, 8]), int(cornpoints[i, 9])), color= (0, 0, 0), markerType=6, markerSize = 30, thickness=2) + # cv2.imwrite(pth + f"/zhuanli/{file}_{track.tid}.png", imgbt) + + return img + +def drawTrack(tracks, img): + # img = cv2.imread("./shopcart/cart_tempt/edgeline.png") + annotator = TrackAnnotator(img, line_width=2) + + for track in tracks: + if isinstance(track, np.ndarray): + annotator.plotting_track(track) + else: + annotator.plotting_track(track.boxes) + img = annotator.result() + + # pth = save_dir.joinpath(f"{filename}") + # cv2.imwrite(pth, img) + + return img + + + + +if __name__ == "__main__": + + y = np.array([5.0, 20, 40, 41, 42, 55, 56]) + + + + diff --git a/tracking/utils/gen.py b/tracking/utils/gen.py new file mode 100644 index 0000000..29d8e49 --- /dev/null +++ b/tracking/utils/gen.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +""" +Created on Tue Jan 16 10:36:38 2024 + +@author: ym +""" +import contextlib +import time + +class Profile(contextlib.ContextDecorator): + # YOLOv5 Profile class. Usage: @Profile() decorator or 'with Profile():' context manager + def __init__(self, t=0.0): + self.t = t + # self.cuda = torch.cuda.is_available() + + def __enter__(self): + self.start = self.time() + return self + + def __exit__(self, type, value, traceback): + self.dt = self.time() - self.start # delta-time + self.t += self.dt # accumulate dt + + def time(self): + # if self.cuda: + # torch.cuda.synchronize() + return time.time() \ No newline at end of file diff --git a/tracking/utils/iterYaml.py b/tracking/utils/iterYaml.py new file mode 100644 index 0000000..4256fc6 --- /dev/null +++ b/tracking/utils/iterYaml.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +""" +Created on Sun Dec 31 17:07:09 2023 + +@author: ym +""" + +from pathlib import Path +from types import SimpleNamespace +import re +import yaml + +class IterableSimpleNamespace(SimpleNamespace): + """ + Ultralytics IterableSimpleNamespace is an extension class of SimpleNamespace that adds iterable functionality and + enables usage with dict() and for loops. + """ + + def __iter__(self): + """Return an iterator of key-value pairs from the namespace's attributes.""" + return iter(vars(self).items()) + + def __str__(self): + """Return a human-readable string representation of the object.""" + return '\n'.join(f'{k}={v}' for k, v in vars(self).items()) + + def __getattr__(self, attr): + """Custom attribute access error message with helpful information.""" + name = self.__class__.__name__ + raise AttributeError(f""" + '{name}' object has no attribute '{attr}'. This may be caused by a modified or out of date ultralytics + 'default.yaml' file.\nPlease update your code with 'pip install -U ultralytics' and if necessary replace + DEFAULT_CFG_PATH with the latest version from + https://github.com/ultralytics/ultralytics/blob/main/ultralytics/cfg/default.yaml + """) + + def get(self, key, default=None): + """Return the value of the specified key if it exists; otherwise, return the default value.""" + return getattr(self, key, default) + + + +def yaml_load(file='data.yaml', append_filename=False): + """ + Load YAML data from a file. + + Args: + file (str, optional): File name. Default is 'data.yaml'. + append_filename (bool): Add the YAML filename to the YAML dictionary. Default is False. + + Returns: + (dict): YAML data and file name. + """ + assert Path(file).suffix in ('.yaml', '.yml'), f'Attempting to load non-YAML file {file} with yaml_load()' + with open(file, errors='ignore', encoding='utf-8') as f: + s = f.read() # string + + # Remove special characters + if not s.isprintable(): + s = re.sub(r'[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD\U00010000-\U0010ffff]+', '', s) + + # Add YAML filename to dict and return + data = yaml.safe_load(s) or {} # always return a dict (yaml.safe_load() may return None for empty files) + if append_filename: + data['yaml_file'] = str(file) + return data \ No newline at end of file diff --git a/tracking/utils/mergetrack.py b/tracking/utils/mergetrack.py new file mode 100644 index 0000000..95787a7 --- /dev/null +++ b/tracking/utils/mergetrack.py @@ -0,0 +1,213 @@ +# -*- coding: utf-8 -*- +""" +Created on Fri Feb 23 11:04:48 2024 + +@author: ym +""" +import numpy as np +import cv2 +from scipy.spatial.distance import cdist +# from trackers.utils import matching + +# TracksDict +def readDict(boxes, TracksDict): + feats = [] + for i in range(boxes.shape[0]): + tid, fid, bid = int(boxes[i, 4]), int(boxes[i, 7]), int(boxes[i, 8]) + + trackdict = TracksDict[f"frame_{fid}"] + if "feats" in trackdict: + feat = trackdict["feats"][bid] + feats.append(feat) + + if "boxes" in trackdict: + box = trackdict["boxes"][bid] + assert (box[:4].astype(int) == boxes[i, :4].astype(int)).all(), f"Please check: frame_{fid}" + + if "imgs" in trackdict: + img = trackdict["imgs"][bid] + cv2.imwrite(f'./data/imgs/{tid}_{fid}_{bid}.png', img) + + return np.asarray(feats, dtype=np.float32) + + + +def track_equal_track(atrack, btrack): + # boxes: [x, y, w, h, track_id, score, cls, frame_index, box_index] + # 0 1 2 3 4 5 6 7 8 + aboxes = atrack.boxes + bboxes = btrack.boxes + + afeat = atrack.features + bfeat = btrack.features + + # afeat = readDict(aboxes, TracksDict) + # bfeat = readDict(bboxes, TracksDict) + + + ''' 1. 判断轨迹在时序上是否有交集 ''' + afids = aboxes[:, 7].astype(np.int_) + bfids = bboxes[:, 7].astype(np.int_) + # 帧索引交集 + interfid = set(afids).intersection(set(bfids)) + + # 或者直接判断帧索引是否有交集，返回 Ture or False + # interfid = set(afids).isdisjoint(set(bfids)) + + if len(interfid): + return False + + ''' 2. 轨迹特征相似度判断''' + feat = np.concatenate((afeat, bfeat), axis=0) + + emb_simil = 1 - np.maximum(0.0, cdist(feat, feat, 'cosine')) + emb_ = 1 - np.maximum(0.0, cdist(np.mean(afeat, axis=0)[None, :], np.mean(bfeat, axis=0)[None, :], 'cosine'))/2 + + if emb_[0, 0]<0.66: + return False + + ''' 3. 轨迹空间iou''' + alabel = np.array([0] * afids.size, dtype=np.int_) + blabel = np.array([1] * bfids.size, dtype=np.int_) + + label = np.concatenate((alabel, blabel), axis=0) + fids = np.concatenate((afids, bfids), axis=0) + indices = np.argsort(fids) + idx_pair = [] + for i in range(len(indices)-1): + idx1, idx2 = indices[i], indices[i+1] + if label[idx1] != label[idx2] and fids[idx2] - fids[idx1] <= 3: + if label[idx1] == 0: + a_idx = idx1 + b_idx = idx2-alabel.size + else: + a_idx = idx2 + b_idx = idx1-alabel.size + + idx_pair.append((a_idx, b_idx)) + + ious = [] + embs = [] + for a, b in idx_pair: + abox, bbox = aboxes[a, :], bboxes[b, :] + + af, bf = afeat[a, :], bfeat[b, :] + + emb_ab = 1 - np.maximum(0.0, cdist(af[None, :], bf[None, :], 'cosine')) + + + xa1, ya1 = abox[0] - abox[2]/2, abox[1] - abox[3]/2 + xa2, ya2 = abox[0] + abox[2]/2, abox[1] + abox[3]/2 + + xb1, yb1 = bbox[0] - bbox[2]/2, bbox[1] - bbox[3]/2 + xb2, yb2 = bbox[0] + bbox[2]/2, bbox[1] + bbox[3]/2 + + + inter = (np.minimum(xb2, xa2) - np.maximum(xb1, xa1)).clip(0) * \ + (np.minimum(yb2, ya2) - np.maximum(yb1, ya1)).clip(0) + + # Union Area + box1_area = abox[2] * abox[3] + box2_area = bbox[2] * bbox[3] + union = box1_area + box2_area - inter + 1e-6 + + ious.append(inter/union) + embs.append(emb_ab[0, 0]) + + ''' 4. 和同一手部关联，如何将该代码和 iou 部分相融合,需进一步完善''' + # ahands = np.array(atrack.Hands) + # bhands = np.array(btrack.Hands) + # ahids = ahands[:, 0] + # bhids = bhands[:, 0] + # interhid = set(ahids).intersection(set(bhids)) + # for hid in interhid: + # aidx = ahands[:, 0] == hid + # bidx = bhands[:, 0] == hid + + # ahfids = ahids[aidx, 1] + # bhfids = bhids[bidx, 1] + + + + + cont = False if len(interfid) else True # fid 无交集 + cont1 = all(emb > 0.5 for emb in embs) + cont2 = all(iou > 0.5 for iou in ious) + # cont = cont and cont2 and cont3 + + cont = cont and cont1 and cont2 + + + return cont + + + +def track_equal_str(atrack, btrack): + if atrack == btrack: + return True + else: + return False + + +def merge_track(Residual): + out_list = [] + alist = [t for t in Residual] + while alist: + atrack = alist[0] + cur_list = [] + cur_list.append(atrack) + alist.pop(0) + + blist = [b for b in alist] + alist = [] + for btrack in blist: + if track_equal_str(atrack, btrack): + cur_list.append(btrack) + else: + alist.append(btrack) + + out_list.append(cur_list) + return out_list + +def main(): + Residual = ['a', 'b', 'c', 'd', 'a', 'b', 'c', 'b', 'c', 'd'] + out_list = merge_track(Residual) + + print(Residual) + print(out_list) + +if __name__ == "__main__": + + main() + + + + +# ============================================================================= +# for i, atrack in enumerate(input_list): +# cur_list = [] +# cur_list.append(atrack) +# del input_list[i] +# +# for j, btrack in enumerate(input_list): +# if track_equal(atrack, btrack): +# cur_list.append(btrack) +# del input_list[j] +# +# out_list.append(cur_list) +# ============================================================================= + + + + + + + + + + + + + + + diff --git a/tracking/utils/plotting.py b/tracking/utils/plotting.py new file mode 100644 index 0000000..9eb4c78 --- /dev/null +++ b/tracking/utils/plotting.py @@ -0,0 +1,364 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import contextlib +import math +import warnings +from pathlib import Path +import os +import cv2 +import matplotlib.pyplot as plt +import numpy as np +import torch +from PIL import Image, ImageDraw, ImageFont +from PIL import __version__ as pil_version +# from utils.general import increment_path + +# from ultralytics.utils import LOGGER, TryExcept, ops, plt_settings, threaded + +# from .checks import check_font, check_version, is_ascii +# from .files import increment_path + + +class Colors: + """ + Ultralytics default color palette https://ultralytics.com/. + + This class provides methods to work with the Ultralytics color palette, including converting hex color codes to + RGB values. + + Attributes: + palette (list of tuple): List of RGB color values. + n (int): The number of colors in the palette. + pose_palette (np.array): A specific color palette array with dtype np.uint8. + """ + + def __init__(self): + """Initialize colors as hex = matplotlib.colors.TABLEAU_COLORS.values().""" + hexs = ('FF3838', 'FF9D97', 'FF701F', 'FFB21D', 'CFD231', '48F90A', '92CC17', '3DDB86', '1A9334', '00D4BB', + '2C99A8', '00C2FF', '344593', '6473FF', '0018EC', '8438FF', '520085', 'CB38FF', 'FF95C8', 'FF37C7') + + self.palette = [self.hex2rgb(f'#{c}') for c in hexs] + self.n = len(self.palette) + self.pose_palette = np.array([[255, 128, 0], [255, 153, 51], [255, 178, 102], [230, 230, 0], [255, 153, 255], + [153, 204, 255], [255, 102, 255], [255, 51, 255], [102, 178, 255], [51, 153, 255], + [255, 153, 153], [255, 102, 102], [255, 51, 51], [153, 255, 153], [102, 255, 102], + [51, 255, 51], [0, 255, 0], [0, 0, 255], [255, 0, 0], [255, 255, 255]], + dtype=np.uint8) + + def __call__(self, i, bgr=False): + """Converts hex color codes to RGB values.""" + c = self.palette[int(i) % self.n] + return (c[2], c[1], c[0]) if bgr else c + + @staticmethod + def hex2rgb(h): + """Converts hex color codes to RGB values (i.e. default PIL order).""" + return tuple(int(h[1 + i:1 + i + 2], 16) for i in (0, 2, 4)) + + +colors = Colors() # create instance for 'from utils.plots import colors' + + +class Annotator: + """ + Ultralytics Annotator for train/val mosaics and JPGs and predictions annotations. + + Attributes: + im (Image.Image or numpy array): The image to annotate. + pil (bool): Whether to use PIL or cv2 for drawing annotations. + font (ImageFont.truetype or ImageFont.load_default): Font used for text annotations. + lw (float): Line width for drawing. + skeleton (List[List[int]]): Skeleton structure for keypoints. + limb_color (List[int]): Color palette for limbs. + kpt_color (List[int]): Color palette for keypoints. + """ + + def __init__(self, im, line_width=None, font_size=None, font='Arial.ttf', pil=False, example='abc'): + """Initialize the Annotator class with image and line width along with color palette for keypoints and limbs.""" + assert im.data.contiguous, 'Image not contiguous. Apply np.ascontiguousarray(im) to Annotator() input images.' + self.im = im + self.lw = line_width or max(round(sum(im.shape) / 2 * 0.003), 2) # line width + # Pose + self.skeleton = [[16, 14], [14, 12], [17, 15], [15, 13], [12, 13], [6, 12], [7, 13], [6, 7], [6, 8], [7, 9], + [8, 10], [9, 11], [2, 3], [1, 2], [1, 3], [2, 4], [3, 5], [4, 6], [5, 7]] + + self.limb_color = colors.pose_palette[[9, 9, 9, 9, 7, 7, 7, 0, 0, 0, 0, 0, 16, 16, 16, 16, 16, 16, 16]] + self.kpt_color = colors.pose_palette[[16, 16, 16, 16, 16, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9]] + + def box_label(self, box, label='', color=(128, 128, 128), txt_color=(255, 255, 255)): + """Add one xyxy box to image with label.""" + if isinstance(box, torch.Tensor): + box = box.tolist() + + p1, p2 = (int(box[0]), int(box[1])), (int(box[2]), int(box[3])) + cv2.rectangle(self.im, p1, p2, color, thickness=self.lw, lineType=cv2.LINE_AA) + if label: + tf = max(self.lw - 1, 1) # font thickness + w, h = cv2.getTextSize(label, 0, fontScale=self.lw / 3, thickness=tf)[0] # text width, height + outside = p1[1] - h >= 3 + p2 = p1[0] + w, p1[1] - h - 3 if outside else p1[1] + h + 3 + cv2.rectangle(self.im, p1, p2, color, -1, cv2.LINE_AA) # filled + cv2.putText(self.im, + label, (p1[0], p1[1] - 2 if outside else p1[1] + h + 2), + 0, + self.lw / 3, + txt_color, + thickness=tf, + lineType=cv2.LINE_AA) + + def masks(self, masks, colors, im_gpu, alpha=0.5, retina_masks=False): + """ + Plot masks on image. + + Args: + masks (tensor): Predicted masks on cuda, shape: [n, h, w] + colors (List[List[Int]]): Colors for predicted masks, [[r, g, b] * n] + im_gpu (tensor): Image is in cuda, shape: [3, h, w], range: [0, 1] + alpha (float): Mask transparency: 0.0 fully transparent, 1.0 opaque + retina_masks (bool): Whether to use high resolution masks or not. Defaults to False. + """ + if self.pil: + # Convert to numpy first + self.im = np.asarray(self.im).copy() + if len(masks) == 0: + self.im[:] = im_gpu.permute(1, 2, 0).contiguous().cpu().numpy() * 255 + if im_gpu.device != masks.device: + im_gpu = im_gpu.to(masks.device) + colors = torch.tensor(colors, device=masks.device, dtype=torch.float32) / 255.0 # shape(n,3) + colors = colors[:, None, None] # shape(n,1,1,3) + masks = masks.unsqueeze(3) # shape(n,h,w,1) + masks_color = masks * (colors * alpha) # shape(n,h,w,3) + + inv_alph_masks = (1 - masks * alpha).cumprod(0) # shape(n,h,w,1) + mcs = masks_color.max(dim=0).values # shape(n,h,w,3) + + im_gpu = im_gpu.flip(dims=[0]) # flip channel + im_gpu = im_gpu.permute(1, 2, 0).contiguous() # shape(h,w,3) + im_gpu = im_gpu * inv_alph_masks[-1] + mcs + im_mask = (im_gpu * 255) + im_mask_np = im_mask.byte().cpu().numpy() + self.im[:] = im_mask_np if retina_masks else scale_image(im_mask_np, self.im.shape) + if self.pil: + # Convert im back to PIL and update draw + self.fromarray(self.im) + + def kpts(self, kpts, shape=(640, 640), radius=5, kpt_line=True): + """ + Plot keypoints on the image. + + Args: + kpts (tensor): Predicted keypoints with shape [17, 3]. Each keypoint has (x, y, confidence). + shape (tuple): Image shape as a tuple (h, w), where h is the height and w is the width. + radius (int, optional): Radius of the drawn keypoints. Default is 5. + kpt_line (bool, optional): If True, the function will draw lines connecting keypoints + for human pose. Default is True. + + Note: `kpt_line=True` currently only supports human pose plotting. + """ + if self.pil: + # Convert to numpy first + self.im = np.asarray(self.im).copy() + nkpt, ndim = kpts.shape + is_pose = nkpt == 17 and ndim == 3 + kpt_line &= is_pose # `kpt_line=True` for now only supports human pose plotting + for i, k in enumerate(kpts): + color_k = [int(x) for x in self.kpt_color[i]] if is_pose else colors(i) + x_coord, y_coord = k[0], k[1] + if x_coord % shape[1] != 0 and y_coord % shape[0] != 0: + if len(k) == 3: + conf = k[2] + if conf < 0.5: + continue + cv2.circle(self.im, (int(x_coord), int(y_coord)), radius, color_k, -1, lineType=cv2.LINE_AA) + + if kpt_line: + ndim = kpts.shape[-1] + for i, sk in enumerate(self.skeleton): + pos1 = (int(kpts[(sk[0] - 1), 0]), int(kpts[(sk[0] - 1), 1])) + pos2 = (int(kpts[(sk[1] - 1), 0]), int(kpts[(sk[1] - 1), 1])) + if ndim == 3: + conf1 = kpts[(sk[0] - 1), 2] + conf2 = kpts[(sk[1] - 1), 2] + if conf1 < 0.5 or conf2 < 0.5: + continue + if pos1[0] % shape[1] == 0 or pos1[1] % shape[0] == 0 or pos1[0] < 0 or pos1[1] < 0: + continue + if pos2[0] % shape[1] == 0 or pos2[1] % shape[0] == 0 or pos2[0] < 0 or pos2[1] < 0: + continue + cv2.line(self.im, pos1, pos2, [int(x) for x in self.limb_color[i]], thickness=2, lineType=cv2.LINE_AA) + if self.pil: + # Convert im back to PIL and update draw + self.fromarray(self.im) + + def rectangle(self, xy, fill=None, outline=None, width=1): + """Add rectangle to image (PIL-only).""" + self.draw.rectangle(xy, fill, outline, width) + + def text(self, xy, text, txt_color=(255, 255, 255), anchor='top', box_style=False): + """Adds text to an image using PIL or cv2.""" + if anchor == 'bottom': # start y from font bottom + w, h = self.font.getsize(text) # text width, height + xy[1] += 1 - h + if self.pil: + if box_style: + w, h = self.font.getsize(text) + self.draw.rectangle((xy[0], xy[1], xy[0] + w + 1, xy[1] + h + 1), fill=txt_color) + # Using `txt_color` for background and draw fg with white color + txt_color = (255, 255, 255) + if '\n' in text: + lines = text.split('\n') + _, h = self.font.getsize(text) + for line in lines: + self.draw.text(xy, line, fill=txt_color, font=self.font) + xy[1] += h + else: + self.draw.text(xy, text, fill=txt_color, font=self.font) + else: + if box_style: + tf = max(self.lw - 1, 1) # font thickness + w, h = cv2.getTextSize(text, 0, fontScale=self.lw / 3, thickness=tf)[0] # text width, height + outside = xy[1] - h >= 3 + p2 = xy[0] + w, xy[1] - h - 3 if outside else xy[1] + h + 3 + cv2.rectangle(self.im, xy, p2, txt_color, -1, cv2.LINE_AA) # filled + # Using `txt_color` for background and draw fg with white color + txt_color = (255, 255, 255) + tf = max(self.lw - 1, 1) # font thickness + cv2.putText(self.im, text, xy, 0, self.lw / 3, txt_color, thickness=tf, lineType=cv2.LINE_AA) + + def fromarray(self, im): + """Update self.im from a numpy array.""" + self.im = im if isinstance(im, Image.Image) else Image.fromarray(im) + self.draw = ImageDraw.Draw(self.im) + + def result(self): + """Return annotated image as array.""" + return np.asarray(self.im) + + +def scale_image(masks, im0_shape, ratio_pad=None): + """ + Takes a mask, and resizes it to the original image size + + Args: + masks (np.ndarray): resized and padded masks/images, [h, w, num]/[h, w, 3]. + im0_shape (tuple): the original image shape + ratio_pad (tuple): the ratio of the padding to the original image. + + Returns: + masks (torch.Tensor): The masks that are being returned. + """ + # Rescale coordinates (xyxy) from im1_shape to im0_shape + im1_shape = masks.shape + if im1_shape[:2] == im0_shape[:2]: + return masks + if ratio_pad is None: # calculate from im0_shape + gain = min(im1_shape[0] / im0_shape[0], im1_shape[1] / im0_shape[1]) # gain = old / new + pad = (im1_shape[1] - im0_shape[1] * gain) / 2, (im1_shape[0] - im0_shape[0] * gain) / 2 # wh padding + else: + gain = ratio_pad[0][0] + pad = ratio_pad[1] + top, left = int(pad[1]), int(pad[0]) # y, x + bottom, right = int(im1_shape[0] - pad[1]), int(im1_shape[1] - pad[0]) + + if len(masks.shape) < 2: + raise ValueError(f'"len of masks shape" should be 2 or 3, but got {len(masks.shape)}') + masks = masks[top:bottom, left:right] + masks = cv2.resize(masks, (im0_shape[1], im0_shape[0])) + if len(masks.shape) == 2: + masks = masks[:, :, None] + + return masks + + +def boxing_img(det, img, line_width=3): + annotator = Annotator(img, line_width) + for *xyxy, id, conf, cls, _, _ in reversed(det): + label = (f'id:{int(id)} '+str(int(cls)) +f' {conf:.2f}') + if cls==0: + color = colors(int(cls), True) + else: + color = colors(int(id), True) + annotator.box_label(xyxy, label, color=color) + + # Save results (image and video with tracking) + imgx = annotator.result() + + return imgx + +def array2list(bboxes): + track_fids = np.unique(bboxes[:, 7].astype(int)) + track_fids.sort() + + lboxes = [] + for f_id in track_fids: + # print(f"The ID is: {t_id}") + idx = np.where(bboxes[:, 7] == f_id)[0] + box = bboxes[idx, :] + lboxes.append(box) + + assert len(set(box[:, 4])) == len(box), "Please check!!!" + + return lboxes + + +def get_subimgs(imgs, tracks, scale=2): + bboxes = [] + if len(tracks): + bboxes = array2list(tracks) + + subimgs = [] + for i, boxes in enumerate(bboxes): + fid = int(boxes[0, 7]) + + for *xyxy, tid, conf, cls, fid, bid in boxes: + pt2 = [p/scale for p in xyxy] + x1, y1, x2, y2 = (int(pt2[0]), int(pt2[1])), (int(pt2[2]), int(pt2[3])) + subimgs.append((int(fid), int(bid), imgs[fid-1][y1:y2, x1:x2])) + + return subimgs + +def draw_tracking_boxes(imgs, tracks, scale=2): + '''需要确保 imgs 覆盖tracks中的帧ID数 + tracks: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] + 0 1 2 3 4 5 6 7 8 + 关键： + (1) imgs中的次序和 track 中的 fid 对应 + (2) img 尺度小对于xyxy减半 + + ''' + + + bboxes = [] + if len(tracks): + bboxes = array2list(tracks) + + # if len(bboxes)!=len(imgs): + # return False, imgs + + annimgs = [] + for i, boxes in enumerate(bboxes): + fid = int(boxes[0, 7]) + annotator = Annotator(imgs[fid-1].copy()) + for *xyxy, tid, conf, cls, fid, bid in boxes: + label = f'id:{int(tid)}_{int(cls)}_{conf:.2f}' + + if cls==0: + color = colors(int(cls), True) + elif tid>0 and cls!=0: + color = colors(int(tid), True) + else: + color = colors(19, True) # 19为调色板的最后一个元素 + + pt2 = [p/scale for p in xyxy] + annotator.box_label(pt2, label, color=color) + + img = annotator.result() + annimgs.append((int(fid), img)) + + return annimgs + + + + + + diff --git a/tracking/utils/proBoxes.py b/tracking/utils/proBoxes.py new file mode 100644 index 0000000..ee8ddaf --- /dev/null +++ b/tracking/utils/proBoxes.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +""" +Created on Sun Dec 31 17:14:37 2023 + +@author: ym +""" +import numpy as np + + +class Boxes: + def __init__(self, boxes, orig_shape=None) -> None: + """Initialize the Boxes class.""" + if boxes.ndim == 1: + boxes = boxes[None, :] + m, n = boxes.shape + assert n in (6, 7), f'expected `n` in [6, 7], but got {n}' # xyxy, track_id, conf, cls + + '''对每一个box进行编号，利用该编号可以索引对应 feature''' + self.data = np.concatenate([boxes[:, :4], np.arange(m).reshape(-1, 1), boxes[:, 4:]], axis=-1) + + self.orig_shape = orig_shape + + def cpu(self): + """Return a copy of the tensor on CPU memory.""" + return self if isinstance(self.data, np.ndarray) else self.__class__(self.data.cpu(), self.orig_shape) + def numpy(self): + """Return a copy of the tensor as a numpy array.""" + return self if isinstance(self.data, np.ndarray) else self.__class__(self.data.numpy(), self.orig_shape) + + @property + def xyxy(self): + """Return the boxes in xyxy format.""" + return self.data[:, :4] + + @property + def xyxyb(self): + """Return the boxes in xyxyb format.""" + return self.data[:, :5] + + @property + def conf(self): + """Return the confidence values of the boxes.""" + return self.data[:, -2] + + @property + def cls(self): + """Return the class values of the boxes.""" + return self.data[:, -1] + + + +# def boxes_add_fid(tboxes): +# ''' +# 将 bboxes 对应的帧索引添加到 boxes 最后一列 +# Return： +# bboxes: [x1, y1, x2, y2, track_id, score, cls, frame_index] +# ''' + +# bboxes = np.empty((0, 8), dtype = np.float32) +# for tbox, f in tboxes: +# data = tbox.numpy() + +# frame = f * np.ones([data.shape[0], 1]) +# bbox = np.concatenate([data, frame], axis=1) +# bboxes = np.concatenate([bboxes, bbox], axis=0) + +# return bboxes + + +def boxes_add_fid(tboxes): + ''' + 将 bboxes 对应的帧索引添加到 boxes 最后一列 + Return： + bboxes: [x1, y1, x2, y2, track_id, score, cls, frame_index] + ''' + + bboxes = np.empty((0, 8), dtype = np.float32) + for data, f in tboxes: + frame = f * np.ones([data.shape[0], 1]) + bbox = np.concatenate([data, frame], axis=1) + bboxes = np.concatenate([bboxes, bbox], axis=0) + + return bboxes + + + + + + + + + + + + diff --git a/tracking/utils/processboxes.py b/tracking/utils/processboxes.py new file mode 100644 index 0000000..384046e --- /dev/null +++ b/tracking/utils/processboxes.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- +""" +Created on Wed Sep 20 14:28:20 2023 + +@author: ym +""" +import numpy as np +from scipy.spatial.distance import cdist + + +def boxes_add_fid(tboxes): + ''' + 将 bboxes 对应的帧索引添加到 boxes 最后一列 + Return： + bboxes: [x1, y1, x2, y2, track_id, score, cls, frame_index] + ''' + + bboxes = np.empty((0, 8), dtype = np.float32) + for tbox, f in tboxes: + data = tbox.numpy() + + frame = f * np.ones([data.shape[0], 1]) + bbox = np.concatenate([data, frame], axis=1) + bboxes = np.concatenate([bboxes, bbox], axis=0) + + return bboxes + + + +def array2list(bboxes): + ''' + 将 bboxes 变换为 track 列表 + bboxes: [x1, y1, x2, y2, track_id, score, cls, frame_index] + Return： + lboxes：列表，列表中元素具有同一 track_id，xywh 格式 + [x, y, w, h, track_id, score, cls, frame_index] + ''' + track_ids = set(bboxes[:, 4]) + lboxes = [] + for t_id in track_ids: + idx = np.where(bboxes[:, 4] == t_id)[0] + box = bboxes[idx, :] + + x = (box[:, 0] + box[:, 2]) / 2 + y = (box[:, 1] + box[:, 3]) / 2 + + # box: [x, y, w, h, track_id, score, cls, frame_index] + box[:, 2] = box[:, 2] - box[:, 0] + box[:, 3] = box[:, 3] - box[:, 1] + box[:, 0] = x + box[:, 1] = y + + lboxes.append(box) + + return lboxes + + +def max_dist_track(tboxes): + + ''' + 计算 tboxes 中最大dist的 track + Return： + ''' + + max_track_dist, max_dist = 0, 0 + for track in tboxes: + box = track[:, :4].astype(int) + + dist = cdist(box[:, :2], box[:, :2]) + + dm = np.max(dist) + if dm > max_dist: + max_dist = dm + max_track = track.copy() + max_track_dist = dist.copy() + + + # 同一 track_id 中目标中心移动最大距离的索引（ix1, ix2） + indx, indy = np.where(dist == dm) + ix1, ix2 = indx[0], indy[0] + # 确保 ix1 < ix2，索引 ix1 是开始时的视频 + if ix1 > ix2: ix1, ix2 = ix2, ix1 + +# ============================================================================= +# # ============================================================================= +# # 逻辑分析 +# # ============================================================================= +# Scanzone = ((0, int(Height/4)), (int(2*Weight/3), Weight)) +# if max_track.shape[0] > 10: +# +# # max_track 视频序列的第一帧索引 idx1 +# frame_1 = int(min(max_track[:, 7])) +# idx1 = np.where(max_track[:, 7] == frame_1)[0][0] +# +# # max_track 视频序列的最后一帧索引 idx2 +# frame_2 = int(max(max_track[:, 7])) +# idx2 = np.where(max_track[:, 7] == frame_2)[0][0] +# +# # max_track 视频序列的第一帧目标位置中心 (x1, y1) +# x1, y1 = max_track[idx1, :2] +# +# # max_track 视频序列的第最后一帧目标位置中心 (x2, y2) +# x2, y2 = max_track[idx2, :2] +# +# +# # track序列第一帧和最后一帧的距离，该距离和 mx_dist 不是一个概念 +# dist_1_2 = max_track_dist[idx1, idx2] +# +# if max_dist < 3 * Height/10: +# State = Uncertain +# +# elif y1 > y2: +# State = TakeOut +# +# elif y1 < y2: +# State = PutIn +# ============================================================================= + return max_track, max_dist diff --git a/tracking/utils/read_data.py b/tracking/utils/read_data.py new file mode 100644 index 0000000..2d9f82f --- /dev/null +++ b/tracking/utils/read_data.py @@ -0,0 +1,750 @@ +# -*- coding: utf-8 -*- +""" +Created on Fri Jul 5 13:59:21 2024 + 函数 读取文件 + extract_data() 0/1_track.data + read_tracking_output() 0/1_tracking_output.data + read_similar() process.data + +@author: ym +""" +import numpy as np +import re +import os +from collections import OrderedDict +import warnings +import matplotlib.pyplot as plt + + +def str_to_float_arr(s): + # 移除字符串末尾的逗号（如果存在） + if s.endswith(','): + s = s[:-1] + + # 使用split()方法分割字符串，然后将每个元素转化为float + float_array = [float(x) for x in s.split(",")] + return float_array + + +def find_samebox_in_array(arr, target): + + for i, st in enumerate(arr): + if st[:4] == target[:4]: + return i + return -1 + +def array2list(boxes, feats): + '''boxes: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index]''' + + trackID = np.unique(boxes[:, 4].astype(int)) + + track_ids = boxes[:, 4].astype(int) + lboxes = [] + for t_id in trackID: + idx = np.where(track_ids == t_id)[0] + box = boxes[idx, :] + feat = feats[idx, :] + + assert len(set(box[:, 7])) == len(box), "Please check!!!" + lboxes.append(box) + + return lboxes + +def extract_data(datapath): + ''' + 0/1_track.data 数据读取 + 对于特征，需要构造两个对象： + (1) tracking输出 boxes，依据 (fid, bid) 找到对应的 feats, tracker_feat_dict 实现较为方便 + (2) frameDictList 中元素为 frameDict，输出同一接口 + ''' + bboxes, ffeats = [], [] + + trackerboxes = np.empty((0, 9), dtype=np.float64) + trackerfeats = np.empty((0, 256), dtype=np.float64) + + boxes, feats, tboxes, tfeats = [], [], [], [] + timestamps, frameIds = [], [] + with open(datapath, 'r', encoding='utf-8') as lines: + for line in lines: + line = line.strip() # 去除行尾的换行符和可能的空白字符 + if not line: # 跳过空行 + continue + + if line.find("CameraId")>=0: + if len(boxes): bboxes.append(np.array(boxes)) + if len(feats): ffeats.append(np.array(feats)) + if len(tboxes): + trackerboxes = np.concatenate((trackerboxes, np.array(tboxes))) + if len(tfeats): + trackerfeats = np.concatenate((trackerfeats, np.array(tfeats))) + + timestamp, frameId = [int(ln.split(":")[1]) for ln in line.split(",")[1:]] + timestamps.append(timestamp) + frameIds.append(frameId) + + boxes, feats, tboxes, tfeats = [], [], [], [] + + if line.find("box:") >= 0 and line.find("output_box:")<0 and line.find("out_boxes")<0: + box = line[line.find("box:") + 4:].strip() + # if len(box)==6: + boxes.append(str_to_float_arr(box)) + + if line.find("feat:") >= 0: + feat = line[line.find("feat:") + 5:].strip() + # if len(feat)==256: + feats.append(str_to_float_arr(feat)) + + if line.find("output_box:") >= 0: + assert(len(boxes)>=0 and len(boxes)==len(feats)), f"{datapath}, {datapath}, len(boxes)!=len(feats)" + + box = str_to_float_arr(line[line.find("output_box:") + 11:].strip()) + index = find_samebox_in_array(boxes, box) + if index >= 0: + tboxes.append(box) # 去掉'output_box:'并去除可能的空白字符 + + # feat_f = str_to_float_arr(input_feats[index]) + feat_f = feats[index] + norm_f = np.linalg.norm(feat_f) + feat_f = feat_f / norm_f + tfeats.append(feat_f) + + + + if len(boxes): bboxes.append(np.array(boxes)) + if len(feats): ffeats.append(np.array(feats)) + if len(tboxes): trackerboxes = np.concatenate((trackerboxes, np.array(tboxes))) + if len(tfeats): trackerfeats = np.concatenate((trackerfeats, np.array(tfeats))) + + assert(len(bboxes)==len(ffeats)), "Error at Yolo output!" + assert(len(trackerboxes)==len(trackerfeats)), "Error at tracker output!" + + ## 生成帧为单位的特征列表 + tracker_feats = {} + frmIDs = np.sort(np.unique(trackerboxes[:, 7].astype(int))) + for fid in frmIDs: + idx = np.where(trackerboxes[:, 7] == fid)[0] + boxes = trackerboxes[idx, :] + feats = trackerfeats[idx, :] + + for i in range(len(boxes)): + f, b = int(boxes[i, 7]), int(boxes[i, 8]) + tracker_feats.update({f"{f}_{b}": feats[i, :]}) + + boxes, trackingboxes= [], [] + tracking_flag = False + with open(datapath, 'r', encoding='utf-8') as lines: + for line in lines: + line = line.strip() # 去除行尾的换行符和可能的空白字符 + if not line: # 跳过空行 + tracking_flag = False + continue + + if tracking_flag: + if line.find("tracking_") >= 0: + tracking_flag = False + else: + box = str_to_float_arr(line) + boxes.append(box) + if line.find("tracking_") >= 0: + tracking_flag = True + if len(boxes): + trackingboxes.append(np.array(boxes)) + boxes = [] + + if len(boxes): + trackingboxes.append(np.array(boxes)) + + # tracking_feat_dict = {} + # try: + # for i, boxes in enumerate(trackingboxes): + # for box in boxes: + # tid, fid, bid = int(box[4]), int(box[7]), int(box[8]) + # if f"track_{tid}" not in tracking_feat_dict: + # tracking_feat_dict[f"track_{tid}"]= {"feats": {}} + # tracking_feat_dict[f"track_{tid}"]["feats"].update({f"{fid}_{bid}": tracker_feat_dict[f"frame_{fid}"]["feats"][bid]}) + # except Exception as e: + # print(f'Path: {datapath}, tracking_feat_dict can not be structured correcttly, Error: {e}') + + # tracker_feat_dict = {} + # tracker_feat_dict["timestamps"] = timestamps + # tracker_feat_dict["frameIds"] = frameIds + + # for i in range(len(trackerboxes)): + # fid, bid = int(trackerboxes[i, 7]), int(trackerboxes[i, 8]) + # if f"frame_{fid}" not in tracker_feat_dict: + # tracker_feat_dict[f"frame_{fid}"]= {"feats": {}} + # tracker_feat_dict[f"frame_{fid}"]["feats"].update({bid: trackerfeats[i, :]}) + + trackingfeats = [] + try: + for i, boxes in enumerate(trackingboxes): + feats = [] + for box in boxes: + fid, bid = int(box[7]), int(box[8]) + feat = tracker_feats[f"{fid}_{bid}"] + feats.append(feat) + trackingfeats.append(np.array(feats)) + except Exception as e: + print(f'Path: {datapath}, trackingfeats can not be structured correcttly, Error: {e}') + + # return bboxes, ffeats, trackerboxes, tracker_feat_dict, trackingboxes, trackingfeats + return bboxes, ffeats, trackerboxes, tracker_feats, trackingboxes, trackingfeats + + + + +def extract_data_realtime(datapath): + boxes, feats = [], [] + tracker_feats = {} + with open(datapath, 'r', encoding='utf-8') as lines: + for line in lines: + line = line.strip() # 去除行尾的换行符和可能的空白字符 + if not line: # 跳过空行 + continue + + if line.endswith(','): + line = line[:-1] + ftlist = [float(x) for x in line.split()] + + if len(ftlist) != 265: + continue + boxes.append(ftlist[:9]) + feats.append(ftlist[9:]) + + + trackerboxes = np.array(boxes) + trackerfeats = np.array(feats) + + if len(trackerboxes)==0 or len(trackerboxes) != len(trackerfeats): + return np.array([]), {} + + frmIDs = np.sort(np.unique(trackerboxes[:, 7].astype(int))) + for fid in frmIDs: + idx = np.where(trackerboxes[:, 7] == fid)[0] + box = trackerboxes[idx, :] + feat = trackerfeats[idx, :] + + for i in range(len(box)): + f, b = int(box[i, 7]), int(box[i, 8]) + tracker_feats.update({f"{f}_{b}": feat[i, :]}) + return trackerboxes, tracker_feats + + +def read_tracking_output_realtime(datapath): + + trackingboxes, trackingfeats = [], [] + tracking_outboxes, tracking_outfeats = [], [] + with open(datapath, 'r', encoding='utf-8') as lines: + boxes, feats = [], [] + Flag = False + for line in lines: + line = line.strip() # 去除行尾的换行符和可能的空白字符 + if not line: # 跳过空行 + continue + if line.endswith(','): + line = line[:-1] + + ftlist = [float(x) for x in line.split()] + if len(ftlist) != 265: continue + + Flag = all(elem == 0 for elem in ftlist) + if Flag: + trackingboxes.append(np.array(boxes)) + trackingfeats.append(np.array(feats)) + boxes, feats = [], [] + continue + + boxes.append(ftlist[:9]) + feats.append(ftlist[9:]) + if len(boxes): + trackingboxes.append(np.array(boxes)) + trackingfeats.append(np.array(feats)) + if len(trackingboxes): + tracking_outboxes = trackingboxes[:1] + tracking_outfeats = trackingfeats[:1] + + return trackingboxes, trackingfeats, tracking_outboxes, tracking_outfeats + + +def read_tracking_output(filepath): + ''' + 0/1_tracking_output.data 数据读取 + ''' + + boxes = [] + feats = [] + if not os.path.isfile(filepath): + return boxes, feats + + with open(filepath, 'r', encoding='utf-8') as file: + for line in file: + line = line.strip() # 去除行尾的换行符和可能的空白字符 + + if not line: + continue + if line.find("gift")>0: continue + + if line.endswith(','): + line = line[:-1] + + data = np.array([float(x) for x in line.split(",")]) + if data.size == 9: + boxes.append(data) + if data.size == 256: + feats.append(data) + + + if len(feats) != len(boxes) or len(boxes)==0: + return [], [] + + return [np.array(boxes)], [np.array(feats)] + + +def read_process(filePath): + timeDict = {} + with open(filePath, 'r', encoding='utf-8') as f: + lines = f.readlines() + clines = [line.strip().replace("'", '').replace('"', '') for line in lines] + for i, line in enumerate(clines): + line = line.strip() + if line.endswith(','): + line = line[:-1] + if not line: continue + + lnList = line.split(":") + if line.find("eventStart")>=0: + timeDict["eventStart"] = int(lnList[-1]) + + if line.find("eventEnd")>=0: + timeDict["eventEnd"] = int(lnList[-1]) + + if line.find("weightValue")>=0: + timeDict["weightValue"] = int(lnList[-1]) + return timeDict + + +def read_similar(filePath): + '''1:n时 Dict['type']字段提取和非全实时不一致，无 "=" 字符 ''' + + SimiDict = {} + SimiDict['one2one'] = [] + SimiDict['one2SN'] = [] + SimiDict['one2n'] = [] + SimiDict['algroStartToEnd'] = -1 + + with open(filePath, 'r', encoding='utf-8') as f: + lines = f.readlines() + clean_lines = [line.strip().replace("'", '').replace('"', '') for line in lines] + one2one_list, one2SN_list, one2n_list = [], [], [] + + Flag_1to1, Flag_1toSN, Flag_1ton = False, False, False + for i, line in enumerate(clean_lines): + line = line.strip() + if line.endswith(','): + line = line[:-1] + Dict = {} + + if not line: + if len(one2one_list): SimiDict['one2one'] = one2one_list + if len(one2SN_list): SimiDict['one2SN'] = one2SN_list + if len(one2n_list): SimiDict['one2n'] = one2n_list + + + one2one_list, one2SN_list, one2n_list = [], [], [] + Flag_1to1, Flag_1toSN, Flag_1ton = False, False, False + continue + + if line.find("algroStartToEnd")>=0: + alg_during = line.split(':')[1].strip() + SimiDict['algroStartToEnd'] = int(alg_during) + + + + + if line.find('oneToOne')>=0: + Flag_1to1, Flag_1toSN, Flag_1ton = True, False,False + continue + if line.find('oneToSN')>=0: + Flag_1to1, Flag_1toSN, Flag_1ton = False, True, False + continue + if line.find('oneTon')>=0: + Flag_1to1, Flag_1toSN, Flag_1ton = False, False, True + continue + + + + if Flag_1to1: + barcode = line.split(',')[0].strip() + value = line.split(',')[1].split(':')[1].strip() + Dict['barcode'] = barcode + Dict['similar'] = float(value) + one2one_list.append(Dict) + continue + + if Flag_1toSN: + barcode = line.split(',')[0].strip() + value = line.split(',')[1].split(':')[1].strip() + Dict['barcode'] = barcode + Dict['similar'] = float(value) + one2SN_list.append(Dict) + continue + + if Flag_1ton: + label = line.split(':')[0].strip() + value = line.split(':')[1].strip() + Dict['barcode'] = '' + if label.find("_") > 0: + bcd = label.split('_')[-1] + if len(bcd)>=8 and bcd.isdigit(): + Dict['barcode'] = bcd + + + + Dict['event'] = label + Dict['similar'] = float(value.split(',')[0]) + + if value.find("=")>0: + Dict['type'] = value.split('=')[-1] + else: + Dict['type'] = value.split(',')[-1] + one2n_list.append(Dict) + + if len(one2one_list): SimiDict['one2one'] = one2one_list + if len(one2n_list): SimiDict['one2n'] = one2n_list + if len(one2SN_list): SimiDict['one2SN'] = one2SN_list + + return SimiDict + +def read_weight_sensor(filepath): + WeightDict = OrderedDict() + with open(filepath, 'r', encoding='utf-8') as f: + lines = f.readlines() + clean_lines = [line.strip().replace("'", '').replace('"', '') for line in lines] + for i, line in enumerate(clean_lines): + line = line.strip() + + if line.find(':') < 0: continue + if line.find("Weight") >= 0: + label = "Weight" + continue + + + keyword = line.split(':')[0] + value = line.split(':')[1] + + if label == "Weight": + vdata = [float(s) for s in value.split(',') if len(s) and s.isdigit()] + WeightDict[keyword] = vdata[-1] + + + weights = [(float(t), w) for t, w in WeightDict.items()] + weights = np.array(weights).astype(np.int64) + + return weights + +def read_weight_timeConsuming(filePth): + WeightDict, SensorDict, ProcessTimeDict = OrderedDict(), OrderedDict(), OrderedDict() + + with open(filePth, 'r', encoding='utf-8') as f: + lines = f.readlines() + # label = '' + for i, line in enumerate(lines): + line = line.strip() + + if line.find(':') < 0: continue + if line.find("Weight") >= 0: + label = "Weight" + continue + if line.find("Sensor") >= 0: + label = "Sensor" + continue + if line.find("processTime") >= 0: + label = "ProcessTime" + continue + + keyword = line.split(':')[0] + value = line.split(':')[1] + + if label == "Weight": + WeightDict[keyword] = float(value.strip(',')) + if label == "Sensor": + SensorDict[keyword] = [float(s) for s in value.split(',') if len(s)] + if label == "ProcessTime": + ProcessTimeDict[keyword] = float(value.strip(',')) + + # print("Done!") + return WeightDict, SensorDict, ProcessTimeDict + + + + +def read_deletedBarcode_file(filePath): + with open(filePath, 'r', encoding='utf-8') as f: + lines = f.readlines() + + split_flag, all_list = False, [] + dict, barcode_list, similarity_list = {}, [], [] + clean_lines = [line.strip().replace("'", '').replace('"', '') for line in lines] + + for i, line in enumerate(clean_lines): + if line.endswith(','): + line = line[:-1] + + stripped_line = line.strip() + if not stripped_line: + if len(barcode_list): dict['barcode'] = barcode_list + if len(similarity_list): dict['similarity'] = similarity_list + if len(dict): all_list.append(dict) + + split_flag = False + dict, barcode_list, similarity_list = {}, [], [] + continue + + if line.find(':')<0: continue + label = line.split(':')[0] + value = line.split(':')[1] + + if label == 'SeqDir': + dict['SeqDir'] = value + dict['filetype'] = "deletedBarcode" + if label == 'Deleted': + dict['Deleted'] = value + if label == 'List': + split_flag = True + continue + if split_flag: + barcode_list.append(label) + similarity_list.append(value) + + if len(barcode_list): dict['barcode'] = barcode_list + if len(similarity_list): dict['similarity'] = similarity_list + if len(dict): all_list.append(dict) + return all_list + + +def read_returnGoods_file(filePath): + ''' + 20241030开始，原 deletedBarcode.txt 中数据格式修改为 returnGoods.txt，读数方式随之变化 + ''' + + with open(filePath, 'r', encoding='utf-8') as f: + lines = f.readlines() + clean_lines = [line.strip().replace("'", '').replace('"', '') for line in lines] + + all_list = [] + split_flag, dict = False, {} + barcode_list, similarity_list = [], [] + event_list, type_list = [], [] + + + for i, line in enumerate(clean_lines): + stripped_line = line.strip() + if line.endswith(','): + line = line[:-1] + + if not stripped_line: + if len(barcode_list): dict['barcode'] = barcode_list + if len(similarity_list): dict['similarity'] = similarity_list + if len(event_list): dict['event'] = event_list + if len(type_list): dict['type'] = type_list + + if len(dict) and dict['SeqDir'].find('*')<0: + all_list.append(dict) + + split_flag, dict = False, {} + barcode_list, similarity_list = [], [] + event_list, type_list = [], [] + continue + + if line.find(':')<0: continue + if line.find('1:n')==0: continue + + label = line.split(':')[0].strip() + value = line.split(':')[1].strip() + + if label == 'SeqDir': + dict['SeqDir'] = value + dict['Deleted'] = value.split('_')[-1] + dict['filetype'] = "returnGoods" + if label == 'List': + split_flag = True + continue + if split_flag: + bcd = label.split('_')[-1] + if len(bcd)<8: continue + # event_list.append(label + '_' + bcd) + event_list.append(label) + barcode_list.append(bcd) + similarity_list.append(value.split(',')[0]) + type_list.append(value.split('=')[-1]) + + + if len(barcode_list): dict['barcode'] = barcode_list + if len(similarity_list): dict['similarity'] = similarity_list + if len(event_list): dict['event'] = event_list + if len(type_list): dict['type'] = type_list + if len(dict) and dict['SeqDir'].find('*')<0: + all_list.append(dict) + + return all_list + + +def analyse_simi(simi_list, one2n_flag=False): + dict_ = {} + # maxSimi_dict = sorted(simi_list, key=lambda x: x['similar'], reverse=True)[0] + + for i, simi_dict in enumerate(simi_list): + if one2n_flag: + event = simi_dict['event'] + simi = simi_dict['similar'] + # print(f"barcode:{barcode}, simi:{simi}") + dict_[event] = simi + else: + barcode = simi_dict['barcode'] + simi = simi_dict['similar'] + # print(f"barcode:{barcode}, simi:{simi}") + dict_[barcode] = simi + return dict_ +def get_process_csv_data(csv_path, event_path): + dict_data = {} + event = event_path.name + process_file = event_path / 'process.data' + # print('**********process_file', process_file) + simidct = read_similar(process_file) + # print('simidct', simidct) + ###========================解析每个相似度====================== + one2one_list = simidct['one2one'] + one2SN_list = simidct['one2SN'] + one2n_list = simidct['one2n'] + # print('one2one_list', one2one_list) + + + if len(one2one_list): + one2one_dict = analyse_simi(one2one_list) + dict_data['one2one'] = str(list(one2one_dict.values())[0]) + # print('one2one_dict', one2one_dict) + if len(one2SN_list): + one2SN_dict = analyse_simi(one2SN_list) + dict_data['one2SN'] = one2SN_dict + # print('one2SN_dict', one2SN_dict) + + if len(one2n_list): + one2n_dict = analyse_simi(one2n_list, one2n_flag=True) + dict_data['one2n'] = one2n_dict + # print('one2n_dict', one2n_dict) + algroStartToEnd = str(simidct['algroStartToEnd']) + dict_data['algroStartToEnd'] = algroStartToEnd + ###========================================================= + + ####人工标注初始化excel文件中已有 + # barcode = read_csv(csv_path, event) + # dict_data['barcode'] = barcode + return dict_data + + + +def plot_sensor_curve(WeightDict, SensorDict, ProcessTimeDict): + + wtime, wdata = [], [] + stime, sdata = [], [] + for key, value in WeightDict.items(): + wtime.append(int(key)) + wdata.append(value) + + for key, value in SensorDict.items(): + if len(value) != 9: continue + + stime.append(int(key)) + sdata.append(np.array(value)) + + static_range = [] + dynamic_range = [] + windth = 8 + nw = len(wdata) + assert(nw) >= 8, "The num of weight data is less than 8!" + + # i1, i2 = 0, 7 + # while i2 < nw: + # data = wdata[i1:(i2+1)] + # max(data) - min(data) + # if i2<7: + # i1 = 0 + # else: + # i1 = i2-windth + + min_t = min(wtime + stime) + wtime = [t-min_t for t in wtime] + stime = [t-min_t for t in stime] + + max_t = max(wtime + stime) + + fig = plt.figure(figsize=(16, 12)) + gs = fig.add_gridspec(2, 1, left=0.1, right=0.9, bottom=0.1, top=0.9, + wspace=0.05, hspace=0.15) + # ax1, ax2 = axs + + ax1 = fig.add_subplot(gs[0,0]) + ax2 = fig.add_subplot(gs[1,0]) + + ax1.plot(wtime, wdata, 'b--', linewidth=2 ) + for i in range(9): + ydata = [s[i] for s in sdata] + ax2.plot(stime, ydata, linewidth=2 ) + + ax1.grid(True), ax1.set_xlim(0, max_t), ax1.set_title('Weight') + ax1.set_label("(Time: ms)") + # ax1.legend() + + ax2.grid(True), ax2.set_xlim(0, max_t), ax2.set_title('IMU') + # ax2.legend() + + plt.show() + + + + + + +def test_process(file_path): + WeightDict, SensorDict, ProcessTimeDict = read_weight_timeConsuming(file_path) + plot_sensor_curve(WeightDict, SensorDict, ProcessTimeDict) + + +def main(): + files_path = r'\\192.168.1.28\share\测试_202406\0814\0814\20240814-102227-62264578-a720-4eb9-b95e-cb8be009aa98_null' + k = 0 + for filename in os.listdir(files_path): + filename = 'process.data' + + file_path = os.path.join(files_path, filename) + if os.path.isfile(file_path) and filename.find("track.data")>0: + extract_data(file_path) + + if os.path.isfile(file_path) and filename.find("process.data")>=0: + test_process(file_path) + + k += 1 + if k == 1: + break + +def main1(): + fpath = r'\\192.168.1.28\share\测试视频数据以及日志\各模块测试记录\比对测试\1209永辉超市测试\20241209-155924-117e1941-70f8-4287-8de1-4866868548a6_6926475209967\process.data' + simidct = read_similar(fpath) + print(simidct) + + +if __name__ == "__main__": + main1() + + + + + + + + + + + + + diff --git a/tracking/utils/read_pipeline_data.py b/tracking/utils/read_pipeline_data.py new file mode 100644 index 0000000..d0de65a --- /dev/null +++ b/tracking/utils/read_pipeline_data.py @@ -0,0 +1,250 @@ +# -*- coding: utf-8 -*- +""" +Created on Tue May 21 15:25:23 2024 +读取 Pipeline 各模块的数据，主代码由 马晓慧 完成 + +@author: ieemoo-zl003 +""" + +import os +import numpy as np + +# 替换为你的目录路径 +files_path = 'D:/contrast/dataset/1_to_n/709/20240709-112658_6903148351833/' + +def str_to_float_arr(s): + # 移除字符串末尾的逗号（如果存在） + if s.endswith(','): + s = s[:-1] + + # 使用split()方法分割字符串，然后将每个元素转化为float + float_array = np.array([float(x) for x in s.split(",")]) + return float_array + +def extract_tracker_input_boxes_feats(file_name): + boxes = [] + feats = [] + with open(file_name, 'r', encoding='utf-8') as file: + for line in file: + line = line.strip() # 去除行尾的换行符和可能的空白字符 + + # 跳过空行 + if not line: + continue + + # 检查是否以'box:'或'feat:'开始 + if line.find("box:") >= 0 and line.find("output_box:") < 0: + box = line[line.find("box:") + 4:].strip() + boxes.append(str_to_float_arr(box)) # 去掉'box:'并去除可能的空白字符 + + if line.find("feat:") >= 0: + feat = line[line.find("feat:") + 5:].strip() + feats.append(str_to_float_arr(feat)) # 去掉'box:'并去除可能的空白字符 + + return np.array(boxes), np.array(feats) + +def find_string_in_array(arr, target): + """ + 在字符串数组中找到目标字符串对应的行（索引）。 + + 参数: + arr -- 字符串数组 + target -- 要查找的目标字符串 + + 返回: + 目标字符串在数组中的索引。如果未找到，则返回-1。 + """ + tg = [float(t) for k, t in enumerate(target.split(',')) if k<4][:4] + for i, st in enumerate(arr): + st = [float(s) for k, s in enumerate(target.split(',')) if k<4][:4] + + if st == tg: + return i + + # if st[:20] == target[:20]: + # return i + return -1 + +def find_samebox_in_array(arr, target): + + for i, st in enumerate(arr): + if all(st[:4] == target[:4]): + return i + return -1 + + + + + + +def extract_tracker_output_boxes_feats(read_file_name): + + input_boxes, input_feats = extract_tracker_input_boxes_feats(read_file_name) + + boxes = [] + feats = [] + with open(read_file_name, 'r', encoding='utf-8') as file: + for line in file: + line = line.strip() # 去除行尾的换行符和可能的空白字符 + + # 跳过空行 + if not line: + continue + + # 检查是否以'output_box:'开始 + if line.find("output_box:") >= 0: + box = str_to_float_arr(line[line.find("output_box:") + 11:].strip()) + boxes.append(box) # 去掉'output_box:'并去除可能的空白字符 + index = find_samebox_in_array(input_boxes, box) + if index >= 0: + # feat_f = str_to_float_arr(input_feats[index]) + feat_f = input_feats[index] + norm_f = np.linalg.norm(feat_f) + feat_f = feat_f / norm_f + feats.append(feat_f) + return input_boxes, input_feats, np.array(boxes), np.array(feats) + +def extract_tracking_output_boxes_feats(read_file_name): + tracker_boxes, tracker_feats, input_boxes, input_feats = extract_tracker_output_boxes_feats(read_file_name) + boxes = [] + feats = [] + + tracking_flag = False + with open(read_file_name, 'r', encoding='utf-8') as file: + for line in file: + line = line.strip() # 去除行尾的换行符和可能的空白字符 + + # 跳过空行 + if not line: + continue + + if tracking_flag: + if line.find("tracking_") >= 0: + tracking_flag = False + else: + box = str_to_float_arr(line) + boxes.append(box) + index = find_samebox_in_array(input_boxes, box) + if index >= 0: + feats.append(input_feats[index]) + # 检查是否以tracking_'开始 + if line.find("tracking_") >= 0: + tracking_flag = True + + assert(len(tracker_boxes)==len(tracker_feats)), "Error at Yolo output" + assert(len(input_boxes)==len(input_feats)), "Error at tracker output" + assert(len(boxes)==len(feats)), "Error at tracking output" + + return tracker_boxes, tracker_feats, input_boxes, input_feats, np.array(boxes), np.array(feats) + +def read_tracking_input(datapath): + with open(datapath, 'r') as file: + lines = file.readlines() + + data = [] + for line in lines: + data.append([s for s in line.split(',') if len(s)>=3]) + # data.append([float(s) for s in line.split(',') if len(s)>=3]) + + # data = np.array(data, dtype = np.float32) + try: + data = np.array(data, dtype = np.float32) + except Exception as e: + data = np.array([], dtype = np.float32) + print('DataError for func: read_tracking_input()') + + + return data + + + +def read_tracker_input(datapath): + with open(datapath, 'r') as file: + lines = file.readlines() + Videos = [] + FrameBoxes, FrameFeats = [], [] + boxes, feats = [], [] + + timestamp = [] + t1 = None + for line in lines: + if line.find('CameraId') >= 0: + t = int(line.split(',')[1].split(':')[1]) + timestamp.append(t) + + if len(boxes) and len(feats): + FrameBoxes.append(np.array(boxes, dtype = np.float32)) + FrameFeats.append(np.array(feats, dtype = np.float32)) + boxes, feats = [], [] + + if t1 and t - t1 > 1e3: + Videos.append((FrameBoxes, FrameFeats)) + FrameBoxes, FrameFeats = [], [] + t1 = int(line.split(',')[1].split(':')[1]) + + if line.find('box') >= 0: + box = line.split(':', )[1].split(',')[:-1] + boxes.append(box) + + + if line.find('feat') >= 0: + feat = line.split(':', )[1].split(',')[:-1] + feats.append(feat) + + FrameBoxes.append(np.array(boxes, dtype = np.float32)) + FrameFeats.append(np.array(feats, dtype = np.float32)) + Videos.append((FrameBoxes, FrameFeats)) + + # TimeStamp = np.array(timestamp, dtype = np.int64) + # DimesDiff = np.diff((TimeStamp)) + # sorted_indices = np.argsort(TimeStamp) + # TimeStamp_sorted = TimeStamp[sorted_indices] + # DimesDiff_sorted = np.diff((TimeStamp_sorted)) + + return Videos + + + +def main(): + files_path = 'D:/contrast/dataset/1_to_n/709/20240709-112658_6903148351833/' + + # 遍历目录下的所有文件和目录 + for filename in os.listdir(files_path): + # 构造完整的文件路径 + file_path = os.path.join(files_path, filename) + if os.path.isfile(file_path) and filename.find("track.data")>0: + tracker_boxes, tracker_feats, tracking_boxes, tracking_feats, output_boxes, output_feats = extract_tracking_output_boxes_feats(file_path) + + print("Done") + + +if __name__ == "__main__": + main() + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tracking/utils/rename.py b/tracking/utils/rename.py new file mode 100644 index 0000000..500a128 --- /dev/null +++ b/tracking/utils/rename.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +""" +Created on Sat Jun 8 09:51:59 2024 + +@author: ym +""" +import os + +def main(): + directory = r'D:\DetectTracking\runs\detect' + directory = r'D:\DetectTracking\tracking\result\tracks' + + suffix = '_' + + for root, dirs, files in os.walk(directory): + for name in dirs: + old_name = os.path.join(root, name) + new_name = os.path.join(root, f"{name}{suffix}") + try: + os.rename(old_name, new_name) + except Exception as e: + print(f"Failed to rename directory '{old_name}': {e}") + + for name in files: + old_name = os.path.join(root, name) + file, ext = os.path.splitext(name) + new_name = os.path.join(root, f"{file}{suffix}{ext}") + try: + os.rename(old_name, new_name) + except Exception as e: + print(f"Failed to rename file '{old_name}': {e}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tracking/utils/showtrack.py b/tracking/utils/showtrack.py new file mode 100644 index 0000000..2245791 --- /dev/null +++ b/tracking/utils/showtrack.py @@ -0,0 +1,361 @@ +# -*- coding: utf-8 -*- +""" +Created on Wed Jul 5 10:01:11 2023 + +@author: ym +""" + +import numpy as np +import os +import cv2 +import sys + +from scipy.spatial.distance import cdist +import matplotlib +import matplotlib.pyplot as plt +from sklearn.decomposition import PCA +# from ultralytics.utils.plotting import Annotator, colors + + +from .annotator import TrackAnnotator +# from .processboxes import array2list + +# boxes Format: [x1, y1, x2, y2, track_id, score, cls, frame_index] +pth = r"D:/DeepLearning/yolov5/tracking/" +colors = np.array([[255, 255, 255], [255, 255, 255], [255, 255, 255], [255, 255, 255], [255, 255, 255], + [0, 0, 255], [0, 255, 0], [255, 51, 255], [102, 178, 255], [51, 153, 255], + [255, 153, 153], [255, 102, 102], [255, 51, 51], [153, 255, 153], [102, 255, 102], + [51, 255, 51], [255, 102, 255], [153, 204, 255], [255, 0, 0], [255, 255, 255]], + dtype=np.uint8) + +def array2list(bboxes): + ''' + 将 bboxes 变换为 track 列表 + bboxes: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] + Return： + lboxes：列表，列表中元素具有同一 track_id，x1y1x2y2 格式 + [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] + ''' + tids = set(bboxes[:, 4]) + track_ids = bboxes[:, 4].astype(int) + lboxes = [] + for t_id in tids: + # print(f"The ID is: {t_id}") + idx = np.where(track_ids == t_id)[0] + box = bboxes[idx, :] + + lboxes.append(box) + + return lboxes + + + +def draw5points(bboxes, file): + """ + 显示中心点、4角点的轨迹，以及轨迹 features + """ + image = cv2.imread(pth + r"/shopcart/cart_tempt/edgeline.png") + + imgx = image.copy() + annotator = TrackAnnotator(imgx, line_width=2) + + lboxes = array2list(bboxes) + + + for k in range(len(lboxes)): + boxes = lboxes[k] + + cls = int(boxes[0, 6]) + tid = int(boxes[0, 4]) + + # print(tid) + + frnum = boxes.shape[0] + cornpoints = np.zeros((frnum, 10)) + + cornpoints[:,0] = (boxes[:, 0] + boxes[:, 2]) / 2 + cornpoints[:,1] = (boxes[:, 1] + boxes[:, 3]) / 2 + cornpoints[:,2], cornpoints[:,3] = boxes[:, 0], boxes[:, 1] + cornpoints[:,4], cornpoints[:,5] = boxes[:, 2], boxes[:, 1] + cornpoints[:,6], cornpoints[:,7] = boxes[:, 0], boxes[:, 3] + cornpoints[:,8], cornpoints[:,9] = boxes[:, 2], boxes[:, 3] + + + x1, y1, x2, y2 = cornpoints[:,2],cornpoints[:,3],cornpoints[:,8],cornpoints[:,9] + BoundPixel = 10 + BoundThresh = 0.4 + cont1 = sum(abs(x1) BoundThresh + cont2 = sum(abs(y1) BoundThresh + cont3 = sum(abs(x2-1024) BoundThresh + cont4 = sum(abs(y2-1280) BoundThresh + isImgBorder = False + if cont1 or cont2 or cont3 or cont4: + isImgBorder = True + +# ============================================================================= +# '''情况1: 在商品运动过程中，商品检测框始终左下角点和图像左下角点重合, 用中心点代替''' +# lfcn_dist = np.linalg.norm(cornpoints[:, 6:8] - [0, 1280], axis=1) +# idx1 = lfcn_dist<10 +# if sum(idx1)/frnum > 0.5: +# cornpoints[:, 6:8] = cornpoints[:, 0:2] +# +# '''情况2: 在商品运动过程中，商品检测框始终右下角点和图像右下角点重合, 用中心点代替''' +# rtcn_dist = np.linalg.norm(cornpoints[:, 8:10] - [1024, 1280], axis=1) +# idx2 = rtcn_dist<10 +# if sum(idx2)/frnum > 0.5: +# cornpoints[:, 8:10] = cornpoints[:, 0:2] +# ============================================================================= + + + mwh = (np.mean(boxes[:, 2]) + np.mean(boxes[:, 3]))/2 + + trajectory = [] + trajlens = [] + trajdist = [] + for k in range(5): + traj = np.linalg.norm(np.diff(cornpoints[:, 2*k:2*(k+1)], axis = 0), axis=1) + trajlen = np.sum(traj) + ptdist = np.max(cdist(cornpoints[:, 2*k:2*(k+1)], cornpoints[:, 2*k:2*(k+1)])) + + trajectory.append(traj) + trajlens.append(trajlen) + trajdist.append(ptdist) + + if not isImgBorder: + idx = trajlens.index(min(trajlens)) + + trajmin = trajectory[idx] + trajlen_min = min(trajlens) + trajdist_min = min(trajdist) + else: + trajmin = trajectory[0] + trajlen_min = trajlens[0] + trajdist_min = trajdist[0] + + + '''最小轨迹长度/最大轨迹长度，越小，代表运动幅度越小''' + trajlen_rate = trajlen_min/(max(trajlens)+0.0001) + + '''最小轨迹欧氏距离/目标框尺度均值''' + trajdist_rate = trajdist_min/(mwh+0.0001) + + + + + # idx = trajlens.index(min(trajlens)) + # trajmin = trajectory[idx] + + # '''最小轨迹长度/最大轨迹长度，越小，代表运动幅度越小''' + # trajlen_rate = min(trajlens)/(max(trajlens)+0.0001) + + # '''最小轨迹欧氏距离，越小，代表运动幅度越小''' + # trajdist_min = min(trajdist) + + # '''最小轨迹欧氏距离 / 目标框尺度均值''' + # mindist_rate = min(trajdist)/(mwh+0.0001) + + + + + img = image.copy() + for i in range(boxes.shape[0]): + cv2.circle(img, (int(cornpoints[i, 0]), int(cornpoints[i, 1])), 6, (255, 255, 255), 2) + cv2.circle(img, (int(cornpoints[i, 2]), int(cornpoints[i, 3])), 6, (255, 0, 255), 2) + cv2.circle(img, (int(cornpoints[i, 4]), int(cornpoints[i, 5])), 6, (0, 255, 0), 2) + cv2.circle(img, (int(cornpoints[i, 6]), int(cornpoints[i, 7])), 6, (64, 128, 255), 2) + cv2.circle(img, (int(cornpoints[i, 8]), int(cornpoints[i, 9])), 6, (255, 128, 64), 2) + + # if frnum>=3: + # cntpoints = cornpoints[:, 0:2].astype(np.int64) + # rect = cv2.minAreaRect(cntpoints) + # box = cv2.boxPoints(rect) + # box = np.int0(box) + # cv2.drawContours(img, [box], 0, (255, 0, 0), 2) + + # img1 = image.copy() + # for i in range(boxes.shape[0]-1): + # pt1 = cornpoints[i, :].astype(np.int64) + # pt2 = cornpoints[i+1, :].astype(np.int64) + # cv2.line(img1, pt1, pt2, color=(255, 255, 255), thickness=2) + # gray = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY) + # _, binary = cv2.threshold(gray, 127, 255, cv2.THRESH_BINARY) + # contours, _ = cv2.findContours(binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + + + + color = ((0, 0, 255), (255, 128, 0)) + label_6 = "PCA(singular_values_) : " + label_7 = "Rect : " + + if frnum>=3: + if isImgBorder: + X = cornpoints[:, 0:2] + else: + X = cornpoints[:, 2*idx:2*(idx+1)] + + pca = PCA() + pca.fit(X) + label_6 = "PCA(variance_ratio) : {:.2f}".format(pca.explained_variance_ratio_[0]) + for i, (comp, var) in enumerate(zip(pca.components_, pca.explained_variance_ratio_)): + pt1 = (pca.mean_ - comp*var*200).astype(np.int64) + pt2 = (pca.mean_ + comp*var*200).astype(np.int64) + cv2.line(img, pt1, pt2, color=color[i], thickness=2) + + rect = cv2.minAreaRect(X.astype(np.int64)) + box = cv2.boxPoints(rect) + box = np.int0(box) + cv2.drawContours(img, [box], 0, (0, 255, 0), 2) + label_7 = "Rect W&H&Ratio: {}, {}, {:.2f}".format(int(rect[1][0]), int(rect[1][1]), min(rect[1])/(max(rect[1])+0.001)) + + + + '''撰写专利需要，生成黑白图像''' + # imgbt = cv2.bitwise_not(img) + # for i in range(box.shape[0]): + # cv2.circle(imgbt, (int(cornpoints[i, 0]), int(cornpoints[i, 1])), 14, (0, 0, 0), 2) + # cv2.drawMarker(imgbt, (int(cornpoints[i, 2]), int(cornpoints[i, 3])), color= (0, 0, 0), markerType=3, markerSize = 30, thickness=2) + # cv2.drawMarker(imgbt, (int(cornpoints[i, 4]), int(cornpoints[i, 5])), color= (0, 0, 0), markerType=4, markerSize = 30, thickness=2) + # cv2.drawMarker(imgbt, (int(cornpoints[i, 6]), int(cornpoints[i, 7])), color= (0, 0, 0), markerType=5, markerSize = 30, thickness=2) + # cv2.drawMarker(imgbt, (int(cornpoints[i, 8]), int(cornpoints[i, 9])), color= (0, 0, 0), markerType=6, markerSize = 30, thickness=2) + # cv2.imwrite(pth + f"/zhuanli/{file}_{tid}.png", imgbt) + + if len(trajmin): + trajstd = np.std(trajmin) + else: + trajstd = 0 + + + + trajlens = [int(t) for t in trajlens] + trajdist = [int(t) for t in trajdist] + label_1 = f"trajlens: {trajlens}, trajlen_min: {int(trajlen_min)}" + label_2 = f"trajdist: {trajdist}: trajdist_min: {int(trajdist_min)}" + label_3 = "trajlen_min/max(trajlens): {:.2f}/{} = {:.2f}".format(trajlen_min, max(trajlens), trajlen_rate) + label_4 = "trajdist_min/mwh : {:.2f}/{} = {:.2f}".format(trajdist_min, int(mwh), trajdist_rate) + label_5 = "std(trajmin) : {:.2f}".format(trajstd) + + label = [label_1, label_2, label_3, label_4, label_5, label_6, label_7] + word = 'abc' + w, h = cv2.getTextSize('abc', 0, fontScale=2, thickness=1)[0] + for i in range(len(label)): + # color = [int(x) for x in colors[i]] + cv2.putText(img, + label[i], + (20, int(50+(i+1)*1.2*h)), + 0, + 1, + [int(x) for x in colors[i]], + 2, + lineType=cv2.LINE_AA) + cv2.imwrite(pth + f"/result/cls11_80212/{file}_{tid}.png", img) + + + + + +def drawtracks(bboxes, imgshow=None): + + """ + Inputs + bboxes: 原始检测跟踪后的结果，变换为 tboxes + image：只用于获取图像的（Width, Height） + Outputs: + imgshow + """ + if imgshow is None: + edgeline = cv2.imread(pth + r"/shopcart/cart_tempt/edgeline.png") + # edgeline = cv2.bitwise_not(edgeline) + H, W = edgeline.shape[0:2] + imgshow= np.zeros((H, W, 3), np.uint8) + + if 'edgeline' in locals().keys(): + imgshow = cv2.add(imgshow, edgeline) + + ## ==== list，其中元素格式: [x, y, w, h, track_id, score, cls, frame_index] + + tboxes = array2list(bboxes) + + # imgshow = cv2.bitwise_not(imgshow) + + annotator = TrackAnnotator(imgshow, line_width=2) + for boxes in tboxes: + annotator.plotting_track(boxes) + + imgshow = annotator.result() + + return imgshow + + + +def writefilename(): + npydir = r"D:\DeepLearning\yolov5\runs\boxes" + files = [f.split('.')[0] for f in os.listdir(npydir)] + with open('data.txt', 'w') as f: + [f.write(f"{file}:\n") for file in files] + + print("len(files)") + # for filename in os.listdir(npydir): + # file, ext = os.path.splitext(filename) + + +def main(): + npydir = r"D:\DeepLearning\yolov5\runs\boxes" + k = 0 + fields = [] + + for filename in os.listdir(npydir): + # filename = "加购_快速置入_12.npy" + print(filename) + + file, ext = os.path.splitext(filename) + filepath = os.path.join(npydir, filename) + + try: + bboxes = np.load(filepath) + + imgshow = drawtracks(bboxes, file) + draw5points(bboxes, file) + + cv2.imwrite(pth + f"/result/cls11_80212/{file}_show.png", imgshow) + except Exception as e: + # print(str(e)) + pass + + # k += 1 + # if k == 1: + # break + + +if __name__ == "__main__": + + main() + + # writefilename() + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tracking/utils/videot.py b/tracking/utils/videot.py new file mode 100644 index 0000000..1895024 --- /dev/null +++ b/tracking/utils/videot.py @@ -0,0 +1,111 @@ +# -*- coding: utf-8 -*- +""" +Created on Wed Sep 20 14:10:09 2023 + +@author: ym +""" + +import numpy as np +import os +import cv2 + +# from pathlib import Path +# import math +# import sys +# from scipy.spatial.distance import cdist + +VideoFormat = ['.mp4', '.avi', '.ts'] +def video2imgs(videof, imgdir): + cap = cv2.VideoCapture(videof) + i = 0 + while True: + ret, frame = cap.read() + if not ret: + break + imgp = os.path.join(imgdir, f"{i}.png") + i += 1 + cv2.imwrite(imgp, frame) + + if i == 400: + break + cap.release() + + print(os.path.basename(videof) + f" haved resolved") + +def videosave(bboxes, videopath="100_1688009697927.mp4"): + + cap = cv2.VideoCapture(videopath) + + fps = int(cap.get(cv2.CAP_PROP_FPS)) # integer required, floats produce error in MP4 codec + width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + + ## =========================================== 在当前模块地址下存储图像和视频 + path = os.path.split(os.path.realpath(__file__))[0] + _, filename = os.path.split(videopath) + file, ext = os.path.splitext(filename) + + ## ======================================================== 视频保存设置 + fourcc = cv2.VideoWriter_fourcc(*'MP4V') + save_video_path = os.path.join(path, "{}_show_1.mp4".format(file)) + vid_writer = cv2.VideoWriter(save_video_path, fourcc, fps, (width, height)) + + ## ======================================================== 图像保存路径设置 + save_img_path = os.path.join(path, "{}_show".format(file)) + if not os.path.exists(save_img_path): + os.makedirs(save_img_path) + + cout = 0 + while cap.isOpened(): + ret, frame = cap.read() + if ret: + + idx = np.where(bboxes[:, 7] == cout)[0] + box = bboxes[idx, 0:4].astype(int) + for i in range(box.shape[0]): + x1, y1 = box[i, :2] + x2, y2 = box[i, 2:4] + cv2.rectangle(frame, (x1, y1), (x2, y2), (0, 125, 255), 2) + + + cv2.imwrite(os.path.join(save_img_path, "{}.png".format(cout)), frame) + vid_writer.write(frame) + + cout += 1 + else: + print("end!!!!!!!!!!!!!!!!!!!") + break + + vid_writer.release() + cap.release() + +def main(): + videopath = r'D:\videos' + savepath = r'D:\videos' + # video2imgs(videopath, savepath) + k = 0 + for filename in os.listdir(videopath): + # filename = "20240929-155533.ts" + + file, ext = os.path.splitext(filename) + if ext not in VideoFormat: + continue + + basename = os.path.basename(videopath) + imgbase = basename + '-&-' + file + imgdir = os.path.join(savepath, imgbase) + if not os.path.exists(imgdir): + os.mkdir(imgdir) + + videof = os.path.join(videopath, filename) + video2imgs(videof, imgdir) + + # k += 1 + # if k == 1: + # break + + + +if __name__ == '__main__': + main() + \ No newline at end of file diff --git a/train.py b/train.py new file mode 100644 index 0000000..004c8ee --- /dev/null +++ b/train.py @@ -0,0 +1,647 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Train a YOLOv5 model on a custom dataset. +Models and datasets download automatically from the latest YOLOv5 release. + +Usage - Single-GPU training: + $ python train.py --data coco128.yaml --weights yolov5s.pt --img 640 # from pretrained (recommended) + $ python train.py --data coco128.yaml --weights '' --cfg yolov5s.yaml --img 640 # from scratch + +Usage - Multi-GPU DDP training: + $ python -m torch.distributed.run --nproc_per_node 4 --master_port 1 train.py --data coco128.yaml --weights yolov5s.pt --img 640 --device 0,1,2,3 + +Models: https://github.com/ultralytics/yolov5/tree/master/models +Datasets: https://github.com/ultralytics/yolov5/tree/master/data +Tutorial: https://docs.ultralytics.com/yolov5/tutorials/train_custom_data +""" + +import argparse +import math +import os +import random +import subprocess +import sys +import time +from copy import deepcopy +from datetime import datetime +from pathlib import Path + +try: + import comet_ml # must be imported before torch (if installed) +except ImportError: + comet_ml = None + +import numpy as np +import torch +import torch.distributed as dist +import torch.nn as nn +import yaml +from torch.optim import lr_scheduler +from tqdm import tqdm + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[0] # YOLOv5 root directory +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) # add ROOT to PATH +ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative + +import val as validate # for end-of-epoch mAP +from models.experimental import attempt_load +from models.yolo import Model +from utils.autoanchor import check_anchors +from utils.autobatch import check_train_batch_size +from utils.callbacks import Callbacks +from utils.dataloaders import create_dataloader +from utils.downloads import attempt_download, is_url +from utils.general import (LOGGER, TQDM_BAR_FORMAT, check_amp, check_dataset, check_file, check_git_info, + check_git_status, check_img_size, check_requirements, check_suffix, check_yaml, colorstr, + get_latest_run, increment_path, init_seeds, intersect_dicts, labels_to_class_weights, + labels_to_image_weights, methods, one_cycle, print_args, print_mutation, strip_optimizer, + yaml_save) +from utils.loggers import Loggers +from utils.loggers.comet.comet_utils import check_comet_resume +from utils.loss import ComputeLoss +from utils.metrics import fitness +from utils.plots import plot_evolve +from utils.torch_utils import (EarlyStopping, ModelEMA, de_parallel, select_device, smart_DDP, smart_optimizer, + smart_resume, torch_distributed_zero_first) + +LOCAL_RANK = int(os.getenv('LOCAL_RANK', -1)) # https://pytorch.org/docs/stable/elastic/run.html +RANK = int(os.getenv('RANK', -1)) +WORLD_SIZE = int(os.getenv('WORLD_SIZE', 1)) +GIT_INFO = check_git_info() + + +def train(hyp, opt, device, callbacks): # hyp is path/to/hyp.yaml or hyp dictionary + save_dir, epochs, batch_size, weights, single_cls, evolve, data, cfg, resume, noval, nosave, workers, freeze = \ + Path(opt.save_dir), opt.epochs, opt.batch_size, opt.weights, opt.single_cls, opt.evolve, opt.data, opt.cfg, \ + opt.resume, opt.noval, opt.nosave, opt.workers, opt.freeze + callbacks.run('on_pretrain_routine_start') + + # Directories + w = save_dir / 'weights' # weights dir + (w.parent if evolve else w).mkdir(parents=True, exist_ok=True) # make dir + last, best = w / 'last.pt', w / 'best.pt' + + # Hyperparameters + if isinstance(hyp, str): + with open(hyp, errors='ignore') as f: + hyp = yaml.safe_load(f) # load hyps dict + LOGGER.info(colorstr('hyperparameters: ') + ', '.join(f'{k}={v}' for k, v in hyp.items())) + opt.hyp = hyp.copy() # for saving hyps to checkpoints + + # Save run settings + if not evolve: + yaml_save(save_dir / 'hyp.yaml', hyp) + yaml_save(save_dir / 'opt.yaml', vars(opt)) + + # Loggers + data_dict = None + if RANK in {-1, 0}: + loggers = Loggers(save_dir, weights, opt, hyp, LOGGER) # loggers instance + + # Register actions + for k in methods(loggers): + callbacks.register_action(k, callback=getattr(loggers, k)) + + # Process custom dataset artifact link + data_dict = loggers.remote_dataset + if resume: # If resuming runs from remote artifact + weights, epochs, hyp, batch_size = opt.weights, opt.epochs, opt.hyp, opt.batch_size + + # Config + plots = not evolve and not opt.noplots # create plots + cuda = device.type != 'cpu' + init_seeds(opt.seed + 1 + RANK, deterministic=True) + with torch_distributed_zero_first(LOCAL_RANK): + data_dict = data_dict or check_dataset(data) # check if None + train_path, val_path = data_dict['train'], data_dict['val'] + nc = 1 if single_cls else int(data_dict['nc']) # number of classes + names = {0: 'item'} if single_cls and len(data_dict['names']) != 1 else data_dict['names'] # class names + is_coco = isinstance(val_path, str) and val_path.endswith('coco/val2017.txt') # COCO dataset + + # Model + check_suffix(weights, '.pt') # check weights + pretrained = weights.endswith('.pt') + if pretrained: + with torch_distributed_zero_first(LOCAL_RANK): + weights = attempt_download(weights) # download if not found locally + ckpt = torch.load(weights, map_location='cpu') # load checkpoint to CPU to avoid CUDA memory leak + model = Model(cfg or ckpt['model'].yaml, ch=3, nc=nc, anchors=hyp.get('anchors')).to(device) # create + exclude = ['anchor'] if (cfg or hyp.get('anchors')) and not resume else [] # exclude keys + csd = ckpt['model'].float().state_dict() # checkpoint state_dict as FP32 + csd = intersect_dicts(csd, model.state_dict(), exclude=exclude) # intersect + model.load_state_dict(csd, strict=False) # load + LOGGER.info(f'Transferred {len(csd)}/{len(model.state_dict())} items from {weights}') # report + else: + model = Model(cfg, ch=3, nc=nc, anchors=hyp.get('anchors')).to(device) # create + amp = check_amp(model) # check AMP + + # Freeze + freeze = [f'model.{x}.' for x in (freeze if len(freeze) > 1 else range(freeze[0]))] # layers to freeze + for k, v in model.named_parameters(): + v.requires_grad = True # train all layers + # v.register_hook(lambda x: torch.nan_to_num(x)) # NaN to 0 (commented for erratic training results) + if any(x in k for x in freeze): + LOGGER.info(f'freezing {k}') + v.requires_grad = False + + # Image size + gs = max(int(model.stride.max()), 32) # grid size (max stride) + imgsz = check_img_size(opt.imgsz, gs, floor=gs * 2) # verify imgsz is gs-multiple + + # Batch size + if RANK == -1 and batch_size == -1: # single-GPU only, estimate best batch size + batch_size = check_train_batch_size(model, imgsz, amp) + loggers.on_params_update({'batch_size': batch_size}) + + # Optimizer + nbs = 64 # nominal batch size + accumulate = max(round(nbs / batch_size), 1) # accumulate loss before optimizing + hyp['weight_decay'] *= batch_size * accumulate / nbs # scale weight_decay + optimizer = smart_optimizer(model, opt.optimizer, hyp['lr0'], hyp['momentum'], hyp['weight_decay']) + + # Scheduler + if opt.cos_lr: + lf = one_cycle(1, hyp['lrf'], epochs) # cosine 1->hyp['lrf'] + else: + lf = lambda x: (1 - x / epochs) * (1.0 - hyp['lrf']) + hyp['lrf'] # linear + scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lf) # plot_lr_scheduler(optimizer, scheduler, epochs) + + # EMA + ema = ModelEMA(model) if RANK in {-1, 0} else None + + # Resume + best_fitness, start_epoch = 0.0, 0 + if pretrained: + if resume: + best_fitness, start_epoch, epochs = smart_resume(ckpt, optimizer, ema, weights, epochs, resume) + del ckpt, csd + + # DP mode + if cuda and RANK == -1 and torch.cuda.device_count() > 1: + LOGGER.warning( + 'WARNING ⚠️ DP not recommended, use torch.distributed.run for best DDP Multi-GPU results.\n' + 'See Multi-GPU Tutorial at https://docs.ultralytics.com/yolov5/tutorials/multi_gpu_training to get started.' + ) + model = torch.nn.DataParallel(model) + + # SyncBatchNorm + if opt.sync_bn and cuda and RANK != -1: + model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model).to(device) + LOGGER.info('Using SyncBatchNorm()') + + # Trainloader + train_loader, dataset = create_dataloader(train_path, + imgsz, + batch_size // WORLD_SIZE, + gs, + single_cls, + hyp=hyp, + augment=True, + cache=None if opt.cache == 'val' else opt.cache, + rect=opt.rect, + rank=LOCAL_RANK, + workers=workers, + image_weights=opt.image_weights, + quad=opt.quad, + prefix=colorstr('train: '), + shuffle=True, + seed=opt.seed) + labels = np.concatenate(dataset.labels, 0) + mlc = int(labels[:, 0].max()) # max label class + assert mlc < nc, f'Label class {mlc} exceeds nc={nc} in {data}. Possible class labels are 0-{nc - 1}' + + # Process 0 + if RANK in {-1, 0}: + val_loader = create_dataloader(val_path, + imgsz, + batch_size // WORLD_SIZE * 2, + gs, + single_cls, + hyp=hyp, + cache=None if noval else opt.cache, + rect=True, + rank=-1, + workers=workers * 2, + pad=0.5, + prefix=colorstr('val: '))[0] + + if not resume: + if not opt.noautoanchor: + check_anchors(dataset, model=model, thr=hyp['anchor_t'], imgsz=imgsz) # run AutoAnchor + model.half().float() # pre-reduce anchor precision + + callbacks.run('on_pretrain_routine_end', labels, names) + + # DDP mode + if cuda and RANK != -1: + model = smart_DDP(model) + + # Model attributes + nl = de_parallel(model).model[-1].nl # number of detection layers (to scale hyps) + hyp['box'] *= 3 / nl # scale to layers + hyp['cls'] *= nc / 80 * 3 / nl # scale to classes and layers + hyp['obj'] *= (imgsz / 640) ** 2 * 3 / nl # scale to image size and layers + hyp['label_smoothing'] = opt.label_smoothing + model.nc = nc # attach number of classes to model + model.hyp = hyp # attach hyperparameters to model + model.class_weights = labels_to_class_weights(dataset.labels, nc).to(device) * nc # attach class weights + model.names = names + + # Start training + t0 = time.time() + nb = len(train_loader) # number of batches + nw = max(round(hyp['warmup_epochs'] * nb), 100) # number of warmup iterations, max(3 epochs, 100 iterations) + # nw = min(nw, (epochs - start_epoch) / 2 * nb) # limit warmup to < 1/2 of training + last_opt_step = -1 + maps = np.zeros(nc) # mAP per class + results = (0, 0, 0, 0, 0, 0, 0) # P, R, mAP@.5, mAP@.5-.95, val_loss(box, obj, cls) + scheduler.last_epoch = start_epoch - 1 # do not move + scaler = torch.cuda.amp.GradScaler(enabled=amp) + stopper, stop = EarlyStopping(patience=opt.patience), False + compute_loss = ComputeLoss(model) # init loss class + callbacks.run('on_train_start') + LOGGER.info(f'Image sizes {imgsz} train, {imgsz} val\n' + f'Using {train_loader.num_workers * WORLD_SIZE} dataloader workers\n' + f"Logging results to {colorstr('bold', save_dir)}\n" + f'Starting training for {epochs} epochs...') + for epoch in range(start_epoch, epochs): # epoch ------------------------------------------------------------------ + callbacks.run('on_train_epoch_start') + model.train() + + # Update image weights (optional, single-GPU only) + if opt.image_weights: + cw = model.class_weights.cpu().numpy() * (1 - maps) ** 2 / nc # class weights + iw = labels_to_image_weights(dataset.labels, nc=nc, class_weights=cw) # image weights + dataset.indices = random.choices(range(dataset.n), weights=iw, k=dataset.n) # rand weighted idx + + # Update mosaic border (optional) + # b = int(random.uniform(0.25 * imgsz, 0.75 * imgsz + gs) // gs * gs) + # dataset.mosaic_border = [b - imgsz, -b] # height, width borders + + mloss = torch.zeros(3, device=device) # mean losses + if RANK != -1: + train_loader.sampler.set_epoch(epoch) + pbar = enumerate(train_loader) + LOGGER.info(('\n' + '%11s' * 7) % ('Epoch', 'GPU_mem', 'box_loss', 'obj_loss', 'cls_loss', 'Instances', 'Size')) + if RANK in {-1, 0}: + pbar = tqdm(pbar, total=nb, bar_format=TQDM_BAR_FORMAT) # progress bar + optimizer.zero_grad() + for i, (imgs, targets, paths, _) in pbar: # batch ------------------------------------------------------------- + callbacks.run('on_train_batch_start') + ni = i + nb * epoch # number integrated batches (since train start) + imgs = imgs.to(device, non_blocking=True).float() / 255 # uint8 to float32, 0-255 to 0.0-1.0 + + # Warmup + if ni <= nw: + xi = [0, nw] # x interp + # compute_loss.gr = np.interp(ni, xi, [0.0, 1.0]) # iou loss ratio (obj_loss = 1.0 or iou) + accumulate = max(1, np.interp(ni, xi, [1, nbs / batch_size]).round()) + for j, x in enumerate(optimizer.param_groups): + # bias lr falls from 0.1 to lr0, all other lrs rise from 0.0 to lr0 + x['lr'] = np.interp(ni, xi, [hyp['warmup_bias_lr'] if j == 0 else 0.0, x['initial_lr'] * lf(epoch)]) + if 'momentum' in x: + x['momentum'] = np.interp(ni, xi, [hyp['warmup_momentum'], hyp['momentum']]) + + # Multi-scale + if opt.multi_scale: + sz = random.randrange(int(imgsz * 0.5), int(imgsz * 1.5) + gs) // gs * gs # size + sf = sz / max(imgs.shape[2:]) # scale factor + if sf != 1: + ns = [math.ceil(x * sf / gs) * gs for x in imgs.shape[2:]] # new shape (stretched to gs-multiple) + imgs = nn.functional.interpolate(imgs, size=ns, mode='bilinear', align_corners=False) + + # Forward + with torch.cuda.amp.autocast(amp): + pred = model(imgs) # forward + loss, loss_items = compute_loss(pred, targets.to(device)) # loss scaled by batch_size + if RANK != -1: + loss *= WORLD_SIZE # gradient averaged between devices in DDP mode + if opt.quad: + loss *= 4. + + # Backward + scaler.scale(loss).backward() + + # Optimize - https://pytorch.org/docs/master/notes/amp_examples.html + if ni - last_opt_step >= accumulate: + scaler.unscale_(optimizer) # unscale gradients + torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=10.0) # clip gradients + scaler.step(optimizer) # optimizer.step + scaler.update() + optimizer.zero_grad() + if ema: + ema.update(model) + last_opt_step = ni + + # Log + if RANK in {-1, 0}: + mloss = (mloss * i + loss_items) / (i + 1) # update mean losses + mem = f'{torch.cuda.memory_reserved() / 1E9 if torch.cuda.is_available() else 0:.3g}G' # (GB) + pbar.set_description(('%11s' * 2 + '%11.4g' * 5) % + (f'{epoch}/{epochs - 1}', mem, *mloss, targets.shape[0], imgs.shape[-1])) + callbacks.run('on_train_batch_end', model, ni, imgs, targets, paths, list(mloss)) + if callbacks.stop_training: + return + # end batch ------------------------------------------------------------------------------------------------ + + # Scheduler + lr = [x['lr'] for x in optimizer.param_groups] # for loggers + scheduler.step() + + if RANK in {-1, 0}: + # mAP + callbacks.run('on_train_epoch_end', epoch=epoch) + ema.update_attr(model, include=['yaml', 'nc', 'hyp', 'names', 'stride', 'class_weights']) + final_epoch = (epoch + 1 == epochs) or stopper.possible_stop + if not noval or final_epoch: # Calculate mAP + results, maps, _ = validate.run(data_dict, + batch_size=batch_size // WORLD_SIZE * 2, + imgsz=imgsz, + half=amp, + model=ema.ema, + single_cls=single_cls, + dataloader=val_loader, + save_dir=save_dir, + plots=False, + callbacks=callbacks, + compute_loss=compute_loss) + + # Update best mAP + fi = fitness(np.array(results).reshape(1, -1)) # weighted combination of [P, R, mAP@.5, mAP@.5-.95] + stop = stopper(epoch=epoch, fitness=fi) # early stop check + if fi > best_fitness: + best_fitness = fi + log_vals = list(mloss) + list(results) + lr + callbacks.run('on_fit_epoch_end', log_vals, epoch, best_fitness, fi) + + # Save model + if (not nosave) or (final_epoch and not evolve): # if save + ckpt = { + 'epoch': epoch, + 'best_fitness': best_fitness, + 'model': deepcopy(de_parallel(model)).half(), + 'ema': deepcopy(ema.ema).half(), + 'updates': ema.updates, + 'optimizer': optimizer.state_dict(), + 'opt': vars(opt), + 'git': GIT_INFO, # {remote, branch, commit} if a git repo + 'date': datetime.now().isoformat()} + + # Save last, best and delete + torch.save(ckpt, last) + if best_fitness == fi: + torch.save(ckpt, best) + if opt.save_period > 0 and epoch % opt.save_period == 0: + torch.save(ckpt, w / f'epoch{epoch}.pt') + del ckpt + callbacks.run('on_model_save', last, epoch, final_epoch, best_fitness, fi) + + # EarlyStopping + if RANK != -1: # if DDP training + broadcast_list = [stop if RANK == 0 else None] + dist.broadcast_object_list(broadcast_list, 0) # broadcast 'stop' to all ranks + if RANK != 0: + stop = broadcast_list[0] + if stop: + break # must break all DDP ranks + + # end epoch ---------------------------------------------------------------------------------------------------- + # end training ----------------------------------------------------------------------------------------------------- + if RANK in {-1, 0}: + LOGGER.info(f'\n{epoch - start_epoch + 1} epochs completed in {(time.time() - t0) / 3600:.3f} hours.') + for f in last, best: + if f.exists(): + strip_optimizer(f) # strip optimizers + if f is best: + LOGGER.info(f'\nValidating {f}...') + results, _, _ = validate.run( + data_dict, + batch_size=batch_size // WORLD_SIZE * 2, + imgsz=imgsz, + model=attempt_load(f, device).half(), + iou_thres=0.65 if is_coco else 0.60, # best pycocotools at iou 0.65 + single_cls=single_cls, + dataloader=val_loader, + save_dir=save_dir, + save_json=is_coco, + verbose=True, + plots=plots, + callbacks=callbacks, + compute_loss=compute_loss) # val best model with plots + if is_coco: + callbacks.run('on_fit_epoch_end', list(mloss) + list(results) + lr, epoch, best_fitness, fi) + + callbacks.run('on_train_end', last, best, epoch, results) + + torch.cuda.empty_cache() + return results + + +def parse_opt(known=False): + parser = argparse.ArgumentParser() + parser.add_argument('--weights', type=str, default=ROOT / 'yolov5s.pt', help='initial weights path') + parser.add_argument('--cfg', type=str, default='', help='model.yaml path') + parser.add_argument('--data', type=str, default=ROOT / 'data/coco128.yaml', help='dataset.yaml path') + parser.add_argument('--hyp', type=str, default=ROOT / 'data/hyps/hyp.scratch-low.yaml', help='hyperparameters path') + parser.add_argument('--epochs', type=int, default=100, help='total training epochs') + parser.add_argument('--batch-size', type=int, default=16, help='total batch size for all GPUs, -1 for autobatch') + parser.add_argument('--imgsz', '--img', '--img-size', type=int, default=640, help='train, val image size (pixels)') + parser.add_argument('--rect', action='store_true', help='rectangular training') + parser.add_argument('--resume', nargs='?', const=True, default=False, help='resume most recent training') + parser.add_argument('--nosave', action='store_true', help='only save final checkpoint') + parser.add_argument('--noval', action='store_true', help='only validate final epoch') + parser.add_argument('--noautoanchor', action='store_true', help='disable AutoAnchor') + parser.add_argument('--noplots', action='store_true', help='save no plot files') + parser.add_argument('--evolve', type=int, nargs='?', const=300, help='evolve hyperparameters for x generations') + parser.add_argument('--bucket', type=str, default='', help='gsutil bucket') + parser.add_argument('--cache', type=str, nargs='?', const='ram', help='image --cache ram/disk') + parser.add_argument('--image-weights', action='store_true', help='use weighted image selection for training') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--multi-scale', action='store_true', help='vary img-size +/- 50%%') + parser.add_argument('--single-cls', action='store_true', help='train multi-class data as single-class') + parser.add_argument('--optimizer', type=str, choices=['SGD', 'Adam', 'AdamW'], default='SGD', help='optimizer') + parser.add_argument('--sync-bn', action='store_true', help='use SyncBatchNorm, only available in DDP mode') + parser.add_argument('--workers', type=int, default=8, help='max dataloader workers (per RANK in DDP mode)') + parser.add_argument('--project', default=ROOT / 'runs/train', help='save to project/name') + parser.add_argument('--name', default='exp', help='save to project/name') + parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment') + parser.add_argument('--quad', action='store_true', help='quad dataloader') + parser.add_argument('--cos-lr', action='store_true', help='cosine LR scheduler') + parser.add_argument('--label-smoothing', type=float, default=0.0, help='Label smoothing epsilon') + parser.add_argument('--patience', type=int, default=100, help='EarlyStopping patience (epochs without improvement)') + parser.add_argument('--freeze', nargs='+', type=int, default=[0], help='Freeze layers: backbone=10, first3=0 1 2') + parser.add_argument('--save-period', type=int, default=-1, help='Save checkpoint every x epochs (disabled if < 1)') + parser.add_argument('--seed', type=int, default=0, help='Global training seed') + parser.add_argument('--local_rank', type=int, default=-1, help='Automatic DDP Multi-GPU argument, do not modify') + + # Logger arguments + parser.add_argument('--entity', default=None, help='Entity') + parser.add_argument('--upload_dataset', nargs='?', const=True, default=False, help='Upload data, "val" option') + parser.add_argument('--bbox_interval', type=int, default=-1, help='Set bounding-box image logging interval') + parser.add_argument('--artifact_alias', type=str, default='latest', help='Version of dataset artifact to use') + + return parser.parse_known_args()[0] if known else parser.parse_args() + + +def main(opt, callbacks=Callbacks()): + # Checks + if RANK in {-1, 0}: + print_args(vars(opt)) + check_git_status() + check_requirements(ROOT / 'requirements.txt') + + # Resume (from specified or most recent last.pt) + if opt.resume and not check_comet_resume(opt) and not opt.evolve: + last = Path(check_file(opt.resume) if isinstance(opt.resume, str) else get_latest_run()) + opt_yaml = last.parent.parent / 'opt.yaml' # train options yaml + opt_data = opt.data # original dataset + if opt_yaml.is_file(): + with open(opt_yaml, errors='ignore') as f: + d = yaml.safe_load(f) + else: + d = torch.load(last, map_location='cpu')['opt'] + opt = argparse.Namespace(**d) # replace + opt.cfg, opt.weights, opt.resume = '', str(last), True # reinstate + if is_url(opt_data): + opt.data = check_file(opt_data) # avoid HUB resume auth timeout + else: + opt.data, opt.cfg, opt.hyp, opt.weights, opt.project = \ + check_file(opt.data), check_yaml(opt.cfg), check_yaml(opt.hyp), str(opt.weights), str(opt.project) # checks + assert len(opt.cfg) or len(opt.weights), 'either --cfg or --weights must be specified' + if opt.evolve: + if opt.project == str(ROOT / 'runs/train'): # if default project name, rename to runs/evolve + opt.project = str(ROOT / 'runs/evolve') + opt.exist_ok, opt.resume = opt.resume, False # pass resume to exist_ok and disable resume + if opt.name == 'cfg': + opt.name = Path(opt.cfg).stem # use model.yaml as name + opt.save_dir = str(increment_path(Path(opt.project) / opt.name, exist_ok=opt.exist_ok)) + + # DDP mode + device = select_device(opt.device, batch_size=opt.batch_size) + if LOCAL_RANK != -1: + msg = 'is not compatible with YOLOv5 Multi-GPU DDP training' + assert not opt.image_weights, f'--image-weights {msg}' + assert not opt.evolve, f'--evolve {msg}' + assert opt.batch_size != -1, f'AutoBatch with --batch-size -1 {msg}, please pass a valid --batch-size' + assert opt.batch_size % WORLD_SIZE == 0, f'--batch-size {opt.batch_size} must be multiple of WORLD_SIZE' + assert torch.cuda.device_count() > LOCAL_RANK, 'insufficient CUDA devices for DDP command' + torch.cuda.set_device(LOCAL_RANK) + device = torch.device('cuda', LOCAL_RANK) + dist.init_process_group(backend='nccl' if dist.is_nccl_available() else 'gloo') + + # Train + if not opt.evolve: + train(opt.hyp, opt, device, callbacks) + + # Evolve hyperparameters (optional) + else: + # Hyperparameter evolution metadata (mutation scale 0-1, lower_limit, upper_limit) + meta = { + 'lr0': (1, 1e-5, 1e-1), # initial learning rate (SGD=1E-2, Adam=1E-3) + 'lrf': (1, 0.01, 1.0), # final OneCycleLR learning rate (lr0 * lrf) + 'momentum': (0.3, 0.6, 0.98), # SGD momentum/Adam beta1 + 'weight_decay': (1, 0.0, 0.001), # optimizer weight decay + 'warmup_epochs': (1, 0.0, 5.0), # warmup epochs (fractions ok) + 'warmup_momentum': (1, 0.0, 0.95), # warmup initial momentum + 'warmup_bias_lr': (1, 0.0, 0.2), # warmup initial bias lr + 'box': (1, 0.02, 0.2), # box loss gain + 'cls': (1, 0.2, 4.0), # cls loss gain + 'cls_pw': (1, 0.5, 2.0), # cls BCELoss positive_weight + 'obj': (1, 0.2, 4.0), # obj loss gain (scale with pixels) + 'obj_pw': (1, 0.5, 2.0), # obj BCELoss positive_weight + 'iou_t': (0, 0.1, 0.7), # IoU training threshold + 'anchor_t': (1, 2.0, 8.0), # anchor-multiple threshold + 'anchors': (2, 2.0, 10.0), # anchors per output grid (0 to ignore) + 'fl_gamma': (0, 0.0, 2.0), # focal loss gamma (efficientDet default gamma=1.5) + 'hsv_h': (1, 0.0, 0.1), # image HSV-Hue augmentation (fraction) + 'hsv_s': (1, 0.0, 0.9), # image HSV-Saturation augmentation (fraction) + 'hsv_v': (1, 0.0, 0.9), # image HSV-Value augmentation (fraction) + 'degrees': (1, 0.0, 45.0), # image rotation (+/- deg) + 'translate': (1, 0.0, 0.9), # image translation (+/- fraction) + 'scale': (1, 0.0, 0.9), # image scale (+/- gain) + 'shear': (1, 0.0, 10.0), # image shear (+/- deg) + 'perspective': (0, 0.0, 0.001), # image perspective (+/- fraction), range 0-0.001 + 'flipud': (1, 0.0, 1.0), # image flip up-down (probability) + 'fliplr': (0, 0.0, 1.0), # image flip left-right (probability) + 'mosaic': (1, 0.0, 1.0), # image mixup (probability) + 'mixup': (1, 0.0, 1.0), # image mixup (probability) + 'copy_paste': (1, 0.0, 1.0)} # segment copy-paste (probability) + + with open(opt.hyp, errors='ignore') as f: + hyp = yaml.safe_load(f) # load hyps dict + if 'anchors' not in hyp: # anchors commented in hyp.yaml + hyp['anchors'] = 3 + if opt.noautoanchor: + del hyp['anchors'], meta['anchors'] + opt.noval, opt.nosave, save_dir = True, True, Path(opt.save_dir) # only val/save final epoch + # ei = [isinstance(x, (int, float)) for x in hyp.values()] # evolvable indices + evolve_yaml, evolve_csv = save_dir / 'hyp_evolve.yaml', save_dir / 'evolve.csv' + if opt.bucket: + # download evolve.csv if exists + subprocess.run([ + 'gsutil', + 'cp', + f'gs://{opt.bucket}/evolve.csv', + str(evolve_csv), ]) + + for _ in range(opt.evolve): # generations to evolve + if evolve_csv.exists(): # if evolve.csv exists: select best hyps and mutate + # Select parent(s) + parent = 'single' # parent selection method: 'single' or 'weighted' + x = np.loadtxt(evolve_csv, ndmin=2, delimiter=',', skiprows=1) + n = min(5, len(x)) # number of previous results to consider + x = x[np.argsort(-fitness(x))][:n] # top n mutations + w = fitness(x) - fitness(x).min() + 1E-6 # weights (sum > 0) + if parent == 'single' or len(x) == 1: + # x = x[random.randint(0, n - 1)] # random selection + x = x[random.choices(range(n), weights=w)[0]] # weighted selection + elif parent == 'weighted': + x = (x * w.reshape(n, 1)).sum(0) / w.sum() # weighted combination + + # Mutate + mp, s = 0.8, 0.2 # mutation probability, sigma + npr = np.random + npr.seed(int(time.time())) + g = np.array([meta[k][0] for k in hyp.keys()]) # gains 0-1 + ng = len(meta) + v = np.ones(ng) + while all(v == 1): # mutate until a change occurs (prevent duplicates) + v = (g * (npr.random(ng) < mp) * npr.randn(ng) * npr.random() * s + 1).clip(0.3, 3.0) + for i, k in enumerate(hyp.keys()): # plt.hist(v.ravel(), 300) + hyp[k] = float(x[i + 7] * v[i]) # mutate + + # Constrain to limits + for k, v in meta.items(): + hyp[k] = max(hyp[k], v[1]) # lower limit + hyp[k] = min(hyp[k], v[2]) # upper limit + hyp[k] = round(hyp[k], 5) # significant digits + + # Train mutation + results = train(hyp.copy(), opt, device, callbacks) + callbacks = Callbacks() + # Write mutation results + keys = ('metrics/precision', 'metrics/recall', 'metrics/mAP_0.5', 'metrics/mAP_0.5:0.95', 'val/box_loss', + 'val/obj_loss', 'val/cls_loss') + print_mutation(keys, results, hyp.copy(), save_dir, opt.bucket) + + # Plot results + plot_evolve(evolve_csv) + LOGGER.info(f'Hyperparameter evolution finished {opt.evolve} generations\n' + f"Results saved to {colorstr('bold', save_dir)}\n" + f'Usage example: $ python train.py --hyp {evolve_yaml}') + + +def run(**kwargs): + # Usage: import train; train.run(data='coco128.yaml', imgsz=320, weights='yolov5m.pt') + opt = parse_opt(True) + for k, v in kwargs.items(): + setattr(opt, k, v) + main(opt) + return opt + + +if __name__ == '__main__': + opt = parse_opt() + main(opt) diff --git a/ultralytics/__init__.py b/ultralytics/__init__.py new file mode 100644 index 0000000..8ff1b4f --- /dev/null +++ b/ultralytics/__init__.py @@ -0,0 +1,27 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +__version__ = "8.1.34" + +from ultralytics.data.explorer.explorer import Explorer +from ultralytics.models import RTDETR, SAM, YOLO, YOLOWorld, YOLOv10 +from ultralytics.models.fastsam import FastSAM +from ultralytics.models.nas import NAS +from ultralytics.utils import ASSETS, SETTINGS as settings +from ultralytics.utils.checks import check_yolo as checks +from ultralytics.utils.downloads import download + +__all__ = ( + "__version__", + "ASSETS", + "YOLO", + "YOLOWorld", + "NAS", + "SAM", + "FastSAM", + "RTDETR", + "checks", + "download", + "settings", + "Explorer", + "YOLOv10" +) diff --git a/ultralytics/__pycache__/__init__.cpython-312.pyc b/ultralytics/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..c4f39ae Binary files /dev/null and b/ultralytics/__pycache__/__init__.cpython-312.pyc differ diff --git a/ultralytics/__pycache__/__init__.cpython-39.pyc b/ultralytics/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..a34898b Binary files /dev/null and b/ultralytics/__pycache__/__init__.cpython-39.pyc differ diff --git a/ultralytics/cfg/__init__.py b/ultralytics/cfg/__init__.py new file mode 100644 index 0000000..c34cc17 --- /dev/null +++ b/ultralytics/cfg/__init__.py @@ -0,0 +1,613 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import contextlib +import os +import shutil +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace +from typing import Dict, List, Union +import re + +from ultralytics.utils import ( + ASSETS, + DEFAULT_CFG, + DEFAULT_CFG_DICT, + DEFAULT_CFG_PATH, + LOGGER, + RANK, + ROOT, + RUNS_DIR, + SETTINGS, + SETTINGS_YAML, + TESTS_RUNNING, + IterableSimpleNamespace, + __version__, + checks, + colorstr, + deprecation_warn, + yaml_load, + yaml_print, +) + +# Define valid tasks and modes +MODES = {"train", "val", "predict", "export", "track", "benchmark"} +TASKS = {"detect", "segment", "classify", "pose", "obb"} +TASK2DATA = { + "detect": "coco8.yaml", + "segment": "coco8-seg.yaml", + "classify": "imagenet10", + "pose": "coco8-pose.yaml", + "obb": "dota8.yaml", +} +TASK2MODEL = { + "detect": "yolov8n.pt", + "segment": "yolov8n-seg.pt", + "classify": "yolov8n-cls.pt", + "pose": "yolov8n-pose.pt", + "obb": "yolov8n-obb.pt", +} +TASK2METRIC = { + "detect": "metrics/mAP50-95(B)", + "segment": "metrics/mAP50-95(M)", + "classify": "metrics/accuracy_top1", + "pose": "metrics/mAP50-95(P)", + "obb": "metrics/mAP50-95(B)", +} + +CLI_HELP_MSG = f""" + Arguments received: {str(['yolo'] + sys.argv[1:])}. Ultralytics 'yolo' commands use the following syntax: + + yolo TASK MODE ARGS + + Where TASK (optional) is one of {TASKS} + MODE (required) is one of {MODES} + ARGS (optional) are any number of custom 'arg=value' pairs like 'imgsz=320' that override defaults. + See all ARGS at https://docs.ultralytics.com/usage/cfg or with 'yolo cfg' + + 1. Train a detection model for 10 epochs with an initial learning_rate of 0.01 + yolo train data=coco128.yaml model=yolov8n.pt epochs=10 lr0=0.01 + + 2. Predict a YouTube video using a pretrained segmentation model at image size 320: + yolo predict model=yolov8n-seg.pt source='https://youtu.be/LNwODJXcvt4' imgsz=320 + + 3. Val a pretrained detection model at batch-size 1 and image size 640: + yolo val model=yolov8n.pt data=coco128.yaml batch=1 imgsz=640 + + 4. Export a YOLOv8n classification model to ONNX format at image size 224 by 128 (no TASK required) + yolo export model=yolov8n-cls.pt format=onnx imgsz=224,128 + + 6. Explore your datasets using semantic search and SQL with a simple GUI powered by Ultralytics Explorer API + yolo explorer + + 5. Run special commands: + yolo help + yolo checks + yolo version + yolo settings + yolo copy-cfg + yolo cfg + + Docs: https://docs.ultralytics.com + Community: https://community.ultralytics.com + GitHub: https://github.com/ultralytics/ultralytics + """ + +# Define keys for arg type checks +CFG_FLOAT_KEYS = {"warmup_epochs", "box", "cls", "dfl", "degrees", "shear", "time"} +CFG_FRACTION_KEYS = { + "dropout", + "iou", + "lr0", + "lrf", + "momentum", + "weight_decay", + "warmup_momentum", + "warmup_bias_lr", + "label_smoothing", + "hsv_h", + "hsv_s", + "hsv_v", + "translate", + "scale", + "perspective", + "flipud", + "fliplr", + "bgr", + "mosaic", + "mixup", + "copy_paste", + "conf", + "iou", + "fraction", +} # fraction floats 0.0 - 1.0 +CFG_INT_KEYS = { + "epochs", + "patience", + "batch", + "workers", + "seed", + "close_mosaic", + "mask_ratio", + "max_det", + "vid_stride", + "line_width", + "workspace", + "nbs", + "save_period", +} +CFG_BOOL_KEYS = { + "save", + "exist_ok", + "verbose", + "deterministic", + "single_cls", + "rect", + "cos_lr", + "overlap_mask", + "val", + "save_json", + "save_hybrid", + "half", + "dnn", + "plots", + "show", + "save_txt", + "save_conf", + "save_crop", + "save_frames", + "show_labels", + "show_conf", + "visualize", + "augment", + "agnostic_nms", + "retina_masks", + "show_boxes", + "keras", + "optimize", + "int8", + "dynamic", + "simplify", + "nms", + "profile", + "multi_scale", +} + + +def cfg2dict(cfg): + """ + Convert a configuration object to a dictionary, whether it is a file path, a string, or a SimpleNamespace object. + + Args: + cfg (str | Path | dict | SimpleNamespace): Configuration object to be converted to a dictionary. + + Returns: + cfg (dict): Configuration object in dictionary format. + """ + if isinstance(cfg, (str, Path)): + cfg = yaml_load(cfg) # load dict + elif isinstance(cfg, SimpleNamespace): + cfg = vars(cfg) # convert to dict + return cfg + + +def get_cfg(cfg: Union[str, Path, Dict, SimpleNamespace] = DEFAULT_CFG_DICT, overrides: Dict = None): + """ + Load and merge configuration data from a file or dictionary. + + Args: + cfg (str | Path | Dict | SimpleNamespace): Configuration data. + overrides (str | Dict | optional): Overrides in the form of a file name or a dictionary. Default is None. + + Returns: + (SimpleNamespace): Training arguments namespace. + """ + cfg = cfg2dict(cfg) + + # Merge overrides + if overrides: + overrides = cfg2dict(overrides) + if "save_dir" not in cfg: + overrides.pop("save_dir", None) # special override keys to ignore + check_dict_alignment(cfg, overrides) + cfg = {**cfg, **overrides} # merge cfg and overrides dicts (prefer overrides) + + # Special handling for numeric project/name + for k in "project", "name": + if k in cfg and isinstance(cfg[k], (int, float)): + cfg[k] = str(cfg[k]) + if cfg.get("name") == "model": # assign model to 'name' arg + cfg["name"] = cfg.get("model", "").split(".")[0] + LOGGER.warning(f"WARNING ⚠️ 'name=model' automatically updated to 'name={cfg['name']}'.") + + # Type and Value checks + check_cfg(cfg) + + # Return instance + return IterableSimpleNamespace(**cfg) + + +def check_cfg(cfg, hard=True): + """Check Ultralytics configuration argument types and values.""" + for k, v in cfg.items(): + if v is not None: # None values may be from optional args + if k in CFG_FLOAT_KEYS and not isinstance(v, (int, float)): + if hard: + raise TypeError( + f"'{k}={v}' is of invalid type {type(v).__name__}. " + f"Valid '{k}' types are int (i.e. '{k}=0') or float (i.e. '{k}=0.5')" + ) + cfg[k] = float(v) + elif k in CFG_FRACTION_KEYS: + if not isinstance(v, (int, float)): + if hard: + raise TypeError( + f"'{k}={v}' is of invalid type {type(v).__name__}. " + f"Valid '{k}' types are int (i.e. '{k}=0') or float (i.e. '{k}=0.5')" + ) + cfg[k] = v = float(v) + if not (0.0 <= v <= 1.0): + raise ValueError(f"'{k}={v}' is an invalid value. " f"Valid '{k}' values are between 0.0 and 1.0.") + elif k in CFG_INT_KEYS and not isinstance(v, int): + if hard: + raise TypeError( + f"'{k}={v}' is of invalid type {type(v).__name__}. " f"'{k}' must be an int (i.e. '{k}=8')" + ) + cfg[k] = int(v) + elif k in CFG_BOOL_KEYS and not isinstance(v, bool): + if hard: + raise TypeError( + f"'{k}={v}' is of invalid type {type(v).__name__}. " + f"'{k}' must be a bool (i.e. '{k}=True' or '{k}=False')" + ) + cfg[k] = bool(v) + + +def get_save_dir(args, name=None): + """Return save_dir as created from train/val/predict arguments.""" + + if getattr(args, "save_dir", None): + save_dir = args.save_dir + else: + from ultralytics.utils.files import increment_path + + project = args.project or (ROOT.parent / "tests/tmp/runs" if TESTS_RUNNING else RUNS_DIR) / args.task + name = name or args.name or f"{args.mode}" + save_dir = increment_path(Path(project) / name, exist_ok=args.exist_ok if RANK in (-1, 0) else True) + + return Path(save_dir) + + +def _handle_deprecation(custom): + """Hardcoded function to handle deprecated config keys.""" + + for key in custom.copy().keys(): + if key == "boxes": + deprecation_warn(key, "show_boxes") + custom["show_boxes"] = custom.pop("boxes") + if key == "hide_labels": + deprecation_warn(key, "show_labels") + custom["show_labels"] = custom.pop("hide_labels") == "False" + if key == "hide_conf": + deprecation_warn(key, "show_conf") + custom["show_conf"] = custom.pop("hide_conf") == "False" + if key == "line_thickness": + deprecation_warn(key, "line_width") + custom["line_width"] = custom.pop("line_thickness") + + return custom + + +def check_dict_alignment(base: Dict, custom: Dict, e=None): + """ + This function checks for any mismatched keys between a custom configuration list and a base configuration list. If + any mismatched keys are found, the function prints out similar keys from the base list and exits the program. + + Args: + custom (dict): a dictionary of custom configuration options + base (dict): a dictionary of base configuration options + e (Error, optional): An optional error that is passed by the calling function. + """ + custom = _handle_deprecation(custom) + base_keys, custom_keys = (set(x.keys()) for x in (base, custom)) + mismatched = [k for k in custom_keys if k not in base_keys] + if mismatched: + from difflib import get_close_matches + + string = "" + for x in mismatched: + matches = get_close_matches(x, base_keys) # key list + matches = [f"{k}={base[k]}" if base.get(k) is not None else k for k in matches] + match_str = f"Similar arguments are i.e. {matches}." if matches else "" + string += f"'{colorstr('red', 'bold', x)}' is not a valid YOLO argument. {match_str}\n" + raise SyntaxError(string + CLI_HELP_MSG) from e + + +def merge_equals_args(args: List[str]) -> List[str]: + """ + Merges arguments around isolated '=' args in a list of strings. The function considers cases where the first + argument ends with '=' or the second starts with '=', as well as when the middle one is an equals sign. + + Args: + args (List[str]): A list of strings where each element is an argument. + + Returns: + (List[str]): A list of strings where the arguments around isolated '=' are merged. + """ + new_args = [] + for i, arg in enumerate(args): + if arg == "=" and 0 < i < len(args) - 1: # merge ['arg', '=', 'val'] + new_args[-1] += f"={args[i + 1]}" + del args[i + 1] + elif arg.endswith("=") and i < len(args) - 1 and "=" not in args[i + 1]: # merge ['arg=', 'val'] + new_args.append(f"{arg}{args[i + 1]}") + del args[i + 1] + elif arg.startswith("=") and i > 0: # merge ['arg', '=val'] + new_args[-1] += arg + else: + new_args.append(arg) + return new_args + + +def handle_yolo_hub(args: List[str]) -> None: + """ + Handle Ultralytics HUB command-line interface (CLI) commands. + + This function processes Ultralytics HUB CLI commands such as login and logout. + It should be called when executing a script with arguments related to HUB authentication. + + Args: + args (List[str]): A list of command line arguments + + Example: + ```bash + python my_script.py hub login your_api_key + ``` + """ + from ultralytics import hub + + if args[0] == "login": + key = args[1] if len(args) > 1 else "" + # Log in to Ultralytics HUB using the provided API key + hub.login(key) + elif args[0] == "logout": + # Log out from Ultralytics HUB + hub.logout() + + +def handle_yolo_settings(args: List[str]) -> None: + """ + Handle YOLO settings command-line interface (CLI) commands. + + This function processes YOLO settings CLI commands such as reset. + It should be called when executing a script with arguments related to YOLO settings management. + + Args: + args (List[str]): A list of command line arguments for YOLO settings management. + + Example: + ```bash + python my_script.py yolo settings reset + ``` + """ + url = "https://docs.ultralytics.com/quickstart/#ultralytics-settings" # help URL + try: + if any(args): + if args[0] == "reset": + SETTINGS_YAML.unlink() # delete the settings file + SETTINGS.reset() # create new settings + LOGGER.info("Settings reset successfully") # inform the user that settings have been reset + else: # save a new setting + new = dict(parse_key_value_pair(a) for a in args) + check_dict_alignment(SETTINGS, new) + SETTINGS.update(new) + + LOGGER.info(f"💡 Learn about settings at {url}") + yaml_print(SETTINGS_YAML) # print the current settings + except Exception as e: + LOGGER.warning(f"WARNING ⚠️ settings error: '{e}'. Please see {url} for help.") + + +def handle_explorer(): + """Open the Ultralytics Explorer GUI.""" + checks.check_requirements("streamlit") + LOGGER.info("💡 Loading Explorer dashboard...") + subprocess.run(["streamlit", "run", ROOT / "data/explorer/gui/dash.py", "--server.maxMessageSize", "2048"]) + + +def parse_key_value_pair(pair): + """Parse one 'key=value' pair and return key and value.""" + k, v = pair.split("=", 1) # split on first '=' sign + k, v = k.strip(), v.strip() # remove spaces + assert v, f"missing '{k}' value" + return k, smart_value(v) + + +def smart_value(v): + """Convert a string to an underlying type such as int, float, bool, etc.""" + v_lower = v.lower() + if v_lower == "none": + return None + elif v_lower == "true": + return True + elif v_lower == "false": + return False + else: + with contextlib.suppress(Exception): + return eval(v) + return v + + +def entrypoint(debug=""): + """ + This function is the ultralytics package entrypoint, it's responsible for parsing the command line arguments passed + to the package. + + This function allows for: + - passing mandatory YOLO args as a list of strings + - specifying the task to be performed, either 'detect', 'segment' or 'classify' + - specifying the mode, either 'train', 'val', 'test', or 'predict' + - running special modes like 'checks' + - passing overrides to the package's configuration + + It uses the package's default cfg and initializes it using the passed overrides. + Then it calls the CLI function with the composed cfg + """ + args = (debug.split(" ") if debug else sys.argv)[1:] + if not args: # no arguments passed + LOGGER.info(CLI_HELP_MSG) + return + + special = { + "help": lambda: LOGGER.info(CLI_HELP_MSG), + "checks": checks.collect_system_info, + "version": lambda: LOGGER.info(__version__), + "settings": lambda: handle_yolo_settings(args[1:]), + "cfg": lambda: yaml_print(DEFAULT_CFG_PATH), + "hub": lambda: handle_yolo_hub(args[1:]), + "login": lambda: handle_yolo_hub(args), + "copy-cfg": copy_default_cfg, + "explorer": lambda: handle_explorer(), + } + full_args_dict = {**DEFAULT_CFG_DICT, **{k: None for k in TASKS}, **{k: None for k in MODES}, **special} + + # Define common misuses of special commands, i.e. -h, -help, --help + special.update({k[0]: v for k, v in special.items()}) # singular + special.update({k[:-1]: v for k, v in special.items() if len(k) > 1 and k.endswith("s")}) # singular + special = {**special, **{f"-{k}": v for k, v in special.items()}, **{f"--{k}": v for k, v in special.items()}} + + overrides = {} # basic overrides, i.e. imgsz=320 + for a in merge_equals_args(args): # merge spaces around '=' sign + if a.startswith("--"): + LOGGER.warning(f"WARNING ⚠️ argument '{a}' does not require leading dashes '--', updating to '{a[2:]}'.") + a = a[2:] + if a.endswith(","): + LOGGER.warning(f"WARNING ⚠️ argument '{a}' does not require trailing comma ',', updating to '{a[:-1]}'.") + a = a[:-1] + if "=" in a: + try: + k, v = parse_key_value_pair(a) + if k == "cfg" and v is not None: # custom.yaml passed + LOGGER.info(f"Overriding {DEFAULT_CFG_PATH} with {v}") + overrides = {k: val for k, val in yaml_load(checks.check_yaml(v)).items() if k != "cfg"} + else: + overrides[k] = v + except (NameError, SyntaxError, ValueError, AssertionError) as e: + check_dict_alignment(full_args_dict, {a: ""}, e) + + elif a in TASKS: + overrides["task"] = a + elif a in MODES: + overrides["mode"] = a + elif a.lower() in special: + special[a.lower()]() + return + elif a in DEFAULT_CFG_DICT and isinstance(DEFAULT_CFG_DICT[a], bool): + overrides[a] = True # auto-True for default bool args, i.e. 'yolo show' sets show=True + elif a in DEFAULT_CFG_DICT: + raise SyntaxError( + f"'{colorstr('red', 'bold', a)}' is a valid YOLO argument but is missing an '=' sign " + f"to set its value, i.e. try '{a}={DEFAULT_CFG_DICT[a]}'\n{CLI_HELP_MSG}" + ) + else: + check_dict_alignment(full_args_dict, {a: ""}) + + # Check keys + check_dict_alignment(full_args_dict, overrides) + + # Mode + mode = overrides.get("mode") + if mode is None: + mode = DEFAULT_CFG.mode or "predict" + LOGGER.warning(f"WARNING ⚠️ 'mode' argument is missing. Valid modes are {MODES}. Using default 'mode={mode}'.") + elif mode not in MODES: + raise ValueError(f"Invalid 'mode={mode}'. Valid modes are {MODES}.\n{CLI_HELP_MSG}") + + # Task + task = overrides.pop("task", None) + if task: + if task not in TASKS: + raise ValueError(f"Invalid 'task={task}'. Valid tasks are {TASKS}.\n{CLI_HELP_MSG}") + if "model" not in overrides: + overrides["model"] = TASK2MODEL[task] + + # Model + model = overrides.pop("model", DEFAULT_CFG.model) + if model is None: + model = "yolov8n.pt" + LOGGER.warning(f"WARNING ⚠️ 'model' argument is missing. Using default 'model={model}'.") + overrides["model"] = model + # stem = Path(model).stem.lower() + stem = model.lower() + if "rtdetr" in stem: # guess architecture + from ultralytics import RTDETR + + model = RTDETR(model) # no task argument + elif "fastsam" in stem: + from ultralytics import FastSAM + + model = FastSAM(model) + elif "sam" in stem: + from ultralytics import SAM + + model = SAM(model) + elif re.search("v3|v5|v6|v8|v9", stem): + from ultralytics import YOLO + + model = YOLO(model, task=task) + else: + from ultralytics import YOLOv10 + + # Special case for the HuggingFace Hub + split_path = model.split('/') + if len(split_path) == 2 and (not os.path.exists(model)): + model = YOLOv10.from_pretrained(model) + else: + model = YOLOv10(model) + if isinstance(overrides.get("pretrained"), str): + model.load(overrides["pretrained"]) + + # Task Update + if task != model.task: + if task: + LOGGER.warning( + f"WARNING ⚠️ conflicting 'task={task}' passed with 'task={model.task}' model. " + f"Ignoring 'task={task}' and updating to 'task={model.task}' to match model." + ) + task = model.task + + # Mode + if mode in ("predict", "track") and "source" not in overrides: + overrides["source"] = DEFAULT_CFG.source or ASSETS + LOGGER.warning(f"WARNING ⚠️ 'source' argument is missing. Using default 'source={overrides['source']}'.") + elif mode in ("train", "val"): + if "data" not in overrides and "resume" not in overrides: + overrides["data"] = DEFAULT_CFG.data or TASK2DATA.get(task or DEFAULT_CFG.task, DEFAULT_CFG.data) + LOGGER.warning(f"WARNING ⚠️ 'data' argument is missing. Using default 'data={overrides['data']}'.") + elif mode == "export": + if "format" not in overrides: + overrides["format"] = DEFAULT_CFG.format or "torchscript" + LOGGER.warning(f"WARNING ⚠️ 'format' argument is missing. Using default 'format={overrides['format']}'.") + + # Run command in python + getattr(model, mode)(**overrides) # default args from model + + # Show help + LOGGER.info(f"💡 Learn more at https://docs.ultralytics.com/modes/{mode}") + + +# Special modes -------------------------------------------------------------------------------------------------------- +def copy_default_cfg(): + """Copy and create a new default configuration file with '_copy' appended to its name.""" + new_file = Path.cwd() / DEFAULT_CFG_PATH.name.replace(".yaml", "_copy.yaml") + shutil.copy2(DEFAULT_CFG_PATH, new_file) + LOGGER.info( + f"{DEFAULT_CFG_PATH} copied to {new_file}\n" + f"Example YOLO command with this new custom cfg:\n yolo cfg='{new_file}' imgsz=320 batch=8" + ) + + +if __name__ == "__main__": + # Example: entrypoint(debug='yolo predict model=yolov8n.pt') + entrypoint(debug="") diff --git a/ultralytics/cfg/__pycache__/__init__.cpython-312.pyc b/ultralytics/cfg/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..b9a6a30 Binary files /dev/null and b/ultralytics/cfg/__pycache__/__init__.cpython-312.pyc differ diff --git a/ultralytics/cfg/__pycache__/__init__.cpython-39.pyc b/ultralytics/cfg/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..14e2e94 Binary files /dev/null and b/ultralytics/cfg/__pycache__/__init__.cpython-39.pyc differ diff --git a/ultralytics/cfg/datasets/Argoverse.yaml b/ultralytics/cfg/datasets/Argoverse.yaml new file mode 100644 index 0000000..43755f7 --- /dev/null +++ b/ultralytics/cfg/datasets/Argoverse.yaml @@ -0,0 +1,74 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# Argoverse-HD dataset (ring-front-center camera) https://www.cs.cmu.edu/~mengtial/proj/streaming/ by Argo AI +# Documentation: https://docs.ultralytics.com/datasets/detect/argoverse/ +# Example usage: yolo train data=Argoverse.yaml +# parent +# ├── ultralytics +# └── datasets +# └── Argoverse ← downloads here (31.5 GB) + +# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..] +path: ../datasets/Argoverse # dataset root dir +train: Argoverse-1.1/images/train/ # train images (relative to 'path') 39384 images +val: Argoverse-1.1/images/val/ # val images (relative to 'path') 15062 images +test: Argoverse-1.1/images/test/ # test images (optional) https://eval.ai/web/challenges/challenge-page/800/overview + +# Classes +names: + 0: person + 1: bicycle + 2: car + 3: motorcycle + 4: bus + 5: truck + 6: traffic_light + 7: stop_sign + +# Download script/URL (optional) --------------------------------------------------------------------------------------- +download: | + import json + from tqdm import tqdm + from ultralytics.utils.downloads import download + from pathlib import Path + + def argoverse2yolo(set): + labels = {} + a = json.load(open(set, "rb")) + for annot in tqdm(a['annotations'], desc=f"Converting {set} to YOLOv5 format..."): + img_id = annot['image_id'] + img_name = a['images'][img_id]['name'] + img_label_name = f'{img_name[:-3]}txt' + + cls = annot['category_id'] # instance class id + x_center, y_center, width, height = annot['bbox'] + x_center = (x_center + width / 2) / 1920.0 # offset and scale + y_center = (y_center + height / 2) / 1200.0 # offset and scale + width /= 1920.0 # scale + height /= 1200.0 # scale + + img_dir = set.parents[2] / 'Argoverse-1.1' / 'labels' / a['seq_dirs'][a['images'][annot['image_id']]['sid']] + if not img_dir.exists(): + img_dir.mkdir(parents=True, exist_ok=True) + + k = str(img_dir / img_label_name) + if k not in labels: + labels[k] = [] + labels[k].append(f"{cls} {x_center} {y_center} {width} {height}\n") + + for k in labels: + with open(k, "w") as f: + f.writelines(labels[k]) + + + # Download 'https://argoverse-hd.s3.us-east-2.amazonaws.com/Argoverse-HD-Full.zip' (deprecated S3 link) + dir = Path(yaml['path']) # dataset root dir + urls = ['https://drive.google.com/file/d/1st9qW3BeIwQsnR0t8mRpvbsSWIo16ACi/view?usp=drive_link'] + print("\n\nWARNING: Argoverse dataset MUST be downloaded manually, autodownload will NOT work.") + print(f"WARNING: Manually download Argoverse dataset '{urls[0]}' to '{dir}' and re-run your command.\n\n") + # download(urls, dir=dir) + + # Convert + annotations_dir = 'Argoverse-HD/annotations/' + (dir / 'Argoverse-1.1' / 'tracking').rename(dir / 'Argoverse-1.1' / 'images') # rename 'tracking' to 'images' + for d in "train.json", "val.json": + argoverse2yolo(dir / annotations_dir / d) # convert Argoverse annotations to YOLO labels diff --git a/ultralytics/cfg/datasets/DOTAv1.5.yaml b/ultralytics/cfg/datasets/DOTAv1.5.yaml new file mode 100644 index 0000000..701535f --- /dev/null +++ b/ultralytics/cfg/datasets/DOTAv1.5.yaml @@ -0,0 +1,36 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# DOTA 1.5 dataset https://captain-whu.github.io/DOTA/index.html for object detection in aerial images by Wuhan University +# Documentation: https://docs.ultralytics.com/datasets/obb/dota-v2/ +# Example usage: yolo train model=yolov8n-obb.pt data=DOTAv1.5.yaml +# parent +# ├── ultralytics +# └── datasets +# └── dota1.5 ← downloads here (2GB) + +# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..] +path: ../datasets/DOTAv1.5 # dataset root dir +train: images/train # train images (relative to 'path') 1411 images +val: images/val # val images (relative to 'path') 458 images +test: images/test # test images (optional) 937 images + +# Classes for DOTA 1.5 +names: + 0: plane + 1: ship + 2: storage tank + 3: baseball diamond + 4: tennis court + 5: basketball court + 6: ground track field + 7: harbor + 8: bridge + 9: large vehicle + 10: small vehicle + 11: helicopter + 12: roundabout + 13: soccer ball field + 14: swimming pool + 15: container crane + +# Download script/URL (optional) +download: https://github.com/ultralytics/yolov5/releases/download/v1.0/DOTAv1.5.zip diff --git a/ultralytics/cfg/datasets/DOTAv1.yaml b/ultralytics/cfg/datasets/DOTAv1.yaml new file mode 100644 index 0000000..f6364d3 --- /dev/null +++ b/ultralytics/cfg/datasets/DOTAv1.yaml @@ -0,0 +1,35 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# DOTA 1.0 dataset https://captain-whu.github.io/DOTA/index.html for object detection in aerial images by Wuhan University +# Documentation: https://docs.ultralytics.com/datasets/obb/dota-v2/ +# Example usage: yolo train model=yolov8n-obb.pt data=DOTAv1.yaml +# parent +# ├── ultralytics +# └── datasets +# └── dota1 ← downloads here (2GB) + +# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..] +path: ../datasets/DOTAv1 # dataset root dir +train: images/train # train images (relative to 'path') 1411 images +val: images/val # val images (relative to 'path') 458 images +test: images/test # test images (optional) 937 images + +# Classes for DOTA 1.0 +names: + 0: plane + 1: ship + 2: storage tank + 3: baseball diamond + 4: tennis court + 5: basketball court + 6: ground track field + 7: harbor + 8: bridge + 9: large vehicle + 10: small vehicle + 11: helicopter + 12: roundabout + 13: soccer ball field + 14: swimming pool + +# Download script/URL (optional) +download: https://github.com/ultralytics/yolov5/releases/download/v1.0/DOTAv1.zip diff --git a/ultralytics/cfg/datasets/GlobalWheat2020.yaml b/ultralytics/cfg/datasets/GlobalWheat2020.yaml new file mode 100644 index 0000000..ae6bfa0 --- /dev/null +++ b/ultralytics/cfg/datasets/GlobalWheat2020.yaml @@ -0,0 +1,53 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# Global Wheat 2020 dataset https://www.global-wheat.com/ by University of Saskatchewan +# Documentation: https://docs.ultralytics.com/datasets/detect/globalwheat2020/ +# Example usage: yolo train data=GlobalWheat2020.yaml +# parent +# ├── ultralytics +# └── datasets +# └── GlobalWheat2020 ← downloads here (7.0 GB) + +# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..] +path: ../datasets/GlobalWheat2020 # dataset root dir +train: # train images (relative to 'path') 3422 images + - images/arvalis_1 + - images/arvalis_2 + - images/arvalis_3 + - images/ethz_1 + - images/rres_1 + - images/inrae_1 + - images/usask_1 +val: # val images (relative to 'path') 748 images (WARNING: train set contains ethz_1) + - images/ethz_1 +test: # test images (optional) 1276 images + - images/utokyo_1 + - images/utokyo_2 + - images/nau_1 + - images/uq_1 + +# Classes +names: + 0: wheat_head + +# Download script/URL (optional) --------------------------------------------------------------------------------------- +download: | + from ultralytics.utils.downloads import download + from pathlib import Path + + # Download + dir = Path(yaml['path']) # dataset root dir + urls = ['https://zenodo.org/record/4298502/files/global-wheat-codalab-official.zip', + 'https://github.com/ultralytics/yolov5/releases/download/v1.0/GlobalWheat2020_labels.zip'] + download(urls, dir=dir) + + # Make Directories + for p in 'annotations', 'images', 'labels': + (dir / p).mkdir(parents=True, exist_ok=True) + + # Move + for p in 'arvalis_1', 'arvalis_2', 'arvalis_3', 'ethz_1', 'rres_1', 'inrae_1', 'usask_1', \ + 'utokyo_1', 'utokyo_2', 'nau_1', 'uq_1': + (dir / 'global-wheat-codalab-official' / p).rename(dir / 'images' / p) # move to /images + f = (dir / 'global-wheat-codalab-official' / p).with_suffix('.json') # json file + if f.exists(): + f.rename((dir / 'annotations' / p).with_suffix('.json')) # move to /annotations diff --git a/ultralytics/cfg/datasets/ImageNet.yaml b/ultralytics/cfg/datasets/ImageNet.yaml new file mode 100644 index 0000000..0dc344a --- /dev/null +++ b/ultralytics/cfg/datasets/ImageNet.yaml @@ -0,0 +1,2024 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# ImageNet-1k dataset https://www.image-net.org/index.php by Stanford University +# Simplified class names from https://github.com/anishathalye/imagenet-simple-labels +# Documentation: https://docs.ultralytics.com/datasets/classify/imagenet/ +# Example usage: yolo train task=classify data=imagenet +# parent +# ├── ultralytics +# └── datasets +# └── imagenet ← downloads here (144 GB) + +# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..] +path: ../datasets/imagenet # dataset root dir +train: train # train images (relative to 'path') 1281167 images +val: val # val images (relative to 'path') 50000 images +test: # test images (optional) + +# Classes +names: + 0: tench + 1: goldfish + 2: great white shark + 3: tiger shark + 4: hammerhead shark + 5: electric ray + 6: stingray + 7: cock + 8: hen + 9: ostrich + 10: brambling + 11: goldfinch + 12: house finch + 13: junco + 14: indigo bunting + 15: American robin + 16: bulbul + 17: jay + 18: magpie + 19: chickadee + 20: American dipper + 21: kite + 22: bald eagle + 23: vulture + 24: great grey owl + 25: fire salamander + 26: smooth newt + 27: newt + 28: spotted salamander + 29: axolotl + 30: American bullfrog + 31: tree frog + 32: tailed frog + 33: loggerhead sea turtle + 34: leatherback sea turtle + 35: mud turtle + 36: terrapin + 37: box turtle + 38: banded gecko + 39: green iguana + 40: Carolina anole + 41: desert grassland whiptail lizard + 42: agama + 43: frilled-necked lizard + 44: alligator lizard + 45: Gila monster + 46: European green lizard + 47: chameleon + 48: Komodo dragon + 49: Nile crocodile + 50: American alligator + 51: triceratops + 52: worm snake + 53: ring-necked snake + 54: eastern hog-nosed snake + 55: smooth green snake + 56: kingsnake + 57: garter snake + 58: water snake + 59: vine snake + 60: night snake + 61: boa constrictor + 62: African rock python + 63: Indian cobra + 64: green mamba + 65: sea snake + 66: Saharan horned viper + 67: eastern diamondback rattlesnake + 68: sidewinder + 69: trilobite + 70: harvestman + 71: scorpion + 72: yellow garden spider + 73: barn spider + 74: European garden spider + 75: southern black widow + 76: tarantula + 77: wolf spider + 78: tick + 79: centipede + 80: black grouse + 81: ptarmigan + 82: ruffed grouse + 83: prairie grouse + 84: peacock + 85: quail + 86: partridge + 87: grey parrot + 88: macaw + 89: sulphur-crested cockatoo + 90: lorikeet + 91: coucal + 92: bee eater + 93: hornbill + 94: hummingbird + 95: jacamar + 96: toucan + 97: duck + 98: red-breasted merganser + 99: goose + 100: black swan + 101: tusker + 102: echidna + 103: platypus + 104: wallaby + 105: koala + 106: wombat + 107: jellyfish + 108: sea anemone + 109: brain coral + 110: flatworm + 111: nematode + 112: conch + 113: snail + 114: slug + 115: sea slug + 116: chiton + 117: chambered nautilus + 118: Dungeness crab + 119: rock crab + 120: fiddler crab + 121: red king crab + 122: American lobster + 123: spiny lobster + 124: crayfish + 125: hermit crab + 126: isopod + 127: white stork + 128: black stork + 129: spoonbill + 130: flamingo + 131: little blue heron + 132: great egret + 133: bittern + 134: crane (bird) + 135: limpkin + 136: common gallinule + 137: American coot + 138: bustard + 139: ruddy turnstone + 140: dunlin + 141: common redshank + 142: dowitcher + 143: oystercatcher + 144: pelican + 145: king penguin + 146: albatross + 147: grey whale + 148: killer whale + 149: dugong + 150: sea lion + 151: Chihuahua + 152: Japanese Chin + 153: Maltese + 154: Pekingese + 155: Shih Tzu + 156: King Charles Spaniel + 157: Papillon + 158: toy terrier + 159: Rhodesian Ridgeback + 160: Afghan Hound + 161: Basset Hound + 162: Beagle + 163: Bloodhound + 164: Bluetick Coonhound + 165: Black and Tan Coonhound + 166: Treeing Walker Coonhound + 167: English foxhound + 168: Redbone Coonhound + 169: borzoi + 170: Irish Wolfhound + 171: Italian Greyhound + 172: Whippet + 173: Ibizan Hound + 174: Norwegian Elkhound + 175: Otterhound + 176: Saluki + 177: Scottish Deerhound + 178: Weimaraner + 179: Staffordshire Bull Terrier + 180: American Staffordshire Terrier + 181: Bedlington Terrier + 182: Border Terrier + 183: Kerry Blue Terrier + 184: Irish Terrier + 185: Norfolk Terrier + 186: Norwich Terrier + 187: Yorkshire Terrier + 188: Wire Fox Terrier + 189: Lakeland Terrier + 190: Sealyham Terrier + 191: Airedale Terrier + 192: Cairn Terrier + 193: Australian Terrier + 194: Dandie Dinmont Terrier + 195: Boston Terrier + 196: Miniature Schnauzer + 197: Giant Schnauzer + 198: Standard Schnauzer + 199: Scottish Terrier + 200: Tibetan Terrier + 201: Australian Silky Terrier + 202: Soft-coated Wheaten Terrier + 203: West Highland White Terrier + 204: Lhasa Apso + 205: Flat-Coated Retriever + 206: Curly-coated Retriever + 207: Golden Retriever + 208: Labrador Retriever + 209: Chesapeake Bay Retriever + 210: German Shorthaired Pointer + 211: Vizsla + 212: English Setter + 213: Irish Setter + 214: Gordon Setter + 215: Brittany + 216: Clumber Spaniel + 217: English Springer Spaniel + 218: Welsh Springer Spaniel + 219: Cocker Spaniels + 220: Sussex Spaniel + 221: Irish Water Spaniel + 222: Kuvasz + 223: Schipperke + 224: Groenendael + 225: Malinois + 226: Briard + 227: Australian Kelpie + 228: Komondor + 229: Old English Sheepdog + 230: Shetland Sheepdog + 231: collie + 232: Border Collie + 233: Bouvier des Flandres + 234: Rottweiler + 235: German Shepherd Dog + 236: Dobermann + 237: Miniature Pinscher + 238: Greater Swiss Mountain Dog + 239: Bernese Mountain Dog + 240: Appenzeller Sennenhund + 241: Entlebucher Sennenhund + 242: Boxer + 243: Bullmastiff + 244: Tibetan Mastiff + 245: French Bulldog + 246: Great Dane + 247: St. Bernard + 248: husky + 249: Alaskan Malamute + 250: Siberian Husky + 251: Dalmatian + 252: Affenpinscher + 253: Basenji + 254: pug + 255: Leonberger + 256: Newfoundland + 257: Pyrenean Mountain Dog + 258: Samoyed + 259: Pomeranian + 260: Chow Chow + 261: Keeshond + 262: Griffon Bruxellois + 263: Pembroke Welsh Corgi + 264: Cardigan Welsh Corgi + 265: Toy Poodle + 266: Miniature Poodle + 267: Standard Poodle + 268: Mexican hairless dog + 269: grey wolf + 270: Alaskan tundra wolf + 271: red wolf + 272: coyote + 273: dingo + 274: dhole + 275: African wild dog + 276: hyena + 277: red fox + 278: kit fox + 279: Arctic fox + 280: grey fox + 281: tabby cat + 282: tiger cat + 283: Persian cat + 284: Siamese cat + 285: Egyptian Mau + 286: cougar + 287: lynx + 288: leopard + 289: snow leopard + 290: jaguar + 291: lion + 292: tiger + 293: cheetah + 294: brown bear + 295: American black bear + 296: polar bear + 297: sloth bear + 298: mongoose + 299: meerkat + 300: tiger beetle + 301: ladybug + 302: ground beetle + 303: longhorn beetle + 304: leaf beetle + 305: dung beetle + 306: rhinoceros beetle + 307: weevil + 308: fly + 309: bee + 310: ant + 311: grasshopper + 312: cricket + 313: stick insect + 314: cockroach + 315: mantis + 316: cicada + 317: leafhopper + 318: lacewing + 319: dragonfly + 320: damselfly + 321: red admiral + 322: ringlet + 323: monarch butterfly + 324: small white + 325: sulphur butterfly + 326: gossamer-winged butterfly + 327: starfish + 328: sea urchin + 329: sea cucumber + 330: cottontail rabbit + 331: hare + 332: Angora rabbit + 333: hamster + 334: porcupine + 335: fox squirrel + 336: marmot + 337: beaver + 338: guinea pig + 339: common sorrel + 340: zebra + 341: pig + 342: wild boar + 343: warthog + 344: hippopotamus + 345: ox + 346: water buffalo + 347: bison + 348: ram + 349: bighorn sheep + 350: Alpine ibex + 351: hartebeest + 352: impala + 353: gazelle + 354: dromedary + 355: llama + 356: weasel + 357: mink + 358: European polecat + 359: black-footed ferret + 360: otter + 361: skunk + 362: badger + 363: armadillo + 364: three-toed sloth + 365: orangutan + 366: gorilla + 367: chimpanzee + 368: gibbon + 369: siamang + 370: guenon + 371: patas monkey + 372: baboon + 373: macaque + 374: langur + 375: black-and-white colobus + 376: proboscis monkey + 377: marmoset + 378: white-headed capuchin + 379: howler monkey + 380: titi + 381: Geoffroy's spider monkey + 382: common squirrel monkey + 383: ring-tailed lemur + 384: indri + 385: Asian elephant + 386: African bush elephant + 387: red panda + 388: giant panda + 389: snoek + 390: eel + 391: coho salmon + 392: rock beauty + 393: clownfish + 394: sturgeon + 395: garfish + 396: lionfish + 397: pufferfish + 398: abacus + 399: abaya + 400: academic gown + 401: accordion + 402: acoustic guitar + 403: aircraft carrier + 404: airliner + 405: airship + 406: altar + 407: ambulance + 408: amphibious vehicle + 409: analog clock + 410: apiary + 411: apron + 412: waste container + 413: assault rifle + 414: backpack + 415: bakery + 416: balance beam + 417: balloon + 418: ballpoint pen + 419: Band-Aid + 420: banjo + 421: baluster + 422: barbell + 423: barber chair + 424: barbershop + 425: barn + 426: barometer + 427: barrel + 428: wheelbarrow + 429: baseball + 430: basketball + 431: bassinet + 432: bassoon + 433: swimming cap + 434: bath towel + 435: bathtub + 436: station wagon + 437: lighthouse + 438: beaker + 439: military cap + 440: beer bottle + 441: beer glass + 442: bell-cot + 443: bib + 444: tandem bicycle + 445: bikini + 446: ring binder + 447: binoculars + 448: birdhouse + 449: boathouse + 450: bobsleigh + 451: bolo tie + 452: poke bonnet + 453: bookcase + 454: bookstore + 455: bottle cap + 456: bow + 457: bow tie + 458: brass + 459: bra + 460: breakwater + 461: breastplate + 462: broom + 463: bucket + 464: buckle + 465: bulletproof vest + 466: high-speed train + 467: butcher shop + 468: taxicab + 469: cauldron + 470: candle + 471: cannon + 472: canoe + 473: can opener + 474: cardigan + 475: car mirror + 476: carousel + 477: tool kit + 478: carton + 479: car wheel + 480: automated teller machine + 481: cassette + 482: cassette player + 483: castle + 484: catamaran + 485: CD player + 486: cello + 487: mobile phone + 488: chain + 489: chain-link fence + 490: chain mail + 491: chainsaw + 492: chest + 493: chiffonier + 494: chime + 495: china cabinet + 496: Christmas stocking + 497: church + 498: movie theater + 499: cleaver + 500: cliff dwelling + 501: cloak + 502: clogs + 503: cocktail shaker + 504: coffee mug + 505: coffeemaker + 506: coil + 507: combination lock + 508: computer keyboard + 509: confectionery store + 510: container ship + 511: convertible + 512: corkscrew + 513: cornet + 514: cowboy boot + 515: cowboy hat + 516: cradle + 517: crane (machine) + 518: crash helmet + 519: crate + 520: infant bed + 521: Crock Pot + 522: croquet ball + 523: crutch + 524: cuirass + 525: dam + 526: desk + 527: desktop computer + 528: rotary dial telephone + 529: diaper + 530: digital clock + 531: digital watch + 532: dining table + 533: dishcloth + 534: dishwasher + 535: disc brake + 536: dock + 537: dog sled + 538: dome + 539: doormat + 540: drilling rig + 541: drum + 542: drumstick + 543: dumbbell + 544: Dutch oven + 545: electric fan + 546: electric guitar + 547: electric locomotive + 548: entertainment center + 549: envelope + 550: espresso machine + 551: face powder + 552: feather boa + 553: filing cabinet + 554: fireboat + 555: fire engine + 556: fire screen sheet + 557: flagpole + 558: flute + 559: folding chair + 560: football helmet + 561: forklift + 562: fountain + 563: fountain pen + 564: four-poster bed + 565: freight car + 566: French horn + 567: frying pan + 568: fur coat + 569: garbage truck + 570: gas mask + 571: gas pump + 572: goblet + 573: go-kart + 574: golf ball + 575: golf cart + 576: gondola + 577: gong + 578: gown + 579: grand piano + 580: greenhouse + 581: grille + 582: grocery store + 583: guillotine + 584: barrette + 585: hair spray + 586: half-track + 587: hammer + 588: hamper + 589: hair dryer + 590: hand-held computer + 591: handkerchief + 592: hard disk drive + 593: harmonica + 594: harp + 595: harvester + 596: hatchet + 597: holster + 598: home theater + 599: honeycomb + 600: hook + 601: hoop skirt + 602: horizontal bar + 603: horse-drawn vehicle + 604: hourglass + 605: iPod + 606: clothes iron + 607: jack-o'-lantern + 608: jeans + 609: jeep + 610: T-shirt + 611: jigsaw puzzle + 612: pulled rickshaw + 613: joystick + 614: kimono + 615: knee pad + 616: knot + 617: lab coat + 618: ladle + 619: lampshade + 620: laptop computer + 621: lawn mower + 622: lens cap + 623: paper knife + 624: library + 625: lifeboat + 626: lighter + 627: limousine + 628: ocean liner + 629: lipstick + 630: slip-on shoe + 631: lotion + 632: speaker + 633: loupe + 634: sawmill + 635: magnetic compass + 636: mail bag + 637: mailbox + 638: tights + 639: tank suit + 640: manhole cover + 641: maraca + 642: marimba + 643: mask + 644: match + 645: maypole + 646: maze + 647: measuring cup + 648: medicine chest + 649: megalith + 650: microphone + 651: microwave oven + 652: military uniform + 653: milk can + 654: minibus + 655: miniskirt + 656: minivan + 657: missile + 658: mitten + 659: mixing bowl + 660: mobile home + 661: Model T + 662: modem + 663: monastery + 664: monitor + 665: moped + 666: mortar + 667: square academic cap + 668: mosque + 669: mosquito net + 670: scooter + 671: mountain bike + 672: tent + 673: computer mouse + 674: mousetrap + 675: moving van + 676: muzzle + 677: nail + 678: neck brace + 679: necklace + 680: nipple + 681: notebook computer + 682: obelisk + 683: oboe + 684: ocarina + 685: odometer + 686: oil filter + 687: organ + 688: oscilloscope + 689: overskirt + 690: bullock cart + 691: oxygen mask + 692: packet + 693: paddle + 694: paddle wheel + 695: padlock + 696: paintbrush + 697: pajamas + 698: palace + 699: pan flute + 700: paper towel + 701: parachute + 702: parallel bars + 703: park bench + 704: parking meter + 705: passenger car + 706: patio + 707: payphone + 708: pedestal + 709: pencil case + 710: pencil sharpener + 711: perfume + 712: Petri dish + 713: photocopier + 714: plectrum + 715: Pickelhaube + 716: picket fence + 717: pickup truck + 718: pier + 719: piggy bank + 720: pill bottle + 721: pillow + 722: ping-pong ball + 723: pinwheel + 724: pirate ship + 725: pitcher + 726: hand plane + 727: planetarium + 728: plastic bag + 729: plate rack + 730: plow + 731: plunger + 732: Polaroid camera + 733: pole + 734: police van + 735: poncho + 736: billiard table + 737: soda bottle + 738: pot + 739: potter's wheel + 740: power drill + 741: prayer rug + 742: printer + 743: prison + 744: projectile + 745: projector + 746: hockey puck + 747: punching bag + 748: purse + 749: quill + 750: quilt + 751: race car + 752: racket + 753: radiator + 754: radio + 755: radio telescope + 756: rain barrel + 757: recreational vehicle + 758: reel + 759: reflex camera + 760: refrigerator + 761: remote control + 762: restaurant + 763: revolver + 764: rifle + 765: rocking chair + 766: rotisserie + 767: eraser + 768: rugby ball + 769: ruler + 770: running shoe + 771: safe + 772: safety pin + 773: salt shaker + 774: sandal + 775: sarong + 776: saxophone + 777: scabbard + 778: weighing scale + 779: school bus + 780: schooner + 781: scoreboard + 782: CRT screen + 783: screw + 784: screwdriver + 785: seat belt + 786: sewing machine + 787: shield + 788: shoe store + 789: shoji + 790: shopping basket + 791: shopping cart + 792: shovel + 793: shower cap + 794: shower curtain + 795: ski + 796: ski mask + 797: sleeping bag + 798: slide rule + 799: sliding door + 800: slot machine + 801: snorkel + 802: snowmobile + 803: snowplow + 804: soap dispenser + 805: soccer ball + 806: sock + 807: solar thermal collector + 808: sombrero + 809: soup bowl + 810: space bar + 811: space heater + 812: space shuttle + 813: spatula + 814: motorboat + 815: spider web + 816: spindle + 817: sports car + 818: spotlight + 819: stage + 820: steam locomotive + 821: through arch bridge + 822: steel drum + 823: stethoscope + 824: scarf + 825: stone wall + 826: stopwatch + 827: stove + 828: strainer + 829: tram + 830: stretcher + 831: couch + 832: stupa + 833: submarine + 834: suit + 835: sundial + 836: sunglass + 837: sunglasses + 838: sunscreen + 839: suspension bridge + 840: mop + 841: sweatshirt + 842: swimsuit + 843: swing + 844: switch + 845: syringe + 846: table lamp + 847: tank + 848: tape player + 849: teapot + 850: teddy bear + 851: television + 852: tennis ball + 853: thatched roof + 854: front curtain + 855: thimble + 856: threshing machine + 857: throne + 858: tile roof + 859: toaster + 860: tobacco shop + 861: toilet seat + 862: torch + 863: totem pole + 864: tow truck + 865: toy store + 866: tractor + 867: semi-trailer truck + 868: tray + 869: trench coat + 870: tricycle + 871: trimaran + 872: tripod + 873: triumphal arch + 874: trolleybus + 875: trombone + 876: tub + 877: turnstile + 878: typewriter keyboard + 879: umbrella + 880: unicycle + 881: upright piano + 882: vacuum cleaner + 883: vase + 884: vault + 885: velvet + 886: vending machine + 887: vestment + 888: viaduct + 889: violin + 890: volleyball + 891: waffle iron + 892: wall clock + 893: wallet + 894: wardrobe + 895: military aircraft + 896: sink + 897: washing machine + 898: water bottle + 899: water jug + 900: water tower + 901: whiskey jug + 902: whistle + 903: wig + 904: window screen + 905: window shade + 906: Windsor tie + 907: wine bottle + 908: wing + 909: wok + 910: wooden spoon + 911: wool + 912: split-rail fence + 913: shipwreck + 914: yawl + 915: yurt + 916: website + 917: comic book + 918: crossword + 919: traffic sign + 920: traffic light + 921: dust jacket + 922: menu + 923: plate + 924: guacamole + 925: consomme + 926: hot pot + 927: trifle + 928: ice cream + 929: ice pop + 930: baguette + 931: bagel + 932: pretzel + 933: cheeseburger + 934: hot dog + 935: mashed potato + 936: cabbage + 937: broccoli + 938: cauliflower + 939: zucchini + 940: spaghetti squash + 941: acorn squash + 942: butternut squash + 943: cucumber + 944: artichoke + 945: bell pepper + 946: cardoon + 947: mushroom + 948: Granny Smith + 949: strawberry + 950: orange + 951: lemon + 952: fig + 953: pineapple + 954: banana + 955: jackfruit + 956: custard apple + 957: pomegranate + 958: hay + 959: carbonara + 960: chocolate syrup + 961: dough + 962: meatloaf + 963: pizza + 964: pot pie + 965: burrito + 966: red wine + 967: espresso + 968: cup + 969: eggnog + 970: alp + 971: bubble + 972: cliff + 973: coral reef + 974: geyser + 975: lakeshore + 976: promontory + 977: shoal + 978: seashore + 979: valley + 980: volcano + 981: baseball player + 982: bridegroom + 983: scuba diver + 984: rapeseed + 985: daisy + 986: yellow lady's slipper + 987: corn + 988: acorn + 989: rose hip + 990: horse chestnut seed + 991: coral fungus + 992: agaric + 993: gyromitra + 994: stinkhorn mushroom + 995: earth star + 996: hen-of-the-woods + 997: bolete + 998: ear + 999: toilet paper + +# Imagenet class codes to human-readable names +map: + n01440764: tench + n01443537: goldfish + n01484850: great_white_shark + n01491361: tiger_shark + n01494475: hammerhead + n01496331: electric_ray + n01498041: stingray + n01514668: cock + n01514859: hen + n01518878: ostrich + n01530575: brambling + n01531178: goldfinch + n01532829: house_finch + n01534433: junco + n01537544: indigo_bunting + n01558993: robin + n01560419: bulbul + n01580077: jay + n01582220: magpie + n01592084: chickadee + n01601694: water_ouzel + n01608432: kite + n01614925: bald_eagle + n01616318: vulture + n01622779: great_grey_owl + n01629819: European_fire_salamander + n01630670: common_newt + n01631663: eft + n01632458: spotted_salamander + n01632777: axolotl + n01641577: bullfrog + n01644373: tree_frog + n01644900: tailed_frog + n01664065: loggerhead + n01665541: leatherback_turtle + n01667114: mud_turtle + n01667778: terrapin + n01669191: box_turtle + n01675722: banded_gecko + n01677366: common_iguana + n01682714: American_chameleon + n01685808: whiptail + n01687978: agama + n01688243: frilled_lizard + n01689811: alligator_lizard + n01692333: Gila_monster + n01693334: green_lizard + n01694178: African_chameleon + n01695060: Komodo_dragon + n01697457: African_crocodile + n01698640: American_alligator + n01704323: triceratops + n01728572: thunder_snake + n01728920: ringneck_snake + n01729322: hognose_snake + n01729977: green_snake + n01734418: king_snake + n01735189: garter_snake + n01737021: water_snake + n01739381: vine_snake + n01740131: night_snake + n01742172: boa_constrictor + n01744401: rock_python + n01748264: Indian_cobra + n01749939: green_mamba + n01751748: sea_snake + n01753488: horned_viper + n01755581: diamondback + n01756291: sidewinder + n01768244: trilobite + n01770081: harvestman + n01770393: scorpion + n01773157: black_and_gold_garden_spider + n01773549: barn_spider + n01773797: garden_spider + n01774384: black_widow + n01774750: tarantula + n01775062: wolf_spider + n01776313: tick + n01784675: centipede + n01795545: black_grouse + n01796340: ptarmigan + n01797886: ruffed_grouse + n01798484: prairie_chicken + n01806143: peacock + n01806567: quail + n01807496: partridge + n01817953: African_grey + n01818515: macaw + n01819313: sulphur-crested_cockatoo + n01820546: lorikeet + n01824575: coucal + n01828970: bee_eater + n01829413: hornbill + n01833805: hummingbird + n01843065: jacamar + n01843383: toucan + n01847000: drake + n01855032: red-breasted_merganser + n01855672: goose + n01860187: black_swan + n01871265: tusker + n01872401: echidna + n01873310: platypus + n01877812: wallaby + n01882714: koala + n01883070: wombat + n01910747: jellyfish + n01914609: sea_anemone + n01917289: brain_coral + n01924916: flatworm + n01930112: nematode + n01943899: conch + n01944390: snail + n01945685: slug + n01950731: sea_slug + n01955084: chiton + n01968897: chambered_nautilus + n01978287: Dungeness_crab + n01978455: rock_crab + n01980166: fiddler_crab + n01981276: king_crab + n01983481: American_lobster + n01984695: spiny_lobster + n01985128: crayfish + n01986214: hermit_crab + n01990800: isopod + n02002556: white_stork + n02002724: black_stork + n02006656: spoonbill + n02007558: flamingo + n02009229: little_blue_heron + n02009912: American_egret + n02011460: bittern + n02012849: crane_(bird) + n02013706: limpkin + n02017213: European_gallinule + n02018207: American_coot + n02018795: bustard + n02025239: ruddy_turnstone + n02027492: red-backed_sandpiper + n02028035: redshank + n02033041: dowitcher + n02037110: oystercatcher + n02051845: pelican + n02056570: king_penguin + n02058221: albatross + n02066245: grey_whale + n02071294: killer_whale + n02074367: dugong + n02077923: sea_lion + n02085620: Chihuahua + n02085782: Japanese_spaniel + n02085936: Maltese_dog + n02086079: Pekinese + n02086240: Shih-Tzu + n02086646: Blenheim_spaniel + n02086910: papillon + n02087046: toy_terrier + n02087394: Rhodesian_ridgeback + n02088094: Afghan_hound + n02088238: basset + n02088364: beagle + n02088466: bloodhound + n02088632: bluetick + n02089078: black-and-tan_coonhound + n02089867: Walker_hound + n02089973: English_foxhound + n02090379: redbone + n02090622: borzoi + n02090721: Irish_wolfhound + n02091032: Italian_greyhound + n02091134: whippet + n02091244: Ibizan_hound + n02091467: Norwegian_elkhound + n02091635: otterhound + n02091831: Saluki + n02092002: Scottish_deerhound + n02092339: Weimaraner + n02093256: Staffordshire_bullterrier + n02093428: American_Staffordshire_terrier + n02093647: Bedlington_terrier + n02093754: Border_terrier + n02093859: Kerry_blue_terrier + n02093991: Irish_terrier + n02094114: Norfolk_terrier + n02094258: Norwich_terrier + n02094433: Yorkshire_terrier + n02095314: wire-haired_fox_terrier + n02095570: Lakeland_terrier + n02095889: Sealyham_terrier + n02096051: Airedale + n02096177: cairn + n02096294: Australian_terrier + n02096437: Dandie_Dinmont + n02096585: Boston_bull + n02097047: miniature_schnauzer + n02097130: giant_schnauzer + n02097209: standard_schnauzer + n02097298: Scotch_terrier + n02097474: Tibetan_terrier + n02097658: silky_terrier + n02098105: soft-coated_wheaten_terrier + n02098286: West_Highland_white_terrier + n02098413: Lhasa + n02099267: flat-coated_retriever + n02099429: curly-coated_retriever + n02099601: golden_retriever + n02099712: Labrador_retriever + n02099849: Chesapeake_Bay_retriever + n02100236: German_short-haired_pointer + n02100583: vizsla + n02100735: English_setter + n02100877: Irish_setter + n02101006: Gordon_setter + n02101388: Brittany_spaniel + n02101556: clumber + n02102040: English_springer + n02102177: Welsh_springer_spaniel + n02102318: cocker_spaniel + n02102480: Sussex_spaniel + n02102973: Irish_water_spaniel + n02104029: kuvasz + n02104365: schipperke + n02105056: groenendael + n02105162: malinois + n02105251: briard + n02105412: kelpie + n02105505: komondor + n02105641: Old_English_sheepdog + n02105855: Shetland_sheepdog + n02106030: collie + n02106166: Border_collie + n02106382: Bouvier_des_Flandres + n02106550: Rottweiler + n02106662: German_shepherd + n02107142: Doberman + n02107312: miniature_pinscher + n02107574: Greater_Swiss_Mountain_dog + n02107683: Bernese_mountain_dog + n02107908: Appenzeller + n02108000: EntleBucher + n02108089: boxer + n02108422: bull_mastiff + n02108551: Tibetan_mastiff + n02108915: French_bulldog + n02109047: Great_Dane + n02109525: Saint_Bernard + n02109961: Eskimo_dog + n02110063: malamute + n02110185: Siberian_husky + n02110341: dalmatian + n02110627: affenpinscher + n02110806: basenji + n02110958: pug + n02111129: Leonberg + n02111277: Newfoundland + n02111500: Great_Pyrenees + n02111889: Samoyed + n02112018: Pomeranian + n02112137: chow + n02112350: keeshond + n02112706: Brabancon_griffon + n02113023: Pembroke + n02113186: Cardigan + n02113624: toy_poodle + n02113712: miniature_poodle + n02113799: standard_poodle + n02113978: Mexican_hairless + n02114367: timber_wolf + n02114548: white_wolf + n02114712: red_wolf + n02114855: coyote + n02115641: dingo + n02115913: dhole + n02116738: African_hunting_dog + n02117135: hyena + n02119022: red_fox + n02119789: kit_fox + n02120079: Arctic_fox + n02120505: grey_fox + n02123045: tabby + n02123159: tiger_cat + n02123394: Persian_cat + n02123597: Siamese_cat + n02124075: Egyptian_cat + n02125311: cougar + n02127052: lynx + n02128385: leopard + n02128757: snow_leopard + n02128925: jaguar + n02129165: lion + n02129604: tiger + n02130308: cheetah + n02132136: brown_bear + n02133161: American_black_bear + n02134084: ice_bear + n02134418: sloth_bear + n02137549: mongoose + n02138441: meerkat + n02165105: tiger_beetle + n02165456: ladybug + n02167151: ground_beetle + n02168699: long-horned_beetle + n02169497: leaf_beetle + n02172182: dung_beetle + n02174001: rhinoceros_beetle + n02177972: weevil + n02190166: fly + n02206856: bee + n02219486: ant + n02226429: grasshopper + n02229544: cricket + n02231487: walking_stick + n02233338: cockroach + n02236044: mantis + n02256656: cicada + n02259212: leafhopper + n02264363: lacewing + n02268443: dragonfly + n02268853: damselfly + n02276258: admiral + n02277742: ringlet + n02279972: monarch + n02280649: cabbage_butterfly + n02281406: sulphur_butterfly + n02281787: lycaenid + n02317335: starfish + n02319095: sea_urchin + n02321529: sea_cucumber + n02325366: wood_rabbit + n02326432: hare + n02328150: Angora + n02342885: hamster + n02346627: porcupine + n02356798: fox_squirrel + n02361337: marmot + n02363005: beaver + n02364673: guinea_pig + n02389026: sorrel + n02391049: zebra + n02395406: hog + n02396427: wild_boar + n02397096: warthog + n02398521: hippopotamus + n02403003: ox + n02408429: water_buffalo + n02410509: bison + n02412080: ram + n02415577: bighorn + n02417914: ibex + n02422106: hartebeest + n02422699: impala + n02423022: gazelle + n02437312: Arabian_camel + n02437616: llama + n02441942: weasel + n02442845: mink + n02443114: polecat + n02443484: black-footed_ferret + n02444819: otter + n02445715: skunk + n02447366: badger + n02454379: armadillo + n02457408: three-toed_sloth + n02480495: orangutan + n02480855: gorilla + n02481823: chimpanzee + n02483362: gibbon + n02483708: siamang + n02484975: guenon + n02486261: patas + n02486410: baboon + n02487347: macaque + n02488291: langur + n02488702: colobus + n02489166: proboscis_monkey + n02490219: marmoset + n02492035: capuchin + n02492660: howler_monkey + n02493509: titi + n02493793: spider_monkey + n02494079: squirrel_monkey + n02497673: Madagascar_cat + n02500267: indri + n02504013: Indian_elephant + n02504458: African_elephant + n02509815: lesser_panda + n02510455: giant_panda + n02514041: barracouta + n02526121: eel + n02536864: coho + n02606052: rock_beauty + n02607072: anemone_fish + n02640242: sturgeon + n02641379: gar + n02643566: lionfish + n02655020: puffer + n02666196: abacus + n02667093: abaya + n02669723: academic_gown + n02672831: accordion + n02676566: acoustic_guitar + n02687172: aircraft_carrier + n02690373: airliner + n02692877: airship + n02699494: altar + n02701002: ambulance + n02704792: amphibian + n02708093: analog_clock + n02727426: apiary + n02730930: apron + n02747177: ashcan + n02749479: assault_rifle + n02769748: backpack + n02776631: bakery + n02777292: balance_beam + n02782093: balloon + n02783161: ballpoint + n02786058: Band_Aid + n02787622: banjo + n02788148: bannister + n02790996: barbell + n02791124: barber_chair + n02791270: barbershop + n02793495: barn + n02794156: barometer + n02795169: barrel + n02797295: barrow + n02799071: baseball + n02802426: basketball + n02804414: bassinet + n02804610: bassoon + n02807133: bathing_cap + n02808304: bath_towel + n02808440: bathtub + n02814533: beach_wagon + n02814860: beacon + n02815834: beaker + n02817516: bearskin + n02823428: beer_bottle + n02823750: beer_glass + n02825657: bell_cote + n02834397: bib + n02835271: bicycle-built-for-two + n02837789: bikini + n02840245: binder + n02841315: binoculars + n02843684: birdhouse + n02859443: boathouse + n02860847: bobsled + n02865351: bolo_tie + n02869837: bonnet + n02870880: bookcase + n02871525: bookshop + n02877765: bottlecap + n02879718: bow + n02883205: bow_tie + n02892201: brass + n02892767: brassiere + n02894605: breakwater + n02895154: breastplate + n02906734: broom + n02909870: bucket + n02910353: buckle + n02916936: bulletproof_vest + n02917067: bullet_train + n02927161: butcher_shop + n02930766: cab + n02939185: caldron + n02948072: candle + n02950826: cannon + n02951358: canoe + n02951585: can_opener + n02963159: cardigan + n02965783: car_mirror + n02966193: carousel + n02966687: carpenter's_kit + n02971356: carton + n02974003: car_wheel + n02977058: cash_machine + n02978881: cassette + n02979186: cassette_player + n02980441: castle + n02981792: catamaran + n02988304: CD_player + n02992211: cello + n02992529: cellular_telephone + n02999410: chain + n03000134: chainlink_fence + n03000247: chain_mail + n03000684: chain_saw + n03014705: chest + n03016953: chiffonier + n03017168: chime + n03018349: china_cabinet + n03026506: Christmas_stocking + n03028079: church + n03032252: cinema + n03041632: cleaver + n03042490: cliff_dwelling + n03045698: cloak + n03047690: clog + n03062245: cocktail_shaker + n03063599: coffee_mug + n03063689: coffeepot + n03065424: coil + n03075370: combination_lock + n03085013: computer_keyboard + n03089624: confectionery + n03095699: container_ship + n03100240: convertible + n03109150: corkscrew + n03110669: cornet + n03124043: cowboy_boot + n03124170: cowboy_hat + n03125729: cradle + n03126707: crane_(machine) + n03127747: crash_helmet + n03127925: crate + n03131574: crib + n03133878: Crock_Pot + n03134739: croquet_ball + n03141823: crutch + n03146219: cuirass + n03160309: dam + n03179701: desk + n03180011: desktop_computer + n03187595: dial_telephone + n03188531: diaper + n03196217: digital_clock + n03197337: digital_watch + n03201208: dining_table + n03207743: dishrag + n03207941: dishwasher + n03208938: disk_brake + n03216828: dock + n03218198: dogsled + n03220513: dome + n03223299: doormat + n03240683: drilling_platform + n03249569: drum + n03250847: drumstick + n03255030: dumbbell + n03259280: Dutch_oven + n03271574: electric_fan + n03272010: electric_guitar + n03272562: electric_locomotive + n03290653: entertainment_center + n03291819: envelope + n03297495: espresso_maker + n03314780: face_powder + n03325584: feather_boa + n03337140: file + n03344393: fireboat + n03345487: fire_engine + n03347037: fire_screen + n03355925: flagpole + n03372029: flute + n03376595: folding_chair + n03379051: football_helmet + n03384352: forklift + n03388043: fountain + n03388183: fountain_pen + n03388549: four-poster + n03393912: freight_car + n03394916: French_horn + n03400231: frying_pan + n03404251: fur_coat + n03417042: garbage_truck + n03424325: gasmask + n03425413: gas_pump + n03443371: goblet + n03444034: go-kart + n03445777: golf_ball + n03445924: golfcart + n03447447: gondola + n03447721: gong + n03450230: gown + n03452741: grand_piano + n03457902: greenhouse + n03459775: grille + n03461385: grocery_store + n03467068: guillotine + n03476684: hair_slide + n03476991: hair_spray + n03478589: half_track + n03481172: hammer + n03482405: hamper + n03483316: hand_blower + n03485407: hand-held_computer + n03485794: handkerchief + n03492542: hard_disc + n03494278: harmonica + n03495258: harp + n03496892: harvester + n03498962: hatchet + n03527444: holster + n03529860: home_theater + n03530642: honeycomb + n03532672: hook + n03534580: hoopskirt + n03535780: horizontal_bar + n03538406: horse_cart + n03544143: hourglass + n03584254: iPod + n03584829: iron + n03590841: jack-o'-lantern + n03594734: jean + n03594945: jeep + n03595614: jersey + n03598930: jigsaw_puzzle + n03599486: jinrikisha + n03602883: joystick + n03617480: kimono + n03623198: knee_pad + n03627232: knot + n03630383: lab_coat + n03633091: ladle + n03637318: lampshade + n03642806: laptop + n03649909: lawn_mower + n03657121: lens_cap + n03658185: letter_opener + n03661043: library + n03662601: lifeboat + n03666591: lighter + n03670208: limousine + n03673027: liner + n03676483: lipstick + n03680355: Loafer + n03690938: lotion + n03691459: loudspeaker + n03692522: loupe + n03697007: lumbermill + n03706229: magnetic_compass + n03709823: mailbag + n03710193: mailbox + n03710637: maillot_(tights) + n03710721: maillot_(tank_suit) + n03717622: manhole_cover + n03720891: maraca + n03721384: marimba + n03724870: mask + n03729826: matchstick + n03733131: maypole + n03733281: maze + n03733805: measuring_cup + n03742115: medicine_chest + n03743016: megalith + n03759954: microphone + n03761084: microwave + n03763968: military_uniform + n03764736: milk_can + n03769881: minibus + n03770439: miniskirt + n03770679: minivan + n03773504: missile + n03775071: mitten + n03775546: mixing_bowl + n03776460: mobile_home + n03777568: Model_T + n03777754: modem + n03781244: monastery + n03782006: monitor + n03785016: moped + n03786901: mortar + n03787032: mortarboard + n03788195: mosque + n03788365: mosquito_net + n03791053: motor_scooter + n03792782: mountain_bike + n03792972: mountain_tent + n03793489: mouse + n03794056: mousetrap + n03796401: moving_van + n03803284: muzzle + n03804744: nail + n03814639: neck_brace + n03814906: necklace + n03825788: nipple + n03832673: notebook + n03837869: obelisk + n03838899: oboe + n03840681: ocarina + n03841143: odometer + n03843555: oil_filter + n03854065: organ + n03857828: oscilloscope + n03866082: overskirt + n03868242: oxcart + n03868863: oxygen_mask + n03871628: packet + n03873416: paddle + n03874293: paddlewheel + n03874599: padlock + n03876231: paintbrush + n03877472: pajama + n03877845: palace + n03884397: panpipe + n03887697: paper_towel + n03888257: parachute + n03888605: parallel_bars + n03891251: park_bench + n03891332: parking_meter + n03895866: passenger_car + n03899768: patio + n03902125: pay-phone + n03903868: pedestal + n03908618: pencil_box + n03908714: pencil_sharpener + n03916031: perfume + n03920288: Petri_dish + n03924679: photocopier + n03929660: pick + n03929855: pickelhaube + n03930313: picket_fence + n03930630: pickup + n03933933: pier + n03935335: piggy_bank + n03937543: pill_bottle + n03938244: pillow + n03942813: ping-pong_ball + n03944341: pinwheel + n03947888: pirate + n03950228: pitcher + n03954731: plane + n03956157: planetarium + n03958227: plastic_bag + n03961711: plate_rack + n03967562: plow + n03970156: plunger + n03976467: Polaroid_camera + n03976657: pole + n03977966: police_van + n03980874: poncho + n03982430: pool_table + n03983396: pop_bottle + n03991062: pot + n03992509: potter's_wheel + n03995372: power_drill + n03998194: prayer_rug + n04004767: printer + n04005630: prison + n04008634: projectile + n04009552: projector + n04019541: puck + n04023962: punching_bag + n04026417: purse + n04033901: quill + n04033995: quilt + n04037443: racer + n04039381: racket + n04040759: radiator + n04041544: radio + n04044716: radio_telescope + n04049303: rain_barrel + n04065272: recreational_vehicle + n04067472: reel + n04069434: reflex_camera + n04070727: refrigerator + n04074963: remote_control + n04081281: restaurant + n04086273: revolver + n04090263: rifle + n04099969: rocking_chair + n04111531: rotisserie + n04116512: rubber_eraser + n04118538: rugby_ball + n04118776: rule + n04120489: running_shoe + n04125021: safe + n04127249: safety_pin + n04131690: saltshaker + n04133789: sandal + n04136333: sarong + n04141076: sax + n04141327: scabbard + n04141975: scale + n04146614: school_bus + n04147183: schooner + n04149813: scoreboard + n04152593: screen + n04153751: screw + n04154565: screwdriver + n04162706: seat_belt + n04179913: sewing_machine + n04192698: shield + n04200800: shoe_shop + n04201297: shoji + n04204238: shopping_basket + n04204347: shopping_cart + n04208210: shovel + n04209133: shower_cap + n04209239: shower_curtain + n04228054: ski + n04229816: ski_mask + n04235860: sleeping_bag + n04238763: slide_rule + n04239074: sliding_door + n04243546: slot + n04251144: snorkel + n04252077: snowmobile + n04252225: snowplow + n04254120: soap_dispenser + n04254680: soccer_ball + n04254777: sock + n04258138: solar_dish + n04259630: sombrero + n04263257: soup_bowl + n04264628: space_bar + n04265275: space_heater + n04266014: space_shuttle + n04270147: spatula + n04273569: speedboat + n04275548: spider_web + n04277352: spindle + n04285008: sports_car + n04286575: spotlight + n04296562: stage + n04310018: steam_locomotive + n04311004: steel_arch_bridge + n04311174: steel_drum + n04317175: stethoscope + n04325704: stole + n04326547: stone_wall + n04328186: stopwatch + n04330267: stove + n04332243: strainer + n04335435: streetcar + n04336792: stretcher + n04344873: studio_couch + n04346328: stupa + n04347754: submarine + n04350905: suit + n04355338: sundial + n04355933: sunglass + n04356056: sunglasses + n04357314: sunscreen + n04366367: suspension_bridge + n04367480: swab + n04370456: sweatshirt + n04371430: swimming_trunks + n04371774: swing + n04372370: switch + n04376876: syringe + n04380533: table_lamp + n04389033: tank + n04392985: tape_player + n04398044: teapot + n04399382: teddy + n04404412: television + n04409515: tennis_ball + n04417672: thatch + n04418357: theater_curtain + n04423845: thimble + n04428191: thresher + n04429376: throne + n04435653: tile_roof + n04442312: toaster + n04443257: tobacco_shop + n04447861: toilet_seat + n04456115: torch + n04458633: totem_pole + n04461696: tow_truck + n04462240: toyshop + n04465501: tractor + n04467665: trailer_truck + n04476259: tray + n04479046: trench_coat + n04482393: tricycle + n04483307: trimaran + n04485082: tripod + n04486054: triumphal_arch + n04487081: trolleybus + n04487394: trombone + n04493381: tub + n04501370: turnstile + n04505470: typewriter_keyboard + n04507155: umbrella + n04509417: unicycle + n04515003: upright + n04517823: vacuum + n04522168: vase + n04523525: vault + n04525038: velvet + n04525305: vending_machine + n04532106: vestment + n04532670: viaduct + n04536866: violin + n04540053: volleyball + n04542943: waffle_iron + n04548280: wall_clock + n04548362: wallet + n04550184: wardrobe + n04552348: warplane + n04553703: washbasin + n04554684: washer + n04557648: water_bottle + n04560804: water_jug + n04562935: water_tower + n04579145: whiskey_jug + n04579432: whistle + n04584207: wig + n04589890: window_screen + n04590129: window_shade + n04591157: Windsor_tie + n04591713: wine_bottle + n04592741: wing + n04596742: wok + n04597913: wooden_spoon + n04599235: wool + n04604644: worm_fence + n04606251: wreck + n04612504: yawl + n04613696: yurt + n06359193: web_site + n06596364: comic_book + n06785654: crossword_puzzle + n06794110: street_sign + n06874185: traffic_light + n07248320: book_jacket + n07565083: menu + n07579787: plate + n07583066: guacamole + n07584110: consomme + n07590611: hot_pot + n07613480: trifle + n07614500: ice_cream + n07615774: ice_lolly + n07684084: French_loaf + n07693725: bagel + n07695742: pretzel + n07697313: cheeseburger + n07697537: hotdog + n07711569: mashed_potato + n07714571: head_cabbage + n07714990: broccoli + n07715103: cauliflower + n07716358: zucchini + n07716906: spaghetti_squash + n07717410: acorn_squash + n07717556: butternut_squash + n07718472: cucumber + n07718747: artichoke + n07720875: bell_pepper + n07730033: cardoon + n07734744: mushroom + n07742313: Granny_Smith + n07745940: strawberry + n07747607: orange + n07749582: lemon + n07753113: fig + n07753275: pineapple + n07753592: banana + n07754684: jackfruit + n07760859: custard_apple + n07768694: pomegranate + n07802026: hay + n07831146: carbonara + n07836838: chocolate_sauce + n07860988: dough + n07871810: meat_loaf + n07873807: pizza + n07875152: potpie + n07880968: burrito + n07892512: red_wine + n07920052: espresso + n07930864: cup + n07932039: eggnog + n09193705: alp + n09229709: bubble + n09246464: cliff + n09256479: coral_reef + n09288635: geyser + n09332890: lakeside + n09399592: promontory + n09421951: sandbar + n09428293: seashore + n09468604: valley + n09472597: volcano + n09835506: ballplayer + n10148035: groom + n10565667: scuba_diver + n11879895: rapeseed + n11939491: daisy + n12057211: yellow_lady's_slipper + n12144580: corn + n12267677: acorn + n12620546: hip + n12768682: buckeye + n12985857: coral_fungus + n12998815: agaric + n13037406: gyromitra + n13040303: stinkhorn + n13044778: earthstar + n13052670: hen-of-the-woods + n13054560: bolete + n13133613: ear + n15075141: toilet_tissue + +# Download script/URL (optional) +download: yolo/data/scripts/get_imagenet.sh diff --git a/ultralytics/cfg/datasets/Objects365.yaml b/ultralytics/cfg/datasets/Objects365.yaml new file mode 100644 index 0000000..9b11720 --- /dev/null +++ b/ultralytics/cfg/datasets/Objects365.yaml @@ -0,0 +1,442 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# Objects365 dataset https://www.objects365.org/ by Megvii +# Documentation: https://docs.ultralytics.com/datasets/detect/objects365/ +# Example usage: yolo train data=Objects365.yaml +# parent +# ├── ultralytics +# └── datasets +# └── Objects365 ← downloads here (712 GB = 367G data + 345G zips) + +# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..] +path: ../datasets/Objects365 # dataset root dir +train: images/train # train images (relative to 'path') 1742289 images +val: images/val # val images (relative to 'path') 80000 images +test: # test images (optional) + +# Classes +names: + 0: Person + 1: Sneakers + 2: Chair + 3: Other Shoes + 4: Hat + 5: Car + 6: Lamp + 7: Glasses + 8: Bottle + 9: Desk + 10: Cup + 11: Street Lights + 12: Cabinet/shelf + 13: Handbag/Satchel + 14: Bracelet + 15: Plate + 16: Picture/Frame + 17: Helmet + 18: Book + 19: Gloves + 20: Storage box + 21: Boat + 22: Leather Shoes + 23: Flower + 24: Bench + 25: Potted Plant + 26: Bowl/Basin + 27: Flag + 28: Pillow + 29: Boots + 30: Vase + 31: Microphone + 32: Necklace + 33: Ring + 34: SUV + 35: Wine Glass + 36: Belt + 37: Monitor/TV + 38: Backpack + 39: Umbrella + 40: Traffic Light + 41: Speaker + 42: Watch + 43: Tie + 44: Trash bin Can + 45: Slippers + 46: Bicycle + 47: Stool + 48: Barrel/bucket + 49: Van + 50: Couch + 51: Sandals + 52: Basket + 53: Drum + 54: Pen/Pencil + 55: Bus + 56: Wild Bird + 57: High Heels + 58: Motorcycle + 59: Guitar + 60: Carpet + 61: Cell Phone + 62: Bread + 63: Camera + 64: Canned + 65: Truck + 66: Traffic cone + 67: Cymbal + 68: Lifesaver + 69: Towel + 70: Stuffed Toy + 71: Candle + 72: Sailboat + 73: Laptop + 74: Awning + 75: Bed + 76: Faucet + 77: Tent + 78: Horse + 79: Mirror + 80: Power outlet + 81: Sink + 82: Apple + 83: Air Conditioner + 84: Knife + 85: Hockey Stick + 86: Paddle + 87: Pickup Truck + 88: Fork + 89: Traffic Sign + 90: Balloon + 91: Tripod + 92: Dog + 93: Spoon + 94: Clock + 95: Pot + 96: Cow + 97: Cake + 98: Dinning Table + 99: Sheep + 100: Hanger + 101: Blackboard/Whiteboard + 102: Napkin + 103: Other Fish + 104: Orange/Tangerine + 105: Toiletry + 106: Keyboard + 107: Tomato + 108: Lantern + 109: Machinery Vehicle + 110: Fan + 111: Green Vegetables + 112: Banana + 113: Baseball Glove + 114: Airplane + 115: Mouse + 116: Train + 117: Pumpkin + 118: Soccer + 119: Skiboard + 120: Luggage + 121: Nightstand + 122: Tea pot + 123: Telephone + 124: Trolley + 125: Head Phone + 126: Sports Car + 127: Stop Sign + 128: Dessert + 129: Scooter + 130: Stroller + 131: Crane + 132: Remote + 133: Refrigerator + 134: Oven + 135: Lemon + 136: Duck + 137: Baseball Bat + 138: Surveillance Camera + 139: Cat + 140: Jug + 141: Broccoli + 142: Piano + 143: Pizza + 144: Elephant + 145: Skateboard + 146: Surfboard + 147: Gun + 148: Skating and Skiing shoes + 149: Gas stove + 150: Donut + 151: Bow Tie + 152: Carrot + 153: Toilet + 154: Kite + 155: Strawberry + 156: Other Balls + 157: Shovel + 158: Pepper + 159: Computer Box + 160: Toilet Paper + 161: Cleaning Products + 162: Chopsticks + 163: Microwave + 164: Pigeon + 165: Baseball + 166: Cutting/chopping Board + 167: Coffee Table + 168: Side Table + 169: Scissors + 170: Marker + 171: Pie + 172: Ladder + 173: Snowboard + 174: Cookies + 175: Radiator + 176: Fire Hydrant + 177: Basketball + 178: Zebra + 179: Grape + 180: Giraffe + 181: Potato + 182: Sausage + 183: Tricycle + 184: Violin + 185: Egg + 186: Fire Extinguisher + 187: Candy + 188: Fire Truck + 189: Billiards + 190: Converter + 191: Bathtub + 192: Wheelchair + 193: Golf Club + 194: Briefcase + 195: Cucumber + 196: Cigar/Cigarette + 197: Paint Brush + 198: Pear + 199: Heavy Truck + 200: Hamburger + 201: Extractor + 202: Extension Cord + 203: Tong + 204: Tennis Racket + 205: Folder + 206: American Football + 207: earphone + 208: Mask + 209: Kettle + 210: Tennis + 211: Ship + 212: Swing + 213: Coffee Machine + 214: Slide + 215: Carriage + 216: Onion + 217: Green beans + 218: Projector + 219: Frisbee + 220: Washing Machine/Drying Machine + 221: Chicken + 222: Printer + 223: Watermelon + 224: Saxophone + 225: Tissue + 226: Toothbrush + 227: Ice cream + 228: Hot-air balloon + 229: Cello + 230: French Fries + 231: Scale + 232: Trophy + 233: Cabbage + 234: Hot dog + 235: Blender + 236: Peach + 237: Rice + 238: Wallet/Purse + 239: Volleyball + 240: Deer + 241: Goose + 242: Tape + 243: Tablet + 244: Cosmetics + 245: Trumpet + 246: Pineapple + 247: Golf Ball + 248: Ambulance + 249: Parking meter + 250: Mango + 251: Key + 252: Hurdle + 253: Fishing Rod + 254: Medal + 255: Flute + 256: Brush + 257: Penguin + 258: Megaphone + 259: Corn + 260: Lettuce + 261: Garlic + 262: Swan + 263: Helicopter + 264: Green Onion + 265: Sandwich + 266: Nuts + 267: Speed Limit Sign + 268: Induction Cooker + 269: Broom + 270: Trombone + 271: Plum + 272: Rickshaw + 273: Goldfish + 274: Kiwi fruit + 275: Router/modem + 276: Poker Card + 277: Toaster + 278: Shrimp + 279: Sushi + 280: Cheese + 281: Notepaper + 282: Cherry + 283: Pliers + 284: CD + 285: Pasta + 286: Hammer + 287: Cue + 288: Avocado + 289: Hamimelon + 290: Flask + 291: Mushroom + 292: Screwdriver + 293: Soap + 294: Recorder + 295: Bear + 296: Eggplant + 297: Board Eraser + 298: Coconut + 299: Tape Measure/Ruler + 300: Pig + 301: Showerhead + 302: Globe + 303: Chips + 304: Steak + 305: Crosswalk Sign + 306: Stapler + 307: Camel + 308: Formula 1 + 309: Pomegranate + 310: Dishwasher + 311: Crab + 312: Hoverboard + 313: Meat ball + 314: Rice Cooker + 315: Tuba + 316: Calculator + 317: Papaya + 318: Antelope + 319: Parrot + 320: Seal + 321: Butterfly + 322: Dumbbell + 323: Donkey + 324: Lion + 325: Urinal + 326: Dolphin + 327: Electric Drill + 328: Hair Dryer + 329: Egg tart + 330: Jellyfish + 331: Treadmill + 332: Lighter + 333: Grapefruit + 334: Game board + 335: Mop + 336: Radish + 337: Baozi + 338: Target + 339: French + 340: Spring Rolls + 341: Monkey + 342: Rabbit + 343: Pencil Case + 344: Yak + 345: Red Cabbage + 346: Binoculars + 347: Asparagus + 348: Barbell + 349: Scallop + 350: Noddles + 351: Comb + 352: Dumpling + 353: Oyster + 354: Table Tennis paddle + 355: Cosmetics Brush/Eyeliner Pencil + 356: Chainsaw + 357: Eraser + 358: Lobster + 359: Durian + 360: Okra + 361: Lipstick + 362: Cosmetics Mirror + 363: Curling + 364: Table Tennis + +# Download script/URL (optional) --------------------------------------------------------------------------------------- +download: | + from tqdm import tqdm + + from ultralytics.utils.checks import check_requirements + from ultralytics.utils.downloads import download + from ultralytics.utils.ops import xyxy2xywhn + + import numpy as np + from pathlib import Path + + check_requirements(('pycocotools>=2.0',)) + from pycocotools.coco import COCO + + # Make Directories + dir = Path(yaml['path']) # dataset root dir + for p in 'images', 'labels': + (dir / p).mkdir(parents=True, exist_ok=True) + for q in 'train', 'val': + (dir / p / q).mkdir(parents=True, exist_ok=True) + + # Train, Val Splits + for split, patches in [('train', 50 + 1), ('val', 43 + 1)]: + print(f"Processing {split} in {patches} patches ...") + images, labels = dir / 'images' / split, dir / 'labels' / split + + # Download + url = f"https://dorc.ks3-cn-beijing.ksyun.com/data-set/2020Objects365%E6%95%B0%E6%8D%AE%E9%9B%86/{split}/" + if split == 'train': + download([f'{url}zhiyuan_objv2_{split}.tar.gz'], dir=dir) # annotations json + download([f'{url}patch{i}.tar.gz' for i in range(patches)], dir=images, curl=True, threads=8) + elif split == 'val': + download([f'{url}zhiyuan_objv2_{split}.json'], dir=dir) # annotations json + download([f'{url}images/v1/patch{i}.tar.gz' for i in range(15 + 1)], dir=images, curl=True, threads=8) + download([f'{url}images/v2/patch{i}.tar.gz' for i in range(16, patches)], dir=images, curl=True, threads=8) + + # Move + for f in tqdm(images.rglob('*.jpg'), desc=f'Moving {split} images'): + f.rename(images / f.name) # move to /images/{split} + + # Labels + coco = COCO(dir / f'zhiyuan_objv2_{split}.json') + names = [x["name"] for x in coco.loadCats(coco.getCatIds())] + for cid, cat in enumerate(names): + catIds = coco.getCatIds(catNms=[cat]) + imgIds = coco.getImgIds(catIds=catIds) + for im in tqdm(coco.loadImgs(imgIds), desc=f'Class {cid + 1}/{len(names)} {cat}'): + width, height = im["width"], im["height"] + path = Path(im["file_name"]) # image filename + try: + with open(labels / path.with_suffix('.txt').name, 'a') as file: + annIds = coco.getAnnIds(imgIds=im["id"], catIds=catIds, iscrowd=None) + for a in coco.loadAnns(annIds): + x, y, w, h = a['bbox'] # bounding box in xywh (xy top-left corner) + xyxy = np.array([x, y, x + w, y + h])[None] # pixels(1,4) + x, y, w, h = xyxy2xywhn(xyxy, w=width, h=height, clip=True)[0] # normalized and clipped + file.write(f"{cid} {x:.5f} {y:.5f} {w:.5f} {h:.5f}\n") + except Exception as e: + print(e) diff --git a/ultralytics/cfg/datasets/SKU-110K.yaml b/ultralytics/cfg/datasets/SKU-110K.yaml new file mode 100644 index 0000000..fff1baa --- /dev/null +++ b/ultralytics/cfg/datasets/SKU-110K.yaml @@ -0,0 +1,57 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# SKU-110K retail items dataset https://github.com/eg4000/SKU110K_CVPR19 by Trax Retail +# Documentation: https://docs.ultralytics.com/datasets/detect/sku-110k/ +# Example usage: yolo train data=SKU-110K.yaml +# parent +# ├── ultralytics +# └── datasets +# └── SKU-110K ← downloads here (13.6 GB) + +# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..] +path: ../datasets/SKU-110K # dataset root dir +train: train.txt # train images (relative to 'path') 8219 images +val: val.txt # val images (relative to 'path') 588 images +test: test.txt # test images (optional) 2936 images + +# Classes +names: + 0: object + +# Download script/URL (optional) --------------------------------------------------------------------------------------- +download: | + import shutil + from pathlib import Path + + import numpy as np + import pandas as pd + from tqdm import tqdm + + from ultralytics.utils.downloads import download + from ultralytics.utils.ops import xyxy2xywh + + # Download + dir = Path(yaml['path']) # dataset root dir + parent = Path(dir.parent) # download dir + urls = ['http://trax-geometry.s3.amazonaws.com/cvpr_challenge/SKU110K_fixed.tar.gz'] + download(urls, dir=parent) + + # Rename directories + if dir.exists(): + shutil.rmtree(dir) + (parent / 'SKU110K_fixed').rename(dir) # rename dir + (dir / 'labels').mkdir(parents=True, exist_ok=True) # create labels dir + + # Convert labels + names = 'image', 'x1', 'y1', 'x2', 'y2', 'class', 'image_width', 'image_height' # column names + for d in 'annotations_train.csv', 'annotations_val.csv', 'annotations_test.csv': + x = pd.read_csv(dir / 'annotations' / d, names=names).values # annotations + images, unique_images = x[:, 0], np.unique(x[:, 0]) + with open((dir / d).with_suffix('.txt').__str__().replace('annotations_', ''), 'w') as f: + f.writelines(f'./images/{s}\n' for s in unique_images) + for im in tqdm(unique_images, desc=f'Converting {dir / d}'): + cls = 0 # single-class dataset + with open((dir / 'labels' / im).with_suffix('.txt'), 'a') as f: + for r in x[images == im]: + w, h = r[6], r[7] # image width, height + xywh = xyxy2xywh(np.array([[r[1] / w, r[2] / h, r[3] / w, r[4] / h]]))[0] # instance + f.write(f"{cls} {xywh[0]:.5f} {xywh[1]:.5f} {xywh[2]:.5f} {xywh[3]:.5f}\n") # write label diff --git a/ultralytics/cfg/datasets/VOC.yaml b/ultralytics/cfg/datasets/VOC.yaml new file mode 100644 index 0000000..cd6d5ad --- /dev/null +++ b/ultralytics/cfg/datasets/VOC.yaml @@ -0,0 +1,99 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# PASCAL VOC dataset http://host.robots.ox.ac.uk/pascal/VOC by University of Oxford +# Documentation: # Documentation: https://docs.ultralytics.com/datasets/detect/voc/ +# Example usage: yolo train data=VOC.yaml +# parent +# ├── ultralytics +# └── datasets +# └── VOC ← downloads here (2.8 GB) + +# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..] +path: ../datasets/VOC +train: # train images (relative to 'path') 16551 images + - images/train2012 + - images/train2007 + - images/val2012 + - images/val2007 +val: # val images (relative to 'path') 4952 images + - images/test2007 +test: # test images (optional) + - images/test2007 + +# Classes +names: + 0: aeroplane + 1: bicycle + 2: bird + 3: boat + 4: bottle + 5: bus + 6: car + 7: cat + 8: chair + 9: cow + 10: diningtable + 11: dog + 12: horse + 13: motorbike + 14: person + 15: pottedplant + 16: sheep + 17: sofa + 18: train + 19: tvmonitor + +# Download script/URL (optional) --------------------------------------------------------------------------------------- +download: | + import xml.etree.ElementTree as ET + + from tqdm import tqdm + from ultralytics.utils.downloads import download + from pathlib import Path + + def convert_label(path, lb_path, year, image_id): + def convert_box(size, box): + dw, dh = 1. / size[0], 1. / size[1] + x, y, w, h = (box[0] + box[1]) / 2.0 - 1, (box[2] + box[3]) / 2.0 - 1, box[1] - box[0], box[3] - box[2] + return x * dw, y * dh, w * dw, h * dh + + in_file = open(path / f'VOC{year}/Annotations/{image_id}.xml') + out_file = open(lb_path, 'w') + tree = ET.parse(in_file) + root = tree.getroot() + size = root.find('size') + w = int(size.find('width').text) + h = int(size.find('height').text) + + names = list(yaml['names'].values()) # names list + for obj in root.iter('object'): + cls = obj.find('name').text + if cls in names and int(obj.find('difficult').text) != 1: + xmlbox = obj.find('bndbox') + bb = convert_box((w, h), [float(xmlbox.find(x).text) for x in ('xmin', 'xmax', 'ymin', 'ymax')]) + cls_id = names.index(cls) # class id + out_file.write(" ".join(str(a) for a in (cls_id, *bb)) + '\n') + + + # Download + dir = Path(yaml['path']) # dataset root dir + url = 'https://github.com/ultralytics/yolov5/releases/download/v1.0/' + urls = [f'{url}VOCtrainval_06-Nov-2007.zip', # 446MB, 5012 images + f'{url}VOCtest_06-Nov-2007.zip', # 438MB, 4953 images + f'{url}VOCtrainval_11-May-2012.zip'] # 1.95GB, 17126 images + download(urls, dir=dir / 'images', curl=True, threads=3, exist_ok=True) # download and unzip over existing paths (required) + + # Convert + path = dir / 'images/VOCdevkit' + for year, image_set in ('2012', 'train'), ('2012', 'val'), ('2007', 'train'), ('2007', 'val'), ('2007', 'test'): + imgs_path = dir / 'images' / f'{image_set}{year}' + lbs_path = dir / 'labels' / f'{image_set}{year}' + imgs_path.mkdir(exist_ok=True, parents=True) + lbs_path.mkdir(exist_ok=True, parents=True) + + with open(path / f'VOC{year}/ImageSets/Main/{image_set}.txt') as f: + image_ids = f.read().strip().split() + for id in tqdm(image_ids, desc=f'{image_set}{year}'): + f = path / f'VOC{year}/JPEGImages/{id}.jpg' # old img path + lb_path = (lbs_path / f.name).with_suffix('.txt') # new label path + f.rename(imgs_path / f.name) # move image + convert_label(path, lb_path, year, id) # convert labels to YOLO format diff --git a/ultralytics/cfg/datasets/VisDrone.yaml b/ultralytics/cfg/datasets/VisDrone.yaml new file mode 100644 index 0000000..773f0b0 --- /dev/null +++ b/ultralytics/cfg/datasets/VisDrone.yaml @@ -0,0 +1,72 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# VisDrone2019-DET dataset https://github.com/VisDrone/VisDrone-Dataset by Tianjin University +# Documentation: https://docs.ultralytics.com/datasets/detect/visdrone/ +# Example usage: yolo train data=VisDrone.yaml +# parent +# ├── ultralytics +# └── datasets +# └── VisDrone ← downloads here (2.3 GB) + +# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..] +path: ../datasets/VisDrone # dataset root dir +train: VisDrone2019-DET-train/images # train images (relative to 'path') 6471 images +val: VisDrone2019-DET-val/images # val images (relative to 'path') 548 images +test: VisDrone2019-DET-test-dev/images # test images (optional) 1610 images + +# Classes +names: + 0: pedestrian + 1: people + 2: bicycle + 3: car + 4: van + 5: truck + 6: tricycle + 7: awning-tricycle + 8: bus + 9: motor + +# Download script/URL (optional) --------------------------------------------------------------------------------------- +download: | + import os + from pathlib import Path + + from ultralytics.utils.downloads import download + + def visdrone2yolo(dir): + from PIL import Image + from tqdm import tqdm + + def convert_box(size, box): + # Convert VisDrone box to YOLO xywh box + dw = 1. / size[0] + dh = 1. / size[1] + return (box[0] + box[2] / 2) * dw, (box[1] + box[3] / 2) * dh, box[2] * dw, box[3] * dh + + (dir / 'labels').mkdir(parents=True, exist_ok=True) # make labels directory + pbar = tqdm((dir / 'annotations').glob('*.txt'), desc=f'Converting {dir}') + for f in pbar: + img_size = Image.open((dir / 'images' / f.name).with_suffix('.jpg')).size + lines = [] + with open(f, 'r') as file: # read annotation.txt + for row in [x.split(',') for x in file.read().strip().splitlines()]: + if row[4] == '0': # VisDrone 'ignored regions' class 0 + continue + cls = int(row[5]) - 1 + box = convert_box(img_size, tuple(map(int, row[:4]))) + lines.append(f"{cls} {' '.join(f'{x:.6f}' for x in box)}\n") + with open(str(f).replace(f'{os.sep}annotations{os.sep}', f'{os.sep}labels{os.sep}'), 'w') as fl: + fl.writelines(lines) # write label.txt + + + # Download + dir = Path(yaml['path']) # dataset root dir + urls = ['https://github.com/ultralytics/yolov5/releases/download/v1.0/VisDrone2019-DET-train.zip', + 'https://github.com/ultralytics/yolov5/releases/download/v1.0/VisDrone2019-DET-val.zip', + 'https://github.com/ultralytics/yolov5/releases/download/v1.0/VisDrone2019-DET-test-dev.zip', + 'https://github.com/ultralytics/yolov5/releases/download/v1.0/VisDrone2019-DET-test-challenge.zip'] + download(urls, dir=dir, curl=True, threads=4) + + # Convert + for d in 'VisDrone2019-DET-train', 'VisDrone2019-DET-val', 'VisDrone2019-DET-test-dev': + visdrone2yolo(dir / d) # convert VisDrone annotations to YOLO labels diff --git a/ultralytics/cfg/datasets/african-wildlife.yaml b/ultralytics/cfg/datasets/african-wildlife.yaml new file mode 100644 index 0000000..af8af36 --- /dev/null +++ b/ultralytics/cfg/datasets/african-wildlife.yaml @@ -0,0 +1,24 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# African-wildlife dataset by Ultralytics +# Documentation: https://docs.ultralytics.com/datasets/detect/african-wildlife/ +# Example usage: yolo train data=african-wildlife.yaml +# parent +# ├── ultralytics +# └── datasets +# └── african-wildlife ← downloads here (100 MB) + +# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..] +path: ../datasets/african-wildlife # dataset root dir +train: train/images # train images (relative to 'path') 1052 images +val: valid/images # val images (relative to 'path') 225 images +test: test/images # test images (relative to 'path') 227 images + +# Classes +names: + 0: buffalo + 1: elephant + 2: rhino + 3: zebra + +# Download script/URL (optional) +download: https://ultralytics.com/assets/african-wildlife.zip diff --git a/ultralytics/cfg/datasets/brain-tumor.yaml b/ultralytics/cfg/datasets/brain-tumor.yaml new file mode 100644 index 0000000..be61098 --- /dev/null +++ b/ultralytics/cfg/datasets/brain-tumor.yaml @@ -0,0 +1,22 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# Brain-tumor dataset by Ultralytics +# Documentation: https://docs.ultralytics.com/datasets/detect/brain-tumor/ +# Example usage: yolo train data=brain-tumor.yaml +# parent +# ├── ultralytics +# └── datasets +# └── brain-tumor ← downloads here (4.05 MB) + +# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..] +path: ../datasets/brain-tumor # dataset root dir +train: train/images # train images (relative to 'path') 893 images +val: valid/images # val images (relative to 'path') 223 images +test: # test images (relative to 'path') + +# Classes +names: + 0: negative + 1: positive + +# Download script/URL (optional) +download: https://ultralytics.com/assets/brain-tumor.zip diff --git a/ultralytics/cfg/datasets/carparts-seg.yaml b/ultralytics/cfg/datasets/carparts-seg.yaml new file mode 100644 index 0000000..a1c25ba --- /dev/null +++ b/ultralytics/cfg/datasets/carparts-seg.yaml @@ -0,0 +1,43 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# Carparts-seg dataset by Ultralytics +# Documentation: https://docs.ultralytics.com/datasets/segment/carparts-seg/ +# Example usage: yolo train data=carparts-seg.yaml +# parent +# ├── ultralytics +# └── datasets +# └── carparts-seg ← downloads here (132 MB) + +# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..] +path: ../datasets/carparts-seg # dataset root dir +train: train/images # train images (relative to 'path') 3516 images +val: valid/images # val images (relative to 'path') 276 images +test: test/images # test images (relative to 'path') 401 images + +# Classes +names: + 0: back_bumper + 1: back_door + 2: back_glass + 3: back_left_door + 4: back_left_light + 5: back_light + 6: back_right_door + 7: back_right_light + 8: front_bumper + 9: front_door + 10: front_glass + 11: front_left_door + 12: front_left_light + 13: front_light + 14: front_right_door + 15: front_right_light + 16: hood + 17: left_mirror + 18: object + 19: right_mirror + 20: tailgate + 21: trunk + 22: wheel + +# Download script/URL (optional) +download: https://ultralytics.com/assets/carparts-seg.zip diff --git a/ultralytics/cfg/datasets/coco-pose.yaml b/ultralytics/cfg/datasets/coco-pose.yaml new file mode 100644 index 0000000..b50b7a5 --- /dev/null +++ b/ultralytics/cfg/datasets/coco-pose.yaml @@ -0,0 +1,38 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# COCO 2017 dataset https://cocodataset.org by Microsoft +# Documentation: https://docs.ultralytics.com/datasets/pose/coco/ +# Example usage: yolo train data=coco-pose.yaml +# parent +# ├── ultralytics +# └── datasets +# └── coco-pose ← downloads here (20.1 GB) + +# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..] +path: ../datasets/coco-pose # dataset root dir +train: train2017.txt # train images (relative to 'path') 118287 images +val: val2017.txt # val images (relative to 'path') 5000 images +test: test-dev2017.txt # 20288 of 40670 images, submit to https://competitions.codalab.org/competitions/20794 + +# Keypoints +kpt_shape: [17, 3] # number of keypoints, number of dims (2 for x,y or 3 for x,y,visible) +flip_idx: [0, 2, 1, 4, 3, 6, 5, 8, 7, 10, 9, 12, 11, 14, 13, 16, 15] + +# Classes +names: + 0: person + +# Download script/URL (optional) +download: | + from ultralytics.utils.downloads import download + from pathlib import Path + + # Download labels + dir = Path(yaml['path']) # dataset root dir + url = 'https://github.com/ultralytics/yolov5/releases/download/v1.0/' + urls = [url + 'coco2017labels-pose.zip'] # labels + download(urls, dir=dir.parent) + # Download data + urls = ['http://images.cocodataset.org/zips/train2017.zip', # 19G, 118k images + 'http://images.cocodataset.org/zips/val2017.zip', # 1G, 5k images + 'http://images.cocodataset.org/zips/test2017.zip'] # 7G, 41k images (optional) + download(urls, dir=dir / 'images', threads=3) diff --git a/ultralytics/cfg/datasets/coco.yaml b/ultralytics/cfg/datasets/coco.yaml new file mode 100644 index 0000000..d0297f7 --- /dev/null +++ b/ultralytics/cfg/datasets/coco.yaml @@ -0,0 +1,114 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# COCO 2017 dataset https://cocodataset.org by Microsoft +# Documentation: https://docs.ultralytics.com/datasets/detect/coco/ +# Example usage: yolo train data=coco.yaml +# parent +# ├── ultralytics +# └── datasets +# └── coco ← downloads here (20.1 GB) + +# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..] +path: ../datasets/coco # dataset root dir +train: train2017.txt # train images (relative to 'path') 118287 images +val: val2017.txt # val images (relative to 'path') 5000 images +test: test-dev2017.txt # 20288 of 40670 images, submit to https://competitions.codalab.org/competitions/20794 + +# Classes +names: + 0: person + 1: bicycle + 2: car + 3: motorcycle + 4: airplane + 5: bus + 6: train + 7: truck + 8: boat + 9: traffic light + 10: fire hydrant + 11: stop sign + 12: parking meter + 13: bench + 14: bird + 15: cat + 16: dog + 17: horse + 18: sheep + 19: cow + 20: elephant + 21: bear + 22: zebra + 23: giraffe + 24: backpack + 25: umbrella + 26: handbag + 27: tie + 28: suitcase + 29: frisbee + 30: skis + 31: snowboard + 32: sports ball + 33: kite + 34: baseball bat + 35: baseball glove + 36: skateboard + 37: surfboard + 38: tennis racket + 39: bottle + 40: wine glass + 41: cup + 42: fork + 43: knife + 44: spoon + 45: bowl + 46: banana + 47: apple + 48: sandwich + 49: orange + 50: broccoli + 51: carrot + 52: hot dog + 53: pizza + 54: donut + 55: cake + 56: chair + 57: couch + 58: potted plant + 59: bed + 60: dining table + 61: toilet + 62: tv + 63: laptop + 64: mouse + 65: remote + 66: keyboard + 67: cell phone + 68: microwave + 69: oven + 70: toaster + 71: sink + 72: refrigerator + 73: book + 74: clock + 75: vase + 76: scissors + 77: teddy bear + 78: hair drier + 79: toothbrush + +# Download script/URL (optional) +download: | + from ultralytics.utils.downloads import download + from pathlib import Path + + # Download labels + segments = True # segment or box labels + dir = Path(yaml['path']) # dataset root dir + url = 'https://github.com/ultralytics/yolov5/releases/download/v1.0/' + urls = [url + ('coco2017labels-segments.zip' if segments else 'coco2017labels.zip')] # labels + download(urls, dir=dir.parent) + # Download data + urls = ['http://images.cocodataset.org/zips/train2017.zip', # 19G, 118k images + 'http://images.cocodataset.org/zips/val2017.zip', # 1G, 5k images + 'http://images.cocodataset.org/zips/test2017.zip'] # 7G, 41k images (optional) + download(urls, dir=dir / 'images', threads=3) diff --git a/ultralytics/cfg/datasets/coco128-seg.yaml b/ultralytics/cfg/datasets/coco128-seg.yaml new file mode 100644 index 0000000..e898a40 --- /dev/null +++ b/ultralytics/cfg/datasets/coco128-seg.yaml @@ -0,0 +1,100 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# COCO128-seg dataset https://www.kaggle.com/ultralytics/coco128 (first 128 images from COCO train2017) by Ultralytics +# Documentation: https://docs.ultralytics.com/datasets/segment/coco/ +# Example usage: yolo train data=coco128.yaml +# parent +# ├── ultralytics +# └── datasets +# └── coco128-seg ← downloads here (7 MB) + +# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..] +path: ../datasets/coco128-seg # dataset root dir +train: images/train2017 # train images (relative to 'path') 128 images +val: images/train2017 # val images (relative to 'path') 128 images +test: # test images (optional) + +# Classes +names: + 0: person + 1: bicycle + 2: car + 3: motorcycle + 4: airplane + 5: bus + 6: train + 7: truck + 8: boat + 9: traffic light + 10: fire hydrant + 11: stop sign + 12: parking meter + 13: bench + 14: bird + 15: cat + 16: dog + 17: horse + 18: sheep + 19: cow + 20: elephant + 21: bear + 22: zebra + 23: giraffe + 24: backpack + 25: umbrella + 26: handbag + 27: tie + 28: suitcase + 29: frisbee + 30: skis + 31: snowboard + 32: sports ball + 33: kite + 34: baseball bat + 35: baseball glove + 36: skateboard + 37: surfboard + 38: tennis racket + 39: bottle + 40: wine glass + 41: cup + 42: fork + 43: knife + 44: spoon + 45: bowl + 46: banana + 47: apple + 48: sandwich + 49: orange + 50: broccoli + 51: carrot + 52: hot dog + 53: pizza + 54: donut + 55: cake + 56: chair + 57: couch + 58: potted plant + 59: bed + 60: dining table + 61: toilet + 62: tv + 63: laptop + 64: mouse + 65: remote + 66: keyboard + 67: cell phone + 68: microwave + 69: oven + 70: toaster + 71: sink + 72: refrigerator + 73: book + 74: clock + 75: vase + 76: scissors + 77: teddy bear + 78: hair drier + 79: toothbrush + +# Download script/URL (optional) +download: https://ultralytics.com/assets/coco128-seg.zip diff --git a/ultralytics/cfg/datasets/coco128.yaml b/ultralytics/cfg/datasets/coco128.yaml new file mode 100644 index 0000000..8d47ee0 --- /dev/null +++ b/ultralytics/cfg/datasets/coco128.yaml @@ -0,0 +1,100 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# COCO128 dataset https://www.kaggle.com/ultralytics/coco128 (first 128 images from COCO train2017) by Ultralytics +# Documentation: https://docs.ultralytics.com/datasets/detect/coco/ +# Example usage: yolo train data=coco128.yaml +# parent +# ├── ultralytics +# └── datasets +# └── coco128 ← downloads here (7 MB) + +# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..] +path: ../datasets/coco128 # dataset root dir +train: images/train2017 # train images (relative to 'path') 128 images +val: images/train2017 # val images (relative to 'path') 128 images +test: # test images (optional) + +# Classes +names: + 0: person + 1: bicycle + 2: car + 3: motorcycle + 4: airplane + 5: bus + 6: train + 7: truck + 8: boat + 9: traffic light + 10: fire hydrant + 11: stop sign + 12: parking meter + 13: bench + 14: bird + 15: cat + 16: dog + 17: horse + 18: sheep + 19: cow + 20: elephant + 21: bear + 22: zebra + 23: giraffe + 24: backpack + 25: umbrella + 26: handbag + 27: tie + 28: suitcase + 29: frisbee + 30: skis + 31: snowboard + 32: sports ball + 33: kite + 34: baseball bat + 35: baseball glove + 36: skateboard + 37: surfboard + 38: tennis racket + 39: bottle + 40: wine glass + 41: cup + 42: fork + 43: knife + 44: spoon + 45: bowl + 46: banana + 47: apple + 48: sandwich + 49: orange + 50: broccoli + 51: carrot + 52: hot dog + 53: pizza + 54: donut + 55: cake + 56: chair + 57: couch + 58: potted plant + 59: bed + 60: dining table + 61: toilet + 62: tv + 63: laptop + 64: mouse + 65: remote + 66: keyboard + 67: cell phone + 68: microwave + 69: oven + 70: toaster + 71: sink + 72: refrigerator + 73: book + 74: clock + 75: vase + 76: scissors + 77: teddy bear + 78: hair drier + 79: toothbrush + +# Download script/URL (optional) +download: https://ultralytics.com/assets/coco128.zip diff --git a/ultralytics/cfg/datasets/coco8-pose.yaml b/ultralytics/cfg/datasets/coco8-pose.yaml new file mode 100644 index 0000000..4dee5be --- /dev/null +++ b/ultralytics/cfg/datasets/coco8-pose.yaml @@ -0,0 +1,25 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# COCO8-pose dataset (first 8 images from COCO train2017) by Ultralytics +# Documentation: https://docs.ultralytics.com/datasets/pose/coco8-pose/ +# Example usage: yolo train data=coco8-pose.yaml +# parent +# ├── ultralytics +# └── datasets +# └── coco8-pose ← downloads here (1 MB) + +# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..] +path: ../datasets/coco8-pose # dataset root dir +train: images/train # train images (relative to 'path') 4 images +val: images/val # val images (relative to 'path') 4 images +test: # test images (optional) + +# Keypoints +kpt_shape: [17, 3] # number of keypoints, number of dims (2 for x,y or 3 for x,y,visible) +flip_idx: [0, 2, 1, 4, 3, 6, 5, 8, 7, 10, 9, 12, 11, 14, 13, 16, 15] + +# Classes +names: + 0: person + +# Download script/URL (optional) +download: https://ultralytics.com/assets/coco8-pose.zip diff --git a/ultralytics/cfg/datasets/coco8-seg.yaml b/ultralytics/cfg/datasets/coco8-seg.yaml new file mode 100644 index 0000000..d8b6ed2 --- /dev/null +++ b/ultralytics/cfg/datasets/coco8-seg.yaml @@ -0,0 +1,100 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# COCO8-seg dataset (first 8 images from COCO train2017) by Ultralytics +# Documentation: https://docs.ultralytics.com/datasets/segment/coco8-seg/ +# Example usage: yolo train data=coco8-seg.yaml +# parent +# ├── ultralytics +# └── datasets +# └── coco8-seg ← downloads here (1 MB) + +# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..] +path: ../datasets/coco8-seg # dataset root dir +train: images/train # train images (relative to 'path') 4 images +val: images/val # val images (relative to 'path') 4 images +test: # test images (optional) + +# Classes +names: + 0: person + 1: bicycle + 2: car + 3: motorcycle + 4: airplane + 5: bus + 6: train + 7: truck + 8: boat + 9: traffic light + 10: fire hydrant + 11: stop sign + 12: parking meter + 13: bench + 14: bird + 15: cat + 16: dog + 17: horse + 18: sheep + 19: cow + 20: elephant + 21: bear + 22: zebra + 23: giraffe + 24: backpack + 25: umbrella + 26: handbag + 27: tie + 28: suitcase + 29: frisbee + 30: skis + 31: snowboard + 32: sports ball + 33: kite + 34: baseball bat + 35: baseball glove + 36: skateboard + 37: surfboard + 38: tennis racket + 39: bottle + 40: wine glass + 41: cup + 42: fork + 43: knife + 44: spoon + 45: bowl + 46: banana + 47: apple + 48: sandwich + 49: orange + 50: broccoli + 51: carrot + 52: hot dog + 53: pizza + 54: donut + 55: cake + 56: chair + 57: couch + 58: potted plant + 59: bed + 60: dining table + 61: toilet + 62: tv + 63: laptop + 64: mouse + 65: remote + 66: keyboard + 67: cell phone + 68: microwave + 69: oven + 70: toaster + 71: sink + 72: refrigerator + 73: book + 74: clock + 75: vase + 76: scissors + 77: teddy bear + 78: hair drier + 79: toothbrush + +# Download script/URL (optional) +download: https://ultralytics.com/assets/coco8-seg.zip diff --git a/ultralytics/cfg/datasets/coco8.yaml b/ultralytics/cfg/datasets/coco8.yaml new file mode 100644 index 0000000..2925f81 --- /dev/null +++ b/ultralytics/cfg/datasets/coco8.yaml @@ -0,0 +1,100 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# COCO8 dataset (first 8 images from COCO train2017) by Ultralytics +# Documentation: https://docs.ultralytics.com/datasets/detect/coco8/ +# Example usage: yolo train data=coco8.yaml +# parent +# ├── ultralytics +# └── datasets +# └── coco8 ← downloads here (1 MB) + +# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..] +path: ../datasets/coco8 # dataset root dir +train: images/train # train images (relative to 'path') 4 images +val: images/val # val images (relative to 'path') 4 images +test: # test images (optional) + +# Classes +names: + 0: person + 1: bicycle + 2: car + 3: motorcycle + 4: airplane + 5: bus + 6: train + 7: truck + 8: boat + 9: traffic light + 10: fire hydrant + 11: stop sign + 12: parking meter + 13: bench + 14: bird + 15: cat + 16: dog + 17: horse + 18: sheep + 19: cow + 20: elephant + 21: bear + 22: zebra + 23: giraffe + 24: backpack + 25: umbrella + 26: handbag + 27: tie + 28: suitcase + 29: frisbee + 30: skis + 31: snowboard + 32: sports ball + 33: kite + 34: baseball bat + 35: baseball glove + 36: skateboard + 37: surfboard + 38: tennis racket + 39: bottle + 40: wine glass + 41: cup + 42: fork + 43: knife + 44: spoon + 45: bowl + 46: banana + 47: apple + 48: sandwich + 49: orange + 50: broccoli + 51: carrot + 52: hot dog + 53: pizza + 54: donut + 55: cake + 56: chair + 57: couch + 58: potted plant + 59: bed + 60: dining table + 61: toilet + 62: tv + 63: laptop + 64: mouse + 65: remote + 66: keyboard + 67: cell phone + 68: microwave + 69: oven + 70: toaster + 71: sink + 72: refrigerator + 73: book + 74: clock + 75: vase + 76: scissors + 77: teddy bear + 78: hair drier + 79: toothbrush + +# Download script/URL (optional) +download: https://ultralytics.com/assets/coco8.zip diff --git a/ultralytics/cfg/datasets/crack-seg.yaml b/ultralytics/cfg/datasets/crack-seg.yaml new file mode 100644 index 0000000..2054f62 --- /dev/null +++ b/ultralytics/cfg/datasets/crack-seg.yaml @@ -0,0 +1,21 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# Crack-seg dataset by Ultralytics +# Documentation: https://docs.ultralytics.com/datasets/segment/crack-seg/ +# Example usage: yolo train data=crack-seg.yaml +# parent +# ├── ultralytics +# └── datasets +# └── crack-seg ← downloads here (91.2 MB) + +# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..] +path: ../datasets/crack-seg # dataset root dir +train: train/images # train images (relative to 'path') 3717 images +val: valid/images # val images (relative to 'path') 112 images +test: test/images # test images (relative to 'path') 200 images + +# Classes +names: + 0: crack + +# Download script/URL (optional) +download: https://ultralytics.com/assets/crack-seg.zip diff --git a/ultralytics/cfg/datasets/dota8.yaml b/ultralytics/cfg/datasets/dota8.yaml new file mode 100644 index 0000000..f58b501 --- /dev/null +++ b/ultralytics/cfg/datasets/dota8.yaml @@ -0,0 +1,34 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# DOTA8 dataset 8 images from split DOTAv1 dataset by Ultralytics +# Documentation: https://docs.ultralytics.com/datasets/obb/dota8/ +# Example usage: yolo train model=yolov8n-obb.pt data=dota8.yaml +# parent +# ├── ultralytics +# └── datasets +# └── dota8 ← downloads here (1MB) + +# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..] +path: ../datasets/dota8 # dataset root dir +train: images/train # train images (relative to 'path') 4 images +val: images/val # val images (relative to 'path') 4 images + +# Classes for DOTA 1.0 +names: + 0: plane + 1: ship + 2: storage tank + 3: baseball diamond + 4: tennis court + 5: basketball court + 6: ground track field + 7: harbor + 8: bridge + 9: large vehicle + 10: small vehicle + 11: helicopter + 12: roundabout + 13: soccer ball field + 14: swimming pool + +# Download script/URL (optional) +download: https://github.com/ultralytics/yolov5/releases/download/v1.0/dota8.zip diff --git a/ultralytics/cfg/datasets/open-images-v7.yaml b/ultralytics/cfg/datasets/open-images-v7.yaml new file mode 100644 index 0000000..d9cad9f --- /dev/null +++ b/ultralytics/cfg/datasets/open-images-v7.yaml @@ -0,0 +1,660 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# Open Images v7 dataset https://storage.googleapis.com/openimages/web/index.html by Google +# Documentation: https://docs.ultralytics.com/datasets/detect/open-images-v7/ +# Example usage: yolo train data=open-images-v7.yaml +# parent +# ├── ultralytics +# └── datasets +# └── open-images-v7 ← downloads here (561 GB) + +# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..] +path: ../datasets/open-images-v7 # dataset root dir +train: images/train # train images (relative to 'path') 1743042 images +val: images/val # val images (relative to 'path') 41620 images +test: # test images (optional) + +# Classes +names: + 0: Accordion + 1: Adhesive tape + 2: Aircraft + 3: Airplane + 4: Alarm clock + 5: Alpaca + 6: Ambulance + 7: Animal + 8: Ant + 9: Antelope + 10: Apple + 11: Armadillo + 12: Artichoke + 13: Auto part + 14: Axe + 15: Backpack + 16: Bagel + 17: Baked goods + 18: Balance beam + 19: Ball + 20: Balloon + 21: Banana + 22: Band-aid + 23: Banjo + 24: Barge + 25: Barrel + 26: Baseball bat + 27: Baseball glove + 28: Bat (Animal) + 29: Bathroom accessory + 30: Bathroom cabinet + 31: Bathtub + 32: Beaker + 33: Bear + 34: Bed + 35: Bee + 36: Beehive + 37: Beer + 38: Beetle + 39: Bell pepper + 40: Belt + 41: Bench + 42: Bicycle + 43: Bicycle helmet + 44: Bicycle wheel + 45: Bidet + 46: Billboard + 47: Billiard table + 48: Binoculars + 49: Bird + 50: Blender + 51: Blue jay + 52: Boat + 53: Bomb + 54: Book + 55: Bookcase + 56: Boot + 57: Bottle + 58: Bottle opener + 59: Bow and arrow + 60: Bowl + 61: Bowling equipment + 62: Box + 63: Boy + 64: Brassiere + 65: Bread + 66: Briefcase + 67: Broccoli + 68: Bronze sculpture + 69: Brown bear + 70: Building + 71: Bull + 72: Burrito + 73: Bus + 74: Bust + 75: Butterfly + 76: Cabbage + 77: Cabinetry + 78: Cake + 79: Cake stand + 80: Calculator + 81: Camel + 82: Camera + 83: Can opener + 84: Canary + 85: Candle + 86: Candy + 87: Cannon + 88: Canoe + 89: Cantaloupe + 90: Car + 91: Carnivore + 92: Carrot + 93: Cart + 94: Cassette deck + 95: Castle + 96: Cat + 97: Cat furniture + 98: Caterpillar + 99: Cattle + 100: Ceiling fan + 101: Cello + 102: Centipede + 103: Chainsaw + 104: Chair + 105: Cheese + 106: Cheetah + 107: Chest of drawers + 108: Chicken + 109: Chime + 110: Chisel + 111: Chopsticks + 112: Christmas tree + 113: Clock + 114: Closet + 115: Clothing + 116: Coat + 117: Cocktail + 118: Cocktail shaker + 119: Coconut + 120: Coffee + 121: Coffee cup + 122: Coffee table + 123: Coffeemaker + 124: Coin + 125: Common fig + 126: Common sunflower + 127: Computer keyboard + 128: Computer monitor + 129: Computer mouse + 130: Container + 131: Convenience store + 132: Cookie + 133: Cooking spray + 134: Corded phone + 135: Cosmetics + 136: Couch + 137: Countertop + 138: Cowboy hat + 139: Crab + 140: Cream + 141: Cricket ball + 142: Crocodile + 143: Croissant + 144: Crown + 145: Crutch + 146: Cucumber + 147: Cupboard + 148: Curtain + 149: Cutting board + 150: Dagger + 151: Dairy Product + 152: Deer + 153: Desk + 154: Dessert + 155: Diaper + 156: Dice + 157: Digital clock + 158: Dinosaur + 159: Dishwasher + 160: Dog + 161: Dog bed + 162: Doll + 163: Dolphin + 164: Door + 165: Door handle + 166: Doughnut + 167: Dragonfly + 168: Drawer + 169: Dress + 170: Drill (Tool) + 171: Drink + 172: Drinking straw + 173: Drum + 174: Duck + 175: Dumbbell + 176: Eagle + 177: Earrings + 178: Egg (Food) + 179: Elephant + 180: Envelope + 181: Eraser + 182: Face powder + 183: Facial tissue holder + 184: Falcon + 185: Fashion accessory + 186: Fast food + 187: Fax + 188: Fedora + 189: Filing cabinet + 190: Fire hydrant + 191: Fireplace + 192: Fish + 193: Flag + 194: Flashlight + 195: Flower + 196: Flowerpot + 197: Flute + 198: Flying disc + 199: Food + 200: Food processor + 201: Football + 202: Football helmet + 203: Footwear + 204: Fork + 205: Fountain + 206: Fox + 207: French fries + 208: French horn + 209: Frog + 210: Fruit + 211: Frying pan + 212: Furniture + 213: Garden Asparagus + 214: Gas stove + 215: Giraffe + 216: Girl + 217: Glasses + 218: Glove + 219: Goat + 220: Goggles + 221: Goldfish + 222: Golf ball + 223: Golf cart + 224: Gondola + 225: Goose + 226: Grape + 227: Grapefruit + 228: Grinder + 229: Guacamole + 230: Guitar + 231: Hair dryer + 232: Hair spray + 233: Hamburger + 234: Hammer + 235: Hamster + 236: Hand dryer + 237: Handbag + 238: Handgun + 239: Harbor seal + 240: Harmonica + 241: Harp + 242: Harpsichord + 243: Hat + 244: Headphones + 245: Heater + 246: Hedgehog + 247: Helicopter + 248: Helmet + 249: High heels + 250: Hiking equipment + 251: Hippopotamus + 252: Home appliance + 253: Honeycomb + 254: Horizontal bar + 255: Horse + 256: Hot dog + 257: House + 258: Houseplant + 259: Human arm + 260: Human beard + 261: Human body + 262: Human ear + 263: Human eye + 264: Human face + 265: Human foot + 266: Human hair + 267: Human hand + 268: Human head + 269: Human leg + 270: Human mouth + 271: Human nose + 272: Humidifier + 273: Ice cream + 274: Indoor rower + 275: Infant bed + 276: Insect + 277: Invertebrate + 278: Ipod + 279: Isopod + 280: Jacket + 281: Jacuzzi + 282: Jaguar (Animal) + 283: Jeans + 284: Jellyfish + 285: Jet ski + 286: Jug + 287: Juice + 288: Kangaroo + 289: Kettle + 290: Kitchen & dining room table + 291: Kitchen appliance + 292: Kitchen knife + 293: Kitchen utensil + 294: Kitchenware + 295: Kite + 296: Knife + 297: Koala + 298: Ladder + 299: Ladle + 300: Ladybug + 301: Lamp + 302: Land vehicle + 303: Lantern + 304: Laptop + 305: Lavender (Plant) + 306: Lemon + 307: Leopard + 308: Light bulb + 309: Light switch + 310: Lighthouse + 311: Lily + 312: Limousine + 313: Lion + 314: Lipstick + 315: Lizard + 316: Lobster + 317: Loveseat + 318: Luggage and bags + 319: Lynx + 320: Magpie + 321: Mammal + 322: Man + 323: Mango + 324: Maple + 325: Maracas + 326: Marine invertebrates + 327: Marine mammal + 328: Measuring cup + 329: Mechanical fan + 330: Medical equipment + 331: Microphone + 332: Microwave oven + 333: Milk + 334: Miniskirt + 335: Mirror + 336: Missile + 337: Mixer + 338: Mixing bowl + 339: Mobile phone + 340: Monkey + 341: Moths and butterflies + 342: Motorcycle + 343: Mouse + 344: Muffin + 345: Mug + 346: Mule + 347: Mushroom + 348: Musical instrument + 349: Musical keyboard + 350: Nail (Construction) + 351: Necklace + 352: Nightstand + 353: Oboe + 354: Office building + 355: Office supplies + 356: Orange + 357: Organ (Musical Instrument) + 358: Ostrich + 359: Otter + 360: Oven + 361: Owl + 362: Oyster + 363: Paddle + 364: Palm tree + 365: Pancake + 366: Panda + 367: Paper cutter + 368: Paper towel + 369: Parachute + 370: Parking meter + 371: Parrot + 372: Pasta + 373: Pastry + 374: Peach + 375: Pear + 376: Pen + 377: Pencil case + 378: Pencil sharpener + 379: Penguin + 380: Perfume + 381: Person + 382: Personal care + 383: Personal flotation device + 384: Piano + 385: Picnic basket + 386: Picture frame + 387: Pig + 388: Pillow + 389: Pineapple + 390: Pitcher (Container) + 391: Pizza + 392: Pizza cutter + 393: Plant + 394: Plastic bag + 395: Plate + 396: Platter + 397: Plumbing fixture + 398: Polar bear + 399: Pomegranate + 400: Popcorn + 401: Porch + 402: Porcupine + 403: Poster + 404: Potato + 405: Power plugs and sockets + 406: Pressure cooker + 407: Pretzel + 408: Printer + 409: Pumpkin + 410: Punching bag + 411: Rabbit + 412: Raccoon + 413: Racket + 414: Radish + 415: Ratchet (Device) + 416: Raven + 417: Rays and skates + 418: Red panda + 419: Refrigerator + 420: Remote control + 421: Reptile + 422: Rhinoceros + 423: Rifle + 424: Ring binder + 425: Rocket + 426: Roller skates + 427: Rose + 428: Rugby ball + 429: Ruler + 430: Salad + 431: Salt and pepper shakers + 432: Sandal + 433: Sandwich + 434: Saucer + 435: Saxophone + 436: Scale + 437: Scarf + 438: Scissors + 439: Scoreboard + 440: Scorpion + 441: Screwdriver + 442: Sculpture + 443: Sea lion + 444: Sea turtle + 445: Seafood + 446: Seahorse + 447: Seat belt + 448: Segway + 449: Serving tray + 450: Sewing machine + 451: Shark + 452: Sheep + 453: Shelf + 454: Shellfish + 455: Shirt + 456: Shorts + 457: Shotgun + 458: Shower + 459: Shrimp + 460: Sink + 461: Skateboard + 462: Ski + 463: Skirt + 464: Skull + 465: Skunk + 466: Skyscraper + 467: Slow cooker + 468: Snack + 469: Snail + 470: Snake + 471: Snowboard + 472: Snowman + 473: Snowmobile + 474: Snowplow + 475: Soap dispenser + 476: Sock + 477: Sofa bed + 478: Sombrero + 479: Sparrow + 480: Spatula + 481: Spice rack + 482: Spider + 483: Spoon + 484: Sports equipment + 485: Sports uniform + 486: Squash (Plant) + 487: Squid + 488: Squirrel + 489: Stairs + 490: Stapler + 491: Starfish + 492: Stationary bicycle + 493: Stethoscope + 494: Stool + 495: Stop sign + 496: Strawberry + 497: Street light + 498: Stretcher + 499: Studio couch + 500: Submarine + 501: Submarine sandwich + 502: Suit + 503: Suitcase + 504: Sun hat + 505: Sunglasses + 506: Surfboard + 507: Sushi + 508: Swan + 509: Swim cap + 510: Swimming pool + 511: Swimwear + 512: Sword + 513: Syringe + 514: Table + 515: Table tennis racket + 516: Tablet computer + 517: Tableware + 518: Taco + 519: Tank + 520: Tap + 521: Tart + 522: Taxi + 523: Tea + 524: Teapot + 525: Teddy bear + 526: Telephone + 527: Television + 528: Tennis ball + 529: Tennis racket + 530: Tent + 531: Tiara + 532: Tick + 533: Tie + 534: Tiger + 535: Tin can + 536: Tire + 537: Toaster + 538: Toilet + 539: Toilet paper + 540: Tomato + 541: Tool + 542: Toothbrush + 543: Torch + 544: Tortoise + 545: Towel + 546: Tower + 547: Toy + 548: Traffic light + 549: Traffic sign + 550: Train + 551: Training bench + 552: Treadmill + 553: Tree + 554: Tree house + 555: Tripod + 556: Trombone + 557: Trousers + 558: Truck + 559: Trumpet + 560: Turkey + 561: Turtle + 562: Umbrella + 563: Unicycle + 564: Van + 565: Vase + 566: Vegetable + 567: Vehicle + 568: Vehicle registration plate + 569: Violin + 570: Volleyball (Ball) + 571: Waffle + 572: Waffle iron + 573: Wall clock + 574: Wardrobe + 575: Washing machine + 576: Waste container + 577: Watch + 578: Watercraft + 579: Watermelon + 580: Weapon + 581: Whale + 582: Wheel + 583: Wheelchair + 584: Whisk + 585: Whiteboard + 586: Willow + 587: Window + 588: Window blind + 589: Wine + 590: Wine glass + 591: Wine rack + 592: Winter melon + 593: Wok + 594: Woman + 595: Wood-burning stove + 596: Woodpecker + 597: Worm + 598: Wrench + 599: Zebra + 600: Zucchini + +# Download script/URL (optional) --------------------------------------------------------------------------------------- +download: | + from ultralytics.utils import LOGGER, SETTINGS, Path, is_ubuntu, get_ubuntu_version + from ultralytics.utils.checks import check_requirements, check_version + + check_requirements('fiftyone') + if is_ubuntu() and check_version(get_ubuntu_version(), '>=22.04'): + # Ubuntu>=22.04 patch https://github.com/voxel51/fiftyone/issues/2961#issuecomment-1666519347 + check_requirements('fiftyone-db-ubuntu2204') + + import fiftyone as fo + import fiftyone.zoo as foz + import warnings + + name = 'open-images-v7' + fraction = 1.0 # fraction of full dataset to use + LOGGER.warning('WARNING ⚠️ Open Images V7 dataset requires at least **561 GB of free space. Starting download...') + for split in 'train', 'validation': # 1743042 train, 41620 val images + train = split == 'train' + + # Load Open Images dataset + dataset = foz.load_zoo_dataset(name, + split=split, + label_types=['detections'], + dataset_dir=Path(SETTINGS['datasets_dir']) / 'fiftyone' / name, + max_samples=round((1743042 if train else 41620) * fraction)) + + # Define classes + if train: + classes = dataset.default_classes # all classes + # classes = dataset.distinct('ground_truth.detections.label') # only observed classes + + # Export to YOLO format + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=UserWarning, module="fiftyone.utils.yolo") + dataset.export(export_dir=str(Path(SETTINGS['datasets_dir']) / name), + dataset_type=fo.types.YOLOv5Dataset, + label_field='ground_truth', + split='val' if split == 'validation' else split, + classes=classes, + overwrite=train) diff --git a/ultralytics/cfg/datasets/package-seg.yaml b/ultralytics/cfg/datasets/package-seg.yaml new file mode 100644 index 0000000..44fe550 --- /dev/null +++ b/ultralytics/cfg/datasets/package-seg.yaml @@ -0,0 +1,21 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# Package-seg dataset by Ultralytics +# Documentation: https://docs.ultralytics.com/datasets/segment/package-seg/ +# Example usage: yolo train data=package-seg.yaml +# parent +# ├── ultralytics +# └── datasets +# └── package-seg ← downloads here (102 MB) + +# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..] +path: ../datasets/package-seg # dataset root dir +train: images/train # train images (relative to 'path') 1920 images +val: images/val # val images (relative to 'path') 89 images +test: test/images # test images (relative to 'path') 188 images + +# Classes +names: + 0: package + +# Download script/URL (optional) +download: https://ultralytics.com/assets/package-seg.zip diff --git a/ultralytics/cfg/datasets/tiger-pose.yaml b/ultralytics/cfg/datasets/tiger-pose.yaml new file mode 100644 index 0000000..d37df04 --- /dev/null +++ b/ultralytics/cfg/datasets/tiger-pose.yaml @@ -0,0 +1,24 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# Tiger Pose dataset by Ultralytics +# Documentation: https://docs.ultralytics.com/datasets/pose/tiger-pose/ +# Example usage: yolo train data=tiger-pose.yaml +# parent +# ├── ultralytics +# └── datasets +# └── tiger-pose ← downloads here (75.3 MB) + +# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..] +path: ../datasets/tiger-pose # dataset root dir +train: train # train images (relative to 'path') 210 images +val: val # val images (relative to 'path') 53 images + +# Keypoints +kpt_shape: [12, 2] # number of keypoints, number of dims (2 for x,y or 3 for x,y,visible) +flip_idx: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] + +# Classes +names: + 0: tiger + +# Download script/URL (optional) +download: https://ultralytics.com/assets/tiger-pose.zip diff --git a/ultralytics/cfg/datasets/xView.yaml b/ultralytics/cfg/datasets/xView.yaml new file mode 100644 index 0000000..d2e957a --- /dev/null +++ b/ultralytics/cfg/datasets/xView.yaml @@ -0,0 +1,152 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# DIUx xView 2018 Challenge https://challenge.xviewdataset.org by U.S. National Geospatial-Intelligence Agency (NGA) +# -------- DOWNLOAD DATA MANUALLY and jar xf val_images.zip to 'datasets/xView' before running train command! -------- +# Documentation: https://docs.ultralytics.com/datasets/detect/xview/ +# Example usage: yolo train data=xView.yaml +# parent +# ├── ultralytics +# └── datasets +# └── xView ← downloads here (20.7 GB) + +# Train/val/test sets as 1) dir: path/to/imgs, 2) file: path/to/imgs.txt, or 3) list: [path/to/imgs1, path/to/imgs2, ..] +path: ../datasets/xView # dataset root dir +train: images/autosplit_train.txt # train images (relative to 'path') 90% of 847 train images +val: images/autosplit_val.txt # train images (relative to 'path') 10% of 847 train images + +# Classes +names: + 0: Fixed-wing Aircraft + 1: Small Aircraft + 2: Cargo Plane + 3: Helicopter + 4: Passenger Vehicle + 5: Small Car + 6: Bus + 7: Pickup Truck + 8: Utility Truck + 9: Truck + 10: Cargo Truck + 11: Truck w/Box + 12: Truck Tractor + 13: Trailer + 14: Truck w/Flatbed + 15: Truck w/Liquid + 16: Crane Truck + 17: Railway Vehicle + 18: Passenger Car + 19: Cargo Car + 20: Flat Car + 21: Tank car + 22: Locomotive + 23: Maritime Vessel + 24: Motorboat + 25: Sailboat + 26: Tugboat + 27: Barge + 28: Fishing Vessel + 29: Ferry + 30: Yacht + 31: Container Ship + 32: Oil Tanker + 33: Engineering Vehicle + 34: Tower crane + 35: Container Crane + 36: Reach Stacker + 37: Straddle Carrier + 38: Mobile Crane + 39: Dump Truck + 40: Haul Truck + 41: Scraper/Tractor + 42: Front loader/Bulldozer + 43: Excavator + 44: Cement Mixer + 45: Ground Grader + 46: Hut/Tent + 47: Shed + 48: Building + 49: Aircraft Hangar + 50: Damaged Building + 51: Facility + 52: Construction Site + 53: Vehicle Lot + 54: Helipad + 55: Storage Tank + 56: Shipping container lot + 57: Shipping Container + 58: Pylon + 59: Tower + +# Download script/URL (optional) --------------------------------------------------------------------------------------- +download: | + import json + import os + from pathlib import Path + + import numpy as np + from PIL import Image + from tqdm import tqdm + + from ultralytics.data.utils import autosplit + from ultralytics.utils.ops import xyxy2xywhn + + + def convert_labels(fname=Path('xView/xView_train.geojson')): + # Convert xView geoJSON labels to YOLO format + path = fname.parent + with open(fname) as f: + print(f'Loading {fname}...') + data = json.load(f) + + # Make dirs + labels = Path(path / 'labels' / 'train') + os.system(f'rm -rf {labels}') + labels.mkdir(parents=True, exist_ok=True) + + # xView classes 11-94 to 0-59 + xview_class2index = [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 1, 2, -1, 3, -1, 4, 5, 6, 7, 8, -1, 9, 10, 11, + 12, 13, 14, 15, -1, -1, 16, 17, 18, 19, 20, 21, 22, -1, 23, 24, 25, -1, 26, 27, -1, 28, -1, + 29, 30, 31, 32, 33, 34, 35, 36, 37, -1, 38, 39, 40, 41, 42, 43, 44, 45, -1, -1, -1, -1, 46, + 47, 48, 49, -1, 50, 51, -1, 52, -1, -1, -1, 53, 54, -1, 55, -1, -1, 56, -1, 57, -1, 58, 59] + + shapes = {} + for feature in tqdm(data['features'], desc=f'Converting {fname}'): + p = feature['properties'] + if p['bounds_imcoords']: + id = p['image_id'] + file = path / 'train_images' / id + if file.exists(): # 1395.tif missing + try: + box = np.array([int(num) for num in p['bounds_imcoords'].split(",")]) + assert box.shape[0] == 4, f'incorrect box shape {box.shape[0]}' + cls = p['type_id'] + cls = xview_class2index[int(cls)] # xView class to 0-60 + assert 59 >= cls >= 0, f'incorrect class index {cls}' + + # Write YOLO label + if id not in shapes: + shapes[id] = Image.open(file).size + box = xyxy2xywhn(box[None].astype(np.float), w=shapes[id][0], h=shapes[id][1], clip=True) + with open((labels / id).with_suffix('.txt'), 'a') as f: + f.write(f"{cls} {' '.join(f'{x:.6f}' for x in box[0])}\n") # write label.txt + except Exception as e: + print(f'WARNING: skipping one label for {file}: {e}') + + + # Download manually from https://challenge.xviewdataset.org + dir = Path(yaml['path']) # dataset root dir + # urls = ['https://d307kc0mrhucc3.cloudfront.net/train_labels.zip', # train labels + # 'https://d307kc0mrhucc3.cloudfront.net/train_images.zip', # 15G, 847 train images + # 'https://d307kc0mrhucc3.cloudfront.net/val_images.zip'] # 5G, 282 val images (no labels) + # download(urls, dir=dir) + + # Convert labels + convert_labels(dir / 'xView_train.geojson') + + # Move images + images = Path(dir / 'images') + images.mkdir(parents=True, exist_ok=True) + Path(dir / 'train_images').rename(dir / 'images' / 'train') + Path(dir / 'val_images').rename(dir / 'images' / 'val') + + # Split + autosplit(dir / 'images' / 'train') diff --git a/ultralytics/cfg/default.yaml b/ultralytics/cfg/default.yaml new file mode 100644 index 0000000..bd074b1 --- /dev/null +++ b/ultralytics/cfg/default.yaml @@ -0,0 +1,127 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# Default training settings and hyperparameters for medium-augmentation COCO training + +task: detect # (str) YOLO task, i.e. detect, segment, classify, pose +mode: train # (str) YOLO mode, i.e. train, val, predict, export, track, benchmark + +# Train settings ------------------------------------------------------------------------------------------------------- +model: # (str, optional) path to model file, i.e. yolov8n.pt, yolov8n.yaml +data: # (str, optional) path to data file, i.e. coco128.yaml +epochs: 100 # (int) number of epochs to train for +time: # (float, optional) number of hours to train for, overrides epochs if supplied +patience: 100 # (int) epochs to wait for no observable improvement for early stopping of training +batch: 16 # (int) number of images per batch (-1 for AutoBatch) +imgsz: 640 # (int | list) input images size as int for train and val modes, or list[w,h] for predict and export modes +save: True # (bool) save train checkpoints and predict results +save_period: -1 # (int) Save checkpoint every x epochs (disabled if < 1) +val_period: 1 # (int) Validation every x epochs +cache: False # (bool) True/ram, disk or False. Use cache for data loading +device: # (int | str | list, optional) device to run on, i.e. cuda device=0 or device=0,1,2,3 or device=cpu +workers: 8 # (int) number of worker threads for data loading (per RANK if DDP) +project: # (str, optional) project name +name: # (str, optional) experiment name, results saved to 'project/name' directory +exist_ok: False # (bool) whether to overwrite existing experiment +pretrained: True # (bool | str) whether to use a pretrained model (bool) or a model to load weights from (str) +optimizer: auto # (str) optimizer to use, choices=[SGD, Adam, Adamax, AdamW, NAdam, RAdam, RMSProp, auto] +verbose: True # (bool) whether to print verbose output +seed: 0 # (int) random seed for reproducibility +deterministic: True # (bool) whether to enable deterministic mode +single_cls: False # (bool) train multi-class data as single-class +rect: False # (bool) rectangular training if mode='train' or rectangular validation if mode='val' +cos_lr: False # (bool) use cosine learning rate scheduler +close_mosaic: 10 # (int) disable mosaic augmentation for final epochs (0 to disable) +resume: False # (bool) resume training from last checkpoint +amp: True # (bool) Automatic Mixed Precision (AMP) training, choices=[True, False], True runs AMP check +fraction: 1.0 # (float) dataset fraction to train on (default is 1.0, all images in train set) +profile: False # (bool) profile ONNX and TensorRT speeds during training for loggers +freeze: None # (int | list, optional) freeze first n layers, or freeze list of layer indices during training +multi_scale: False # (bool) Whether to use multiscale during training +# Segmentation +overlap_mask: True # (bool) masks should overlap during training (segment train only) +mask_ratio: 4 # (int) mask downsample ratio (segment train only) +# Classification +dropout: 0.0 # (float) use dropout regularization (classify train only) + +# Val/Test settings ---------------------------------------------------------------------------------------------------- +val: True # (bool) validate/test during training +split: val # (str) dataset split to use for validation, i.e. 'val', 'test' or 'train' +save_json: False # (bool) save results to JSON file +save_hybrid: False # (bool) save hybrid version of labels (labels + additional predictions) +conf: # (float, optional) object confidence threshold for detection (default 0.25 predict, 0.001 val) +iou: 0.7 # (float) intersection over union (IoU) threshold for NMS +max_det: 300 # (int) maximum number of detections per image +half: False # (bool) use half precision (FP16) +dnn: False # (bool) use OpenCV DNN for ONNX inference +plots: True # (bool) save plots and images during train/val + +# Predict settings ----------------------------------------------------------------------------------------------------- +source: # (str, optional) source directory for images or videos +vid_stride: 1 # (int) video frame-rate stride +stream_buffer: False # (bool) buffer all streaming frames (True) or return the most recent frame (False) +visualize: False # (bool) visualize model features +augment: False # (bool) apply image augmentation to prediction sources +agnostic_nms: False # (bool) class-agnostic NMS +classes: # (int | list[int], optional) filter results by class, i.e. classes=0, or classes=[0,2,3] +retina_masks: False # (bool) use high-resolution segmentation masks +embed: # (list[int], optional) return feature vectors/embeddings from given layers + +# Visualize settings --------------------------------------------------------------------------------------------------- +show: False # (bool) show predicted images and videos if environment allows +save_frames: False # (bool) save predicted individual video frames +save_txt: False # (bool) save results as .txt file +save_conf: False # (bool) save results with confidence scores +save_crop: False # (bool) save cropped images with results +show_labels: True # (bool) show prediction labels, i.e. 'person' +show_conf: True # (bool) show prediction confidence, i.e. '0.99' +show_boxes: True # (bool) show prediction boxes +line_width: # (int, optional) line width of the bounding boxes. Scaled to image size if None. + +# Export settings ------------------------------------------------------------------------------------------------------ +format: torchscript # (str) format to export to, choices at https://docs.ultralytics.com/modes/export/#export-formats +keras: False # (bool) use Kera=s +optimize: False # (bool) TorchScript: optimize for mobile +int8: False # (bool) CoreML/TF INT8 quantization +dynamic: False # (bool) ONNX/TF/TensorRT: dynamic axes +simplify: False # (bool) ONNX: simplify model using `onnxslim` +opset: # (int, optional) ONNX: opset version +workspace: 4 # (int) TensorRT: workspace size (GB) +nms: False # (bool) CoreML: add NMS + +# Hyperparameters ------------------------------------------------------------------------------------------------------ +lr0: 0.01 # (float) initial learning rate (i.e. SGD=1E-2, Adam=1E-3) +lrf: 0.01 # (float) final learning rate (lr0 * lrf) +momentum: 0.937 # (float) SGD momentum/Adam beta1 +weight_decay: 0.0005 # (float) optimizer weight decay 5e-4 +warmup_epochs: 3.0 # (float) warmup epochs (fractions ok) +warmup_momentum: 0.8 # (float) warmup initial momentum +warmup_bias_lr: 0.1 # (float) warmup initial bias lr +box: 7.5 # (float) box loss gain +cls: 0.5 # (float) cls loss gain (scale with pixels) +dfl: 1.5 # (float) dfl loss gain +pose: 12.0 # (float) pose loss gain +kobj: 1.0 # (float) keypoint obj loss gain +label_smoothing: 0.0 # (float) label smoothing (fraction) +nbs: 64 # (int) nominal batch size +hsv_h: 0.015 # (float) image HSV-Hue augmentation (fraction) +hsv_s: 0.7 # (float) image HSV-Saturation augmentation (fraction) +hsv_v: 0.4 # (float) image HSV-Value augmentation (fraction) +degrees: 0.0 # (float) image rotation (+/- deg) +translate: 0.1 # (float) image translation (+/- fraction) +scale: 0.5 # (float) image scale (+/- gain) +shear: 0.0 # (float) image shear (+/- deg) +perspective: 0.0 # (float) image perspective (+/- fraction), range 0-0.001 +flipud: 0.0 # (float) image flip up-down (probability) +fliplr: 0.5 # (float) image flip left-right (probability) +bgr: 0.0 # (float) image channel BGR (probability) +mosaic: 1.0 # (float) image mosaic (probability) +mixup: 0.0 # (float) image mixup (probability) +copy_paste: 0.0 # (float) segment copy-paste (probability) +auto_augment: randaugment # (str) auto augmentation policy for classification (randaugment, autoaugment, augmix) +erasing: 0.4 # (float) probability of random erasing during classification training (0-1) +crop_fraction: 1.0 # (float) image crop fraction for classification evaluation/inference (0-1) + +# Custom config.yaml --------------------------------------------------------------------------------------------------- +cfg: # (str, optional) for overriding defaults.yaml + +# Tracker settings ------------------------------------------------------------------------------------------------------ +tracker: botsort.yaml # (str) tracker type, choices=[botsort.yaml, bytetrack.yaml] diff --git a/ultralytics/cfg/models/README.md b/ultralytics/cfg/models/README.md new file mode 100644 index 0000000..c022fb5 --- /dev/null +++ b/ultralytics/cfg/models/README.md @@ -0,0 +1,40 @@ +## Models + +Welcome to the Ultralytics Models directory! Here you will find a wide variety of pre-configured model configuration files (`*.yaml`s) that can be used to create custom YOLO models. The models in this directory have been expertly crafted and fine-tuned by the Ultralytics team to provide the best performance for a wide range of object detection and image segmentation tasks. + +These model configurations cover a wide range of scenarios, from simple object detection to more complex tasks like instance segmentation and object tracking. They are also designed to run efficiently on a variety of hardware platforms, from CPUs to GPUs. Whether you are a seasoned machine learning practitioner or just getting started with YOLO, this directory provides a great starting point for your custom model development needs. + +To get started, simply browse through the models in this directory and find one that best suits your needs. Once you've selected a model, you can use the provided `*.yaml` file to train and deploy your custom YOLO model with ease. See full details at the Ultralytics [Docs](https://docs.ultralytics.com/models), and if you need help or have any questions, feel free to reach out to the Ultralytics team for support. So, don't wait, start creating your custom YOLO model now! + +### Usage + +Model `*.yaml` files may be used directly in the Command Line Interface (CLI) with a `yolo` command: + +```bash +yolo task=detect mode=train model=yolov8n.yaml data=coco128.yaml epochs=100 +``` + +They may also be used directly in a Python environment, and accepts the same [arguments](https://docs.ultralytics.com/usage/cfg/) as in the CLI example above: + +```python +from ultralytics import YOLO + +model = YOLO("model.yaml") # build a YOLOv8n model from scratch +# YOLO("model.pt") use pre-trained model if available +model.info() # display model information +model.train(data="coco128.yaml", epochs=100) # train the model +``` + +## Pre-trained Model Architectures + +Ultralytics supports many model architectures. Visit https://docs.ultralytics.com/models to view detailed information and usage. Any of these models can be used by loading their configs or pretrained checkpoints if available. + +## Contribute New Models + +Have you trained a new YOLO variant or achieved state-of-the-art performance with specific tuning? We'd love to showcase your work in our Models section! Contributions from the community in the form of new models, architectures, or optimizations are highly valued and can significantly enrich our repository. + +By contributing to this section, you're helping us offer a wider array of model choices and configurations to the community. It's a fantastic way to share your knowledge and expertise while making the Ultralytics YOLO ecosystem even more versatile. + +To get started, please consult our [Contributing Guide](https://docs.ultralytics.com/help/contributing) for step-by-step instructions on how to submit a Pull Request (PR) 🛠️. Your contributions are eagerly awaited! + +Let's join hands to extend the range and capabilities of the Ultralytics YOLO models 🙏! diff --git a/ultralytics/cfg/models/rt-detr/rtdetr-l.yaml b/ultralytics/cfg/models/rt-detr/rtdetr-l.yaml new file mode 100644 index 0000000..c6eb0b3 --- /dev/null +++ b/ultralytics/cfg/models/rt-detr/rtdetr-l.yaml @@ -0,0 +1,50 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# RT-DETR-l object detection model with P3-P5 outputs. For details see https://docs.ultralytics.com/models/rtdetr + +# Parameters +nc: 80 # number of classes +scales: # model compound scaling constants, i.e. 'model=yolov8n-cls.yaml' will call yolov8-cls.yaml with scale 'n' + # [depth, width, max_channels] + l: [1.00, 1.00, 1024] + +backbone: + # [from, repeats, module, args] + - [-1, 1, HGStem, [32, 48]] # 0-P2/4 + - [-1, 6, HGBlock, [48, 128, 3]] # stage 1 + + - [-1, 1, DWConv, [128, 3, 2, 1, False]] # 2-P3/8 + - [-1, 6, HGBlock, [96, 512, 3]] # stage 2 + + - [-1, 1, DWConv, [512, 3, 2, 1, False]] # 4-P3/16 + - [-1, 6, HGBlock, [192, 1024, 5, True, False]] # cm, c2, k, light, shortcut + - [-1, 6, HGBlock, [192, 1024, 5, True, True]] + - [-1, 6, HGBlock, [192, 1024, 5, True, True]] # stage 3 + + - [-1, 1, DWConv, [1024, 3, 2, 1, False]] # 8-P4/32 + - [-1, 6, HGBlock, [384, 2048, 5, True, False]] # stage 4 + +head: + - [-1, 1, Conv, [256, 1, 1, None, 1, 1, False]] # 10 input_proj.2 + - [-1, 1, AIFI, [1024, 8]] + - [-1, 1, Conv, [256, 1, 1]] # 12, Y5, lateral_convs.0 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [7, 1, Conv, [256, 1, 1, None, 1, 1, False]] # 14 input_proj.1 + - [[-2, -1], 1, Concat, [1]] + - [-1, 3, RepC3, [256]] # 16, fpn_blocks.0 + - [-1, 1, Conv, [256, 1, 1]] # 17, Y4, lateral_convs.1 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [3, 1, Conv, [256, 1, 1, None, 1, 1, False]] # 19 input_proj.0 + - [[-2, -1], 1, Concat, [1]] # cat backbone P4 + - [-1, 3, RepC3, [256]] # X3 (21), fpn_blocks.1 + + - [-1, 1, Conv, [256, 3, 2]] # 22, downsample_convs.0 + - [[-1, 17], 1, Concat, [1]] # cat Y4 + - [-1, 3, RepC3, [256]] # F4 (24), pan_blocks.0 + + - [-1, 1, Conv, [256, 3, 2]] # 25, downsample_convs.1 + - [[-1, 12], 1, Concat, [1]] # cat Y5 + - [-1, 3, RepC3, [256]] # F5 (27), pan_blocks.1 + + - [[21, 24, 27], 1, RTDETRDecoder, [nc]] # Detect(P3, P4, P5) diff --git a/ultralytics/cfg/models/rt-detr/rtdetr-resnet101.yaml b/ultralytics/cfg/models/rt-detr/rtdetr-resnet101.yaml new file mode 100644 index 0000000..a68bb5d --- /dev/null +++ b/ultralytics/cfg/models/rt-detr/rtdetr-resnet101.yaml @@ -0,0 +1,42 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# RT-DETR-ResNet101 object detection model with P3-P5 outputs. + +# Parameters +nc: 80 # number of classes +scales: # model compound scaling constants, i.e. 'model=yolov8n-cls.yaml' will call yolov8-cls.yaml with scale 'n' + # [depth, width, max_channels] + l: [1.00, 1.00, 1024] + +backbone: + # [from, repeats, module, args] + - [-1, 1, ResNetLayer, [3, 64, 1, True, 1]] # 0 + - [-1, 1, ResNetLayer, [64, 64, 1, False, 3]] # 1 + - [-1, 1, ResNetLayer, [256, 128, 2, False, 4]] # 2 + - [-1, 1, ResNetLayer, [512, 256, 2, False, 23]] # 3 + - [-1, 1, ResNetLayer, [1024, 512, 2, False, 3]] # 4 + +head: + - [-1, 1, Conv, [256, 1, 1, None, 1, 1, False]] # 5 + - [-1, 1, AIFI, [1024, 8]] + - [-1, 1, Conv, [256, 1, 1]] # 7 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [3, 1, Conv, [256, 1, 1, None, 1, 1, False]] # 9 + - [[-2, -1], 1, Concat, [1]] + - [-1, 3, RepC3, [256]] # 11 + - [-1, 1, Conv, [256, 1, 1]] # 12 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [2, 1, Conv, [256, 1, 1, None, 1, 1, False]] # 14 + - [[-2, -1], 1, Concat, [1]] # cat backbone P4 + - [-1, 3, RepC3, [256]] # X3 (16), fpn_blocks.1 + + - [-1, 1, Conv, [256, 3, 2]] # 17, downsample_convs.0 + - [[-1, 12], 1, Concat, [1]] # cat Y4 + - [-1, 3, RepC3, [256]] # F4 (19), pan_blocks.0 + + - [-1, 1, Conv, [256, 3, 2]] # 20, downsample_convs.1 + - [[-1, 7], 1, Concat, [1]] # cat Y5 + - [-1, 3, RepC3, [256]] # F5 (22), pan_blocks.1 + + - [[16, 19, 22], 1, RTDETRDecoder, [nc]] # Detect(P3, P4, P5) diff --git a/ultralytics/cfg/models/rt-detr/rtdetr-resnet50.yaml b/ultralytics/cfg/models/rt-detr/rtdetr-resnet50.yaml new file mode 100644 index 0000000..7145910 --- /dev/null +++ b/ultralytics/cfg/models/rt-detr/rtdetr-resnet50.yaml @@ -0,0 +1,42 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# RT-DETR-ResNet50 object detection model with P3-P5 outputs. + +# Parameters +nc: 80 # number of classes +scales: # model compound scaling constants, i.e. 'model=yolov8n-cls.yaml' will call yolov8-cls.yaml with scale 'n' + # [depth, width, max_channels] + l: [1.00, 1.00, 1024] + +backbone: + # [from, repeats, module, args] + - [-1, 1, ResNetLayer, [3, 64, 1, True, 1]] # 0 + - [-1, 1, ResNetLayer, [64, 64, 1, False, 3]] # 1 + - [-1, 1, ResNetLayer, [256, 128, 2, False, 4]] # 2 + - [-1, 1, ResNetLayer, [512, 256, 2, False, 6]] # 3 + - [-1, 1, ResNetLayer, [1024, 512, 2, False, 3]] # 4 + +head: + - [-1, 1, Conv, [256, 1, 1, None, 1, 1, False]] # 5 + - [-1, 1, AIFI, [1024, 8]] + - [-1, 1, Conv, [256, 1, 1]] # 7 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [3, 1, Conv, [256, 1, 1, None, 1, 1, False]] # 9 + - [[-2, -1], 1, Concat, [1]] + - [-1, 3, RepC3, [256]] # 11 + - [-1, 1, Conv, [256, 1, 1]] # 12 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [2, 1, Conv, [256, 1, 1, None, 1, 1, False]] # 14 + - [[-2, -1], 1, Concat, [1]] # cat backbone P4 + - [-1, 3, RepC3, [256]] # X3 (16), fpn_blocks.1 + + - [-1, 1, Conv, [256, 3, 2]] # 17, downsample_convs.0 + - [[-1, 12], 1, Concat, [1]] # cat Y4 + - [-1, 3, RepC3, [256]] # F4 (19), pan_blocks.0 + + - [-1, 1, Conv, [256, 3, 2]] # 20, downsample_convs.1 + - [[-1, 7], 1, Concat, [1]] # cat Y5 + - [-1, 3, RepC3, [256]] # F5 (22), pan_blocks.1 + + - [[16, 19, 22], 1, RTDETRDecoder, [nc]] # Detect(P3, P4, P5) diff --git a/ultralytics/cfg/models/rt-detr/rtdetr-x.yaml b/ultralytics/cfg/models/rt-detr/rtdetr-x.yaml new file mode 100644 index 0000000..0e819b0 --- /dev/null +++ b/ultralytics/cfg/models/rt-detr/rtdetr-x.yaml @@ -0,0 +1,54 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# RT-DETR-x object detection model with P3-P5 outputs. For details see https://docs.ultralytics.com/models/rtdetr + +# Parameters +nc: 80 # number of classes +scales: # model compound scaling constants, i.e. 'model=yolov8n-cls.yaml' will call yolov8-cls.yaml with scale 'n' + # [depth, width, max_channels] + x: [1.00, 1.00, 2048] + +backbone: + # [from, repeats, module, args] + - [-1, 1, HGStem, [32, 64]] # 0-P2/4 + - [-1, 6, HGBlock, [64, 128, 3]] # stage 1 + + - [-1, 1, DWConv, [128, 3, 2, 1, False]] # 2-P3/8 + - [-1, 6, HGBlock, [128, 512, 3]] + - [-1, 6, HGBlock, [128, 512, 3, False, True]] # 4-stage 2 + + - [-1, 1, DWConv, [512, 3, 2, 1, False]] # 5-P3/16 + - [-1, 6, HGBlock, [256, 1024, 5, True, False]] # cm, c2, k, light, shortcut + - [-1, 6, HGBlock, [256, 1024, 5, True, True]] + - [-1, 6, HGBlock, [256, 1024, 5, True, True]] + - [-1, 6, HGBlock, [256, 1024, 5, True, True]] + - [-1, 6, HGBlock, [256, 1024, 5, True, True]] # 10-stage 3 + + - [-1, 1, DWConv, [1024, 3, 2, 1, False]] # 11-P4/32 + - [-1, 6, HGBlock, [512, 2048, 5, True, False]] + - [-1, 6, HGBlock, [512, 2048, 5, True, True]] # 13-stage 4 + +head: + - [-1, 1, Conv, [384, 1, 1, None, 1, 1, False]] # 14 input_proj.2 + - [-1, 1, AIFI, [2048, 8]] + - [-1, 1, Conv, [384, 1, 1]] # 16, Y5, lateral_convs.0 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [10, 1, Conv, [384, 1, 1, None, 1, 1, False]] # 18 input_proj.1 + - [[-2, -1], 1, Concat, [1]] + - [-1, 3, RepC3, [384]] # 20, fpn_blocks.0 + - [-1, 1, Conv, [384, 1, 1]] # 21, Y4, lateral_convs.1 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [4, 1, Conv, [384, 1, 1, None, 1, 1, False]] # 23 input_proj.0 + - [[-2, -1], 1, Concat, [1]] # cat backbone P4 + - [-1, 3, RepC3, [384]] # X3 (25), fpn_blocks.1 + + - [-1, 1, Conv, [384, 3, 2]] # 26, downsample_convs.0 + - [[-1, 21], 1, Concat, [1]] # cat Y4 + - [-1, 3, RepC3, [384]] # F4 (28), pan_blocks.0 + + - [-1, 1, Conv, [384, 3, 2]] # 29, downsample_convs.1 + - [[-1, 16], 1, Concat, [1]] # cat Y5 + - [-1, 3, RepC3, [384]] # F5 (31), pan_blocks.1 + + - [[25, 28, 31], 1, RTDETRDecoder, [nc]] # Detect(P3, P4, P5) diff --git a/ultralytics/cfg/models/v10/yolov10b.yaml b/ultralytics/cfg/models/v10/yolov10b.yaml new file mode 100644 index 0000000..a9dc721 --- /dev/null +++ b/ultralytics/cfg/models/v10/yolov10b.yaml @@ -0,0 +1,40 @@ +# Parameters +nc: 80 # number of classes +scales: # model compound scaling constants, i.e. 'model=yolov8n.yaml' will call yolov8.yaml with scale 'n' + # [depth, width, max_channels] + b: [0.67, 1.00, 512] + +# YOLOv8.0n backbone +backbone: + # [from, repeats, module, args] + - [-1, 1, Conv, [64, 3, 2]] # 0-P1/2 + - [-1, 1, Conv, [128, 3, 2]] # 1-P2/4 + - [-1, 3, C2f, [128, True]] + - [-1, 1, Conv, [256, 3, 2]] # 3-P3/8 + - [-1, 6, C2f, [256, True]] + - [-1, 1, SCDown, [512, 3, 2]] # 5-P4/16 + - [-1, 6, C2f, [512, True]] + - [-1, 1, SCDown, [1024, 3, 2]] # 7-P5/32 + - [-1, 3, C2fCIB, [1024, True]] + - [-1, 1, SPPF, [1024, 5]] # 9 + - [-1, 1, PSA, [1024]] # 10 + +# YOLOv8.0n head +head: + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 6], 1, Concat, [1]] # cat backbone P4 + - [-1, 3, C2fCIB, [512, True]] # 13 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 4], 1, Concat, [1]] # cat backbone P3 + - [-1, 3, C2f, [256]] # 16 (P3/8-small) + + - [-1, 1, Conv, [256, 3, 2]] + - [[-1, 13], 1, Concat, [1]] # cat head P4 + - [-1, 3, C2fCIB, [512, True]] # 19 (P4/16-medium) + + - [-1, 1, SCDown, [512, 3, 2]] + - [[-1, 10], 1, Concat, [1]] # cat head P5 + - [-1, 3, C2fCIB, [1024, True]] # 22 (P5/32-large) + + - [[16, 19, 22], 1, v10Detect, [nc]] # Detect(P3, P4, P5) diff --git a/ultralytics/cfg/models/v10/yolov10l.yaml b/ultralytics/cfg/models/v10/yolov10l.yaml new file mode 100644 index 0000000..047de26 --- /dev/null +++ b/ultralytics/cfg/models/v10/yolov10l.yaml @@ -0,0 +1,40 @@ +# Parameters +nc: 80 # number of classes +scales: # model compound scaling constants, i.e. 'model=yolov8n.yaml' will call yolov8.yaml with scale 'n' + # [depth, width, max_channels] + l: [1.00, 1.00, 512] # YOLOv8l summary: 365 layers, 43691520 parameters, 43691504 gradients, 165.7 GFLOPs + +# YOLOv8.0n backbone +backbone: + # [from, repeats, module, args] + - [-1, 1, Conv, [64, 3, 2]] # 0-P1/2 + - [-1, 1, Conv, [128, 3, 2]] # 1-P2/4 + - [-1, 3, C2f, [128, True]] + - [-1, 1, Conv, [256, 3, 2]] # 3-P3/8 + - [-1, 6, C2f, [256, True]] + - [-1, 1, SCDown, [512, 3, 2]] # 5-P4/16 + - [-1, 6, C2f, [512, True]] + - [-1, 1, SCDown, [1024, 3, 2]] # 7-P5/32 + - [-1, 3, C2fCIB, [1024, True]] + - [-1, 1, SPPF, [1024, 5]] # 9 + - [-1, 1, PSA, [1024]] # 10 + +# YOLOv8.0n head +head: + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 6], 1, Concat, [1]] # cat backbone P4 + - [-1, 3, C2fCIB, [512, True]] # 13 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 4], 1, Concat, [1]] # cat backbone P3 + - [-1, 3, C2f, [256]] # 16 (P3/8-small) + + - [-1, 1, Conv, [256, 3, 2]] + - [[-1, 13], 1, Concat, [1]] # cat head P4 + - [-1, 3, C2fCIB, [512, True]] # 19 (P4/16-medium) + + - [-1, 1, SCDown, [512, 3, 2]] + - [[-1, 10], 1, Concat, [1]] # cat head P5 + - [-1, 3, C2fCIB, [1024, True]] # 22 (P5/32-large) + + - [[16, 19, 22], 1, v10Detect, [nc]] # Detect(P3, P4, P5) diff --git a/ultralytics/cfg/models/v10/yolov10m.yaml b/ultralytics/cfg/models/v10/yolov10m.yaml new file mode 100644 index 0000000..5bdb5bf --- /dev/null +++ b/ultralytics/cfg/models/v10/yolov10m.yaml @@ -0,0 +1,43 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# YOLOv8 object detection model with P3-P5 outputs. For Usage examples see https://docs.ultralytics.com/tasks/detect + +# Parameters +nc: 80 # number of classes +scales: # model compound scaling constants, i.e. 'model=yolov8n.yaml' will call yolov8.yaml with scale 'n' + # [depth, width, max_channels] + m: [0.67, 0.75, 768] # YOLOv8m summary: 295 layers, 25902640 parameters, 25902624 gradients, 79.3 GFLOPs + +# YOLOv8.0n backbone +backbone: + # [from, repeats, module, args] + - [-1, 1, Conv, [64, 3, 2]] # 0-P1/2 + - [-1, 1, Conv, [128, 3, 2]] # 1-P2/4 + - [-1, 3, C2f, [128, True]] + - [-1, 1, Conv, [256, 3, 2]] # 3-P3/8 + - [-1, 6, C2f, [256, True]] + - [-1, 1, SCDown, [512, 3, 2]] # 5-P4/16 + - [-1, 6, C2f, [512, True]] + - [-1, 1, SCDown, [1024, 3, 2]] # 7-P5/32 + - [-1, 3, C2fCIB, [1024, True]] + - [-1, 1, SPPF, [1024, 5]] # 9 + - [-1, 1, PSA, [1024]] # 10 + +# YOLOv8.0n head +head: + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 6], 1, Concat, [1]] # cat backbone P4 + - [-1, 3, C2f, [512]] # 13 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 4], 1, Concat, [1]] # cat backbone P3 + - [-1, 3, C2f, [256]] # 16 (P3/8-small) + + - [-1, 1, Conv, [256, 3, 2]] + - [[-1, 13], 1, Concat, [1]] # cat head P4 + - [-1, 3, C2fCIB, [512, True]] # 19 (P4/16-medium) + + - [-1, 1, SCDown, [512, 3, 2]] + - [[-1, 10], 1, Concat, [1]] # cat head P5 + - [-1, 3, C2fCIB, [1024, True]] # 22 (P5/32-large) + + - [[16, 19, 22], 1, v10Detect, [nc]] # Detect(P3, P4, P5) diff --git a/ultralytics/cfg/models/v10/yolov10n.yaml b/ultralytics/cfg/models/v10/yolov10n.yaml new file mode 100644 index 0000000..1ee7437 --- /dev/null +++ b/ultralytics/cfg/models/v10/yolov10n.yaml @@ -0,0 +1,40 @@ +# Parameters +nc: 80 # number of classes +scales: # model compound scaling constants, i.e. 'model=yolov8n.yaml' will call yolov8.yaml with scale 'n' + # [depth, width, max_channels] + n: [0.33, 0.25, 1024] + +# YOLOv8.0n backbone +backbone: + # [from, repeats, module, args] + - [-1, 1, Conv, [64, 3, 2]] # 0-P1/2 + - [-1, 1, Conv, [128, 3, 2]] # 1-P2/4 + - [-1, 3, C2f, [128, True]] + - [-1, 1, Conv, [256, 3, 2]] # 3-P3/8 + - [-1, 6, C2f, [256, True]] + - [-1, 1, SCDown, [512, 3, 2]] # 5-P4/16 + - [-1, 6, C2f, [512, True]] + - [-1, 1, SCDown, [1024, 3, 2]] # 7-P5/32 + - [-1, 3, C2f, [1024, True]] + - [-1, 1, SPPF, [1024, 5]] # 9 + - [-1, 1, PSA, [1024]] # 10 + +# YOLOv8.0n head +head: + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 6], 1, Concat, [1]] # cat backbone P4 + - [-1, 3, C2f, [512]] # 13 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 4], 1, Concat, [1]] # cat backbone P3 + - [-1, 3, C2f, [256]] # 16 (P3/8-small) + + - [-1, 1, Conv, [256, 3, 2]] + - [[-1, 13], 1, Concat, [1]] # cat head P4 + - [-1, 3, C2f, [512]] # 19 (P4/16-medium) + + - [-1, 1, SCDown, [512, 3, 2]] + - [[-1, 10], 1, Concat, [1]] # cat head P5 + - [-1, 3, C2fCIB, [1024, True, True]] # 22 (P5/32-large) + + - [[16, 19, 22], 1, v10Detect, [nc]] # Detect(P3, P4, P5) diff --git a/ultralytics/cfg/models/v10/yolov10s.yaml b/ultralytics/cfg/models/v10/yolov10s.yaml new file mode 100644 index 0000000..c61e08c --- /dev/null +++ b/ultralytics/cfg/models/v10/yolov10s.yaml @@ -0,0 +1,39 @@ +# Parameters +nc: 80 # number of classes +scales: # model compound scaling constants, i.e. 'model=yolov8n.yaml' will call yolov8.yaml with scale 'n' + # [depth, width, max_channels] + s: [0.33, 0.50, 1024] + +backbone: + # [from, repeats, module, args] + - [-1, 1, Conv, [64, 3, 2]] # 0-P1/2 + - [-1, 1, Conv, [128, 3, 2]] # 1-P2/4 + - [-1, 3, C2f, [128, True]] + - [-1, 1, Conv, [256, 3, 2]] # 3-P3/8 + - [-1, 6, C2f, [256, True]] + - [-1, 1, SCDown, [512, 3, 2]] # 5-P4/16 + - [-1, 6, C2f, [512, True]] + - [-1, 1, SCDown, [1024, 3, 2]] # 7-P5/32 + - [-1, 3, C2fCIB, [1024, True, True]] + - [-1, 1, SPPF, [1024, 5]] # 9 + - [-1, 1, PSA, [1024]] # 10 + +# YOLOv8.0n head +head: + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 6], 1, Concat, [1]] # cat backbone P4 + - [-1, 3, C2f, [512]] # 13 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 4], 1, Concat, [1]] # cat backbone P3 + - [-1, 3, C2f, [256]] # 16 (P3/8-small) + + - [-1, 1, Conv, [256, 3, 2]] + - [[-1, 13], 1, Concat, [1]] # cat head P4 + - [-1, 3, C2f, [512]] # 19 (P4/16-medium) + + - [-1, 1, SCDown, [512, 3, 2]] + - [[-1, 10], 1, Concat, [1]] # cat head P5 + - [-1, 3, C2fCIB, [1024, True, True]] # 22 (P5/32-large) + + - [[16, 19, 22], 1, v10Detect, [nc]] # Detect(P3, P4, P5) diff --git a/ultralytics/cfg/models/v10/yolov10x.yaml b/ultralytics/cfg/models/v10/yolov10x.yaml new file mode 100644 index 0000000..ab5fc8f --- /dev/null +++ b/ultralytics/cfg/models/v10/yolov10x.yaml @@ -0,0 +1,40 @@ +# Parameters +nc: 80 # number of classes +scales: # model compound scaling constants, i.e. 'model=yolov8n.yaml' will call yolov8.yaml with scale 'n' + # [depth, width, max_channels] + x: [1.00, 1.25, 512] + +# YOLOv8.0n backbone +backbone: + # [from, repeats, module, args] + - [-1, 1, Conv, [64, 3, 2]] # 0-P1/2 + - [-1, 1, Conv, [128, 3, 2]] # 1-P2/4 + - [-1, 3, C2f, [128, True]] + - [-1, 1, Conv, [256, 3, 2]] # 3-P3/8 + - [-1, 6, C2f, [256, True]] + - [-1, 1, SCDown, [512, 3, 2]] # 5-P4/16 + - [-1, 6, C2fCIB, [512, True]] + - [-1, 1, SCDown, [1024, 3, 2]] # 7-P5/32 + - [-1, 3, C2fCIB, [1024, True]] + - [-1, 1, SPPF, [1024, 5]] # 9 + - [-1, 1, PSA, [1024]] # 10 + +# YOLOv8.0n head +head: + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 6], 1, Concat, [1]] # cat backbone P4 + - [-1, 3, C2fCIB, [512, True]] # 13 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 4], 1, Concat, [1]] # cat backbone P3 + - [-1, 3, C2f, [256]] # 16 (P3/8-small) + + - [-1, 1, Conv, [256, 3, 2]] + - [[-1, 13], 1, Concat, [1]] # cat head P4 + - [-1, 3, C2fCIB, [512, True]] # 19 (P4/16-medium) + + - [-1, 1, SCDown, [512, 3, 2]] + - [[-1, 10], 1, Concat, [1]] # cat head P5 + - [-1, 3, C2fCIB, [1024, True]] # 22 (P5/32-large) + + - [[16, 19, 22], 1, v10Detect, [nc]] # Detect(P3, P4, P5) diff --git a/ultralytics/cfg/models/v3/yolov3-spp.yaml b/ultralytics/cfg/models/v3/yolov3-spp.yaml new file mode 100644 index 0000000..6724f4e --- /dev/null +++ b/ultralytics/cfg/models/v3/yolov3-spp.yaml @@ -0,0 +1,46 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# YOLOv3-SPP object detection model with P3-P5 outputs. For details see https://docs.ultralytics.com/models/yolov3 + +# Parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple + +# darknet53 backbone +backbone: + # [from, number, module, args] + - [-1, 1, Conv, [32, 3, 1]] # 0 + - [-1, 1, Conv, [64, 3, 2]] # 1-P1/2 + - [-1, 1, Bottleneck, [64]] + - [-1, 1, Conv, [128, 3, 2]] # 3-P2/4 + - [-1, 2, Bottleneck, [128]] + - [-1, 1, Conv, [256, 3, 2]] # 5-P3/8 + - [-1, 8, Bottleneck, [256]] + - [-1, 1, Conv, [512, 3, 2]] # 7-P4/16 + - [-1, 8, Bottleneck, [512]] + - [-1, 1, Conv, [1024, 3, 2]] # 9-P5/32 + - [-1, 4, Bottleneck, [1024]] # 10 + +# YOLOv3-SPP head +head: + - [-1, 1, Bottleneck, [1024, False]] + - [-1, 1, SPP, [512, [5, 9, 13]]] + - [-1, 1, Conv, [1024, 3, 1]] + - [-1, 1, Conv, [512, 1, 1]] + - [-1, 1, Conv, [1024, 3, 1]] # 15 (P5/32-large) + + - [-2, 1, Conv, [256, 1, 1]] + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 8], 1, Concat, [1]] # cat backbone P4 + - [-1, 1, Bottleneck, [512, False]] + - [-1, 1, Bottleneck, [512, False]] + - [-1, 1, Conv, [256, 1, 1]] + - [-1, 1, Conv, [512, 3, 1]] # 22 (P4/16-medium) + + - [-2, 1, Conv, [128, 1, 1]] + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 6], 1, Concat, [1]] # cat backbone P3 + - [-1, 1, Bottleneck, [256, False]] + - [-1, 2, Bottleneck, [256, False]] # 27 (P3/8-small) + + - [[27, 22, 15], 1, Detect, [nc]] # Detect(P3, P4, P5) diff --git a/ultralytics/cfg/models/v3/yolov3-tiny.yaml b/ultralytics/cfg/models/v3/yolov3-tiny.yaml new file mode 100644 index 0000000..f3fe257 --- /dev/null +++ b/ultralytics/cfg/models/v3/yolov3-tiny.yaml @@ -0,0 +1,37 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# YOLOv3-tiny object detection model with P4-P5 outputs. For details see https://docs.ultralytics.com/models/yolov3 + +# Parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple + +# YOLOv3-tiny backbone +backbone: + # [from, number, module, args] + - [-1, 1, Conv, [16, 3, 1]] # 0 + - [-1, 1, nn.MaxPool2d, [2, 2, 0]] # 1-P1/2 + - [-1, 1, Conv, [32, 3, 1]] + - [-1, 1, nn.MaxPool2d, [2, 2, 0]] # 3-P2/4 + - [-1, 1, Conv, [64, 3, 1]] + - [-1, 1, nn.MaxPool2d, [2, 2, 0]] # 5-P3/8 + - [-1, 1, Conv, [128, 3, 1]] + - [-1, 1, nn.MaxPool2d, [2, 2, 0]] # 7-P4/16 + - [-1, 1, Conv, [256, 3, 1]] + - [-1, 1, nn.MaxPool2d, [2, 2, 0]] # 9-P5/32 + - [-1, 1, Conv, [512, 3, 1]] + - [-1, 1, nn.ZeroPad2d, [[0, 1, 0, 1]]] # 11 + - [-1, 1, nn.MaxPool2d, [2, 1, 0]] # 12 + +# YOLOv3-tiny head +head: + - [-1, 1, Conv, [1024, 3, 1]] + - [-1, 1, Conv, [256, 1, 1]] + - [-1, 1, Conv, [512, 3, 1]] # 15 (P5/32-large) + + - [-2, 1, Conv, [128, 1, 1]] + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 8], 1, Concat, [1]] # cat backbone P4 + - [-1, 1, Conv, [256, 3, 1]] # 19 (P4/16-medium) + + - [[19, 15], 1, Detect, [nc]] # Detect(P4, P5) diff --git a/ultralytics/cfg/models/v3/yolov3.yaml b/ultralytics/cfg/models/v3/yolov3.yaml new file mode 100644 index 0000000..716866a --- /dev/null +++ b/ultralytics/cfg/models/v3/yolov3.yaml @@ -0,0 +1,46 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# YOLOv3 object detection model with P3-P5 outputs. For details see https://docs.ultralytics.com/models/yolov3 + +# Parameters +nc: 80 # number of classes +depth_multiple: 1.0 # model depth multiple +width_multiple: 1.0 # layer channel multiple + +# darknet53 backbone +backbone: + # [from, number, module, args] + - [-1, 1, Conv, [32, 3, 1]] # 0 + - [-1, 1, Conv, [64, 3, 2]] # 1-P1/2 + - [-1, 1, Bottleneck, [64]] + - [-1, 1, Conv, [128, 3, 2]] # 3-P2/4 + - [-1, 2, Bottleneck, [128]] + - [-1, 1, Conv, [256, 3, 2]] # 5-P3/8 + - [-1, 8, Bottleneck, [256]] + - [-1, 1, Conv, [512, 3, 2]] # 7-P4/16 + - [-1, 8, Bottleneck, [512]] + - [-1, 1, Conv, [1024, 3, 2]] # 9-P5/32 + - [-1, 4, Bottleneck, [1024]] # 10 + +# YOLOv3 head +head: + - [-1, 1, Bottleneck, [1024, False]] + - [-1, 1, Conv, [512, 1, 1]] + - [-1, 1, Conv, [1024, 3, 1]] + - [-1, 1, Conv, [512, 1, 1]] + - [-1, 1, Conv, [1024, 3, 1]] # 15 (P5/32-large) + + - [-2, 1, Conv, [256, 1, 1]] + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 8], 1, Concat, [1]] # cat backbone P4 + - [-1, 1, Bottleneck, [512, False]] + - [-1, 1, Bottleneck, [512, False]] + - [-1, 1, Conv, [256, 1, 1]] + - [-1, 1, Conv, [512, 3, 1]] # 22 (P4/16-medium) + + - [-2, 1, Conv, [128, 1, 1]] + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 6], 1, Concat, [1]] # cat backbone P3 + - [-1, 1, Bottleneck, [256, False]] + - [-1, 2, Bottleneck, [256, False]] # 27 (P3/8-small) + + - [[27, 22, 15], 1, Detect, [nc]] # Detect(P3, P4, P5) diff --git a/ultralytics/cfg/models/v5/yolov5-p6.yaml b/ultralytics/cfg/models/v5/yolov5-p6.yaml new file mode 100644 index 0000000..2fd3ac7 --- /dev/null +++ b/ultralytics/cfg/models/v5/yolov5-p6.yaml @@ -0,0 +1,59 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# YOLOv5 object detection model with P3-P6 outputs. For details see https://docs.ultralytics.com/models/yolov5 + +# Parameters +nc: 80 # number of classes +scales: # model compound scaling constants, i.e. 'model=yolov5n-p6.yaml' will call yolov5-p6.yaml with scale 'n' + # [depth, width, max_channels] + n: [0.33, 0.25, 1024] + s: [0.33, 0.50, 1024] + m: [0.67, 0.75, 1024] + l: [1.00, 1.00, 1024] + x: [1.33, 1.25, 1024] + +# YOLOv5 v6.0 backbone +backbone: + # [from, number, module, args] + - [-1, 1, Conv, [64, 6, 2, 2]] # 0-P1/2 + - [-1, 1, Conv, [128, 3, 2]] # 1-P2/4 + - [-1, 3, C3, [128]] + - [-1, 1, Conv, [256, 3, 2]] # 3-P3/8 + - [-1, 6, C3, [256]] + - [-1, 1, Conv, [512, 3, 2]] # 5-P4/16 + - [-1, 9, C3, [512]] + - [-1, 1, Conv, [768, 3, 2]] # 7-P5/32 + - [-1, 3, C3, [768]] + - [-1, 1, Conv, [1024, 3, 2]] # 9-P6/64 + - [-1, 3, C3, [1024]] + - [-1, 1, SPPF, [1024, 5]] # 11 + +# YOLOv5 v6.0 head +head: + - [-1, 1, Conv, [768, 1, 1]] + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 8], 1, Concat, [1]] # cat backbone P5 + - [-1, 3, C3, [768, False]] # 15 + + - [-1, 1, Conv, [512, 1, 1]] + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 6], 1, Concat, [1]] # cat backbone P4 + - [-1, 3, C3, [512, False]] # 19 + + - [-1, 1, Conv, [256, 1, 1]] + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 4], 1, Concat, [1]] # cat backbone P3 + - [-1, 3, C3, [256, False]] # 23 (P3/8-small) + + - [-1, 1, Conv, [256, 3, 2]] + - [[-1, 20], 1, Concat, [1]] # cat head P4 + - [-1, 3, C3, [512, False]] # 26 (P4/16-medium) + + - [-1, 1, Conv, [512, 3, 2]] + - [[-1, 16], 1, Concat, [1]] # cat head P5 + - [-1, 3, C3, [768, False]] # 29 (P5/32-large) + + - [-1, 1, Conv, [768, 3, 2]] + - [[-1, 12], 1, Concat, [1]] # cat head P6 + - [-1, 3, C3, [1024, False]] # 32 (P6/64-xlarge) + + - [[23, 26, 29, 32], 1, Detect, [nc]] # Detect(P3, P4, P5, P6) diff --git a/ultralytics/cfg/models/v5/yolov5.yaml b/ultralytics/cfg/models/v5/yolov5.yaml new file mode 100644 index 0000000..8fdc79e --- /dev/null +++ b/ultralytics/cfg/models/v5/yolov5.yaml @@ -0,0 +1,48 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# YOLOv5 object detection model with P3-P5 outputs. For details see https://docs.ultralytics.com/models/yolov5 + +# Parameters +nc: 80 # number of classes +scales: # model compound scaling constants, i.e. 'model=yolov5n.yaml' will call yolov5.yaml with scale 'n' + # [depth, width, max_channels] + n: [0.33, 0.25, 1024] + s: [0.33, 0.50, 1024] + m: [0.67, 0.75, 1024] + l: [1.00, 1.00, 1024] + x: [1.33, 1.25, 1024] + +# YOLOv5 v6.0 backbone +backbone: + # [from, number, module, args] + - [-1, 1, Conv, [64, 6, 2, 2]] # 0-P1/2 + - [-1, 1, Conv, [128, 3, 2]] # 1-P2/4 + - [-1, 3, C3, [128]] + - [-1, 1, Conv, [256, 3, 2]] # 3-P3/8 + - [-1, 6, C3, [256]] + - [-1, 1, Conv, [512, 3, 2]] # 5-P4/16 + - [-1, 9, C3, [512]] + - [-1, 1, Conv, [1024, 3, 2]] # 7-P5/32 + - [-1, 3, C3, [1024]] + - [-1, 1, SPPF, [1024, 5]] # 9 + +# YOLOv5 v6.0 head +head: + - [-1, 1, Conv, [512, 1, 1]] + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 6], 1, Concat, [1]] # cat backbone P4 + - [-1, 3, C3, [512, False]] # 13 + + - [-1, 1, Conv, [256, 1, 1]] + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 4], 1, Concat, [1]] # cat backbone P3 + - [-1, 3, C3, [256, False]] # 17 (P3/8-small) + + - [-1, 1, Conv, [256, 3, 2]] + - [[-1, 14], 1, Concat, [1]] # cat head P4 + - [-1, 3, C3, [512, False]] # 20 (P4/16-medium) + + - [-1, 1, Conv, [512, 3, 2]] + - [[-1, 10], 1, Concat, [1]] # cat head P5 + - [-1, 3, C3, [1024, False]] # 23 (P5/32-large) + + - [[17, 20, 23], 1, Detect, [nc]] # Detect(P3, P4, P5) diff --git a/ultralytics/cfg/models/v6/yolov6.yaml b/ultralytics/cfg/models/v6/yolov6.yaml new file mode 100644 index 0000000..f39dfb4 --- /dev/null +++ b/ultralytics/cfg/models/v6/yolov6.yaml @@ -0,0 +1,53 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# YOLOv6 object detection model with P3-P5 outputs. For Usage examples see https://docs.ultralytics.com/models/yolov6 + +# Parameters +nc: 80 # number of classes +activation: nn.ReLU() # (optional) model default activation function +scales: # model compound scaling constants, i.e. 'model=yolov6n.yaml' will call yolov8.yaml with scale 'n' + # [depth, width, max_channels] + n: [0.33, 0.25, 1024] + s: [0.33, 0.50, 1024] + m: [0.67, 0.75, 768] + l: [1.00, 1.00, 512] + x: [1.00, 1.25, 512] + +# YOLOv6-3.0s backbone +backbone: + # [from, repeats, module, args] + - [-1, 1, Conv, [64, 3, 2]] # 0-P1/2 + - [-1, 1, Conv, [128, 3, 2]] # 1-P2/4 + - [-1, 6, Conv, [128, 3, 1]] + - [-1, 1, Conv, [256, 3, 2]] # 3-P3/8 + - [-1, 12, Conv, [256, 3, 1]] + - [-1, 1, Conv, [512, 3, 2]] # 5-P4/16 + - [-1, 18, Conv, [512, 3, 1]] + - [-1, 1, Conv, [1024, 3, 2]] # 7-P5/32 + - [-1, 6, Conv, [1024, 3, 1]] + - [-1, 1, SPPF, [1024, 5]] # 9 + +# YOLOv6-3.0s head +head: + - [-1, 1, Conv, [256, 1, 1]] + - [-1, 1, nn.ConvTranspose2d, [256, 2, 2, 0]] + - [[-1, 6], 1, Concat, [1]] # cat backbone P4 + - [-1, 1, Conv, [256, 3, 1]] + - [-1, 9, Conv, [256, 3, 1]] # 14 + + - [-1, 1, Conv, [128, 1, 1]] + - [-1, 1, nn.ConvTranspose2d, [128, 2, 2, 0]] + - [[-1, 4], 1, Concat, [1]] # cat backbone P3 + - [-1, 1, Conv, [128, 3, 1]] + - [-1, 9, Conv, [128, 3, 1]] # 19 + + - [-1, 1, Conv, [128, 3, 2]] + - [[-1, 15], 1, Concat, [1]] # cat head P4 + - [-1, 1, Conv, [256, 3, 1]] + - [-1, 9, Conv, [256, 3, 1]] # 23 + + - [-1, 1, Conv, [256, 3, 2]] + - [[-1, 10], 1, Concat, [1]] # cat head P5 + - [-1, 1, Conv, [512, 3, 1]] + - [-1, 9, Conv, [512, 3, 1]] # 27 + + - [[19, 23, 27], 1, Detect, [nc]] # Detect(P3, P4, P5) diff --git a/ultralytics/cfg/models/v8/yolov8-cls-resnet101.yaml b/ultralytics/cfg/models/v8/yolov8-cls-resnet101.yaml new file mode 100644 index 0000000..6867f88 --- /dev/null +++ b/ultralytics/cfg/models/v8/yolov8-cls-resnet101.yaml @@ -0,0 +1,25 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# YOLOv8-cls image classification model. For Usage examples see https://docs.ultralytics.com/tasks/classify + +# Parameters +nc: 1000 # number of classes +scales: # model compound scaling constants, i.e. 'model=yolov8n-cls.yaml' will call yolov8-cls.yaml with scale 'n' + # [depth, width, max_channels] + n: [0.33, 0.25, 1024] + s: [0.33, 0.50, 1024] + m: [0.67, 0.75, 1024] + l: [1.00, 1.00, 1024] + x: [1.00, 1.25, 1024] + +# YOLOv8.0n backbone +backbone: + # [from, repeats, module, args] + - [-1, 1, ResNetLayer, [3, 64, 1, True, 1]] # 0-P1/2 + - [-1, 1, ResNetLayer, [64, 64, 1, False, 3]] # 1-P2/4 + - [-1, 1, ResNetLayer, [256, 128, 2, False, 4]] # 2-P3/8 + - [-1, 1, ResNetLayer, [512, 256, 2, False, 23]] # 3-P4/16 + - [-1, 1, ResNetLayer, [1024, 512, 2, False, 3]] # 4-P5/32 + +# YOLOv8.0n head +head: + - [-1, 1, Classify, [nc]] # Classify diff --git a/ultralytics/cfg/models/v8/yolov8-cls-resnet50.yaml b/ultralytics/cfg/models/v8/yolov8-cls-resnet50.yaml new file mode 100644 index 0000000..8ffd111 --- /dev/null +++ b/ultralytics/cfg/models/v8/yolov8-cls-resnet50.yaml @@ -0,0 +1,25 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# YOLOv8-cls image classification model. For Usage examples see https://docs.ultralytics.com/tasks/classify + +# Parameters +nc: 1000 # number of classes +scales: # model compound scaling constants, i.e. 'model=yolov8n-cls.yaml' will call yolov8-cls.yaml with scale 'n' + # [depth, width, max_channels] + n: [0.33, 0.25, 1024] + s: [0.33, 0.50, 1024] + m: [0.67, 0.75, 1024] + l: [1.00, 1.00, 1024] + x: [1.00, 1.25, 1024] + +# YOLOv8.0n backbone +backbone: + # [from, repeats, module, args] + - [-1, 1, ResNetLayer, [3, 64, 1, True, 1]] # 0-P1/2 + - [-1, 1, ResNetLayer, [64, 64, 1, False, 3]] # 1-P2/4 + - [-1, 1, ResNetLayer, [256, 128, 2, False, 4]] # 2-P3/8 + - [-1, 1, ResNetLayer, [512, 256, 2, False, 6]] # 3-P4/16 + - [-1, 1, ResNetLayer, [1024, 512, 2, False, 3]] # 4-P5/32 + +# YOLOv8.0n head +head: + - [-1, 1, Classify, [nc]] # Classify diff --git a/ultralytics/cfg/models/v8/yolov8-cls.yaml b/ultralytics/cfg/models/v8/yolov8-cls.yaml new file mode 100644 index 0000000..180fc65 --- /dev/null +++ b/ultralytics/cfg/models/v8/yolov8-cls.yaml @@ -0,0 +1,29 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# YOLOv8-cls image classification model. For Usage examples see https://docs.ultralytics.com/tasks/classify + +# Parameters +nc: 1000 # number of classes +scales: # model compound scaling constants, i.e. 'model=yolov8n-cls.yaml' will call yolov8-cls.yaml with scale 'n' + # [depth, width, max_channels] + n: [0.33, 0.25, 1024] + s: [0.33, 0.50, 1024] + m: [0.67, 0.75, 1024] + l: [1.00, 1.00, 1024] + x: [1.00, 1.25, 1024] + +# YOLOv8.0n backbone +backbone: + # [from, repeats, module, args] + - [-1, 1, Conv, [64, 3, 2]] # 0-P1/2 + - [-1, 1, Conv, [128, 3, 2]] # 1-P2/4 + - [-1, 3, C2f, [128, True]] + - [-1, 1, Conv, [256, 3, 2]] # 3-P3/8 + - [-1, 6, C2f, [256, True]] + - [-1, 1, Conv, [512, 3, 2]] # 5-P4/16 + - [-1, 6, C2f, [512, True]] + - [-1, 1, Conv, [1024, 3, 2]] # 7-P5/32 + - [-1, 3, C2f, [1024, True]] + +# YOLOv8.0n head +head: + - [-1, 1, Classify, [nc]] # Classify diff --git a/ultralytics/cfg/models/v8/yolov8-ghost-p2.yaml b/ultralytics/cfg/models/v8/yolov8-ghost-p2.yaml new file mode 100644 index 0000000..aee2093 --- /dev/null +++ b/ultralytics/cfg/models/v8/yolov8-ghost-p2.yaml @@ -0,0 +1,54 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# YOLOv8 object detection model with P2-P5 outputs. For Usage examples see https://docs.ultralytics.com/tasks/detect + +# Parameters +nc: 80 # number of classes +scales: # model compound scaling constants, i.e. 'model=yolov8n.yaml' will call yolov8.yaml with scale 'n' + # [depth, width, max_channels] + n: [0.33, 0.25, 1024] # YOLOv8n-ghost-p2 summary: 491 layers, 2033944 parameters, 2033928 gradients, 13.8 GFLOPs + s: [0.33, 0.50, 1024] # YOLOv8s-ghost-p2 summary: 491 layers, 5562080 parameters, 5562064 gradients, 25.1 GFLOPs + m: [0.67, 0.75, 768] # YOLOv8m-ghost-p2 summary: 731 layers, 9031728 parameters, 9031712 gradients, 42.8 GFLOPs + l: [1.00, 1.00, 512] # YOLOv8l-ghost-p2 summary: 971 layers, 12214448 parameters, 12214432 gradients, 69.1 GFLOPs + x: [1.00, 1.25, 512] # YOLOv8x-ghost-p2 summary: 971 layers, 18664776 parameters, 18664760 gradients, 103.3 GFLOPs + +# YOLOv8.0-ghost backbone +backbone: + # [from, repeats, module, args] + - [-1, 1, Conv, [64, 3, 2]] # 0-P1/2 + - [-1, 1, GhostConv, [128, 3, 2]] # 1-P2/4 + - [-1, 3, C3Ghost, [128, True]] + - [-1, 1, GhostConv, [256, 3, 2]] # 3-P3/8 + - [-1, 6, C3Ghost, [256, True]] + - [-1, 1, GhostConv, [512, 3, 2]] # 5-P4/16 + - [-1, 6, C3Ghost, [512, True]] + - [-1, 1, GhostConv, [1024, 3, 2]] # 7-P5/32 + - [-1, 3, C3Ghost, [1024, True]] + - [-1, 1, SPPF, [1024, 5]] # 9 + +# YOLOv8.0-ghost-p2 head +head: + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 6], 1, Concat, [1]] # cat backbone P4 + - [-1, 3, C3Ghost, [512]] # 12 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 4], 1, Concat, [1]] # cat backbone P3 + - [-1, 3, C3Ghost, [256]] # 15 (P3/8-small) + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 2], 1, Concat, [1]] # cat backbone P2 + - [-1, 3, C3Ghost, [128]] # 18 (P2/4-xsmall) + + - [-1, 1, GhostConv, [128, 3, 2]] + - [[-1, 15], 1, Concat, [1]] # cat head P3 + - [-1, 3, C3Ghost, [256]] # 21 (P3/8-small) + + - [-1, 1, GhostConv, [256, 3, 2]] + - [[-1, 12], 1, Concat, [1]] # cat head P4 + - [-1, 3, C3Ghost, [512]] # 24 (P4/16-medium) + + - [-1, 1, GhostConv, [512, 3, 2]] + - [[-1, 9], 1, Concat, [1]] # cat head P5 + - [-1, 3, C3Ghost, [1024]] # 27 (P5/32-large) + + - [[18, 21, 24, 27], 1, Detect, [nc]] # Detect(P2, P3, P4, P5) diff --git a/ultralytics/cfg/models/v8/yolov8-ghost-p6.yaml b/ultralytics/cfg/models/v8/yolov8-ghost-p6.yaml new file mode 100644 index 0000000..b35f4cd --- /dev/null +++ b/ultralytics/cfg/models/v8/yolov8-ghost-p6.yaml @@ -0,0 +1,56 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# YOLOv8 object detection model with P3-P6 outputs. For Usage examples see https://docs.ultralytics.com/tasks/detect + +# Parameters +nc: 80 # number of classes +scales: # model compound scaling constants, i.e. 'model=yolov8n-p6.yaml' will call yolov8-p6.yaml with scale 'n' + # [depth, width, max_channels] + n: [0.33, 0.25, 1024] # YOLOv8n-ghost-p6 summary: 529 layers, 2901100 parameters, 2901084 gradients, 5.8 GFLOPs + s: [0.33, 0.50, 1024] # YOLOv8s-ghost-p6 summary: 529 layers, 9520008 parameters, 9519992 gradients, 16.4 GFLOPs + m: [0.67, 0.75, 768] # YOLOv8m-ghost-p6 summary: 789 layers, 18002904 parameters, 18002888 gradients, 34.4 GFLOPs + l: [1.00, 1.00, 512] # YOLOv8l-ghost-p6 summary: 1049 layers, 21227584 parameters, 21227568 gradients, 55.3 GFLOPs + x: [1.00, 1.25, 512] # YOLOv8x-ghost-p6 summary: 1049 layers, 33057852 parameters, 33057836 gradients, 85.7 GFLOPs + +# YOLOv8.0-ghost backbone +backbone: + # [from, repeats, module, args] + - [-1, 1, Conv, [64, 3, 2]] # 0-P1/2 + - [-1, 1, GhostConv, [128, 3, 2]] # 1-P2/4 + - [-1, 3, C3Ghost, [128, True]] + - [-1, 1, GhostConv, [256, 3, 2]] # 3-P3/8 + - [-1, 6, C3Ghost, [256, True]] + - [-1, 1, GhostConv, [512, 3, 2]] # 5-P4/16 + - [-1, 6, C3Ghost, [512, True]] + - [-1, 1, GhostConv, [768, 3, 2]] # 7-P5/32 + - [-1, 3, C3Ghost, [768, True]] + - [-1, 1, GhostConv, [1024, 3, 2]] # 9-P6/64 + - [-1, 3, C3Ghost, [1024, True]] + - [-1, 1, SPPF, [1024, 5]] # 11 + +# YOLOv8.0-ghost-p6 head +head: + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 8], 1, Concat, [1]] # cat backbone P5 + - [-1, 3, C3Ghost, [768]] # 14 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 6], 1, Concat, [1]] # cat backbone P4 + - [-1, 3, C3Ghost, [512]] # 17 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 4], 1, Concat, [1]] # cat backbone P3 + - [-1, 3, C3Ghost, [256]] # 20 (P3/8-small) + + - [-1, 1, GhostConv, [256, 3, 2]] + - [[-1, 17], 1, Concat, [1]] # cat head P4 + - [-1, 3, C3Ghost, [512]] # 23 (P4/16-medium) + + - [-1, 1, GhostConv, [512, 3, 2]] + - [[-1, 14], 1, Concat, [1]] # cat head P5 + - [-1, 3, C3Ghost, [768]] # 26 (P5/32-large) + + - [-1, 1, GhostConv, [768, 3, 2]] + - [[-1, 11], 1, Concat, [1]] # cat head P6 + - [-1, 3, C3Ghost, [1024]] # 29 (P6/64-xlarge) + + - [[20, 23, 26, 29], 1, Detect, [nc]] # Detect(P3, P4, P5, P6) diff --git a/ultralytics/cfg/models/v8/yolov8-ghost.yaml b/ultralytics/cfg/models/v8/yolov8-ghost.yaml new file mode 100644 index 0000000..adc1802 --- /dev/null +++ b/ultralytics/cfg/models/v8/yolov8-ghost.yaml @@ -0,0 +1,47 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# YOLOv8 object detection model with P3-P5 outputs. For Usage examples see https://docs.ultralytics.com/tasks/detect +# Employs Ghost convolutions and modules proposed in Huawei's GhostNet in https://arxiv.org/abs/1911.11907v2 + +# Parameters +nc: 80 # number of classes +scales: # model compound scaling constants, i.e. 'model=yolov8n.yaml' will call yolov8.yaml with scale 'n' + # [depth, width, max_channels] + n: [0.33, 0.25, 1024] # YOLOv8n-ghost summary: 403 layers, 1865316 parameters, 1865300 gradients, 5.8 GFLOPs + s: [0.33, 0.50, 1024] # YOLOv8s-ghost summary: 403 layers, 5960072 parameters, 5960056 gradients, 16.4 GFLOPs + m: [0.67, 0.75, 768] # YOLOv8m-ghost summary: 603 layers, 10336312 parameters, 10336296 gradients, 32.7 GFLOPs + l: [1.00, 1.00, 512] # YOLOv8l-ghost summary: 803 layers, 14277872 parameters, 14277856 gradients, 53.7 GFLOPs + x: [1.00, 1.25, 512] # YOLOv8x-ghost summary: 803 layers, 22229308 parameters, 22229292 gradients, 83.3 GFLOPs + +# YOLOv8.0n-ghost backbone +backbone: + # [from, repeats, module, args] + - [-1, 1, Conv, [64, 3, 2]] # 0-P1/2 + - [-1, 1, GhostConv, [128, 3, 2]] # 1-P2/4 + - [-1, 3, C3Ghost, [128, True]] + - [-1, 1, GhostConv, [256, 3, 2]] # 3-P3/8 + - [-1, 6, C3Ghost, [256, True]] + - [-1, 1, GhostConv, [512, 3, 2]] # 5-P4/16 + - [-1, 6, C3Ghost, [512, True]] + - [-1, 1, GhostConv, [1024, 3, 2]] # 7-P5/32 + - [-1, 3, C3Ghost, [1024, True]] + - [-1, 1, SPPF, [1024, 5]] # 9 + +# YOLOv8.0n head +head: + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 6], 1, Concat, [1]] # cat backbone P4 + - [-1, 3, C3Ghost, [512]] # 12 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 4], 1, Concat, [1]] # cat backbone P3 + - [-1, 3, C3Ghost, [256]] # 15 (P3/8-small) + + - [-1, 1, GhostConv, [256, 3, 2]] + - [[-1, 12], 1, Concat, [1]] # cat head P4 + - [-1, 3, C3Ghost, [512]] # 18 (P4/16-medium) + + - [-1, 1, GhostConv, [512, 3, 2]] + - [[-1, 9], 1, Concat, [1]] # cat head P5 + - [-1, 3, C3Ghost, [1024]] # 21 (P5/32-large) + + - [[15, 18, 21], 1, Detect, [nc]] # Detect(P3, P4, P5) diff --git a/ultralytics/cfg/models/v8/yolov8-obb.yaml b/ultralytics/cfg/models/v8/yolov8-obb.yaml new file mode 100644 index 0000000..7a7f60c --- /dev/null +++ b/ultralytics/cfg/models/v8/yolov8-obb.yaml @@ -0,0 +1,46 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# YOLOv8 Oriented Bounding Boxes (OBB) model with P3-P5 outputs. For Usage examples see https://docs.ultralytics.com/tasks/detect + +# Parameters +nc: 80 # number of classes +scales: # model compound scaling constants, i.e. 'model=yolov8n.yaml' will call yolov8.yaml with scale 'n' + # [depth, width, max_channels] + n: [0.33, 0.25, 1024] # YOLOv8n summary: 225 layers, 3157200 parameters, 3157184 gradients, 8.9 GFLOPs + s: [0.33, 0.50, 1024] # YOLOv8s summary: 225 layers, 11166560 parameters, 11166544 gradients, 28.8 GFLOPs + m: [0.67, 0.75, 768] # YOLOv8m summary: 295 layers, 25902640 parameters, 25902624 gradients, 79.3 GFLOPs + l: [1.00, 1.00, 512] # YOLOv8l summary: 365 layers, 43691520 parameters, 43691504 gradients, 165.7 GFLOPs + x: [1.00, 1.25, 512] # YOLOv8x summary: 365 layers, 68229648 parameters, 68229632 gradients, 258.5 GFLOPs + +# YOLOv8.0n backbone +backbone: + # [from, repeats, module, args] + - [-1, 1, Conv, [64, 3, 2]] # 0-P1/2 + - [-1, 1, Conv, [128, 3, 2]] # 1-P2/4 + - [-1, 3, C2f, [128, True]] + - [-1, 1, Conv, [256, 3, 2]] # 3-P3/8 + - [-1, 6, C2f, [256, True]] + - [-1, 1, Conv, [512, 3, 2]] # 5-P4/16 + - [-1, 6, C2f, [512, True]] + - [-1, 1, Conv, [1024, 3, 2]] # 7-P5/32 + - [-1, 3, C2f, [1024, True]] + - [-1, 1, SPPF, [1024, 5]] # 9 + +# YOLOv8.0n head +head: + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 6], 1, Concat, [1]] # cat backbone P4 + - [-1, 3, C2f, [512]] # 12 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 4], 1, Concat, [1]] # cat backbone P3 + - [-1, 3, C2f, [256]] # 15 (P3/8-small) + + - [-1, 1, Conv, [256, 3, 2]] + - [[-1, 12], 1, Concat, [1]] # cat head P4 + - [-1, 3, C2f, [512]] # 18 (P4/16-medium) + + - [-1, 1, Conv, [512, 3, 2]] + - [[-1, 9], 1, Concat, [1]] # cat head P5 + - [-1, 3, C2f, [1024]] # 21 (P5/32-large) + + - [[15, 18, 21], 1, OBB, [nc, 1]] # OBB(P3, P4, P5) diff --git a/ultralytics/cfg/models/v8/yolov8-p2.yaml b/ultralytics/cfg/models/v8/yolov8-p2.yaml new file mode 100644 index 0000000..5392774 --- /dev/null +++ b/ultralytics/cfg/models/v8/yolov8-p2.yaml @@ -0,0 +1,54 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# YOLOv8 object detection model with P2-P5 outputs. For Usage examples see https://docs.ultralytics.com/tasks/detect + +# Parameters +nc: 80 # number of classes +scales: # model compound scaling constants, i.e. 'model=yolov8n.yaml' will call yolov8.yaml with scale 'n' + # [depth, width, max_channels] + n: [0.33, 0.25, 1024] + s: [0.33, 0.50, 1024] + m: [0.67, 0.75, 768] + l: [1.00, 1.00, 512] + x: [1.00, 1.25, 512] + +# YOLOv8.0 backbone +backbone: + # [from, repeats, module, args] + - [-1, 1, Conv, [64, 3, 2]] # 0-P1/2 + - [-1, 1, Conv, [128, 3, 2]] # 1-P2/4 + - [-1, 3, C2f, [128, True]] + - [-1, 1, Conv, [256, 3, 2]] # 3-P3/8 + - [-1, 6, C2f, [256, True]] + - [-1, 1, Conv, [512, 3, 2]] # 5-P4/16 + - [-1, 6, C2f, [512, True]] + - [-1, 1, Conv, [1024, 3, 2]] # 7-P5/32 + - [-1, 3, C2f, [1024, True]] + - [-1, 1, SPPF, [1024, 5]] # 9 + +# YOLOv8.0-p2 head +head: + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 6], 1, Concat, [1]] # cat backbone P4 + - [-1, 3, C2f, [512]] # 12 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 4], 1, Concat, [1]] # cat backbone P3 + - [-1, 3, C2f, [256]] # 15 (P3/8-small) + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 2], 1, Concat, [1]] # cat backbone P2 + - [-1, 3, C2f, [128]] # 18 (P2/4-xsmall) + + - [-1, 1, Conv, [128, 3, 2]] + - [[-1, 15], 1, Concat, [1]] # cat head P3 + - [-1, 3, C2f, [256]] # 21 (P3/8-small) + + - [-1, 1, Conv, [256, 3, 2]] + - [[-1, 12], 1, Concat, [1]] # cat head P4 + - [-1, 3, C2f, [512]] # 24 (P4/16-medium) + + - [-1, 1, Conv, [512, 3, 2]] + - [[-1, 9], 1, Concat, [1]] # cat head P5 + - [-1, 3, C2f, [1024]] # 27 (P5/32-large) + + - [[18, 21, 24, 27], 1, Detect, [nc]] # Detect(P2, P3, P4, P5) diff --git a/ultralytics/cfg/models/v8/yolov8-p6.yaml b/ultralytics/cfg/models/v8/yolov8-p6.yaml new file mode 100644 index 0000000..2d6d5f9 --- /dev/null +++ b/ultralytics/cfg/models/v8/yolov8-p6.yaml @@ -0,0 +1,56 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# YOLOv8 object detection model with P3-P6 outputs. For Usage examples see https://docs.ultralytics.com/tasks/detect + +# Parameters +nc: 80 # number of classes +scales: # model compound scaling constants, i.e. 'model=yolov8n-p6.yaml' will call yolov8-p6.yaml with scale 'n' + # [depth, width, max_channels] + n: [0.33, 0.25, 1024] + s: [0.33, 0.50, 1024] + m: [0.67, 0.75, 768] + l: [1.00, 1.00, 512] + x: [1.00, 1.25, 512] + +# YOLOv8.0x6 backbone +backbone: + # [from, repeats, module, args] + - [-1, 1, Conv, [64, 3, 2]] # 0-P1/2 + - [-1, 1, Conv, [128, 3, 2]] # 1-P2/4 + - [-1, 3, C2f, [128, True]] + - [-1, 1, Conv, [256, 3, 2]] # 3-P3/8 + - [-1, 6, C2f, [256, True]] + - [-1, 1, Conv, [512, 3, 2]] # 5-P4/16 + - [-1, 6, C2f, [512, True]] + - [-1, 1, Conv, [768, 3, 2]] # 7-P5/32 + - [-1, 3, C2f, [768, True]] + - [-1, 1, Conv, [1024, 3, 2]] # 9-P6/64 + - [-1, 3, C2f, [1024, True]] + - [-1, 1, SPPF, [1024, 5]] # 11 + +# YOLOv8.0x6 head +head: + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 8], 1, Concat, [1]] # cat backbone P5 + - [-1, 3, C2, [768, False]] # 14 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 6], 1, Concat, [1]] # cat backbone P4 + - [-1, 3, C2, [512, False]] # 17 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 4], 1, Concat, [1]] # cat backbone P3 + - [-1, 3, C2, [256, False]] # 20 (P3/8-small) + + - [-1, 1, Conv, [256, 3, 2]] + - [[-1, 17], 1, Concat, [1]] # cat head P4 + - [-1, 3, C2, [512, False]] # 23 (P4/16-medium) + + - [-1, 1, Conv, [512, 3, 2]] + - [[-1, 14], 1, Concat, [1]] # cat head P5 + - [-1, 3, C2, [768, False]] # 26 (P5/32-large) + + - [-1, 1, Conv, [768, 3, 2]] + - [[-1, 11], 1, Concat, [1]] # cat head P6 + - [-1, 3, C2, [1024, False]] # 29 (P6/64-xlarge) + + - [[20, 23, 26, 29], 1, Detect, [nc]] # Detect(P3, P4, P5, P6) diff --git a/ultralytics/cfg/models/v8/yolov8-pose-p6.yaml b/ultralytics/cfg/models/v8/yolov8-pose-p6.yaml new file mode 100644 index 0000000..60007ac --- /dev/null +++ b/ultralytics/cfg/models/v8/yolov8-pose-p6.yaml @@ -0,0 +1,57 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# YOLOv8-pose-p6 keypoints/pose estimation model. For Usage examples see https://docs.ultralytics.com/tasks/pose + +# Parameters +nc: 1 # number of classes +kpt_shape: [17, 3] # number of keypoints, number of dims (2 for x,y or 3 for x,y,visible) +scales: # model compound scaling constants, i.e. 'model=yolov8n-p6.yaml' will call yolov8-p6.yaml with scale 'n' + # [depth, width, max_channels] + n: [0.33, 0.25, 1024] + s: [0.33, 0.50, 1024] + m: [0.67, 0.75, 768] + l: [1.00, 1.00, 512] + x: [1.00, 1.25, 512] + +# YOLOv8.0x6 backbone +backbone: + # [from, repeats, module, args] + - [-1, 1, Conv, [64, 3, 2]] # 0-P1/2 + - [-1, 1, Conv, [128, 3, 2]] # 1-P2/4 + - [-1, 3, C2f, [128, True]] + - [-1, 1, Conv, [256, 3, 2]] # 3-P3/8 + - [-1, 6, C2f, [256, True]] + - [-1, 1, Conv, [512, 3, 2]] # 5-P4/16 + - [-1, 6, C2f, [512, True]] + - [-1, 1, Conv, [768, 3, 2]] # 7-P5/32 + - [-1, 3, C2f, [768, True]] + - [-1, 1, Conv, [1024, 3, 2]] # 9-P6/64 + - [-1, 3, C2f, [1024, True]] + - [-1, 1, SPPF, [1024, 5]] # 11 + +# YOLOv8.0x6 head +head: + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 8], 1, Concat, [1]] # cat backbone P5 + - [-1, 3, C2, [768, False]] # 14 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 6], 1, Concat, [1]] # cat backbone P4 + - [-1, 3, C2, [512, False]] # 17 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 4], 1, Concat, [1]] # cat backbone P3 + - [-1, 3, C2, [256, False]] # 20 (P3/8-small) + + - [-1, 1, Conv, [256, 3, 2]] + - [[-1, 17], 1, Concat, [1]] # cat head P4 + - [-1, 3, C2, [512, False]] # 23 (P4/16-medium) + + - [-1, 1, Conv, [512, 3, 2]] + - [[-1, 14], 1, Concat, [1]] # cat head P5 + - [-1, 3, C2, [768, False]] # 26 (P5/32-large) + + - [-1, 1, Conv, [768, 3, 2]] + - [[-1, 11], 1, Concat, [1]] # cat head P6 + - [-1, 3, C2, [1024, False]] # 29 (P6/64-xlarge) + + - [[20, 23, 26, 29], 1, Pose, [nc, kpt_shape]] # Pose(P3, P4, P5, P6) diff --git a/ultralytics/cfg/models/v8/yolov8-pose.yaml b/ultralytics/cfg/models/v8/yolov8-pose.yaml new file mode 100644 index 0000000..60388ef --- /dev/null +++ b/ultralytics/cfg/models/v8/yolov8-pose.yaml @@ -0,0 +1,47 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# YOLOv8-pose keypoints/pose estimation model. For Usage examples see https://docs.ultralytics.com/tasks/pose + +# Parameters +nc: 1 # number of classes +kpt_shape: [17, 3] # number of keypoints, number of dims (2 for x,y or 3 for x,y,visible) +scales: # model compound scaling constants, i.e. 'model=yolov8n-pose.yaml' will call yolov8-pose.yaml with scale 'n' + # [depth, width, max_channels] + n: [0.33, 0.25, 1024] + s: [0.33, 0.50, 1024] + m: [0.67, 0.75, 768] + l: [1.00, 1.00, 512] + x: [1.00, 1.25, 512] + +# YOLOv8.0n backbone +backbone: + # [from, repeats, module, args] + - [-1, 1, Conv, [64, 3, 2]] # 0-P1/2 + - [-1, 1, Conv, [128, 3, 2]] # 1-P2/4 + - [-1, 3, C2f, [128, True]] + - [-1, 1, Conv, [256, 3, 2]] # 3-P3/8 + - [-1, 6, C2f, [256, True]] + - [-1, 1, Conv, [512, 3, 2]] # 5-P4/16 + - [-1, 6, C2f, [512, True]] + - [-1, 1, Conv, [1024, 3, 2]] # 7-P5/32 + - [-1, 3, C2f, [1024, True]] + - [-1, 1, SPPF, [1024, 5]] # 9 + +# YOLOv8.0n head +head: + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 6], 1, Concat, [1]] # cat backbone P4 + - [-1, 3, C2f, [512]] # 12 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 4], 1, Concat, [1]] # cat backbone P3 + - [-1, 3, C2f, [256]] # 15 (P3/8-small) + + - [-1, 1, Conv, [256, 3, 2]] + - [[-1, 12], 1, Concat, [1]] # cat head P4 + - [-1, 3, C2f, [512]] # 18 (P4/16-medium) + + - [-1, 1, Conv, [512, 3, 2]] + - [[-1, 9], 1, Concat, [1]] # cat head P5 + - [-1, 3, C2f, [1024]] # 21 (P5/32-large) + + - [[15, 18, 21], 1, Pose, [nc, kpt_shape]] # Pose(P3, P4, P5) diff --git a/ultralytics/cfg/models/v8/yolov8-rtdetr.yaml b/ultralytics/cfg/models/v8/yolov8-rtdetr.yaml new file mode 100644 index 0000000..27b790b --- /dev/null +++ b/ultralytics/cfg/models/v8/yolov8-rtdetr.yaml @@ -0,0 +1,46 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# YOLOv8 object detection model with P3-P5 outputs. For Usage examples see https://docs.ultralytics.com/tasks/detect + +# Parameters +nc: 80 # number of classes +scales: # model compound scaling constants, i.e. 'model=yolov8n.yaml' will call yolov8.yaml with scale 'n' + # [depth, width, max_channels] + n: [0.33, 0.25, 1024] # YOLOv8n summary: 225 layers, 3157200 parameters, 3157184 gradients, 8.9 GFLOPs + s: [0.33, 0.50, 1024] # YOLOv8s summary: 225 layers, 11166560 parameters, 11166544 gradients, 28.8 GFLOPs + m: [0.67, 0.75, 768] # YOLOv8m summary: 295 layers, 25902640 parameters, 25902624 gradients, 79.3 GFLOPs + l: [1.00, 1.00, 512] # YOLOv8l summary: 365 layers, 43691520 parameters, 43691504 gradients, 165.7 GFLOPs + x: [1.00, 1.25, 512] # YOLOv8x summary: 365 layers, 68229648 parameters, 68229632 gradients, 258.5 GFLOPs + +# YOLOv8.0n backbone +backbone: + # [from, repeats, module, args] + - [-1, 1, Conv, [64, 3, 2]] # 0-P1/2 + - [-1, 1, Conv, [128, 3, 2]] # 1-P2/4 + - [-1, 3, C2f, [128, True]] + - [-1, 1, Conv, [256, 3, 2]] # 3-P3/8 + - [-1, 6, C2f, [256, True]] + - [-1, 1, Conv, [512, 3, 2]] # 5-P4/16 + - [-1, 6, C2f, [512, True]] + - [-1, 1, Conv, [1024, 3, 2]] # 7-P5/32 + - [-1, 3, C2f, [1024, True]] + - [-1, 1, SPPF, [1024, 5]] # 9 + +# YOLOv8.0n head +head: + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 6], 1, Concat, [1]] # cat backbone P4 + - [-1, 3, C2f, [512]] # 12 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 4], 1, Concat, [1]] # cat backbone P3 + - [-1, 3, C2f, [256]] # 15 (P3/8-small) + + - [-1, 1, Conv, [256, 3, 2]] + - [[-1, 12], 1, Concat, [1]] # cat head P4 + - [-1, 3, C2f, [512]] # 18 (P4/16-medium) + + - [-1, 1, Conv, [512, 3, 2]] + - [[-1, 9], 1, Concat, [1]] # cat head P5 + - [-1, 3, C2f, [1024]] # 21 (P5/32-large) + + - [[15, 18, 21], 1, RTDETRDecoder, [nc]] # Detect(P3, P4, P5) diff --git a/ultralytics/cfg/models/v8/yolov8-seg-p6.yaml b/ultralytics/cfg/models/v8/yolov8-seg-p6.yaml new file mode 100644 index 0000000..78c0444 --- /dev/null +++ b/ultralytics/cfg/models/v8/yolov8-seg-p6.yaml @@ -0,0 +1,56 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# YOLOv8-seg-p6 instance segmentation model. For Usage examples see https://docs.ultralytics.com/tasks/segment + +# Parameters +nc: 80 # number of classes +scales: # model compound scaling constants, i.e. 'model=yolov8n-seg-p6.yaml' will call yolov8-seg-p6.yaml with scale 'n' + # [depth, width, max_channels] + n: [0.33, 0.25, 1024] + s: [0.33, 0.50, 1024] + m: [0.67, 0.75, 768] + l: [1.00, 1.00, 512] + x: [1.00, 1.25, 512] + +# YOLOv8.0x6 backbone +backbone: + # [from, repeats, module, args] + - [-1, 1, Conv, [64, 3, 2]] # 0-P1/2 + - [-1, 1, Conv, [128, 3, 2]] # 1-P2/4 + - [-1, 3, C2f, [128, True]] + - [-1, 1, Conv, [256, 3, 2]] # 3-P3/8 + - [-1, 6, C2f, [256, True]] + - [-1, 1, Conv, [512, 3, 2]] # 5-P4/16 + - [-1, 6, C2f, [512, True]] + - [-1, 1, Conv, [768, 3, 2]] # 7-P5/32 + - [-1, 3, C2f, [768, True]] + - [-1, 1, Conv, [1024, 3, 2]] # 9-P6/64 + - [-1, 3, C2f, [1024, True]] + - [-1, 1, SPPF, [1024, 5]] # 11 + +# YOLOv8.0x6 head +head: + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 8], 1, Concat, [1]] # cat backbone P5 + - [-1, 3, C2, [768, False]] # 14 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 6], 1, Concat, [1]] # cat backbone P4 + - [-1, 3, C2, [512, False]] # 17 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 4], 1, Concat, [1]] # cat backbone P3 + - [-1, 3, C2, [256, False]] # 20 (P3/8-small) + + - [-1, 1, Conv, [256, 3, 2]] + - [[-1, 17], 1, Concat, [1]] # cat head P4 + - [-1, 3, C2, [512, False]] # 23 (P4/16-medium) + + - [-1, 1, Conv, [512, 3, 2]] + - [[-1, 14], 1, Concat, [1]] # cat head P5 + - [-1, 3, C2, [768, False]] # 26 (P5/32-large) + + - [-1, 1, Conv, [768, 3, 2]] + - [[-1, 11], 1, Concat, [1]] # cat head P6 + - [-1, 3, C2, [1024, False]] # 29 (P6/64-xlarge) + + - [[20, 23, 26, 29], 1, Segment, [nc, 32, 256]] # Pose(P3, P4, P5, P6) diff --git a/ultralytics/cfg/models/v8/yolov8-seg.yaml b/ultralytics/cfg/models/v8/yolov8-seg.yaml new file mode 100644 index 0000000..700b795 --- /dev/null +++ b/ultralytics/cfg/models/v8/yolov8-seg.yaml @@ -0,0 +1,46 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# YOLOv8-seg instance segmentation model. For Usage examples see https://docs.ultralytics.com/tasks/segment + +# Parameters +nc: 80 # number of classes +scales: # model compound scaling constants, i.e. 'model=yolov8n-seg.yaml' will call yolov8-seg.yaml with scale 'n' + # [depth, width, max_channels] + n: [0.33, 0.25, 1024] + s: [0.33, 0.50, 1024] + m: [0.67, 0.75, 768] + l: [1.00, 1.00, 512] + x: [1.00, 1.25, 512] + +# YOLOv8.0n backbone +backbone: + # [from, repeats, module, args] + - [-1, 1, Conv, [64, 3, 2]] # 0-P1/2 + - [-1, 1, Conv, [128, 3, 2]] # 1-P2/4 + - [-1, 3, C2f, [128, True]] + - [-1, 1, Conv, [256, 3, 2]] # 3-P3/8 + - [-1, 6, C2f, [256, True]] + - [-1, 1, Conv, [512, 3, 2]] # 5-P4/16 + - [-1, 6, C2f, [512, True]] + - [-1, 1, Conv, [1024, 3, 2]] # 7-P5/32 + - [-1, 3, C2f, [1024, True]] + - [-1, 1, SPPF, [1024, 5]] # 9 + +# YOLOv8.0n head +head: + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 6], 1, Concat, [1]] # cat backbone P4 + - [-1, 3, C2f, [512]] # 12 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 4], 1, Concat, [1]] # cat backbone P3 + - [-1, 3, C2f, [256]] # 15 (P3/8-small) + + - [-1, 1, Conv, [256, 3, 2]] + - [[-1, 12], 1, Concat, [1]] # cat head P4 + - [-1, 3, C2f, [512]] # 18 (P4/16-medium) + + - [-1, 1, Conv, [512, 3, 2]] + - [[-1, 9], 1, Concat, [1]] # cat head P5 + - [-1, 3, C2f, [1024]] # 21 (P5/32-large) + + - [[15, 18, 21], 1, Segment, [nc, 32, 256]] # Segment(P3, P4, P5) diff --git a/ultralytics/cfg/models/v8/yolov8-world.yaml b/ultralytics/cfg/models/v8/yolov8-world.yaml new file mode 100644 index 0000000..c21a7f0 --- /dev/null +++ b/ultralytics/cfg/models/v8/yolov8-world.yaml @@ -0,0 +1,48 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# YOLOv8-World object detection model with P3-P5 outputs. For details see https://docs.ultralytics.com/tasks/detect + +# Parameters +nc: 80 # number of classes +scales: # model compound scaling constants, i.e. 'model=yolov8n.yaml' will call yolov8.yaml with scale 'n' + # [depth, width, max_channels] + n: [0.33, 0.25, 1024] # YOLOv8n summary: 225 layers, 3157200 parameters, 3157184 gradients, 8.9 GFLOPs + s: [0.33, 0.50, 1024] # YOLOv8s summary: 225 layers, 11166560 parameters, 11166544 gradients, 28.8 GFLOPs + m: [0.67, 0.75, 768] # YOLOv8m summary: 295 layers, 25902640 parameters, 25902624 gradients, 79.3 GFLOPs + l: [1.00, 1.00, 512] # YOLOv8l summary: 365 layers, 43691520 parameters, 43691504 gradients, 165.7 GFLOPs + x: [1.00, 1.25, 512] # YOLOv8x summary: 365 layers, 68229648 parameters, 68229632 gradients, 258.5 GFLOPs + +# YOLOv8.0n backbone +backbone: + # [from, repeats, module, args] + - [-1, 1, Conv, [64, 3, 2]] # 0-P1/2 + - [-1, 1, Conv, [128, 3, 2]] # 1-P2/4 + - [-1, 3, C2f, [128, True]] + - [-1, 1, Conv, [256, 3, 2]] # 3-P3/8 + - [-1, 6, C2f, [256, True]] + - [-1, 1, Conv, [512, 3, 2]] # 5-P4/16 + - [-1, 6, C2f, [512, True]] + - [-1, 1, Conv, [1024, 3, 2]] # 7-P5/32 + - [-1, 3, C2f, [1024, True]] + - [-1, 1, SPPF, [1024, 5]] # 9 + +# YOLOv8.0n head +head: + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 6], 1, Concat, [1]] # cat backbone P4 + - [-1, 3, C2fAttn, [512, 256, 8]] # 12 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 4], 1, Concat, [1]] # cat backbone P3 + - [-1, 3, C2fAttn, [256, 128, 4]] # 15 (P3/8-small) + + - [[15, 12, 9], 1, ImagePoolingAttn, [256]] # 16 (P3/8-small) + + - [15, 1, Conv, [256, 3, 2]] + - [[-1, 12], 1, Concat, [1]] # cat head P4 + - [-1, 3, C2fAttn, [512, 256, 8]] # 19 (P4/16-medium) + + - [-1, 1, Conv, [512, 3, 2]] + - [[-1, 9], 1, Concat, [1]] # cat head P5 + - [-1, 3, C2fAttn, [1024, 512, 16]] # 22 (P5/32-large) + + - [[15, 19, 22], 1, WorldDetect, [nc, 512, False]] # Detect(P3, P4, P5) diff --git a/ultralytics/cfg/models/v8/yolov8-worldv2.yaml b/ultralytics/cfg/models/v8/yolov8-worldv2.yaml new file mode 100644 index 0000000..322b97d --- /dev/null +++ b/ultralytics/cfg/models/v8/yolov8-worldv2.yaml @@ -0,0 +1,46 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# YOLOv8-World-v2 object detection model with P3-P5 outputs. For details see https://docs.ultralytics.com/tasks/detect + +# Parameters +nc: 80 # number of classes +scales: # model compound scaling constants, i.e. 'model=yolov8n.yaml' will call yolov8.yaml with scale 'n' + # [depth, width, max_channels] + n: [0.33, 0.25, 1024] # YOLOv8n summary: 225 layers, 3157200 parameters, 3157184 gradients, 8.9 GFLOPs + s: [0.33, 0.50, 1024] # YOLOv8s summary: 225 layers, 11166560 parameters, 11166544 gradients, 28.8 GFLOPs + m: [0.67, 0.75, 768] # YOLOv8m summary: 295 layers, 25902640 parameters, 25902624 gradients, 79.3 GFLOPs + l: [1.00, 1.00, 512] # YOLOv8l summary: 365 layers, 43691520 parameters, 43691504 gradients, 165.7 GFLOPs + x: [1.00, 1.25, 512] # YOLOv8x summary: 365 layers, 68229648 parameters, 68229632 gradients, 258.5 GFLOPs + +# YOLOv8.0n backbone +backbone: + # [from, repeats, module, args] + - [-1, 1, Conv, [64, 3, 2]] # 0-P1/2 + - [-1, 1, Conv, [128, 3, 2]] # 1-P2/4 + - [-1, 3, C2f, [128, True]] + - [-1, 1, Conv, [256, 3, 2]] # 3-P3/8 + - [-1, 6, C2f, [256, True]] + - [-1, 1, Conv, [512, 3, 2]] # 5-P4/16 + - [-1, 6, C2f, [512, True]] + - [-1, 1, Conv, [1024, 3, 2]] # 7-P5/32 + - [-1, 3, C2f, [1024, True]] + - [-1, 1, SPPF, [1024, 5]] # 9 + +# YOLOv8.0n head +head: + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 6], 1, Concat, [1]] # cat backbone P4 + - [-1, 3, C2fAttn, [512, 256, 8]] # 12 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 4], 1, Concat, [1]] # cat backbone P3 + - [-1, 3, C2fAttn, [256, 128, 4]] # 15 (P3/8-small) + + - [15, 1, Conv, [256, 3, 2]] + - [[-1, 12], 1, Concat, [1]] # cat head P4 + - [-1, 3, C2fAttn, [512, 256, 8]] # 18 (P4/16-medium) + + - [-1, 1, Conv, [512, 3, 2]] + - [[-1, 9], 1, Concat, [1]] # cat head P5 + - [-1, 3, C2fAttn, [1024, 512, 16]] # 21 (P5/32-large) + + - [[15, 18, 21], 1, WorldDetect, [nc, 512, True]] # Detect(P3, P4, P5) diff --git a/ultralytics/cfg/models/v8/yolov8.yaml b/ultralytics/cfg/models/v8/yolov8.yaml new file mode 100644 index 0000000..b328e98 --- /dev/null +++ b/ultralytics/cfg/models/v8/yolov8.yaml @@ -0,0 +1,46 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# YOLOv8 object detection model with P3-P5 outputs. For Usage examples see https://docs.ultralytics.com/tasks/detect + +# Parameters +nc: 80 # number of classes +scales: # model compound scaling constants, i.e. 'model=yolov8n.yaml' will call yolov8.yaml with scale 'n' + # [depth, width, max_channels] + n: [0.33, 0.25, 1024] # YOLOv8n summary: 225 layers, 3157200 parameters, 3157184 gradients, 8.9 GFLOPs + s: [0.33, 0.50, 1024] # YOLOv8s summary: 225 layers, 11166560 parameters, 11166544 gradients, 28.8 GFLOPs + m: [0.67, 0.75, 768] # YOLOv8m summary: 295 layers, 25902640 parameters, 25902624 gradients, 79.3 GFLOPs + l: [1.00, 1.00, 512] # YOLOv8l summary: 365 layers, 43691520 parameters, 43691504 gradients, 165.7 GFLOPs + x: [1.00, 1.25, 512] # YOLOv8x summary: 365 layers, 68229648 parameters, 68229632 gradients, 258.5 GFLOPs + +# YOLOv8.0n backbone +backbone: + # [from, repeats, module, args] + - [-1, 1, Conv, [64, 3, 2]] # 0-P1/2 + - [-1, 1, Conv, [128, 3, 2]] # 1-P2/4 + - [-1, 3, C2f, [128, True]] + - [-1, 1, Conv, [256, 3, 2]] # 3-P3/8 + - [-1, 6, C2f, [256, True]] + - [-1, 1, Conv, [512, 3, 2]] # 5-P4/16 + - [-1, 6, C2f, [512, True]] + - [-1, 1, Conv, [1024, 3, 2]] # 7-P5/32 + - [-1, 3, C2f, [1024, True]] + - [-1, 1, SPPF, [1024, 5]] # 9 + +# YOLOv8.0n head +head: + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 6], 1, Concat, [1]] # cat backbone P4 + - [-1, 3, C2f, [512]] # 12 + + - [-1, 1, nn.Upsample, [None, 2, "nearest"]] + - [[-1, 4], 1, Concat, [1]] # cat backbone P3 + - [-1, 3, C2f, [256]] # 15 (P3/8-small) + + - [-1, 1, Conv, [256, 3, 2]] + - [[-1, 12], 1, Concat, [1]] # cat head P4 + - [-1, 3, C2f, [512]] # 18 (P4/16-medium) + + - [-1, 1, Conv, [512, 3, 2]] + - [[-1, 9], 1, Concat, [1]] # cat head P5 + - [-1, 3, C2f, [1024]] # 21 (P5/32-large) + + - [[15, 18, 21], 1, Detect, [nc]] # Detect(P3, P4, P5) diff --git a/ultralytics/cfg/models/v9/yolov9c.yaml b/ultralytics/cfg/models/v9/yolov9c.yaml new file mode 100644 index 0000000..66c02d6 --- /dev/null +++ b/ultralytics/cfg/models/v9/yolov9c.yaml @@ -0,0 +1,36 @@ +# YOLOv9 + +# parameters +nc: 80 # number of classes + +# gelan backbone +backbone: + - [-1, 1, Conv, [64, 3, 2]] # 0-P1/2 + - [-1, 1, Conv, [128, 3, 2]] # 1-P2/4 + - [-1, 1, RepNCSPELAN4, [256, 128, 64, 1]] # 2 + - [-1, 1, ADown, [256]] # 3-P3/8 + - [-1, 1, RepNCSPELAN4, [512, 256, 128, 1]] # 4 + - [-1, 1, ADown, [512]] # 5-P4/16 + - [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]] # 6 + - [-1, 1, ADown, [512]] # 7-P5/32 + - [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]] # 8 + - [-1, 1, SPPELAN, [512, 256]] # 9 + +head: + - [-1, 1, nn.Upsample, [None, 2, 'nearest']] + - [[-1, 6], 1, Concat, [1]] # cat backbone P4 + - [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]] # 12 + + - [-1, 1, nn.Upsample, [None, 2, 'nearest']] + - [[-1, 4], 1, Concat, [1]] # cat backbone P3 + - [-1, 1, RepNCSPELAN4, [256, 256, 128, 1]] # 15 (P3/8-small) + + - [-1, 1, ADown, [256]] + - [[-1, 12], 1, Concat, [1]] # cat head P4 + - [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]] # 18 (P4/16-medium) + + - [-1, 1, ADown, [512]] + - [[-1, 9], 1, Concat, [1]] # cat head P5 + - [-1, 1, RepNCSPELAN4, [512, 512, 256, 1]] # 21 (P5/32-large) + + - [[15, 18, 21], 1, Detect, [nc]] # DDetect(P3, P4, P5) diff --git a/ultralytics/cfg/models/v9/yolov9e.yaml b/ultralytics/cfg/models/v9/yolov9e.yaml new file mode 100644 index 0000000..8e15a42 --- /dev/null +++ b/ultralytics/cfg/models/v9/yolov9e.yaml @@ -0,0 +1,60 @@ +# YOLOv9 + +# parameters +nc: 80 # number of classes + +# gelan backbone +backbone: + - [-1, 1, Silence, []] + - [-1, 1, Conv, [64, 3, 2]] # 1-P1/2 + - [-1, 1, Conv, [128, 3, 2]] # 2-P2/4 + - [-1, 1, RepNCSPELAN4, [256, 128, 64, 2]] # 3 + - [-1, 1, ADown, [256]] # 4-P3/8 + - [-1, 1, RepNCSPELAN4, [512, 256, 128, 2]] # 5 + - [-1, 1, ADown, [512]] # 6-P4/16 + - [-1, 1, RepNCSPELAN4, [1024, 512, 256, 2]] # 7 + - [-1, 1, ADown, [1024]] # 8-P5/32 + - [-1, 1, RepNCSPELAN4, [1024, 512, 256, 2]] # 9 + + - [1, 1, CBLinear, [[64]]] # 10 + - [3, 1, CBLinear, [[64, 128]]] # 11 + - [5, 1, CBLinear, [[64, 128, 256]]] # 12 + - [7, 1, CBLinear, [[64, 128, 256, 512]]] # 13 + - [9, 1, CBLinear, [[64, 128, 256, 512, 1024]]] # 14 + + - [0, 1, Conv, [64, 3, 2]] # 15-P1/2 + - [[10, 11, 12, 13, 14, -1], 1, CBFuse, [[0, 0, 0, 0, 0]]] # 16 + - [-1, 1, Conv, [128, 3, 2]] # 17-P2/4 + - [[11, 12, 13, 14, -1], 1, CBFuse, [[1, 1, 1, 1]]] # 18 + - [-1, 1, RepNCSPELAN4, [256, 128, 64, 2]] # 19 + - [-1, 1, ADown, [256]] # 20-P3/8 + - [[12, 13, 14, -1], 1, CBFuse, [[2, 2, 2]]] # 21 + - [-1, 1, RepNCSPELAN4, [512, 256, 128, 2]] # 22 + - [-1, 1, ADown, [512]] # 23-P4/16 + - [[13, 14, -1], 1, CBFuse, [[3, 3]]] # 24 + - [-1, 1, RepNCSPELAN4, [1024, 512, 256, 2]] # 25 + - [-1, 1, ADown, [1024]] # 26-P5/32 + - [[14, -1], 1, CBFuse, [[4]]] # 27 + - [-1, 1, RepNCSPELAN4, [1024, 512, 256, 2]] # 28 + - [-1, 1, SPPELAN, [512, 256]] # 29 + +# gelan head +head: + - [-1, 1, nn.Upsample, [None, 2, 'nearest']] + - [[-1, 25], 1, Concat, [1]] # cat backbone P4 + - [-1, 1, RepNCSPELAN4, [512, 512, 256, 2]] # 32 + + - [-1, 1, nn.Upsample, [None, 2, 'nearest']] + - [[-1, 22], 1, Concat, [1]] # cat backbone P3 + - [-1, 1, RepNCSPELAN4, [256, 256, 128, 2]] # 35 (P3/8-small) + + - [-1, 1, ADown, [256]] + - [[-1, 32], 1, Concat, [1]] # cat head P4 + - [-1, 1, RepNCSPELAN4, [512, 512, 256, 2]] # 38 (P4/16-medium) + + - [-1, 1, ADown, [512]] + - [[-1, 29], 1, Concat, [1]] # cat head P5 + - [-1, 1, RepNCSPELAN4, [512, 1024, 512, 2]] # 41 (P5/32-large) + + # detect + - [[35, 38, 41], 1, Detect, [nc]] # Detect(P3, P4, P5) diff --git a/ultralytics/cfg/trackers/botsort.yaml b/ultralytics/cfg/trackers/botsort.yaml new file mode 100644 index 0000000..0c66dc6 --- /dev/null +++ b/ultralytics/cfg/trackers/botsort.yaml @@ -0,0 +1,18 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# Default YOLO tracker settings for BoT-SORT tracker https://github.com/NirAharon/BoT-SORT + +tracker_type: botsort # tracker type, ['botsort', 'bytetrack'] +track_high_thresh: 0.5 # threshold for the first association +track_low_thresh: 0.1 # threshold for the second association +new_track_thresh: 0.6 # threshold for init new track if the detection does not match any tracks +track_buffer: 30 # buffer to calculate the time when to remove tracks +match_thresh: 0.8 # threshold for matching tracks +# min_box_area: 10 # threshold for min box areas(for tracker evaluation, not used for now) +# mot20: False # for tracker evaluation(not used for now) + +# BoT-SORT settings +gmc_method: sparseOptFlow # method of global motion compensation +# ReID model related thresh (not supported yet) +proximity_thresh: 0.5 +appearance_thresh: 0.25 +with_reid: False diff --git a/ultralytics/cfg/trackers/bytetrack.yaml b/ultralytics/cfg/trackers/bytetrack.yaml new file mode 100644 index 0000000..29d352c --- /dev/null +++ b/ultralytics/cfg/trackers/bytetrack.yaml @@ -0,0 +1,11 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +# Default YOLO tracker settings for ByteTrack tracker https://github.com/ifzhang/ByteTrack + +tracker_type: bytetrack # tracker type, ['botsort', 'bytetrack'] +track_high_thresh: 0.5 # threshold for the first association +track_low_thresh: 0.1 # threshold for the second association +new_track_thresh: 0.6 # threshold for init new track if the detection does not match any tracks +track_buffer: 30 # buffer to calculate the time when to remove tracks +match_thresh: 0.8 # threshold for matching tracks +# min_box_area: 10 # threshold for min box areas(for tracker evaluation, not used for now) +# mot20: False # for tracker evaluation(not used for now) diff --git a/ultralytics/data/__init__.py b/ultralytics/data/__init__.py new file mode 100644 index 0000000..9f91ce9 --- /dev/null +++ b/ultralytics/data/__init__.py @@ -0,0 +1,15 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from .base import BaseDataset +from .build import build_dataloader, build_yolo_dataset, load_inference_source +from .dataset import ClassificationDataset, SemanticDataset, YOLODataset + +__all__ = ( + "BaseDataset", + "ClassificationDataset", + "SemanticDataset", + "YOLODataset", + "build_yolo_dataset", + "build_dataloader", + "load_inference_source", +) diff --git a/ultralytics/data/__pycache__/__init__.cpython-312.pyc b/ultralytics/data/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..60fbf48 Binary files /dev/null and b/ultralytics/data/__pycache__/__init__.cpython-312.pyc differ diff --git a/ultralytics/data/__pycache__/__init__.cpython-39.pyc b/ultralytics/data/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..f451229 Binary files /dev/null and b/ultralytics/data/__pycache__/__init__.cpython-39.pyc differ diff --git a/ultralytics/data/__pycache__/augment.cpython-312.pyc b/ultralytics/data/__pycache__/augment.cpython-312.pyc new file mode 100644 index 0000000..4a02210 Binary files /dev/null and b/ultralytics/data/__pycache__/augment.cpython-312.pyc differ diff --git a/ultralytics/data/__pycache__/augment.cpython-39.pyc b/ultralytics/data/__pycache__/augment.cpython-39.pyc new file mode 100644 index 0000000..bebc23e Binary files /dev/null and b/ultralytics/data/__pycache__/augment.cpython-39.pyc differ diff --git a/ultralytics/data/__pycache__/base.cpython-312.pyc b/ultralytics/data/__pycache__/base.cpython-312.pyc new file mode 100644 index 0000000..f24113e Binary files /dev/null and b/ultralytics/data/__pycache__/base.cpython-312.pyc differ diff --git a/ultralytics/data/__pycache__/base.cpython-39.pyc b/ultralytics/data/__pycache__/base.cpython-39.pyc new file mode 100644 index 0000000..f403862 Binary files /dev/null and b/ultralytics/data/__pycache__/base.cpython-39.pyc differ diff --git a/ultralytics/data/__pycache__/build.cpython-312.pyc b/ultralytics/data/__pycache__/build.cpython-312.pyc new file mode 100644 index 0000000..6a29242 Binary files /dev/null and b/ultralytics/data/__pycache__/build.cpython-312.pyc differ diff --git a/ultralytics/data/__pycache__/build.cpython-39.pyc b/ultralytics/data/__pycache__/build.cpython-39.pyc new file mode 100644 index 0000000..a437e16 Binary files /dev/null and b/ultralytics/data/__pycache__/build.cpython-39.pyc differ diff --git a/ultralytics/data/__pycache__/converter.cpython-312.pyc b/ultralytics/data/__pycache__/converter.cpython-312.pyc new file mode 100644 index 0000000..b2e5e40 Binary files /dev/null and b/ultralytics/data/__pycache__/converter.cpython-312.pyc differ diff --git a/ultralytics/data/__pycache__/converter.cpython-39.pyc b/ultralytics/data/__pycache__/converter.cpython-39.pyc new file mode 100644 index 0000000..ff6cf6d Binary files /dev/null and b/ultralytics/data/__pycache__/converter.cpython-39.pyc differ diff --git a/ultralytics/data/__pycache__/dataset.cpython-312.pyc b/ultralytics/data/__pycache__/dataset.cpython-312.pyc new file mode 100644 index 0000000..115fc37 Binary files /dev/null and b/ultralytics/data/__pycache__/dataset.cpython-312.pyc differ diff --git a/ultralytics/data/__pycache__/dataset.cpython-39.pyc b/ultralytics/data/__pycache__/dataset.cpython-39.pyc new file mode 100644 index 0000000..91bfcba Binary files /dev/null and b/ultralytics/data/__pycache__/dataset.cpython-39.pyc differ diff --git a/ultralytics/data/__pycache__/loaders.cpython-312.pyc b/ultralytics/data/__pycache__/loaders.cpython-312.pyc new file mode 100644 index 0000000..435a16f Binary files /dev/null and b/ultralytics/data/__pycache__/loaders.cpython-312.pyc differ diff --git a/ultralytics/data/__pycache__/loaders.cpython-39.pyc b/ultralytics/data/__pycache__/loaders.cpython-39.pyc new file mode 100644 index 0000000..cabbb34 Binary files /dev/null and b/ultralytics/data/__pycache__/loaders.cpython-39.pyc differ diff --git a/ultralytics/data/__pycache__/utils.cpython-312.pyc b/ultralytics/data/__pycache__/utils.cpython-312.pyc new file mode 100644 index 0000000..b9326f3 Binary files /dev/null and b/ultralytics/data/__pycache__/utils.cpython-312.pyc differ diff --git a/ultralytics/data/__pycache__/utils.cpython-39.pyc b/ultralytics/data/__pycache__/utils.cpython-39.pyc new file mode 100644 index 0000000..1347543 Binary files /dev/null and b/ultralytics/data/__pycache__/utils.cpython-39.pyc differ diff --git a/ultralytics/data/annotator.py b/ultralytics/data/annotator.py new file mode 100644 index 0000000..b5b899c --- /dev/null +++ b/ultralytics/data/annotator.py @@ -0,0 +1,50 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from pathlib import Path + +from ultralytics import SAM, YOLO + + +def auto_annotate(data, det_model="yolov8x.pt", sam_model="sam_b.pt", device="", output_dir=None): + """ + Automatically annotates images using a YOLO object detection model and a SAM segmentation model. + + Args: + data (str): Path to a folder containing images to be annotated. + det_model (str, optional): Pre-trained YOLO detection model. Defaults to 'yolov8x.pt'. + sam_model (str, optional): Pre-trained SAM segmentation model. Defaults to 'sam_b.pt'. + device (str, optional): Device to run the models on. Defaults to an empty string (CPU or GPU, if available). + output_dir (str | None | optional): Directory to save the annotated results. + Defaults to a 'labels' folder in the same directory as 'data'. + + Example: + ```python + from ultralytics.data.annotator import auto_annotate + + auto_annotate(data='ultralytics/assets', det_model='yolov8n.pt', sam_model='mobile_sam.pt') + ``` + """ + det_model = YOLO(det_model) + sam_model = SAM(sam_model) + + data = Path(data) + if not output_dir: + output_dir = data.parent / f"{data.stem}_auto_annotate_labels" + Path(output_dir).mkdir(exist_ok=True, parents=True) + + det_results = det_model(data, stream=True, device=device) + + for result in det_results: + class_ids = result.boxes.cls.int().tolist() # noqa + if len(class_ids): + boxes = result.boxes.xyxy # Boxes object for bbox outputs + sam_results = sam_model(result.orig_img, bboxes=boxes, verbose=False, save=False, device=device) + segments = sam_results[0].masks.xyn # noqa + + with open(f"{Path(output_dir) / Path(result.path).stem}.txt", "w") as f: + for i in range(len(segments)): + s = segments[i] + if len(s) == 0: + continue + segment = map(str, segments[i].reshape(-1).tolist()) + f.write(f"{class_ids[i]} " + " ".join(segment) + "\n") diff --git a/ultralytics/data/augment.py b/ultralytics/data/augment.py new file mode 100644 index 0000000..aab3e62 --- /dev/null +++ b/ultralytics/data/augment.py @@ -0,0 +1,1254 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import math +import random +from copy import deepcopy + +import cv2 +import numpy as np +import torch +import torchvision.transforms as T + +from ultralytics.utils import LOGGER, colorstr +from ultralytics.utils.checks import check_version +from ultralytics.utils.instance import Instances +from ultralytics.utils.metrics import bbox_ioa +from ultralytics.utils.ops import segment2box, xyxyxyxy2xywhr +from ultralytics.utils.torch_utils import TORCHVISION_0_10, TORCHVISION_0_11, TORCHVISION_0_13 +from .utils import polygons2masks, polygons2masks_overlap + +DEFAULT_MEAN = (0.0, 0.0, 0.0) +DEFAULT_STD = (1.0, 1.0, 1.0) +DEFAULT_CROP_FTACTION = 1.0 + + +# TODO: we might need a BaseTransform to make all these augments be compatible with both classification and semantic +class BaseTransform: + """ + Base class for image transformations. + + This is a generic transformation class that can be extended for specific image processing needs. + The class is designed to be compatible with both classification and semantic segmentation tasks. + + Methods: + __init__: Initializes the BaseTransform object. + apply_image: Applies image transformation to labels. + apply_instances: Applies transformations to object instances in labels. + apply_semantic: Applies semantic segmentation to an image. + __call__: Applies all label transformations to an image, instances, and semantic masks. + """ + + def __init__(self) -> None: + """Initializes the BaseTransform object.""" + pass + + def apply_image(self, labels): + """Applies image transformations to labels.""" + pass + + def apply_instances(self, labels): + """Applies transformations to object instances in labels.""" + pass + + def apply_semantic(self, labels): + """Applies semantic segmentation to an image.""" + pass + + def __call__(self, labels): + """Applies all label transformations to an image, instances, and semantic masks.""" + self.apply_image(labels) + self.apply_instances(labels) + self.apply_semantic(labels) + + +class Compose: + """Class for composing multiple image transformations.""" + + def __init__(self, transforms): + """Initializes the Compose object with a list of transforms.""" + self.transforms = transforms + + def __call__(self, data): + """Applies a series of transformations to input data.""" + for t in self.transforms: + data = t(data) + return data + + def append(self, transform): + """Appends a new transform to the existing list of transforms.""" + self.transforms.append(transform) + + def tolist(self): + """Converts the list of transforms to a standard Python list.""" + return self.transforms + + def __repr__(self): + """Returns a string representation of the object.""" + return f"{self.__class__.__name__}({', '.join([f'{t}' for t in self.transforms])})" + + +class BaseMixTransform: + """ + Class for base mix (MixUp/Mosaic) transformations. + + This implementation is from mmyolo. + """ + + def __init__(self, dataset, pre_transform=None, p=0.0) -> None: + """Initializes the BaseMixTransform object with dataset, pre_transform, and probability.""" + self.dataset = dataset + self.pre_transform = pre_transform + self.p = p + + def __call__(self, labels): + """Applies pre-processing transforms and mixup/mosaic transforms to labels data.""" + if random.uniform(0, 1) > self.p: + return labels + + # Get index of one or three other images + indexes = self.get_indexes() + if isinstance(indexes, int): + indexes = [indexes] + + # Get images information will be used for Mosaic or MixUp + mix_labels = [self.dataset.get_image_and_label(i) for i in indexes] + + if self.pre_transform is not None: + for i, data in enumerate(mix_labels): + mix_labels[i] = self.pre_transform(data) + labels["mix_labels"] = mix_labels + + # Mosaic or MixUp + labels = self._mix_transform(labels) + labels.pop("mix_labels", None) + return labels + + def _mix_transform(self, labels): + """Applies MixUp or Mosaic augmentation to the label dictionary.""" + raise NotImplementedError + + def get_indexes(self): + """Gets a list of shuffled indexes for mosaic augmentation.""" + raise NotImplementedError + + +class Mosaic(BaseMixTransform): + """ + Mosaic augmentation. + + This class performs mosaic augmentation by combining multiple (4 or 9) images into a single mosaic image. + The augmentation is applied to a dataset with a given probability. + + Attributes: + dataset: The dataset on which the mosaic augmentation is applied. + imgsz (int, optional): Image size (height and width) after mosaic pipeline of a single image. Default to 640. + p (float, optional): Probability of applying the mosaic augmentation. Must be in the range 0-1. Default to 1.0. + n (int, optional): The grid size, either 4 (for 2x2) or 9 (for 3x3). + """ + + def __init__(self, dataset, imgsz=640, p=1.0, n=4): + """Initializes the object with a dataset, image size, probability, and border.""" + assert 0 <= p <= 1.0, f"The probability should be in range [0, 1], but got {p}." + assert n in (4, 9), "grid must be equal to 4 or 9." + super().__init__(dataset=dataset, p=p) + self.dataset = dataset + self.imgsz = imgsz + self.border = (-imgsz // 2, -imgsz // 2) # width, height + self.n = n + + def get_indexes(self, buffer=True): + """Return a list of random indexes from the dataset.""" + if buffer: # select images from buffer + return random.choices(list(self.dataset.buffer), k=self.n - 1) + else: # select any images + return [random.randint(0, len(self.dataset) - 1) for _ in range(self.n - 1)] + + def _mix_transform(self, labels): + """Apply mixup transformation to the input image and labels.""" + assert labels.get("rect_shape", None) is None, "rect and mosaic are mutually exclusive." + assert len(labels.get("mix_labels", [])), "There are no other images for mosaic augment." + return ( + self._mosaic3(labels) if self.n == 3 else self._mosaic4(labels) if self.n == 4 else self._mosaic9(labels) + ) # This code is modified for mosaic3 method. + + def _mosaic3(self, labels): + """Create a 1x3 image mosaic.""" + mosaic_labels = [] + s = self.imgsz + for i in range(3): + labels_patch = labels if i == 0 else labels["mix_labels"][i - 1] + # Load image + img = labels_patch["img"] + h, w = labels_patch.pop("resized_shape") + + # Place img in img3 + if i == 0: # center + img3 = np.full((s * 3, s * 3, img.shape[2]), 114, dtype=np.uint8) # base image with 3 tiles + h0, w0 = h, w + c = s, s, s + w, s + h # xmin, ymin, xmax, ymax (base) coordinates + elif i == 1: # right + c = s + w0, s, s + w0 + w, s + h + elif i == 2: # left + c = s - w, s + h0 - h, s, s + h0 + + padw, padh = c[:2] + x1, y1, x2, y2 = (max(x, 0) for x in c) # allocate coords + + img3[y1:y2, x1:x2] = img[y1 - padh :, x1 - padw :] # img3[ymin:ymax, xmin:xmax] + # hp, wp = h, w # height, width previous for next iteration + + # Labels assuming imgsz*2 mosaic size + labels_patch = self._update_labels(labels_patch, padw + self.border[0], padh + self.border[1]) + mosaic_labels.append(labels_patch) + final_labels = self._cat_labels(mosaic_labels) + + final_labels["img"] = img3[-self.border[0] : self.border[0], -self.border[1] : self.border[1]] + return final_labels + + def _mosaic4(self, labels): + """Create a 2x2 image mosaic.""" + mosaic_labels = [] + s = self.imgsz + yc, xc = (int(random.uniform(-x, 2 * s + x)) for x in self.border) # mosaic center x, y + for i in range(4): + labels_patch = labels if i == 0 else labels["mix_labels"][i - 1] + # Load image + img = labels_patch["img"] + h, w = labels_patch.pop("resized_shape") + + # Place img in img4 + if i == 0: # top left + img4 = np.full((s * 2, s * 2, img.shape[2]), 114, dtype=np.uint8) # base image with 4 tiles + x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc # xmin, ymin, xmax, ymax (large image) + x1b, y1b, x2b, y2b = w - (x2a - x1a), h - (y2a - y1a), w, h # xmin, ymin, xmax, ymax (small image) + elif i == 1: # top right + x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc + x1b, y1b, x2b, y2b = 0, h - (y2a - y1a), min(w, x2a - x1a), h + elif i == 2: # bottom left + x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h) + x1b, y1b, x2b, y2b = w - (x2a - x1a), 0, w, min(y2a - y1a, h) + elif i == 3: # bottom right + x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h) + x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h) + + img4[y1a:y2a, x1a:x2a] = img[y1b:y2b, x1b:x2b] # img4[ymin:ymax, xmin:xmax] + padw = x1a - x1b + padh = y1a - y1b + + labels_patch = self._update_labels(labels_patch, padw, padh) + mosaic_labels.append(labels_patch) + final_labels = self._cat_labels(mosaic_labels) + final_labels["img"] = img4 + return final_labels + + def _mosaic9(self, labels): + """Create a 3x3 image mosaic.""" + mosaic_labels = [] + s = self.imgsz + hp, wp = -1, -1 # height, width previous + for i in range(9): + labels_patch = labels if i == 0 else labels["mix_labels"][i - 1] + # Load image + img = labels_patch["img"] + h, w = labels_patch.pop("resized_shape") + + # Place img in img9 + if i == 0: # center + img9 = np.full((s * 3, s * 3, img.shape[2]), 114, dtype=np.uint8) # base image with 4 tiles + h0, w0 = h, w + c = s, s, s + w, s + h # xmin, ymin, xmax, ymax (base) coordinates + elif i == 1: # top + c = s, s - h, s + w, s + elif i == 2: # top right + c = s + wp, s - h, s + wp + w, s + elif i == 3: # right + c = s + w0, s, s + w0 + w, s + h + elif i == 4: # bottom right + c = s + w0, s + hp, s + w0 + w, s + hp + h + elif i == 5: # bottom + c = s + w0 - w, s + h0, s + w0, s + h0 + h + elif i == 6: # bottom left + c = s + w0 - wp - w, s + h0, s + w0 - wp, s + h0 + h + elif i == 7: # left + c = s - w, s + h0 - h, s, s + h0 + elif i == 8: # top left + c = s - w, s + h0 - hp - h, s, s + h0 - hp + + padw, padh = c[:2] + x1, y1, x2, y2 = (max(x, 0) for x in c) # allocate coords + + # Image + img9[y1:y2, x1:x2] = img[y1 - padh :, x1 - padw :] # img9[ymin:ymax, xmin:xmax] + hp, wp = h, w # height, width previous for next iteration + + # Labels assuming imgsz*2 mosaic size + labels_patch = self._update_labels(labels_patch, padw + self.border[0], padh + self.border[1]) + mosaic_labels.append(labels_patch) + final_labels = self._cat_labels(mosaic_labels) + + final_labels["img"] = img9[-self.border[0] : self.border[0], -self.border[1] : self.border[1]] + return final_labels + + @staticmethod + def _update_labels(labels, padw, padh): + """Update labels.""" + nh, nw = labels["img"].shape[:2] + labels["instances"].convert_bbox(format="xyxy") + labels["instances"].denormalize(nw, nh) + labels["instances"].add_padding(padw, padh) + return labels + + def _cat_labels(self, mosaic_labels): + """Return labels with mosaic border instances clipped.""" + if len(mosaic_labels) == 0: + return {} + cls = [] + instances = [] + imgsz = self.imgsz * 2 # mosaic imgsz + for labels in mosaic_labels: + cls.append(labels["cls"]) + instances.append(labels["instances"]) + # Final labels + final_labels = { + "im_file": mosaic_labels[0]["im_file"], + "ori_shape": mosaic_labels[0]["ori_shape"], + "resized_shape": (imgsz, imgsz), + "cls": np.concatenate(cls, 0), + "instances": Instances.concatenate(instances, axis=0), + "mosaic_border": self.border, + } + final_labels["instances"].clip(imgsz, imgsz) + good = final_labels["instances"].remove_zero_area_boxes() + final_labels["cls"] = final_labels["cls"][good] + return final_labels + + +class MixUp(BaseMixTransform): + """Class for applying MixUp augmentation to the dataset.""" + + def __init__(self, dataset, pre_transform=None, p=0.0) -> None: + """Initializes MixUp object with dataset, pre_transform, and probability of applying MixUp.""" + super().__init__(dataset=dataset, pre_transform=pre_transform, p=p) + + def get_indexes(self): + """Get a random index from the dataset.""" + return random.randint(0, len(self.dataset) - 1) + + def _mix_transform(self, labels): + """Applies MixUp augmentation as per https://arxiv.org/pdf/1710.09412.pdf.""" + r = np.random.beta(32.0, 32.0) # mixup ratio, alpha=beta=32.0 + labels2 = labels["mix_labels"][0] + labels["img"] = (labels["img"] * r + labels2["img"] * (1 - r)).astype(np.uint8) + labels["instances"] = Instances.concatenate([labels["instances"], labels2["instances"]], axis=0) + labels["cls"] = np.concatenate([labels["cls"], labels2["cls"]], 0) + return labels + + +class RandomPerspective: + """ + Implements random perspective and affine transformations on images and corresponding bounding boxes, segments, and + keypoints. These transformations include rotation, translation, scaling, and shearing. The class also offers the + option to apply these transformations conditionally with a specified probability. + + Attributes: + degrees (float): Degree range for random rotations. + translate (float): Fraction of total width and height for random translation. + scale (float): Scaling factor interval, e.g., a scale factor of 0.1 allows a resize between 90%-110%. + shear (float): Shear intensity (angle in degrees). + perspective (float): Perspective distortion factor. + border (tuple): Tuple specifying mosaic border. + pre_transform (callable): A function/transform to apply to the image before starting the random transformation. + + Methods: + affine_transform(img, border): Applies a series of affine transformations to the image. + apply_bboxes(bboxes, M): Transforms bounding boxes using the calculated affine matrix. + apply_segments(segments, M): Transforms segments and generates new bounding boxes. + apply_keypoints(keypoints, M): Transforms keypoints. + __call__(labels): Main method to apply transformations to both images and their corresponding annotations. + box_candidates(box1, box2): Filters out bounding boxes that don't meet certain criteria post-transformation. + """ + + def __init__( + self, degrees=0.0, translate=0.1, scale=0.5, shear=0.0, perspective=0.0, border=(0, 0), pre_transform=None + ): + """Initializes RandomPerspective object with transformation parameters.""" + + self.degrees = degrees + self.translate = translate + self.scale = scale + self.shear = shear + self.perspective = perspective + self.border = border # mosaic border + self.pre_transform = pre_transform + + def affine_transform(self, img, border): + """ + Applies a sequence of affine transformations centered around the image center. + + Args: + img (ndarray): Input image. + border (tuple): Border dimensions. + + Returns: + img (ndarray): Transformed image. + M (ndarray): Transformation matrix. + s (float): Scale factor. + """ + + # Center + C = np.eye(3, dtype=np.float32) + + C[0, 2] = -img.shape[1] / 2 # x translation (pixels) + C[1, 2] = -img.shape[0] / 2 # y translation (pixels) + + # Perspective + P = np.eye(3, dtype=np.float32) + P[2, 0] = random.uniform(-self.perspective, self.perspective) # x perspective (about y) + P[2, 1] = random.uniform(-self.perspective, self.perspective) # y perspective (about x) + + # Rotation and Scale + R = np.eye(3, dtype=np.float32) + a = random.uniform(-self.degrees, self.degrees) + # a += random.choice([-180, -90, 0, 90]) # add 90deg rotations to small rotations + s = random.uniform(1 - self.scale, 1 + self.scale) + # s = 2 ** random.uniform(-scale, scale) + R[:2] = cv2.getRotationMatrix2D(angle=a, center=(0, 0), scale=s) + + # Shear + S = np.eye(3, dtype=np.float32) + S[0, 1] = math.tan(random.uniform(-self.shear, self.shear) * math.pi / 180) # x shear (deg) + S[1, 0] = math.tan(random.uniform(-self.shear, self.shear) * math.pi / 180) # y shear (deg) + + # Translation + T = np.eye(3, dtype=np.float32) + T[0, 2] = random.uniform(0.5 - self.translate, 0.5 + self.translate) * self.size[0] # x translation (pixels) + T[1, 2] = random.uniform(0.5 - self.translate, 0.5 + self.translate) * self.size[1] # y translation (pixels) + + # Combined rotation matrix + M = T @ S @ R @ P @ C # order of operations (right to left) is IMPORTANT + # Affine image + if (border[0] != 0) or (border[1] != 0) or (M != np.eye(3)).any(): # image changed + if self.perspective: + img = cv2.warpPerspective(img, M, dsize=self.size, borderValue=(114, 114, 114)) + else: # affine + img = cv2.warpAffine(img, M[:2], dsize=self.size, borderValue=(114, 114, 114)) + return img, M, s + + def apply_bboxes(self, bboxes, M): + """ + Apply affine to bboxes only. + + Args: + bboxes (ndarray): list of bboxes, xyxy format, with shape (num_bboxes, 4). + M (ndarray): affine matrix. + + Returns: + new_bboxes (ndarray): bboxes after affine, [num_bboxes, 4]. + """ + n = len(bboxes) + if n == 0: + return bboxes + + xy = np.ones((n * 4, 3), dtype=bboxes.dtype) + xy[:, :2] = bboxes[:, [0, 1, 2, 3, 0, 3, 2, 1]].reshape(n * 4, 2) # x1y1, x2y2, x1y2, x2y1 + xy = xy @ M.T # transform + xy = (xy[:, :2] / xy[:, 2:3] if self.perspective else xy[:, :2]).reshape(n, 8) # perspective rescale or affine + + # Create new boxes + x = xy[:, [0, 2, 4, 6]] + y = xy[:, [1, 3, 5, 7]] + return np.concatenate((x.min(1), y.min(1), x.max(1), y.max(1)), dtype=bboxes.dtype).reshape(4, n).T + + def apply_segments(self, segments, M): + """ + Apply affine to segments and generate new bboxes from segments. + + Args: + segments (ndarray): list of segments, [num_samples, 500, 2]. + M (ndarray): affine matrix. + + Returns: + new_segments (ndarray): list of segments after affine, [num_samples, 500, 2]. + new_bboxes (ndarray): bboxes after affine, [N, 4]. + """ + n, num = segments.shape[:2] + if n == 0: + return [], segments + + xy = np.ones((n * num, 3), dtype=segments.dtype) + segments = segments.reshape(-1, 2) + xy[:, :2] = segments + xy = xy @ M.T # transform + xy = xy[:, :2] / xy[:, 2:3] + segments = xy.reshape(n, -1, 2) + bboxes = np.stack([segment2box(xy, self.size[0], self.size[1]) for xy in segments], 0) + segments[..., 0] = segments[..., 0].clip(bboxes[:, 0:1], bboxes[:, 2:3]) + segments[..., 1] = segments[..., 1].clip(bboxes[:, 1:2], bboxes[:, 3:4]) + return bboxes, segments + + def apply_keypoints(self, keypoints, M): + """ + Apply affine to keypoints. + + Args: + keypoints (ndarray): keypoints, [N, 17, 3]. + M (ndarray): affine matrix. + + Returns: + new_keypoints (ndarray): keypoints after affine, [N, 17, 3]. + """ + n, nkpt = keypoints.shape[:2] + if n == 0: + return keypoints + xy = np.ones((n * nkpt, 3), dtype=keypoints.dtype) + visible = keypoints[..., 2].reshape(n * nkpt, 1) + xy[:, :2] = keypoints[..., :2].reshape(n * nkpt, 2) + xy = xy @ M.T # transform + xy = xy[:, :2] / xy[:, 2:3] # perspective rescale or affine + out_mask = (xy[:, 0] < 0) | (xy[:, 1] < 0) | (xy[:, 0] > self.size[0]) | (xy[:, 1] > self.size[1]) + visible[out_mask] = 0 + return np.concatenate([xy, visible], axis=-1).reshape(n, nkpt, 3) + + def __call__(self, labels): + """ + Affine images and targets. + + Args: + labels (dict): a dict of `bboxes`, `segments`, `keypoints`. + """ + if self.pre_transform and "mosaic_border" not in labels: + labels = self.pre_transform(labels) + labels.pop("ratio_pad", None) # do not need ratio pad + + img = labels["img"] + cls = labels["cls"] + instances = labels.pop("instances") + # Make sure the coord formats are right + instances.convert_bbox(format="xyxy") + instances.denormalize(*img.shape[:2][::-1]) + + border = labels.pop("mosaic_border", self.border) + self.size = img.shape[1] + border[1] * 2, img.shape[0] + border[0] * 2 # w, h + # M is affine matrix + # Scale for func:`box_candidates` + img, M, scale = self.affine_transform(img, border) + + bboxes = self.apply_bboxes(instances.bboxes, M) + + segments = instances.segments + keypoints = instances.keypoints + # Update bboxes if there are segments. + if len(segments): + bboxes, segments = self.apply_segments(segments, M) + + if keypoints is not None: + keypoints = self.apply_keypoints(keypoints, M) + new_instances = Instances(bboxes, segments, keypoints, bbox_format="xyxy", normalized=False) + # Clip + new_instances.clip(*self.size) + + # Filter instances + instances.scale(scale_w=scale, scale_h=scale, bbox_only=True) + # Make the bboxes have the same scale with new_bboxes + i = self.box_candidates( + box1=instances.bboxes.T, box2=new_instances.bboxes.T, area_thr=0.01 if len(segments) else 0.10 + ) + labels["instances"] = new_instances[i] + labels["cls"] = cls[i] + labels["img"] = img + labels["resized_shape"] = img.shape[:2] + return labels + + def box_candidates(self, box1, box2, wh_thr=2, ar_thr=100, area_thr=0.1, eps=1e-16): + """ + Compute box candidates based on a set of thresholds. This method compares the characteristics of the boxes + before and after augmentation to decide whether a box is a candidate for further processing. + + Args: + box1 (numpy.ndarray): The 4,n bounding box before augmentation, represented as [x1, y1, x2, y2]. + box2 (numpy.ndarray): The 4,n bounding box after augmentation, represented as [x1, y1, x2, y2]. + wh_thr (float, optional): The width and height threshold in pixels. Default is 2. + ar_thr (float, optional): The aspect ratio threshold. Default is 100. + area_thr (float, optional): The area ratio threshold. Default is 0.1. + eps (float, optional): A small epsilon value to prevent division by zero. Default is 1e-16. + + Returns: + (numpy.ndarray): A boolean array indicating which boxes are candidates based on the given thresholds. + """ + w1, h1 = box1[2] - box1[0], box1[3] - box1[1] + w2, h2 = box2[2] - box2[0], box2[3] - box2[1] + ar = np.maximum(w2 / (h2 + eps), h2 / (w2 + eps)) # aspect ratio + return (w2 > wh_thr) & (h2 > wh_thr) & (w2 * h2 / (w1 * h1 + eps) > area_thr) & (ar < ar_thr) # candidates + + +class RandomHSV: + """ + This class is responsible for performing random adjustments to the Hue, Saturation, and Value (HSV) channels of an + image. + + The adjustments are random but within limits set by hgain, sgain, and vgain. + """ + + def __init__(self, hgain=0.5, sgain=0.5, vgain=0.5) -> None: + """ + Initialize RandomHSV class with gains for each HSV channel. + + Args: + hgain (float, optional): Maximum variation for hue. Default is 0.5. + sgain (float, optional): Maximum variation for saturation. Default is 0.5. + vgain (float, optional): Maximum variation for value. Default is 0.5. + """ + self.hgain = hgain + self.sgain = sgain + self.vgain = vgain + + def __call__(self, labels): + """ + Applies random HSV augmentation to an image within the predefined limits. + + The modified image replaces the original image in the input 'labels' dict. + """ + img = labels["img"] + if self.hgain or self.sgain or self.vgain: + r = np.random.uniform(-1, 1, 3) * [self.hgain, self.sgain, self.vgain] + 1 # random gains + hue, sat, val = cv2.split(cv2.cvtColor(img, cv2.COLOR_BGR2HSV)) + dtype = img.dtype # uint8 + + x = np.arange(0, 256, dtype=r.dtype) + lut_hue = ((x * r[0]) % 180).astype(dtype) + lut_sat = np.clip(x * r[1], 0, 255).astype(dtype) + lut_val = np.clip(x * r[2], 0, 255).astype(dtype) + + im_hsv = cv2.merge((cv2.LUT(hue, lut_hue), cv2.LUT(sat, lut_sat), cv2.LUT(val, lut_val))) + cv2.cvtColor(im_hsv, cv2.COLOR_HSV2BGR, dst=img) # no return needed + return labels + + +class RandomFlip: + """ + Applies a random horizontal or vertical flip to an image with a given probability. + + Also updates any instances (bounding boxes, keypoints, etc.) accordingly. + """ + + def __init__(self, p=0.5, direction="horizontal", flip_idx=None) -> None: + """ + Initializes the RandomFlip class with probability and direction. + + Args: + p (float, optional): The probability of applying the flip. Must be between 0 and 1. Default is 0.5. + direction (str, optional): The direction to apply the flip. Must be 'horizontal' or 'vertical'. + Default is 'horizontal'. + flip_idx (array-like, optional): Index mapping for flipping keypoints, if any. + """ + assert direction in ["horizontal", "vertical"], f"Support direction `horizontal` or `vertical`, got {direction}" + assert 0 <= p <= 1.0 + + self.p = p + self.direction = direction + self.flip_idx = flip_idx + + def __call__(self, labels): + """ + Applies random flip to an image and updates any instances like bounding boxes or keypoints accordingly. + + Args: + labels (dict): A dictionary containing the keys 'img' and 'instances'. 'img' is the image to be flipped. + 'instances' is an object containing bounding boxes and optionally keypoints. + + Returns: + (dict): The same dict with the flipped image and updated instances under the 'img' and 'instances' keys. + """ + img = labels["img"] + instances = labels.pop("instances") + instances.convert_bbox(format="xywh") + h, w = img.shape[:2] + h = 1 if instances.normalized else h + w = 1 if instances.normalized else w + + # Flip up-down + if self.direction == "vertical" and random.random() < self.p: + img = np.flipud(img) + instances.flipud(h) + if self.direction == "horizontal" and random.random() < self.p: + img = np.fliplr(img) + instances.fliplr(w) + # For keypoints + if self.flip_idx is not None and instances.keypoints is not None: + instances.keypoints = np.ascontiguousarray(instances.keypoints[:, self.flip_idx, :]) + labels["img"] = np.ascontiguousarray(img) + labels["instances"] = instances + return labels + + +class LetterBox: + """Resize image and padding for detection, instance segmentation, pose.""" + + def __init__(self, new_shape=(640, 640), auto=False, scaleFill=False, scaleup=True, center=True, stride=32): + """Initialize LetterBox object with specific parameters.""" + self.new_shape = new_shape + self.auto = auto + self.scaleFill = scaleFill + self.scaleup = scaleup + self.stride = stride + self.center = center # Put the image in the middle or top-left + + def __call__(self, labels=None, image=None): + """Return updated labels and image with added border.""" + if labels is None: + labels = {} + img = labels.get("img") if image is None else image + shape = img.shape[:2] # current shape [height, width] + new_shape = labels.pop("rect_shape", self.new_shape) + if isinstance(new_shape, int): + new_shape = (new_shape, new_shape) + + # Scale ratio (new / old) + r = min(new_shape[0] / shape[0], new_shape[1] / shape[1]) + if not self.scaleup: # only scale down, do not scale up (for better val mAP) + r = min(r, 1.0) + + # Compute padding + ratio = r, r # width, height ratios + new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r)) + dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1] # wh padding + if self.auto: # minimum rectangle + dw, dh = np.mod(dw, self.stride), np.mod(dh, self.stride) # wh padding + elif self.scaleFill: # stretch + dw, dh = 0.0, 0.0 + new_unpad = (new_shape[1], new_shape[0]) + ratio = new_shape[1] / shape[1], new_shape[0] / shape[0] # width, height ratios + + if self.center: + dw /= 2 # divide padding into 2 sides + dh /= 2 + + if shape[::-1] != new_unpad: # resize + img = cv2.resize(img, new_unpad, interpolation=cv2.INTER_LINEAR) + top, bottom = int(round(dh - 0.1)) if self.center else 0, int(round(dh + 0.1)) + left, right = int(round(dw - 0.1)) if self.center else 0, int(round(dw + 0.1)) + img = cv2.copyMakeBorder( + img, top, bottom, left, right, cv2.BORDER_CONSTANT, value=(114, 114, 114) + ) # add border + if labels.get("ratio_pad"): + labels["ratio_pad"] = (labels["ratio_pad"], (left, top)) # for evaluation + + if len(labels): + labels = self._update_labels(labels, ratio, dw, dh) + labels["img"] = img + labels["resized_shape"] = new_shape + return labels + else: + return img + + def _update_labels(self, labels, ratio, padw, padh): + """Update labels.""" + labels["instances"].convert_bbox(format="xyxy") + labels["instances"].denormalize(*labels["img"].shape[:2][::-1]) + labels["instances"].scale(*ratio) + labels["instances"].add_padding(padw, padh) + return labels + + +class CopyPaste: + """ + Implements the Copy-Paste augmentation as described in the paper https://arxiv.org/abs/2012.07177. This class is + responsible for applying the Copy-Paste augmentation on images and their corresponding instances. + """ + + def __init__(self, p=0.5) -> None: + """ + Initializes the CopyPaste class with a given probability. + + Args: + p (float, optional): The probability of applying the Copy-Paste augmentation. Must be between 0 and 1. + Default is 0.5. + """ + self.p = p + + def __call__(self, labels): + """ + Applies the Copy-Paste augmentation to the given image and instances. + + Args: + labels (dict): A dictionary containing: + - 'img': The image to augment. + - 'cls': Class labels associated with the instances. + - 'instances': Object containing bounding boxes, and optionally, keypoints and segments. + + Returns: + (dict): Dict with augmented image and updated instances under the 'img', 'cls', and 'instances' keys. + + Notes: + 1. Instances are expected to have 'segments' as one of their attributes for this augmentation to work. + 2. This method modifies the input dictionary 'labels' in place. + """ + im = labels["img"] + cls = labels["cls"] + h, w = im.shape[:2] + instances = labels.pop("instances") + instances.convert_bbox(format="xyxy") + instances.denormalize(w, h) + if self.p and len(instances.segments): + n = len(instances) + _, w, _ = im.shape # height, width, channels + im_new = np.zeros(im.shape, np.uint8) + + # Calculate ioa first then select indexes randomly + ins_flip = deepcopy(instances) + ins_flip.fliplr(w) + + ioa = bbox_ioa(ins_flip.bboxes, instances.bboxes) # intersection over area, (N, M) + indexes = np.nonzero((ioa < 0.30).all(1))[0] # (N, ) + n = len(indexes) + for j in random.sample(list(indexes), k=round(self.p * n)): + cls = np.concatenate((cls, cls[[j]]), axis=0) + instances = Instances.concatenate((instances, ins_flip[[j]]), axis=0) + cv2.drawContours(im_new, instances.segments[[j]].astype(np.int32), -1, (1, 1, 1), cv2.FILLED) + + result = cv2.flip(im, 1) # augment segments (flip left-right) + i = cv2.flip(im_new, 1).astype(bool) + im[i] = result[i] + + labels["img"] = im + labels["cls"] = cls + labels["instances"] = instances + return labels + + +class Albumentations: + """ + Albumentations transformations. + + Optional, uninstall package to disable. Applies Blur, Median Blur, convert to grayscale, Contrast Limited Adaptive + Histogram Equalization, random change of brightness and contrast, RandomGamma and lowering of image quality by + compression. + """ + + def __init__(self, p=1.0): + """Initialize the transform object for YOLO bbox formatted params.""" + self.p = p + self.transform = None + prefix = colorstr("albumentations: ") + try: + import albumentations as A + + check_version(A.__version__, "1.0.3", hard=True) # version requirement + + # Transforms + T = [ + A.Blur(p=0.01), + A.MedianBlur(p=0.01), + A.ToGray(p=0.01), + A.CLAHE(p=0.01), + A.RandomBrightnessContrast(p=0.0), + A.RandomGamma(p=0.0), + A.ImageCompression(quality_lower=75, p=0.0), + ] + self.transform = A.Compose(T, bbox_params=A.BboxParams(format="yolo", label_fields=["class_labels"])) + + LOGGER.info(prefix + ", ".join(f"{x}".replace("always_apply=False, ", "") for x in T if x.p)) + except ImportError: # package not installed, skip + pass + except Exception as e: + LOGGER.info(f"{prefix}{e}") + + def __call__(self, labels): + """Generates object detections and returns a dictionary with detection results.""" + im = labels["img"] + cls = labels["cls"] + if len(cls): + labels["instances"].convert_bbox("xywh") + labels["instances"].normalize(*im.shape[:2][::-1]) + bboxes = labels["instances"].bboxes + # TODO: add supports of segments and keypoints + if self.transform and random.random() < self.p: + new = self.transform(image=im, bboxes=bboxes, class_labels=cls) # transformed + if len(new["class_labels"]) > 0: # skip update if no bbox in new im + labels["img"] = new["image"] + labels["cls"] = np.array(new["class_labels"]) + bboxes = np.array(new["bboxes"], dtype=np.float32) + labels["instances"].update(bboxes=bboxes) + return labels + + +# TODO: technically this is not an augmentation, maybe we should put this to another files +class Format: + """ + Formats image annotations for object detection, instance segmentation, and pose estimation tasks. The class + standardizes the image and instance annotations to be used by the `collate_fn` in PyTorch DataLoader. + + Attributes: + bbox_format (str): Format for bounding boxes. Default is 'xywh'. + normalize (bool): Whether to normalize bounding boxes. Default is True. + return_mask (bool): Return instance masks for segmentation. Default is False. + return_keypoint (bool): Return keypoints for pose estimation. Default is False. + mask_ratio (int): Downsample ratio for masks. Default is 4. + mask_overlap (bool): Whether to overlap masks. Default is True. + batch_idx (bool): Keep batch indexes. Default is True. + bgr (float): The probability to return BGR images. Default is 0.0. + """ + + def __init__( + self, + bbox_format="xywh", + normalize=True, + return_mask=False, + return_keypoint=False, + return_obb=False, + mask_ratio=4, + mask_overlap=True, + batch_idx=True, + bgr=0.0, + ): + """Initializes the Format class with given parameters.""" + self.bbox_format = bbox_format + self.normalize = normalize + self.return_mask = return_mask # set False when training detection only + self.return_keypoint = return_keypoint + self.return_obb = return_obb + self.mask_ratio = mask_ratio + self.mask_overlap = mask_overlap + self.batch_idx = batch_idx # keep the batch indexes + self.bgr = bgr + + def __call__(self, labels): + """Return formatted image, classes, bounding boxes & keypoints to be used by 'collate_fn'.""" + img = labels.pop("img") + h, w = img.shape[:2] + cls = labels.pop("cls") + instances = labels.pop("instances") + instances.convert_bbox(format=self.bbox_format) + instances.denormalize(w, h) + nl = len(instances) + + if self.return_mask: + if nl: + masks, instances, cls = self._format_segments(instances, cls, w, h) + masks = torch.from_numpy(masks) + else: + masks = torch.zeros( + 1 if self.mask_overlap else nl, img.shape[0] // self.mask_ratio, img.shape[1] // self.mask_ratio + ) + labels["masks"] = masks + if self.normalize: + instances.normalize(w, h) + labels["img"] = self._format_img(img) + labels["cls"] = torch.from_numpy(cls) if nl else torch.zeros(nl) + labels["bboxes"] = torch.from_numpy(instances.bboxes) if nl else torch.zeros((nl, 4)) + if self.return_keypoint: + labels["keypoints"] = torch.from_numpy(instances.keypoints) + if self.return_obb: + labels["bboxes"] = ( + xyxyxyxy2xywhr(torch.from_numpy(instances.segments)) if len(instances.segments) else torch.zeros((0, 5)) + ) + # Then we can use collate_fn + if self.batch_idx: + labels["batch_idx"] = torch.zeros(nl) + return labels + + def _format_img(self, img): + """Format the image for YOLO from Numpy array to PyTorch tensor.""" + if len(img.shape) < 3: + img = np.expand_dims(img, -1) + img = img.transpose(2, 0, 1) + img = np.ascontiguousarray(img[::-1] if random.uniform(0, 1) > self.bgr else img) + img = torch.from_numpy(img) + return img + + def _format_segments(self, instances, cls, w, h): + """Convert polygon points to bitmap.""" + segments = instances.segments + if self.mask_overlap: + masks, sorted_idx = polygons2masks_overlap((h, w), segments, downsample_ratio=self.mask_ratio) + masks = masks[None] # (640, 640) -> (1, 640, 640) + instances = instances[sorted_idx] + cls = cls[sorted_idx] + else: + masks = polygons2masks((h, w), segments, color=1, downsample_ratio=self.mask_ratio) + + return masks, instances, cls + + +def v8_transforms(dataset, imgsz, hyp, stretch=False): + """Convert images to a size suitable for YOLOv8 training.""" + pre_transform = Compose( + [ + Mosaic(dataset, imgsz=imgsz, p=hyp.mosaic), + CopyPaste(p=hyp.copy_paste), + RandomPerspective( + degrees=hyp.degrees, + translate=hyp.translate, + scale=hyp.scale, + shear=hyp.shear, + perspective=hyp.perspective, + pre_transform=None if stretch else LetterBox(new_shape=(imgsz, imgsz)), + ), + ] + ) + flip_idx = dataset.data.get("flip_idx", []) # for keypoints augmentation + if dataset.use_keypoints: + kpt_shape = dataset.data.get("kpt_shape", None) + if len(flip_idx) == 0 and hyp.fliplr > 0.0: + hyp.fliplr = 0.0 + LOGGER.warning("WARNING ⚠️ No 'flip_idx' array defined in data.yaml, setting augmentation 'fliplr=0.0'") + elif flip_idx and (len(flip_idx) != kpt_shape[0]): + raise ValueError(f"data.yaml flip_idx={flip_idx} length must be equal to kpt_shape[0]={kpt_shape[0]}") + + return Compose( + [ + pre_transform, + MixUp(dataset, pre_transform=pre_transform, p=hyp.mixup), + Albumentations(p=1.0), + RandomHSV(hgain=hyp.hsv_h, sgain=hyp.hsv_s, vgain=hyp.hsv_v), + RandomFlip(direction="vertical", p=hyp.flipud), + RandomFlip(direction="horizontal", p=hyp.fliplr, flip_idx=flip_idx), + ] + ) # transforms + + +# Classification augmentations ----------------------------------------------------------------------------------------- +def classify_transforms( + size=224, + mean=DEFAULT_MEAN, + std=DEFAULT_STD, + interpolation: T.InterpolationMode = T.InterpolationMode.BILINEAR, + crop_fraction: float = DEFAULT_CROP_FTACTION, +): + """ + Classification transforms for evaluation/inference. Inspired by timm/data/transforms_factory.py. + + Args: + size (int): image size + mean (tuple): mean values of RGB channels + std (tuple): std values of RGB channels + interpolation (T.InterpolationMode): interpolation mode. default is T.InterpolationMode.BILINEAR. + crop_fraction (float): fraction of image to crop. default is 1.0. + + Returns: + (T.Compose): torchvision transforms + """ + + if isinstance(size, (tuple, list)): + assert len(size) == 2 + scale_size = tuple(math.floor(x / crop_fraction) for x in size) + else: + scale_size = math.floor(size / crop_fraction) + scale_size = (scale_size, scale_size) + + # aspect ratio is preserved, crops center within image, no borders are added, image is lost + if scale_size[0] == scale_size[1]: + # simple case, use torchvision built-in Resize w/ shortest edge mode (scalar size arg) + tfl = [T.Resize(scale_size[0], interpolation=interpolation)] + else: + # resize shortest edge to matching target dim for non-square target + tfl = [T.Resize(scale_size)] + tfl += [T.CenterCrop(size)] + + tfl += [ + T.ToTensor(), + T.Normalize( + mean=torch.tensor(mean), + std=torch.tensor(std), + ), + ] + + return T.Compose(tfl) + + +# Classification augmentations train --------------------------------------------------------------------------------------- +def classify_augmentations( + size=224, + mean=DEFAULT_MEAN, + std=DEFAULT_STD, + scale=None, + ratio=None, + hflip=0.5, + vflip=0.0, + auto_augment=None, + hsv_h=0.015, # image HSV-Hue augmentation (fraction) + hsv_s=0.4, # image HSV-Saturation augmentation (fraction) + hsv_v=0.4, # image HSV-Value augmentation (fraction) + force_color_jitter=False, + erasing=0.0, + interpolation: T.InterpolationMode = T.InterpolationMode.BILINEAR, +): + """ + Classification transforms with augmentation for training. Inspired by timm/data/transforms_factory.py. + + Args: + size (int): image size + scale (tuple): scale range of the image. default is (0.08, 1.0) + ratio (tuple): aspect ratio range of the image. default is (3./4., 4./3.) + mean (tuple): mean values of RGB channels + std (tuple): std values of RGB channels + hflip (float): probability of horizontal flip + vflip (float): probability of vertical flip + auto_augment (str): auto augmentation policy. can be 'randaugment', 'augmix', 'autoaugment' or None. + hsv_h (float): image HSV-Hue augmentation (fraction) + hsv_s (float): image HSV-Saturation augmentation (fraction) + hsv_v (float): image HSV-Value augmentation (fraction) + force_color_jitter (bool): force to apply color jitter even if auto augment is enabled + erasing (float): probability of random erasing + interpolation (T.InterpolationMode): interpolation mode. default is T.InterpolationMode.BILINEAR. + + Returns: + (T.Compose): torchvision transforms + """ + # Transforms to apply if albumentations not installed + if not isinstance(size, int): + raise TypeError(f"classify_transforms() size {size} must be integer, not (list, tuple)") + scale = tuple(scale or (0.08, 1.0)) # default imagenet scale range + ratio = tuple(ratio or (3.0 / 4.0, 4.0 / 3.0)) # default imagenet ratio range + primary_tfl = [T.RandomResizedCrop(size, scale=scale, ratio=ratio, interpolation=interpolation)] + if hflip > 0.0: + primary_tfl += [T.RandomHorizontalFlip(p=hflip)] + if vflip > 0.0: + primary_tfl += [T.RandomVerticalFlip(p=vflip)] + + secondary_tfl = [] + disable_color_jitter = False + if auto_augment: + assert isinstance(auto_augment, str) + # color jitter is typically disabled if AA/RA on, + # this allows override without breaking old hparm cfgs + disable_color_jitter = not force_color_jitter + + if auto_augment == "randaugment": + if TORCHVISION_0_11: + secondary_tfl += [T.RandAugment(interpolation=interpolation)] + else: + LOGGER.warning('"auto_augment=randaugment" requires torchvision >= 0.11.0. Disabling it.') + + elif auto_augment == "augmix": + if TORCHVISION_0_13: + secondary_tfl += [T.AugMix(interpolation=interpolation)] + else: + LOGGER.warning('"auto_augment=augmix" requires torchvision >= 0.13.0. Disabling it.') + + elif auto_augment == "autoaugment": + if TORCHVISION_0_10: + secondary_tfl += [T.AutoAugment(interpolation=interpolation)] + else: + LOGGER.warning('"auto_augment=autoaugment" requires torchvision >= 0.10.0. Disabling it.') + + else: + raise ValueError( + f'Invalid auto_augment policy: {auto_augment}. Should be one of "randaugment", ' + f'"augmix", "autoaugment" or None' + ) + + if not disable_color_jitter: + secondary_tfl += [T.ColorJitter(brightness=hsv_v, contrast=hsv_v, saturation=hsv_s, hue=hsv_h)] + + final_tfl = [ + T.ToTensor(), + T.Normalize(mean=torch.tensor(mean), std=torch.tensor(std)), + T.RandomErasing(p=erasing, inplace=True), + ] + + return T.Compose(primary_tfl + secondary_tfl + final_tfl) + + +# NOTE: keep this class for backward compatibility +class ClassifyLetterBox: + """ + YOLOv8 LetterBox class for image preprocessing, designed to be part of a transformation pipeline, e.g., + T.Compose([LetterBox(size), ToTensor()]). + + Attributes: + h (int): Target height of the image. + w (int): Target width of the image. + auto (bool): If True, automatically solves for short side using stride. + stride (int): The stride value, used when 'auto' is True. + """ + + def __init__(self, size=(640, 640), auto=False, stride=32): + """ + Initializes the ClassifyLetterBox class with a target size, auto-flag, and stride. + + Args: + size (Union[int, Tuple[int, int]]): The target dimensions (height, width) for the letterbox. + auto (bool): If True, automatically calculates the short side based on stride. + stride (int): The stride value, used when 'auto' is True. + """ + super().__init__() + self.h, self.w = (size, size) if isinstance(size, int) else size + self.auto = auto # pass max size integer, automatically solve for short side using stride + self.stride = stride # used with auto + + def __call__(self, im): + """ + Resizes the image and pads it with a letterbox method. + + Args: + im (numpy.ndarray): The input image as a numpy array of shape HWC. + + Returns: + (numpy.ndarray): The letterboxed and resized image as a numpy array. + """ + imh, imw = im.shape[:2] + r = min(self.h / imh, self.w / imw) # ratio of new/old dimensions + h, w = round(imh * r), round(imw * r) # resized image dimensions + + # Calculate padding dimensions + hs, ws = (math.ceil(x / self.stride) * self.stride for x in (h, w)) if self.auto else (self.h, self.w) + top, left = round((hs - h) / 2 - 0.1), round((ws - w) / 2 - 0.1) + + # Create padded image + im_out = np.full((hs, ws, 3), 114, dtype=im.dtype) + im_out[top : top + h, left : left + w] = cv2.resize(im, (w, h), interpolation=cv2.INTER_LINEAR) + return im_out + + +# NOTE: keep this class for backward compatibility +class CenterCrop: + """YOLOv8 CenterCrop class for image preprocessing, designed to be part of a transformation pipeline, e.g., + T.Compose([CenterCrop(size), ToTensor()]). + """ + + def __init__(self, size=640): + """Converts an image from numpy array to PyTorch tensor.""" + super().__init__() + self.h, self.w = (size, size) if isinstance(size, int) else size + + def __call__(self, im): + """ + Resizes and crops the center of the image using a letterbox method. + + Args: + im (numpy.ndarray): The input image as a numpy array of shape HWC. + + Returns: + (numpy.ndarray): The center-cropped and resized image as a numpy array. + """ + imh, imw = im.shape[:2] + m = min(imh, imw) # min dimension + top, left = (imh - m) // 2, (imw - m) // 2 + return cv2.resize(im[top : top + m, left : left + m], (self.w, self.h), interpolation=cv2.INTER_LINEAR) + + +# NOTE: keep this class for backward compatibility +class ToTensor: + """YOLOv8 ToTensor class for image preprocessing, i.e., T.Compose([LetterBox(size), ToTensor()]).""" + + def __init__(self, half=False): + """Initialize YOLOv8 ToTensor object with optional half-precision support.""" + super().__init__() + self.half = half + + def __call__(self, im): + """ + Transforms an image from a numpy array to a PyTorch tensor, applying optional half-precision and normalization. + + Args: + im (numpy.ndarray): Input image as a numpy array with shape (H, W, C) in BGR order. + + Returns: + (torch.Tensor): The transformed image as a PyTorch tensor in float32 or float16, normalized to [0, 1]. + """ + im = np.ascontiguousarray(im.transpose((2, 0, 1))[::-1]) # HWC to CHW -> BGR to RGB -> contiguous + im = torch.from_numpy(im) # to torch + im = im.half() if self.half else im.float() # uint8 to fp16/32 + im /= 255.0 # 0-255 to 0.0-1.0 + return im diff --git a/ultralytics/data/base.py b/ultralytics/data/base.py new file mode 100644 index 0000000..6af8d3c --- /dev/null +++ b/ultralytics/data/base.py @@ -0,0 +1,311 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import glob +import math +import os +import random +from copy import deepcopy +from multiprocessing.pool import ThreadPool +from pathlib import Path +from typing import Optional + +import cv2 +import numpy as np +import psutil +from torch.utils.data import Dataset + +from ultralytics.utils import DEFAULT_CFG, LOCAL_RANK, LOGGER, NUM_THREADS, TQDM +from .utils import HELP_URL, IMG_FORMATS + + +class BaseDataset(Dataset): + """ + Base dataset class for loading and processing image data. + + Args: + img_path (str): Path to the folder containing images. + imgsz (int, optional): Image size. Defaults to 640. + cache (bool, optional): Cache images to RAM or disk during training. Defaults to False. + augment (bool, optional): If True, data augmentation is applied. Defaults to True. + hyp (dict, optional): Hyperparameters to apply data augmentation. Defaults to None. + prefix (str, optional): Prefix to print in log messages. Defaults to ''. + rect (bool, optional): If True, rectangular training is used. Defaults to False. + batch_size (int, optional): Size of batches. Defaults to None. + stride (int, optional): Stride. Defaults to 32. + pad (float, optional): Padding. Defaults to 0.0. + single_cls (bool, optional): If True, single class training is used. Defaults to False. + classes (list): List of included classes. Default is None. + fraction (float): Fraction of dataset to utilize. Default is 1.0 (use all data). + + Attributes: + im_files (list): List of image file paths. + labels (list): List of label data dictionaries. + ni (int): Number of images in the dataset. + ims (list): List of loaded images. + npy_files (list): List of numpy file paths. + transforms (callable): Image transformation function. + """ + + def __init__( + self, + img_path, + imgsz=640, + cache=False, + augment=True, + hyp=DEFAULT_CFG, + prefix="", + rect=False, + batch_size=16, + stride=32, + pad=0.5, + single_cls=False, + classes=None, + fraction=1.0, + ): + """Initialize BaseDataset with given configuration and options.""" + super().__init__() + self.img_path = img_path + self.imgsz = imgsz + self.augment = augment + self.single_cls = single_cls + self.prefix = prefix + self.fraction = fraction + self.im_files = self.get_img_files(self.img_path) + self.labels = self.get_labels() + self.update_labels(include_class=classes) # single_cls and include_class + self.ni = len(self.labels) # number of images + self.rect = rect + self.batch_size = batch_size + self.stride = stride + self.pad = pad + if self.rect: + assert self.batch_size is not None + self.set_rectangle() + + # Buffer thread for mosaic images + self.buffer = [] # buffer size = batch size + self.max_buffer_length = min((self.ni, self.batch_size * 8, 1000)) if self.augment else 0 + + # Cache images + if cache == "ram" and not self.check_cache_ram(): + cache = False + self.ims, self.im_hw0, self.im_hw = [None] * self.ni, [None] * self.ni, [None] * self.ni + self.npy_files = [Path(f).with_suffix(".npy") for f in self.im_files] + if cache: + self.cache_images(cache) + + # Transforms + self.transforms = self.build_transforms(hyp=hyp) + + def get_img_files(self, img_path): + """Read image files.""" + try: + f = [] # image files + for p in img_path if isinstance(img_path, list) else [img_path]: + p = Path(p) # os-agnostic + if p.is_dir(): # dir + f += glob.glob(str(p / "**" / "*.*"), recursive=True) + # F = list(p.rglob('*.*')) # pathlib + elif p.is_file(): # file + with open(p) as t: + t = t.read().strip().splitlines() + parent = str(p.parent) + os.sep + f += [x.replace("./", parent) if x.startswith("./") else x for x in t] # local to global path + # F += [p.parent / x.lstrip(os.sep) for x in t] # local to global path (pathlib) + else: + raise FileNotFoundError(f"{self.prefix}{p} does not exist") + im_files = sorted(x.replace("/", os.sep) for x in f if x.split(".")[-1].lower() in IMG_FORMATS) + # self.img_files = sorted([x for x in f if x.suffix[1:].lower() in IMG_FORMATS]) # pathlib + assert im_files, f"{self.prefix}No images found in {img_path}" + except Exception as e: + raise FileNotFoundError(f"{self.prefix}Error loading data from {img_path}\n{HELP_URL}") from e + if self.fraction < 1: + # im_files = im_files[: round(len(im_files) * self.fraction)] + num_elements_to_select = round(len(im_files) * self.fraction) + im_files = random.sample(im_files, num_elements_to_select) + return im_files + + def update_labels(self, include_class: Optional[list]): + """Update labels to include only these classes (optional).""" + include_class_array = np.array(include_class).reshape(1, -1) + for i in range(len(self.labels)): + if include_class is not None: + cls = self.labels[i]["cls"] + bboxes = self.labels[i]["bboxes"] + segments = self.labels[i]["segments"] + keypoints = self.labels[i]["keypoints"] + j = (cls == include_class_array).any(1) + self.labels[i]["cls"] = cls[j] + self.labels[i]["bboxes"] = bboxes[j] + if segments: + self.labels[i]["segments"] = [segments[si] for si, idx in enumerate(j) if idx] + if keypoints is not None: + self.labels[i]["keypoints"] = keypoints[j] + if self.single_cls: + self.labels[i]["cls"][:, 0] = 0 + + def load_image(self, i, rect_mode=True): + """Loads 1 image from dataset index 'i', returns (im, resized hw).""" + im, f, fn = self.ims[i], self.im_files[i], self.npy_files[i] + if im is None: # not cached in RAM + if fn.exists(): # load npy + try: + im = np.load(fn) + except Exception as e: + LOGGER.warning(f"{self.prefix}WARNING ⚠️ Removing corrupt *.npy image file {fn} due to: {e}") + Path(fn).unlink(missing_ok=True) + im = cv2.imread(f) # BGR + else: # read image + im = cv2.imread(f) # BGR + if im is None: + raise FileNotFoundError(f"Image Not Found {f}") + + h0, w0 = im.shape[:2] # orig hw + if rect_mode: # resize long side to imgsz while maintaining aspect ratio + r = self.imgsz / max(h0, w0) # ratio + if r != 1: # if sizes are not equal + w, h = (min(math.ceil(w0 * r), self.imgsz), min(math.ceil(h0 * r), self.imgsz)) + im = cv2.resize(im, (w, h), interpolation=cv2.INTER_LINEAR) + elif not (h0 == w0 == self.imgsz): # resize by stretching image to square imgsz + im = cv2.resize(im, (self.imgsz, self.imgsz), interpolation=cv2.INTER_LINEAR) + + # Add to buffer if training with augmentations + if self.augment: + self.ims[i], self.im_hw0[i], self.im_hw[i] = im, (h0, w0), im.shape[:2] # im, hw_original, hw_resized + self.buffer.append(i) + if len(self.buffer) >= self.max_buffer_length: + j = self.buffer.pop(0) + self.ims[j], self.im_hw0[j], self.im_hw[j] = None, None, None + + return im, (h0, w0), im.shape[:2] + + return self.ims[i], self.im_hw0[i], self.im_hw[i] + + def cache_images(self, cache): + """Cache images to memory or disk.""" + b, gb = 0, 1 << 30 # bytes of cached images, bytes per gigabytes + fcn = self.cache_images_to_disk if cache == "disk" else self.load_image + with ThreadPool(NUM_THREADS) as pool: + results = pool.imap(fcn, range(self.ni)) + pbar = TQDM(enumerate(results), total=self.ni, disable=LOCAL_RANK > 0) + for i, x in pbar: + if cache == "disk": + b += self.npy_files[i].stat().st_size + else: # 'ram' + self.ims[i], self.im_hw0[i], self.im_hw[i] = x # im, hw_orig, hw_resized = load_image(self, i) + b += self.ims[i].nbytes + pbar.desc = f"{self.prefix}Caching images ({b / gb:.1f}GB {cache})" + pbar.close() + + def cache_images_to_disk(self, i): + """Saves an image as an *.npy file for faster loading.""" + f = self.npy_files[i] + if not f.exists(): + np.save(f.as_posix(), cv2.imread(self.im_files[i]), allow_pickle=False) + + def check_cache_ram(self, safety_margin=0.5): + """Check image caching requirements vs available memory.""" + b, gb = 0, 1 << 30 # bytes of cached images, bytes per gigabytes + n = min(self.ni, 30) # extrapolate from 30 random images + for _ in range(n): + im = cv2.imread(random.choice(self.im_files)) # sample image + ratio = self.imgsz / max(im.shape[0], im.shape[1]) # max(h, w) # ratio + b += im.nbytes * ratio**2 + mem_required = b * self.ni / n * (1 + safety_margin) # GB required to cache dataset into RAM + mem = psutil.virtual_memory() + cache = mem_required < mem.available # to cache or not to cache, that is the question + if not cache: + LOGGER.info( + f'{self.prefix}{mem_required / gb:.1f}GB RAM required to cache images ' + f'with {int(safety_margin * 100)}% safety margin but only ' + f'{mem.available / gb:.1f}/{mem.total / gb:.1f}GB available, ' + f"{'caching images ✅' if cache else 'not caching images ⚠️'}" + ) + return cache + + def set_rectangle(self): + """Sets the shape of bounding boxes for YOLO detections as rectangles.""" + bi = np.floor(np.arange(self.ni) / self.batch_size).astype(int) # batch index + nb = bi[-1] + 1 # number of batches + + s = np.array([x.pop("shape") for x in self.labels]) # hw + ar = s[:, 0] / s[:, 1] # aspect ratio + irect = ar.argsort() + self.im_files = [self.im_files[i] for i in irect] + self.labels = [self.labels[i] for i in irect] + ar = ar[irect] + + # Set training image shapes + shapes = [[1, 1]] * nb + for i in range(nb): + ari = ar[bi == i] + mini, maxi = ari.min(), ari.max() + if maxi < 1: + shapes[i] = [maxi, 1] + elif mini > 1: + shapes[i] = [1, 1 / mini] + + self.batch_shapes = np.ceil(np.array(shapes) * self.imgsz / self.stride + self.pad).astype(int) * self.stride + self.batch = bi # batch index of image + + def __getitem__(self, index): + """Returns transformed label information for given index.""" + return self.transforms(self.get_image_and_label(index)) + + def get_image_and_label(self, index): + """Get and return label information from the dataset.""" + label = deepcopy(self.labels[index]) # requires deepcopy() https://github.com/ultralytics/ultralytics/pull/1948 + label.pop("shape", None) # shape is for rect, remove it + label["img"], label["ori_shape"], label["resized_shape"] = self.load_image(index) + label["ratio_pad"] = ( + label["resized_shape"][0] / label["ori_shape"][0], + label["resized_shape"][1] / label["ori_shape"][1], + ) # for evaluation + if self.rect: + label["rect_shape"] = self.batch_shapes[self.batch[index]] + return self.update_labels_info(label) + + def __len__(self): + """Returns the length of the labels list for the dataset.""" + return len(self.labels) + + def update_labels_info(self, label): + """Custom your label format here.""" + return label + + def build_transforms(self, hyp=None): + """ + Users can customize augmentations here. + + Example: + ```python + if self.augment: + # Training transforms + return Compose([]) + else: + # Val transforms + return Compose([]) + ``` + """ + raise NotImplementedError + + def get_labels(self): + """ + Users can customize their own format here. + + Note: + Ensure output is a dictionary with the following keys: + ```python + dict( + im_file=im_file, + shape=shape, # format: (height, width) + cls=cls, + bboxes=bboxes, # xywh + segments=segments, # xy + keypoints=keypoints, # xy + normalized=True, # or False + bbox_format="xyxy", # or xywh, ltwh + ) + ``` + """ + raise NotImplementedError diff --git a/ultralytics/data/build.py b/ultralytics/data/build.py new file mode 100644 index 0000000..6bfb48f --- /dev/null +++ b/ultralytics/data/build.py @@ -0,0 +1,186 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import os +import random +from pathlib import Path + +import numpy as np +import torch +from PIL import Image +from torch.utils.data import dataloader, distributed + +from ultralytics.data.loaders import ( + LOADERS, + LoadImagesAndVideos, + LoadPilAndNumpy, + LoadScreenshots, + LoadStreams, + LoadTensor, + SourceTypes, + autocast_list, +) +from ultralytics.data.utils import IMG_FORMATS, VID_FORMATS +from ultralytics.utils import RANK, colorstr +from ultralytics.utils.checks import check_file +from .dataset import YOLODataset +from .utils import PIN_MEMORY + + +class InfiniteDataLoader(dataloader.DataLoader): + """ + Dataloader that reuses workers. + + Uses same syntax as vanilla DataLoader. + """ + + def __init__(self, *args, **kwargs): + """Dataloader that infinitely recycles workers, inherits from DataLoader.""" + super().__init__(*args, **kwargs) + object.__setattr__(self, "batch_sampler", _RepeatSampler(self.batch_sampler)) + self.iterator = super().__iter__() + + def __len__(self): + """Returns the length of the batch sampler's sampler.""" + return len(self.batch_sampler.sampler) + + def __iter__(self): + """Creates a sampler that repeats indefinitely.""" + for _ in range(len(self)): + yield next(self.iterator) + + def reset(self): + """ + Reset iterator. + + This is useful when we want to modify settings of dataset while training. + """ + self.iterator = self._get_iterator() + + +class _RepeatSampler: + """ + Sampler that repeats forever. + + Args: + sampler (Dataset.sampler): The sampler to repeat. + """ + + def __init__(self, sampler): + """Initializes an object that repeats a given sampler indefinitely.""" + self.sampler = sampler + + def __iter__(self): + """Iterates over the 'sampler' and yields its contents.""" + while True: + yield from iter(self.sampler) + + +def seed_worker(worker_id): # noqa + """Set dataloader worker seed https://pytorch.org/docs/stable/notes/randomness.html#dataloader.""" + worker_seed = torch.initial_seed() % 2**32 + np.random.seed(worker_seed) + random.seed(worker_seed) + + +def build_yolo_dataset(cfg, img_path, batch, data, mode="train", rect=False, stride=32): + """Build YOLO Dataset.""" + return YOLODataset( + img_path=img_path, + imgsz=cfg.imgsz, + batch_size=batch, + augment=mode == "train", # augmentation + hyp=cfg, # TODO: probably add a get_hyps_from_cfg function + rect=cfg.rect or rect, # rectangular batches + cache=cfg.cache or None, + single_cls=cfg.single_cls or False, + stride=int(stride), + pad=0.0 if mode == "train" else 0.5, + prefix=colorstr(f"{mode}: "), + task=cfg.task, + classes=cfg.classes, + data=data, + fraction=cfg.fraction if mode == "train" else 1.0, + ) + + +def build_dataloader(dataset, batch, workers, shuffle=True, rank=-1): + """Return an InfiniteDataLoader or DataLoader for training or validation set.""" + batch = min(batch, len(dataset)) + nd = torch.cuda.device_count() # number of CUDA devices + nw = min([os.cpu_count() // max(nd, 1), workers]) # number of workers + sampler = None if rank == -1 else distributed.DistributedSampler(dataset, shuffle=shuffle) + generator = torch.Generator() + generator.manual_seed(6148914691236517205 + RANK) + return InfiniteDataLoader( + dataset=dataset, + batch_size=batch, + shuffle=shuffle and sampler is None, + num_workers=nw, + sampler=sampler, + pin_memory=PIN_MEMORY, + collate_fn=getattr(dataset, "collate_fn", None), + worker_init_fn=seed_worker, + generator=generator, + ) + + +def check_source(source): + """Check source type and return corresponding flag values.""" + webcam, screenshot, from_img, in_memory, tensor = False, False, False, False, False + if isinstance(source, (str, int, Path)): # int for local usb camera + source = str(source) + is_file = Path(source).suffix[1:] in (IMG_FORMATS | VID_FORMATS) + is_url = source.lower().startswith(("https://", "http://", "rtsp://", "rtmp://", "tcp://")) + webcam = source.isnumeric() or source.endswith(".streams") or (is_url and not is_file) + screenshot = source.lower() == "screen" + if is_url and is_file: + source = check_file(source) # download + elif isinstance(source, LOADERS): + in_memory = True + elif isinstance(source, (list, tuple)): + source = autocast_list(source) # convert all list elements to PIL or np arrays + from_img = True + elif isinstance(source, (Image.Image, np.ndarray)): + from_img = True + elif isinstance(source, torch.Tensor): + tensor = True + else: + raise TypeError("Unsupported image type. For supported types see https://docs.ultralytics.com/modes/predict") + + return source, webcam, screenshot, from_img, in_memory, tensor + + +def load_inference_source(source=None, batch=1, vid_stride=1, buffer=False): + """ + Loads an inference source for object detection and applies necessary transformations. + + Args: + source (str, Path, Tensor, PIL.Image, np.ndarray): The input source for inference. + batch (int, optional): Batch size for dataloaders. Default is 1. + vid_stride (int, optional): The frame interval for video sources. Default is 1. + buffer (bool, optional): Determined whether stream frames will be buffered. Default is False. + + Returns: + dataset (Dataset): A dataset object for the specified input source. + """ + source, stream, screenshot, from_img, in_memory, tensor = check_source(source) + source_type = source.source_type if in_memory else SourceTypes(stream, screenshot, from_img, tensor) + + # Dataloader + if tensor: + dataset = LoadTensor(source) + elif in_memory: + dataset = source + elif stream: + dataset = LoadStreams(source, vid_stride=vid_stride, buffer=buffer) + elif screenshot: + dataset = LoadScreenshots(source) + elif from_img: + dataset = LoadPilAndNumpy(source) + else: + dataset = LoadImagesAndVideos(source, batch=batch, vid_stride=vid_stride) + + # Attach source types to the dataset + setattr(dataset, "source_type", source_type) + + return dataset diff --git a/ultralytics/data/converter.py b/ultralytics/data/converter.py new file mode 100644 index 0000000..eff4dac --- /dev/null +++ b/ultralytics/data/converter.py @@ -0,0 +1,542 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import json +from collections import defaultdict +from pathlib import Path + +import cv2 +import numpy as np + +from ultralytics.utils import LOGGER, TQDM +from ultralytics.utils.files import increment_path + + +def coco91_to_coco80_class(): + """ + Converts 91-index COCO class IDs to 80-index COCO class IDs. + + Returns: + (list): A list of 91 class IDs where the index represents the 80-index class ID and the value is the + corresponding 91-index class ID. + """ + return [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + None, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + None, + 24, + 25, + None, + None, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + None, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + None, + 60, + None, + None, + 61, + None, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + None, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + None, + ] + + +def coco80_to_coco91_class(): + """ + Converts 80-index (val2014) to 91-index (paper). + For details see https://tech.amikelive.com/node-718/what-object-categories-labels-are-in-coco-dataset/. + + Example: + ```python + import numpy as np + + a = np.loadtxt('data/coco.names', dtype='str', delimiter='\n') + b = np.loadtxt('data/coco_paper.names', dtype='str', delimiter='\n') + x1 = [list(a[i] == b).index(True) + 1 for i in range(80)] # darknet to coco + x2 = [list(b[i] == a).index(True) if any(b[i] == a) else None for i in range(91)] # coco to darknet + ``` + """ + return [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 27, + 28, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 67, + 70, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + ] + + +def convert_coco( + labels_dir="../coco/annotations/", + save_dir="coco_converted/", + use_segments=False, + use_keypoints=False, + cls91to80=True, +): + """ + Converts COCO dataset annotations to a YOLO annotation format suitable for training YOLO models. + + Args: + labels_dir (str, optional): Path to directory containing COCO dataset annotation files. + save_dir (str, optional): Path to directory to save results to. + use_segments (bool, optional): Whether to include segmentation masks in the output. + use_keypoints (bool, optional): Whether to include keypoint annotations in the output. + cls91to80 (bool, optional): Whether to map 91 COCO class IDs to the corresponding 80 COCO class IDs. + + Example: + ```python + from ultralytics.data.converter import convert_coco + + convert_coco('../datasets/coco/annotations/', use_segments=True, use_keypoints=False, cls91to80=True) + ``` + + Output: + Generates output files in the specified output directory. + """ + + # Create dataset directory + save_dir = increment_path(save_dir) # increment if save directory already exists + for p in save_dir / "labels", save_dir / "images": + p.mkdir(parents=True, exist_ok=True) # make dir + + # Convert classes + coco80 = coco91_to_coco80_class() + + # Import json + for json_file in sorted(Path(labels_dir).resolve().glob("*.json")): + fn = Path(save_dir) / "labels" / json_file.stem.replace("instances_", "") # folder name + fn.mkdir(parents=True, exist_ok=True) + with open(json_file) as f: + data = json.load(f) + + # Create image dict + images = {f'{x["id"]:d}': x for x in data["images"]} + # Create image-annotations dict + imgToAnns = defaultdict(list) + for ann in data["annotations"]: + imgToAnns[ann["image_id"]].append(ann) + + # Write labels file + for img_id, anns in TQDM(imgToAnns.items(), desc=f"Annotations {json_file}"): + img = images[f"{img_id:d}"] + h, w, f = img["height"], img["width"], img["file_name"] + + bboxes = [] + segments = [] + keypoints = [] + for ann in anns: + if ann["iscrowd"]: + continue + # The COCO box format is [top left x, top left y, width, height] + box = np.array(ann["bbox"], dtype=np.float64) + box[:2] += box[2:] / 2 # xy top-left corner to center + box[[0, 2]] /= w # normalize x + box[[1, 3]] /= h # normalize y + if box[2] <= 0 or box[3] <= 0: # if w <= 0 and h <= 0 + continue + + cls = coco80[ann["category_id"] - 1] if cls91to80 else ann["category_id"] - 1 # class + box = [cls] + box.tolist() + if box not in bboxes: + bboxes.append(box) + if use_segments and ann.get("segmentation") is not None: + if len(ann["segmentation"]) == 0: + segments.append([]) + continue + elif len(ann["segmentation"]) > 1: + s = merge_multi_segment(ann["segmentation"]) + s = (np.concatenate(s, axis=0) / np.array([w, h])).reshape(-1).tolist() + else: + s = [j for i in ann["segmentation"] for j in i] # all segments concatenated + s = (np.array(s).reshape(-1, 2) / np.array([w, h])).reshape(-1).tolist() + s = [cls] + s + segments.append(s) + if use_keypoints and ann.get("keypoints") is not None: + keypoints.append( + box + (np.array(ann["keypoints"]).reshape(-1, 3) / np.array([w, h, 1])).reshape(-1).tolist() + ) + + # Write + with open((fn / f).with_suffix(".txt"), "a") as file: + for i in range(len(bboxes)): + if use_keypoints: + line = (*(keypoints[i]),) # cls, box, keypoints + else: + line = ( + *(segments[i] if use_segments and len(segments[i]) > 0 else bboxes[i]), + ) # cls, box or segments + file.write(("%g " * len(line)).rstrip() % line + "\n") + + LOGGER.info(f"COCO data converted successfully.\nResults saved to {save_dir.resolve()}") + + +def convert_dota_to_yolo_obb(dota_root_path: str): + """ + Converts DOTA dataset annotations to YOLO OBB (Oriented Bounding Box) format. + + The function processes images in the 'train' and 'val' folders of the DOTA dataset. For each image, it reads the + associated label from the original labels directory and writes new labels in YOLO OBB format to a new directory. + + Args: + dota_root_path (str): The root directory path of the DOTA dataset. + + Example: + ```python + from ultralytics.data.converter import convert_dota_to_yolo_obb + + convert_dota_to_yolo_obb('path/to/DOTA') + ``` + + Notes: + The directory structure assumed for the DOTA dataset: + + - DOTA + ├─ images + │ ├─ train + │ └─ val + └─ labels + ├─ train_original + └─ val_original + + After execution, the function will organize the labels into: + + - DOTA + └─ labels + ├─ train + └─ val + """ + dota_root_path = Path(dota_root_path) + + # Class names to indices mapping + class_mapping = { + "plane": 0, + "ship": 1, + "storage-tank": 2, + "baseball-diamond": 3, + "tennis-court": 4, + "basketball-court": 5, + "ground-track-field": 6, + "harbor": 7, + "bridge": 8, + "large-vehicle": 9, + "small-vehicle": 10, + "helicopter": 11, + "roundabout": 12, + "soccer-ball-field": 13, + "swimming-pool": 14, + "container-crane": 15, + "airport": 16, + "helipad": 17, + } + + def convert_label(image_name, image_width, image_height, orig_label_dir, save_dir): + """Converts a single image's DOTA annotation to YOLO OBB format and saves it to a specified directory.""" + orig_label_path = orig_label_dir / f"{image_name}.txt" + save_path = save_dir / f"{image_name}.txt" + + with orig_label_path.open("r") as f, save_path.open("w") as g: + lines = f.readlines() + for line in lines: + parts = line.strip().split() + if len(parts) < 9: + continue + class_name = parts[8] + class_idx = class_mapping[class_name] + coords = [float(p) for p in parts[:8]] + normalized_coords = [ + coords[i] / image_width if i % 2 == 0 else coords[i] / image_height for i in range(8) + ] + formatted_coords = ["{:.6g}".format(coord) for coord in normalized_coords] + g.write(f"{class_idx} {' '.join(formatted_coords)}\n") + + for phase in ["train", "val"]: + image_dir = dota_root_path / "images" / phase + orig_label_dir = dota_root_path / "labels" / f"{phase}_original" + save_dir = dota_root_path / "labels" / phase + + save_dir.mkdir(parents=True, exist_ok=True) + + image_paths = list(image_dir.iterdir()) + for image_path in TQDM(image_paths, desc=f"Processing {phase} images"): + if image_path.suffix != ".png": + continue + image_name_without_ext = image_path.stem + img = cv2.imread(str(image_path)) + h, w = img.shape[:2] + convert_label(image_name_without_ext, w, h, orig_label_dir, save_dir) + + +def min_index(arr1, arr2): + """ + Find a pair of indexes with the shortest distance between two arrays of 2D points. + + Args: + arr1 (np.ndarray): A NumPy array of shape (N, 2) representing N 2D points. + arr2 (np.ndarray): A NumPy array of shape (M, 2) representing M 2D points. + + Returns: + (tuple): A tuple containing the indexes of the points with the shortest distance in arr1 and arr2 respectively. + """ + dis = ((arr1[:, None, :] - arr2[None, :, :]) ** 2).sum(-1) + return np.unravel_index(np.argmin(dis, axis=None), dis.shape) + + +def merge_multi_segment(segments): + """ + Merge multiple segments into one list by connecting the coordinates with the minimum distance between each segment. + This function connects these coordinates with a thin line to merge all segments into one. + + Args: + segments (List[List]): Original segmentations in COCO's JSON file. + Each element is a list of coordinates, like [segmentation1, segmentation2,...]. + + Returns: + s (List[np.ndarray]): A list of connected segments represented as NumPy arrays. + """ + s = [] + segments = [np.array(i).reshape(-1, 2) for i in segments] + idx_list = [[] for _ in range(len(segments))] + + # Record the indexes with min distance between each segment + for i in range(1, len(segments)): + idx1, idx2 = min_index(segments[i - 1], segments[i]) + idx_list[i - 1].append(idx1) + idx_list[i].append(idx2) + + # Use two round to connect all the segments + for k in range(2): + # Forward connection + if k == 0: + for i, idx in enumerate(idx_list): + # Middle segments have two indexes, reverse the index of middle segments + if len(idx) == 2 and idx[0] > idx[1]: + idx = idx[::-1] + segments[i] = segments[i][::-1, :] + + segments[i] = np.roll(segments[i], -idx[0], axis=0) + segments[i] = np.concatenate([segments[i], segments[i][:1]]) + # Deal with the first segment and the last one + if i in [0, len(idx_list) - 1]: + s.append(segments[i]) + else: + idx = [0, idx[1] - idx[0]] + s.append(segments[i][idx[0] : idx[1] + 1]) + + else: + for i in range(len(idx_list) - 1, -1, -1): + if i not in [0, len(idx_list) - 1]: + idx = idx_list[i] + nidx = abs(idx[1] - idx[0]) + s.append(segments[i][nidx:]) + return s + + +def yolo_bbox2segment(im_dir, save_dir=None, sam_model="sam_b.pt"): + """ + Converts existing object detection dataset (bounding boxes) to segmentation dataset or oriented bounding box (OBB) + in YOLO format. Generates segmentation data using SAM auto-annotator as needed. + + Args: + im_dir (str | Path): Path to image directory to convert. + save_dir (str | Path): Path to save the generated labels, labels will be saved + into `labels-segment` in the same directory level of `im_dir` if save_dir is None. Default: None. + sam_model (str): Segmentation model to use for intermediate segmentation data; optional. + + Notes: + The input directory structure assumed for dataset: + + - im_dir + ├─ 001.jpg + ├─ .. + └─ NNN.jpg + - labels + ├─ 001.txt + ├─ .. + └─ NNN.txt + """ + from ultralytics.data import YOLODataset + from ultralytics.utils.ops import xywh2xyxy + from ultralytics.utils import LOGGER + from ultralytics import SAM + from tqdm import tqdm + + # NOTE: add placeholder to pass class index check + dataset = YOLODataset(im_dir, data=dict(names=list(range(1000)))) + if len(dataset.labels[0]["segments"]) > 0: # if it's segment data + LOGGER.info("Segmentation labels detected, no need to generate new ones!") + return + + LOGGER.info("Detection labels detected, generating segment labels by SAM model!") + sam_model = SAM(sam_model) + for l in tqdm(dataset.labels, total=len(dataset.labels), desc="Generating segment labels"): + h, w = l["shape"] + boxes = l["bboxes"] + if len(boxes) == 0: # skip empty labels + continue + boxes[:, [0, 2]] *= w + boxes[:, [1, 3]] *= h + im = cv2.imread(l["im_file"]) + sam_results = sam_model(im, bboxes=xywh2xyxy(boxes), verbose=False, save=False) + l["segments"] = sam_results[0].masks.xyn + + save_dir = Path(save_dir) if save_dir else Path(im_dir).parent / "labels-segment" + save_dir.mkdir(parents=True, exist_ok=True) + for l in dataset.labels: + texts = [] + lb_name = Path(l["im_file"]).with_suffix(".txt").name + txt_file = save_dir / lb_name + cls = l["cls"] + for i, s in enumerate(l["segments"]): + line = (int(cls[i]), *s.reshape(-1)) + texts.append(("%g " * len(line)).rstrip() % line) + if texts: + with open(txt_file, "a") as f: + f.writelines(text + "\n" for text in texts) + LOGGER.info(f"Generated segment labels saved in {save_dir}") diff --git a/ultralytics/data/dataset.py b/ultralytics/data/dataset.py new file mode 100644 index 0000000..42b7cc1 --- /dev/null +++ b/ultralytics/data/dataset.py @@ -0,0 +1,383 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +import contextlib +from itertools import repeat +from multiprocessing.pool import ThreadPool +from pathlib import Path + +import cv2 +import numpy as np +import torch +import torchvision +from PIL import Image + +from ultralytics.utils import LOCAL_RANK, NUM_THREADS, TQDM, colorstr, is_dir_writeable +from ultralytics.utils.ops import resample_segments +from .augment import Compose, Format, Instances, LetterBox, classify_augmentations, classify_transforms, v8_transforms +from .base import BaseDataset +from .utils import HELP_URL, LOGGER, get_hash, img2label_paths, verify_image, verify_image_label + +# Ultralytics dataset *.cache version, >= 1.0.0 for YOLOv8 +DATASET_CACHE_VERSION = "1.0.3" + + +class YOLODataset(BaseDataset): + """ + Dataset class for loading object detection and/or segmentation labels in YOLO format. + + Args: + data (dict, optional): A dataset YAML dictionary. Defaults to None. + task (str): An explicit arg to point current task, Defaults to 'detect'. + + Returns: + (torch.utils.data.Dataset): A PyTorch dataset object that can be used for training an object detection model. + """ + + def __init__(self, *args, data=None, task="detect", **kwargs): + """Initializes the YOLODataset with optional configurations for segments and keypoints.""" + self.use_segments = task == "segment" + self.use_keypoints = task == "pose" + self.use_obb = task == "obb" + self.data = data + assert not (self.use_segments and self.use_keypoints), "Can not use both segments and keypoints." + super().__init__(*args, **kwargs) + + def cache_labels(self, path=Path("./labels.cache")): + """ + Cache dataset labels, check images and read shapes. + + Args: + path (Path): Path where to save the cache file. Default is Path('./labels.cache'). + + Returns: + (dict): labels. + """ + x = {"labels": []} + nm, nf, ne, nc, msgs = 0, 0, 0, 0, [] # number missing, found, empty, corrupt, messages + desc = f"{self.prefix}Scanning {path.parent / path.stem}..." + total = len(self.im_files) + nkpt, ndim = self.data.get("kpt_shape", (0, 0)) + if self.use_keypoints and (nkpt <= 0 or ndim not in (2, 3)): + raise ValueError( + "'kpt_shape' in data.yaml missing or incorrect. Should be a list with [number of " + "keypoints, number of dims (2 for x,y or 3 for x,y,visible)], i.e. 'kpt_shape: [17, 3]'" + ) + with ThreadPool(NUM_THREADS) as pool: + results = pool.imap( + func=verify_image_label, + iterable=zip( + self.im_files, + self.label_files, + repeat(self.prefix), + repeat(self.use_keypoints), + repeat(len(self.data["names"])), + repeat(nkpt), + repeat(ndim), + ), + ) + pbar = TQDM(results, desc=desc, total=total) + for im_file, lb, shape, segments, keypoint, nm_f, nf_f, ne_f, nc_f, msg in pbar: + nm += nm_f + nf += nf_f + ne += ne_f + nc += nc_f + if im_file: + x["labels"].append( + dict( + im_file=im_file, + shape=shape, + cls=lb[:, 0:1], # n, 1 + bboxes=lb[:, 1:], # n, 4 + segments=segments, + keypoints=keypoint, + normalized=True, + bbox_format="xywh", + ) + ) + if msg: + msgs.append(msg) + pbar.desc = f"{desc} {nf} images, {nm + ne} backgrounds, {nc} corrupt" + pbar.close() + + if msgs: + LOGGER.info("\n".join(msgs)) + if nf == 0: + LOGGER.warning(f"{self.prefix}WARNING ⚠️ No labels found in {path}. {HELP_URL}") + x["hash"] = get_hash(self.label_files + self.im_files) + x["results"] = nf, nm, ne, nc, len(self.im_files) + x["msgs"] = msgs # warnings + save_dataset_cache_file(self.prefix, path, x) + return x + + def get_labels(self): + """Returns dictionary of labels for YOLO training.""" + self.label_files = img2label_paths(self.im_files) + cache_path = Path(self.label_files[0]).parent.with_suffix(".cache") + try: + cache, exists = load_dataset_cache_file(cache_path), True # attempt to load a *.cache file + assert cache["version"] == DATASET_CACHE_VERSION # matches current version + assert cache["hash"] == get_hash(self.label_files + self.im_files) # identical hash + except (FileNotFoundError, AssertionError, AttributeError): + cache, exists = self.cache_labels(cache_path), False # run cache ops + + # Display cache + nf, nm, ne, nc, n = cache.pop("results") # found, missing, empty, corrupt, total + if exists and LOCAL_RANK in (-1, 0): + d = f"Scanning {cache_path}... {nf} images, {nm + ne} backgrounds, {nc} corrupt" + TQDM(None, desc=self.prefix + d, total=n, initial=n) # display results + if cache["msgs"]: + LOGGER.info("\n".join(cache["msgs"])) # display warnings + + # Read cache + [cache.pop(k) for k in ("hash", "version", "msgs")] # remove items + labels = cache["labels"] + if not labels: + LOGGER.warning(f"WARNING ⚠️ No images found in {cache_path}, training may not work correctly. {HELP_URL}") + self.im_files = [lb["im_file"] for lb in labels] # update im_files + + # Check if the dataset is all boxes or all segments + lengths = ((len(lb["cls"]), len(lb["bboxes"]), len(lb["segments"])) for lb in labels) + len_cls, len_boxes, len_segments = (sum(x) for x in zip(*lengths)) + if len_segments and len_boxes != len_segments: + LOGGER.warning( + f"WARNING ⚠️ Box and segment counts should be equal, but got len(segments) = {len_segments}, " + f"len(boxes) = {len_boxes}. To resolve this only boxes will be used and all segments will be removed. " + "To avoid this please supply either a detect or segment dataset, not a detect-segment mixed dataset." + ) + for lb in labels: + lb["segments"] = [] + if len_cls == 0: + LOGGER.warning(f"WARNING ⚠️ No labels found in {cache_path}, training may not work correctly. {HELP_URL}") + return labels + + def build_transforms(self, hyp=None): + """Builds and appends transforms to the list.""" + if self.augment: + hyp.mosaic = hyp.mosaic if self.augment and not self.rect else 0.0 + hyp.mixup = hyp.mixup if self.augment and not self.rect else 0.0 + transforms = v8_transforms(self, self.imgsz, hyp) + else: + transforms = Compose([LetterBox(new_shape=(self.imgsz, self.imgsz), scaleup=False)]) + transforms.append( + Format( + bbox_format="xywh", + normalize=True, + return_mask=self.use_segments, + return_keypoint=self.use_keypoints, + return_obb=self.use_obb, + batch_idx=True, + mask_ratio=hyp.mask_ratio, + mask_overlap=hyp.overlap_mask, + bgr=hyp.bgr if self.augment else 0.0, # only affect training. + ) + ) + return transforms + + def close_mosaic(self, hyp): + """Sets mosaic, copy_paste and mixup options to 0.0 and builds transformations.""" + hyp.mosaic = 0.0 # set mosaic ratio=0.0 + hyp.copy_paste = 0.0 # keep the same behavior as previous v8 close-mosaic + hyp.mixup = 0.0 # keep the same behavior as previous v8 close-mosaic + self.transforms = self.build_transforms(hyp) + + def update_labels_info(self, label): + """ + Custom your label format here. + + Note: + cls is not with bboxes now, classification and semantic segmentation need an independent cls label + Can also support classification and semantic segmentation by adding or removing dict keys there. + """ + bboxes = label.pop("bboxes") + segments = label.pop("segments", []) + keypoints = label.pop("keypoints", None) + bbox_format = label.pop("bbox_format") + normalized = label.pop("normalized") + + # NOTE: do NOT resample oriented boxes + segment_resamples = 100 if self.use_obb else 1000 + if len(segments) > 0: + # list[np.array(1000, 2)] * num_samples + # (N, 1000, 2) + segments = np.stack(resample_segments(segments, n=segment_resamples), axis=0) + else: + segments = np.zeros((0, segment_resamples, 2), dtype=np.float32) + label["instances"] = Instances(bboxes, segments, keypoints, bbox_format=bbox_format, normalized=normalized) + return label + + @staticmethod + def collate_fn(batch): + """Collates data samples into batches.""" + new_batch = {} + keys = batch[0].keys() + values = list(zip(*[list(b.values()) for b in batch])) + for i, k in enumerate(keys): + value = values[i] + if k == "img": + value = torch.stack(value, 0) + if k in ["masks", "keypoints", "bboxes", "cls", "segments", "obb"]: + value = torch.cat(value, 0) + new_batch[k] = value + new_batch["batch_idx"] = list(new_batch["batch_idx"]) + for i in range(len(new_batch["batch_idx"])): + new_batch["batch_idx"][i] += i # add target image index for build_targets() + new_batch["batch_idx"] = torch.cat(new_batch["batch_idx"], 0) + return new_batch + + +# Classification dataloaders ------------------------------------------------------------------------------------------- +class ClassificationDataset(torchvision.datasets.ImageFolder): + """ + Extends torchvision ImageFolder to support YOLO classification tasks, offering functionalities like image + augmentation, caching, and verification. It's designed to efficiently handle large datasets for training deep + learning models, with optional image transformations and caching mechanisms to speed up training. + + This class allows for augmentations using both torchvision and Albumentations libraries, and supports caching images + in RAM or on disk to reduce IO overhead during training. Additionally, it implements a robust verification process + to ensure data integrity and consistency. + + Attributes: + cache_ram (bool): Indicates if caching in RAM is enabled. + cache_disk (bool): Indicates if caching on disk is enabled. + samples (list): A list of tuples, each containing the path to an image, its class index, path to its .npy cache + file (if caching on disk), and optionally the loaded image array (if caching in RAM). + torch_transforms (callable): PyTorch transforms to be applied to the images. + """ + + def __init__(self, root, args, augment=False, prefix=""): + """ + Initialize YOLO object with root, image size, augmentations, and cache settings. + + Args: + root (str): Path to the dataset directory where images are stored in a class-specific folder structure. + args (Namespace): Configuration containing dataset-related settings such as image size, augmentation + parameters, and cache settings. It includes attributes like `imgsz` (image size), `fraction` (fraction + of data to use), `scale`, `fliplr`, `flipud`, `cache` (disk or RAM caching for faster training), + `auto_augment`, `hsv_h`, `hsv_s`, `hsv_v`, and `crop_fraction`. + augment (bool, optional): Whether to apply augmentations to the dataset. Default is False. + prefix (str, optional): Prefix for logging and cache filenames, aiding in dataset identification and + debugging. Default is an empty string. + """ + super().__init__(root=root) + if augment and args.fraction < 1.0: # reduce training fraction + self.samples = self.samples[: round(len(self.samples) * args.fraction)] + self.prefix = colorstr(f"{prefix}: ") if prefix else "" + self.cache_ram = args.cache is True or args.cache == "ram" # cache images into RAM + self.cache_disk = args.cache == "disk" # cache images on hard drive as uncompressed *.npy files + self.samples = self.verify_images() # filter out bad images + self.samples = [list(x) + [Path(x[0]).with_suffix(".npy"), None] for x in self.samples] # file, index, npy, im + scale = (1.0 - args.scale, 1.0) # (0.08, 1.0) + self.torch_transforms = ( + classify_augmentations( + size=args.imgsz, + scale=scale, + hflip=args.fliplr, + vflip=args.flipud, + erasing=args.erasing, + auto_augment=args.auto_augment, + hsv_h=args.hsv_h, + hsv_s=args.hsv_s, + hsv_v=args.hsv_v, + ) + if augment + else classify_transforms(size=args.imgsz, crop_fraction=args.crop_fraction) + ) + + def __getitem__(self, i): + """Returns subset of data and targets corresponding to given indices.""" + f, j, fn, im = self.samples[i] # filename, index, filename.with_suffix('.npy'), image + if self.cache_ram and im is None: + im = self.samples[i][3] = cv2.imread(f) + elif self.cache_disk: + if not fn.exists(): # load npy + np.save(fn.as_posix(), cv2.imread(f), allow_pickle=False) + im = np.load(fn) + else: # read image + im = cv2.imread(f) # BGR + # Convert NumPy array to PIL image + im = Image.fromarray(cv2.cvtColor(im, cv2.COLOR_BGR2RGB)) + sample = self.torch_transforms(im) + return {"img": sample, "cls": j} + + def __len__(self) -> int: + """Return the total number of samples in the dataset.""" + return len(self.samples) + + def verify_images(self): + """Verify all images in dataset.""" + desc = f"{self.prefix}Scanning {self.root}..." + path = Path(self.root).with_suffix(".cache") # *.cache file path + + with contextlib.suppress(FileNotFoundError, AssertionError, AttributeError): + cache = load_dataset_cache_file(path) # attempt to load a *.cache file + assert cache["version"] == DATASET_CACHE_VERSION # matches current version + assert cache["hash"] == get_hash([x[0] for x in self.samples]) # identical hash + nf, nc, n, samples = cache.pop("results") # found, missing, empty, corrupt, total + if LOCAL_RANK in (-1, 0): + d = f"{desc} {nf} images, {nc} corrupt" + TQDM(None, desc=d, total=n, initial=n) + if cache["msgs"]: + LOGGER.info("\n".join(cache["msgs"])) # display warnings + return samples + + # Run scan if *.cache retrieval failed + nf, nc, msgs, samples, x = 0, 0, [], [], {} + with ThreadPool(NUM_THREADS) as pool: + results = pool.imap(func=verify_image, iterable=zip(self.samples, repeat(self.prefix))) + pbar = TQDM(results, desc=desc, total=len(self.samples)) + for sample, nf_f, nc_f, msg in pbar: + if nf_f: + samples.append(sample) + if msg: + msgs.append(msg) + nf += nf_f + nc += nc_f + pbar.desc = f"{desc} {nf} images, {nc} corrupt" + pbar.close() + if msgs: + LOGGER.info("\n".join(msgs)) + x["hash"] = get_hash([x[0] for x in self.samples]) + x["results"] = nf, nc, len(samples), samples + x["msgs"] = msgs # warnings + save_dataset_cache_file(self.prefix, path, x) + return samples + + +def load_dataset_cache_file(path): + """Load an Ultralytics *.cache dictionary from path.""" + import gc + + gc.disable() # reduce pickle load time https://github.com/ultralytics/ultralytics/pull/1585 + cache = np.load(str(path), allow_pickle=True).item() # load dict + gc.enable() + return cache + + +def save_dataset_cache_file(prefix, path, x): + """Save an Ultralytics dataset *.cache dictionary x to path.""" + x["version"] = DATASET_CACHE_VERSION # add cache version + if is_dir_writeable(path.parent): + if path.exists(): + path.unlink() # remove *.cache file if exists + np.save(str(path), x) # save cache for next time + path.with_suffix(".cache.npy").rename(path) # remove .npy suffix + LOGGER.info(f"{prefix}New cache created: {path}") + else: + LOGGER.warning(f"{prefix}WARNING ⚠️ Cache directory {path.parent} is not writeable, cache not saved.") + + +# TODO: support semantic segmentation +class SemanticDataset(BaseDataset): + """ + Semantic Segmentation Dataset. + + This class is responsible for handling datasets used for semantic segmentation tasks. It inherits functionalities + from the BaseDataset class. + + Note: + This class is currently a placeholder and needs to be populated with methods and attributes for supporting + semantic segmentation tasks. + """ + + def __init__(self): + """Initialize a SemanticDataset object.""" + super().__init__() diff --git a/ultralytics/data/explorer/__init__.py b/ultralytics/data/explorer/__init__.py new file mode 100644 index 0000000..ce594dc --- /dev/null +++ b/ultralytics/data/explorer/__init__.py @@ -0,0 +1,5 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from .utils import plot_query_result + +__all__ = ["plot_query_result"] diff --git a/ultralytics/data/explorer/__pycache__/__init__.cpython-312.pyc b/ultralytics/data/explorer/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..79ed03e Binary files /dev/null and b/ultralytics/data/explorer/__pycache__/__init__.cpython-312.pyc differ diff --git a/ultralytics/data/explorer/__pycache__/__init__.cpython-39.pyc b/ultralytics/data/explorer/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..93c0a20 Binary files /dev/null and b/ultralytics/data/explorer/__pycache__/__init__.cpython-39.pyc differ diff --git a/ultralytics/data/explorer/__pycache__/explorer.cpython-312.pyc b/ultralytics/data/explorer/__pycache__/explorer.cpython-312.pyc new file mode 100644 index 0000000..8940c45 Binary files /dev/null and b/ultralytics/data/explorer/__pycache__/explorer.cpython-312.pyc differ diff --git a/ultralytics/data/explorer/__pycache__/explorer.cpython-39.pyc b/ultralytics/data/explorer/__pycache__/explorer.cpython-39.pyc new file mode 100644 index 0000000..c0d300d Binary files /dev/null and b/ultralytics/data/explorer/__pycache__/explorer.cpython-39.pyc differ diff --git a/ultralytics/data/explorer/__pycache__/utils.cpython-312.pyc b/ultralytics/data/explorer/__pycache__/utils.cpython-312.pyc new file mode 100644 index 0000000..7806bbf Binary files /dev/null and b/ultralytics/data/explorer/__pycache__/utils.cpython-312.pyc differ diff --git a/ultralytics/data/explorer/__pycache__/utils.cpython-39.pyc b/ultralytics/data/explorer/__pycache__/utils.cpython-39.pyc new file mode 100644 index 0000000..c17a0e4 Binary files /dev/null and b/ultralytics/data/explorer/__pycache__/utils.cpython-39.pyc differ diff --git a/ultralytics/data/explorer/explorer.py b/ultralytics/data/explorer/explorer.py new file mode 100644 index 0000000..d21a5c2 --- /dev/null +++ b/ultralytics/data/explorer/explorer.py @@ -0,0 +1,472 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from io import BytesIO +from pathlib import Path +from typing import Any, List, Tuple, Union + +import cv2 +import numpy as np +import torch +from PIL import Image +from matplotlib import pyplot as plt +from pandas import DataFrame +from tqdm import tqdm + +from ultralytics.data.augment import Format +from ultralytics.data.dataset import YOLODataset +from ultralytics.data.utils import check_det_dataset +from ultralytics.models.yolo.model import YOLO +from ultralytics.utils import LOGGER, IterableSimpleNamespace, checks, USER_CONFIG_DIR +from .utils import get_sim_index_schema, get_table_schema, plot_query_result, prompt_sql_query, sanitize_batch + + +class ExplorerDataset(YOLODataset): + def __init__(self, *args, data: dict = None, **kwargs) -> None: + super().__init__(*args, data=data, **kwargs) + + def load_image(self, i: int) -> Union[Tuple[np.ndarray, Tuple[int, int], Tuple[int, int]], Tuple[None, None, None]]: + """Loads 1 image from dataset index 'i' without any resize ops.""" + im, f, fn = self.ims[i], self.im_files[i], self.npy_files[i] + if im is None: # not cached in RAM + if fn.exists(): # load npy + im = np.load(fn) + else: # read image + im = cv2.imread(f) # BGR + if im is None: + raise FileNotFoundError(f"Image Not Found {f}") + h0, w0 = im.shape[:2] # orig hw + return im, (h0, w0), im.shape[:2] + + return self.ims[i], self.im_hw0[i], self.im_hw[i] + + def build_transforms(self, hyp: IterableSimpleNamespace = None): + """Creates transforms for dataset images without resizing.""" + return Format( + bbox_format="xyxy", + normalize=False, + return_mask=self.use_segments, + return_keypoint=self.use_keypoints, + batch_idx=True, + mask_ratio=hyp.mask_ratio, + mask_overlap=hyp.overlap_mask, + ) + + +class Explorer: + def __init__( + self, + data: Union[str, Path] = "coco128.yaml", + model: str = "yolov8n.pt", + uri: str = USER_CONFIG_DIR / "explorer", + ) -> None: + # Note duckdb==0.10.0 bug https://github.com/ultralytics/ultralytics/pull/8181 + checks.check_requirements(["lancedb>=0.4.3", "duckdb<=0.9.2"]) + import lancedb + + self.connection = lancedb.connect(uri) + self.table_name = Path(data).name.lower() + "_" + model.lower() + self.sim_idx_base_name = ( + f"{self.table_name}_sim_idx".lower() + ) # Use this name and append thres and top_k to reuse the table + self.model = YOLO(model) + self.data = data # None + self.choice_set = None + + self.table = None + self.progress = 0 + + def create_embeddings_table(self, force: bool = False, split: str = "train") -> None: + """ + Create LanceDB table containing the embeddings of the images in the dataset. The table will be reused if it + already exists. Pass force=True to overwrite the existing table. + + Args: + force (bool): Whether to overwrite the existing table or not. Defaults to False. + split (str): Split of the dataset to use. Defaults to 'train'. + + Example: + ```python + exp = Explorer() + exp.create_embeddings_table() + ``` + """ + if self.table is not None and not force: + LOGGER.info("Table already exists. Reusing it. Pass force=True to overwrite it.") + return + if self.table_name in self.connection.table_names() and not force: + LOGGER.info(f"Table {self.table_name} already exists. Reusing it. Pass force=True to overwrite it.") + self.table = self.connection.open_table(self.table_name) + self.progress = 1 + return + if self.data is None: + raise ValueError("Data must be provided to create embeddings table") + + data_info = check_det_dataset(self.data) + if split not in data_info: + raise ValueError( + f"Split {split} is not found in the dataset. Available keys in the dataset are {list(data_info.keys())}" + ) + + choice_set = data_info[split] + choice_set = choice_set if isinstance(choice_set, list) else [choice_set] + self.choice_set = choice_set + dataset = ExplorerDataset(img_path=choice_set, data=data_info, augment=False, cache=False, task=self.model.task) + + # Create the table schema + batch = dataset[0] + vector_size = self.model.embed(batch["im_file"], verbose=False)[0].shape[0] + table = self.connection.create_table(self.table_name, schema=get_table_schema(vector_size), mode="overwrite") + table.add( + self._yield_batches( + dataset, + data_info, + self.model, + exclude_keys=["img", "ratio_pad", "resized_shape", "ori_shape", "batch_idx"], + ) + ) + + self.table = table + + def _yield_batches(self, dataset: ExplorerDataset, data_info: dict, model: YOLO, exclude_keys: List[str]): + """Generates batches of data for embedding, excluding specified keys.""" + for i in tqdm(range(len(dataset))): + self.progress = float(i + 1) / len(dataset) + batch = dataset[i] + for k in exclude_keys: + batch.pop(k, None) + batch = sanitize_batch(batch, data_info) + batch["vector"] = model.embed(batch["im_file"], verbose=False)[0].detach().tolist() + yield [batch] + + def query( + self, imgs: Union[str, np.ndarray, List[str], List[np.ndarray]] = None, limit: int = 25 + ) -> Any: # pyarrow.Table + """ + Query the table for similar images. Accepts a single image or a list of images. + + Args: + imgs (str or list): Path to the image or a list of paths to the images. + limit (int): Number of results to return. + + Returns: + (pyarrow.Table): An arrow table containing the results. Supports converting to: + - pandas dataframe: `result.to_pandas()` + - dict of lists: `result.to_pydict()` + + Example: + ```python + exp = Explorer() + exp.create_embeddings_table() + similar = exp.query(img='https://ultralytics.com/images/zidane.jpg') + ``` + """ + if self.table is None: + raise ValueError("Table is not created. Please create the table first.") + if isinstance(imgs, str): + imgs = [imgs] + assert isinstance(imgs, list), f"img must be a string or a list of strings. Got {type(imgs)}" + embeds = self.model.embed(imgs) + # Get avg if multiple images are passed (len > 1) + embeds = torch.mean(torch.stack(embeds), 0).cpu().numpy() if len(embeds) > 1 else embeds[0].cpu().numpy() + return self.table.search(embeds).limit(limit).to_arrow() + + def sql_query( + self, query: str, return_type: str = "pandas" + ) -> Union[DataFrame, Any, None]: # pandas.dataframe or pyarrow.Table + """ + Run a SQL-Like query on the table. Utilizes LanceDB predicate pushdown. + + Args: + query (str): SQL query to run. + return_type (str): Type of the result to return. Can be either 'pandas' or 'arrow'. Defaults to 'pandas'. + + Returns: + (pyarrow.Table): An arrow table containing the results. + + Example: + ```python + exp = Explorer() + exp.create_embeddings_table() + query = "SELECT * FROM 'table' WHERE labels LIKE '%person%'" + result = exp.sql_query(query) + ``` + """ + assert return_type in { + "pandas", + "arrow", + }, f"Return type should be either `pandas` or `arrow`, but got {return_type}" + import duckdb + + if self.table is None: + raise ValueError("Table is not created. Please create the table first.") + + # Note: using filter pushdown would be a better long term solution. Temporarily using duckdb for this. + table = self.table.to_arrow() # noqa NOTE: Don't comment this. This line is used by DuckDB + if not query.startswith("SELECT") and not query.startswith("WHERE"): + raise ValueError( + f"Query must start with SELECT or WHERE. You can either pass the entire query or just the WHERE clause. found {query}" + ) + if query.startswith("WHERE"): + query = f"SELECT * FROM 'table' {query}" + LOGGER.info(f"Running query: {query}") + + rs = duckdb.sql(query) + if return_type == "arrow": + return rs.arrow() + elif return_type == "pandas": + return rs.df() + + def plot_sql_query(self, query: str, labels: bool = True) -> Image.Image: + """ + Plot the results of a SQL-Like query on the table. + Args: + query (str): SQL query to run. + labels (bool): Whether to plot the labels or not. + + Returns: + (PIL.Image): Image containing the plot. + + Example: + ```python + exp = Explorer() + exp.create_embeddings_table() + query = "SELECT * FROM 'table' WHERE labels LIKE '%person%'" + result = exp.plot_sql_query(query) + ``` + """ + result = self.sql_query(query, return_type="arrow") + if len(result) == 0: + LOGGER.info("No results found.") + return None + img = plot_query_result(result, plot_labels=labels) + return Image.fromarray(img) + + def get_similar( + self, + img: Union[str, np.ndarray, List[str], List[np.ndarray]] = None, + idx: Union[int, List[int]] = None, + limit: int = 25, + return_type: str = "pandas", + ) -> Union[DataFrame, Any]: # pandas.dataframe or pyarrow.Table + """ + Query the table for similar images. Accepts a single image or a list of images. + + Args: + img (str or list): Path to the image or a list of paths to the images. + idx (int or list): Index of the image in the table or a list of indexes. + limit (int): Number of results to return. Defaults to 25. + return_type (str): Type of the result to return. Can be either 'pandas' or 'arrow'. Defaults to 'pandas'. + + Returns: + (pandas.DataFrame): A dataframe containing the results. + + Example: + ```python + exp = Explorer() + exp.create_embeddings_table() + similar = exp.get_similar(img='https://ultralytics.com/images/zidane.jpg') + ``` + """ + assert return_type in { + "pandas", + "arrow", + }, f"Return type should be either `pandas` or `arrow`, but got {return_type}" + img = self._check_imgs_or_idxs(img, idx) + similar = self.query(img, limit=limit) + + if return_type == "arrow": + return similar + elif return_type == "pandas": + return similar.to_pandas() + + def plot_similar( + self, + img: Union[str, np.ndarray, List[str], List[np.ndarray]] = None, + idx: Union[int, List[int]] = None, + limit: int = 25, + labels: bool = True, + ) -> Image.Image: + """ + Plot the similar images. Accepts images or indexes. + + Args: + img (str or list): Path to the image or a list of paths to the images. + idx (int or list): Index of the image in the table or a list of indexes. + labels (bool): Whether to plot the labels or not. + limit (int): Number of results to return. Defaults to 25. + + Returns: + (PIL.Image): Image containing the plot. + + Example: + ```python + exp = Explorer() + exp.create_embeddings_table() + similar = exp.plot_similar(img='https://ultralytics.com/images/zidane.jpg') + ``` + """ + similar = self.get_similar(img, idx, limit, return_type="arrow") + if len(similar) == 0: + LOGGER.info("No results found.") + return None + img = plot_query_result(similar, plot_labels=labels) + return Image.fromarray(img) + + def similarity_index(self, max_dist: float = 0.2, top_k: float = None, force: bool = False) -> DataFrame: + """ + Calculate the similarity index of all the images in the table. Here, the index will contain the data points that + are max_dist or closer to the image in the embedding space at a given index. + + Args: + max_dist (float): maximum L2 distance between the embeddings to consider. Defaults to 0.2. + top_k (float): Percentage of the closest data points to consider when counting. Used to apply limit when running + vector search. Defaults: None. + force (bool): Whether to overwrite the existing similarity index or not. Defaults to True. + + Returns: + (pandas.DataFrame): A dataframe containing the similarity index. Each row corresponds to an image, and columns + include indices of similar images and their respective distances. + + Example: + ```python + exp = Explorer() + exp.create_embeddings_table() + sim_idx = exp.similarity_index() + ``` + """ + if self.table is None: + raise ValueError("Table is not created. Please create the table first.") + sim_idx_table_name = f"{self.sim_idx_base_name}_thres_{max_dist}_top_{top_k}".lower() + if sim_idx_table_name in self.connection.table_names() and not force: + LOGGER.info("Similarity matrix already exists. Reusing it. Pass force=True to overwrite it.") + return self.connection.open_table(sim_idx_table_name).to_pandas() + + if top_k and not (1.0 >= top_k >= 0.0): + raise ValueError(f"top_k must be between 0.0 and 1.0. Got {top_k}") + if max_dist < 0.0: + raise ValueError(f"max_dist must be greater than 0. Got {max_dist}") + + top_k = int(top_k * len(self.table)) if top_k else len(self.table) + top_k = max(top_k, 1) + features = self.table.to_lance().to_table(columns=["vector", "im_file"]).to_pydict() + im_files = features["im_file"] + embeddings = features["vector"] + + sim_table = self.connection.create_table(sim_idx_table_name, schema=get_sim_index_schema(), mode="overwrite") + + def _yield_sim_idx(): + """Generates a dataframe with similarity indices and distances for images.""" + for i in tqdm(range(len(embeddings))): + sim_idx = self.table.search(embeddings[i]).limit(top_k).to_pandas().query(f"_distance <= {max_dist}") + yield [ + { + "idx": i, + "im_file": im_files[i], + "count": len(sim_idx), + "sim_im_files": sim_idx["im_file"].tolist(), + } + ] + + sim_table.add(_yield_sim_idx()) + self.sim_index = sim_table + return sim_table.to_pandas() + + def plot_similarity_index(self, max_dist: float = 0.2, top_k: float = None, force: bool = False) -> Image: + """ + Plot the similarity index of all the images in the table. Here, the index will contain the data points that are + max_dist or closer to the image in the embedding space at a given index. + + Args: + max_dist (float): maximum L2 distance between the embeddings to consider. Defaults to 0.2. + top_k (float): Percentage of closest data points to consider when counting. Used to apply limit when + running vector search. Defaults to 0.01. + force (bool): Whether to overwrite the existing similarity index or not. Defaults to True. + + Returns: + (PIL.Image): Image containing the plot. + + Example: + ```python + exp = Explorer() + exp.create_embeddings_table() + + similarity_idx_plot = exp.plot_similarity_index() + similarity_idx_plot.show() # view image preview + similarity_idx_plot.save('path/to/save/similarity_index_plot.png') # save contents to file + ``` + """ + sim_idx = self.similarity_index(max_dist=max_dist, top_k=top_k, force=force) + sim_count = sim_idx["count"].tolist() + sim_count = np.array(sim_count) + + indices = np.arange(len(sim_count)) + + # Create the bar plot + plt.bar(indices, sim_count) + + # Customize the plot (optional) + plt.xlabel("data idx") + plt.ylabel("Count") + plt.title("Similarity Count") + buffer = BytesIO() + plt.savefig(buffer, format="png") + buffer.seek(0) + + # Use Pillow to open the image from the buffer + return Image.fromarray(np.array(Image.open(buffer))) + + def _check_imgs_or_idxs( + self, img: Union[str, np.ndarray, List[str], List[np.ndarray], None], idx: Union[None, int, List[int]] + ) -> List[np.ndarray]: + if img is None and idx is None: + raise ValueError("Either img or idx must be provided.") + if img is not None and idx is not None: + raise ValueError("Only one of img or idx must be provided.") + if idx is not None: + idx = idx if isinstance(idx, list) else [idx] + img = self.table.to_lance().take(idx, columns=["im_file"]).to_pydict()["im_file"] + + return img if isinstance(img, list) else [img] + + def ask_ai(self, query): + """ + Ask AI a question. + + Args: + query (str): Question to ask. + + Returns: + (pandas.DataFrame): A dataframe containing filtered results to the SQL query. + + Example: + ```python + exp = Explorer() + exp.create_embeddings_table() + answer = exp.ask_ai('Show images with 1 person and 2 dogs') + ``` + """ + result = prompt_sql_query(query) + try: + df = self.sql_query(result) + except Exception as e: + LOGGER.error("AI generated query is not valid. Please try again with a different prompt") + LOGGER.error(e) + return None + return df + + def visualize(self, result): + """ + Visualize the results of a query. TODO. + + Args: + result (pyarrow.Table): Table containing the results of a query. + """ + pass + + def generate_report(self, result): + """ + Generate a report of the dataset. + + TODO + """ + pass diff --git a/ultralytics/data/explorer/gui/__init__.py b/ultralytics/data/explorer/gui/__init__.py new file mode 100644 index 0000000..9e68dc1 --- /dev/null +++ b/ultralytics/data/explorer/gui/__init__.py @@ -0,0 +1 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license diff --git a/ultralytics/data/explorer/gui/dash.py b/ultralytics/data/explorer/gui/dash.py new file mode 100644 index 0000000..b082d49 --- /dev/null +++ b/ultralytics/data/explorer/gui/dash.py @@ -0,0 +1,268 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import time +from threading import Thread + +import pandas as pd + +from ultralytics import Explorer +from ultralytics.utils import ROOT, SETTINGS +from ultralytics.utils.checks import check_requirements + +check_requirements(("streamlit>=1.29.0", "streamlit-select>=0.3")) + +import streamlit as st +from streamlit_select import image_select + + +def _get_explorer(): + """Initializes and returns an instance of the Explorer class.""" + exp = Explorer(data=st.session_state.get("dataset"), model=st.session_state.get("model")) + thread = Thread( + target=exp.create_embeddings_table, kwargs={"force": st.session_state.get("force_recreate_embeddings")} + ) + thread.start() + progress_bar = st.progress(0, text="Creating embeddings table...") + while exp.progress < 1: + time.sleep(0.1) + progress_bar.progress(exp.progress, text=f"Progress: {exp.progress * 100}%") + thread.join() + st.session_state["explorer"] = exp + progress_bar.empty() + + +def init_explorer_form(): + """Initializes an Explorer instance and creates embeddings table with progress tracking.""" + datasets = ROOT / "cfg" / "datasets" + ds = [d.name for d in datasets.glob("*.yaml")] + models = [ + "yolov8n.pt", + "yolov8s.pt", + "yolov8m.pt", + "yolov8l.pt", + "yolov8x.pt", + "yolov8n-seg.pt", + "yolov8s-seg.pt", + "yolov8m-seg.pt", + "yolov8l-seg.pt", + "yolov8x-seg.pt", + "yolov8n-pose.pt", + "yolov8s-pose.pt", + "yolov8m-pose.pt", + "yolov8l-pose.pt", + "yolov8x-pose.pt", + ] + with st.form(key="explorer_init_form"): + col1, col2 = st.columns(2) + with col1: + st.selectbox("Select dataset", ds, key="dataset", index=ds.index("coco128.yaml")) + with col2: + st.selectbox("Select model", models, key="model") + st.checkbox("Force recreate embeddings", key="force_recreate_embeddings") + + st.form_submit_button("Explore", on_click=_get_explorer) + + +def query_form(): + """Sets up a form in Streamlit to initialize Explorer with dataset and model selection.""" + with st.form("query_form"): + col1, col2 = st.columns([0.8, 0.2]) + with col1: + st.text_input( + "Query", + "WHERE labels LIKE '%person%' AND labels LIKE '%dog%'", + label_visibility="collapsed", + key="query", + ) + with col2: + st.form_submit_button("Query", on_click=run_sql_query) + + +def ai_query_form(): + """Sets up a Streamlit form for user input to initialize Explorer with dataset and model selection.""" + with st.form("ai_query_form"): + col1, col2 = st.columns([0.8, 0.2]) + with col1: + st.text_input("Query", "Show images with 1 person and 1 dog", label_visibility="collapsed", key="ai_query") + with col2: + st.form_submit_button("Ask AI", on_click=run_ai_query) + + +def find_similar_imgs(imgs): + """Initializes a Streamlit form for AI-based image querying with custom input.""" + exp = st.session_state["explorer"] + similar = exp.get_similar(img=imgs, limit=st.session_state.get("limit"), return_type="arrow") + paths = similar.to_pydict()["im_file"] + st.session_state["imgs"] = paths + st.session_state["res"] = similar + + +def similarity_form(selected_imgs): + """Initializes a form for AI-based image querying with custom input in Streamlit.""" + st.write("Similarity Search") + with st.form("similarity_form"): + subcol1, subcol2 = st.columns([1, 1]) + with subcol1: + st.number_input( + "limit", min_value=None, max_value=None, value=25, label_visibility="collapsed", key="limit" + ) + + with subcol2: + disabled = not len(selected_imgs) + st.write("Selected: ", len(selected_imgs)) + st.form_submit_button( + "Search", + disabled=disabled, + on_click=find_similar_imgs, + args=(selected_imgs,), + ) + if disabled: + st.error("Select at least one image to search.") + + +# def persist_reset_form(): +# with st.form("persist_reset"): +# col1, col2 = st.columns([1, 1]) +# with col1: +# st.form_submit_button("Reset", on_click=reset) +# +# with col2: +# st.form_submit_button("Persist", on_click=update_state, args=("PERSISTING", True)) + + +def run_sql_query(): + """Executes an SQL query and returns the results.""" + st.session_state["error"] = None + query = st.session_state.get("query") + if query.rstrip().lstrip(): + exp = st.session_state["explorer"] + res = exp.sql_query(query, return_type="arrow") + st.session_state["imgs"] = res.to_pydict()["im_file"] + st.session_state["res"] = res + + +def run_ai_query(): + """Execute SQL query and update session state with query results.""" + if not SETTINGS["openai_api_key"]: + st.session_state["error"] = ( + 'OpenAI API key not found in settings. Please run yolo settings openai_api_key="..."' + ) + return + st.session_state["error"] = None + query = st.session_state.get("ai_query") + if query.rstrip().lstrip(): + exp = st.session_state["explorer"] + res = exp.ask_ai(query) + if not isinstance(res, pd.DataFrame) or res.empty: + st.session_state["error"] = "No results found using AI generated query. Try another query or rerun it." + return + st.session_state["imgs"] = res["im_file"].to_list() + st.session_state["res"] = res + + +def reset_explorer(): + """Resets the explorer to its initial state by clearing session variables.""" + st.session_state["explorer"] = None + st.session_state["imgs"] = None + st.session_state["error"] = None + + +def utralytics_explorer_docs_callback(): + """Resets the explorer to its initial state by clearing session variables.""" + with st.container(border=True): + st.image( + "https://raw.githubusercontent.com/ultralytics/assets/main/logo/Ultralytics_Logotype_Original.svg", + width=100, + ) + st.markdown( + "

This demo is built using Ultralytics Explorer API. Visit API docs to try examples & learn more

", + unsafe_allow_html=True, + help=None, + ) + st.link_button("Ultrlaytics Explorer API", "https://docs.ultralytics.com/datasets/explorer/") + + +def layout(): + """Resets explorer session variables and provides documentation with a link to API docs.""" + st.set_page_config(layout="wide", initial_sidebar_state="collapsed") + st.markdown("

Ultralytics Explorer Demo

", unsafe_allow_html=True) + + if st.session_state.get("explorer") is None: + init_explorer_form() + return + + st.button(":arrow_backward: Select Dataset", on_click=reset_explorer) + exp = st.session_state.get("explorer") + col1, col2 = st.columns([0.75, 0.25], gap="small") + imgs = [] + if st.session_state.get("error"): + st.error(st.session_state["error"]) + else: + if st.session_state.get("imgs"): + imgs = st.session_state.get("imgs") + else: + imgs = exp.table.to_lance().to_table(columns=["im_file"]).to_pydict()["im_file"] + st.session_state["res"] = exp.table.to_arrow() + total_imgs, selected_imgs = len(imgs), [] + with col1: + subcol1, subcol2, subcol3, subcol4, subcol5 = st.columns(5) + with subcol1: + st.write("Max Images Displayed:") + with subcol2: + num = st.number_input( + "Max Images Displayed", + min_value=0, + max_value=total_imgs, + value=min(500, total_imgs), + key="num_imgs_displayed", + label_visibility="collapsed", + ) + with subcol3: + st.write("Start Index:") + with subcol4: + start_idx = st.number_input( + "Start Index", + min_value=0, + max_value=total_imgs, + value=0, + key="start_index", + label_visibility="collapsed", + ) + with subcol5: + reset = st.button("Reset", use_container_width=False, key="reset") + if reset: + st.session_state["imgs"] = None + st.experimental_rerun() + + query_form() + ai_query_form() + if total_imgs: + labels, boxes, masks, kpts, classes = None, None, None, None, None + task = exp.model.task + if st.session_state.get("display_labels"): + labels = st.session_state.get("res").to_pydict()["labels"][start_idx : start_idx + num] + boxes = st.session_state.get("res").to_pydict()["bboxes"][start_idx : start_idx + num] + masks = st.session_state.get("res").to_pydict()["masks"][start_idx : start_idx + num] + kpts = st.session_state.get("res").to_pydict()["keypoints"][start_idx : start_idx + num] + classes = st.session_state.get("res").to_pydict()["cls"][start_idx : start_idx + num] + imgs_displayed = imgs[start_idx : start_idx + num] + selected_imgs = image_select( + f"Total samples: {total_imgs}", + images=imgs_displayed, + use_container_width=False, + # indices=[i for i in range(num)] if select_all else None, + labels=labels, + classes=classes, + bboxes=boxes, + masks=masks if task == "segment" else None, + kpts=kpts if task == "pose" else None, + ) + + with col2: + similarity_form(selected_imgs) + display_labels = st.checkbox("Labels", value=False, key="display_labels") + utralytics_explorer_docs_callback() + + +if __name__ == "__main__": + layout() diff --git a/ultralytics/data/explorer/utils.py b/ultralytics/data/explorer/utils.py new file mode 100644 index 0000000..d1c4b9b --- /dev/null +++ b/ultralytics/data/explorer/utils.py @@ -0,0 +1,166 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import getpass +from typing import List + +import cv2 +import numpy as np +import pandas as pd + +from ultralytics.data.augment import LetterBox +from ultralytics.utils import LOGGER as logger +from ultralytics.utils import SETTINGS +from ultralytics.utils.checks import check_requirements +from ultralytics.utils.ops import xyxy2xywh +from ultralytics.utils.plotting import plot_images + + +def get_table_schema(vector_size): + """Extracts and returns the schema of a database table.""" + from lancedb.pydantic import LanceModel, Vector + + class Schema(LanceModel): + im_file: str + labels: List[str] + cls: List[int] + bboxes: List[List[float]] + masks: List[List[List[int]]] + keypoints: List[List[List[float]]] + vector: Vector(vector_size) + + return Schema + + +def get_sim_index_schema(): + """Returns a LanceModel schema for a database table with specified vector size.""" + from lancedb.pydantic import LanceModel + + class Schema(LanceModel): + idx: int + im_file: str + count: int + sim_im_files: List[str] + + return Schema + + +def sanitize_batch(batch, dataset_info): + """Sanitizes input batch for inference, ensuring correct format and dimensions.""" + batch["cls"] = batch["cls"].flatten().int().tolist() + box_cls_pair = sorted(zip(batch["bboxes"].tolist(), batch["cls"]), key=lambda x: x[1]) + batch["bboxes"] = [box for box, _ in box_cls_pair] + batch["cls"] = [cls for _, cls in box_cls_pair] + batch["labels"] = [dataset_info["names"][i] for i in batch["cls"]] + batch["masks"] = batch["masks"].tolist() if "masks" in batch else [[[]]] + batch["keypoints"] = batch["keypoints"].tolist() if "keypoints" in batch else [[[]]] + return batch + + +def plot_query_result(similar_set, plot_labels=True): + """ + Plot images from the similar set. + + Args: + similar_set (list): Pyarrow or pandas object containing the similar data points + plot_labels (bool): Whether to plot labels or not + """ + similar_set = ( + similar_set.to_dict(orient="list") if isinstance(similar_set, pd.DataFrame) else similar_set.to_pydict() + ) + empty_masks = [[[]]] + empty_boxes = [[]] + images = similar_set.get("im_file", []) + bboxes = similar_set.get("bboxes", []) if similar_set.get("bboxes") is not empty_boxes else [] + masks = similar_set.get("masks") if similar_set.get("masks")[0] != empty_masks else [] + kpts = similar_set.get("keypoints") if similar_set.get("keypoints")[0] != empty_masks else [] + cls = similar_set.get("cls", []) + + plot_size = 640 + imgs, batch_idx, plot_boxes, plot_masks, plot_kpts = [], [], [], [], [] + for i, imf in enumerate(images): + im = cv2.imread(imf) + im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) + h, w = im.shape[:2] + r = min(plot_size / h, plot_size / w) + imgs.append(LetterBox(plot_size, center=False)(image=im).transpose(2, 0, 1)) + if plot_labels: + if len(bboxes) > i and len(bboxes[i]) > 0: + box = np.array(bboxes[i], dtype=np.float32) + box[:, [0, 2]] *= r + box[:, [1, 3]] *= r + plot_boxes.append(box) + if len(masks) > i and len(masks[i]) > 0: + mask = np.array(masks[i], dtype=np.uint8)[0] + plot_masks.append(LetterBox(plot_size, center=False)(image=mask)) + if len(kpts) > i and kpts[i] is not None: + kpt = np.array(kpts[i], dtype=np.float32) + kpt[:, :, :2] *= r + plot_kpts.append(kpt) + batch_idx.append(np.ones(len(np.array(bboxes[i], dtype=np.float32))) * i) + imgs = np.stack(imgs, axis=0) + masks = np.stack(plot_masks, axis=0) if plot_masks else np.zeros(0, dtype=np.uint8) + kpts = np.concatenate(plot_kpts, axis=0) if plot_kpts else np.zeros((0, 51), dtype=np.float32) + boxes = xyxy2xywh(np.concatenate(plot_boxes, axis=0)) if plot_boxes else np.zeros(0, dtype=np.float32) + batch_idx = np.concatenate(batch_idx, axis=0) + cls = np.concatenate([np.array(c, dtype=np.int32) for c in cls], axis=0) + + return plot_images( + imgs, batch_idx, cls, bboxes=boxes, masks=masks, kpts=kpts, max_subplots=len(images), save=False, threaded=False + ) + + +def prompt_sql_query(query): + """Plots images with optional labels from a similar data set.""" + check_requirements("openai>=1.6.1") + from openai import OpenAI + + if not SETTINGS["openai_api_key"]: + logger.warning("OpenAI API key not found in settings. Please enter your API key below.") + openai_api_key = getpass.getpass("OpenAI API key: ") + SETTINGS.update({"openai_api_key": openai_api_key}) + openai = OpenAI(api_key=SETTINGS["openai_api_key"]) + + messages = [ + { + "role": "system", + "content": """ + You are a helpful data scientist proficient in SQL. You need to output exactly one SQL query based on + the following schema and a user request. You only need to output the format with fixed selection + statement that selects everything from "'table'", like `SELECT * from 'table'` + + Schema: + im_file: string not null + labels: list not null + child 0, item: string + cls: list not null + child 0, item: int64 + bboxes: list> not null + child 0, item: list + child 0, item: double + masks: list>> not null + child 0, item: list> + child 0, item: list + child 0, item: int64 + keypoints: list>> not null + child 0, item: list> + child 0, item: list + child 0, item: double + vector: fixed_size_list[256] not null + child 0, item: float + + Some details about the schema: + - the "labels" column contains the string values like 'person' and 'dog' for the respective objects + in each image + - the "cls" column contains the integer values on these classes that map them the labels + + Example of a correct query: + request - Get all data points that contain 2 or more people and at least one dog + correct query- + SELECT * FROM 'table' WHERE ARRAY_LENGTH(cls) >= 2 AND ARRAY_LENGTH(FILTER(labels, x -> x = 'person')) >= 2 AND ARRAY_LENGTH(FILTER(labels, x -> x = 'dog')) >= 1; + """, + }, + {"role": "user", "content": f"{query}"}, + ] + + response = openai.chat.completions.create(model="gpt-3.5-turbo", messages=messages) + return response.choices[0].message.content diff --git a/ultralytics/data/loaders.py b/ultralytics/data/loaders.py new file mode 100644 index 0000000..7d82e77 --- /dev/null +++ b/ultralytics/data/loaders.py @@ -0,0 +1,581 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import glob +import math +import os +import time +from dataclasses import dataclass +from pathlib import Path +from threading import Thread +from urllib.parse import urlparse + +import cv2 +import numpy as np +import requests +import torch +from PIL import Image + +from ultralytics.data.utils import IMG_FORMATS, VID_FORMATS +from ultralytics.utils import LOGGER, is_colab, is_kaggle, ops +from ultralytics.utils.checks import check_requirements + +import subprocess +import json + +@dataclass +class SourceTypes: + """Class to represent various types of input sources for predictions.""" + + stream: bool = False + screenshot: bool = False + from_img: bool = False + tensor: bool = False + + +class LoadStreams: + """ + Stream Loader for various types of video streams, Supports RTSP, RTMP, HTTP, and TCP streams. + + Attributes: + sources (str): The source input paths or URLs for the video streams. + vid_stride (int): Video frame-rate stride, defaults to 1. + buffer (bool): Whether to buffer input streams, defaults to False. + running (bool): Flag to indicate if the streaming thread is running. + mode (str): Set to 'stream' indicating real-time capture. + imgs (list): List of image frames for each stream. + fps (list): List of FPS for each stream. + frames (list): List of total frames for each stream. + threads (list): List of threads for each stream. + shape (list): List of shapes for each stream. + caps (list): List of cv2.VideoCapture objects for each stream. + bs (int): Batch size for processing. + + Methods: + __init__: Initialize the stream loader. + update: Read stream frames in daemon thread. + close: Close stream loader and release resources. + __iter__: Returns an iterator object for the class. + __next__: Returns source paths, transformed, and original images for processing. + __len__: Return the length of the sources object. + + Example: + ```bash + yolo predict source='rtsp://example.com/media.mp4' + ``` + """ + + def __init__(self, sources="file.streams", vid_stride=1, buffer=False): + """Initialize instance variables and check for consistent input stream shapes.""" + torch.backends.cudnn.benchmark = True # faster for fixed-size inference + self.buffer = buffer # buffer input streams + self.running = True # running flag for Thread + self.mode = "stream" + self.vid_stride = vid_stride # video frame-rate stride + + sources = Path(sources).read_text().rsplit() if os.path.isfile(sources) else [sources] + n = len(sources) + self.bs = n + self.fps = [0] * n # frames per second + self.frames = [0] * n + self.threads = [None] * n + self.caps = [None] * n # video capture objects + self.imgs = [[] for _ in range(n)] # images + self.shape = [[] for _ in range(n)] # image shapes + self.sources = [ops.clean_str(x) for x in sources] # clean source names for later + for i, s in enumerate(sources): # index, source + # Start thread to read frames from video stream + st = f"{i + 1}/{n}: {s}... " + if urlparse(s).hostname in ("www.youtube.com", "youtube.com", "youtu.be"): # if source is YouTube video + # YouTube format i.e. 'https://www.youtube.com/watch?v=Zgi9g1ksQHc' or 'https://youtu.be/LNwODJXcvt4' + s = get_best_youtube_url(s) + s = eval(s) if s.isnumeric() else s # i.e. s = '0' local webcam + if s == 0 and (is_colab() or is_kaggle()): + raise NotImplementedError( + "'source=0' webcam not supported in Colab and Kaggle notebooks. " + "Try running 'source=0' in a local environment." + ) + self.caps[i] = cv2.VideoCapture(s) # store video capture object + if not self.caps[i].isOpened(): + raise ConnectionError(f"{st}Failed to open {s}") + w = int(self.caps[i].get(cv2.CAP_PROP_FRAME_WIDTH)) + h = int(self.caps[i].get(cv2.CAP_PROP_FRAME_HEIGHT)) + fps = self.caps[i].get(cv2.CAP_PROP_FPS) # warning: may return 0 or nan + self.frames[i] = max(int(self.caps[i].get(cv2.CAP_PROP_FRAME_COUNT)), 0) or float( + "inf" + ) # infinite stream fallback + self.fps[i] = max((fps if math.isfinite(fps) else 0) % 100, 0) or 30 # 30 FPS fallback + + success, im = self.caps[i].read() # guarantee first frame + if not success or im is None: + raise ConnectionError(f"{st}Failed to read images from {s}") + self.imgs[i].append(im) + self.shape[i] = im.shape + self.threads[i] = Thread(target=self.update, args=([i, self.caps[i], s]), daemon=True) + LOGGER.info(f"{st}Success ✅ ({self.frames[i]} frames of shape {w}x{h} at {self.fps[i]:.2f} FPS)") + self.threads[i].start() + LOGGER.info("") # newline + + def update(self, i, cap, stream): + """Read stream `i` frames in daemon thread.""" + n, f = 0, self.frames[i] # frame number, frame array + while self.running and cap.isOpened() and n < (f - 1): + if len(self.imgs[i]) < 30: # keep a <=30-image buffer + n += 1 + cap.grab() # .read() = .grab() followed by .retrieve() + if n % self.vid_stride == 0: + success, im = cap.retrieve() + if not success: + im = np.zeros(self.shape[i], dtype=np.uint8) + LOGGER.warning("WARNING ⚠️ Video stream unresponsive, please check your IP camera connection.") + cap.open(stream) # re-open stream if signal was lost + if self.buffer: + self.imgs[i].append(im) + else: + self.imgs[i] = [im] + else: + time.sleep(0.01) # wait until the buffer is empty + + def close(self): + """Close stream loader and release resources.""" + self.running = False # stop flag for Thread + for thread in self.threads: + if thread.is_alive(): + thread.join(timeout=5) # Add timeout + for cap in self.caps: # Iterate through the stored VideoCapture objects + try: + cap.release() # release video capture + except Exception as e: + LOGGER.warning(f"WARNING ⚠️ Could not release VideoCapture object: {e}") + cv2.destroyAllWindows() + + def __iter__(self): + """Iterates through YOLO image feed and re-opens unresponsive streams.""" + self.count = -1 + return self + + def __next__(self): + """Returns source paths, transformed and original images for processing.""" + self.count += 1 + + images = [] + for i, x in enumerate(self.imgs): + # Wait until a frame is available in each buffer + while not x: + if not self.threads[i].is_alive() or cv2.waitKey(1) == ord("q"): # q to quit + self.close() + raise StopIteration + time.sleep(1 / min(self.fps)) + x = self.imgs[i] + if not x: + LOGGER.warning(f"WARNING ⚠️ Waiting for stream {i}") + + # Get and remove the first frame from imgs buffer + if self.buffer: + images.append(x.pop(0)) + + # Get the last frame, and clear the rest from the imgs buffer + else: + images.append(x.pop(-1) if x else np.zeros(self.shape[i], dtype=np.uint8)) + x.clear() + + return self.sources, images, [""] * self.bs + + def __len__(self): + """Return the length of the sources object.""" + return self.bs # 1E12 frames = 32 streams at 30 FPS for 30 years + + +class LoadScreenshots: + """ + YOLOv8 screenshot dataloader. + + This class manages the loading of screenshot images for processing with YOLOv8. + Suitable for use with `yolo predict source=screen`. + + Attributes: + source (str): The source input indicating which screen to capture. + screen (int): The screen number to capture. + left (int): The left coordinate for screen capture area. + top (int): The top coordinate for screen capture area. + width (int): The width of the screen capture area. + height (int): The height of the screen capture area. + mode (str): Set to 'stream' indicating real-time capture. + frame (int): Counter for captured frames. + sct (mss.mss): Screen capture object from `mss` library. + bs (int): Batch size, set to 1. + monitor (dict): Monitor configuration details. + + Methods: + __iter__: Returns an iterator object. + __next__: Captures the next screenshot and returns it. + """ + + def __init__(self, source): + """Source = [screen_number left top width height] (pixels).""" + check_requirements("mss") + import mss # noqa + + source, *params = source.split() + self.screen, left, top, width, height = 0, None, None, None, None # default to full screen 0 + if len(params) == 1: + self.screen = int(params[0]) + elif len(params) == 4: + left, top, width, height = (int(x) for x in params) + elif len(params) == 5: + self.screen, left, top, width, height = (int(x) for x in params) + self.mode = "stream" + self.frame = 0 + self.sct = mss.mss() + self.bs = 1 + self.fps = 30 + + # Parse monitor shape + monitor = self.sct.monitors[self.screen] + self.top = monitor["top"] if top is None else (monitor["top"] + top) + self.left = monitor["left"] if left is None else (monitor["left"] + left) + self.width = width or monitor["width"] + self.height = height or monitor["height"] + self.monitor = {"left": self.left, "top": self.top, "width": self.width, "height": self.height} + + def __iter__(self): + """Returns an iterator of the object.""" + return self + + def __next__(self): + """mss screen capture: get raw pixels from the screen as np array.""" + im0 = np.asarray(self.sct.grab(self.monitor))[:, :, :3] # BGRA to BGR + s = f"screen {self.screen} (LTWH): {self.left},{self.top},{self.width},{self.height}: " + + self.frame += 1 + return [str(self.screen)], [im0], [s] # screen, img, string + + +class LoadImagesAndVideos: + """ + YOLOv8 image/video dataloader. + + This class manages the loading and pre-processing of image and video data for YOLOv8. It supports loading from + various formats, including single image files, video files, and lists of image and video paths. + + Attributes: + files (list): List of image and video file paths. + nf (int): Total number of files (images and videos). + video_flag (list): Flags indicating whether a file is a video (True) or an image (False). + mode (str): Current mode, 'image' or 'video'. + vid_stride (int): Stride for video frame-rate, defaults to 1. + bs (int): Batch size, set to 1 for this class. + cap (cv2.VideoCapture): Video capture object for OpenCV. + frame (int): Frame counter for video. + frames (int): Total number of frames in the video. + count (int): Counter for iteration, initialized at 0 during `__iter__()`. + + Methods: + _new_video(path): Create a new cv2.VideoCapture object for a given video path. + """ + + def __init__(self, path, batch=1, vid_stride=1): + """Initialize the Dataloader and raise FileNotFoundError if file not found.""" + parent = None + if isinstance(path, str) and Path(path).suffix == ".txt": # *.txt file with img/vid/dir on each line + parent = Path(path).parent + path = Path(path).read_text().splitlines() # list of sources + files = [] + for p in sorted(path) if isinstance(path, (list, tuple)) else [path]: + a = str(Path(p).absolute()) # do not use .resolve() https://github.com/ultralytics/ultralytics/issues/2912 + if "*" in a: + files.extend(sorted(glob.glob(a, recursive=True))) # glob + elif os.path.isdir(a): + files.extend(sorted(glob.glob(os.path.join(a, "*.*")))) # dir + elif os.path.isfile(a): + files.append(a) # files (absolute or relative to CWD) + elif parent and (parent / p).is_file(): + files.append(str((parent / p).absolute())) # files (relative to *.txt file parent) + else: + raise FileNotFoundError(f"{p} does not exist") + + images = [x for x in files if x.split(".")[-1].lower() in IMG_FORMATS] + videos = [x for x in files if x.split(".")[-1].lower() in VID_FORMATS] + ni, nv = len(images), len(videos) + + self.files = images + videos + self.nf = ni + nv # number of files + self.ni = ni # number of images + self.video_flag = [False] * ni + [True] * nv + self.mode = "image" + self.vid_stride = vid_stride # video frame-rate stride + self.bs = batch + if any(videos): + self._new_video(videos[0]) # new video + else: + self.cap = None + if self.nf == 0: + raise FileNotFoundError( + f"No images or videos found in {p}. " + f"Supported formats are:\nimages: {IMG_FORMATS}\nvideos: {VID_FORMATS}" + ) + + def __iter__(self): + """Returns an iterator object for VideoStream or ImageFolder.""" + self.count = 0 + return self + + def __next__(self): + """Returns the next batch of images or video frames along with their paths and metadata.""" + paths, imgs, info = [], [], [] + while len(imgs) < self.bs: + if self.count >= self.nf: # end of file list + if len(imgs) > 0: + return paths, imgs, info # return last partial batch + else: + raise StopIteration + + path = self.files[self.count] + if self.video_flag[self.count]: + self.mode = "video" + if not self.cap or not self.cap.isOpened(): + self._new_video(path) + + for _ in range(self.vid_stride): + success = self.cap.grab() + if not success: + break # end of video or failure + + if success: + success, im0 = self.cap.retrieve() + ##====================== + '''判断视频是否含旋转信息''' + rotation = self.get_rotation(path) + if rotation == 270: + im0 = cv2.rotate(im0, cv2.ROTATE_90_COUNTERCLOCKWISE) + ###====================== + if success: + self.frame += 1 + paths.append(path) + imgs.append(im0) + info.append(f"video {self.count + 1}/{self.nf} (frame {self.frame}/{self.frames}) {path}: ") + if self.frame == self.frames: # end of video + self.count += 1 + self.cap.release() + else: + # Move to the next file if the current video ended or failed to open + self.count += 1 + if self.cap: + self.cap.release() + if self.count < self.nf: + self._new_video(self.files[self.count]) + + else: + self.mode = "image" + im0 = cv2.imread(path) # BGR + if im0 is None: + raise FileNotFoundError(f"Image Not Found {path}") + paths.append(path) + imgs.append(im0) + info.append(f"image {self.count + 1}/{self.nf} {path}: ") + self.count += 1 # move to the next file + if self.count >= self.ni: # end of image list + break + + return paths, imgs, info + + def _new_video(self, path): + """Creates a new video capture object for the given path.""" + self.frame = 0 + self.cap = cv2.VideoCapture(path) + self.fps = int(self.cap.get(cv2.CAP_PROP_FPS)) + if not self.cap.isOpened(): + raise FileNotFoundError(f"Failed to open video {path}") + self.frames = int(self.cap.get(cv2.CAP_PROP_FRAME_COUNT) / self.vid_stride) + + def get_rotation(self, filename): + cmd = [ + "ffprobe", # 注意是 ffprobe，不是 ffmpeg + "-v", "error", + "-select_streams", "v:0", + "-show_entries", "stream_tags=rotate", + "-of", "json", + filename + ] + result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if result.returncode == 0: + metadata = json.loads(result.stdout) + rotation = metadata.get("streams", [{}])[0].get("tags", {}).get("rotate", 0) + return int(rotation) + else: + return 0 + + def __len__(self): + """Returns the number of batches in the object.""" + return math.ceil(self.nf / self.bs) # number of files + + +class LoadPilAndNumpy: + """ + Load images from PIL and Numpy arrays for batch processing. + + This class is designed to manage loading and pre-processing of image data from both PIL and Numpy formats. + It performs basic validation and format conversion to ensure that the images are in the required format for + downstream processing. + + Attributes: + paths (list): List of image paths or autogenerated filenames. + im0 (list): List of images stored as Numpy arrays. + mode (str): Type of data being processed, defaults to 'image'. + bs (int): Batch size, equivalent to the length of `im0`. + + Methods: + _single_check(im): Validate and format a single image to a Numpy array. + """ + + def __init__(self, im0): + """Initialize PIL and Numpy Dataloader.""" + if not isinstance(im0, list): + im0 = [im0] + self.paths = [getattr(im, "filename", f"image{i}.jpg") for i, im in enumerate(im0)] + self.im0 = [self._single_check(im) for im in im0] + self.mode = "image" + self.bs = len(self.im0) + + @staticmethod + def _single_check(im): + """Validate and format an image to numpy array.""" + assert isinstance(im, (Image.Image, np.ndarray)), f"Expected PIL/np.ndarray image type, but got {type(im)}" + if isinstance(im, Image.Image): + if im.mode != "RGB": + im = im.convert("RGB") + im = np.asarray(im)[:, :, ::-1] + im = np.ascontiguousarray(im) # contiguous + return im + + def __len__(self): + """Returns the length of the 'im0' attribute.""" + return len(self.im0) + + def __next__(self): + """Returns batch paths, images, processed images, None, ''.""" + if self.count == 1: # loop only once as it's batch inference + raise StopIteration + self.count += 1 + return self.paths, self.im0, [""] * self.bs + + def __iter__(self): + """Enables iteration for class LoadPilAndNumpy.""" + self.count = 0 + return self + + +class LoadTensor: + """ + Load images from torch.Tensor data. + + This class manages the loading and pre-processing of image data from PyTorch tensors for further processing. + + Attributes: + im0 (torch.Tensor): The input tensor containing the image(s). + bs (int): Batch size, inferred from the shape of `im0`. + mode (str): Current mode, set to 'image'. + paths (list): List of image paths or filenames. + count (int): Counter for iteration, initialized at 0 during `__iter__()`. + + Methods: + _single_check(im, stride): Validate and possibly modify the input tensor. + """ + + def __init__(self, im0) -> None: + """Initialize Tensor Dataloader.""" + self.im0 = self._single_check(im0) + self.bs = self.im0.shape[0] + self.mode = "image" + self.paths = [getattr(im, "filename", f"image{i}.jpg") for i, im in enumerate(im0)] + + @staticmethod + def _single_check(im, stride=32): + """Validate and format an image to torch.Tensor.""" + s = ( + f"WARNING ⚠️ torch.Tensor inputs should be BCHW i.e. shape(1, 3, 640, 640) " + f"divisible by stride {stride}. Input shape{tuple(im.shape)} is incompatible." + ) + if len(im.shape) != 4: + if len(im.shape) != 3: + raise ValueError(s) + LOGGER.warning(s) + im = im.unsqueeze(0) + if im.shape[2] % stride or im.shape[3] % stride: + raise ValueError(s) + if im.max() > 1.0 + torch.finfo(im.dtype).eps: # torch.float32 eps is 1.2e-07 + LOGGER.warning( + f"WARNING ⚠️ torch.Tensor inputs should be normalized 0.0-1.0 but max value is {im.max()}. " + f"Dividing input by 255." + ) + im = im.float() / 255.0 + + return im + + def __iter__(self): + """Returns an iterator object.""" + self.count = 0 + return self + + def __next__(self): + """Return next item in the iterator.""" + if self.count == 1: + raise StopIteration + self.count += 1 + return self.paths, self.im0, [""] * self.bs + + def __len__(self): + """Returns the batch size.""" + return self.bs + + +def autocast_list(source): + """Merges a list of source of different types into a list of numpy arrays or PIL images.""" + files = [] + for im in source: + if isinstance(im, (str, Path)): # filename or uri + files.append(Image.open(requests.get(im, stream=True).raw if str(im).startswith("http") else im)) + elif isinstance(im, (Image.Image, np.ndarray)): # PIL or np Image + files.append(im) + else: + raise TypeError( + f"type {type(im).__name__} is not a supported Ultralytics prediction source type. \n" + f"See https://docs.ultralytics.com/modes/predict for supported source types." + ) + + return files + + +def get_best_youtube_url(url, use_pafy=True): + """ + Retrieves the URL of the best quality MP4 video stream from a given YouTube video. + + This function uses the pafy or yt_dlp library to extract the video info from YouTube. It then finds the highest + quality MP4 format that has video codec but no audio codec, and returns the URL of this video stream. + + Args: + url (str): The URL of the YouTube video. + use_pafy (bool): Use the pafy package, default=True, otherwise use yt_dlp package. + + Returns: + (str): The URL of the best quality MP4 video stream, or None if no suitable stream is found. + """ + if use_pafy: + check_requirements(("pafy", "youtube_dl==2020.12.2")) + import pafy # noqa + + return pafy.new(url).getbestvideo(preftype="mp4").url + else: + check_requirements("yt-dlp") + import yt_dlp + + with yt_dlp.YoutubeDL({"quiet": True}) as ydl: + info_dict = ydl.extract_info(url, download=False) # extract info + for f in reversed(info_dict.get("formats", [])): # reversed because best is usually last + # Find a format with video codec, no audio, *.mp4 extension at least 1920x1080 size + good_size = (f.get("width") or 0) >= 1920 or (f.get("height") or 0) >= 1080 + if good_size and f["vcodec"] != "none" and f["acodec"] == "none" and f["ext"] == "mp4": + return f.get("url") + + +# Define constants +LOADERS = (LoadStreams, LoadPilAndNumpy, LoadImagesAndVideos, LoadScreenshots) diff --git a/ultralytics/data/scripts/download_weights.sh b/ultralytics/data/scripts/download_weights.sh new file mode 100644 index 0000000..87db31f --- /dev/null +++ b/ultralytics/data/scripts/download_weights.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Ultralytics YOLO 🚀, AGPL-3.0 license +# Download latest models from https://github.com/ultralytics/assets/releases +# Example usage: bash ultralytics/data/scripts/download_weights.sh +# parent +# └── weights +# ├── yolov8n.pt ← downloads here +# ├── yolov8s.pt +# └── ... + +python - < gap, f"invalid crop_size gap pair [{crop_size} {gap}]" + step = crop_size - gap + + xn = 1 if w <= crop_size else ceil((w - crop_size) / step + 1) + xs = [step * i for i in range(xn)] + if len(xs) > 1 and xs[-1] + crop_size > w: + xs[-1] = w - crop_size + + yn = 1 if h <= crop_size else ceil((h - crop_size) / step + 1) + ys = [step * i for i in range(yn)] + if len(ys) > 1 and ys[-1] + crop_size > h: + ys[-1] = h - crop_size + + start = np.array(list(itertools.product(xs, ys)), dtype=np.int64) + stop = start + crop_size + windows.append(np.concatenate([start, stop], axis=1)) + windows = np.concatenate(windows, axis=0) + + im_in_wins = windows.copy() + im_in_wins[:, 0::2] = np.clip(im_in_wins[:, 0::2], 0, w) + im_in_wins[:, 1::2] = np.clip(im_in_wins[:, 1::2], 0, h) + im_areas = (im_in_wins[:, 2] - im_in_wins[:, 0]) * (im_in_wins[:, 3] - im_in_wins[:, 1]) + win_areas = (windows[:, 2] - windows[:, 0]) * (windows[:, 3] - windows[:, 1]) + im_rates = im_areas / win_areas + if not (im_rates > im_rate_thr).any(): + max_rate = im_rates.max() + im_rates[abs(im_rates - max_rate) < eps] = 1 + return windows[im_rates > im_rate_thr] + + +def get_window_obj(anno, windows, iof_thr=0.7): + """Get objects for each window.""" + h, w = anno["ori_size"] + label = anno["label"] + if len(label): + label[:, 1::2] *= w + label[:, 2::2] *= h + iofs = bbox_iof(label[:, 1:], windows) + # Unnormalized and misaligned coordinates + return [(label[iofs[:, i] >= iof_thr]) for i in range(len(windows))] # window_anns + else: + return [np.zeros((0, 9), dtype=np.float32) for _ in range(len(windows))] # window_anns + + +def crop_and_save(anno, windows, window_objs, im_dir, lb_dir): + """ + Crop images and save new labels. + + Args: + anno (dict): Annotation dict, including `filepath`, `label`, `ori_size` as its keys. + windows (list): A list of windows coordinates. + window_objs (list): A list of labels inside each window. + im_dir (str): The output directory path of images. + lb_dir (str): The output directory path of labels. + + Notes: + The directory structure assumed for the DOTA dataset: + - data_root + - images + - train + - val + - labels + - train + - val + """ + im = cv2.imread(anno["filepath"]) + name = Path(anno["filepath"]).stem + for i, window in enumerate(windows): + x_start, y_start, x_stop, y_stop = window.tolist() + new_name = f"{name}__{x_stop - x_start}__{x_start}___{y_start}" + patch_im = im[y_start:y_stop, x_start:x_stop] + ph, pw = patch_im.shape[:2] + + cv2.imwrite(str(Path(im_dir) / f"{new_name}.jpg"), patch_im) + label = window_objs[i] + if len(label) == 0: + continue + label[:, 1::2] -= x_start + label[:, 2::2] -= y_start + label[:, 1::2] /= pw + label[:, 2::2] /= ph + + with open(Path(lb_dir) / f"{new_name}.txt", "w") as f: + for lb in label: + formatted_coords = ["{:.6g}".format(coord) for coord in lb[1:]] + f.write(f"{int(lb[0])} {' '.join(formatted_coords)}\n") + + +def split_images_and_labels(data_root, save_dir, split="train", crop_sizes=[1024], gaps=[200]): + """ + Split both images and labels. + + Notes: + The directory structure assumed for the DOTA dataset: + - data_root + - images + - split + - labels + - split + and the output directory structure is: + - save_dir + - images + - split + - labels + - split + """ + im_dir = Path(save_dir) / "images" / split + im_dir.mkdir(parents=True, exist_ok=True) + lb_dir = Path(save_dir) / "labels" / split + lb_dir.mkdir(parents=True, exist_ok=True) + + annos = load_yolo_dota(data_root, split=split) + for anno in tqdm(annos, total=len(annos), desc=split): + windows = get_windows(anno["ori_size"], crop_sizes, gaps) + window_objs = get_window_obj(anno, windows) + crop_and_save(anno, windows, window_objs, str(im_dir), str(lb_dir)) + + +def split_trainval(data_root, save_dir, crop_size=1024, gap=200, rates=[1.0]): + """ + Split train and val set of DOTA. + + Notes: + The directory structure assumed for the DOTA dataset: + - data_root + - images + - train + - val + - labels + - train + - val + and the output directory structure is: + - save_dir + - images + - train + - val + - labels + - train + - val + """ + crop_sizes, gaps = [], [] + for r in rates: + crop_sizes.append(int(crop_size / r)) + gaps.append(int(gap / r)) + for split in ["train", "val"]: + split_images_and_labels(data_root, save_dir, split, crop_sizes, gaps) + + +def split_test(data_root, save_dir, crop_size=1024, gap=200, rates=[1.0]): + """ + Split test set of DOTA, labels are not included within this set. + + Notes: + The directory structure assumed for the DOTA dataset: + - data_root + - images + - test + and the output directory structure is: + - save_dir + - images + - test + """ + crop_sizes, gaps = [], [] + for r in rates: + crop_sizes.append(int(crop_size / r)) + gaps.append(int(gap / r)) + save_dir = Path(save_dir) / "images" / "test" + save_dir.mkdir(parents=True, exist_ok=True) + + im_dir = Path(data_root) / "images" / "test" + assert im_dir.exists(), f"Can't find {im_dir}, please check your data root." + im_files = glob(str(im_dir / "*")) + for im_file in tqdm(im_files, total=len(im_files), desc="test"): + w, h = exif_size(Image.open(im_file)) + windows = get_windows((h, w), crop_sizes=crop_sizes, gaps=gaps) + im = cv2.imread(im_file) + name = Path(im_file).stem + for window in windows: + x_start, y_start, x_stop, y_stop = window.tolist() + new_name = f"{name}__{x_stop - x_start}__{x_start}___{y_start}" + patch_im = im[y_start:y_stop, x_start:x_stop] + cv2.imwrite(str(save_dir / f"{new_name}.jpg"), patch_im) + + +if __name__ == "__main__": + split_trainval(data_root="DOTAv2", save_dir="DOTAv2-split") + split_test(data_root="DOTAv2", save_dir="DOTAv2-split") diff --git a/ultralytics/data/utils.py b/ultralytics/data/utils.py new file mode 100644 index 0000000..c0a0773 --- /dev/null +++ b/ultralytics/data/utils.py @@ -0,0 +1,651 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import contextlib +import hashlib +import json +import os +import random +import subprocess +import time +import zipfile +from multiprocessing.pool import ThreadPool +from pathlib import Path +from tarfile import is_tarfile + +import cv2 +import numpy as np +from PIL import Image, ImageOps + +from ultralytics.nn.autobackend import check_class_names +from ultralytics.utils import ( + DATASETS_DIR, + LOGGER, + NUM_THREADS, + ROOT, + SETTINGS_YAML, + TQDM, + clean_url, + colorstr, + emojis, + yaml_load, + yaml_save, +) +from ultralytics.utils.checks import check_file, check_font, is_ascii +from ultralytics.utils.downloads import download, safe_download, unzip_file +from ultralytics.utils.ops import segments2boxes + +HELP_URL = "See https://docs.ultralytics.com/datasets/detect for dataset formatting guidance." +IMG_FORMATS = {"bmp", "dng", "jpeg", "jpg", "mpo", "png", "tif", "tiff", "webp", "pfm"} # image suffixes +VID_FORMATS = {"asf", "avi", "gif", "m4v", "mkv", "mov", "mp4", "mpeg", "mpg", "ts", "wmv", "webm"} # video suffixes +PIN_MEMORY = str(os.getenv("PIN_MEMORY", True)).lower() == "true" # global pin_memory for dataloaders + + +def img2label_paths(img_paths): + """Define label paths as a function of image paths.""" + sa, sb = f"{os.sep}images{os.sep}", f"{os.sep}labels{os.sep}" # /images/, /labels/ substrings + return [sb.join(x.rsplit(sa, 1)).rsplit(".", 1)[0] + ".txt" for x in img_paths] + + +def get_hash(paths): + """Returns a single hash value of a list of paths (files or dirs).""" + size = sum(os.path.getsize(p) for p in paths if os.path.exists(p)) # sizes + h = hashlib.sha256(str(size).encode()) # hash sizes + h.update("".join(paths).encode()) # hash paths + return h.hexdigest() # return hash + + +def exif_size(img: Image.Image): + """Returns exif-corrected PIL size.""" + s = img.size # (width, height) + if img.format == "JPEG": # only support JPEG images + with contextlib.suppress(Exception): + exif = img.getexif() + if exif: + rotation = exif.get(274, None) # the EXIF key for the orientation tag is 274 + if rotation in [6, 8]: # rotation 270 or 90 + s = s[1], s[0] + return s + + +def verify_image(args): + """Verify one image.""" + (im_file, cls), prefix = args + # Number (found, corrupt), message + nf, nc, msg = 0, 0, "" + try: + im = Image.open(im_file) + im.verify() # PIL verify + shape = exif_size(im) # image size + shape = (shape[1], shape[0]) # hw + assert (shape[0] > 9) & (shape[1] > 9), f"image size {shape} <10 pixels" + assert im.format.lower() in IMG_FORMATS, f"invalid image format {im.format}" + if im.format.lower() in ("jpg", "jpeg"): + with open(im_file, "rb") as f: + f.seek(-2, 2) + if f.read() != b"\xff\xd9": # corrupt JPEG + ImageOps.exif_transpose(Image.open(im_file)).save(im_file, "JPEG", subsampling=0, quality=100) + msg = f"{prefix}WARNING ⚠️ {im_file}: corrupt JPEG restored and saved" + nf = 1 + except Exception as e: + nc = 1 + msg = f"{prefix}WARNING ⚠️ {im_file}: ignoring corrupt image/label: {e}" + return (im_file, cls), nf, nc, msg + + +def verify_image_label(args): + """Verify one image-label pair.""" + im_file, lb_file, prefix, keypoint, num_cls, nkpt, ndim = args + # Number (missing, found, empty, corrupt), message, segments, keypoints + nm, nf, ne, nc, msg, segments, keypoints = 0, 0, 0, 0, "", [], None + try: + # Verify images + im = Image.open(im_file) + im.verify() # PIL verify + shape = exif_size(im) # image size + shape = (shape[1], shape[0]) # hw + assert (shape[0] > 9) & (shape[1] > 9), f"image size {shape} <10 pixels" + assert im.format.lower() in IMG_FORMATS, f"invalid image format {im.format}" + if im.format.lower() in ("jpg", "jpeg"): + with open(im_file, "rb") as f: + f.seek(-2, 2) + if f.read() != b"\xff\xd9": # corrupt JPEG + ImageOps.exif_transpose(Image.open(im_file)).save(im_file, "JPEG", subsampling=0, quality=100) + msg = f"{prefix}WARNING ⚠️ {im_file}: corrupt JPEG restored and saved" + + # Verify labels + if os.path.isfile(lb_file): + nf = 1 # label found + with open(lb_file) as f: + lb = [x.split() for x in f.read().strip().splitlines() if len(x)] + if any(len(x) > 6 for x in lb) and (not keypoint): # is segment + classes = np.array([x[0] for x in lb], dtype=np.float32) + segments = [np.array(x[1:], dtype=np.float32).reshape(-1, 2) for x in lb] # (cls, xy1...) + lb = np.concatenate((classes.reshape(-1, 1), segments2boxes(segments)), 1) # (cls, xywh) + lb = np.array(lb, dtype=np.float32) + nl = len(lb) + if nl: + if keypoint: + assert lb.shape[1] == (5 + nkpt * ndim), f"labels require {(5 + nkpt * ndim)} columns each" + points = lb[:, 5:].reshape(-1, ndim)[:, :2] + else: + assert lb.shape[1] == 5, f"labels require 5 columns, {lb.shape[1]} columns detected" + points = lb[:, 1:] + assert points.max() <= 1, f"non-normalized or out of bounds coordinates {points[points > 1]}" + assert lb.min() >= 0, f"negative label values {lb[lb < 0]}" + + # All labels + max_cls = lb[:, 0].max() # max label count + assert max_cls <= num_cls, ( + f"Label class {int(max_cls)} exceeds dataset class count {num_cls}. " + f"Possible class labels are 0-{num_cls - 1}" + ) + _, i = np.unique(lb, axis=0, return_index=True) + if len(i) < nl: # duplicate row check + lb = lb[i] # remove duplicates + if segments: + segments = [segments[x] for x in i] + msg = f"{prefix}WARNING ⚠️ {im_file}: {nl - len(i)} duplicate labels removed" + else: + ne = 1 # label empty + lb = np.zeros((0, (5 + nkpt * ndim) if keypoint else 5), dtype=np.float32) + else: + nm = 1 # label missing + lb = np.zeros((0, (5 + nkpt * ndim) if keypoints else 5), dtype=np.float32) + if keypoint: + keypoints = lb[:, 5:].reshape(-1, nkpt, ndim) + if ndim == 2: + kpt_mask = np.where((keypoints[..., 0] < 0) | (keypoints[..., 1] < 0), 0.0, 1.0).astype(np.float32) + keypoints = np.concatenate([keypoints, kpt_mask[..., None]], axis=-1) # (nl, nkpt, 3) + lb = lb[:, :5] + return im_file, lb, shape, segments, keypoints, nm, nf, ne, nc, msg + except Exception as e: + nc = 1 + msg = f"{prefix}WARNING ⚠️ {im_file}: ignoring corrupt image/label: {e}" + return [None, None, None, None, None, nm, nf, ne, nc, msg] + + +def polygon2mask(imgsz, polygons, color=1, downsample_ratio=1): + """ + Convert a list of polygons to a binary mask of the specified image size. + + Args: + imgsz (tuple): The size of the image as (height, width). + polygons (list[np.ndarray]): A list of polygons. Each polygon is an array with shape [N, M], where + N is the number of polygons, and M is the number of points such that M % 2 = 0. + color (int, optional): The color value to fill in the polygons on the mask. Defaults to 1. + downsample_ratio (int, optional): Factor by which to downsample the mask. Defaults to 1. + + Returns: + (np.ndarray): A binary mask of the specified image size with the polygons filled in. + """ + mask = np.zeros(imgsz, dtype=np.uint8) + polygons = np.asarray(polygons, dtype=np.int32) + polygons = polygons.reshape((polygons.shape[0], -1, 2)) + cv2.fillPoly(mask, polygons, color=color) + nh, nw = (imgsz[0] // downsample_ratio, imgsz[1] // downsample_ratio) + # Note: fillPoly first then resize is trying to keep the same loss calculation method when mask-ratio=1 + return cv2.resize(mask, (nw, nh)) + + +def polygons2masks(imgsz, polygons, color, downsample_ratio=1): + """ + Convert a list of polygons to a set of binary masks of the specified image size. + + Args: + imgsz (tuple): The size of the image as (height, width). + polygons (list[np.ndarray]): A list of polygons. Each polygon is an array with shape [N, M], where + N is the number of polygons, and M is the number of points such that M % 2 = 0. + color (int): The color value to fill in the polygons on the masks. + downsample_ratio (int, optional): Factor by which to downsample each mask. Defaults to 1. + + Returns: + (np.ndarray): A set of binary masks of the specified image size with the polygons filled in. + """ + return np.array([polygon2mask(imgsz, [x.reshape(-1)], color, downsample_ratio) for x in polygons]) + + +def polygons2masks_overlap(imgsz, segments, downsample_ratio=1): + """Return a (640, 640) overlap mask.""" + masks = np.zeros( + (imgsz[0] // downsample_ratio, imgsz[1] // downsample_ratio), + dtype=np.int32 if len(segments) > 255 else np.uint8, + ) + areas = [] + ms = [] + for si in range(len(segments)): + mask = polygon2mask(imgsz, [segments[si].reshape(-1)], downsample_ratio=downsample_ratio, color=1) + ms.append(mask) + areas.append(mask.sum()) + areas = np.asarray(areas) + index = np.argsort(-areas) + ms = np.array(ms)[index] + for i in range(len(segments)): + mask = ms[i] * (i + 1) + masks = masks + mask + masks = np.clip(masks, a_min=0, a_max=i + 1) + return masks, index + + +def find_dataset_yaml(path: Path) -> Path: + """ + Find and return the YAML file associated with a Detect, Segment or Pose dataset. + + This function searches for a YAML file at the root level of the provided directory first, and if not found, it + performs a recursive search. It prefers YAML files that have the same stem as the provided path. An AssertionError + is raised if no YAML file is found or if multiple YAML files are found. + + Args: + path (Path): The directory path to search for the YAML file. + + Returns: + (Path): The path of the found YAML file. + """ + files = list(path.glob("*.yaml")) or list(path.rglob("*.yaml")) # try root level first and then recursive + assert files, f"No YAML file found in '{path.resolve()}'" + if len(files) > 1: + files = [f for f in files if f.stem == path.stem] # prefer *.yaml files that match + assert len(files) == 1, f"Expected 1 YAML file in '{path.resolve()}', but found {len(files)}.\n{files}" + return files[0] + + +def check_det_dataset(dataset, autodownload=True): + """ + Download, verify, and/or unzip a dataset if not found locally. + + This function checks the availability of a specified dataset, and if not found, it has the option to download and + unzip the dataset. It then reads and parses the accompanying YAML data, ensuring key requirements are met and also + resolves paths related to the dataset. + + Args: + dataset (str): Path to the dataset or dataset descriptor (like a YAML file). + autodownload (bool, optional): Whether to automatically download the dataset if not found. Defaults to True. + + Returns: + (dict): Parsed dataset information and paths. + """ + + file = check_file(dataset) + + # Download (optional) + extract_dir = "" + if zipfile.is_zipfile(file) or is_tarfile(file): + new_dir = safe_download(file, dir=DATASETS_DIR, unzip=True, delete=False) + file = find_dataset_yaml(DATASETS_DIR / new_dir) + extract_dir, autodownload = file.parent, False + + # Read YAML + data = yaml_load(file, append_filename=True) # dictionary + + # Checks + for k in "train", "val": + if k not in data: + if k != "val" or "validation" not in data: + raise SyntaxError( + emojis(f"{dataset} '{k}:' key missing ❌.\n'train' and 'val' are required in all data YAMLs.") + ) + LOGGER.info("WARNING ⚠️ renaming data YAML 'validation' key to 'val' to match YOLO format.") + data["val"] = data.pop("validation") # replace 'validation' key with 'val' key + if "names" not in data and "nc" not in data: + raise SyntaxError(emojis(f"{dataset} key missing ❌.\n either 'names' or 'nc' are required in all data YAMLs.")) + if "names" in data and "nc" in data and len(data["names"]) != data["nc"]: + raise SyntaxError(emojis(f"{dataset} 'names' length {len(data['names'])} and 'nc: {data['nc']}' must match.")) + if "names" not in data: + data["names"] = [f"class_{i}" for i in range(data["nc"])] + else: + data["nc"] = len(data["names"]) + + data["names"] = check_class_names(data["names"]) + + # Resolve paths + path = Path(extract_dir or data.get("path") or Path(data.get("yaml_file", "")).parent) # dataset root + if not path.is_absolute(): + path = (DATASETS_DIR / path).resolve() + + # Set paths + data["path"] = path # download scripts + for k in "train", "val", "test": + if data.get(k): # prepend path + if isinstance(data[k], str): + x = (path / data[k]).resolve() + if not x.exists() and data[k].startswith("../"): + x = (path / data[k][3:]).resolve() + data[k] = str(x) + else: + data[k] = [str((path / x).resolve()) for x in data[k]] + + # Parse YAML + val, s = (data.get(x) for x in ("val", "download")) + if val: + val = [Path(x).resolve() for x in (val if isinstance(val, list) else [val])] # val path + if not all(x.exists() for x in val): + name = clean_url(dataset) # dataset name with URL auth stripped + m = f"\nDataset '{name}' images not found ⚠️, missing path '{[x for x in val if not x.exists()][0]}'" + if s and autodownload: + LOGGER.warning(m) + else: + m += f"\nNote dataset download directory is '{DATASETS_DIR}'. You can update this in '{SETTINGS_YAML}'" + raise FileNotFoundError(m) + t = time.time() + r = None # success + if s.startswith("http") and s.endswith(".zip"): # URL + safe_download(url=s, dir=DATASETS_DIR, delete=True) + elif s.startswith("bash "): # bash script + LOGGER.info(f"Running {s} ...") + r = os.system(s) + else: # python script + exec(s, {"yaml": data}) + dt = f"({round(time.time() - t, 1)}s)" + s = f"success ✅ {dt}, saved to {colorstr('bold', DATASETS_DIR)}" if r in (0, None) else f"failure {dt} ❌" + LOGGER.info(f"Dataset download {s}\n") + check_font("Arial.ttf" if is_ascii(data["names"]) else "Arial.Unicode.ttf") # download fonts + + return data # dictionary + + +def check_cls_dataset(dataset, split=""): + """ + Checks a classification dataset such as Imagenet. + + This function accepts a `dataset` name and attempts to retrieve the corresponding dataset information. + If the dataset is not found locally, it attempts to download the dataset from the internet and save it locally. + + Args: + dataset (str | Path): The name of the dataset. + split (str, optional): The split of the dataset. Either 'val', 'test', or ''. Defaults to ''. + + Returns: + (dict): A dictionary containing the following keys: + - 'train' (Path): The directory path containing the training set of the dataset. + - 'val' (Path): The directory path containing the validation set of the dataset. + - 'test' (Path): The directory path containing the test set of the dataset. + - 'nc' (int): The number of classes in the dataset. + - 'names' (dict): A dictionary of class names in the dataset. + """ + + # Download (optional if dataset=https://file.zip is passed directly) + if str(dataset).startswith(("http:/", "https:/")): + dataset = safe_download(dataset, dir=DATASETS_DIR, unzip=True, delete=False) + elif Path(dataset).suffix in (".zip", ".tar", ".gz"): + file = check_file(dataset) + dataset = safe_download(file, dir=DATASETS_DIR, unzip=True, delete=False) + + dataset = Path(dataset) + data_dir = (dataset if dataset.is_dir() else (DATASETS_DIR / dataset)).resolve() + if not data_dir.is_dir(): + LOGGER.warning(f"\nDataset not found ⚠️, missing path {data_dir}, attempting download...") + t = time.time() + if str(dataset) == "imagenet": + subprocess.run(f"bash {ROOT / 'data/scripts/get_imagenet.sh'}", shell=True, check=True) + else: + url = f"https://github.com/ultralytics/yolov5/releases/download/v1.0/{dataset}.zip" + download(url, dir=data_dir.parent) + s = f"Dataset download success ✅ ({time.time() - t:.1f}s), saved to {colorstr('bold', data_dir)}\n" + LOGGER.info(s) + train_set = data_dir / "train" + val_set = ( + data_dir / "val" + if (data_dir / "val").exists() + else data_dir / "validation" + if (data_dir / "validation").exists() + else None + ) # data/test or data/val + test_set = data_dir / "test" if (data_dir / "test").exists() else None # data/val or data/test + if split == "val" and not val_set: + LOGGER.warning("WARNING ⚠️ Dataset 'split=val' not found, using 'split=test' instead.") + elif split == "test" and not test_set: + LOGGER.warning("WARNING ⚠️ Dataset 'split=test' not found, using 'split=val' instead.") + + nc = len([x for x in (data_dir / "train").glob("*") if x.is_dir()]) # number of classes + names = [x.name for x in (data_dir / "train").iterdir() if x.is_dir()] # class names list + names = dict(enumerate(sorted(names))) + + # Print to console + for k, v in {"train": train_set, "val": val_set, "test": test_set}.items(): + prefix = f'{colorstr(f"{k}:")} {v}...' + if v is None: + LOGGER.info(prefix) + else: + files = [path for path in v.rglob("*.*") if path.suffix[1:].lower() in IMG_FORMATS] + nf = len(files) # number of files + nd = len({file.parent for file in files}) # number of directories + if nf == 0: + if k == "train": + raise FileNotFoundError(emojis(f"{dataset} '{k}:' no training images found ❌ ")) + else: + LOGGER.warning(f"{prefix} found {nf} images in {nd} classes: WARNING ⚠️ no images found") + elif nd != nc: + LOGGER.warning(f"{prefix} found {nf} images in {nd} classes: ERROR ❌️ requires {nc} classes, not {nd}") + else: + LOGGER.info(f"{prefix} found {nf} images in {nd} classes ✅ ") + + return {"train": train_set, "val": val_set, "test": test_set, "nc": nc, "names": names} + + +class HUBDatasetStats: + """ + A class for generating HUB dataset JSON and `-hub` dataset directory. + + Args: + path (str): Path to data.yaml or data.zip (with data.yaml inside data.zip). Default is 'coco8.yaml'. + task (str): Dataset task. Options are 'detect', 'segment', 'pose', 'classify'. Default is 'detect'. + autodownload (bool): Attempt to download dataset if not found locally. Default is False. + + Example: + Download *.zip files from https://github.com/ultralytics/hub/tree/main/example_datasets + i.e. https://github.com/ultralytics/hub/raw/main/example_datasets/coco8.zip for coco8.zip. + ```python + from ultralytics.data.utils import HUBDatasetStats + + stats = HUBDatasetStats('path/to/coco8.zip', task='detect') # detect dataset + stats = HUBDatasetStats('path/to/coco8-seg.zip', task='segment') # segment dataset + stats = HUBDatasetStats('path/to/coco8-pose.zip', task='pose') # pose dataset + stats = HUBDatasetStats('path/to/imagenet10.zip', task='classify') # classification dataset + + stats.get_json(save=True) + stats.process_images() + ``` + """ + + def __init__(self, path="coco8.yaml", task="detect", autodownload=False): + """Initialize class.""" + path = Path(path).resolve() + LOGGER.info(f"Starting HUB dataset checks for {path}....") + + self.task = task # detect, segment, pose, classify + if self.task == "classify": + unzip_dir = unzip_file(path) + data = check_cls_dataset(unzip_dir) + data["path"] = unzip_dir + else: # detect, segment, pose + _, data_dir, yaml_path = self._unzip(Path(path)) + try: + # Load YAML with checks + data = yaml_load(yaml_path) + data["path"] = "" # strip path since YAML should be in dataset root for all HUB datasets + yaml_save(yaml_path, data) + data = check_det_dataset(yaml_path, autodownload) # dict + data["path"] = data_dir # YAML path should be set to '' (relative) or parent (absolute) + except Exception as e: + raise Exception("error/HUB/dataset_stats/init") from e + + self.hub_dir = Path(f'{data["path"]}-hub') + self.im_dir = self.hub_dir / "images" + self.stats = {"nc": len(data["names"]), "names": list(data["names"].values())} # statistics dictionary + self.data = data + + @staticmethod + def _unzip(path): + """Unzip data.zip.""" + if not str(path).endswith(".zip"): # path is data.yaml + return False, None, path + unzip_dir = unzip_file(path, path=path.parent) + assert unzip_dir.is_dir(), ( + f"Error unzipping {path}, {unzip_dir} not found. " f"path/to/abc.zip MUST unzip to path/to/abc/" + ) + return True, str(unzip_dir), find_dataset_yaml(unzip_dir) # zipped, data_dir, yaml_path + + def _hub_ops(self, f): + """Saves a compressed image for HUB previews.""" + compress_one_image(f, self.im_dir / Path(f).name) # save to dataset-hub + + def get_json(self, save=False, verbose=False): + """Return dataset JSON for Ultralytics HUB.""" + + def _round(labels): + """Update labels to integer class and 4 decimal place floats.""" + if self.task == "detect": + coordinates = labels["bboxes"] + elif self.task == "segment": + coordinates = [x.flatten() for x in labels["segments"]] + elif self.task == "pose": + n = labels["keypoints"].shape[0] + coordinates = np.concatenate((labels["bboxes"], labels["keypoints"].reshape(n, -1)), 1) + else: + raise ValueError("Undefined dataset task.") + zipped = zip(labels["cls"], coordinates) + return [[int(c[0]), *(round(float(x), 4) for x in points)] for c, points in zipped] + + for split in "train", "val", "test": + self.stats[split] = None # predefine + path = self.data.get(split) + + # Check split + if path is None: # no split + continue + files = [f for f in Path(path).rglob("*.*") if f.suffix[1:].lower() in IMG_FORMATS] # image files in split + if not files: # no images + continue + + # Get dataset statistics + if self.task == "classify": + from torchvision.datasets import ImageFolder + + dataset = ImageFolder(self.data[split]) + + x = np.zeros(len(dataset.classes)).astype(int) + for im in dataset.imgs: + x[im[1]] += 1 + + self.stats[split] = { + "instance_stats": {"total": len(dataset), "per_class": x.tolist()}, + "image_stats": {"total": len(dataset), "unlabelled": 0, "per_class": x.tolist()}, + "labels": [{Path(k).name: v} for k, v in dataset.imgs], + } + else: + from ultralytics.data import YOLODataset + + dataset = YOLODataset(img_path=self.data[split], data=self.data, task=self.task) + x = np.array( + [ + np.bincount(label["cls"].astype(int).flatten(), minlength=self.data["nc"]) + for label in TQDM(dataset.labels, total=len(dataset), desc="Statistics") + ] + ) # shape(128x80) + self.stats[split] = { + "instance_stats": {"total": int(x.sum()), "per_class": x.sum(0).tolist()}, + "image_stats": { + "total": len(dataset), + "unlabelled": int(np.all(x == 0, 1).sum()), + "per_class": (x > 0).sum(0).tolist(), + }, + "labels": [{Path(k).name: _round(v)} for k, v in zip(dataset.im_files, dataset.labels)], + } + + # Save, print and return + if save: + self.hub_dir.mkdir(parents=True, exist_ok=True) # makes dataset-hub/ + stats_path = self.hub_dir / "stats.json" + LOGGER.info(f"Saving {stats_path.resolve()}...") + with open(stats_path, "w") as f: + json.dump(self.stats, f) # save stats.json + if verbose: + LOGGER.info(json.dumps(self.stats, indent=2, sort_keys=False)) + return self.stats + + def process_images(self): + """Compress images for Ultralytics HUB.""" + from ultralytics.data import YOLODataset # ClassificationDataset + + self.im_dir.mkdir(parents=True, exist_ok=True) # makes dataset-hub/images/ + for split in "train", "val", "test": + if self.data.get(split) is None: + continue + dataset = YOLODataset(img_path=self.data[split], data=self.data) + with ThreadPool(NUM_THREADS) as pool: + for _ in TQDM(pool.imap(self._hub_ops, dataset.im_files), total=len(dataset), desc=f"{split} images"): + pass + LOGGER.info(f"Done. All images saved to {self.im_dir}") + return self.im_dir + + +def compress_one_image(f, f_new=None, max_dim=1920, quality=50): + """ + Compresses a single image file to reduced size while preserving its aspect ratio and quality using either the Python + Imaging Library (PIL) or OpenCV library. If the input image is smaller than the maximum dimension, it will not be + resized. + + Args: + f (str): The path to the input image file. + f_new (str, optional): The path to the output image file. If not specified, the input file will be overwritten. + max_dim (int, optional): The maximum dimension (width or height) of the output image. Default is 1920 pixels. + quality (int, optional): The image compression quality as a percentage. Default is 50%. + + Example: + ```python + from pathlib import Path + from ultralytics.data.utils import compress_one_image + + for f in Path('path/to/dataset').rglob('*.jpg'): + compress_one_image(f) + ``` + """ + + try: # use PIL + im = Image.open(f) + r = max_dim / max(im.height, im.width) # ratio + if r < 1.0: # image too large + im = im.resize((int(im.width * r), int(im.height * r))) + im.save(f_new or f, "JPEG", quality=quality, optimize=True) # save + except Exception as e: # use OpenCV + LOGGER.info(f"WARNING ⚠️ HUB ops PIL failure {f}: {e}") + im = cv2.imread(f) + im_height, im_width = im.shape[:2] + r = max_dim / max(im_height, im_width) # ratio + if r < 1.0: # image too large + im = cv2.resize(im, (int(im_width * r), int(im_height * r)), interpolation=cv2.INTER_AREA) + cv2.imwrite(str(f_new or f), im) + + +def autosplit(path=DATASETS_DIR / "coco8/images", weights=(0.9, 0.1, 0.0), annotated_only=False): + """ + Automatically split a dataset into train/val/test splits and save the resulting splits into autosplit_*.txt files. + + Args: + path (Path, optional): Path to images directory. Defaults to DATASETS_DIR / 'coco8/images'. + weights (list | tuple, optional): Train, validation, and test split fractions. Defaults to (0.9, 0.1, 0.0). + annotated_only (bool, optional): If True, only images with an associated txt file are used. Defaults to False. + + Example: + ```python + from ultralytics.data.utils import autosplit + + autosplit() + ``` + """ + + path = Path(path) # images dir + files = sorted(x for x in path.rglob("*.*") if x.suffix[1:].lower() in IMG_FORMATS) # image files only + n = len(files) # number of files + random.seed(0) # for reproducibility + indices = random.choices([0, 1, 2], weights=weights, k=n) # assign each image to a split + + txt = ["autosplit_train.txt", "autosplit_val.txt", "autosplit_test.txt"] # 3 txt files + for x in txt: + if (path.parent / x).exists(): + (path.parent / x).unlink() # remove existing + + LOGGER.info(f"Autosplitting images from {path}" + ", using *.txt labeled images only" * annotated_only) + for i, img in TQDM(zip(indices, files), total=n): + if not annotated_only or Path(img2label_paths([str(img)])[0]).exists(): # check label + with open(path.parent / txt[i], "a") as f: + f.write(f"./{img.relative_to(path.parent).as_posix()}" + "\n") # add image to txt file diff --git a/ultralytics/engine/__init__.py b/ultralytics/engine/__init__.py new file mode 100644 index 0000000..9e68dc1 --- /dev/null +++ b/ultralytics/engine/__init__.py @@ -0,0 +1 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license diff --git a/ultralytics/engine/__pycache__/__init__.cpython-312.pyc b/ultralytics/engine/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..b4f9384 Binary files /dev/null and b/ultralytics/engine/__pycache__/__init__.cpython-312.pyc differ diff --git a/ultralytics/engine/__pycache__/__init__.cpython-39.pyc b/ultralytics/engine/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..d1d5e84 Binary files /dev/null and b/ultralytics/engine/__pycache__/__init__.cpython-39.pyc differ diff --git a/ultralytics/engine/__pycache__/exporter.cpython-312.pyc b/ultralytics/engine/__pycache__/exporter.cpython-312.pyc new file mode 100644 index 0000000..016503f Binary files /dev/null and b/ultralytics/engine/__pycache__/exporter.cpython-312.pyc differ diff --git a/ultralytics/engine/__pycache__/exporter.cpython-39.pyc b/ultralytics/engine/__pycache__/exporter.cpython-39.pyc new file mode 100644 index 0000000..dc597f0 Binary files /dev/null and b/ultralytics/engine/__pycache__/exporter.cpython-39.pyc differ diff --git a/ultralytics/engine/__pycache__/model.cpython-312.pyc b/ultralytics/engine/__pycache__/model.cpython-312.pyc new file mode 100644 index 0000000..4ea6cac Binary files /dev/null and b/ultralytics/engine/__pycache__/model.cpython-312.pyc differ diff --git a/ultralytics/engine/__pycache__/model.cpython-39.pyc b/ultralytics/engine/__pycache__/model.cpython-39.pyc new file mode 100644 index 0000000..b1d5c97 Binary files /dev/null and b/ultralytics/engine/__pycache__/model.cpython-39.pyc differ diff --git a/ultralytics/engine/__pycache__/predictor.cpython-312.pyc b/ultralytics/engine/__pycache__/predictor.cpython-312.pyc new file mode 100644 index 0000000..d5cde2b Binary files /dev/null and b/ultralytics/engine/__pycache__/predictor.cpython-312.pyc differ diff --git a/ultralytics/engine/__pycache__/predictor.cpython-39.pyc b/ultralytics/engine/__pycache__/predictor.cpython-39.pyc new file mode 100644 index 0000000..958802e Binary files /dev/null and b/ultralytics/engine/__pycache__/predictor.cpython-39.pyc differ diff --git a/ultralytics/engine/__pycache__/results.cpython-312.pyc b/ultralytics/engine/__pycache__/results.cpython-312.pyc new file mode 100644 index 0000000..ff65abe Binary files /dev/null and b/ultralytics/engine/__pycache__/results.cpython-312.pyc differ diff --git a/ultralytics/engine/__pycache__/results.cpython-39.pyc b/ultralytics/engine/__pycache__/results.cpython-39.pyc new file mode 100644 index 0000000..a8f289a Binary files /dev/null and b/ultralytics/engine/__pycache__/results.cpython-39.pyc differ diff --git a/ultralytics/engine/__pycache__/trainer.cpython-312.pyc b/ultralytics/engine/__pycache__/trainer.cpython-312.pyc new file mode 100644 index 0000000..08e65b6 Binary files /dev/null and b/ultralytics/engine/__pycache__/trainer.cpython-312.pyc differ diff --git a/ultralytics/engine/__pycache__/trainer.cpython-39.pyc b/ultralytics/engine/__pycache__/trainer.cpython-39.pyc new file mode 100644 index 0000000..6ab5ba2 Binary files /dev/null and b/ultralytics/engine/__pycache__/trainer.cpython-39.pyc differ diff --git a/ultralytics/engine/__pycache__/validator.cpython-312.pyc b/ultralytics/engine/__pycache__/validator.cpython-312.pyc new file mode 100644 index 0000000..0da7649 Binary files /dev/null and b/ultralytics/engine/__pycache__/validator.cpython-312.pyc differ diff --git a/ultralytics/engine/__pycache__/validator.cpython-39.pyc b/ultralytics/engine/__pycache__/validator.cpython-39.pyc new file mode 100644 index 0000000..469bb61 Binary files /dev/null and b/ultralytics/engine/__pycache__/validator.cpython-39.pyc differ diff --git a/ultralytics/engine/exporter.py b/ultralytics/engine/exporter.py new file mode 100644 index 0000000..6ac170c --- /dev/null +++ b/ultralytics/engine/exporter.py @@ -0,0 +1,1145 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +""" +Export a YOLOv8 PyTorch model to other formats. TensorFlow exports authored by https://github.com/zldrobit + +Format | `format=argument` | Model +--- | --- | --- +PyTorch | - | yolov8n.pt +TorchScript | `torchscript` | yolov8n.torchscript +ONNX | `onnx` | yolov8n.onnx +OpenVINO | `openvino` | yolov8n_openvino_model/ +TensorRT | `engine` | yolov8n.engine +CoreML | `coreml` | yolov8n.mlpackage +TensorFlow SavedModel | `saved_model` | yolov8n_saved_model/ +TensorFlow GraphDef | `pb` | yolov8n.pb +TensorFlow Lite | `tflite` | yolov8n.tflite +TensorFlow Edge TPU | `edgetpu` | yolov8n_edgetpu.tflite +TensorFlow.js | `tfjs` | yolov8n_web_model/ +PaddlePaddle | `paddle` | yolov8n_paddle_model/ +NCNN | `ncnn` | yolov8n_ncnn_model/ + +Requirements: + $ pip install "ultralytics[export]" + +Python: + from ultralytics import YOLO + model = YOLO('yolov8n.pt') + results = model.export(format='onnx') + +CLI: + $ yolo mode=export model=yolov8n.pt format=onnx + +Inference: + $ yolo predict model=yolov8n.pt # PyTorch + yolov8n.torchscript # TorchScript + yolov8n.onnx # ONNX Runtime or OpenCV DNN with dnn=True + yolov8n_openvino_model # OpenVINO + yolov8n.engine # TensorRT + yolov8n.mlpackage # CoreML (macOS-only) + yolov8n_saved_model # TensorFlow SavedModel + yolov8n.pb # TensorFlow GraphDef + yolov8n.tflite # TensorFlow Lite + yolov8n_edgetpu.tflite # TensorFlow Edge TPU + yolov8n_paddle_model # PaddlePaddle + yolov8n_ncnn_model # NCNN + +TensorFlow.js: + $ cd .. && git clone https://github.com/zldrobit/tfjs-yolov5-example.git && cd tfjs-yolov5-example + $ npm install + $ ln -s ../../yolov5/yolov8n_web_model public/yolov8n_web_model + $ npm start +""" + +import json +import os +import shutil +import subprocess +import time +import warnings +from copy import deepcopy +from datetime import datetime +from pathlib import Path + +import numpy as np +import torch + +from ultralytics.cfg import get_cfg +from ultralytics.data.dataset import YOLODataset +from ultralytics.data.utils import check_det_dataset +from ultralytics.nn.autobackend import check_class_names, default_class_names +from ultralytics.nn.modules import C2f, Detect, RTDETRDecoder, v10Detect +from ultralytics.nn.tasks import DetectionModel, SegmentationModel, WorldModel +from ultralytics.utils import ( + ARM64, + DEFAULT_CFG, + LINUX, + LOGGER, + MACOS, + ROOT, + WINDOWS, + __version__, + callbacks, + colorstr, + get_default_args, + yaml_save, +) +from ultralytics.utils.checks import PYTHON_VERSION, check_imgsz, check_is_path_safe, check_requirements, check_version +from ultralytics.utils.downloads import attempt_download_asset, get_github_assets +from ultralytics.utils.files import file_size, spaces_in_path +from ultralytics.utils.ops import Profile +from ultralytics.utils.torch_utils import TORCH_1_13, get_latest_opset, select_device, smart_inference_mode + + +def export_formats(): + """YOLOv8 export formats.""" + import pandas + + x = [ + ["PyTorch", "-", ".pt", True, True], + ["TorchScript", "torchscript", ".torchscript", True, True], + ["ONNX", "onnx", ".onnx", True, True], + ["OpenVINO", "openvino", "_openvino_model", True, False], + ["TensorRT", "engine", ".engine", False, True], + ["CoreML", "coreml", ".mlpackage", True, False], + ["TensorFlow SavedModel", "saved_model", "_saved_model", True, True], + ["TensorFlow GraphDef", "pb", ".pb", True, True], + ["TensorFlow Lite", "tflite", ".tflite", True, False], + ["TensorFlow Edge TPU", "edgetpu", "_edgetpu.tflite", True, False], + ["TensorFlow.js", "tfjs", "_web_model", True, False], + ["PaddlePaddle", "paddle", "_paddle_model", True, True], + ["NCNN", "ncnn", "_ncnn_model", True, True], + ] + return pandas.DataFrame(x, columns=["Format", "Argument", "Suffix", "CPU", "GPU"]) + + +def gd_outputs(gd): + """TensorFlow GraphDef model output node names.""" + name_list, input_list = [], [] + for node in gd.node: # tensorflow.core.framework.node_def_pb2.NodeDef + name_list.append(node.name) + input_list.extend(node.input) + return sorted(f"{x}:0" for x in list(set(name_list) - set(input_list)) if not x.startswith("NoOp")) + + +def try_export(inner_func): + """YOLOv8 export decorator, i..e @try_export.""" + inner_args = get_default_args(inner_func) + + def outer_func(*args, **kwargs): + """Export a model.""" + prefix = inner_args["prefix"] + try: + with Profile() as dt: + f, model = inner_func(*args, **kwargs) + LOGGER.info(f"{prefix} export success ✅ {dt.t:.1f}s, saved as '{f}' ({file_size(f):.1f} MB)") + return f, model + except Exception as e: + LOGGER.info(f"{prefix} export failure ❌ {dt.t:.1f}s: {e}") + raise e + + return outer_func + + +class Exporter: + """ + A class for exporting a model. + + Attributes: + args (SimpleNamespace): Configuration for the exporter. + callbacks (list, optional): List of callback functions. Defaults to None. + """ + + def __init__(self, cfg=DEFAULT_CFG, overrides=None, _callbacks=None): + """ + Initializes the Exporter class. + + Args: + cfg (str, optional): Path to a configuration file. Defaults to DEFAULT_CFG. + overrides (dict, optional): Configuration overrides. Defaults to None. + _callbacks (dict, optional): Dictionary of callback functions. Defaults to None. + """ + self.args = get_cfg(cfg, overrides) + if self.args.format.lower() in ("coreml", "mlmodel"): # fix attempt for protobuf<3.20.x errors + os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python" # must run before TensorBoard callback + + self.callbacks = _callbacks or callbacks.get_default_callbacks() + callbacks.add_integration_callbacks(self) + + @smart_inference_mode() + def __call__(self, model=None): + """Returns list of exported files/dirs after running callbacks.""" + self.run_callbacks("on_export_start") + t = time.time() + fmt = self.args.format.lower() # to lowercase + if fmt in ("tensorrt", "trt"): # 'engine' aliases + fmt = "engine" + if fmt in ("mlmodel", "mlpackage", "mlprogram", "apple", "ios", "coreml"): # 'coreml' aliases + fmt = "coreml" + fmts = tuple(export_formats()["Argument"][1:]) # available export formats + flags = [x == fmt for x in fmts] + if sum(flags) != 1: + raise ValueError(f"Invalid export format='{fmt}'. Valid formats are {fmts}") + jit, onnx, xml, engine, coreml, saved_model, pb, tflite, edgetpu, tfjs, paddle, ncnn = flags # export booleans + + # Device + if fmt == "engine" and self.args.device is None: + LOGGER.warning("WARNING ⚠️ TensorRT requires GPU export, automatically assigning device=0") + self.args.device = "0" + self.device = select_device("cpu" if self.args.device is None else self.args.device) + + # Checks + if not hasattr(model, "names"): + model.names = default_class_names() + model.names = check_class_names(model.names) + if self.args.half and onnx and self.device.type == "cpu": + LOGGER.warning("WARNING ⚠️ half=True only compatible with GPU export, i.e. use device=0") + self.args.half = False + assert not self.args.dynamic, "half=True not compatible with dynamic=True, i.e. use only one." + self.imgsz = check_imgsz(self.args.imgsz, stride=model.stride, min_dim=2) # check image size + if self.args.optimize: + assert not ncnn, "optimize=True not compatible with format='ncnn', i.e. use optimize=False" + assert self.device.type == "cpu", "optimize=True not compatible with cuda devices, i.e. use device='cpu'" + if edgetpu and not LINUX: + raise SystemError("Edge TPU export only supported on Linux. See https://coral.ai/docs/edgetpu/compiler/") + if isinstance(model, WorldModel): + LOGGER.warning( + "WARNING ⚠️ YOLOWorld (original version) export is not supported to any format.\n" + "WARNING ⚠️ YOLOWorldv2 models (i.e. 'yolov8s-worldv2.pt') only support export to " + "(torchscript, onnx, openvino, engine, coreml) formats. " + "See https://docs.ultralytics.com/models/yolo-world for details." + ) + + # Input + im = torch.zeros(self.args.batch, 3, *self.imgsz).to(self.device) + file = Path( + getattr(model, "pt_path", None) or getattr(model, "yaml_file", None) or model.yaml.get("yaml_file", "") + ) + if file.suffix in {".yaml", ".yml"}: + file = Path(file.name) + + # Update model + model = deepcopy(model).to(self.device) + for p in model.parameters(): + p.requires_grad = False + model.eval() + model.float() + model = model.fuse() + for m in model.modules(): + if isinstance(m, (Detect, RTDETRDecoder)): # includes all Detect subclasses like Segment, Pose, OBB + m.dynamic = self.args.dynamic + m.export = True + m.format = self.args.format + if isinstance(m, v10Detect): + m.max_det = self.args.max_det + + elif isinstance(m, C2f) and not any((saved_model, pb, tflite, edgetpu, tfjs)): + # EdgeTPU does not support FlexSplitV while split provides cleaner ONNX graph + m.forward = m.forward_split + + y = None + for _ in range(2): + y = model(im) # dry runs + if self.args.half and onnx and self.device.type != "cpu": + im, model = im.half(), model.half() # to FP16 + + # Filter warnings + warnings.filterwarnings("ignore", category=torch.jit.TracerWarning) # suppress TracerWarning + warnings.filterwarnings("ignore", category=UserWarning) # suppress shape prim::Constant missing ONNX warning + warnings.filterwarnings("ignore", category=DeprecationWarning) # suppress CoreML np.bool deprecation warning + + # Assign + self.im = im + self.model = model + self.file = file + self.output_shape = ( + tuple(y.shape) + if isinstance(y, torch.Tensor) + else tuple(tuple(x.shape if isinstance(x, torch.Tensor) else []) for x in y) + ) + self.pretty_name = Path(self.model.yaml.get("yaml_file", self.file)).stem.replace("yolo", "YOLO") + data = model.args["data"] if hasattr(model, "args") and isinstance(model.args, dict) else "" + description = f'Ultralytics {self.pretty_name} model {f"trained on {data}" if data else ""}' + self.metadata = { + "description": description, + "author": "Ultralytics", + "date": datetime.now().isoformat(), + "version": __version__, + "license": "AGPL-3.0 License (https://ultralytics.com/license)", + "docs": "https://docs.ultralytics.com", + "stride": int(max(model.stride)), + "task": model.task, + "batch": self.args.batch, + "imgsz": self.imgsz, + "names": model.names, + } # model metadata + if model.task == "pose": + self.metadata["kpt_shape"] = model.model[-1].kpt_shape + + LOGGER.info( + f"\n{colorstr('PyTorch:')} starting from '{file}' with input shape {tuple(im.shape)} BCHW and " + f'output shape(s) {self.output_shape} ({file_size(file):.1f} MB)' + ) + + # Exports + f = [""] * len(fmts) # exported filenames + if jit or ncnn: # TorchScript + f[0], _ = self.export_torchscript() + if engine: # TensorRT required before ONNX + f[1], _ = self.export_engine() + if onnx: # ONNX + f[2], _ = self.export_onnx() + if xml: # OpenVINO + f[3], _ = self.export_openvino() + if coreml: # CoreML + f[4], _ = self.export_coreml() + if any((saved_model, pb, tflite, edgetpu, tfjs)): # TensorFlow formats + self.args.int8 |= edgetpu + f[5], keras_model = self.export_saved_model() + if pb or tfjs: # pb prerequisite to tfjs + f[6], _ = self.export_pb(keras_model=keras_model) + if tflite: + f[7], _ = self.export_tflite(keras_model=keras_model, nms=False, agnostic_nms=self.args.agnostic_nms) + if edgetpu: + f[8], _ = self.export_edgetpu(tflite_model=Path(f[5]) / f"{self.file.stem}_full_integer_quant.tflite") + if tfjs: + f[9], _ = self.export_tfjs() + if paddle: # PaddlePaddle + f[10], _ = self.export_paddle() + if ncnn: # NCNN + f[11], _ = self.export_ncnn() + + # Finish + f = [str(x) for x in f if x] # filter out '' and None + if any(f): + f = str(Path(f[-1])) + square = self.imgsz[0] == self.imgsz[1] + s = ( + "" + if square + else f"WARNING ⚠️ non-PyTorch val requires square images, 'imgsz={self.imgsz}' will not " + f"work. Use export 'imgsz={max(self.imgsz)}' if val is required." + ) + imgsz = self.imgsz[0] if square else str(self.imgsz)[1:-1].replace(" ", "") + predict_data = f"data={data}" if model.task == "segment" and fmt == "pb" else "" + q = "int8" if self.args.int8 else "half" if self.args.half else "" # quantization + LOGGER.info( + f'\nExport complete ({time.time() - t:.1f}s)' + f"\nResults saved to {colorstr('bold', file.parent.resolve())}" + f'\nPredict: yolo predict task={model.task} model={f} imgsz={imgsz} {q} {predict_data}' + f'\nValidate: yolo val task={model.task} model={f} imgsz={imgsz} data={data} {q} {s}' + f'\nVisualize: https://netron.app' + ) + + self.run_callbacks("on_export_end") + return f # return list of exported files/dirs + + @try_export + def export_torchscript(self, prefix=colorstr("TorchScript:")): + """YOLOv8 TorchScript model export.""" + LOGGER.info(f"\n{prefix} starting export with torch {torch.__version__}...") + f = self.file.with_suffix(".torchscript") + + ts = torch.jit.trace(self.model, self.im, strict=False) + extra_files = {"config.txt": json.dumps(self.metadata)} # torch._C.ExtraFilesMap() + if self.args.optimize: # https://pytorch.org/tutorials/recipes/mobile_interpreter.html + LOGGER.info(f"{prefix} optimizing for mobile...") + from torch.utils.mobile_optimizer import optimize_for_mobile + + optimize_for_mobile(ts)._save_for_lite_interpreter(str(f), _extra_files=extra_files) + else: + ts.save(str(f), _extra_files=extra_files) + return f, None + + @try_export + def export_onnx(self, prefix=colorstr("ONNX:")): + """YOLOv8 ONNX export.""" + requirements = ["onnx>=1.12.0"] + if self.args.simplify: + requirements += ["onnxslim==0.1.31", "onnxruntime" + ("-gpu" if torch.cuda.is_available() else "")] + check_requirements(requirements) + import onnx # noqa + + opset_version = self.args.opset or get_latest_opset() + LOGGER.info(f"\n{prefix} starting export with onnx {onnx.__version__} opset {opset_version}...") + f = str(self.file.with_suffix(".onnx")) + + output_names = ["output0", "output1"] if isinstance(self.model, SegmentationModel) else ["output0"] + dynamic = self.args.dynamic + if dynamic: + dynamic = {"images": {0: "batch", 2: "height", 3: "width"}} # shape(1,3,640,640) + if isinstance(self.model, SegmentationModel): + dynamic["output0"] = {0: "batch", 2: "anchors"} # shape(1, 116, 8400) + dynamic["output1"] = {0: "batch", 2: "mask_height", 3: "mask_width"} # shape(1,32,160,160) + elif isinstance(self.model, DetectionModel): + dynamic["output0"] = {0: "batch", 2: "anchors"} # shape(1, 84, 8400) + + torch.onnx.export( + self.model.cpu() if dynamic else self.model, # dynamic=True only compatible with cpu + self.im.cpu() if dynamic else self.im, + f, + verbose=False, + opset_version=opset_version, + do_constant_folding=True, # WARNING: DNN inference with torch>=1.12 may require do_constant_folding=False + input_names=["images"], + output_names=output_names, + dynamic_axes=dynamic or None, + ) + + # Checks + model_onnx = onnx.load(f) # load onnx model + # onnx.checker.check_model(model_onnx) # check onnx model + + # Simplify + if self.args.simplify: + try: + import onnxslim + + LOGGER.info(f"{prefix} simplifying with onnxslim {onnxslim.__version__}...") + model_onnx = onnxslim.slim(model_onnx) + + # ONNX Simplifier (deprecated as must be compiled with 'cmake' in aarch64 and Conda CI environments) + # import onnxsim + # model_onnx, check = onnxsim.simplify(model_onnx) + # assert check, "Simplified ONNX model could not be validated" + except Exception as e: + LOGGER.warning(f"{prefix} simplifier failure: {e}") + + # Metadata + for k, v in self.metadata.items(): + meta = model_onnx.metadata_props.add() + meta.key, meta.value = k, str(v) + + onnx.save(model_onnx, f) + return f, model_onnx + + @try_export + def export_openvino(self, prefix=colorstr("OpenVINO:")): + """YOLOv8 OpenVINO export.""" + check_requirements("openvino>=2024.0.0") # requires openvino: https://pypi.org/project/openvino/ + import openvino as ov + + LOGGER.info(f"\n{prefix} starting export with openvino {ov.__version__}...") + assert TORCH_1_13, f"OpenVINO export requires torch>=1.13.0 but torch=={torch.__version__} is installed" + ov_model = ov.convert_model( + self.model.cpu(), + input=None if self.args.dynamic else [self.im.shape], + example_input=self.im, + ) + + def serialize(ov_model, file): + """Set RT info, serialize and save metadata YAML.""" + ov_model.set_rt_info("YOLOv8", ["model_info", "model_type"]) + ov_model.set_rt_info(True, ["model_info", "reverse_input_channels"]) + ov_model.set_rt_info(114, ["model_info", "pad_value"]) + ov_model.set_rt_info([255.0], ["model_info", "scale_values"]) + ov_model.set_rt_info(self.args.iou, ["model_info", "iou_threshold"]) + ov_model.set_rt_info([v.replace(" ", "_") for v in self.model.names.values()], ["model_info", "labels"]) + if self.model.task != "classify": + ov_model.set_rt_info("fit_to_window_letterbox", ["model_info", "resize_type"]) + + ov.runtime.save_model(ov_model, file, compress_to_fp16=self.args.half) + yaml_save(Path(file).parent / "metadata.yaml", self.metadata) # add metadata.yaml + + if self.args.int8: + fq = str(self.file).replace(self.file.suffix, f"_int8_openvino_model{os.sep}") + fq_ov = str(Path(fq) / self.file.with_suffix(".xml").name) + if not self.args.data: + self.args.data = DEFAULT_CFG.data or "coco128.yaml" + LOGGER.warning( + f"{prefix} WARNING ⚠️ INT8 export requires a missing 'data' arg for calibration. " + f"Using default 'data={self.args.data}'." + ) + check_requirements("nncf>=2.8.0") + import nncf + + def transform_fn(data_item): + """Quantization transform function.""" + assert ( + data_item["img"].dtype == torch.uint8 + ), "Input image must be uint8 for the quantization preprocessing" + im = data_item["img"].numpy().astype(np.float32) / 255.0 # uint8 to fp16/32 and 0 - 255 to 0.0 - 1.0 + return np.expand_dims(im, 0) if im.ndim == 3 else im + + # Generate calibration data for integer quantization + LOGGER.info(f"{prefix} collecting INT8 calibration images from 'data={self.args.data}'") + data = check_det_dataset(self.args.data) + dataset = YOLODataset(data["val"], data=data, imgsz=self.imgsz[0], augment=False) + n = len(dataset) + if n < 300: + LOGGER.warning(f"{prefix} WARNING ⚠️ >300 images recommended for INT8 calibration, found {n} images.") + quantization_dataset = nncf.Dataset(dataset, transform_fn) + + ignored_scope = None + if isinstance(self.model.model[-1], Detect): + # Includes all Detect subclasses like Segment, Pose, OBB, WorldDetect + head_module_name = ".".join(list(self.model.named_modules())[-1][0].split(".")[:2]) + + ignored_scope = nncf.IgnoredScope( # ignore operations + patterns=[ + f".*{head_module_name}/.*/Add", + f".*{head_module_name}/.*/Sub*", + f".*{head_module_name}/.*/Mul*", + f".*{head_module_name}/.*/Div*", + f".*{head_module_name}\\.dfl.*", + ], + types=["Sigmoid"], + ) + + quantized_ov_model = nncf.quantize( + ov_model, quantization_dataset, preset=nncf.QuantizationPreset.MIXED, ignored_scope=ignored_scope + ) + serialize(quantized_ov_model, fq_ov) + return fq, None + + f = str(self.file).replace(self.file.suffix, f"_openvino_model{os.sep}") + f_ov = str(Path(f) / self.file.with_suffix(".xml").name) + + serialize(ov_model, f_ov) + return f, None + + @try_export + def export_paddle(self, prefix=colorstr("PaddlePaddle:")): + """YOLOv8 Paddle export.""" + check_requirements(("paddlepaddle", "x2paddle")) + import x2paddle # noqa + from x2paddle.convert import pytorch2paddle # noqa + + LOGGER.info(f"\n{prefix} starting export with X2Paddle {x2paddle.__version__}...") + f = str(self.file).replace(self.file.suffix, f"_paddle_model{os.sep}") + + pytorch2paddle(module=self.model, save_dir=f, jit_type="trace", input_examples=[self.im]) # export + yaml_save(Path(f) / "metadata.yaml", self.metadata) # add metadata.yaml + return f, None + + @try_export + def export_ncnn(self, prefix=colorstr("NCNN:")): + """ + YOLOv8 NCNN export using PNNX https://github.com/pnnx/pnnx. + """ + check_requirements("ncnn") + import ncnn # noqa + + LOGGER.info(f"\n{prefix} starting export with NCNN {ncnn.__version__}...") + f = Path(str(self.file).replace(self.file.suffix, f"_ncnn_model{os.sep}")) + f_ts = self.file.with_suffix(".torchscript") + + name = Path("pnnx.exe" if WINDOWS else "pnnx") # PNNX filename + pnnx = name if name.is_file() else ROOT / name + if not pnnx.is_file(): + LOGGER.warning( + f"{prefix} WARNING ⚠️ PNNX not found. Attempting to download binary file from " + "https://github.com/pnnx/pnnx/.\nNote PNNX Binary file must be placed in current working directory " + f"or in {ROOT}. See PNNX repo for full installation instructions." + ) + system = "macos" if MACOS else "windows" if WINDOWS else "linux-aarch64" if ARM64 else "linux" + _, assets = get_github_assets(repo="pnnx/pnnx", retry=True) + if assets: + url = [x for x in assets if f"{system}.zip" in x][0] + else: + url = f"https://github.com/pnnx/pnnx/releases/download/20240226/pnnx-20240226-{system}.zip" + LOGGER.warning(f"{prefix} WARNING ⚠️ PNNX GitHub assets not found, using default {url}") + asset = attempt_download_asset(url, repo="pnnx/pnnx", release="latest") + if check_is_path_safe(Path.cwd(), asset): # avoid path traversal security vulnerability + unzip_dir = Path(asset).with_suffix("") + (unzip_dir / name).rename(pnnx) # move binary to ROOT + shutil.rmtree(unzip_dir) # delete unzip dir + Path(asset).unlink() # delete zip + pnnx.chmod(0o777) # set read, write, and execute permissions for everyone + + ncnn_args = [ + f'ncnnparam={f / "model.ncnn.param"}', + f'ncnnbin={f / "model.ncnn.bin"}', + f'ncnnpy={f / "model_ncnn.py"}', + ] + + pnnx_args = [ + f'pnnxparam={f / "model.pnnx.param"}', + f'pnnxbin={f / "model.pnnx.bin"}', + f'pnnxpy={f / "model_pnnx.py"}', + f'pnnxonnx={f / "model.pnnx.onnx"}', + ] + + cmd = [ + str(pnnx), + str(f_ts), + *ncnn_args, + *pnnx_args, + f"fp16={int(self.args.half)}", + f"device={self.device.type}", + f'inputshape="{[self.args.batch, 3, *self.imgsz]}"', + ] + f.mkdir(exist_ok=True) # make ncnn_model directory + LOGGER.info(f"{prefix} running '{' '.join(cmd)}'") + subprocess.run(cmd, check=True) + + # Remove debug files + pnnx_files = [x.split("=")[-1] for x in pnnx_args] + for f_debug in ("debug.bin", "debug.param", "debug2.bin", "debug2.param", *pnnx_files): + Path(f_debug).unlink(missing_ok=True) + + yaml_save(f / "metadata.yaml", self.metadata) # add metadata.yaml + return str(f), None + + @try_export + def export_coreml(self, prefix=colorstr("CoreML:")): + """YOLOv8 CoreML export.""" + mlmodel = self.args.format.lower() == "mlmodel" # legacy *.mlmodel export format requested + check_requirements("coremltools>=6.0,<=6.2" if mlmodel else "coremltools>=7.0") + import coremltools as ct # noqa + + LOGGER.info(f"\n{prefix} starting export with coremltools {ct.__version__}...") + assert not WINDOWS, "CoreML export is not supported on Windows, please run on macOS or Linux." + f = self.file.with_suffix(".mlmodel" if mlmodel else ".mlpackage") + if f.is_dir(): + shutil.rmtree(f) + + bias = [0.0, 0.0, 0.0] + scale = 1 / 255 + classifier_config = None + if self.model.task == "classify": + classifier_config = ct.ClassifierConfig(list(self.model.names.values())) if self.args.nms else None + model = self.model + elif self.model.task == "detect": + model = IOSDetectModel(self.model, self.im) if self.args.nms else self.model + else: + if self.args.nms: + LOGGER.warning(f"{prefix} WARNING ⚠️ 'nms=True' is only available for Detect models like 'yolov8n.pt'.") + # TODO CoreML Segment and Pose model pipelining + model = self.model + + ts = torch.jit.trace(model.eval(), self.im, strict=False) # TorchScript model + ct_model = ct.convert( + ts, + inputs=[ct.ImageType("image", shape=self.im.shape, scale=scale, bias=bias)], + classifier_config=classifier_config, + convert_to="neuralnetwork" if mlmodel else "mlprogram", + ) + bits, mode = (8, "kmeans") if self.args.int8 else (16, "linear") if self.args.half else (32, None) + if bits < 32: + if "kmeans" in mode: + check_requirements("scikit-learn") # scikit-learn package required for k-means quantization + if mlmodel: + ct_model = ct.models.neural_network.quantization_utils.quantize_weights(ct_model, bits, mode) + elif bits == 8: # mlprogram already quantized to FP16 + import coremltools.optimize.coreml as cto + + op_config = cto.OpPalettizerConfig(mode="kmeans", nbits=bits, weight_threshold=512) + config = cto.OptimizationConfig(global_config=op_config) + ct_model = cto.palettize_weights(ct_model, config=config) + if self.args.nms and self.model.task == "detect": + if mlmodel: + # coremltools<=6.2 NMS export requires Python<3.11 + check_version(PYTHON_VERSION, "<3.11", name="Python ", hard=True) + weights_dir = None + else: + ct_model.save(str(f)) # save otherwise weights_dir does not exist + weights_dir = str(f / "Data/com.apple.CoreML/weights") + ct_model = self._pipeline_coreml(ct_model, weights_dir=weights_dir) + + m = self.metadata # metadata dict + ct_model.short_description = m.pop("description") + ct_model.author = m.pop("author") + ct_model.license = m.pop("license") + ct_model.version = m.pop("version") + ct_model.user_defined_metadata.update({k: str(v) for k, v in m.items()}) + try: + ct_model.save(str(f)) # save *.mlpackage + except Exception as e: + LOGGER.warning( + f"{prefix} WARNING ⚠️ CoreML export to *.mlpackage failed ({e}), reverting to *.mlmodel export. " + f"Known coremltools Python 3.11 and Windows bugs https://github.com/apple/coremltools/issues/1928." + ) + f = f.with_suffix(".mlmodel") + ct_model.save(str(f)) + return f, ct_model + + @try_export + def export_engine(self, prefix=colorstr("TensorRT:")): + """YOLOv8 TensorRT export https://developer.nvidia.com/tensorrt.""" + assert self.im.device.type != "cpu", "export running on CPU but must be on GPU, i.e. use 'device=0'" + f_onnx, _ = self.export_onnx() # run before TRT import https://github.com/ultralytics/ultralytics/issues/7016 + + try: + import tensorrt as trt # noqa + except ImportError: + if LINUX: + check_requirements("nvidia-tensorrt", cmds="-U --index-url https://pypi.ngc.nvidia.com") + import tensorrt as trt # noqa + + check_version(trt.__version__, "7.0.0", hard=True) # require tensorrt>=7.0.0 + + self.args.simplify = True + + LOGGER.info(f"\n{prefix} starting export with TensorRT {trt.__version__}...") + assert Path(f_onnx).exists(), f"failed to export ONNX file: {f_onnx}" + f = self.file.with_suffix(".engine") # TensorRT engine file + logger = trt.Logger(trt.Logger.INFO) + if self.args.verbose: + logger.min_severity = trt.Logger.Severity.VERBOSE + + builder = trt.Builder(logger) + config = builder.create_builder_config() + config.max_workspace_size = self.args.workspace * 1 << 30 + # config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, workspace << 30) # fix TRT 8.4 deprecation notice + + flag = 1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH) + network = builder.create_network(flag) + parser = trt.OnnxParser(network, logger) + if not parser.parse_from_file(f_onnx): + raise RuntimeError(f"failed to load ONNX file: {f_onnx}") + + inputs = [network.get_input(i) for i in range(network.num_inputs)] + outputs = [network.get_output(i) for i in range(network.num_outputs)] + for inp in inputs: + LOGGER.info(f'{prefix} input "{inp.name}" with shape{inp.shape} {inp.dtype}') + for out in outputs: + LOGGER.info(f'{prefix} output "{out.name}" with shape{out.shape} {out.dtype}') + + if self.args.dynamic: + shape = self.im.shape + if shape[0] <= 1: + LOGGER.warning(f"{prefix} WARNING ⚠️ 'dynamic=True' model requires max batch size, i.e. 'batch=16'") + profile = builder.create_optimization_profile() + for inp in inputs: + profile.set_shape(inp.name, (1, *shape[1:]), (max(1, shape[0] // 2), *shape[1:]), shape) + config.add_optimization_profile(profile) + + LOGGER.info( + f"{prefix} building FP{16 if builder.platform_has_fast_fp16 and self.args.half else 32} engine as {f}" + ) + if builder.platform_has_fast_fp16 and self.args.half: + config.set_flag(trt.BuilderFlag.FP16) + + del self.model + torch.cuda.empty_cache() + + # Write file + with builder.build_engine(network, config) as engine, open(f, "wb") as t: + # Metadata + meta = json.dumps(self.metadata) + t.write(len(meta).to_bytes(4, byteorder="little", signed=True)) + t.write(meta.encode()) + # Model + t.write(engine.serialize()) + + return f, None + + @try_export + def export_saved_model(self, prefix=colorstr("TensorFlow SavedModel:")): + """YOLOv8 TensorFlow SavedModel export.""" + cuda = torch.cuda.is_available() + try: + import tensorflow as tf # noqa + except ImportError: + suffix = "-macos" if MACOS else "-aarch64" if ARM64 else "" if cuda else "-cpu" + version = "" if ARM64 else "<=2.13.1" + check_requirements(f"tensorflow{suffix}{version}") + import tensorflow as tf # noqa + if ARM64: + check_requirements("cmake") # 'cmake' is needed to build onnxsim on aarch64 + check_requirements( + ( + "onnx>=1.12.0", + "onnx2tf>=1.15.4,<=1.17.5", + "sng4onnx>=1.0.1", + "onnxslim==0.1.31", + "onnx_graphsurgeon>=0.3.26", + "tflite_support", + "flatbuffers>=23.5.26,<100", # update old 'flatbuffers' included inside tensorflow package + "onnxruntime-gpu" if cuda else "onnxruntime", + ), + cmds="--extra-index-url https://pypi.ngc.nvidia.com", + ) # onnx_graphsurgeon only on NVIDIA + + LOGGER.info(f"\n{prefix} starting export with tensorflow {tf.__version__}...") + check_version( + tf.__version__, + "<=2.13.1", + name="tensorflow", + verbose=True, + msg="https://github.com/ultralytics/ultralytics/issues/5161", + ) + import onnx2tf + + f = Path(str(self.file).replace(self.file.suffix, "_saved_model")) + if f.is_dir(): + shutil.rmtree(f) # delete output folder + + # Pre-download calibration file to fix https://github.com/PINTO0309/onnx2tf/issues/545 + onnx2tf_file = Path("calibration_image_sample_data_20x128x128x3_float32.npy") + if not onnx2tf_file.exists(): + attempt_download_asset(f"{onnx2tf_file}.zip", unzip=True, delete=True) + + # Export to ONNX + self.args.simplify = True + f_onnx, _ = self.export_onnx() + + # Export to TF + tmp_file = f / "tmp_tflite_int8_calibration_images.npy" # int8 calibration images file + np_data = None + if self.args.int8: + verbosity = "info" + if self.args.data: + # Generate calibration data for integer quantization + LOGGER.info(f"{prefix} collecting INT8 calibration images from 'data={self.args.data}'") + data = check_det_dataset(self.args.data) + dataset = YOLODataset(data["val"], data=data, imgsz=self.imgsz[0], augment=False) + images = [] + for i, batch in enumerate(dataset): + if i >= 100: # maximum number of calibration images + break + im = batch["img"].permute(1, 2, 0)[None] # list to nparray, CHW to BHWC + images.append(im) + f.mkdir() + images = torch.cat(images, 0).float() + # mean = images.view(-1, 3).mean(0) # imagenet mean [123.675, 116.28, 103.53] + # std = images.view(-1, 3).std(0) # imagenet std [58.395, 57.12, 57.375] + np.save(str(tmp_file), images.numpy()) # BHWC + np_data = [["images", tmp_file, [[[[0, 0, 0]]]], [[[[255, 255, 255]]]]]] + else: + verbosity = "error" + + LOGGER.info(f"{prefix} starting TFLite export with onnx2tf {onnx2tf.__version__}...") + onnx2tf.convert( + input_onnx_file_path=f_onnx, + output_folder_path=str(f), + not_use_onnxsim=True, + verbosity=verbosity, + output_integer_quantized_tflite=self.args.int8, + quant_type="per-tensor", # "per-tensor" (faster) or "per-channel" (slower but more accurate) + custom_input_op_name_np_data_path=np_data, + ) + yaml_save(f / "metadata.yaml", self.metadata) # add metadata.yaml + + # Remove/rename TFLite models + if self.args.int8: + tmp_file.unlink(missing_ok=True) + for file in f.rglob("*_dynamic_range_quant.tflite"): + file.rename(file.with_name(file.stem.replace("_dynamic_range_quant", "_int8") + file.suffix)) + for file in f.rglob("*_integer_quant_with_int16_act.tflite"): + file.unlink() # delete extra fp16 activation TFLite files + + # Add TFLite metadata + for file in f.rglob("*.tflite"): + f.unlink() if "quant_with_int16_act.tflite" in str(f) else self._add_tflite_metadata(file) + + return str(f), tf.saved_model.load(f, tags=None, options=None) # load saved_model as Keras model + + @try_export + def export_pb(self, keras_model, prefix=colorstr("TensorFlow GraphDef:")): + """YOLOv8 TensorFlow GraphDef *.pb export https://github.com/leimao/Frozen_Graph_TensorFlow.""" + import tensorflow as tf # noqa + from tensorflow.python.framework.convert_to_constants import convert_variables_to_constants_v2 # noqa + + LOGGER.info(f"\n{prefix} starting export with tensorflow {tf.__version__}...") + f = self.file.with_suffix(".pb") + + m = tf.function(lambda x: keras_model(x)) # full model + m = m.get_concrete_function(tf.TensorSpec(keras_model.inputs[0].shape, keras_model.inputs[0].dtype)) + frozen_func = convert_variables_to_constants_v2(m) + frozen_func.graph.as_graph_def() + tf.io.write_graph(graph_or_graph_def=frozen_func.graph, logdir=str(f.parent), name=f.name, as_text=False) + return f, None + + @try_export + def export_tflite(self, keras_model, nms, agnostic_nms, prefix=colorstr("TensorFlow Lite:")): + """YOLOv8 TensorFlow Lite export.""" + import tensorflow as tf # noqa + + LOGGER.info(f"\n{prefix} starting export with tensorflow {tf.__version__}...") + saved_model = Path(str(self.file).replace(self.file.suffix, "_saved_model")) + if self.args.int8: + f = saved_model / f"{self.file.stem}_int8.tflite" # fp32 in/out + elif self.args.half: + f = saved_model / f"{self.file.stem}_float16.tflite" # fp32 in/out + else: + f = saved_model / f"{self.file.stem}_float32.tflite" + return str(f), None + + @try_export + def export_edgetpu(self, tflite_model="", prefix=colorstr("Edge TPU:")): + """YOLOv8 Edge TPU export https://coral.ai/docs/edgetpu/models-intro/.""" + LOGGER.warning(f"{prefix} WARNING ⚠️ Edge TPU known bug https://github.com/ultralytics/ultralytics/issues/1185") + + cmd = "edgetpu_compiler --version" + help_url = "https://coral.ai/docs/edgetpu/compiler/" + assert LINUX, f"export only supported on Linux. See {help_url}" + if subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=True).returncode != 0: + LOGGER.info(f"\n{prefix} export requires Edge TPU compiler. Attempting install from {help_url}") + sudo = subprocess.run("sudo --version >/dev/null", shell=True).returncode == 0 # sudo installed on system + for c in ( + "curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -", + 'echo "deb https://packages.cloud.google.com/apt coral-edgetpu-stable main" | ' + "sudo tee /etc/apt/sources.list.d/coral-edgetpu.list", + "sudo apt-get update", + "sudo apt-get install edgetpu-compiler", + ): + subprocess.run(c if sudo else c.replace("sudo ", ""), shell=True, check=True) + ver = subprocess.run(cmd, shell=True, capture_output=True, check=True).stdout.decode().split()[-1] + + LOGGER.info(f"\n{prefix} starting export with Edge TPU compiler {ver}...") + f = str(tflite_model).replace(".tflite", "_edgetpu.tflite") # Edge TPU model + + cmd = f'edgetpu_compiler -s -d -k 10 --out_dir "{Path(f).parent}" "{tflite_model}"' + LOGGER.info(f"{prefix} running '{cmd}'") + subprocess.run(cmd, shell=True) + self._add_tflite_metadata(f) + return f, None + + @try_export + def export_tfjs(self, prefix=colorstr("TensorFlow.js:")): + """YOLOv8 TensorFlow.js export.""" + check_requirements("tensorflowjs") + if ARM64: + # Fix error: `np.object` was a deprecated alias for the builtin `object` when exporting to TF.js on ARM64 + check_requirements("numpy==1.23.5") + import tensorflow as tf + import tensorflowjs as tfjs # noqa + + LOGGER.info(f"\n{prefix} starting export with tensorflowjs {tfjs.__version__}...") + f = str(self.file).replace(self.file.suffix, "_web_model") # js dir + f_pb = str(self.file.with_suffix(".pb")) # *.pb path + + gd = tf.Graph().as_graph_def() # TF GraphDef + with open(f_pb, "rb") as file: + gd.ParseFromString(file.read()) + outputs = ",".join(gd_outputs(gd)) + LOGGER.info(f"\n{prefix} output node names: {outputs}") + + quantization = "--quantize_float16" if self.args.half else "--quantize_uint8" if self.args.int8 else "" + with spaces_in_path(f_pb) as fpb_, spaces_in_path(f) as f_: # exporter can not handle spaces in path + cmd = ( + "tensorflowjs_converter " + f'--input_format=tf_frozen_model {quantization} --output_node_names={outputs} "{fpb_}" "{f_}"' + ) + LOGGER.info(f"{prefix} running '{cmd}'") + subprocess.run(cmd, shell=True) + + if " " in f: + LOGGER.warning(f"{prefix} WARNING ⚠️ your model may not work correctly with spaces in path '{f}'.") + + # f_json = Path(f) / 'model.json' # *.json path + # with open(f_json, 'w') as j: # sort JSON Identity_* in ascending order + # subst = re.sub( + # r'{"outputs": {"Identity.?.?": {"name": "Identity.?.?"}, ' + # r'"Identity.?.?": {"name": "Identity.?.?"}, ' + # r'"Identity.?.?": {"name": "Identity.?.?"}, ' + # r'"Identity.?.?": {"name": "Identity.?.?"}}}', + # r'{"outputs": {"Identity": {"name": "Identity"}, ' + # r'"Identity_1": {"name": "Identity_1"}, ' + # r'"Identity_2": {"name": "Identity_2"}, ' + # r'"Identity_3": {"name": "Identity_3"}}}', + # f_json.read_text(), + # ) + # j.write(subst) + yaml_save(Path(f) / "metadata.yaml", self.metadata) # add metadata.yaml + return f, None + + def _add_tflite_metadata(self, file): + """Add metadata to *.tflite models per https://www.tensorflow.org/lite/models/convert/metadata.""" + from tflite_support import flatbuffers # noqa + from tflite_support import metadata as _metadata # noqa + from tflite_support import metadata_schema_py_generated as _metadata_fb # noqa + + # Create model info + model_meta = _metadata_fb.ModelMetadataT() + model_meta.name = self.metadata["description"] + model_meta.version = self.metadata["version"] + model_meta.author = self.metadata["author"] + model_meta.license = self.metadata["license"] + + # Label file + tmp_file = Path(file).parent / "temp_meta.txt" + with open(tmp_file, "w") as f: + f.write(str(self.metadata)) + + label_file = _metadata_fb.AssociatedFileT() + label_file.name = tmp_file.name + label_file.type = _metadata_fb.AssociatedFileType.TENSOR_AXIS_LABELS + + # Create input info + input_meta = _metadata_fb.TensorMetadataT() + input_meta.name = "image" + input_meta.description = "Input image to be detected." + input_meta.content = _metadata_fb.ContentT() + input_meta.content.contentProperties = _metadata_fb.ImagePropertiesT() + input_meta.content.contentProperties.colorSpace = _metadata_fb.ColorSpaceType.RGB + input_meta.content.contentPropertiesType = _metadata_fb.ContentProperties.ImageProperties + + # Create output info + output1 = _metadata_fb.TensorMetadataT() + output1.name = "output" + output1.description = "Coordinates of detected objects, class labels, and confidence score" + output1.associatedFiles = [label_file] + if self.model.task == "segment": + output2 = _metadata_fb.TensorMetadataT() + output2.name = "output" + output2.description = "Mask protos" + output2.associatedFiles = [label_file] + + # Create subgraph info + subgraph = _metadata_fb.SubGraphMetadataT() + subgraph.inputTensorMetadata = [input_meta] + subgraph.outputTensorMetadata = [output1, output2] if self.model.task == "segment" else [output1] + model_meta.subgraphMetadata = [subgraph] + + b = flatbuffers.Builder(0) + b.Finish(model_meta.Pack(b), _metadata.MetadataPopulator.METADATA_FILE_IDENTIFIER) + metadata_buf = b.Output() + + populator = _metadata.MetadataPopulator.with_model_file(str(file)) + populator.load_metadata_buffer(metadata_buf) + populator.load_associated_files([str(tmp_file)]) + populator.populate() + tmp_file.unlink() + + def _pipeline_coreml(self, model, weights_dir=None, prefix=colorstr("CoreML Pipeline:")): + """YOLOv8 CoreML pipeline.""" + import coremltools as ct # noqa + + LOGGER.info(f"{prefix} starting pipeline with coremltools {ct.__version__}...") + _, _, h, w = list(self.im.shape) # BCHW + + # Output shapes + spec = model.get_spec() + out0, out1 = iter(spec.description.output) + if MACOS: + from PIL import Image + + img = Image.new("RGB", (w, h)) # w=192, h=320 + out = model.predict({"image": img}) + out0_shape = out[out0.name].shape # (3780, 80) + out1_shape = out[out1.name].shape # (3780, 4) + else: # linux and windows can not run model.predict(), get sizes from PyTorch model output y + out0_shape = self.output_shape[2], self.output_shape[1] - 4 # (3780, 80) + out1_shape = self.output_shape[2], 4 # (3780, 4) + + # Checks + names = self.metadata["names"] + nx, ny = spec.description.input[0].type.imageType.width, spec.description.input[0].type.imageType.height + _, nc = out0_shape # number of anchors, number of classes + # _, nc = out0.type.multiArrayType.shape + assert len(names) == nc, f"{len(names)} names found for nc={nc}" # check + + # Define output shapes (missing) + out0.type.multiArrayType.shape[:] = out0_shape # (3780, 80) + out1.type.multiArrayType.shape[:] = out1_shape # (3780, 4) + # spec.neuralNetwork.preprocessing[0].featureName = '0' + + # Flexible input shapes + # from coremltools.models.neural_network import flexible_shape_utils + # s = [] # shapes + # s.append(flexible_shape_utils.NeuralNetworkImageSize(320, 192)) + # s.append(flexible_shape_utils.NeuralNetworkImageSize(640, 384)) # (height, width) + # flexible_shape_utils.add_enumerated_image_sizes(spec, feature_name='image', sizes=s) + # r = flexible_shape_utils.NeuralNetworkImageSizeRange() # shape ranges + # r.add_height_range((192, 640)) + # r.add_width_range((192, 640)) + # flexible_shape_utils.update_image_size_range(spec, feature_name='image', size_range=r) + + # Print + # print(spec.description) + + # Model from spec + model = ct.models.MLModel(spec, weights_dir=weights_dir) + + # 3. Create NMS protobuf + nms_spec = ct.proto.Model_pb2.Model() + nms_spec.specificationVersion = 5 + for i in range(2): + decoder_output = model._spec.description.output[i].SerializeToString() + nms_spec.description.input.add() + nms_spec.description.input[i].ParseFromString(decoder_output) + nms_spec.description.output.add() + nms_spec.description.output[i].ParseFromString(decoder_output) + + nms_spec.description.output[0].name = "confidence" + nms_spec.description.output[1].name = "coordinates" + + output_sizes = [nc, 4] + for i in range(2): + ma_type = nms_spec.description.output[i].type.multiArrayType + ma_type.shapeRange.sizeRanges.add() + ma_type.shapeRange.sizeRanges[0].lowerBound = 0 + ma_type.shapeRange.sizeRanges[0].upperBound = -1 + ma_type.shapeRange.sizeRanges.add() + ma_type.shapeRange.sizeRanges[1].lowerBound = output_sizes[i] + ma_type.shapeRange.sizeRanges[1].upperBound = output_sizes[i] + del ma_type.shape[:] + + nms = nms_spec.nonMaximumSuppression + nms.confidenceInputFeatureName = out0.name # 1x507x80 + nms.coordinatesInputFeatureName = out1.name # 1x507x4 + nms.confidenceOutputFeatureName = "confidence" + nms.coordinatesOutputFeatureName = "coordinates" + nms.iouThresholdInputFeatureName = "iouThreshold" + nms.confidenceThresholdInputFeatureName = "confidenceThreshold" + nms.iouThreshold = 0.45 + nms.confidenceThreshold = 0.25 + nms.pickTop.perClass = True + nms.stringClassLabels.vector.extend(names.values()) + nms_model = ct.models.MLModel(nms_spec) + + # 4. Pipeline models together + pipeline = ct.models.pipeline.Pipeline( + input_features=[ + ("image", ct.models.datatypes.Array(3, ny, nx)), + ("iouThreshold", ct.models.datatypes.Double()), + ("confidenceThreshold", ct.models.datatypes.Double()), + ], + output_features=["confidence", "coordinates"], + ) + pipeline.add_model(model) + pipeline.add_model(nms_model) + + # Correct datatypes + pipeline.spec.description.input[0].ParseFromString(model._spec.description.input[0].SerializeToString()) + pipeline.spec.description.output[0].ParseFromString(nms_model._spec.description.output[0].SerializeToString()) + pipeline.spec.description.output[1].ParseFromString(nms_model._spec.description.output[1].SerializeToString()) + + # Update metadata + pipeline.spec.specificationVersion = 5 + pipeline.spec.description.metadata.userDefined.update( + {"IoU threshold": str(nms.iouThreshold), "Confidence threshold": str(nms.confidenceThreshold)} + ) + + # Save the model + model = ct.models.MLModel(pipeline.spec, weights_dir=weights_dir) + model.input_description["image"] = "Input image" + model.input_description["iouThreshold"] = f"(optional) IoU threshold override (default: {nms.iouThreshold})" + model.input_description["confidenceThreshold"] = ( + f"(optional) Confidence threshold override (default: {nms.confidenceThreshold})" + ) + model.output_description["confidence"] = 'Boxes × Class confidence (see user-defined metadata "classes")' + model.output_description["coordinates"] = "Boxes × [x, y, width, height] (relative to image size)" + LOGGER.info(f"{prefix} pipeline success") + return model + + def add_callback(self, event: str, callback): + """Appends the given callback.""" + self.callbacks[event].append(callback) + + def run_callbacks(self, event: str): + """Execute all callbacks for a given event.""" + for callback in self.callbacks.get(event, []): + callback(self) + + +class IOSDetectModel(torch.nn.Module): + """Wrap an Ultralytics YOLO model for Apple iOS CoreML export.""" + + def __init__(self, model, im): + """Initialize the IOSDetectModel class with a YOLO model and example image.""" + super().__init__() + _, _, h, w = im.shape # batch, channel, height, width + self.model = model + self.nc = len(model.names) # number of classes + if w == h: + self.normalize = 1.0 / w # scalar + else: + self.normalize = torch.tensor([1.0 / w, 1.0 / h, 1.0 / w, 1.0 / h]) # broadcast (slower, smaller) + + def forward(self, x): + """Normalize predictions of object detection model with input size-dependent factors.""" + xywh, cls = self.model(x)[0].transpose(0, 1).split((4, self.nc), 1) + return cls, xywh * self.normalize # confidence (3780, 80), coordinates (3780, 4) diff --git a/ultralytics/engine/model.py b/ultralytics/engine/model.py new file mode 100644 index 0000000..ef5c93c --- /dev/null +++ b/ultralytics/engine/model.py @@ -0,0 +1,823 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import inspect +import sys +from pathlib import Path +from typing import Union + +import numpy as np +import torch + +from ultralytics.cfg import TASK2DATA, get_cfg, get_save_dir +from ultralytics.hub.utils import HUB_WEB_ROOT +from ultralytics.nn.tasks import attempt_load_one_weight, guess_model_task, nn, yaml_model_load +from ultralytics.utils import ASSETS, DEFAULT_CFG_DICT, LOGGER, RANK, SETTINGS, callbacks, checks, emojis, yaml_load + + +class Model(nn.Module): + """ + A base class for implementing YOLO models, unifying APIs across different model types. + + This class provides a common interface for various operations related to YOLO models, such as training, + validation, prediction, exporting, and benchmarking. It handles different types of models, including those + loaded from local files, Ultralytics HUB, or Triton Server. The class is designed to be flexible and + extendable for different tasks and model configurations. + + Args: + model (Union[str, Path], optional): Path or name of the model to load or create. This can be a local file + path, a model name from Ultralytics HUB, or a Triton Server model. Defaults to 'yolov8n.pt'. + task (Any, optional): The task type associated with the YOLO model. This can be used to specify the model's + application domain, such as object detection, segmentation, etc. Defaults to None. + verbose (bool, optional): If True, enables verbose output during the model's operations. Defaults to False. + + Attributes: + callbacks (dict): A dictionary of callback functions for various events during model operations. + predictor (BasePredictor): The predictor object used for making predictions. + model (nn.Module): The underlying PyTorch model. + trainer (BaseTrainer): The trainer object used for training the model. + ckpt (dict): The checkpoint data if the model is loaded from a *.pt file. + cfg (str): The configuration of the model if loaded from a *.yaml file. + ckpt_path (str): The path to the checkpoint file. + overrides (dict): A dictionary of overrides for model configuration. + metrics (dict): The latest training/validation metrics. + session (HUBTrainingSession): The Ultralytics HUB session, if applicable. + task (str): The type of task the model is intended for. + model_name (str): The name of the model. + + Methods: + __call__: Alias for the predict method, enabling the model instance to be callable. + _new: Initializes a new model based on a configuration file. + _load: Loads a model from a checkpoint file. + _check_is_pytorch_model: Ensures that the model is a PyTorch model. + reset_weights: Resets the model's weights to their initial state. + load: Loads model weights from a specified file. + save: Saves the current state of the model to a file. + info: Logs or returns information about the model. + fuse: Fuses Conv2d and BatchNorm2d layers for optimized inference. + predict: Performs object detection predictions. + track: Performs object tracking. + val: Validates the model on a dataset. + benchmark: Benchmarks the model on various export formats. + export: Exports the model to different formats. + train: Trains the model on a dataset. + tune: Performs hyperparameter tuning. + _apply: Applies a function to the model's tensors. + add_callback: Adds a callback function for an event. + clear_callback: Clears all callbacks for an event. + reset_callbacks: Resets all callbacks to their default functions. + _get_hub_session: Retrieves or creates an Ultralytics HUB session. + is_triton_model: Checks if a model is a Triton Server model. + is_hub_model: Checks if a model is an Ultralytics HUB model. + _reset_ckpt_args: Resets checkpoint arguments when loading a PyTorch model. + _smart_load: Loads the appropriate module based on the model task. + task_map: Provides a mapping from model tasks to corresponding classes. + + Raises: + FileNotFoundError: If the specified model file does not exist or is inaccessible. + ValueError: If the model file or configuration is invalid or unsupported. + ImportError: If required dependencies for specific model types (like HUB SDK) are not installed. + TypeError: If the model is not a PyTorch model when required. + AttributeError: If required attributes or methods are not implemented or available. + NotImplementedError: If a specific model task or mode is not supported. + """ + + def __init__( + self, + model: Union[str, Path] = "yolov8n.pt", + task: str = None, + verbose: bool = False, + ) -> None: + """ + Initializes a new instance of the YOLO model class. + + This constructor sets up the model based on the provided model path or name. It handles various types of model + sources, including local files, Ultralytics HUB models, and Triton Server models. The method initializes several + important attributes of the model and prepares it for operations like training, prediction, or export. + + Args: + model (Union[str, Path], optional): The path or model file to load or create. This can be a local + file path, a model name from Ultralytics HUB, or a Triton Server model. Defaults to 'yolov8n.pt'. + task (Any, optional): The task type associated with the YOLO model, specifying its application domain. + Defaults to None. + verbose (bool, optional): If True, enables verbose output during the model's initialization and subsequent + operations. Defaults to False. + + Raises: + FileNotFoundError: If the specified model file does not exist or is inaccessible. + ValueError: If the model file or configuration is invalid or unsupported. + ImportError: If required dependencies for specific model types (like HUB SDK) are not installed. + """ + super().__init__() + self.callbacks = callbacks.get_default_callbacks() + self.predictor = None # reuse predictor + self.model = None # model object + self.trainer = None # trainer object + self.ckpt = None # if loaded from *.pt + self.cfg = None # if loaded from *.yaml + self.ckpt_path = None + self.overrides = {} # overrides for trainer object + self.metrics = None # validation/training metrics + self.session = None # HUB session + self.task = task # task type + model = str(model).strip() + + # Check if Ultralytics HUB model from https://hub.ultralytics.com + if self.is_hub_model(model): + # Fetch model from HUB + checks.check_requirements("hub-sdk>=0.0.6") + self.session = self._get_hub_session(model) + model = self.session.model_file + + # Check if Triton Server model + elif self.is_triton_model(model): + self.model_name = self.model = model + self.task = task + return + + # Load or create new YOLO model + if Path(model).suffix in (".yaml", ".yml"): + self._new(model, task=task, verbose=verbose) + else: + self._load(model, task=task) + + def __call__( + self, + source: Union[str, Path, int, list, tuple, np.ndarray, torch.Tensor] = None, + stream: bool = False, + **kwargs, + ) -> list: + """ + An alias for the predict method, enabling the model instance to be callable. + + This method simplifies the process of making predictions by allowing the model instance to be called directly + with the required arguments for prediction. + + Args: + source (str | Path | int | PIL.Image | np.ndarray, optional): The source of the image for making + predictions. Accepts various types, including file paths, URLs, PIL images, and numpy arrays. + Defaults to None. + stream (bool, optional): If True, treats the input source as a continuous stream for predictions. + Defaults to False. + **kwargs (any): Additional keyword arguments for configuring the prediction process. + + Returns: + (List[ultralytics.engine.results.Results]): A list of prediction results, encapsulated in the Results class. + """ + return self.predict(source, stream, **kwargs) + + @staticmethod + def _get_hub_session(model: str): + """Creates a session for Hub Training.""" + from ultralytics.hub.session import HUBTrainingSession + + session = HUBTrainingSession(model) + return session if session.client.authenticated else None + + @staticmethod + def is_triton_model(model: str) -> bool: + """Is model a Triton Server URL string, i.e. :////""" + from urllib.parse import urlsplit + + url = urlsplit(model) + return url.netloc and url.path and url.scheme in {"http", "grpc"} + + @staticmethod + def is_hub_model(model: str) -> bool: + """Check if the provided model is a HUB model.""" + return any( + ( + model.startswith(f"{HUB_WEB_ROOT}/models/"), # i.e. https://hub.ultralytics.com/models/MODEL_ID + [len(x) for x in model.split("_")] == [42, 20], # APIKEY_MODEL + len(model) == 20 and not Path(model).exists() and all(x not in model for x in "./\\"), # MODEL + ) + ) + + def _new(self, cfg: str, task=None, model=None, verbose=False) -> None: + """ + Initializes a new model and infers the task type from the model definitions. + + Args: + cfg (str): model configuration file + task (str | None): model task + model (BaseModel): Customized model. + verbose (bool): display model info on load + """ + cfg_dict = yaml_model_load(cfg) + self.cfg = cfg + self.task = task or guess_model_task(cfg_dict) + self.model = (model or self._smart_load("model"))(cfg_dict, verbose=verbose and RANK == -1) # build model + self.overrides["model"] = self.cfg + self.overrides["task"] = self.task + + # Below added to allow export from YAMLs + self.model.args = {**DEFAULT_CFG_DICT, **self.overrides} # combine default and model args (prefer model args) + self.model.task = self.task + self.model_name = cfg + + def _load(self, weights: str, task=None) -> None: + """ + Initializes a new model and infers the task type from the model head. + + Args: + weights (str): model checkpoint to be loaded + task (str | None): model task + """ + if weights.lower().startswith(("https://", "http://", "rtsp://", "rtmp://", "tcp://")): + weights = checks.check_file(weights) # automatically download and return local filename + weights = checks.check_model_file_from_stem(weights) # add suffix, i.e. yolov8n -> yolov8n.pt + + if Path(weights).suffix == ".pt": + self.model, self.ckpt = attempt_load_one_weight(weights) + self.task = self.model.args["task"] + self.overrides = self.model.args = self._reset_ckpt_args(self.model.args) + self.ckpt_path = self.model.pt_path + else: + weights = checks.check_file(weights) # runs in all cases, not redundant with above call + self.model, self.ckpt = weights, None + self.task = task or guess_model_task(weights) + self.ckpt_path = weights + self.overrides["model"] = weights + self.overrides["task"] = self.task + self.model_name = weights + + def _check_is_pytorch_model(self) -> None: + """Raises TypeError is model is not a PyTorch model.""" + pt_str = isinstance(self.model, (str, Path)) and Path(self.model).suffix == ".pt" + pt_module = isinstance(self.model, nn.Module) + if not (pt_module or pt_str): + raise TypeError( + f"model='{self.model}' should be a *.pt PyTorch model to run this method, but is a different format. " + f"PyTorch models can train, val, predict and export, i.e. 'model.train(data=...)', but exported " + f"formats like ONNX, TensorRT etc. only support 'predict' and 'val' modes, " + f"i.e. 'yolo predict model=yolov8n.onnx'.\nTo run CUDA or MPS inference please pass the device " + f"argument directly in your inference command, i.e. 'model.predict(source=..., device=0)'" + ) + + def reset_weights(self) -> "Model": + """ + Resets the model parameters to randomly initialized values, effectively discarding all training information. + + This method iterates through all modules in the model and resets their parameters if they have a + 'reset_parameters' method. It also ensures that all parameters have 'requires_grad' set to True, enabling them + to be updated during training. + + Returns: + self (ultralytics.engine.model.Model): The instance of the class with reset weights. + + Raises: + AssertionError: If the model is not a PyTorch model. + """ + self._check_is_pytorch_model() + for m in self.model.modules(): + if hasattr(m, "reset_parameters"): + m.reset_parameters() + for p in self.model.parameters(): + p.requires_grad = True + return self + + def load(self, weights: Union[str, Path] = "yolov8n.pt") -> "Model": + """ + Loads parameters from the specified weights file into the model. + + This method supports loading weights from a file or directly from a weights object. It matches parameters by + name and shape and transfers them to the model. + + Args: + weights (str | Path): Path to the weights file or a weights object. Defaults to 'yolov8n.pt'. + + Returns: + self (ultralytics.engine.model.Model): The instance of the class with loaded weights. + + Raises: + AssertionError: If the model is not a PyTorch model. + """ + self._check_is_pytorch_model() + if isinstance(weights, (str, Path)): + weights, self.ckpt = attempt_load_one_weight(weights) + self.model.load(weights) + return self + + def save(self, filename: Union[str, Path] = "saved_model.pt", use_dill=True) -> None: + """ + Saves the current model state to a file. + + This method exports the model's checkpoint (ckpt) to the specified filename. + + Args: + filename (str | Path): The name of the file to save the model to. Defaults to 'saved_model.pt'. + use_dill (bool): Whether to try using dill for serialization if available. Defaults to True. + + Raises: + AssertionError: If the model is not a PyTorch model. + """ + self._check_is_pytorch_model() + from ultralytics import __version__ + from datetime import datetime + + updates = { + "date": datetime.now().isoformat(), + "version": __version__, + "license": "AGPL-3.0 License (https://ultralytics.com/license)", + "docs": "https://docs.ultralytics.com", + } + torch.save({**self.ckpt, **updates}, filename, use_dill=use_dill) + + def info(self, detailed: bool = False, verbose: bool = True): + """ + Logs or returns model information. + + This method provides an overview or detailed information about the model, depending on the arguments passed. + It can control the verbosity of the output. + + Args: + detailed (bool): If True, shows detailed information about the model. Defaults to False. + verbose (bool): If True, prints the information. If False, returns the information. Defaults to True. + + Returns: + (list): Various types of information about the model, depending on the 'detailed' and 'verbose' parameters. + + Raises: + AssertionError: If the model is not a PyTorch model. + """ + self._check_is_pytorch_model() + return self.model.info(detailed=detailed, verbose=verbose) + + def fuse(self): + """ + Fuses Conv2d and BatchNorm2d layers in the model. + + This method optimizes the model by fusing Conv2d and BatchNorm2d layers, which can improve inference speed. + + Raises: + AssertionError: If the model is not a PyTorch model. + """ + self._check_is_pytorch_model() + self.model.fuse() + + def embed( + self, + source: Union[str, Path, int, list, tuple, np.ndarray, torch.Tensor] = None, + stream: bool = False, + **kwargs, + ) -> list: + """ + Generates image embeddings based on the provided source. + + This method is a wrapper around the 'predict()' method, focusing on generating embeddings from an image source. + It allows customization of the embedding process through various keyword arguments. + + Args: + source (str | int | PIL.Image | np.ndarray): The source of the image for generating embeddings. + The source can be a file path, URL, PIL image, numpy array, etc. Defaults to None. + stream (bool): If True, predictions are streamed. Defaults to False. + **kwargs (any): Additional keyword arguments for configuring the embedding process. + + Returns: + (List[torch.Tensor]): A list containing the image embeddings. + + Raises: + AssertionError: If the model is not a PyTorch model. + """ + if not kwargs.get("embed"): + kwargs["embed"] = [len(self.model.model) - 2] # embed second-to-last layer if no indices passed + return self.predict(source, stream, **kwargs) + + def predict( + self, + source: Union[str, Path, int, list, tuple, np.ndarray, torch.Tensor] = None, + stream: bool = False, + predictor=None, + **kwargs, + ) -> list: + """ + Performs predictions on the given image source using the YOLO model. + + This method facilitates the prediction process, allowing various configurations through keyword arguments. + It supports predictions with custom predictors or the default predictor method. The method handles different + types of image sources and can operate in a streaming mode. It also provides support for SAM-type models + through 'prompts'. + + The method sets up a new predictor if not already present and updates its arguments with each call. + It also issues a warning and uses default assets if the 'source' is not provided. The method determines if it + is being called from the command line interface and adjusts its behavior accordingly, including setting defaults + for confidence threshold and saving behavior. + + Args: + source (str | int | PIL.Image | np.ndarray, optional): The source of the image for making predictions. + Accepts various types, including file paths, URLs, PIL images, and numpy arrays. Defaults to ASSETS. + stream (bool, optional): Treats the input source as a continuous stream for predictions. Defaults to False. + predictor (BasePredictor, optional): An instance of a custom predictor class for making predictions. + If None, the method uses a default predictor. Defaults to None. + **kwargs (any): Additional keyword arguments for configuring the prediction process. These arguments allow + for further customization of the prediction behavior. + + Returns: + (List[ultralytics.engine.results.Results]): A list of prediction results, encapsulated in the Results class. + + Raises: + AttributeError: If the predictor is not properly set up. + """ + if source is None: + source = ASSETS + LOGGER.warning(f"WARNING ⚠️ 'source' is missing. Using 'source={source}'.") + + is_cli = (sys.argv[0].endswith("yolo") or sys.argv[0].endswith("ultralytics")) and any( + x in sys.argv for x in ("predict", "track", "mode=predict", "mode=track") + ) + + custom = {"conf": 0.25, "batch": 1, "save": is_cli, "mode": "predict"} # method defaults + args = {**self.overrides, **custom, **kwargs} # highest priority args on the right + prompts = args.pop("prompts", None) # for SAM-type models + + if not self.predictor: + self.predictor = predictor or self._smart_load("predictor")(overrides=args, _callbacks=self.callbacks) + self.predictor.setup_model(model=self.model, verbose=is_cli) + else: # only update args if predictor is already setup + self.predictor.args = get_cfg(self.predictor.args, args) + if "project" in args or "name" in args: + self.predictor.save_dir = get_save_dir(self.predictor.args) + if prompts and hasattr(self.predictor, "set_prompts"): # for SAM-type models + self.predictor.set_prompts(prompts) + return self.predictor.predict_cli(source=source) if is_cli else self.predictor(source=source, stream=stream) + + def track( + self, + source: Union[str, Path, int, list, tuple, np.ndarray, torch.Tensor] = None, + stream: bool = False, + persist: bool = False, + **kwargs, + ) -> list: + """ + Conducts object tracking on the specified input source using the registered trackers. + + This method performs object tracking using the model's predictors and optionally registered trackers. It is + capable of handling different types of input sources such as file paths or video streams. The method supports + customization of the tracking process through various keyword arguments. It registers trackers if they are not + already present and optionally persists them based on the 'persist' flag. + + The method sets a default confidence threshold specifically for ByteTrack-based tracking, which requires low + confidence predictions as input. The tracking mode is explicitly set in the keyword arguments. + + Args: + source (str, optional): The input source for object tracking. It can be a file path, URL, or video stream. + stream (bool, optional): Treats the input source as a continuous video stream. Defaults to False. + persist (bool, optional): Persists the trackers between different calls to this method. Defaults to False. + **kwargs (any): Additional keyword arguments for configuring the tracking process. These arguments allow + for further customization of the tracking behavior. + + Returns: + (List[ultralytics.engine.results.Results]): A list of tracking results, encapsulated in the Results class. + + Raises: + AttributeError: If the predictor does not have registered trackers. + """ + if not hasattr(self.predictor, "trackers"): + from ultralytics.trackers import register_tracker + + register_tracker(self, persist) + kwargs["conf"] = kwargs.get("conf") or 0.1 # ByteTrack-based method needs low confidence predictions as input + kwargs["batch"] = kwargs.get("batch") or 1 # batch-size 1 for tracking in videos + kwargs["mode"] = "track" + return self.predict(source=source, stream=stream, **kwargs) + + def val( + self, + validator=None, + **kwargs, + ): + """ + Validates the model using a specified dataset and validation configuration. + + This method facilitates the model validation process, allowing for a range of customization through various + settings and configurations. It supports validation with a custom validator or the default validation approach. + The method combines default configurations, method-specific defaults, and user-provided arguments to configure + the validation process. After validation, it updates the model's metrics with the results obtained from the + validator. + + The method supports various arguments that allow customization of the validation process. For a comprehensive + list of all configurable options, users should refer to the 'configuration' section in the documentation. + + Args: + validator (BaseValidator, optional): An instance of a custom validator class for validating the model. If + None, the method uses a default validator. Defaults to None. + **kwargs (any): Arbitrary keyword arguments representing the validation configuration. These arguments are + used to customize various aspects of the validation process. + + Returns: + (dict): Validation metrics obtained from the validation process. + + Raises: + AssertionError: If the model is not a PyTorch model. + """ + custom = {"rect": True} # method defaults + args = {**self.overrides, **custom, **kwargs, "mode": "val"} # highest priority args on the right + + validator = (validator or self._smart_load("validator"))(args=args, _callbacks=self.callbacks) + validator(model=self.model) + self.metrics = validator.metrics + return validator.metrics + + def benchmark( + self, + **kwargs, + ): + """ + Benchmarks the model across various export formats to evaluate performance. + + This method assesses the model's performance in different export formats, such as ONNX, TorchScript, etc. + It uses the 'benchmark' function from the ultralytics.utils.benchmarks module. The benchmarking is configured + using a combination of default configuration values, model-specific arguments, method-specific defaults, and + any additional user-provided keyword arguments. + + The method supports various arguments that allow customization of the benchmarking process, such as dataset + choice, image size, precision modes, device selection, and verbosity. For a comprehensive list of all + configurable options, users should refer to the 'configuration' section in the documentation. + + Args: + **kwargs (any): Arbitrary keyword arguments to customize the benchmarking process. These are combined with + default configurations, model-specific arguments, and method defaults. + + Returns: + (dict): A dictionary containing the results of the benchmarking process. + + Raises: + AssertionError: If the model is not a PyTorch model. + """ + self._check_is_pytorch_model() + from ultralytics.utils.benchmarks import benchmark + + custom = {"verbose": False} # method defaults + args = {**DEFAULT_CFG_DICT, **self.model.args, **custom, **kwargs, "mode": "benchmark"} + return benchmark( + model=self, + data=kwargs.get("data"), # if no 'data' argument passed set data=None for default datasets + imgsz=args["imgsz"], + half=args["half"], + int8=args["int8"], + device=args["device"], + verbose=kwargs.get("verbose"), + ) + + def export( + self, + **kwargs, + ): + """ + Exports the model to a different format suitable for deployment. + + This method facilitates the export of the model to various formats (e.g., ONNX, TorchScript) for deployment + purposes. It uses the 'Exporter' class for the export process, combining model-specific overrides, method + defaults, and any additional arguments provided. The combined arguments are used to configure export settings. + + The method supports a wide range of arguments to customize the export process. For a comprehensive list of all + possible arguments, refer to the 'configuration' section in the documentation. + + Args: + **kwargs (any): Arbitrary keyword arguments to customize the export process. These are combined with the + model's overrides and method defaults. + + Returns: + (object): The exported model in the specified format, or an object related to the export process. + + Raises: + AssertionError: If the model is not a PyTorch model. + """ + self._check_is_pytorch_model() + from .exporter import Exporter + + custom = {"imgsz": self.model.args["imgsz"], "batch": 1, "data": None, "verbose": False} # method defaults + args = {**self.overrides, **custom, **kwargs, "mode": "export"} # highest priority args on the right + return Exporter(overrides=args, _callbacks=self.callbacks)(model=self.model) + + def train( + self, + trainer=None, + **kwargs, + ): + """ + Trains the model using the specified dataset and training configuration. + + This method facilitates model training with a range of customizable settings and configurations. It supports + training with a custom trainer or the default training approach defined in the method. The method handles + different scenarios, such as resuming training from a checkpoint, integrating with Ultralytics HUB, and + updating model and configuration after training. + + When using Ultralytics HUB, if the session already has a loaded model, the method prioritizes HUB training + arguments and issues a warning if local arguments are provided. It checks for pip updates and combines default + configurations, method-specific defaults, and user-provided arguments to configure the training process. After + training, it updates the model and its configurations, and optionally attaches metrics. + + Args: + trainer (BaseTrainer, optional): An instance of a custom trainer class for training the model. If None, the + method uses a default trainer. Defaults to None. + **kwargs (any): Arbitrary keyword arguments representing the training configuration. These arguments are + used to customize various aspects of the training process. + + Returns: + (dict | None): Training metrics if available and training is successful; otherwise, None. + + Raises: + AssertionError: If the model is not a PyTorch model. + PermissionError: If there is a permission issue with the HUB session. + ModuleNotFoundError: If the HUB SDK is not installed. + """ + self._check_is_pytorch_model() + if hasattr(self.session, "model") and self.session.model.id: # Ultralytics HUB session with loaded model + if any(kwargs): + LOGGER.warning("WARNING ⚠️ using HUB training arguments, ignoring local training arguments.") + kwargs = self.session.train_args # overwrite kwargs + + checks.check_pip_update_available() + + overrides = yaml_load(checks.check_yaml(kwargs["cfg"])) if kwargs.get("cfg") else self.overrides + custom = {"data": DEFAULT_CFG_DICT["data"] or TASK2DATA[self.task]} # method defaults + args = {**overrides, **custom, **kwargs, "mode": "train"} # highest priority args on the right + if args.get("resume"): + args["resume"] = self.ckpt_path + + self.trainer = (trainer or self._smart_load("trainer"))(overrides=args, _callbacks=self.callbacks) + if not args.get("resume"): # manually set model only if not resuming + self.trainer.model = self.trainer.get_model(weights=self.model if self.ckpt else None, cfg=self.model.yaml) + self.model = self.trainer.model + + if SETTINGS["hub"] is True and not self.session: + # Create a model in HUB + try: + self.session = self._get_hub_session(self.model_name) + if self.session: + self.session.create_model(args) + # Check model was created + if not getattr(self.session.model, "id", None): + self.session = None + except (PermissionError, ModuleNotFoundError): + # Ignore PermissionError and ModuleNotFoundError which indicates hub-sdk not installed + pass + + self.trainer.hub_session = self.session # attach optional HUB session + self.trainer.train() + # Update model and cfg after training + if RANK in (-1, 0): + ckpt = self.trainer.best if self.trainer.best.exists() else self.trainer.last + self.model, _ = attempt_load_one_weight(ckpt) + self.overrides = self.model.args + self.metrics = getattr(self.trainer.validator, "metrics", None) # TODO: no metrics returned by DDP + return self.metrics + + def tune( + self, + use_ray=False, + iterations=10, + *args, + **kwargs, + ): + """ + Conducts hyperparameter tuning for the model, with an option to use Ray Tune. + + This method supports two modes of hyperparameter tuning: using Ray Tune or a custom tuning method. + When Ray Tune is enabled, it leverages the 'run_ray_tune' function from the ultralytics.utils.tuner module. + Otherwise, it uses the internal 'Tuner' class for tuning. The method combines default, overridden, and + custom arguments to configure the tuning process. + + Args: + use_ray (bool): If True, uses Ray Tune for hyperparameter tuning. Defaults to False. + iterations (int): The number of tuning iterations to perform. Defaults to 10. + *args (list): Variable length argument list for additional arguments. + **kwargs (any): Arbitrary keyword arguments. These are combined with the model's overrides and defaults. + + Returns: + (dict): A dictionary containing the results of the hyperparameter search. + + Raises: + AssertionError: If the model is not a PyTorch model. + """ + self._check_is_pytorch_model() + if use_ray: + from ultralytics.utils.tuner import run_ray_tune + + return run_ray_tune(self, max_samples=iterations, *args, **kwargs) + else: + from .tuner import Tuner + + custom = {} # method defaults + args = {**self.overrides, **custom, **kwargs, "mode": "train"} # highest priority args on the right + return Tuner(args=args, _callbacks=self.callbacks)(model=self, iterations=iterations) + + def _apply(self, fn) -> "Model": + """Apply to(), cpu(), cuda(), half(), float() to model tensors that are not parameters or registered buffers.""" + self._check_is_pytorch_model() + self = super()._apply(fn) # noqa + self.predictor = None # reset predictor as device may have changed + self.overrides["device"] = self.device # was str(self.device) i.e. device(type='cuda', index=0) -> 'cuda:0' + return self + + @property + def names(self) -> list: + """ + Retrieves the class names associated with the loaded model. + + This property returns the class names if they are defined in the model. It checks the class names for validity + using the 'check_class_names' function from the ultralytics.nn.autobackend module. + + Returns: + (list | None): The class names of the model if available, otherwise None. + """ + from ultralytics.nn.autobackend import check_class_names + + return check_class_names(self.model.names) if hasattr(self.model, "names") else None + + @property + def device(self) -> torch.device: + """ + Retrieves the device on which the model's parameters are allocated. + + This property is used to determine whether the model's parameters are on CPU or GPU. It only applies to models + that are instances of nn.Module. + + Returns: + (torch.device | None): The device (CPU/GPU) of the model if it is a PyTorch model, otherwise None. + """ + return next(self.model.parameters()).device if isinstance(self.model, nn.Module) else None + + @property + def transforms(self): + """ + Retrieves the transformations applied to the input data of the loaded model. + + This property returns the transformations if they are defined in the model. + + Returns: + (object | None): The transform object of the model if available, otherwise None. + """ + return self.model.transforms if hasattr(self.model, "transforms") else None + + def add_callback(self, event: str, func) -> None: + """ + Adds a callback function for a specified event. + + This method allows the user to register a custom callback function that is triggered on a specific event during + model training or inference. + + Args: + event (str): The name of the event to attach the callback to. + func (callable): The callback function to be registered. + + Raises: + ValueError: If the event name is not recognized. + """ + self.callbacks[event].append(func) + + def clear_callback(self, event: str) -> None: + """ + Clears all callback functions registered for a specified event. + + This method removes all custom and default callback functions associated with the given event. + + Args: + event (str): The name of the event for which to clear the callbacks. + + Raises: + ValueError: If the event name is not recognized. + """ + self.callbacks[event] = [] + + def reset_callbacks(self) -> None: + """ + Resets all callbacks to their default functions. + + This method reinstates the default callback functions for all events, removing any custom callbacks that were + added previously. + """ + for event in callbacks.default_callbacks.keys(): + self.callbacks[event] = [callbacks.default_callbacks[event][0]] + + @staticmethod + def _reset_ckpt_args(args: dict) -> dict: + """Reset arguments when loading a PyTorch model.""" + include = {"imgsz", "data", "task", "single_cls"} # only remember these arguments when loading a PyTorch model + return {k: v for k, v in args.items() if k in include} + + # def __getattr__(self, attr): + # """Raises error if object has no requested attribute.""" + # name = self.__class__.__name__ + # raise AttributeError(f"'{name}' object has no attribute '{attr}'. See valid attributes below.\n{self.__doc__}") + + def _smart_load(self, key: str): + """Load model/trainer/validator/predictor.""" + try: + return self.task_map[self.task][key] + except Exception as e: + name = self.__class__.__name__ + mode = inspect.stack()[1][3] # get the function name. + raise NotImplementedError( + emojis(f"WARNING ⚠️ '{name}' model does not support '{mode}' mode for '{self.task}' task yet.") + ) from e + + @property + def task_map(self) -> dict: + """ + Map head to model, trainer, validator, and predictor classes. + + Returns: + task_map (dict): The map of model task to mode classes. + """ + raise NotImplementedError("Please provide task map for your model!") diff --git a/ultralytics/engine/predictor.py b/ultralytics/engine/predictor.py new file mode 100644 index 0000000..9ec803a --- /dev/null +++ b/ultralytics/engine/predictor.py @@ -0,0 +1,397 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +""" +Run prediction on images, videos, directories, globs, YouTube, webcam, streams, etc. + +Usage - sources: + $ yolo mode=predict model=yolov8n.pt source=0 # webcam + img.jpg # image + vid.mp4 # video + screen # screenshot + path/ # directory + list.txt # list of images + list.streams # list of streams + 'path/*.jpg' # glob + 'https://youtu.be/LNwODJXcvt4' # YouTube + 'rtsp://example.com/media.mp4' # RTSP, RTMP, HTTP, TCP stream + +Usage - formats: + $ yolo mode=predict model=yolov8n.pt # PyTorch + yolov8n.torchscript # TorchScript + yolov8n.onnx # ONNX Runtime or OpenCV DNN with dnn=True + yolov8n_openvino_model # OpenVINO + yolov8n.engine # TensorRT + yolov8n.mlpackage # CoreML (macOS-only) + yolov8n_saved_model # TensorFlow SavedModel + yolov8n.pb # TensorFlow GraphDef + yolov8n.tflite # TensorFlow Lite + yolov8n_edgetpu.tflite # TensorFlow Edge TPU + yolov8n_paddle_model # PaddlePaddle + yolov8n_ncnn_model # NCNN +""" + +import platform +import re +import threading +from pathlib import Path + +import cv2 +import numpy as np +import torch + +from ultralytics.cfg import get_cfg, get_save_dir +from ultralytics.data import load_inference_source +from ultralytics.data.augment import LetterBox, classify_transforms +from ultralytics.nn.autobackend import AutoBackend +from ultralytics.utils import DEFAULT_CFG, LOGGER, MACOS, WINDOWS, callbacks, colorstr, ops +from ultralytics.utils.checks import check_imgsz, check_imshow +from ultralytics.utils.files import increment_path +from ultralytics.utils.torch_utils import select_device, smart_inference_mode + +STREAM_WARNING = """ +WARNING ⚠️ inference results will accumulate in RAM unless `stream=True` is passed, causing potential out-of-memory +errors for large sources or long-running streams and videos. See https://docs.ultralytics.com/modes/predict/ for help. + +Example: + results = model(source=..., stream=True) # generator of Results objects + for r in results: + boxes = r.boxes # Boxes object for bbox outputs + masks = r.masks # Masks object for segment masks outputs + probs = r.probs # Class probabilities for classification outputs +""" + + +class BasePredictor: + """ + BasePredictor. + + A base class for creating predictors. + + Attributes: + args (SimpleNamespace): Configuration for the predictor. + save_dir (Path): Directory to save results. + done_warmup (bool): Whether the predictor has finished setup. + model (nn.Module): Model used for prediction. + data (dict): Data configuration. + device (torch.device): Device used for prediction. + dataset (Dataset): Dataset used for prediction. + vid_writer (dict): Dictionary of {save_path: video_writer, ...} writer for saving video output. + """ + + def __init__(self, cfg=DEFAULT_CFG, overrides=None, _callbacks=None): + """ + Initializes the BasePredictor class. + + Args: + cfg (str, optional): Path to a configuration file. Defaults to DEFAULT_CFG. + overrides (dict, optional): Configuration overrides. Defaults to None. + """ + self.args = get_cfg(cfg, overrides) + self.save_dir = get_save_dir(self.args) + if self.args.conf is None: + self.args.conf = 0.25 # default conf=0.25 + self.done_warmup = False + if self.args.show: + self.args.show = check_imshow(warn=True) + + # Usable if setup is done + self.model = None + self.data = self.args.data # data_dict + self.imgsz = None + self.device = None + self.dataset = None + self.vid_writer = {} # dict of {save_path: video_writer, ...} + self.plotted_img = None + self.source_type = None + self.seen = 0 + self.windows = [] + self.batch = None + self.results = None + self.transforms = None + self.callbacks = _callbacks or callbacks.get_default_callbacks() + self.txt_path = None + self._lock = threading.Lock() # for automatic thread-safe inference + callbacks.add_integration_callbacks(self) + + def preprocess(self, im): + """ + Prepares input image before inference. + + Args: + im (torch.Tensor | List(np.ndarray)): BCHW for tensor, [(HWC) x B] for list. + """ + not_tensor = not isinstance(im, torch.Tensor) + if not_tensor: + im = np.stack(self.pre_transform(im)) + im = im[..., ::-1].transpose((0, 3, 1, 2)) # BGR to RGB, BHWC to BCHW, (n, 3, h, w) + im = np.ascontiguousarray(im) # contiguous + im = torch.from_numpy(im) + + im = im.to(self.device) + im = im.half() if self.model.fp16 else im.float() # uint8 to fp16/32 + if not_tensor: + im /= 255 # 0 - 255 to 0.0 - 1.0 + return im + + def inference(self, im, *args, **kwargs): + """Runs inference on a given image using the specified model and arguments.""" + visualize = ( + increment_path(self.save_dir / Path(self.batch[0][0]).stem, mkdir=True) + if self.args.visualize and (not self.source_type.tensor) + else False + ) + return self.model(im, augment=self.args.augment, visualize=visualize, embed=self.args.embed, *args, **kwargs) + + def pre_transform(self, im): + """ + Pre-transform input image before inference. + + Args: + im (List(np.ndarray)): (N, 3, h, w) for tensor, [(h, w, 3) x N] for list. + + Returns: + (list): A list of transformed images. + """ + same_shapes = len({x.shape for x in im}) == 1 + letterbox = LetterBox(self.imgsz, auto=same_shapes and self.model.pt, stride=self.model.stride) + return [letterbox(image=x) for x in im] + + def postprocess(self, preds, img, orig_imgs): + """Post-processes predictions for an image and returns them.""" + return preds + + def __call__(self, source=None, model=None, stream=False, *args, **kwargs): + """Performs inference on an image or stream.""" + self.stream = stream + if stream: + return self.stream_inference(source, model, *args, **kwargs) + else: + return list(self.stream_inference(source, model, *args, **kwargs)) # merge list of Result into one + + def predict_cli(self, source=None, model=None): + """ + Method used for CLI prediction. + + It uses always generator as outputs as not required by CLI mode. + """ + gen = self.stream_inference(source, model) + for _ in gen: # noqa, running CLI inference without accumulating any outputs (do not modify) + pass + + def setup_source(self, source): + """Sets up source and inference mode.""" + self.imgsz = check_imgsz(self.args.imgsz, stride=self.model.stride, min_dim=2) # check image size + self.transforms = ( + getattr( + self.model.model, + "transforms", + classify_transforms(self.imgsz[0], crop_fraction=self.args.crop_fraction), + ) + if self.args.task == "classify" + else None + ) + self.dataset = load_inference_source( + source=source, + batch=self.args.batch, + vid_stride=self.args.vid_stride, + buffer=self.args.stream_buffer, + ) + self.source_type = self.dataset.source_type + if not getattr(self, "stream", True) and ( + self.source_type.stream + or self.source_type.screenshot + or len(self.dataset) > 1000 # many images + or any(getattr(self.dataset, "video_flag", [False])) + ): # videos + LOGGER.warning(STREAM_WARNING) + self.vid_writer = {} + + @smart_inference_mode() + def stream_inference(self, source=None, model=None, *args, **kwargs): + """Streams real-time inference on camera feed and saves results to file.""" + if self.args.verbose: + LOGGER.info("") + + # Setup model + if not self.model: + self.setup_model(model) + + with self._lock: # for thread-safe inference + # Setup source every time predict is called + self.setup_source(source if source is not None else self.args.source) + + # Check if save_dir/ label file exists + if self.args.save or self.args.save_txt: + (self.save_dir / "labels" if self.args.save_txt else self.save_dir).mkdir(parents=True, exist_ok=True) + + # Warmup model + if not self.done_warmup: + self.model.warmup(imgsz=(1 if self.model.pt or self.model.triton else self.dataset.bs, 3, *self.imgsz)) + self.done_warmup = True + + self.seen, self.windows, self.batch = 0, [], None + profilers = ( + ops.Profile(device=self.device), + ops.Profile(device=self.device), + ops.Profile(device=self.device), + ) + self.run_callbacks("on_predict_start") + for self.batch in self.dataset: + self.run_callbacks("on_predict_batch_start") + paths, im0s, s = self.batch + + # Preprocess + with profilers[0]: + im = self.preprocess(im0s) + + # Inference + with profilers[1]: + preds = self.inference(im, *args, **kwargs) + if self.args.embed: + yield from [preds] if isinstance(preds, torch.Tensor) else preds # yield embedding tensors + continue + + # Postprocess + with profilers[2]: + self.results = self.postprocess(preds, im, im0s) + self.run_callbacks("on_predict_postprocess_end") + + # Visualize, save, write results + n = len(im0s) + for i in range(n): + self.seen += 1 + self.results[i].speed = { + "preprocess": profilers[0].dt * 1e3 / n, + "inference": profilers[1].dt * 1e3 / n, + "postprocess": profilers[2].dt * 1e3 / n, + } + if self.args.verbose or self.args.save or self.args.save_txt or self.args.show: + s[i] += self.write_results(i, Path(paths[i]), im, s) + + # Print batch results + if self.args.verbose: + LOGGER.info("\n".join(s)) + + self.run_callbacks("on_predict_batch_end") + yield from self.results + + # Release assets + for v in self.vid_writer.values(): + if isinstance(v, cv2.VideoWriter): + v.release() + + # Print final results + if self.args.verbose and self.seen: + t = tuple(x.t / self.seen * 1e3 for x in profilers) # speeds per image + LOGGER.info( + f"Speed: %.1fms preprocess, %.1fms inference, %.1fms postprocess per image at shape " + f"{(min(self.args.batch, self.seen), 3, *im.shape[2:])}" % t + ) + if self.args.save or self.args.save_txt or self.args.save_crop: + nl = len(list(self.save_dir.glob("labels/*.txt"))) # number of labels + s = f"\n{nl} label{'s' * (nl > 1)} saved to {self.save_dir / 'labels'}" if self.args.save_txt else "" + LOGGER.info(f"Results saved to {colorstr('bold', self.save_dir)}{s}") + self.run_callbacks("on_predict_end") + + def setup_model(self, model, verbose=True): + """Initialize YOLO model with given parameters and set it to evaluation mode.""" + self.model = AutoBackend( + weights=model or self.args.model, + device=select_device(self.args.device, verbose=verbose), + dnn=self.args.dnn, + data=self.args.data, + fp16=self.args.half, + batch=self.args.batch, + fuse=True, + verbose=verbose, + ) + + self.device = self.model.device # update device + self.args.half = self.model.fp16 # update half + self.model.eval() + + def write_results(self, i, p, im, s): + """Write inference results to a file or directory.""" + string = "" # print string + if len(im.shape) == 3: + im = im[None] # expand for batch dim + if self.source_type.stream or self.source_type.from_img or self.source_type.tensor: # batch_size >= 1 + string += f"{i}: " + frame = self.dataset.count + else: + match = re.search(r"frame (\d+)/", s[i]) + frame = int(match.group(1)) if match else None # 0 if frame undetermined + + self.txt_path = self.save_dir / "labels" / (p.stem + ("" if self.dataset.mode == "image" else f"_{frame}")) + string += "%gx%g " % im.shape[2:] + result = self.results[i] + result.save_dir = self.save_dir.__str__() # used in other locations + string += result.verbose() + f"{result.speed['inference']:.1f}ms" + + # Add predictions to image + if self.args.save or self.args.show: + self.plotted_img = result.plot( + line_width=self.args.line_width, + boxes=self.args.show_boxes, + conf=self.args.show_conf, + labels=self.args.show_labels, + im_gpu=None if self.args.retina_masks else im[i], + ) + + # Save results + if self.args.save_txt: + result.save_txt(f"{self.txt_path}.txt", save_conf=self.args.save_conf) + if self.args.save_crop: + result.save_crop(save_dir=self.save_dir / "crops", file_name=self.txt_path.stem) + if self.args.show: + self.show(str(p)) + if self.args.save: + self.save_predicted_images(str(self.save_dir / (p.name or "tmp.jpg")), frame) + + return string + + def save_predicted_images(self, save_path="", frame=0): + """Save video predictions as mp4 at specified path.""" + im = self.plotted_img + + # Save videos and streams + if self.dataset.mode in {"stream", "video"}: + fps = self.dataset.fps if self.dataset.mode == "video" else 30 + frames_path = f'{save_path.split(".", 1)[0]}_frames/' + if save_path not in self.vid_writer: # new video + if self.args.save_frames: + Path(frames_path).mkdir(parents=True, exist_ok=True) + suffix, fourcc = (".mp4", "avc1") if MACOS else (".avi", "WMV2") if WINDOWS else (".avi", "MJPG") + self.vid_writer[save_path] = cv2.VideoWriter( + filename=str(Path(save_path).with_suffix(suffix)), + fourcc=cv2.VideoWriter_fourcc(*fourcc), + fps=fps, # integer required, floats produce error in MP4 codec + frameSize=(im.shape[1], im.shape[0]), # (width, height) + ) + + # Save video + self.vid_writer[save_path].write(im) + if self.args.save_frames: + cv2.imwrite(f"{frames_path}{frame}.jpg", im) + + # Save images + else: + cv2.imwrite(save_path, im) + + def show(self, p=""): + """Display an image in a window using OpenCV imshow().""" + im = self.plotted_img + if platform.system() == "Linux" and p not in self.windows: + self.windows.append(p) + cv2.namedWindow(p, cv2.WINDOW_NORMAL | cv2.WINDOW_KEEPRATIO) # allow window resize (Linux) + cv2.resizeWindow(p, im.shape[1], im.shape[0]) # (width, height) + cv2.imshow(p, im) + cv2.waitKey(300 if self.dataset.mode == "image" else 1) # 1 millisecond + + def run_callbacks(self, event: str): + """Runs all registered callbacks for a specific event.""" + for callback in self.callbacks.get(event, []): + callback(self) + + def add_callback(self, event: str, func): + """Add callback.""" + self.callbacks[event].append(func) diff --git a/ultralytics/engine/results.py b/ultralytics/engine/results.py new file mode 100644 index 0000000..85849c3 --- /dev/null +++ b/ultralytics/engine/results.py @@ -0,0 +1,743 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +""" +Ultralytics Results, Boxes and Masks classes for handling inference results. + +Usage: See https://docs.ultralytics.com/modes/predict/ +""" + +from copy import deepcopy +from functools import lru_cache +from pathlib import Path + +import numpy as np +import torch + +from ultralytics.data.augment import LetterBox +from ultralytics.utils import LOGGER, SimpleClass, ops +from ultralytics.utils.plotting import Annotator, colors, save_one_box +from ultralytics.utils.torch_utils import smart_inference_mode + + +class BaseTensor(SimpleClass): + """Base tensor class with additional methods for easy manipulation and device handling.""" + + def __init__(self, data, orig_shape) -> None: + """ + Initialize BaseTensor with data and original shape. + + Args: + data (torch.Tensor | np.ndarray): Predictions, such as bboxes, masks and keypoints. + orig_shape (tuple): Original shape of image. + """ + assert isinstance(data, (torch.Tensor, np.ndarray)) + self.data = data + self.orig_shape = orig_shape + + @property + def shape(self): + """Return the shape of the data tensor.""" + return self.data.shape + + def cpu(self): + """Return a copy of the tensor on CPU memory.""" + return self if isinstance(self.data, np.ndarray) else self.__class__(self.data.cpu(), self.orig_shape) + + def numpy(self): + """Return a copy of the tensor as a numpy array.""" + return self if isinstance(self.data, np.ndarray) else self.__class__(self.data.numpy(), self.orig_shape) + + def cuda(self): + """Return a copy of the tensor on GPU memory.""" + return self.__class__(torch.as_tensor(self.data).cuda(), self.orig_shape) + + def to(self, *args, **kwargs): + """Return a copy of the tensor with the specified device and dtype.""" + return self.__class__(torch.as_tensor(self.data).to(*args, **kwargs), self.orig_shape) + + def __len__(self): # override len(results) + """Return the length of the data tensor.""" + return len(self.data) + + def __getitem__(self, idx): + """Return a BaseTensor with the specified index of the data tensor.""" + return self.__class__(self.data[idx], self.orig_shape) + + +class Results(SimpleClass): + """ + A class for storing and manipulating inference results. + + Attributes: + orig_img (numpy.ndarray): Original image as a numpy array. + orig_shape (tuple): Original image shape in (height, width) format. + boxes (Boxes, optional): Object containing detection bounding boxes. + masks (Masks, optional): Object containing detection masks. + probs (Probs, optional): Object containing class probabilities for classification tasks. + keypoints (Keypoints, optional): Object containing detected keypoints for each object. + speed (dict): Dictionary of preprocess, inference, and postprocess speeds (ms/image). + names (dict): Dictionary of class names. + path (str): Path to the image file. + + Methods: + update(boxes=None, masks=None, probs=None, obb=None): Updates object attributes with new detection results. + cpu(): Returns a copy of the Results object with all tensors on CPU memory. + numpy(): Returns a copy of the Results object with all tensors as numpy arrays. + cuda(): Returns a copy of the Results object with all tensors on GPU memory. + to(*args, **kwargs): Returns a copy of the Results object with tensors on a specified device and dtype. + new(): Returns a new Results object with the same image, path, and names. + plot(...): Plots detection results on an input image, returning an annotated image. + show(): Show annotated results to screen. + save(filename): Save annotated results to file. + verbose(): Returns a log string for each task, detailing detections and classifications. + save_txt(txt_file, save_conf=False): Saves detection results to a text file. + save_crop(save_dir, file_name=Path("im.jpg")): Saves cropped detection images. + tojson(normalize=False): Converts detection results to JSON format. + """ + + def __init__(self, orig_img, path, names, boxes=None, masks=None, probs=None, keypoints=None, obb=None) -> None: + """ + Initialize the Results class. + + Args: + orig_img (numpy.ndarray): The original image as a numpy array. + path (str): The path to the image file. + names (dict): A dictionary of class names. + boxes (torch.tensor, optional): A 2D tensor of bounding box coordinates for each detection. + masks (torch.tensor, optional): A 3D tensor of detection masks, where each mask is a binary image. + probs (torch.tensor, optional): A 1D tensor of probabilities of each class for classification task. + keypoints (torch.tensor, optional): A 2D tensor of keypoint coordinates for each detection. + obb (torch.tensor, optional): A 2D tensor of oriented bounding box coordinates for each detection. + """ + self.orig_img = orig_img + self.orig_shape = orig_img.shape[:2] + self.boxes = Boxes(boxes, self.orig_shape) if boxes is not None else None # native size boxes + self.masks = Masks(masks, self.orig_shape) if masks is not None else None # native size or imgsz masks + self.probs = Probs(probs) if probs is not None else None + self.keypoints = Keypoints(keypoints, self.orig_shape) if keypoints is not None else None + self.obb = OBB(obb, self.orig_shape) if obb is not None else None + self.speed = {"preprocess": None, "inference": None, "postprocess": None} # milliseconds per image + self.names = names + self.path = path + self.save_dir = None + self._keys = "boxes", "masks", "probs", "keypoints", "obb" + + def __getitem__(self, idx): + """Return a Results object for the specified index.""" + return self._apply("__getitem__", idx) + + def __len__(self): + """Return the number of detections in the Results object.""" + for k in self._keys: + v = getattr(self, k) + if v is not None: + return len(v) + + def update(self, boxes=None, masks=None, probs=None, obb=None): + """Update the boxes, masks, and probs attributes of the Results object.""" + if boxes is not None: + self.boxes = Boxes(ops.clip_boxes(boxes, self.orig_shape), self.orig_shape) + if masks is not None: + self.masks = Masks(masks, self.orig_shape) + if probs is not None: + self.probs = probs + if obb is not None: + self.obb = OBB(obb, self.orig_shape) + + def _apply(self, fn, *args, **kwargs): + """ + Applies a function to all non-empty attributes and returns a new Results object with modified attributes. This + function is internally called by methods like .to(), .cuda(), .cpu(), etc. + + Args: + fn (str): The name of the function to apply. + *args: Variable length argument list to pass to the function. + **kwargs: Arbitrary keyword arguments to pass to the function. + + Returns: + Results: A new Results object with attributes modified by the applied function. + """ + r = self.new() + for k in self._keys: + v = getattr(self, k) + if v is not None: + setattr(r, k, getattr(v, fn)(*args, **kwargs)) + return r + + def cpu(self): + """Return a copy of the Results object with all tensors on CPU memory.""" + return self._apply("cpu") + + def numpy(self): + """Return a copy of the Results object with all tensors as numpy arrays.""" + return self._apply("numpy") + + def cuda(self): + """Return a copy of the Results object with all tensors on GPU memory.""" + return self._apply("cuda") + + def to(self, *args, **kwargs): + """Return a copy of the Results object with tensors on the specified device and dtype.""" + return self._apply("to", *args, **kwargs) + + def new(self): + """Return a new Results object with the same image, path, and names.""" + return Results(orig_img=self.orig_img, path=self.path, names=self.names) + + def plot( + self, + conf=True, + line_width=None, + font_size=None, + font="Arial.ttf", + pil=False, + img=None, + im_gpu=None, + kpt_radius=5, + kpt_line=True, + labels=True, + boxes=True, + masks=True, + probs=True, + show=False, + save=False, + filename=None, + ): + """ + Plots the detection results on an input RGB image. Accepts a numpy array (cv2) or a PIL Image. + + Args: + conf (bool): Whether to plot the detection confidence score. + line_width (float, optional): The line width of the bounding boxes. If None, it is scaled to the image size. + font_size (float, optional): The font size of the text. If None, it is scaled to the image size. + font (str): The font to use for the text. + pil (bool): Whether to return the image as a PIL Image. + img (numpy.ndarray): Plot to another image. if not, plot to original image. + im_gpu (torch.Tensor): Normalized image in gpu with shape (1, 3, 640, 640), for faster mask plotting. + kpt_radius (int, optional): Radius of the drawn keypoints. Default is 5. + kpt_line (bool): Whether to draw lines connecting keypoints. + labels (bool): Whether to plot the label of bounding boxes. + boxes (bool): Whether to plot the bounding boxes. + masks (bool): Whether to plot the masks. + probs (bool): Whether to plot classification probability + show (bool): Whether to display the annotated image directly. + save (bool): Whether to save the annotated image to `filename`. + filename (str): Filename to save image to if save is True. + + Returns: + (numpy.ndarray): A numpy array of the annotated image. + + Example: + ```python + from PIL import Image + from ultralytics import YOLO + + model = YOLO('yolov8n.pt') + results = model('bus.jpg') # results list + for r in results: + im_array = r.plot() # plot a BGR numpy array of predictions + im = Image.fromarray(im_array[..., ::-1]) # RGB PIL image + im.show() # show image + im.save('results.jpg') # save image + ``` + """ + if img is None and isinstance(self.orig_img, torch.Tensor): + img = (self.orig_img[0].detach().permute(1, 2, 0).contiguous() * 255).to(torch.uint8).cpu().numpy() + + names = self.names + is_obb = self.obb is not None + pred_boxes, show_boxes = self.obb if is_obb else self.boxes, boxes + pred_masks, show_masks = self.masks, masks + pred_probs, show_probs = self.probs, probs + annotator = Annotator( + deepcopy(self.orig_img if img is None else img), + line_width, + font_size, + font, + pil or (pred_probs is not None and show_probs), # Classify tasks default to pil=True + example=names, + ) + + # Plot Segment results + if pred_masks and show_masks: + if im_gpu is None: + img = LetterBox(pred_masks.shape[1:])(image=annotator.result()) + im_gpu = ( + torch.as_tensor(img, dtype=torch.float16, device=pred_masks.data.device) + .permute(2, 0, 1) + .flip(0) + .contiguous() + / 255 + ) + idx = pred_boxes.cls if pred_boxes else range(len(pred_masks)) + annotator.masks(pred_masks.data, colors=[colors(x, True) for x in idx], im_gpu=im_gpu) + + # Plot Detect results + if pred_boxes is not None and show_boxes: + for d in reversed(pred_boxes): + c, conf, id = int(d.cls), float(d.conf) if conf else None, None if d.id is None else int(d.id.item()) + name = ("" if id is None else f"id:{id} ") + names[c] + label = (f"{name} {conf:.2f}" if conf else name) if labels else None + box = d.xyxyxyxy.reshape(-1, 4, 2).squeeze() if is_obb else d.xyxy.squeeze() + annotator.box_label(box, label, color=colors(c, True), rotated=is_obb) + + # Plot Classify results + if pred_probs is not None and show_probs: + text = ",\n".join(f"{names[j] if names else j} {pred_probs.data[j]:.2f}" for j in pred_probs.top5) + x = round(self.orig_shape[0] * 0.03) + annotator.text([x, x], text, txt_color=(255, 255, 255)) # TODO: allow setting colors + + # Plot Pose results + if self.keypoints is not None: + for k in reversed(self.keypoints.data): + annotator.kpts(k, self.orig_shape, radius=kpt_radius, kpt_line=kpt_line) + + # Show results + if show: + annotator.show(self.path) + + # Save results + if save: + annotator.save(filename) + + return annotator.result() + + def show(self, *args, **kwargs): + """Show annotated results image.""" + self.plot(show=True, *args, **kwargs) + + def save(self, filename=None, *args, **kwargs): + """Save annotated results image.""" + if not filename: + filename = f"results_{Path(self.path).name}" + self.plot(save=True, filename=filename, *args, **kwargs) + return filename + + def verbose(self): + """Return log string for each task.""" + log_string = "" + probs = self.probs + boxes = self.boxes + if len(self) == 0: + return log_string if probs is not None else f"{log_string}(no detections), " + if probs is not None: + log_string += f"{', '.join(f'{self.names[j]} {probs.data[j]:.2f}' for j in probs.top5)}, " + if boxes: + for c in boxes.cls.unique(): + n = (boxes.cls == c).sum() # detections per class + log_string += f"{n} {self.names[int(c)]}{'s' * (n > 1)}, " + return log_string + + def save_txt(self, txt_file, save_conf=False): + """ + Save predictions into txt file. + + Args: + txt_file (str): txt file path. + save_conf (bool): save confidence score or not. + """ + is_obb = self.obb is not None + boxes = self.obb if is_obb else self.boxes + masks = self.masks + probs = self.probs + kpts = self.keypoints + texts = [] + if probs is not None: + # Classify + [texts.append(f"{probs.data[j]:.2f} {self.names[j]}") for j in probs.top5] + elif boxes: + # Detect/segment/pose + for j, d in enumerate(boxes): + c, conf, id = int(d.cls), float(d.conf), None if d.id is None else int(d.id.item()) + line = (c, *(d.xyxyxyxyn.view(-1) if is_obb else d.xywhn.view(-1))) + if masks: + seg = masks[j].xyn[0].copy().reshape(-1) # reversed mask.xyn, (n,2) to (n*2) + line = (c, *seg) + if kpts is not None: + kpt = torch.cat((kpts[j].xyn, kpts[j].conf[..., None]), 2) if kpts[j].has_visible else kpts[j].xyn + line += (*kpt.reshape(-1).tolist(),) + line += (conf,) * save_conf + (() if id is None else (id,)) + texts.append(("%g " * len(line)).rstrip() % line) + + if texts: + Path(txt_file).parent.mkdir(parents=True, exist_ok=True) # make directory + with open(txt_file, "a") as f: + f.writelines(text + "\n" for text in texts) + + def save_crop(self, save_dir, file_name=Path("im.jpg")): + """ + Save cropped predictions to `save_dir/cls/file_name.jpg`. + + Args: + save_dir (str | pathlib.Path): Save path. + file_name (str | pathlib.Path): File name. + """ + if self.probs is not None: + LOGGER.warning("WARNING ⚠️ Classify task do not support `save_crop`.") + return + if self.obb is not None: + LOGGER.warning("WARNING ⚠️ OBB task do not support `save_crop`.") + return + for d in self.boxes: + save_one_box( + d.xyxy, + self.orig_img.copy(), + file=Path(save_dir) / self.names[int(d.cls)] / f"{Path(file_name)}.jpg", + BGR=True, + ) + + def summary(self, normalize=False, decimals=5): + """Convert the results to a summarized format.""" + if self.probs is not None: + LOGGER.warning("Warning: Classify results do not support the `summary()` method yet.") + return + + # Create list of detection dictionaries + results = [] + data = self.boxes.data.cpu().tolist() + h, w = self.orig_shape if normalize else (1, 1) + for i, row in enumerate(data): # xyxy, track_id if tracking, conf, class_id + box = { + "x1": round(row[0] / w, decimals), + "y1": round(row[1] / h, decimals), + "x2": round(row[2] / w, decimals), + "y2": round(row[3] / h, decimals), + } + conf = round(row[-2], decimals) + class_id = int(row[-1]) + result = {"name": self.names[class_id], "class": class_id, "confidence": conf, "box": box} + if self.boxes.is_track: + result["track_id"] = int(row[-3]) # track ID + if self.masks: + result["segments"] = { + "x": (self.masks.xy[i][:, 0] / w).round(decimals).tolist(), + "y": (self.masks.xy[i][:, 1] / h).round(decimals).tolist(), + } + if self.keypoints is not None: + x, y, visible = self.keypoints[i].data[0].cpu().unbind(dim=1) # torch Tensor + result["keypoints"] = { + "x": (x / w).numpy().round(decimals).tolist(), # decimals named argument required + "y": (y / h).numpy().round(decimals).tolist(), + "visible": visible.numpy().round(decimals).tolist(), + } + results.append(result) + + return results + + def tojson(self, normalize=False, decimals=5): + """Convert the results to JSON format.""" + import json + + return json.dumps(self.summary(normalize=normalize, decimals=decimals), indent=2) + + +class Boxes(BaseTensor): + """ + Manages detection boxes, providing easy access and manipulation of box coordinates, confidence scores, class + identifiers, and optional tracking IDs. Supports multiple formats for box coordinates, including both absolute and + normalized forms. + + Attributes: + data (torch.Tensor): The raw tensor containing detection boxes and their associated data. + orig_shape (tuple): The original image size as a tuple (height, width), used for normalization. + is_track (bool): Indicates whether tracking IDs are included in the box data. + + Properties: + xyxy (torch.Tensor | numpy.ndarray): Boxes in [x1, y1, x2, y2] format. + conf (torch.Tensor | numpy.ndarray): Confidence scores for each box. + cls (torch.Tensor | numpy.ndarray): Class labels for each box. + id (torch.Tensor | numpy.ndarray, optional): Tracking IDs for each box, if available. + xywh (torch.Tensor | numpy.ndarray): Boxes in [x, y, width, height] format, calculated on demand. + xyxyn (torch.Tensor | numpy.ndarray): Normalized [x1, y1, x2, y2] boxes, relative to `orig_shape`. + xywhn (torch.Tensor | numpy.ndarray): Normalized [x, y, width, height] boxes, relative to `orig_shape`. + + Methods: + cpu(): Moves the boxes to CPU memory. + numpy(): Converts the boxes to a numpy array format. + cuda(): Moves the boxes to CUDA (GPU) memory. + to(device, dtype=None): Moves the boxes to the specified device. + """ + + def __init__(self, boxes, orig_shape) -> None: + """ + Initialize the Boxes class. + + Args: + boxes (torch.Tensor | numpy.ndarray): A tensor or numpy array containing the detection boxes, with + shape (num_boxes, 6) or (num_boxes, 7). The last two columns contain confidence and class values. + If present, the third last column contains track IDs. + orig_shape (tuple): Original image size, in the format (height, width). + """ + if boxes.ndim == 1: + boxes = boxes[None, :] + n = boxes.shape[-1] + assert n in (6, 7), f"expected 6 or 7 values but got {n}" # xyxy, track_id, conf, cls + super().__init__(boxes, orig_shape) + self.is_track = n == 7 + self.orig_shape = orig_shape + + @property + def xyxy(self): + """Return the boxes in xyxy format.""" + return self.data[:, :4] + + @property + def conf(self): + """Return the confidence values of the boxes.""" + return self.data[:, -2] + + @property + def cls(self): + """Return the class values of the boxes.""" + return self.data[:, -1] + + @property + def id(self): + """Return the track IDs of the boxes (if available).""" + return self.data[:, -3] if self.is_track else None + + @property + @lru_cache(maxsize=2) # maxsize 1 should suffice + def xywh(self): + """Return the boxes in xywh format.""" + return ops.xyxy2xywh(self.xyxy) + + @property + @lru_cache(maxsize=2) + def xyxyn(self): + """Return the boxes in xyxy format normalized by original image size.""" + xyxy = self.xyxy.clone() if isinstance(self.xyxy, torch.Tensor) else np.copy(self.xyxy) + xyxy[..., [0, 2]] /= self.orig_shape[1] + xyxy[..., [1, 3]] /= self.orig_shape[0] + return xyxy + + @property + @lru_cache(maxsize=2) + def xywhn(self): + """Return the boxes in xywh format normalized by original image size.""" + xywh = ops.xyxy2xywh(self.xyxy) + xywh[..., [0, 2]] /= self.orig_shape[1] + xywh[..., [1, 3]] /= self.orig_shape[0] + return xywh + + +class Masks(BaseTensor): + """ + A class for storing and manipulating detection masks. + + Attributes: + xy (list): A list of segments in pixel coordinates. + xyn (list): A list of normalized segments. + + Methods: + cpu(): Returns the masks tensor on CPU memory. + numpy(): Returns the masks tensor as a numpy array. + cuda(): Returns the masks tensor on GPU memory. + to(device, dtype): Returns the masks tensor with the specified device and dtype. + """ + + def __init__(self, masks, orig_shape) -> None: + """Initialize the Masks class with the given masks tensor and original image shape.""" + if masks.ndim == 2: + masks = masks[None, :] + super().__init__(masks, orig_shape) + + @property + @lru_cache(maxsize=1) + def xyn(self): + """Return normalized segments.""" + return [ + ops.scale_coords(self.data.shape[1:], x, self.orig_shape, normalize=True) + for x in ops.masks2segments(self.data) + ] + + @property + @lru_cache(maxsize=1) + def xy(self): + """Return segments in pixel coordinates.""" + return [ + ops.scale_coords(self.data.shape[1:], x, self.orig_shape, normalize=False) + for x in ops.masks2segments(self.data) + ] + + +class Keypoints(BaseTensor): + """ + A class for storing and manipulating detection keypoints. + + Attributes: + xy (torch.Tensor): A collection of keypoints containing x, y coordinates for each detection. + xyn (torch.Tensor): A normalized version of xy with coordinates in the range [0, 1]. + conf (torch.Tensor): Confidence values associated with keypoints if available, otherwise None. + + Methods: + cpu(): Returns a copy of the keypoints tensor on CPU memory. + numpy(): Returns a copy of the keypoints tensor as a numpy array. + cuda(): Returns a copy of the keypoints tensor on GPU memory. + to(device, dtype): Returns a copy of the keypoints tensor with the specified device and dtype. + """ + + @smart_inference_mode() # avoid keypoints < conf in-place error + def __init__(self, keypoints, orig_shape) -> None: + """Initializes the Keypoints object with detection keypoints and original image size.""" + if keypoints.ndim == 2: + keypoints = keypoints[None, :] + if keypoints.shape[2] == 3: # x, y, conf + mask = keypoints[..., 2] < 0.5 # points with conf < 0.5 (not visible) + keypoints[..., :2][mask] = 0 + super().__init__(keypoints, orig_shape) + self.has_visible = self.data.shape[-1] == 3 + + @property + @lru_cache(maxsize=1) + def xy(self): + """Returns x, y coordinates of keypoints.""" + return self.data[..., :2] + + @property + @lru_cache(maxsize=1) + def xyn(self): + """Returns normalized x, y coordinates of keypoints.""" + xy = self.xy.clone() if isinstance(self.xy, torch.Tensor) else np.copy(self.xy) + xy[..., 0] /= self.orig_shape[1] + xy[..., 1] /= self.orig_shape[0] + return xy + + @property + @lru_cache(maxsize=1) + def conf(self): + """Returns confidence values of keypoints if available, else None.""" + return self.data[..., 2] if self.has_visible else None + + +class Probs(BaseTensor): + """ + A class for storing and manipulating classification predictions. + + Attributes: + top1 (int): Index of the top 1 class. + top5 (list[int]): Indices of the top 5 classes. + top1conf (torch.Tensor): Confidence of the top 1 class. + top5conf (torch.Tensor): Confidences of the top 5 classes. + + Methods: + cpu(): Returns a copy of the probs tensor on CPU memory. + numpy(): Returns a copy of the probs tensor as a numpy array. + cuda(): Returns a copy of the probs tensor on GPU memory. + to(): Returns a copy of the probs tensor with the specified device and dtype. + """ + + def __init__(self, probs, orig_shape=None) -> None: + """Initialize the Probs class with classification probabilities and optional original shape of the image.""" + super().__init__(probs, orig_shape) + + @property + @lru_cache(maxsize=1) + def top1(self): + """Return the index of top 1.""" + return int(self.data.argmax()) + + @property + @lru_cache(maxsize=1) + def top5(self): + """Return the indices of top 5.""" + return (-self.data).argsort(0)[:5].tolist() # this way works with both torch and numpy. + + @property + @lru_cache(maxsize=1) + def top1conf(self): + """Return the confidence of top 1.""" + return self.data[self.top1] + + @property + @lru_cache(maxsize=1) + def top5conf(self): + """Return the confidences of top 5.""" + return self.data[self.top5] + + +class OBB(BaseTensor): + """ + A class for storing and manipulating Oriented Bounding Boxes (OBB). + + Args: + boxes (torch.Tensor | numpy.ndarray): A tensor or numpy array containing the detection boxes, + with shape (num_boxes, 7) or (num_boxes, 8). The last two columns contain confidence and class values. + If present, the third last column contains track IDs, and the fifth column from the left contains rotation. + orig_shape (tuple): Original image size, in the format (height, width). + + Attributes: + xywhr (torch.Tensor | numpy.ndarray): The boxes in [x_center, y_center, width, height, rotation] format. + conf (torch.Tensor | numpy.ndarray): The confidence values of the boxes. + cls (torch.Tensor | numpy.ndarray): The class values of the boxes. + id (torch.Tensor | numpy.ndarray): The track IDs of the boxes (if available). + xyxyxyxyn (torch.Tensor | numpy.ndarray): The rotated boxes in xyxyxyxy format normalized by orig image size. + xyxyxyxy (torch.Tensor | numpy.ndarray): The rotated boxes in xyxyxyxy format. + xyxy (torch.Tensor | numpy.ndarray): The horizontal boxes in xyxyxyxy format. + data (torch.Tensor): The raw OBB tensor (alias for `boxes`). + + Methods: + cpu(): Move the object to CPU memory. + numpy(): Convert the object to a numpy array. + cuda(): Move the object to CUDA memory. + to(*args, **kwargs): Move the object to the specified device. + """ + + def __init__(self, boxes, orig_shape) -> None: + """Initialize the Boxes class.""" + if boxes.ndim == 1: + boxes = boxes[None, :] + n = boxes.shape[-1] + assert n in (7, 8), f"expected 7 or 8 values but got {n}" # xywh, rotation, track_id, conf, cls + super().__init__(boxes, orig_shape) + self.is_track = n == 8 + self.orig_shape = orig_shape + + @property + def xywhr(self): + """Return the rotated boxes in xywhr format.""" + return self.data[:, :5] + + @property + def conf(self): + """Return the confidence values of the boxes.""" + return self.data[:, -2] + + @property + def cls(self): + """Return the class values of the boxes.""" + return self.data[:, -1] + + @property + def id(self): + """Return the track IDs of the boxes (if available).""" + return self.data[:, -3] if self.is_track else None + + @property + @lru_cache(maxsize=2) + def xyxyxyxy(self): + """Return the boxes in xyxyxyxy format, (N, 4, 2).""" + return ops.xywhr2xyxyxyxy(self.xywhr) + + @property + @lru_cache(maxsize=2) + def xyxyxyxyn(self): + """Return the boxes in xyxyxyxy format, (N, 4, 2).""" + xyxyxyxyn = self.xyxyxyxy.clone() if isinstance(self.xyxyxyxy, torch.Tensor) else np.copy(self.xyxyxyxy) + xyxyxyxyn[..., 0] /= self.orig_shape[1] + xyxyxyxyn[..., 1] /= self.orig_shape[0] + return xyxyxyxyn + + @property + @lru_cache(maxsize=2) + def xyxy(self): + """ + Return the horizontal boxes in xyxy format, (N, 4). + + Accepts both torch and numpy boxes. + """ + x1 = self.xyxyxyxy[..., 0].min(1).values + x2 = self.xyxyxyxy[..., 0].max(1).values + y1 = self.xyxyxyxy[..., 1].min(1).values + y2 = self.xyxyxyxy[..., 1].max(1).values + xyxy = [x1, y1, x2, y2] + return np.stack(xyxy, axis=-1) if isinstance(self.data, np.ndarray) else torch.stack(xyxy, dim=-1) diff --git a/ultralytics/engine/trainer.py b/ultralytics/engine/trainer.py new file mode 100644 index 0000000..2e7a7db --- /dev/null +++ b/ultralytics/engine/trainer.py @@ -0,0 +1,756 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +""" +Train a model on a dataset. + +Usage: + $ yolo mode=train model=yolov8n.pt data=coco128.yaml imgsz=640 epochs=100 batch=16 +""" + +import math +import os +import subprocess +import time +import warnings +from copy import deepcopy +from datetime import datetime, timedelta +from pathlib import Path + +import numpy as np +import torch +from torch import distributed as dist +from torch import nn, optim + +from ultralytics.cfg import get_cfg, get_save_dir +from ultralytics.data.utils import check_cls_dataset, check_det_dataset +from ultralytics.nn.tasks import attempt_load_one_weight, attempt_load_weights +from ultralytics.utils import ( + DEFAULT_CFG, + LOGGER, + RANK, + TQDM, + __version__, + callbacks, + clean_url, + colorstr, + emojis, + yaml_save, +) +from ultralytics.utils.autobatch import check_train_batch_size +from ultralytics.utils.checks import check_amp, check_file, check_imgsz, check_model_file_from_stem, print_args +from ultralytics.utils.dist import ddp_cleanup, generate_ddp_command +from ultralytics.utils.files import get_latest_run +from ultralytics.utils.torch_utils import ( + EarlyStopping, + ModelEMA, + de_parallel, + init_seeds, + one_cycle, + select_device, + strip_optimizer, +) + + +class BaseTrainer: + """ + BaseTrainer. + + A base class for creating trainers. + + Attributes: + args (SimpleNamespace): Configuration for the trainer. + validator (BaseValidator): Validator instance. + model (nn.Module): Model instance. + callbacks (defaultdict): Dictionary of callbacks. + save_dir (Path): Directory to save results. + wdir (Path): Directory to save weights. + last (Path): Path to the last checkpoint. + best (Path): Path to the best checkpoint. + save_period (int): Save checkpoint every x epochs (disabled if < 1). + batch_size (int): Batch size for training. + epochs (int): Number of epochs to train for. + start_epoch (int): Starting epoch for training. + device (torch.device): Device to use for training. + amp (bool): Flag to enable AMP (Automatic Mixed Precision). + scaler (amp.GradScaler): Gradient scaler for AMP. + data (str): Path to data. + trainset (torch.utils.data.Dataset): Training dataset. + testset (torch.utils.data.Dataset): Testing dataset. + ema (nn.Module): EMA (Exponential Moving Average) of the model. + resume (bool): Resume training from a checkpoint. + lf (nn.Module): Loss function. + scheduler (torch.optim.lr_scheduler._LRScheduler): Learning rate scheduler. + best_fitness (float): The best fitness value achieved. + fitness (float): Current fitness value. + loss (float): Current loss value. + tloss (float): Total loss value. + loss_names (list): List of loss names. + csv (Path): Path to results CSV file. + """ + + def __init__(self, cfg=DEFAULT_CFG, overrides=None, _callbacks=None): + """ + Initializes the BaseTrainer class. + + Args: + cfg (str, optional): Path to a configuration file. Defaults to DEFAULT_CFG. + overrides (dict, optional): Configuration overrides. Defaults to None. + """ + self.args = get_cfg(cfg, overrides) + self.check_resume(overrides) + self.device = select_device(self.args.device, self.args.batch) + self.validator = None + self.metrics = None + self.plots = {} + init_seeds(self.args.seed + 1 + RANK, deterministic=self.args.deterministic) + + # Dirs + self.save_dir = get_save_dir(self.args) + self.args.name = self.save_dir.name # update name for loggers + self.wdir = self.save_dir / "weights" # weights dir + if RANK in (-1, 0): + self.wdir.mkdir(parents=True, exist_ok=True) # make dir + self.args.save_dir = str(self.save_dir) + yaml_save(self.save_dir / "args.yaml", vars(self.args)) # save run args + self.last, self.best = self.wdir / "last.pt", self.wdir / "best.pt" # checkpoint paths + self.save_period = self.args.save_period + + self.batch_size = self.args.batch + self.epochs = self.args.epochs + self.start_epoch = 0 + if RANK == -1: + print_args(vars(self.args)) + + # Device + if self.device.type in ("cpu", "mps"): + self.args.workers = 0 # faster CPU training as time dominated by inference, not dataloading + + # Model and Dataset + self.model = check_model_file_from_stem(self.args.model) # add suffix, i.e. yolov8n -> yolov8n.pt + try: + if self.args.task == "classify": + self.data = check_cls_dataset(self.args.data) + elif self.args.data.split(".")[-1] in ("yaml", "yml") or self.args.task in ( + "detect", + "segment", + "pose", + "obb", + ): + self.data = check_det_dataset(self.args.data) + if "yaml_file" in self.data: + self.args.data = self.data["yaml_file"] # for validating 'yolo train data=url.zip' usage + except Exception as e: + raise RuntimeError(emojis(f"Dataset '{clean_url(self.args.data)}' error ❌ {e}")) from e + + self.trainset, self.testset = self.get_dataset(self.data) + self.ema = None + + # Optimization utils init + self.lf = None + self.scheduler = None + + # Epoch level metrics + self.best_fitness = None + self.fitness = None + self.loss = None + self.tloss = None + self.loss_names = ["Loss"] + self.csv = self.save_dir / "results.csv" + self.plot_idx = [0, 1, 2] + + # Callbacks + self.callbacks = _callbacks or callbacks.get_default_callbacks() + if RANK in (-1, 0): + callbacks.add_integration_callbacks(self) + + def add_callback(self, event: str, callback): + """Appends the given callback.""" + self.callbacks[event].append(callback) + + def set_callback(self, event: str, callback): + """Overrides the existing callbacks with the given callback.""" + self.callbacks[event] = [callback] + + def run_callbacks(self, event: str): + """Run all existing callbacks associated with a particular event.""" + for callback in self.callbacks.get(event, []): + callback(self) + + def train(self): + """Allow device='', device=None on Multi-GPU systems to default to device=0.""" + if isinstance(self.args.device, str) and len(self.args.device): # i.e. device='0' or device='0,1,2,3' + world_size = len(self.args.device.split(",")) + elif isinstance(self.args.device, (tuple, list)): # i.e. device=[0, 1, 2, 3] (multi-GPU from CLI is list) + world_size = len(self.args.device) + elif torch.cuda.is_available(): # i.e. device=None or device='' or device=number + world_size = 1 # default to device 0 + else: # i.e. device='cpu' or 'mps' + world_size = 0 + + # Run subprocess if DDP training, else train normally + if world_size > 1 and "LOCAL_RANK" not in os.environ: + # Argument checks + if self.args.rect: + LOGGER.warning("WARNING ⚠️ 'rect=True' is incompatible with Multi-GPU training, setting 'rect=False'") + self.args.rect = False + if self.args.batch == -1: + LOGGER.warning( + "WARNING ⚠️ 'batch=-1' for AutoBatch is incompatible with Multi-GPU training, setting " + "default 'batch=16'" + ) + self.args.batch = 16 + + # Command + cmd, file = generate_ddp_command(world_size, self) + try: + LOGGER.info(f'{colorstr("DDP:")} debug command {" ".join(cmd)}') + subprocess.run(cmd, check=True) + except Exception as e: + raise e + finally: + ddp_cleanup(self, str(file)) + + else: + self._do_train(world_size) + + def _setup_scheduler(self): + """Initialize training learning rate scheduler.""" + if self.args.cos_lr: + self.lf = one_cycle(1, self.args.lrf, self.epochs) # cosine 1->hyp['lrf'] + else: + self.lf = lambda x: max(1 - x / self.epochs, 0) * (1.0 - self.args.lrf) + self.args.lrf # linear + self.scheduler = optim.lr_scheduler.LambdaLR(self.optimizer, lr_lambda=self.lf) + + def _setup_ddp(self, world_size): + """Initializes and sets the DistributedDataParallel parameters for training.""" + torch.cuda.set_device(RANK) + self.device = torch.device("cuda", RANK) + # LOGGER.info(f'DDP info: RANK {RANK}, WORLD_SIZE {world_size}, DEVICE {self.device}') + os.environ["NCCL_BLOCKING_WAIT"] = "1" # set to enforce timeout + dist.init_process_group( + backend="nccl" if dist.is_nccl_available() else "gloo", + timeout=timedelta(seconds=10800), # 3 hours + rank=RANK, + world_size=world_size, + ) + + def _setup_train(self, world_size): + """Builds dataloaders and optimizer on correct rank process.""" + + # Model + self.run_callbacks("on_pretrain_routine_start") + ckpt = self.setup_model() + self.model = self.model.to(self.device) + self.set_model_attributes() + + # Freeze layers + freeze_list = ( + self.args.freeze + if isinstance(self.args.freeze, list) + else range(self.args.freeze) + if isinstance(self.args.freeze, int) + else [] + ) + always_freeze_names = [".dfl"] # always freeze these layers + freeze_layer_names = [f"model.{x}." for x in freeze_list] + always_freeze_names + for k, v in self.model.named_parameters(): + # v.register_hook(lambda x: torch.nan_to_num(x)) # NaN to 0 (commented for erratic training results) + if any(x in k for x in freeze_layer_names): + LOGGER.info(f"Freezing layer '{k}'") + v.requires_grad = False + elif not v.requires_grad and v.dtype.is_floating_point: # only floating point Tensor can require gradients + LOGGER.info( + f"WARNING ⚠️ setting 'requires_grad=True' for frozen layer '{k}'. " + "See ultralytics.engine.trainer for customization of frozen layers." + ) + v.requires_grad = True + + # Check AMP + self.amp = torch.tensor(self.args.amp).to(self.device) # True or False + if self.amp and RANK in (-1, 0): # Single-GPU and DDP + callbacks_backup = callbacks.default_callbacks.copy() # backup callbacks as check_amp() resets them + self.amp = torch.tensor(check_amp(self.model), device=self.device) + callbacks.default_callbacks = callbacks_backup # restore callbacks + if RANK > -1 and world_size > 1: # DDP + dist.broadcast(self.amp, src=0) # broadcast the tensor from rank 0 to all other ranks (returns None) + self.amp = bool(self.amp) # as boolean + self.scaler = torch.cuda.amp.GradScaler(enabled=self.amp) + if world_size > 1: + self.model = nn.parallel.DistributedDataParallel(self.model, device_ids=[RANK]) + + # Check imgsz + gs = max(int(self.model.stride.max() if hasattr(self.model, "stride") else 32), 32) # grid size (max stride) + self.args.imgsz = check_imgsz(self.args.imgsz, stride=gs, floor=gs, max_dim=1) + self.stride = gs # for multiscale training + + # Batch size + if self.batch_size == -1 and RANK == -1: # single-GPU only, estimate best batch size + self.args.batch = self.batch_size = check_train_batch_size(self.model, self.args.imgsz, self.amp) + + # Dataloaders + batch_size = self.batch_size // max(world_size, 1) + self.train_loader = self.get_dataloader(self.trainset, batch_size=batch_size, rank=RANK, mode="train") + if RANK in (-1, 0): + # Note: When training DOTA dataset, double batch size could get OOM on images with >2000 objects. + self.test_loader = self.get_dataloader( + self.testset, batch_size=batch_size if self.args.task == "obb" else batch_size * 2, rank=-1, mode="val" + ) + self.validator = self.get_validator() + metric_keys = self.validator.metrics.keys + self.label_loss_items(prefix="val") + self.metrics = dict(zip(metric_keys, [0] * len(metric_keys))) + self.ema = ModelEMA(self.model) + if self.args.plots: + self.plot_training_labels() + + # Optimizer + self.accumulate = max(round(self.args.nbs / self.batch_size), 1) # accumulate loss before optimizing + weight_decay = self.args.weight_decay * self.batch_size * self.accumulate / self.args.nbs # scale weight_decay + iterations = math.ceil(len(self.train_loader.dataset) / max(self.batch_size, self.args.nbs)) * self.epochs + self.optimizer = self.build_optimizer( + model=self.model, + name=self.args.optimizer, + lr=self.args.lr0, + momentum=self.args.momentum, + decay=weight_decay, + iterations=iterations, + ) + # Scheduler + self._setup_scheduler() + self.stopper, self.stop = EarlyStopping(patience=self.args.patience), False + self.resume_training(ckpt) + self.scheduler.last_epoch = self.start_epoch - 1 # do not move + self.run_callbacks("on_pretrain_routine_end") + + def _do_train(self, world_size=1): + """Train completed, evaluate and plot if specified by arguments.""" + if world_size > 1: + self._setup_ddp(world_size) + self._setup_train(world_size) + + nb = len(self.train_loader) # number of batches + nw = max(round(self.args.warmup_epochs * nb), 100) if self.args.warmup_epochs > 0 else -1 # warmup iterations + last_opt_step = -1 + self.epoch_time = None + self.epoch_time_start = time.time() + self.train_time_start = time.time() + self.run_callbacks("on_train_start") + LOGGER.info( + f'Image sizes {self.args.imgsz} train, {self.args.imgsz} val\n' + f'Using {self.train_loader.num_workers * (world_size or 1)} dataloader workers\n' + f"Logging results to {colorstr('bold', self.save_dir)}\n" + f'Starting training for ' + (f"{self.args.time} hours..." if self.args.time else f"{self.epochs} epochs...") + ) + if self.args.close_mosaic: + base_idx = (self.epochs - self.args.close_mosaic) * nb + self.plot_idx.extend([base_idx, base_idx + 1, base_idx + 2]) + epoch = self.start_epoch + while True: + self.epoch = epoch + self.run_callbacks("on_train_epoch_start") + self.model.train() + if RANK != -1: + self.train_loader.sampler.set_epoch(epoch) + pbar = enumerate(self.train_loader) + # Update dataloader attributes (optional) + if epoch == (self.epochs - self.args.close_mosaic): + self._close_dataloader_mosaic() + self.train_loader.reset() + + if RANK in (-1, 0): + LOGGER.info(self.progress_string()) + pbar = TQDM(enumerate(self.train_loader), total=nb) + self.tloss = None + self.optimizer.zero_grad() + for i, batch in pbar: + self.run_callbacks("on_train_batch_start") + # Warmup + ni = i + nb * epoch + if ni <= nw: + xi = [0, nw] # x interp + self.accumulate = max(1, int(np.interp(ni, xi, [1, self.args.nbs / self.batch_size]).round())) + for j, x in enumerate(self.optimizer.param_groups): + # Bias lr falls from 0.1 to lr0, all other lrs rise from 0.0 to lr0 + x["lr"] = np.interp( + ni, xi, [self.args.warmup_bias_lr if j == 0 else 0.0, x["initial_lr"] * self.lf(epoch)] + ) + if "momentum" in x: + x["momentum"] = np.interp(ni, xi, [self.args.warmup_momentum, self.args.momentum]) + + # Forward + with torch.cuda.amp.autocast(self.amp): + batch = self.preprocess_batch(batch) + self.loss, self.loss_items = self.model(batch) + if RANK != -1: + self.loss *= world_size + self.tloss = ( + (self.tloss * i + self.loss_items) / (i + 1) if self.tloss is not None else self.loss_items + ) + + # Backward + self.scaler.scale(self.loss).backward() + + # Optimize - https://pytorch.org/docs/master/notes/amp_examples.html + if ni - last_opt_step >= self.accumulate: + self.optimizer_step() + last_opt_step = ni + + # Timed stopping + if self.args.time: + self.stop = (time.time() - self.train_time_start) > (self.args.time * 3600) + if RANK != -1: # if DDP training + broadcast_list = [self.stop if RANK == 0 else None] + dist.broadcast_object_list(broadcast_list, 0) # broadcast 'stop' to all ranks + self.stop = broadcast_list[0] + if self.stop: # training time exceeded + break + + # Log + mem = f"{torch.cuda.memory_reserved() / 1E9 if torch.cuda.is_available() else 0:.3g}G" # (GB) + loss_len = self.tloss.shape[0] if len(self.tloss.shape) else 1 + losses = self.tloss if loss_len > 1 else torch.unsqueeze(self.tloss, 0) + if RANK in (-1, 0): + pbar.set_description( + ("%11s" * 2 + "%11.4g" * (2 + loss_len)) + % (f"{epoch + 1}/{self.epochs}", mem, *losses, batch["cls"].shape[0], batch["img"].shape[-1]) + ) + self.run_callbacks("on_batch_end") + if self.args.plots and ni in self.plot_idx: + self.plot_training_samples(batch, ni) + + self.run_callbacks("on_train_batch_end") + + self.lr = {f"lr/pg{ir}": x["lr"] for ir, x in enumerate(self.optimizer.param_groups)} # for loggers + self.run_callbacks("on_train_epoch_end") + if RANK in (-1, 0): + final_epoch = epoch + 1 == self.epochs + self.ema.update_attr(self.model, include=["yaml", "nc", "args", "names", "stride", "class_weights"]) + + # Validation + if (self.args.val and (((epoch+1) % self.args.val_period == 0) or (self.epochs - epoch) <= 10)) \ + or final_epoch or self.stopper.possible_stop or self.stop: + self.metrics, self.fitness = self.validate() + self.save_metrics(metrics={**self.label_loss_items(self.tloss), **self.metrics, **self.lr}) + self.stop |= self.stopper(epoch + 1, self.fitness) or final_epoch + if self.args.time: + self.stop |= (time.time() - self.train_time_start) > (self.args.time * 3600) + + # Save model + if self.args.save or final_epoch: + self.save_model() + self.run_callbacks("on_model_save") + + # Scheduler + t = time.time() + self.epoch_time = t - self.epoch_time_start + self.epoch_time_start = t + with warnings.catch_warnings(): + warnings.simplefilter("ignore") # suppress 'Detected lr_scheduler.step() before optimizer.step()' + if self.args.time: + mean_epoch_time = (t - self.train_time_start) / (epoch - self.start_epoch + 1) + self.epochs = self.args.epochs = math.ceil(self.args.time * 3600 / mean_epoch_time) + self._setup_scheduler() + self.scheduler.last_epoch = self.epoch # do not move + self.stop |= epoch >= self.epochs # stop if exceeded epochs + self.scheduler.step() + self.run_callbacks("on_fit_epoch_end") + torch.cuda.empty_cache() # clear GPU memory at end of epoch, may help reduce CUDA out of memory errors + + # Early Stopping + if RANK != -1: # if DDP training + broadcast_list = [self.stop if RANK == 0 else None] + dist.broadcast_object_list(broadcast_list, 0) # broadcast 'stop' to all ranks + self.stop = broadcast_list[0] + if self.stop: + break # must break all DDP ranks + epoch += 1 + + if RANK in (-1, 0): + # Do final val with best.pt + LOGGER.info( + f"\n{epoch - self.start_epoch + 1} epochs completed in " + f"{(time.time() - self.train_time_start) / 3600:.3f} hours." + ) + self.final_eval() + if self.args.plots: + self.plot_metrics() + self.run_callbacks("on_train_end") + torch.cuda.empty_cache() + self.run_callbacks("teardown") + + def save_model(self): + """Save model training checkpoints with additional metadata.""" + import pandas as pd # scope for faster startup + + metrics = {**self.metrics, **{"fitness": self.fitness}} + results = {k.strip(): v for k, v in pd.read_csv(self.csv).to_dict(orient="list").items()} + ckpt = { + "epoch": self.epoch, + "best_fitness": self.best_fitness, + "model": deepcopy(de_parallel(self.model)).half(), + "ema": deepcopy(self.ema.ema).half(), + "updates": self.ema.updates, + "optimizer": self.optimizer.state_dict(), + "train_args": vars(self.args), # save as dict + "train_metrics": metrics, + "train_results": results, + "date": datetime.now().isoformat(), + "version": __version__, + "license": "AGPL-3.0 (https://ultralytics.com/license)", + "docs": "https://docs.ultralytics.com", + } + + # Save last and best + torch.save(ckpt, self.last) + if self.best_fitness == self.fitness: + torch.save(ckpt, self.best) + if (self.save_period > 0) and (self.epoch > 0) and (self.epoch % self.save_period == 0): + torch.save(ckpt, self.wdir / f"epoch{self.epoch}.pt") + + @staticmethod + def get_dataset(data): + """ + Get train, val path from data dict if it exists. + + Returns None if data format is not recognized. + """ + return data["train"], data.get("val") or data.get("test") + + def setup_model(self): + """Load/create/download model for any task.""" + if isinstance(self.model, torch.nn.Module): # if model is loaded beforehand. No setup needed + return + + model, weights = self.model, None + ckpt = None + if str(model).endswith(".pt"): + weights, ckpt = attempt_load_one_weight(model) + cfg = ckpt["model"].yaml + else: + cfg = model + self.model = self.get_model(cfg=cfg, weights=weights, verbose=RANK == -1) # calls Model(cfg, weights) + return ckpt + + def optimizer_step(self): + """Perform a single step of the training optimizer with gradient clipping and EMA update.""" + self.scaler.unscale_(self.optimizer) # unscale gradients + torch.nn.utils.clip_grad_norm_(self.model.parameters(), max_norm=10.0) # clip gradients + self.scaler.step(self.optimizer) + self.scaler.update() + self.optimizer.zero_grad() + if self.ema: + self.ema.update(self.model) + + def preprocess_batch(self, batch): + """Allows custom preprocessing model inputs and ground truths depending on task type.""" + return batch + + def validate(self): + """ + Runs validation on test set using self.validator. + + The returned dict is expected to contain "fitness" key. + """ + metrics = self.validator(self) + fitness = metrics.pop("fitness", -self.loss.detach().cpu().numpy()) # use loss as fitness measure if not found + if not self.best_fitness or self.best_fitness < fitness: + self.best_fitness = fitness + return metrics, fitness + + def get_model(self, cfg=None, weights=None, verbose=True): + """Get model and raise NotImplementedError for loading cfg files.""" + raise NotImplementedError("This task trainer doesn't support loading cfg files") + + def get_validator(self): + """Returns a NotImplementedError when the get_validator function is called.""" + raise NotImplementedError("get_validator function not implemented in trainer") + + def get_dataloader(self, dataset_path, batch_size=16, rank=0, mode="train"): + """Returns dataloader derived from torch.data.Dataloader.""" + raise NotImplementedError("get_dataloader function not implemented in trainer") + + def build_dataset(self, img_path, mode="train", batch=None): + """Build dataset.""" + raise NotImplementedError("build_dataset function not implemented in trainer") + + def label_loss_items(self, loss_items=None, prefix="train"): + """ + Returns a loss dict with labelled training loss items tensor. + + Note: + This is not needed for classification but necessary for segmentation & detection + """ + return {"loss": loss_items} if loss_items is not None else ["loss"] + + def set_model_attributes(self): + """To set or update model parameters before training.""" + self.model.names = self.data["names"] + + def build_targets(self, preds, targets): + """Builds target tensors for training YOLO model.""" + pass + + def progress_string(self): + """Returns a string describing training progress.""" + return "" + + # TODO: may need to put these following functions into callback + def plot_training_samples(self, batch, ni): + """Plots training samples during YOLO training.""" + pass + + def plot_training_labels(self): + """Plots training labels for YOLO model.""" + pass + + def save_metrics(self, metrics): + """Saves training metrics to a CSV file.""" + keys, vals = list(metrics.keys()), list(metrics.values()) + n = len(metrics) + 1 # number of cols + s = "" if self.csv.exists() else (("%23s," * n % tuple(["epoch"] + keys)).rstrip(",") + "\n") # header + with open(self.csv, "a") as f: + f.write(s + ("%23.5g," * n % tuple([self.epoch + 1] + vals)).rstrip(",") + "\n") + + def plot_metrics(self): + """Plot and display metrics visually.""" + pass + + def on_plot(self, name, data=None): + """Registers plots (e.g. to be consumed in callbacks)""" + path = Path(name) + self.plots[path] = {"data": data, "timestamp": time.time()} + + def final_eval(self): + """Performs final evaluation and validation for object detection YOLO model.""" + for f in self.last, self.best: + if f.exists(): + strip_optimizer(f) # strip optimizers + if f is self.best: + LOGGER.info(f"\nValidating {f}...") + self.validator.args.plots = self.args.plots + self.metrics = self.validator(model=f) + self.metrics.pop("fitness", None) + self.run_callbacks("on_fit_epoch_end") + + def check_resume(self, overrides): + """Check if resume checkpoint exists and update arguments accordingly.""" + resume = self.args.resume + if resume: + try: + exists = isinstance(resume, (str, Path)) and Path(resume).exists() + last = Path(check_file(resume) if exists else get_latest_run()) + + # Check that resume data YAML exists, otherwise strip to force re-download of dataset + ckpt_args = attempt_load_weights(last).args + if not Path(ckpt_args["data"]).exists(): + ckpt_args["data"] = self.args.data + + resume = True + self.args = get_cfg(ckpt_args) + self.args.model = self.args.resume = str(last) # reinstate model + for k in "imgsz", "batch", "device": # allow arg updates to reduce memory or update device on resume + if k in overrides: + setattr(self.args, k, overrides[k]) + + except Exception as e: + raise FileNotFoundError( + "Resume checkpoint not found. Please pass a valid checkpoint to resume from, " + "i.e. 'yolo train resume model=path/to/last.pt'" + ) from e + self.resume = resume + + def resume_training(self, ckpt): + """Resume YOLO training from given epoch and best fitness.""" + if ckpt is None or not self.resume: + return + best_fitness = 0.0 + start_epoch = ckpt["epoch"] + 1 + if ckpt["optimizer"] is not None: + self.optimizer.load_state_dict(ckpt["optimizer"]) # optimizer + best_fitness = ckpt["best_fitness"] + if self.ema and ckpt.get("ema"): + self.ema.ema.load_state_dict(ckpt["ema"].float().state_dict()) # EMA + self.ema.updates = ckpt["updates"] + assert start_epoch > 0, ( + f"{self.args.model} training to {self.epochs} epochs is finished, nothing to resume.\n" + f"Start a new training without resuming, i.e. 'yolo train model={self.args.model}'" + ) + LOGGER.info(f"Resuming training {self.args.model} from epoch {start_epoch + 1} to {self.epochs} total epochs") + if self.epochs < start_epoch: + LOGGER.info( + f"{self.model} has been trained for {ckpt['epoch']} epochs. Fine-tuning for {self.epochs} more epochs." + ) + self.epochs += ckpt["epoch"] # finetune additional epochs + self.best_fitness = best_fitness + self.start_epoch = start_epoch + if start_epoch > (self.epochs - self.args.close_mosaic): + self._close_dataloader_mosaic() + + def _close_dataloader_mosaic(self): + """Update dataloaders to stop using mosaic augmentation.""" + if hasattr(self.train_loader.dataset, "mosaic"): + self.train_loader.dataset.mosaic = False + if hasattr(self.train_loader.dataset, "close_mosaic"): + LOGGER.info("Closing dataloader mosaic") + self.train_loader.dataset.close_mosaic(hyp=self.args) + + def build_optimizer(self, model, name="auto", lr=0.001, momentum=0.9, decay=1e-5, iterations=1e5): + """ + Constructs an optimizer for the given model, based on the specified optimizer name, learning rate, momentum, + weight decay, and number of iterations. + + Args: + model (torch.nn.Module): The model for which to build an optimizer. + name (str, optional): The name of the optimizer to use. If 'auto', the optimizer is selected + based on the number of iterations. Default: 'auto'. + lr (float, optional): The learning rate for the optimizer. Default: 0.001. + momentum (float, optional): The momentum factor for the optimizer. Default: 0.9. + decay (float, optional): The weight decay for the optimizer. Default: 1e-5. + iterations (float, optional): The number of iterations, which determines the optimizer if + name is 'auto'. Default: 1e5. + + Returns: + (torch.optim.Optimizer): The constructed optimizer. + """ + + g = [], [], [] # optimizer parameter groups + bn = tuple(v for k, v in nn.__dict__.items() if "Norm" in k) # normalization layers, i.e. BatchNorm2d() + if name == "auto": + LOGGER.info( + f"{colorstr('optimizer:')} 'optimizer=auto' found, " + f"ignoring 'lr0={self.args.lr0}' and 'momentum={self.args.momentum}' and " + f"determining best 'optimizer', 'lr0' and 'momentum' automatically... " + ) + nc = getattr(model, "nc", 10) # number of classes + lr_fit = round(0.002 * 5 / (4 + nc), 6) # lr0 fit equation to 6 decimal places + name, lr, momentum = ("SGD", 0.01, 0.9) if iterations > 10000 else ("AdamW", lr_fit, 0.9) + self.args.warmup_bias_lr = 0.0 # no higher than 0.01 for Adam + + for module_name, module in model.named_modules(): + for param_name, param in module.named_parameters(recurse=False): + fullname = f"{module_name}.{param_name}" if module_name else param_name + if "bias" in fullname: # bias (no decay) + g[2].append(param) + elif isinstance(module, bn): # weight (no decay) + g[1].append(param) + else: # weight (with decay) + g[0].append(param) + + if name in ("Adam", "Adamax", "AdamW", "NAdam", "RAdam"): + optimizer = getattr(optim, name, optim.Adam)(g[2], lr=lr, betas=(momentum, 0.999), weight_decay=0.0) + elif name == "RMSProp": + optimizer = optim.RMSprop(g[2], lr=lr, momentum=momentum) + elif name == "SGD": + optimizer = optim.SGD(g[2], lr=lr, momentum=momentum, nesterov=True) + else: + raise NotImplementedError( + f"Optimizer '{name}' not found in list of available optimizers " + f"[Adam, AdamW, NAdam, RAdam, RMSProp, SGD, auto]." + "To request support for addition optimizers please visit https://github.com/ultralytics/ultralytics." + ) + + optimizer.add_param_group({"params": g[0], "weight_decay": decay}) # add g0 with weight_decay + optimizer.add_param_group({"params": g[1], "weight_decay": 0.0}) # add g1 (BatchNorm2d weights) + LOGGER.info( + f"{colorstr('optimizer:')} {type(optimizer).__name__}(lr={lr}, momentum={momentum}) with parameter groups " + f'{len(g[1])} weight(decay=0.0), {len(g[0])} weight(decay={decay}), {len(g[2])} bias(decay=0.0)' + ) + return optimizer diff --git a/ultralytics/engine/tuner.py b/ultralytics/engine/tuner.py new file mode 100644 index 0000000..f4fe57e --- /dev/null +++ b/ultralytics/engine/tuner.py @@ -0,0 +1,242 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +""" +This module provides functionalities for hyperparameter tuning of the Ultralytics YOLO models for object detection, +instance segmentation, image classification, pose estimation, and multi-object tracking. + +Hyperparameter tuning is the process of systematically searching for the optimal set of hyperparameters +that yield the best model performance. This is particularly crucial in deep learning models like YOLO, +where small changes in hyperparameters can lead to significant differences in model accuracy and efficiency. + +Example: + Tune hyperparameters for YOLOv8n on COCO8 at imgsz=640 and epochs=30 for 300 tuning iterations. + ```python + from ultralytics import YOLO + + model = YOLO('yolov8n.pt') + model.tune(data='coco8.yaml', epochs=10, iterations=300, optimizer='AdamW', plots=False, save=False, val=False) + ``` +""" + +import random +import shutil +import subprocess +import time + +import numpy as np +import torch + +from ultralytics.cfg import get_cfg, get_save_dir +from ultralytics.utils import DEFAULT_CFG, LOGGER, callbacks, colorstr, remove_colorstr, yaml_print, yaml_save +from ultralytics.utils.plotting import plot_tune_results + + +class Tuner: + """ + Class responsible for hyperparameter tuning of YOLO models. + + The class evolves YOLO model hyperparameters over a given number of iterations + by mutating them according to the search space and retraining the model to evaluate their performance. + + Attributes: + space (dict): Hyperparameter search space containing bounds and scaling factors for mutation. + tune_dir (Path): Directory where evolution logs and results will be saved. + tune_csv (Path): Path to the CSV file where evolution logs are saved. + + Methods: + _mutate(hyp: dict) -> dict: + Mutates the given hyperparameters within the bounds specified in `self.space`. + + __call__(): + Executes the hyperparameter evolution across multiple iterations. + + Example: + Tune hyperparameters for YOLOv8n on COCO8 at imgsz=640 and epochs=30 for 300 tuning iterations. + ```python + from ultralytics import YOLO + + model = YOLO('yolov8n.pt') + model.tune(data='coco8.yaml', epochs=10, iterations=300, optimizer='AdamW', plots=False, save=False, val=False) + ``` + + Tune with custom search space. + ```python + from ultralytics import YOLO + + model = YOLO('yolov8n.pt') + model.tune(space={key1: val1, key2: val2}) # custom search space dictionary + ``` + """ + + def __init__(self, args=DEFAULT_CFG, _callbacks=None): + """ + Initialize the Tuner with configurations. + + Args: + args (dict, optional): Configuration for hyperparameter evolution. + """ + self.space = args.pop("space", None) or { # key: (min, max, gain(optional)) + # 'optimizer': tune.choice(['SGD', 'Adam', 'AdamW', 'NAdam', 'RAdam', 'RMSProp']), + "lr0": (1e-5, 1e-1), # initial learning rate (i.e. SGD=1E-2, Adam=1E-3) + "lrf": (0.0001, 0.1), # final OneCycleLR learning rate (lr0 * lrf) + "momentum": (0.7, 0.98, 0.3), # SGD momentum/Adam beta1 + "weight_decay": (0.0, 0.001), # optimizer weight decay 5e-4 + "warmup_epochs": (0.0, 5.0), # warmup epochs (fractions ok) + "warmup_momentum": (0.0, 0.95), # warmup initial momentum + "box": (1.0, 20.0), # box loss gain + "cls": (0.2, 4.0), # cls loss gain (scale with pixels) + "dfl": (0.4, 6.0), # dfl loss gain + "hsv_h": (0.0, 0.1), # image HSV-Hue augmentation (fraction) + "hsv_s": (0.0, 0.9), # image HSV-Saturation augmentation (fraction) + "hsv_v": (0.0, 0.9), # image HSV-Value augmentation (fraction) + "degrees": (0.0, 45.0), # image rotation (+/- deg) + "translate": (0.0, 0.9), # image translation (+/- fraction) + "scale": (0.0, 0.95), # image scale (+/- gain) + "shear": (0.0, 10.0), # image shear (+/- deg) + "perspective": (0.0, 0.001), # image perspective (+/- fraction), range 0-0.001 + "flipud": (0.0, 1.0), # image flip up-down (probability) + "fliplr": (0.0, 1.0), # image flip left-right (probability) + "bgr": (0.0, 1.0), # image channel bgr (probability) + "mosaic": (0.0, 1.0), # image mixup (probability) + "mixup": (0.0, 1.0), # image mixup (probability) + "copy_paste": (0.0, 1.0), # segment copy-paste (probability) + } + self.args = get_cfg(overrides=args) + self.tune_dir = get_save_dir(self.args, name="tune") + self.tune_csv = self.tune_dir / "tune_results.csv" + self.callbacks = _callbacks or callbacks.get_default_callbacks() + self.prefix = colorstr("Tuner: ") + callbacks.add_integration_callbacks(self) + LOGGER.info( + f"{self.prefix}Initialized Tuner instance with 'tune_dir={self.tune_dir}'\n" + f"{self.prefix}💡 Learn about tuning at https://docs.ultralytics.com/guides/hyperparameter-tuning" + ) + + def _mutate(self, parent="single", n=5, mutation=0.8, sigma=0.2): + """ + Mutates the hyperparameters based on bounds and scaling factors specified in `self.space`. + + Args: + parent (str): Parent selection method: 'single' or 'weighted'. + n (int): Number of parents to consider. + mutation (float): Probability of a parameter mutation in any given iteration. + sigma (float): Standard deviation for Gaussian random number generator. + + Returns: + (dict): A dictionary containing mutated hyperparameters. + """ + if self.tune_csv.exists(): # if CSV file exists: select best hyps and mutate + # Select parent(s) + x = np.loadtxt(self.tune_csv, ndmin=2, delimiter=",", skiprows=1) + fitness = x[:, 0] # first column + n = min(n, len(x)) # number of previous results to consider + x = x[np.argsort(-fitness)][:n] # top n mutations + w = x[:, 0] - x[:, 0].min() + 1e-6 # weights (sum > 0) + if parent == "single" or len(x) == 1: + # x = x[random.randint(0, n - 1)] # random selection + x = x[random.choices(range(n), weights=w)[0]] # weighted selection + elif parent == "weighted": + x = (x * w.reshape(n, 1)).sum(0) / w.sum() # weighted combination + + # Mutate + r = np.random # method + r.seed(int(time.time())) + g = np.array([v[2] if len(v) == 3 else 1.0 for k, v in self.space.items()]) # gains 0-1 + ng = len(self.space) + v = np.ones(ng) + while all(v == 1): # mutate until a change occurs (prevent duplicates) + v = (g * (r.random(ng) < mutation) * r.randn(ng) * r.random() * sigma + 1).clip(0.3, 3.0) + hyp = {k: float(x[i + 1] * v[i]) for i, k in enumerate(self.space.keys())} + else: + hyp = {k: getattr(self.args, k) for k in self.space.keys()} + + # Constrain to limits + for k, v in self.space.items(): + hyp[k] = max(hyp[k], v[0]) # lower limit + hyp[k] = min(hyp[k], v[1]) # upper limit + hyp[k] = round(hyp[k], 5) # significant digits + + return hyp + + def __call__(self, model=None, iterations=10, cleanup=True): + """ + Executes the hyperparameter evolution process when the Tuner instance is called. + + This method iterates through the number of iterations, performing the following steps in each iteration: + 1. Load the existing hyperparameters or initialize new ones. + 2. Mutate the hyperparameters using the `mutate` method. + 3. Train a YOLO model with the mutated hyperparameters. + 4. Log the fitness score and mutated hyperparameters to a CSV file. + + Args: + model (Model): A pre-initialized YOLO model to be used for training. + iterations (int): The number of generations to run the evolution for. + cleanup (bool): Whether to delete iteration weights to reduce storage space used during tuning. + + Note: + The method utilizes the `self.tune_csv` Path object to read and log hyperparameters and fitness scores. + Ensure this path is set correctly in the Tuner instance. + """ + + t0 = time.time() + best_save_dir, best_metrics = None, None + (self.tune_dir / "weights").mkdir(parents=True, exist_ok=True) + for i in range(iterations): + # Mutate hyperparameters + mutated_hyp = self._mutate() + LOGGER.info(f"{self.prefix}Starting iteration {i + 1}/{iterations} with hyperparameters: {mutated_hyp}") + + metrics = {} + train_args = {**vars(self.args), **mutated_hyp} + save_dir = get_save_dir(get_cfg(train_args)) + weights_dir = save_dir / "weights" + try: + # Train YOLO model with mutated hyperparameters (run in subprocess to avoid dataloader hang) + cmd = ["yolo", "train", *(f"{k}={v}" for k, v in train_args.items())] + return_code = subprocess.run(cmd, check=True).returncode + ckpt_file = weights_dir / ("best.pt" if (weights_dir / "best.pt").exists() else "last.pt") + metrics = torch.load(ckpt_file)["train_metrics"] + assert return_code == 0, "training failed" + + except Exception as e: + LOGGER.warning(f"WARNING ❌️ training failure for hyperparameter tuning iteration {i + 1}\n{e}") + + # Save results and mutated_hyp to CSV + fitness = metrics.get("fitness", 0.0) + log_row = [round(fitness, 5)] + [mutated_hyp[k] for k in self.space.keys()] + headers = "" if self.tune_csv.exists() else (",".join(["fitness"] + list(self.space.keys())) + "\n") + with open(self.tune_csv, "a") as f: + f.write(headers + ",".join(map(str, log_row)) + "\n") + + # Get best results + x = np.loadtxt(self.tune_csv, ndmin=2, delimiter=",", skiprows=1) + fitness = x[:, 0] # first column + best_idx = fitness.argmax() + best_is_current = best_idx == i + if best_is_current: + best_save_dir = save_dir + best_metrics = {k: round(v, 5) for k, v in metrics.items()} + for ckpt in weights_dir.glob("*.pt"): + shutil.copy2(ckpt, self.tune_dir / "weights") + elif cleanup: + shutil.rmtree(ckpt_file.parent) # remove iteration weights/ dir to reduce storage space + + # Plot tune results + plot_tune_results(self.tune_csv) + + # Save and print tune results + header = ( + f'{self.prefix}{i + 1}/{iterations} iterations complete ✅ ({time.time() - t0:.2f}s)\n' + f'{self.prefix}Results saved to {colorstr("bold", self.tune_dir)}\n' + f'{self.prefix}Best fitness={fitness[best_idx]} observed at iteration {best_idx + 1}\n' + f'{self.prefix}Best fitness metrics are {best_metrics}\n' + f'{self.prefix}Best fitness model is {best_save_dir}\n' + f'{self.prefix}Best fitness hyperparameters are printed below.\n' + ) + LOGGER.info("\n" + header) + data = {k: float(x[best_idx, i + 1]) for i, k in enumerate(self.space.keys())} + yaml_save( + self.tune_dir / "best_hyperparameters.yaml", + data=data, + header=remove_colorstr(header.replace(self.prefix, "# ")) + "\n", + ) + yaml_print(self.tune_dir / "best_hyperparameters.yaml") diff --git a/ultralytics/engine/validator.py b/ultralytics/engine/validator.py new file mode 100644 index 0000000..e6d3f67 --- /dev/null +++ b/ultralytics/engine/validator.py @@ -0,0 +1,345 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +""" +Check a model's accuracy on a test or val split of a dataset. + +Usage: + $ yolo mode=val model=yolov8n.pt data=coco128.yaml imgsz=640 + +Usage - formats: + $ yolo mode=val model=yolov8n.pt # PyTorch + yolov8n.torchscript # TorchScript + yolov8n.onnx # ONNX Runtime or OpenCV DNN with dnn=True + yolov8n_openvino_model # OpenVINO + yolov8n.engine # TensorRT + yolov8n.mlpackage # CoreML (macOS-only) + yolov8n_saved_model # TensorFlow SavedModel + yolov8n.pb # TensorFlow GraphDef + yolov8n.tflite # TensorFlow Lite + yolov8n_edgetpu.tflite # TensorFlow Edge TPU + yolov8n_paddle_model # PaddlePaddle + yolov8n_ncnn_model # NCNN +""" + +import json +import time +from pathlib import Path + +import numpy as np +import torch + +from ultralytics.cfg import get_cfg, get_save_dir +from ultralytics.data.utils import check_cls_dataset, check_det_dataset +from ultralytics.nn.autobackend import AutoBackend +from ultralytics.utils import LOGGER, TQDM, callbacks, colorstr, emojis +from ultralytics.utils.checks import check_imgsz +from ultralytics.utils.ops import Profile +from ultralytics.utils.torch_utils import de_parallel, select_device, smart_inference_mode + + +class BaseValidator: + """ + BaseValidator. + + A base class for creating validators. + + Attributes: + args (SimpleNamespace): Configuration for the validator. + dataloader (DataLoader): Dataloader to use for validation. + pbar (tqdm): Progress bar to update during validation. + model (nn.Module): Model to validate. + data (dict): Data dictionary. + device (torch.device): Device to use for validation. + batch_i (int): Current batch index. + training (bool): Whether the model is in training mode. + names (dict): Class names. + seen: Records the number of images seen so far during validation. + stats: Placeholder for statistics during validation. + confusion_matrix: Placeholder for a confusion matrix. + nc: Number of classes. + iouv: (torch.Tensor): IoU thresholds from 0.50 to 0.95 in spaces of 0.05. + jdict (dict): Dictionary to store JSON validation results. + speed (dict): Dictionary with keys 'preprocess', 'inference', 'loss', 'postprocess' and their respective + batch processing times in milliseconds. + save_dir (Path): Directory to save results. + plots (dict): Dictionary to store plots for visualization. + callbacks (dict): Dictionary to store various callback functions. + """ + + def __init__(self, dataloader=None, save_dir=None, pbar=None, args=None, _callbacks=None): + """ + Initializes a BaseValidator instance. + + Args: + dataloader (torch.utils.data.DataLoader): Dataloader to be used for validation. + save_dir (Path, optional): Directory to save results. + pbar (tqdm.tqdm): Progress bar for displaying progress. + args (SimpleNamespace): Configuration for the validator. + _callbacks (dict): Dictionary to store various callback functions. + """ + self.args = get_cfg(overrides=args) + self.dataloader = dataloader + self.pbar = pbar + self.stride = None + self.data = None + self.device = None + self.batch_i = None + self.training = True + self.names = None + self.seen = None + self.stats = None + self.confusion_matrix = None + self.nc = None + self.iouv = None + self.jdict = None + self.speed = {"preprocess": 0.0, "inference": 0.0, "loss": 0.0, "postprocess": 0.0} + + self.save_dir = save_dir or get_save_dir(self.args) + (self.save_dir / "labels" if self.args.save_txt else self.save_dir).mkdir(parents=True, exist_ok=True) + if self.args.conf is None: + self.args.conf = 0.001 # default conf=0.001 + self.args.imgsz = check_imgsz(self.args.imgsz, max_dim=1) + + self.plots = {} + self.callbacks = _callbacks or callbacks.get_default_callbacks() + + @smart_inference_mode() + def __call__(self, trainer=None, model=None): + """Supports validation of a pre-trained model if passed or a model being trained if trainer is passed (trainer + gets priority). + """ + self.training = trainer is not None + augment = self.args.augment and (not self.training) + if self.training: + self.device = trainer.device + self.data = trainer.data + # self.args.half = self.device.type != "cpu" # force FP16 val during training + model = trainer.ema.ema or trainer.model + model = model.half() if self.args.half else model.float() + # self.model = model + self.loss = torch.zeros_like(trainer.loss_items, device=trainer.device) + self.args.plots &= trainer.stopper.possible_stop or (trainer.epoch == trainer.epochs - 1) + model.eval() + else: + callbacks.add_integration_callbacks(self) + model = AutoBackend( + weights=model or self.args.model, + device=select_device(self.args.device, self.args.batch), + dnn=self.args.dnn, + data=self.args.data, + fp16=self.args.half, + ) + # self.model = model + self.device = model.device # update device + self.args.half = model.fp16 # update half + stride, pt, jit, engine = model.stride, model.pt, model.jit, model.engine + imgsz = check_imgsz(self.args.imgsz, stride=stride) + if engine: + self.args.batch = model.batch_size + elif not pt and not jit: + self.args.batch = 1 # export.py models default to batch-size 1 + LOGGER.info(f"Forcing batch=1 square inference (1,3,{imgsz},{imgsz}) for non-PyTorch models") + + if str(self.args.data).split(".")[-1] in ("yaml", "yml"): + self.data = check_det_dataset(self.args.data) + elif self.args.task == "classify": + self.data = check_cls_dataset(self.args.data, split=self.args.split) + else: + raise FileNotFoundError(emojis(f"Dataset '{self.args.data}' for task={self.args.task} not found ❌")) + + if self.device.type in ("cpu", "mps"): + self.args.workers = 0 # faster CPU val as time dominated by inference, not dataloading + if not pt: + self.args.rect = False + self.stride = model.stride # used in get_dataloader() for padding + self.dataloader = self.dataloader or self.get_dataloader(self.data.get(self.args.split), self.args.batch) + + model.eval() + model.warmup(imgsz=(1 if pt else self.args.batch, 3, imgsz, imgsz)) # warmup + + self.run_callbacks("on_val_start") + dt = ( + Profile(device=self.device), + Profile(device=self.device), + Profile(device=self.device), + Profile(device=self.device), + ) + bar = TQDM(self.dataloader, desc=self.get_desc(), total=len(self.dataloader)) + self.init_metrics(de_parallel(model)) + self.jdict = [] # empty before each val + for batch_i, batch in enumerate(bar): + self.run_callbacks("on_val_batch_start") + self.batch_i = batch_i + # Preprocess + with dt[0]: + batch = self.preprocess(batch) + + # Inference + with dt[1]: + preds = model(batch["img"], augment=augment) + + # Loss + with dt[2]: + if self.training: + self.loss += model.loss(batch, preds)[1] + + # Postprocess + with dt[3]: + preds = self.postprocess(preds) + + self.update_metrics(preds, batch) + if self.args.plots and batch_i < 3: + self.plot_val_samples(batch, batch_i) + self.plot_predictions(batch, preds, batch_i) + + self.run_callbacks("on_val_batch_end") + stats = self.get_stats() + self.check_stats(stats) + self.speed = dict(zip(self.speed.keys(), (x.t / len(self.dataloader.dataset) * 1e3 for x in dt))) + self.finalize_metrics() + if not (self.args.save_json and self.is_coco and len(self.jdict)): + self.print_results() + self.run_callbacks("on_val_end") + if self.training: + model.float() + if self.args.save_json and self.jdict: + with open(str(self.save_dir / "predictions.json"), "w") as f: + LOGGER.info(f"Saving {f.name}...") + json.dump(self.jdict, f) # flatten and save + stats = self.eval_json(stats) # update stats + stats['fitness'] = stats['metrics/mAP50-95(B)'] + results = {**stats, **trainer.label_loss_items(self.loss.cpu() / len(self.dataloader), prefix="val")} + return {k: round(float(v), 5) for k, v in results.items()} # return results as 5 decimal place floats + else: + LOGGER.info( + "Speed: %.1fms preprocess, %.1fms inference, %.1fms loss, %.1fms postprocess per image" + % tuple(self.speed.values()) + ) + if self.args.save_json and self.jdict: + with open(str(self.save_dir / "predictions.json"), "w") as f: + LOGGER.info(f"Saving {f.name}...") + json.dump(self.jdict, f) # flatten and save + stats = self.eval_json(stats) # update stats + if self.args.plots or self.args.save_json: + LOGGER.info(f"Results saved to {colorstr('bold', self.save_dir)}") + return stats + + def match_predictions(self, pred_classes, true_classes, iou, use_scipy=False): + """ + Matches predictions to ground truth objects (pred_classes, true_classes) using IoU. + + Args: + pred_classes (torch.Tensor): Predicted class indices of shape(N,). + true_classes (torch.Tensor): Target class indices of shape(M,). + iou (torch.Tensor): An NxM tensor containing the pairwise IoU values for predictions and ground of truth + use_scipy (bool): Whether to use scipy for matching (more precise). + + Returns: + (torch.Tensor): Correct tensor of shape(N,10) for 10 IoU thresholds. + """ + # Dx10 matrix, where D - detections, 10 - IoU thresholds + correct = np.zeros((pred_classes.shape[0], self.iouv.shape[0])).astype(bool) + # LxD matrix where L - labels (rows), D - detections (columns) + correct_class = true_classes[:, None] == pred_classes + iou = iou * correct_class # zero out the wrong classes + iou = iou.cpu().numpy() + for i, threshold in enumerate(self.iouv.cpu().tolist()): + if use_scipy: + # WARNING: known issue that reduces mAP in https://github.com/ultralytics/ultralytics/pull/4708 + import scipy # scope import to avoid importing for all commands + + cost_matrix = iou * (iou >= threshold) + if cost_matrix.any(): + labels_idx, detections_idx = scipy.optimize.linear_sum_assignment(cost_matrix, maximize=True) + valid = cost_matrix[labels_idx, detections_idx] > 0 + if valid.any(): + correct[detections_idx[valid], i] = True + else: + matches = np.nonzero(iou >= threshold) # IoU > threshold and classes match + matches = np.array(matches).T + if matches.shape[0]: + if matches.shape[0] > 1: + matches = matches[iou[matches[:, 0], matches[:, 1]].argsort()[::-1]] + matches = matches[np.unique(matches[:, 1], return_index=True)[1]] + # matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 0], return_index=True)[1]] + correct[matches[:, 1].astype(int), i] = True + return torch.tensor(correct, dtype=torch.bool, device=pred_classes.device) + + def add_callback(self, event: str, callback): + """Appends the given callback.""" + self.callbacks[event].append(callback) + + def run_callbacks(self, event: str): + """Runs all callbacks associated with a specified event.""" + for callback in self.callbacks.get(event, []): + callback(self) + + def get_dataloader(self, dataset_path, batch_size): + """Get data loader from dataset path and batch size.""" + raise NotImplementedError("get_dataloader function not implemented for this validator") + + def build_dataset(self, img_path): + """Build dataset.""" + raise NotImplementedError("build_dataset function not implemented in validator") + + def preprocess(self, batch): + """Preprocesses an input batch.""" + return batch + + def postprocess(self, preds): + """Describes and summarizes the purpose of 'postprocess()' but no details mentioned.""" + return preds + + def init_metrics(self, model): + """Initialize performance metrics for the YOLO model.""" + pass + + def update_metrics(self, preds, batch): + """Updates metrics based on predictions and batch.""" + pass + + def finalize_metrics(self, *args, **kwargs): + """Finalizes and returns all metrics.""" + pass + + def get_stats(self): + """Returns statistics about the model's performance.""" + return {} + + def check_stats(self, stats): + """Checks statistics.""" + pass + + def print_results(self): + """Prints the results of the model's predictions.""" + pass + + def get_desc(self): + """Get description of the YOLO model.""" + pass + + @property + def metric_keys(self): + """Returns the metric keys used in YOLO training/validation.""" + return [] + + def on_plot(self, name, data=None): + """Registers plots (e.g. to be consumed in callbacks)""" + self.plots[Path(name)] = {"data": data, "timestamp": time.time()} + + # TODO: may need to put these following functions into callback + def plot_val_samples(self, batch, ni): + """Plots validation samples during training.""" + pass + + def plot_predictions(self, batch, preds, ni): + """Plots YOLO model predictions on batch images.""" + pass + + def pred_to_json(self, preds, batch): + """Convert predictions to JSON format.""" + pass + + def eval_json(self, stats): + """Evaluate and return JSON format of prediction statistics.""" + pass diff --git a/ultralytics/hub/__init__.py b/ultralytics/hub/__init__.py new file mode 100644 index 0000000..4ea2fff --- /dev/null +++ b/ultralytics/hub/__init__.py @@ -0,0 +1,128 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import requests + +from ultralytics.data.utils import HUBDatasetStats +from ultralytics.hub.auth import Auth +from ultralytics.hub.utils import HUB_API_ROOT, HUB_WEB_ROOT, PREFIX +from ultralytics.utils import LOGGER, SETTINGS, checks + + +def login(api_key: str = None, save=True) -> bool: + """ + Log in to the Ultralytics HUB API using the provided API key. + + The session is not stored; a new session is created when needed using the saved SETTINGS or the HUB_API_KEY + environment variable if successfully authenticated. + + Args: + api_key (str, optional): API key to use for authentication. + If not provided, it will be retrieved from SETTINGS or HUB_API_KEY environment variable. + save (bool, optional): Whether to save the API key to SETTINGS if authentication is successful. + + Returns: + (bool): True if authentication is successful, False otherwise. + """ + checks.check_requirements("hub-sdk>=0.0.6") + from hub_sdk import HUBClient + + api_key_url = f"{HUB_WEB_ROOT}/settings?tab=api+keys" # set the redirect URL + saved_key = SETTINGS.get("api_key") + active_key = api_key or saved_key + credentials = {"api_key": active_key} if active_key and active_key != "" else None # set credentials + + client = HUBClient(credentials) # initialize HUBClient + + if client.authenticated: + # Successfully authenticated with HUB + + if save and client.api_key != saved_key: + SETTINGS.update({"api_key": client.api_key}) # update settings with valid API key + + # Set message based on whether key was provided or retrieved from settings + log_message = ( + "New authentication successful ✅" if client.api_key == api_key or not credentials else "Authenticated ✅" + ) + LOGGER.info(f"{PREFIX}{log_message}") + + return True + else: + # Failed to authenticate with HUB + LOGGER.info(f"{PREFIX}Get API key from {api_key_url} and then run 'yolo hub login API_KEY'") + return False + + +def logout(): + """ + Log out of Ultralytics HUB by removing the API key from the settings file. To log in again, use 'yolo hub login'. + + Example: + ```python + from ultralytics import hub + + hub.logout() + ``` + """ + SETTINGS["api_key"] = "" + SETTINGS.save() + LOGGER.info(f"{PREFIX}logged out ✅. To log in again, use 'yolo hub login'.") + + +def reset_model(model_id=""): + """Reset a trained model to an untrained state.""" + r = requests.post(f"{HUB_API_ROOT}/model-reset", json={"modelId": model_id}, headers={"x-api-key": Auth().api_key}) + if r.status_code == 200: + LOGGER.info(f"{PREFIX}Model reset successfully") + return + LOGGER.warning(f"{PREFIX}Model reset failure {r.status_code} {r.reason}") + + +def export_fmts_hub(): + """Returns a list of HUB-supported export formats.""" + from ultralytics.engine.exporter import export_formats + + return list(export_formats()["Argument"][1:]) + ["ultralytics_tflite", "ultralytics_coreml"] + + +def export_model(model_id="", format="torchscript"): + """Export a model to all formats.""" + assert format in export_fmts_hub(), f"Unsupported export format '{format}', valid formats are {export_fmts_hub()}" + r = requests.post( + f"{HUB_API_ROOT}/v1/models/{model_id}/export", json={"format": format}, headers={"x-api-key": Auth().api_key} + ) + assert r.status_code == 200, f"{PREFIX}{format} export failure {r.status_code} {r.reason}" + LOGGER.info(f"{PREFIX}{format} export started ✅") + + +def get_export(model_id="", format="torchscript"): + """Get an exported model dictionary with download URL.""" + assert format in export_fmts_hub(), f"Unsupported export format '{format}', valid formats are {export_fmts_hub()}" + r = requests.post( + f"{HUB_API_ROOT}/get-export", + json={"apiKey": Auth().api_key, "modelId": model_id, "format": format}, + headers={"x-api-key": Auth().api_key}, + ) + assert r.status_code == 200, f"{PREFIX}{format} get_export failure {r.status_code} {r.reason}" + return r.json() + + +def check_dataset(path="", task="detect"): + """ + Function for error-checking HUB dataset Zip file before upload. It checks a dataset for errors before it is uploaded + to the HUB. Usage examples are given below. + + Args: + path (str, optional): Path to data.zip (with data.yaml inside data.zip). Defaults to ''. + task (str, optional): Dataset task. Options are 'detect', 'segment', 'pose', 'classify'. Defaults to 'detect'. + + Example: + ```python + from ultralytics.hub import check_dataset + + check_dataset('path/to/coco8.zip', task='detect') # detect dataset + check_dataset('path/to/coco8-seg.zip', task='segment') # segment dataset + check_dataset('path/to/coco8-pose.zip', task='pose') # pose dataset + ``` + """ + HUBDatasetStats(path=path, task=task).get_json() + LOGGER.info(f"Checks completed correctly ✅. Upload this dataset to {HUB_WEB_ROOT}/datasets/.") diff --git a/ultralytics/hub/__pycache__/__init__.cpython-312.pyc b/ultralytics/hub/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..7b0f566 Binary files /dev/null and b/ultralytics/hub/__pycache__/__init__.cpython-312.pyc differ diff --git a/ultralytics/hub/__pycache__/__init__.cpython-39.pyc b/ultralytics/hub/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..3bb4eff Binary files /dev/null and b/ultralytics/hub/__pycache__/__init__.cpython-39.pyc differ diff --git a/ultralytics/hub/__pycache__/auth.cpython-312.pyc b/ultralytics/hub/__pycache__/auth.cpython-312.pyc new file mode 100644 index 0000000..4b61c13 Binary files /dev/null and b/ultralytics/hub/__pycache__/auth.cpython-312.pyc differ diff --git a/ultralytics/hub/__pycache__/auth.cpython-39.pyc b/ultralytics/hub/__pycache__/auth.cpython-39.pyc new file mode 100644 index 0000000..1f909c7 Binary files /dev/null and b/ultralytics/hub/__pycache__/auth.cpython-39.pyc differ diff --git a/ultralytics/hub/__pycache__/utils.cpython-312.pyc b/ultralytics/hub/__pycache__/utils.cpython-312.pyc new file mode 100644 index 0000000..81d7528 Binary files /dev/null and b/ultralytics/hub/__pycache__/utils.cpython-312.pyc differ diff --git a/ultralytics/hub/__pycache__/utils.cpython-39.pyc b/ultralytics/hub/__pycache__/utils.cpython-39.pyc new file mode 100644 index 0000000..007109c Binary files /dev/null and b/ultralytics/hub/__pycache__/utils.cpython-39.pyc differ diff --git a/ultralytics/hub/auth.py b/ultralytics/hub/auth.py new file mode 100644 index 0000000..6ede303 --- /dev/null +++ b/ultralytics/hub/auth.py @@ -0,0 +1,136 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import requests + +from ultralytics.hub.utils import HUB_API_ROOT, HUB_WEB_ROOT, PREFIX, request_with_credentials +from ultralytics.utils import LOGGER, SETTINGS, emojis, is_colab + +API_KEY_URL = f"{HUB_WEB_ROOT}/settings?tab=api+keys" + + +class Auth: + """ + Manages authentication processes including API key handling, cookie-based authentication, and header generation. + + The class supports different methods of authentication: + 1. Directly using an API key. + 2. Authenticating using browser cookies (specifically in Google Colab). + 3. Prompting the user to enter an API key. + + Attributes: + id_token (str or bool): Token used for identity verification, initialized as False. + api_key (str or bool): API key for authentication, initialized as False. + model_key (bool): Placeholder for model key, initialized as False. + """ + + id_token = api_key = model_key = False + + def __init__(self, api_key="", verbose=False): + """ + Initialize the Auth class with an optional API key. + + Args: + api_key (str, optional): May be an API key or a combination API key and model ID, i.e. key_id + """ + # Split the input API key in case it contains a combined key_model and keep only the API key part + api_key = api_key.split("_")[0] + + # Set API key attribute as value passed or SETTINGS API key if none passed + self.api_key = api_key or SETTINGS.get("api_key", "") + + # If an API key is provided + if self.api_key: + # If the provided API key matches the API key in the SETTINGS + if self.api_key == SETTINGS.get("api_key"): + # Log that the user is already logged in + if verbose: + LOGGER.info(f"{PREFIX}Authenticated ✅") + return + else: + # Attempt to authenticate with the provided API key + success = self.authenticate() + # If the API key is not provided and the environment is a Google Colab notebook + elif is_colab(): + # Attempt to authenticate using browser cookies + success = self.auth_with_cookies() + else: + # Request an API key + success = self.request_api_key() + + # Update SETTINGS with the new API key after successful authentication + if success: + SETTINGS.update({"api_key": self.api_key}) + # Log that the new login was successful + if verbose: + LOGGER.info(f"{PREFIX}New authentication successful ✅") + elif verbose: + LOGGER.info(f"{PREFIX}Get API key from {API_KEY_URL} and then run 'yolo hub login API_KEY'") + + def request_api_key(self, max_attempts=3): + """ + Prompt the user to input their API key. + + Returns the model ID. + """ + import getpass + + for attempts in range(max_attempts): + LOGGER.info(f"{PREFIX}Login. Attempt {attempts + 1} of {max_attempts}") + input_key = getpass.getpass(f"Enter API key from {API_KEY_URL} ") + self.api_key = input_key.split("_")[0] # remove model id if present + if self.authenticate(): + return True + raise ConnectionError(emojis(f"{PREFIX}Failed to authenticate ❌")) + + def authenticate(self) -> bool: + """ + Attempt to authenticate with the server using either id_token or API key. + + Returns: + (bool): True if authentication is successful, False otherwise. + """ + try: + if header := self.get_auth_header(): + r = requests.post(f"{HUB_API_ROOT}/v1/auth", headers=header) + if not r.json().get("success", False): + raise ConnectionError("Unable to authenticate.") + return True + raise ConnectionError("User has not authenticated locally.") + except ConnectionError: + self.id_token = self.api_key = False # reset invalid + LOGGER.warning(f"{PREFIX}Invalid API key ⚠️") + return False + + def auth_with_cookies(self) -> bool: + """ + Attempt to fetch authentication via cookies and set id_token. User must be logged in to HUB and running in a + supported browser. + + Returns: + (bool): True if authentication is successful, False otherwise. + """ + if not is_colab(): + return False # Currently only works with Colab + try: + authn = request_with_credentials(f"{HUB_API_ROOT}/v1/auth/auto") + if authn.get("success", False): + self.id_token = authn.get("data", {}).get("idToken", None) + self.authenticate() + return True + raise ConnectionError("Unable to fetch browser authentication details.") + except ConnectionError: + self.id_token = False # reset invalid + return False + + def get_auth_header(self): + """ + Get the authentication header for making API requests. + + Returns: + (dict): The authentication header if id_token or API key is set, None otherwise. + """ + if self.id_token: + return {"authorization": f"Bearer {self.id_token}"} + elif self.api_key: + return {"x-api-key": self.api_key} + # else returns None diff --git a/ultralytics/hub/session.py b/ultralytics/hub/session.py new file mode 100644 index 0000000..ebde7aa --- /dev/null +++ b/ultralytics/hub/session.py @@ -0,0 +1,355 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import threading +import time +from http import HTTPStatus +from pathlib import Path + +import requests + +from ultralytics.hub.utils import HUB_WEB_ROOT, HELP_MSG, PREFIX, TQDM +from ultralytics.utils import LOGGER, SETTINGS, __version__, checks, emojis, is_colab +from ultralytics.utils.errors import HUBModelError + +AGENT_NAME = f"python-{__version__}-colab" if is_colab() else f"python-{__version__}-local" + + +class HUBTrainingSession: + """ + HUB training session for Ultralytics HUB YOLO models. Handles model initialization, heartbeats, and checkpointing. + + Attributes: + agent_id (str): Identifier for the instance communicating with the server. + model_id (str): Identifier for the YOLO model being trained. + model_url (str): URL for the model in Ultralytics HUB. + api_url (str): API URL for the model in Ultralytics HUB. + auth_header (dict): Authentication header for the Ultralytics HUB API requests. + rate_limits (dict): Rate limits for different API calls (in seconds). + timers (dict): Timers for rate limiting. + metrics_queue (dict): Queue for the model's metrics. + model (dict): Model data fetched from Ultralytics HUB. + alive (bool): Indicates if the heartbeat loop is active. + """ + + def __init__(self, identifier): + """ + Initialize the HUBTrainingSession with the provided model identifier. + + Args: + identifier (str): Model identifier used to initialize the HUB training session. + It can be a URL string or a model key with specific format. + + Raises: + ValueError: If the provided model identifier is invalid. + ConnectionError: If connecting with global API key is not supported. + ModuleNotFoundError: If hub-sdk package is not installed. + """ + from hub_sdk import HUBClient + + self.rate_limits = { + "metrics": 3.0, + "ckpt": 900.0, + "heartbeat": 300.0, + } # rate limits (seconds) + self.metrics_queue = {} # holds metrics for each epoch until upload + self.metrics_upload_failed_queue = {} # holds metrics for each epoch if upload failed + self.timers = {} # holds timers in ultralytics/utils/callbacks/hub.py + + # Parse input + api_key, model_id, self.filename = self._parse_identifier(identifier) + + # Get credentials + active_key = api_key or SETTINGS.get("api_key") + credentials = {"api_key": active_key} if active_key else None # set credentials + + # Initialize client + self.client = HUBClient(credentials) + + if model_id: + self.load_model(model_id) # load existing model + else: + self.model = self.client.model() # load empty model + + def load_model(self, model_id): + """Loads an existing model from Ultralytics HUB using the provided model identifier.""" + self.model = self.client.model(model_id) + if not self.model.data: # then model does not exist + raise ValueError(emojis("❌ The specified HUB model does not exist")) # TODO: improve error handling + + self.model_url = f"{HUB_WEB_ROOT}/models/{self.model.id}" + + self._set_train_args() + + # Start heartbeats for HUB to monitor agent + self.model.start_heartbeat(self.rate_limits["heartbeat"]) + LOGGER.info(f"{PREFIX}View model at {self.model_url} 🚀") + + def create_model(self, model_args): + """Initializes a HUB training session with the specified model identifier.""" + payload = { + "config": { + "batchSize": model_args.get("batch", -1), + "epochs": model_args.get("epochs", 300), + "imageSize": model_args.get("imgsz", 640), + "patience": model_args.get("patience", 100), + "device": model_args.get("device", ""), + "cache": model_args.get("cache", "ram"), + }, + "dataset": {"name": model_args.get("data")}, + "lineage": { + "architecture": { + "name": self.filename.replace(".pt", "").replace(".yaml", ""), + }, + "parent": {}, + }, + "meta": {"name": self.filename}, + } + + if self.filename.endswith(".pt"): + payload["lineage"]["parent"]["name"] = self.filename + + self.model.create_model(payload) + + # Model could not be created + # TODO: improve error handling + if not self.model.id: + return + + self.model_url = f"{HUB_WEB_ROOT}/models/{self.model.id}" + + # Start heartbeats for HUB to monitor agent + self.model.start_heartbeat(self.rate_limits["heartbeat"]) + + LOGGER.info(f"{PREFIX}View model at {self.model_url} 🚀") + + def _parse_identifier(self, identifier): + """ + Parses the given identifier to determine the type of identifier and extract relevant components. + + The method supports different identifier formats: + - A HUB URL, which starts with HUB_WEB_ROOT followed by '/models/' + - An identifier containing an API key and a model ID separated by an underscore + - An identifier that is solely a model ID of a fixed length + - A local filename that ends with '.pt' or '.yaml' + + Args: + identifier (str): The identifier string to be parsed. + + Returns: + (tuple): A tuple containing the API key, model ID, and filename as applicable. + + Raises: + HUBModelError: If the identifier format is not recognized. + """ + + # Initialize variables + api_key, model_id, filename = None, None, None + + # Check if identifier is a HUB URL + if identifier.startswith(f"{HUB_WEB_ROOT}/models/"): + # Extract the model_id after the HUB_WEB_ROOT URL + model_id = identifier.split(f"{HUB_WEB_ROOT}/models/")[-1] + else: + # Split the identifier based on underscores only if it's not a HUB URL + parts = identifier.split("_") + + # Check if identifier is in the format of API key and model ID + if len(parts) == 2 and len(parts[0]) == 42 and len(parts[1]) == 20: + api_key, model_id = parts + # Check if identifier is a single model ID + elif len(parts) == 1 and len(parts[0]) == 20: + model_id = parts[0] + # Check if identifier is a local filename + elif identifier.endswith(".pt") or identifier.endswith(".yaml"): + filename = identifier + else: + raise HUBModelError( + f"model='{identifier}' could not be parsed. Check format is correct. " + f"Supported formats are Ultralytics HUB URL, apiKey_modelId, modelId, local pt or yaml file." + ) + + return api_key, model_id, filename + + def _set_train_args(self): + """ + Initializes training arguments and creates a model entry on the Ultralytics HUB. + + This method sets up training arguments based on the model's state and updates them with any additional + arguments provided. It handles different states of the model, such as whether it's resumable, pretrained, + or requires specific file setup. + + Raises: + ValueError: If the model is already trained, if required dataset information is missing, or if there are + issues with the provided training arguments. + """ + if self.model.is_trained(): + raise ValueError(emojis(f"Model is already trained and uploaded to {self.model_url} 🚀")) + + if self.model.is_resumable(): + # Model has saved weights + self.train_args = {"data": self.model.get_dataset_url(), "resume": True} + self.model_file = self.model.get_weights_url("last") + else: + # Model has no saved weights + self.train_args = self.model.data.get("train_args") # new response + + # Set the model file as either a *.pt or *.yaml file + self.model_file = ( + self.model.get_weights_url("parent") if self.model.is_pretrained() else self.model.get_architecture() + ) + + if "data" not in self.train_args: + # RF bug - datasets are sometimes not exported + raise ValueError("Dataset may still be processing. Please wait a minute and try again.") + + self.model_file = checks.check_yolov5u_filename(self.model_file, verbose=False) # YOLOv5->YOLOv5u + self.model_id = self.model.id + + def request_queue( + self, + request_func, + retry=3, + timeout=30, + thread=True, + verbose=True, + progress_total=None, + *args, + **kwargs, + ): + def retry_request(): + """Attempts to call `request_func` with retries, timeout, and optional threading.""" + t0 = time.time() # Record the start time for the timeout + for i in range(retry + 1): + if (time.time() - t0) > timeout: + LOGGER.warning(f"{PREFIX}Timeout for request reached. {HELP_MSG}") + break # Timeout reached, exit loop + + response = request_func(*args, **kwargs) + if response is None: + LOGGER.warning(f"{PREFIX}Received no response from the request. {HELP_MSG}") + time.sleep(2**i) # Exponential backoff before retrying + continue # Skip further processing and retry + + if progress_total: + self._show_upload_progress(progress_total, response) + + if HTTPStatus.OK <= response.status_code < HTTPStatus.MULTIPLE_CHOICES: + # if request related to metrics upload + if kwargs.get("metrics"): + self.metrics_upload_failed_queue = {} + return response # Success, no need to retry + + if i == 0: + # Initial attempt, check status code and provide messages + message = self._get_failure_message(response, retry, timeout) + + if verbose: + LOGGER.warning(f"{PREFIX}{message} {HELP_MSG} ({response.status_code})") + + if not self._should_retry(response.status_code): + LOGGER.warning(f"{PREFIX}Request failed. {HELP_MSG} ({response.status_code}") + break # Not an error that should be retried, exit loop + + time.sleep(2**i) # Exponential backoff for retries + + # if request related to metrics upload and exceed retries + if response is None and kwargs.get("metrics"): + self.metrics_upload_failed_queue.update(kwargs.get("metrics", None)) + + return response + + if thread: + # Start a new thread to run the retry_request function + threading.Thread(target=retry_request, daemon=True).start() + else: + # If running in the main thread, call retry_request directly + return retry_request() + + def _should_retry(self, status_code): + """Determines if a request should be retried based on the HTTP status code.""" + retry_codes = { + HTTPStatus.REQUEST_TIMEOUT, + HTTPStatus.BAD_GATEWAY, + HTTPStatus.GATEWAY_TIMEOUT, + } + return status_code in retry_codes + + def _get_failure_message(self, response: requests.Response, retry: int, timeout: int): + """ + Generate a retry message based on the response status code. + + Args: + response: The HTTP response object. + retry: The number of retry attempts allowed. + timeout: The maximum timeout duration. + + Returns: + (str): The retry message. + """ + if self._should_retry(response.status_code): + return f"Retrying {retry}x for {timeout}s." if retry else "" + elif response.status_code == HTTPStatus.TOO_MANY_REQUESTS: # rate limit + headers = response.headers + return ( + f"Rate limit reached ({headers['X-RateLimit-Remaining']}/{headers['X-RateLimit-Limit']}). " + f"Please retry after {headers['Retry-After']}s." + ) + else: + try: + return response.json().get("message", "No JSON message.") + except AttributeError: + return "Unable to read JSON." + + def upload_metrics(self): + """Upload model metrics to Ultralytics HUB.""" + return self.request_queue(self.model.upload_metrics, metrics=self.metrics_queue.copy(), thread=True) + + def upload_model( + self, + epoch: int, + weights: str, + is_best: bool = False, + map: float = 0.0, + final: bool = False, + ) -> None: + """ + Upload a model checkpoint to Ultralytics HUB. + + Args: + epoch (int): The current training epoch. + weights (str): Path to the model weights file. + is_best (bool): Indicates if the current model is the best one so far. + map (float): Mean average precision of the model. + final (bool): Indicates if the model is the final model after training. + """ + if Path(weights).is_file(): + progress_total = Path(weights).stat().st_size if final else None # Only show progress if final + self.request_queue( + self.model.upload_model, + epoch=epoch, + weights=weights, + is_best=is_best, + map=map, + final=final, + retry=10, + timeout=3600, + thread=not final, + progress_total=progress_total, + ) + else: + LOGGER.warning(f"{PREFIX}WARNING ⚠️ Model upload issue. Missing model {weights}.") + + def _show_upload_progress(self, content_length: int, response: requests.Response) -> None: + """ + Display a progress bar to track the upload progress of a file download. + + Args: + content_length (int): The total size of the content to be downloaded in bytes. + response (requests.Response): The response object from the file download request. + + Returns: + None + """ + with TQDM(total=content_length, unit="B", unit_scale=True, unit_divisor=1024) as pbar: + for data in response.iter_content(chunk_size=1024): + pbar.update(len(data)) diff --git a/ultralytics/hub/utils.py b/ultralytics/hub/utils.py new file mode 100644 index 0000000..5c00076 --- /dev/null +++ b/ultralytics/hub/utils.py @@ -0,0 +1,247 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import os +import platform +import random +import sys +import threading +import time +from pathlib import Path + +import requests + +from ultralytics.utils import ( + ENVIRONMENT, + LOGGER, + ONLINE, + RANK, + SETTINGS, + TESTS_RUNNING, + TQDM, + TryExcept, + __version__, + colorstr, + get_git_origin_url, + is_colab, + is_git_dir, + is_pip_package, +) +from ultralytics.utils.downloads import GITHUB_ASSETS_NAMES + +HUB_API_ROOT = os.environ.get("ULTRALYTICS_HUB_API", "https://api.ultralytics.com") +HUB_WEB_ROOT = os.environ.get("ULTRALYTICS_HUB_WEB", "https://hub.ultralytics.com") + +PREFIX = colorstr("Ultralytics HUB: ") +HELP_MSG = "If this issue persists please visit https://github.com/ultralytics/hub/issues for assistance." + + +def request_with_credentials(url: str) -> any: + """ + Make an AJAX request with cookies attached in a Google Colab environment. + + Args: + url (str): The URL to make the request to. + + Returns: + (any): The response data from the AJAX request. + + Raises: + OSError: If the function is not run in a Google Colab environment. + """ + if not is_colab(): + raise OSError("request_with_credentials() must run in a Colab environment") + from google.colab import output # noqa + from IPython import display # noqa + + display.display( + display.Javascript( + """ + window._hub_tmp = new Promise((resolve, reject) => { + const timeout = setTimeout(() => reject("Failed authenticating existing browser session"), 5000) + fetch("%s", { + method: 'POST', + credentials: 'include' + }) + .then((response) => resolve(response.json())) + .then((json) => { + clearTimeout(timeout); + }).catch((err) => { + clearTimeout(timeout); + reject(err); + }); + }); + """ + % url + ) + ) + return output.eval_js("_hub_tmp") + + +def requests_with_progress(method, url, **kwargs): + """ + Make an HTTP request using the specified method and URL, with an optional progress bar. + + Args: + method (str): The HTTP method to use (e.g. 'GET', 'POST'). + url (str): The URL to send the request to. + **kwargs (any): Additional keyword arguments to pass to the underlying `requests.request` function. + + Returns: + (requests.Response): The response object from the HTTP request. + + Note: + - If 'progress' is set to True, the progress bar will display the download progress for responses with a known + content length. + - If 'progress' is a number then progress bar will display assuming content length = progress. + """ + progress = kwargs.pop("progress", False) + if not progress: + return requests.request(method, url, **kwargs) + response = requests.request(method, url, stream=True, **kwargs) + total = int(response.headers.get("content-length", 0) if isinstance(progress, bool) else progress) # total size + try: + pbar = TQDM(total=total, unit="B", unit_scale=True, unit_divisor=1024) + for data in response.iter_content(chunk_size=1024): + pbar.update(len(data)) + pbar.close() + except requests.exceptions.ChunkedEncodingError: # avoid 'Connection broken: IncompleteRead' warnings + response.close() + return response + + +def smart_request(method, url, retry=3, timeout=30, thread=True, code=-1, verbose=True, progress=False, **kwargs): + """ + Makes an HTTP request using the 'requests' library, with exponential backoff retries up to a specified timeout. + + Args: + method (str): The HTTP method to use for the request. Choices are 'post' and 'get'. + url (str): The URL to make the request to. + retry (int, optional): Number of retries to attempt before giving up. Default is 3. + timeout (int, optional): Timeout in seconds after which the function will give up retrying. Default is 30. + thread (bool, optional): Whether to execute the request in a separate daemon thread. Default is True. + code (int, optional): An identifier for the request, used for logging purposes. Default is -1. + verbose (bool, optional): A flag to determine whether to print out to console or not. Default is True. + progress (bool, optional): Whether to show a progress bar during the request. Default is False. + **kwargs (any): Keyword arguments to be passed to the requests function specified in method. + + Returns: + (requests.Response): The HTTP response object. If the request is executed in a separate thread, returns None. + """ + retry_codes = (408, 500) # retry only these codes + + @TryExcept(verbose=verbose) + def func(func_method, func_url, **func_kwargs): + """Make HTTP requests with retries and timeouts, with optional progress tracking.""" + r = None # response + t0 = time.time() # initial time for timer + for i in range(retry + 1): + if (time.time() - t0) > timeout: + break + r = requests_with_progress(func_method, func_url, **func_kwargs) # i.e. get(url, data, json, files) + if r.status_code < 300: # return codes in the 2xx range are generally considered "good" or "successful" + break + try: + m = r.json().get("message", "No JSON message.") + except AttributeError: + m = "Unable to read JSON." + if i == 0: + if r.status_code in retry_codes: + m += f" Retrying {retry}x for {timeout}s." if retry else "" + elif r.status_code == 429: # rate limit + h = r.headers # response headers + m = ( + f"Rate limit reached ({h['X-RateLimit-Remaining']}/{h['X-RateLimit-Limit']}). " + f"Please retry after {h['Retry-After']}s." + ) + if verbose: + LOGGER.warning(f"{PREFIX}{m} {HELP_MSG} ({r.status_code} #{code})") + if r.status_code not in retry_codes: + return r + time.sleep(2**i) # exponential standoff + return r + + args = method, url + kwargs["progress"] = progress + if thread: + threading.Thread(target=func, args=args, kwargs=kwargs, daemon=True).start() + else: + return func(*args, **kwargs) + + +class Events: + """ + A class for collecting anonymous event analytics. Event analytics are enabled when sync=True in settings and + disabled when sync=False. Run 'yolo settings' to see and update settings YAML file. + + Attributes: + url (str): The URL to send anonymous events. + rate_limit (float): The rate limit in seconds for sending events. + metadata (dict): A dictionary containing metadata about the environment. + enabled (bool): A flag to enable or disable Events based on certain conditions. + """ + + url = "https://www.google-analytics.com/mp/collect?measurement_id=G-X8NCJYTQXM&api_secret=QLQrATrNSwGRFRLE-cbHJw" + + def __init__(self): + """Initializes the Events object with default values for events, rate_limit, and metadata.""" + self.events = [] # events list + self.rate_limit = 60.0 # rate limit (seconds) + self.t = 0.0 # rate limit timer (seconds) + self.metadata = { + "cli": Path(sys.argv[0]).name == "yolo", + "install": "git" if is_git_dir() else "pip" if is_pip_package() else "other", + "python": ".".join(platform.python_version_tuple()[:2]), # i.e. 3.10 + "version": __version__, + "env": ENVIRONMENT, + "session_id": round(random.random() * 1e15), + "engagement_time_msec": 1000, + } + self.enabled = ( + SETTINGS["sync"] + and RANK in (-1, 0) + and not TESTS_RUNNING + and ONLINE + and (is_pip_package() or get_git_origin_url() == "https://github.com/ultralytics/ultralytics.git") + ) + + def __call__(self, cfg): + """ + Attempts to add a new event to the events list and send events if the rate limit is reached. + + Args: + cfg (IterableSimpleNamespace): The configuration object containing mode and task information. + """ + if not self.enabled: + # Events disabled, do nothing + return + + # Attempt to add to events + if len(self.events) < 25: # Events list limited to 25 events (drop any events past this) + params = { + **self.metadata, + "task": cfg.task, + "model": cfg.model if cfg.model in GITHUB_ASSETS_NAMES else "custom", + } + if cfg.mode == "export": + params["format"] = cfg.format + self.events.append({"name": cfg.mode, "params": params}) + + # Check rate limit + t = time.time() + if (t - self.t) < self.rate_limit: + # Time is under rate limiter, wait to send + return + + # Time is over rate limiter, send now + data = {"client_id": SETTINGS["uuid"], "events": self.events} # SHA-256 anonymized UUID hash and events list + + # POST equivalent to requests.post(self.url, json=data) + smart_request("post", self.url, json=data, retry=0, verbose=False) + + # Reset events and rate limit timer + self.events = [] + self.t = t + + +# Run below code on hub/utils init ------------------------------------------------------------------------------------- +events = Events() diff --git a/ultralytics/models/__init__.py b/ultralytics/models/__init__.py new file mode 100644 index 0000000..42de3fb --- /dev/null +++ b/ultralytics/models/__init__.py @@ -0,0 +1,8 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from .rtdetr import RTDETR +from .sam import SAM +from .yolo import YOLO, YOLOWorld +from .yolov10 import YOLOv10 + +__all__ = "YOLO", "RTDETR", "SAM", "YOLOWorld", "YOLOv10" # allow simpler import diff --git a/ultralytics/models/__pycache__/__init__.cpython-312.pyc b/ultralytics/models/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..d9d13e5 Binary files /dev/null and b/ultralytics/models/__pycache__/__init__.cpython-312.pyc differ diff --git a/ultralytics/models/__pycache__/__init__.cpython-39.pyc b/ultralytics/models/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..4b37be5 Binary files /dev/null and b/ultralytics/models/__pycache__/__init__.cpython-39.pyc differ diff --git a/ultralytics/models/fastsam/__init__.py b/ultralytics/models/fastsam/__init__.py new file mode 100644 index 0000000..eabf5b9 --- /dev/null +++ b/ultralytics/models/fastsam/__init__.py @@ -0,0 +1,8 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from .model import FastSAM +from .predict import FastSAMPredictor +from .prompt import FastSAMPrompt +from .val import FastSAMValidator + +__all__ = "FastSAMPredictor", "FastSAM", "FastSAMPrompt", "FastSAMValidator" diff --git a/ultralytics/models/fastsam/__pycache__/__init__.cpython-312.pyc b/ultralytics/models/fastsam/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..0a65702 Binary files /dev/null and b/ultralytics/models/fastsam/__pycache__/__init__.cpython-312.pyc differ diff --git a/ultralytics/models/fastsam/__pycache__/__init__.cpython-39.pyc b/ultralytics/models/fastsam/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..d2dff01 Binary files /dev/null and b/ultralytics/models/fastsam/__pycache__/__init__.cpython-39.pyc differ diff --git a/ultralytics/models/fastsam/__pycache__/model.cpython-312.pyc b/ultralytics/models/fastsam/__pycache__/model.cpython-312.pyc new file mode 100644 index 0000000..2e8fb18 Binary files /dev/null and b/ultralytics/models/fastsam/__pycache__/model.cpython-312.pyc differ diff --git a/ultralytics/models/fastsam/__pycache__/model.cpython-39.pyc b/ultralytics/models/fastsam/__pycache__/model.cpython-39.pyc new file mode 100644 index 0000000..05ea098 Binary files /dev/null and b/ultralytics/models/fastsam/__pycache__/model.cpython-39.pyc differ diff --git a/ultralytics/models/fastsam/__pycache__/predict.cpython-312.pyc b/ultralytics/models/fastsam/__pycache__/predict.cpython-312.pyc new file mode 100644 index 0000000..8cbac10 Binary files /dev/null and b/ultralytics/models/fastsam/__pycache__/predict.cpython-312.pyc differ diff --git a/ultralytics/models/fastsam/__pycache__/predict.cpython-39.pyc b/ultralytics/models/fastsam/__pycache__/predict.cpython-39.pyc new file mode 100644 index 0000000..d526f37 Binary files /dev/null and b/ultralytics/models/fastsam/__pycache__/predict.cpython-39.pyc differ diff --git a/ultralytics/models/fastsam/__pycache__/prompt.cpython-312.pyc b/ultralytics/models/fastsam/__pycache__/prompt.cpython-312.pyc new file mode 100644 index 0000000..eb26bcf Binary files /dev/null and b/ultralytics/models/fastsam/__pycache__/prompt.cpython-312.pyc differ diff --git a/ultralytics/models/fastsam/__pycache__/prompt.cpython-39.pyc b/ultralytics/models/fastsam/__pycache__/prompt.cpython-39.pyc new file mode 100644 index 0000000..c4fd184 Binary files /dev/null and b/ultralytics/models/fastsam/__pycache__/prompt.cpython-39.pyc differ diff --git a/ultralytics/models/fastsam/__pycache__/utils.cpython-312.pyc b/ultralytics/models/fastsam/__pycache__/utils.cpython-312.pyc new file mode 100644 index 0000000..5e73941 Binary files /dev/null and b/ultralytics/models/fastsam/__pycache__/utils.cpython-312.pyc differ diff --git a/ultralytics/models/fastsam/__pycache__/utils.cpython-39.pyc b/ultralytics/models/fastsam/__pycache__/utils.cpython-39.pyc new file mode 100644 index 0000000..2760a0e Binary files /dev/null and b/ultralytics/models/fastsam/__pycache__/utils.cpython-39.pyc differ diff --git a/ultralytics/models/fastsam/__pycache__/val.cpython-312.pyc b/ultralytics/models/fastsam/__pycache__/val.cpython-312.pyc new file mode 100644 index 0000000..0b69c37 Binary files /dev/null and b/ultralytics/models/fastsam/__pycache__/val.cpython-312.pyc differ diff --git a/ultralytics/models/fastsam/__pycache__/val.cpython-39.pyc b/ultralytics/models/fastsam/__pycache__/val.cpython-39.pyc new file mode 100644 index 0000000..5ca0872 Binary files /dev/null and b/ultralytics/models/fastsam/__pycache__/val.cpython-39.pyc differ diff --git a/ultralytics/models/fastsam/model.py b/ultralytics/models/fastsam/model.py new file mode 100644 index 0000000..c01e66b --- /dev/null +++ b/ultralytics/models/fastsam/model.py @@ -0,0 +1,33 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from pathlib import Path + +from ultralytics.engine.model import Model +from .predict import FastSAMPredictor +from .val import FastSAMValidator + + +class FastSAM(Model): + """ + FastSAM model interface. + + Example: + ```python + from ultralytics import FastSAM + + model = FastSAM('last.pt') + results = model.predict('ultralytics/assets/bus.jpg') + ``` + """ + + def __init__(self, model="FastSAM-x.pt"): + """Call the __init__ method of the parent class (YOLO) with the updated default model.""" + if str(model) == "FastSAM.pt": + model = "FastSAM-x.pt" + assert Path(model).suffix not in (".yaml", ".yml"), "FastSAM models only support pre-trained models." + super().__init__(model=model, task="segment") + + @property + def task_map(self): + """Returns a dictionary mapping segment task to corresponding predictor and validator classes.""" + return {"segment": {"predictor": FastSAMPredictor, "validator": FastSAMValidator}} diff --git a/ultralytics/models/fastsam/predict.py b/ultralytics/models/fastsam/predict.py new file mode 100644 index 0000000..0ef1803 --- /dev/null +++ b/ultralytics/models/fastsam/predict.py @@ -0,0 +1,86 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import torch + +from ultralytics.engine.results import Results +from ultralytics.models.fastsam.utils import bbox_iou +from ultralytics.models.yolo.detect.predict import DetectionPredictor +from ultralytics.utils import DEFAULT_CFG, ops + + +class FastSAMPredictor(DetectionPredictor): + """ + FastSAMPredictor is specialized for fast SAM (Segment Anything Model) segmentation prediction tasks in Ultralytics + YOLO framework. + + This class extends the DetectionPredictor, customizing the prediction pipeline specifically for fast SAM. + It adjusts post-processing steps to incorporate mask prediction and non-max suppression while optimizing + for single-class segmentation. + + Attributes: + cfg (dict): Configuration parameters for prediction. + overrides (dict, optional): Optional parameter overrides for custom behavior. + _callbacks (dict, optional): Optional list of callback functions to be invoked during prediction. + """ + + def __init__(self, cfg=DEFAULT_CFG, overrides=None, _callbacks=None): + """ + Initializes the FastSAMPredictor class, inheriting from DetectionPredictor and setting the task to 'segment'. + + Args: + cfg (dict): Configuration parameters for prediction. + overrides (dict, optional): Optional parameter overrides for custom behavior. + _callbacks (dict, optional): Optional list of callback functions to be invoked during prediction. + """ + super().__init__(cfg, overrides, _callbacks) + self.args.task = "segment" + + def postprocess(self, preds, img, orig_imgs): + """ + Perform post-processing steps on predictions, including non-max suppression and scaling boxes to original image + size, and returns the final results. + + Args: + preds (list): The raw output predictions from the model. + img (torch.Tensor): The processed image tensor. + orig_imgs (list | torch.Tensor): The original image or list of images. + + Returns: + (list): A list of Results objects, each containing processed boxes, masks, and other metadata. + """ + p = ops.non_max_suppression( + preds[0], + self.args.conf, + self.args.iou, + agnostic=self.args.agnostic_nms, + max_det=self.args.max_det, + nc=1, # set to 1 class since SAM has no class predictions + classes=self.args.classes, + ) + full_box = torch.zeros(p[0].shape[1], device=p[0].device) + full_box[2], full_box[3], full_box[4], full_box[6:] = img.shape[3], img.shape[2], 1.0, 1.0 + full_box = full_box.view(1, -1) + critical_iou_index = bbox_iou(full_box[0][:4], p[0][:, :4], iou_thres=0.9, image_shape=img.shape[2:]) + if critical_iou_index.numel() != 0: + full_box[0][4] = p[0][critical_iou_index][:, 4] + full_box[0][6:] = p[0][critical_iou_index][:, 6:] + p[0][critical_iou_index] = full_box + + if not isinstance(orig_imgs, list): # input images are a torch.Tensor, not a list + orig_imgs = ops.convert_torch2numpy_batch(orig_imgs) + + results = [] + proto = preds[1][-1] if len(preds[1]) == 3 else preds[1] # second output is len 3 if pt, but only 1 if exported + for i, pred in enumerate(p): + orig_img = orig_imgs[i] + img_path = self.batch[0][i] + if not len(pred): # save empty boxes + masks = None + elif self.args.retina_masks: + pred[:, :4] = ops.scale_boxes(img.shape[2:], pred[:, :4], orig_img.shape) + masks = ops.process_mask_native(proto[i], pred[:, 6:], pred[:, :4], orig_img.shape[:2]) # HWC + else: + masks = ops.process_mask(proto[i], pred[:, 6:], pred[:, :4], img.shape[2:], upsample=True) # HWC + pred[:, :4] = ops.scale_boxes(img.shape[2:], pred[:, :4], orig_img.shape) + results.append(Results(orig_img, path=img_path, names=self.model.names, boxes=pred[:, :6], masks=masks)) + return results diff --git a/ultralytics/models/fastsam/prompt.py b/ultralytics/models/fastsam/prompt.py new file mode 100644 index 0000000..f7bf5ad --- /dev/null +++ b/ultralytics/models/fastsam/prompt.py @@ -0,0 +1,357 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import os +from pathlib import Path + +import cv2 +import matplotlib.pyplot as plt +import numpy as np +import torch +from PIL import Image + +from ultralytics.utils import TQDM + + +class FastSAMPrompt: + """ + Fast Segment Anything Model class for image annotation and visualization. + + Attributes: + device (str): Computing device ('cuda' or 'cpu'). + results: Object detection or segmentation results. + source: Source image or image path. + clip: CLIP model for linear assignment. + """ + + def __init__(self, source, results, device="cuda") -> None: + """Initializes FastSAMPrompt with given source, results and device, and assigns clip for linear assignment.""" + self.device = device + self.results = results + self.source = source + + # Import and assign clip + try: + import clip + except ImportError: + from ultralytics.utils.checks import check_requirements + + check_requirements("git+https://github.com/openai/CLIP.git") + import clip + self.clip = clip + + @staticmethod + def _segment_image(image, bbox): + """Segments the given image according to the provided bounding box coordinates.""" + image_array = np.array(image) + segmented_image_array = np.zeros_like(image_array) + x1, y1, x2, y2 = bbox + segmented_image_array[y1:y2, x1:x2] = image_array[y1:y2, x1:x2] + segmented_image = Image.fromarray(segmented_image_array) + black_image = Image.new("RGB", image.size, (255, 255, 255)) + # transparency_mask = np.zeros_like((), dtype=np.uint8) + transparency_mask = np.zeros((image_array.shape[0], image_array.shape[1]), dtype=np.uint8) + transparency_mask[y1:y2, x1:x2] = 255 + transparency_mask_image = Image.fromarray(transparency_mask, mode="L") + black_image.paste(segmented_image, mask=transparency_mask_image) + return black_image + + @staticmethod + def _format_results(result, filter=0): + """Formats detection results into list of annotations each containing ID, segmentation, bounding box, score and + area. + """ + annotations = [] + n = len(result.masks.data) if result.masks is not None else 0 + for i in range(n): + mask = result.masks.data[i] == 1.0 + if torch.sum(mask) >= filter: + annotation = { + "id": i, + "segmentation": mask.cpu().numpy(), + "bbox": result.boxes.data[i], + "score": result.boxes.conf[i], + } + annotation["area"] = annotation["segmentation"].sum() + annotations.append(annotation) + return annotations + + @staticmethod + def _get_bbox_from_mask(mask): + """Applies morphological transformations to the mask, displays it, and if with_contours is True, draws + contours. + """ + mask = mask.astype(np.uint8) + contours, hierarchy = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + x1, y1, w, h = cv2.boundingRect(contours[0]) + x2, y2 = x1 + w, y1 + h + if len(contours) > 1: + for b in contours: + x_t, y_t, w_t, h_t = cv2.boundingRect(b) + x1 = min(x1, x_t) + y1 = min(y1, y_t) + x2 = max(x2, x_t + w_t) + y2 = max(y2, y_t + h_t) + return [x1, y1, x2, y2] + + def plot( + self, + annotations, + output, + bbox=None, + points=None, + point_label=None, + mask_random_color=True, + better_quality=True, + retina=False, + with_contours=True, + ): + """ + Plots annotations, bounding boxes, and points on images and saves the output. + + Args: + annotations (list): Annotations to be plotted. + output (str or Path): Output directory for saving the plots. + bbox (list, optional): Bounding box coordinates [x1, y1, x2, y2]. Defaults to None. + points (list, optional): Points to be plotted. Defaults to None. + point_label (list, optional): Labels for the points. Defaults to None. + mask_random_color (bool, optional): Whether to use random color for masks. Defaults to True. + better_quality (bool, optional): Whether to apply morphological transformations for better mask quality. Defaults to True. + retina (bool, optional): Whether to use retina mask. Defaults to False. + with_contours (bool, optional): Whether to plot contours. Defaults to True. + """ + pbar = TQDM(annotations, total=len(annotations)) + for ann in pbar: + result_name = os.path.basename(ann.path) + image = ann.orig_img[..., ::-1] # BGR to RGB + original_h, original_w = ann.orig_shape + # For macOS only + # plt.switch_backend('TkAgg') + plt.figure(figsize=(original_w / 100, original_h / 100)) + # Add subplot with no margin. + plt.subplots_adjust(top=1, bottom=0, right=1, left=0, hspace=0, wspace=0) + plt.margins(0, 0) + plt.gca().xaxis.set_major_locator(plt.NullLocator()) + plt.gca().yaxis.set_major_locator(plt.NullLocator()) + plt.imshow(image) + + if ann.masks is not None: + masks = ann.masks.data + if better_quality: + if isinstance(masks[0], torch.Tensor): + masks = np.array(masks.cpu()) + for i, mask in enumerate(masks): + mask = cv2.morphologyEx(mask.astype(np.uint8), cv2.MORPH_CLOSE, np.ones((3, 3), np.uint8)) + masks[i] = cv2.morphologyEx(mask.astype(np.uint8), cv2.MORPH_OPEN, np.ones((8, 8), np.uint8)) + + self.fast_show_mask( + masks, + plt.gca(), + random_color=mask_random_color, + bbox=bbox, + points=points, + pointlabel=point_label, + retinamask=retina, + target_height=original_h, + target_width=original_w, + ) + + if with_contours: + contour_all = [] + temp = np.zeros((original_h, original_w, 1)) + for i, mask in enumerate(masks): + mask = mask.astype(np.uint8) + if not retina: + mask = cv2.resize(mask, (original_w, original_h), interpolation=cv2.INTER_NEAREST) + contours, _ = cv2.findContours(mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + contour_all.extend(iter(contours)) + cv2.drawContours(temp, contour_all, -1, (255, 255, 255), 2) + color = np.array([0 / 255, 0 / 255, 1.0, 0.8]) + contour_mask = temp / 255 * color.reshape(1, 1, -1) + plt.imshow(contour_mask) + + # Save the figure + save_path = Path(output) / result_name + save_path.parent.mkdir(exist_ok=True, parents=True) + plt.axis("off") + plt.savefig(save_path, bbox_inches="tight", pad_inches=0, transparent=True) + plt.close() + pbar.set_description(f"Saving {result_name} to {save_path}") + + @staticmethod + def fast_show_mask( + annotation, + ax, + random_color=False, + bbox=None, + points=None, + pointlabel=None, + retinamask=True, + target_height=960, + target_width=960, + ): + """ + Quickly shows the mask annotations on the given matplotlib axis. + + Args: + annotation (array-like): Mask annotation. + ax (matplotlib.axes.Axes): Matplotlib axis. + random_color (bool, optional): Whether to use random color for masks. Defaults to False. + bbox (list, optional): Bounding box coordinates [x1, y1, x2, y2]. Defaults to None. + points (list, optional): Points to be plotted. Defaults to None. + pointlabel (list, optional): Labels for the points. Defaults to None. + retinamask (bool, optional): Whether to use retina mask. Defaults to True. + target_height (int, optional): Target height for resizing. Defaults to 960. + target_width (int, optional): Target width for resizing. Defaults to 960. + """ + n, h, w = annotation.shape # batch, height, width + + areas = np.sum(annotation, axis=(1, 2)) + annotation = annotation[np.argsort(areas)] + + index = (annotation != 0).argmax(axis=0) + if random_color: + color = np.random.random((n, 1, 1, 3)) + else: + color = np.ones((n, 1, 1, 3)) * np.array([30 / 255, 144 / 255, 1.0]) + transparency = np.ones((n, 1, 1, 1)) * 0.6 + visual = np.concatenate([color, transparency], axis=-1) + mask_image = np.expand_dims(annotation, -1) * visual + + show = np.zeros((h, w, 4)) + h_indices, w_indices = np.meshgrid(np.arange(h), np.arange(w), indexing="ij") + indices = (index[h_indices, w_indices], h_indices, w_indices, slice(None)) + + show[h_indices, w_indices, :] = mask_image[indices] + if bbox is not None: + x1, y1, x2, y2 = bbox + ax.add_patch(plt.Rectangle((x1, y1), x2 - x1, y2 - y1, fill=False, edgecolor="b", linewidth=1)) + # Draw point + if points is not None: + plt.scatter( + [point[0] for i, point in enumerate(points) if pointlabel[i] == 1], + [point[1] for i, point in enumerate(points) if pointlabel[i] == 1], + s=20, + c="y", + ) + plt.scatter( + [point[0] for i, point in enumerate(points) if pointlabel[i] == 0], + [point[1] for i, point in enumerate(points) if pointlabel[i] == 0], + s=20, + c="m", + ) + + if not retinamask: + show = cv2.resize(show, (target_width, target_height), interpolation=cv2.INTER_NEAREST) + ax.imshow(show) + + @torch.no_grad() + def retrieve(self, model, preprocess, elements, search_text: str, device) -> int: + """Processes images and text with a model, calculates similarity, and returns softmax score.""" + preprocessed_images = [preprocess(image).to(device) for image in elements] + tokenized_text = self.clip.tokenize([search_text]).to(device) + stacked_images = torch.stack(preprocessed_images) + image_features = model.encode_image(stacked_images) + text_features = model.encode_text(tokenized_text) + image_features /= image_features.norm(dim=-1, keepdim=True) + text_features /= text_features.norm(dim=-1, keepdim=True) + probs = 100.0 * image_features @ text_features.T + return probs[:, 0].softmax(dim=0) + + def _crop_image(self, format_results): + """Crops an image based on provided annotation format and returns cropped images and related data.""" + if os.path.isdir(self.source): + raise ValueError(f"'{self.source}' is a directory, not a valid source for this function.") + image = Image.fromarray(cv2.cvtColor(self.results[0].orig_img, cv2.COLOR_BGR2RGB)) + ori_w, ori_h = image.size + annotations = format_results + mask_h, mask_w = annotations[0]["segmentation"].shape + if ori_w != mask_w or ori_h != mask_h: + image = image.resize((mask_w, mask_h)) + cropped_boxes = [] + cropped_images = [] + not_crop = [] + filter_id = [] + for _, mask in enumerate(annotations): + if np.sum(mask["segmentation"]) <= 100: + filter_id.append(_) + continue + bbox = self._get_bbox_from_mask(mask["segmentation"]) # bbox from mask + cropped_boxes.append(self._segment_image(image, bbox)) # save cropped image + cropped_images.append(bbox) # save cropped image bbox + + return cropped_boxes, cropped_images, not_crop, filter_id, annotations + + def box_prompt(self, bbox): + """Modifies the bounding box properties and calculates IoU between masks and bounding box.""" + if self.results[0].masks is not None: + assert bbox[2] != 0 and bbox[3] != 0 + if os.path.isdir(self.source): + raise ValueError(f"'{self.source}' is a directory, not a valid source for this function.") + masks = self.results[0].masks.data + target_height, target_width = self.results[0].orig_shape + h = masks.shape[1] + w = masks.shape[2] + if h != target_height or w != target_width: + bbox = [ + int(bbox[0] * w / target_width), + int(bbox[1] * h / target_height), + int(bbox[2] * w / target_width), + int(bbox[3] * h / target_height), + ] + bbox[0] = max(round(bbox[0]), 0) + bbox[1] = max(round(bbox[1]), 0) + bbox[2] = min(round(bbox[2]), w) + bbox[3] = min(round(bbox[3]), h) + + # IoUs = torch.zeros(len(masks), dtype=torch.float32) + bbox_area = (bbox[3] - bbox[1]) * (bbox[2] - bbox[0]) + + masks_area = torch.sum(masks[:, bbox[1] : bbox[3], bbox[0] : bbox[2]], dim=(1, 2)) + orig_masks_area = torch.sum(masks, dim=(1, 2)) + + union = bbox_area + orig_masks_area - masks_area + iou = masks_area / union + max_iou_index = torch.argmax(iou) + + self.results[0].masks.data = torch.tensor(np.array([masks[max_iou_index].cpu().numpy()])) + return self.results + + def point_prompt(self, points, pointlabel): # numpy + """Adjusts points on detected masks based on user input and returns the modified results.""" + if self.results[0].masks is not None: + if os.path.isdir(self.source): + raise ValueError(f"'{self.source}' is a directory, not a valid source for this function.") + masks = self._format_results(self.results[0], 0) + target_height, target_width = self.results[0].orig_shape + h = masks[0]["segmentation"].shape[0] + w = masks[0]["segmentation"].shape[1] + if h != target_height or w != target_width: + points = [[int(point[0] * w / target_width), int(point[1] * h / target_height)] for point in points] + onemask = np.zeros((h, w)) + for annotation in masks: + mask = annotation["segmentation"] if isinstance(annotation, dict) else annotation + for i, point in enumerate(points): + if mask[point[1], point[0]] == 1 and pointlabel[i] == 1: + onemask += mask + if mask[point[1], point[0]] == 1 and pointlabel[i] == 0: + onemask -= mask + onemask = onemask >= 1 + self.results[0].masks.data = torch.tensor(np.array([onemask])) + return self.results + + def text_prompt(self, text): + """Processes a text prompt, applies it to existing results and returns the updated results.""" + if self.results[0].masks is not None: + format_results = self._format_results(self.results[0], 0) + cropped_boxes, cropped_images, not_crop, filter_id, annotations = self._crop_image(format_results) + clip_model, preprocess = self.clip.load("ViT-B/32", device=self.device) + scores = self.retrieve(clip_model, preprocess, cropped_boxes, text, device=self.device) + max_idx = scores.argsort() + max_idx = max_idx[-1] + max_idx += sum(np.array(filter_id) <= int(max_idx)) + self.results[0].masks.data = torch.tensor(np.array([annotations[max_idx]["segmentation"]])) + return self.results + + def everything_prompt(self): + """Returns the processed results from the previous methods in the class.""" + return self.results diff --git a/ultralytics/models/fastsam/utils.py b/ultralytics/models/fastsam/utils.py new file mode 100644 index 0000000..480e903 --- /dev/null +++ b/ultralytics/models/fastsam/utils.py @@ -0,0 +1,67 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import torch + + +def adjust_bboxes_to_image_border(boxes, image_shape, threshold=20): + """ + Adjust bounding boxes to stick to image border if they are within a certain threshold. + + Args: + boxes (torch.Tensor): (n, 4) + image_shape (tuple): (height, width) + threshold (int): pixel threshold + + Returns: + adjusted_boxes (torch.Tensor): adjusted bounding boxes + """ + + # Image dimensions + h, w = image_shape + + # Adjust boxes + boxes[boxes[:, 0] < threshold, 0] = 0 # x1 + boxes[boxes[:, 1] < threshold, 1] = 0 # y1 + boxes[boxes[:, 2] > w - threshold, 2] = w # x2 + boxes[boxes[:, 3] > h - threshold, 3] = h # y2 + return boxes + + +def bbox_iou(box1, boxes, iou_thres=0.9, image_shape=(640, 640), raw_output=False): + """ + Compute the Intersection-Over-Union of a bounding box with respect to an array of other bounding boxes. + + Args: + box1 (torch.Tensor): (4, ) + boxes (torch.Tensor): (n, 4) + iou_thres (float): IoU threshold + image_shape (tuple): (height, width) + raw_output (bool): If True, return the raw IoU values instead of the indices + + Returns: + high_iou_indices (torch.Tensor): Indices of boxes with IoU > thres + """ + boxes = adjust_bboxes_to_image_border(boxes, image_shape) + # Obtain coordinates for intersections + x1 = torch.max(box1[0], boxes[:, 0]) + y1 = torch.max(box1[1], boxes[:, 1]) + x2 = torch.min(box1[2], boxes[:, 2]) + y2 = torch.min(box1[3], boxes[:, 3]) + + # Compute the area of intersection + intersection = (x2 - x1).clamp(0) * (y2 - y1).clamp(0) + + # Compute the area of both individual boxes + box1_area = (box1[2] - box1[0]) * (box1[3] - box1[1]) + box2_area = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1]) + + # Compute the area of union + union = box1_area + box2_area - intersection + + # Compute the IoU + iou = intersection / union # Should be shape (n, ) + if raw_output: + return 0 if iou.numel() == 0 else iou + + # return indices of boxes with IoU > thres + return torch.nonzero(iou > iou_thres).flatten() diff --git a/ultralytics/models/fastsam/val.py b/ultralytics/models/fastsam/val.py new file mode 100644 index 0000000..9014b27 --- /dev/null +++ b/ultralytics/models/fastsam/val.py @@ -0,0 +1,40 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from ultralytics.models.yolo.segment import SegmentationValidator +from ultralytics.utils.metrics import SegmentMetrics + + +class FastSAMValidator(SegmentationValidator): + """ + Custom validation class for fast SAM (Segment Anything Model) segmentation in Ultralytics YOLO framework. + + Extends the SegmentationValidator class, customizing the validation process specifically for fast SAM. This class + sets the task to 'segment' and uses the SegmentMetrics for evaluation. Additionally, plotting features are disabled + to avoid errors during validation. + + Attributes: + dataloader: The data loader object used for validation. + save_dir (str): The directory where validation results will be saved. + pbar: A progress bar object. + args: Additional arguments for customization. + _callbacks: List of callback functions to be invoked during validation. + """ + + def __init__(self, dataloader=None, save_dir=None, pbar=None, args=None, _callbacks=None): + """ + Initialize the FastSAMValidator class, setting the task to 'segment' and metrics to SegmentMetrics. + + Args: + dataloader (torch.utils.data.DataLoader): Dataloader to be used for validation. + save_dir (Path, optional): Directory to save results. + pbar (tqdm.tqdm): Progress bar for displaying progress. + args (SimpleNamespace): Configuration for the validator. + _callbacks (dict): Dictionary to store various callback functions. + + Notes: + Plots for ConfusionMatrix and other related metrics are disabled in this class to avoid errors. + """ + super().__init__(dataloader, save_dir, pbar, args, _callbacks) + self.args.task = "segment" + self.args.plots = False # disable ConfusionMatrix and other plots to avoid errors + self.metrics = SegmentMetrics(save_dir=self.save_dir, on_plot=self.on_plot) diff --git a/ultralytics/models/nas/__init__.py b/ultralytics/models/nas/__init__.py new file mode 100644 index 0000000..b095a05 --- /dev/null +++ b/ultralytics/models/nas/__init__.py @@ -0,0 +1,7 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from .model import NAS +from .predict import NASPredictor +from .val import NASValidator + +__all__ = "NASPredictor", "NASValidator", "NAS" diff --git a/ultralytics/models/nas/__pycache__/__init__.cpython-312.pyc b/ultralytics/models/nas/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..ea40f77 Binary files /dev/null and b/ultralytics/models/nas/__pycache__/__init__.cpython-312.pyc differ diff --git a/ultralytics/models/nas/__pycache__/__init__.cpython-39.pyc b/ultralytics/models/nas/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..f9e576e Binary files /dev/null and b/ultralytics/models/nas/__pycache__/__init__.cpython-39.pyc differ diff --git a/ultralytics/models/nas/__pycache__/model.cpython-312.pyc b/ultralytics/models/nas/__pycache__/model.cpython-312.pyc new file mode 100644 index 0000000..903852b Binary files /dev/null and b/ultralytics/models/nas/__pycache__/model.cpython-312.pyc differ diff --git a/ultralytics/models/nas/__pycache__/model.cpython-39.pyc b/ultralytics/models/nas/__pycache__/model.cpython-39.pyc new file mode 100644 index 0000000..5dcd9ff Binary files /dev/null and b/ultralytics/models/nas/__pycache__/model.cpython-39.pyc differ diff --git a/ultralytics/models/nas/__pycache__/predict.cpython-312.pyc b/ultralytics/models/nas/__pycache__/predict.cpython-312.pyc new file mode 100644 index 0000000..ef003d2 Binary files /dev/null and b/ultralytics/models/nas/__pycache__/predict.cpython-312.pyc differ diff --git a/ultralytics/models/nas/__pycache__/predict.cpython-39.pyc b/ultralytics/models/nas/__pycache__/predict.cpython-39.pyc new file mode 100644 index 0000000..96c5834 Binary files /dev/null and b/ultralytics/models/nas/__pycache__/predict.cpython-39.pyc differ diff --git a/ultralytics/models/nas/__pycache__/val.cpython-312.pyc b/ultralytics/models/nas/__pycache__/val.cpython-312.pyc new file mode 100644 index 0000000..d58c3d6 Binary files /dev/null and b/ultralytics/models/nas/__pycache__/val.cpython-312.pyc differ diff --git a/ultralytics/models/nas/__pycache__/val.cpython-39.pyc b/ultralytics/models/nas/__pycache__/val.cpython-39.pyc new file mode 100644 index 0000000..c8aa1ce Binary files /dev/null and b/ultralytics/models/nas/__pycache__/val.cpython-39.pyc differ diff --git a/ultralytics/models/nas/model.py b/ultralytics/models/nas/model.py new file mode 100644 index 0000000..7997e96 --- /dev/null +++ b/ultralytics/models/nas/model.py @@ -0,0 +1,83 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +""" +YOLO-NAS model interface. + +Example: + ```python + from ultralytics import NAS + + model = NAS('yolo_nas_s') + results = model.predict('ultralytics/assets/bus.jpg') + ``` +""" + +from pathlib import Path + +import torch + +from ultralytics.engine.model import Model +from ultralytics.utils.torch_utils import model_info, smart_inference_mode +from .predict import NASPredictor +from .val import NASValidator + + +class NAS(Model): + """ + YOLO NAS model for object detection. + + This class provides an interface for the YOLO-NAS models and extends the `Model` class from Ultralytics engine. + It is designed to facilitate the task of object detection using pre-trained or custom-trained YOLO-NAS models. + + Example: + ```python + from ultralytics import NAS + + model = NAS('yolo_nas_s') + results = model.predict('ultralytics/assets/bus.jpg') + ``` + + Attributes: + model (str): Path to the pre-trained model or model name. Defaults to 'yolo_nas_s.pt'. + + Note: + YOLO-NAS models only support pre-trained models. Do not provide YAML configuration files. + """ + + def __init__(self, model="yolo_nas_s.pt") -> None: + """Initializes the NAS model with the provided or default 'yolo_nas_s.pt' model.""" + assert Path(model).suffix not in (".yaml", ".yml"), "YOLO-NAS models only support pre-trained models." + super().__init__(model, task="detect") + + @smart_inference_mode() + def _load(self, weights: str, task: str): + """Loads an existing NAS model weights or creates a new NAS model with pretrained weights if not provided.""" + import super_gradients + + suffix = Path(weights).suffix + if suffix == ".pt": + self.model = torch.load(weights) + elif suffix == "": + self.model = super_gradients.training.models.get(weights, pretrained_weights="coco") + # Standardize model + self.model.fuse = lambda verbose=True: self.model + self.model.stride = torch.tensor([32]) + self.model.names = dict(enumerate(self.model._class_names)) + self.model.is_fused = lambda: False # for info() + self.model.yaml = {} # for info() + self.model.pt_path = weights # for export() + self.model.task = "detect" # for export() + + def info(self, detailed=False, verbose=True): + """ + Logs model info. + + Args: + detailed (bool): Show detailed information about model. + verbose (bool): Controls verbosity. + """ + return model_info(self.model, detailed=detailed, verbose=verbose, imgsz=640) + + @property + def task_map(self): + """Returns a dictionary mapping tasks to respective predictor and validator classes.""" + return {"detect": {"predictor": NASPredictor, "validator": NASValidator}} diff --git a/ultralytics/models/nas/predict.py b/ultralytics/models/nas/predict.py new file mode 100644 index 0000000..2e48546 --- /dev/null +++ b/ultralytics/models/nas/predict.py @@ -0,0 +1,60 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import torch + +from ultralytics.engine.predictor import BasePredictor +from ultralytics.engine.results import Results +from ultralytics.utils import ops + + +class NASPredictor(BasePredictor): + """ + Ultralytics YOLO NAS Predictor for object detection. + + This class extends the `BasePredictor` from Ultralytics engine and is responsible for post-processing the + raw predictions generated by the YOLO NAS models. It applies operations like non-maximum suppression and + scaling the bounding boxes to fit the original image dimensions. + + Attributes: + args (Namespace): Namespace containing various configurations for post-processing. + + Example: + ```python + from ultralytics import NAS + + model = NAS('yolo_nas_s') + predictor = model.predictor + # Assumes that raw_preds, img, orig_imgs are available + results = predictor.postprocess(raw_preds, img, orig_imgs) + ``` + + Note: + Typically, this class is not instantiated directly. It is used internally within the `NAS` class. + """ + + def postprocess(self, preds_in, img, orig_imgs): + """Postprocess predictions and returns a list of Results objects.""" + + # Cat boxes and class scores + boxes = ops.xyxy2xywh(preds_in[0][0]) + preds = torch.cat((boxes, preds_in[0][1]), -1).permute(0, 2, 1) + + preds = ops.non_max_suppression( + preds, + self.args.conf, + self.args.iou, + agnostic=self.args.agnostic_nms, + max_det=self.args.max_det, + classes=self.args.classes, + ) + + if not isinstance(orig_imgs, list): # input images are a torch.Tensor, not a list + orig_imgs = ops.convert_torch2numpy_batch(orig_imgs) + + results = [] + for i, pred in enumerate(preds): + orig_img = orig_imgs[i] + pred[:, :4] = ops.scale_boxes(img.shape[2:], pred[:, :4], orig_img.shape) + img_path = self.batch[0][i] + results.append(Results(orig_img, path=img_path, names=self.model.names, boxes=pred)) + return results diff --git a/ultralytics/models/nas/val.py b/ultralytics/models/nas/val.py new file mode 100644 index 0000000..a4a4f99 --- /dev/null +++ b/ultralytics/models/nas/val.py @@ -0,0 +1,50 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import torch + +from ultralytics.models.yolo.detect import DetectionValidator +from ultralytics.utils import ops + +__all__ = ["NASValidator"] + + +class NASValidator(DetectionValidator): + """ + Ultralytics YOLO NAS Validator for object detection. + + Extends `DetectionValidator` from the Ultralytics models package and is designed to post-process the raw predictions + generated by YOLO NAS models. It performs non-maximum suppression to remove overlapping and low-confidence boxes, + ultimately producing the final detections. + + Attributes: + args (Namespace): Namespace containing various configurations for post-processing, such as confidence and IoU thresholds. + lb (torch.Tensor): Optional tensor for multilabel NMS. + + Example: + ```python + from ultralytics import NAS + + model = NAS('yolo_nas_s') + validator = model.validator + # Assumes that raw_preds are available + final_preds = validator.postprocess(raw_preds) + ``` + + Note: + This class is generally not instantiated directly but is used internally within the `NAS` class. + """ + + def postprocess(self, preds_in): + """Apply Non-maximum suppression to prediction outputs.""" + boxes = ops.xyxy2xywh(preds_in[0][0]) + preds = torch.cat((boxes, preds_in[0][1]), -1).permute(0, 2, 1) + return ops.non_max_suppression( + preds, + self.args.conf, + self.args.iou, + labels=self.lb, + multi_label=False, + agnostic=self.args.single_cls, + max_det=self.args.max_det, + max_time_img=0.5, + ) diff --git a/ultralytics/models/rtdetr/__init__.py b/ultralytics/models/rtdetr/__init__.py new file mode 100644 index 0000000..172c74b --- /dev/null +++ b/ultralytics/models/rtdetr/__init__.py @@ -0,0 +1,7 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from .model import RTDETR +from .predict import RTDETRPredictor +from .val import RTDETRValidator + +__all__ = "RTDETRPredictor", "RTDETRValidator", "RTDETR" diff --git a/ultralytics/models/rtdetr/__pycache__/__init__.cpython-312.pyc b/ultralytics/models/rtdetr/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..6bab8ac Binary files /dev/null and b/ultralytics/models/rtdetr/__pycache__/__init__.cpython-312.pyc differ diff --git a/ultralytics/models/rtdetr/__pycache__/__init__.cpython-39.pyc b/ultralytics/models/rtdetr/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..1224ec8 Binary files /dev/null and b/ultralytics/models/rtdetr/__pycache__/__init__.cpython-39.pyc differ diff --git a/ultralytics/models/rtdetr/__pycache__/model.cpython-312.pyc b/ultralytics/models/rtdetr/__pycache__/model.cpython-312.pyc new file mode 100644 index 0000000..61f42c4 Binary files /dev/null and b/ultralytics/models/rtdetr/__pycache__/model.cpython-312.pyc differ diff --git a/ultralytics/models/rtdetr/__pycache__/model.cpython-39.pyc b/ultralytics/models/rtdetr/__pycache__/model.cpython-39.pyc new file mode 100644 index 0000000..ae5cd9d Binary files /dev/null and b/ultralytics/models/rtdetr/__pycache__/model.cpython-39.pyc differ diff --git a/ultralytics/models/rtdetr/__pycache__/predict.cpython-312.pyc b/ultralytics/models/rtdetr/__pycache__/predict.cpython-312.pyc new file mode 100644 index 0000000..319b9e0 Binary files /dev/null and b/ultralytics/models/rtdetr/__pycache__/predict.cpython-312.pyc differ diff --git a/ultralytics/models/rtdetr/__pycache__/predict.cpython-39.pyc b/ultralytics/models/rtdetr/__pycache__/predict.cpython-39.pyc new file mode 100644 index 0000000..23bf393 Binary files /dev/null and b/ultralytics/models/rtdetr/__pycache__/predict.cpython-39.pyc differ diff --git a/ultralytics/models/rtdetr/__pycache__/train.cpython-312.pyc b/ultralytics/models/rtdetr/__pycache__/train.cpython-312.pyc new file mode 100644 index 0000000..57ed321 Binary files /dev/null and b/ultralytics/models/rtdetr/__pycache__/train.cpython-312.pyc differ diff --git a/ultralytics/models/rtdetr/__pycache__/train.cpython-39.pyc b/ultralytics/models/rtdetr/__pycache__/train.cpython-39.pyc new file mode 100644 index 0000000..5847ea7 Binary files /dev/null and b/ultralytics/models/rtdetr/__pycache__/train.cpython-39.pyc differ diff --git a/ultralytics/models/rtdetr/__pycache__/val.cpython-312.pyc b/ultralytics/models/rtdetr/__pycache__/val.cpython-312.pyc new file mode 100644 index 0000000..6352f6a Binary files /dev/null and b/ultralytics/models/rtdetr/__pycache__/val.cpython-312.pyc differ diff --git a/ultralytics/models/rtdetr/__pycache__/val.cpython-39.pyc b/ultralytics/models/rtdetr/__pycache__/val.cpython-39.pyc new file mode 100644 index 0000000..230f46c Binary files /dev/null and b/ultralytics/models/rtdetr/__pycache__/val.cpython-39.pyc differ diff --git a/ultralytics/models/rtdetr/model.py b/ultralytics/models/rtdetr/model.py new file mode 100644 index 0000000..440df17 --- /dev/null +++ b/ultralytics/models/rtdetr/model.py @@ -0,0 +1,54 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +""" +Interface for Baidu's RT-DETR, a Vision Transformer-based real-time object detector. RT-DETR offers real-time +performance and high accuracy, excelling in accelerated backends like CUDA with TensorRT. It features an efficient +hybrid encoder and IoU-aware query selection for enhanced detection accuracy. + +For more information on RT-DETR, visit: https://arxiv.org/pdf/2304.08069.pdf +""" + +from ultralytics.engine.model import Model +from ultralytics.nn.tasks import RTDETRDetectionModel + +from .predict import RTDETRPredictor +from .train import RTDETRTrainer +from .val import RTDETRValidator + + +class RTDETR(Model): + """ + Interface for Baidu's RT-DETR model. This Vision Transformer-based object detector provides real-time performance + with high accuracy. It supports efficient hybrid encoding, IoU-aware query selection, and adaptable inference speed. + + Attributes: + model (str): Path to the pre-trained model. Defaults to 'rtdetr-l.pt'. + """ + + def __init__(self, model="rtdetr-l.pt") -> None: + """ + Initializes the RT-DETR model with the given pre-trained model file. Supports .pt and .yaml formats. + + Args: + model (str): Path to the pre-trained model. Defaults to 'rtdetr-l.pt'. + + Raises: + NotImplementedError: If the model file extension is not 'pt', 'yaml', or 'yml'. + """ + super().__init__(model=model, task="detect") + + @property + def task_map(self) -> dict: + """ + Returns a task map for RT-DETR, associating tasks with corresponding Ultralytics classes. + + Returns: + dict: A dictionary mapping task names to Ultralytics task classes for the RT-DETR model. + """ + return { + "detect": { + "predictor": RTDETRPredictor, + "validator": RTDETRValidator, + "trainer": RTDETRTrainer, + "model": RTDETRDetectionModel, + } + } diff --git a/ultralytics/models/rtdetr/predict.py b/ultralytics/models/rtdetr/predict.py new file mode 100644 index 0000000..7fc918b --- /dev/null +++ b/ultralytics/models/rtdetr/predict.py @@ -0,0 +1,86 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import torch + +from ultralytics.data.augment import LetterBox +from ultralytics.engine.predictor import BasePredictor +from ultralytics.engine.results import Results +from ultralytics.utils import ops + + +class RTDETRPredictor(BasePredictor): + """ + RT-DETR (Real-Time Detection Transformer) Predictor extending the BasePredictor class for making predictions using + Baidu's RT-DETR model. + + This class leverages the power of Vision Transformers to provide real-time object detection while maintaining + high accuracy. It supports key features like efficient hybrid encoding and IoU-aware query selection. + + Example: + ```python + from ultralytics.utils import ASSETS + from ultralytics.models.rtdetr import RTDETRPredictor + + args = dict(model='rtdetr-l.pt', source=ASSETS) + predictor = RTDETRPredictor(overrides=args) + predictor.predict_cli() + ``` + + Attributes: + imgsz (int): Image size for inference (must be square and scale-filled). + args (dict): Argument overrides for the predictor. + """ + + def postprocess(self, preds, img, orig_imgs): + """ + Postprocess the raw predictions from the model to generate bounding boxes and confidence scores. + + The method filters detections based on confidence and class if specified in `self.args`. + + Args: + preds (list): List of [predictions, extra] from the model. + img (torch.Tensor): Processed input images. + orig_imgs (list or torch.Tensor): Original, unprocessed images. + + Returns: + (list[Results]): A list of Results objects containing the post-processed bounding boxes, confidence scores, + and class labels. + """ + if not isinstance(preds, (list, tuple)): # list for PyTorch inference but list[0] Tensor for export inference + preds = [preds, None] + + nd = preds[0].shape[-1] + bboxes, scores = preds[0].split((4, nd - 4), dim=-1) + + if not isinstance(orig_imgs, list): # input images are a torch.Tensor, not a list + orig_imgs = ops.convert_torch2numpy_batch(orig_imgs) + + results = [] + for i, bbox in enumerate(bboxes): # (300, 4) + bbox = ops.xywh2xyxy(bbox) + score, cls = scores[i].max(-1, keepdim=True) # (300, 1) + idx = score.squeeze(-1) > self.args.conf # (300, ) + if self.args.classes is not None: + idx = (cls == torch.tensor(self.args.classes, device=cls.device)).any(1) & idx + pred = torch.cat([bbox, score, cls], dim=-1)[idx] # filter + orig_img = orig_imgs[i] + oh, ow = orig_img.shape[:2] + pred[..., [0, 2]] *= ow + pred[..., [1, 3]] *= oh + img_path = self.batch[0][i] + results.append(Results(orig_img, path=img_path, names=self.model.names, boxes=pred)) + return results + + def pre_transform(self, im): + """ + Pre-transforms the input images before feeding them into the model for inference. The input images are + letterboxed to ensure a square aspect ratio and scale-filled. The size must be square(640) and scaleFilled. + + Args: + im (list[np.ndarray] |torch.Tensor): Input images of shape (N,3,h,w) for tensor, [(h,w,3) x N] for list. + + Returns: + (list): List of pre-transformed images ready for model inference. + """ + letterbox = LetterBox(self.imgsz, auto=False, scaleFill=True) + return [letterbox(image=x) for x in im] diff --git a/ultralytics/models/rtdetr/train.py b/ultralytics/models/rtdetr/train.py new file mode 100644 index 0000000..10a8f9b --- /dev/null +++ b/ultralytics/models/rtdetr/train.py @@ -0,0 +1,101 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from copy import copy + +import torch + +from ultralytics.models.yolo.detect import DetectionTrainer +from ultralytics.nn.tasks import RTDETRDetectionModel +from ultralytics.utils import RANK, colorstr +from .val import RTDETRDataset, RTDETRValidator + + +class RTDETRTrainer(DetectionTrainer): + """ + Trainer class for the RT-DETR model developed by Baidu for real-time object detection. Extends the DetectionTrainer + class for YOLO to adapt to the specific features and architecture of RT-DETR. This model leverages Vision + Transformers and has capabilities like IoU-aware query selection and adaptable inference speed. + + Notes: + - F.grid_sample used in RT-DETR does not support the `deterministic=True` argument. + - AMP training can lead to NaN outputs and may produce errors during bipartite graph matching. + + Example: + ```python + from ultralytics.models.rtdetr.train import RTDETRTrainer + + args = dict(model='rtdetr-l.yaml', data='coco8.yaml', imgsz=640, epochs=3) + trainer = RTDETRTrainer(overrides=args) + trainer.train() + ``` + """ + + def get_model(self, cfg=None, weights=None, verbose=True): + """ + Initialize and return an RT-DETR model for object detection tasks. + + Args: + cfg (dict, optional): Model configuration. Defaults to None. + weights (str, optional): Path to pre-trained model weights. Defaults to None. + verbose (bool): Verbose logging if True. Defaults to True. + + Returns: + (RTDETRDetectionModel): Initialized model. + """ + model = RTDETRDetectionModel(cfg, nc=self.data["nc"], verbose=verbose and RANK == -1) + if weights: + model.load(weights) + return model + + def build_dataset(self, img_path, mode="val", batch=None): + """ + Build and return an RT-DETR dataset for training or validation. + + Args: + img_path (str): Path to the folder containing images. + mode (str): Dataset mode, either 'train' or 'val'. + batch (int, optional): Batch size for rectangle training. Defaults to None. + + Returns: + (RTDETRDataset): Dataset object for the specific mode. + """ + return RTDETRDataset( + img_path=img_path, + imgsz=self.args.imgsz, + batch_size=batch, + augment=mode == "train", + hyp=self.args, + rect=False, + cache=self.args.cache or None, + prefix=colorstr(f"{mode}: "), + data=self.data, + ) + + def get_validator(self): + """ + Returns a DetectionValidator suitable for RT-DETR model validation. + + Returns: + (RTDETRValidator): Validator object for model validation. + """ + self.loss_names = "giou_loss", "cls_loss", "l1_loss" + return RTDETRValidator(self.test_loader, save_dir=self.save_dir, args=copy(self.args)) + + def preprocess_batch(self, batch): + """ + Preprocess a batch of images. Scales and converts the images to float format. + + Args: + batch (dict): Dictionary containing a batch of images, bboxes, and labels. + + Returns: + (dict): Preprocessed batch. + """ + batch = super().preprocess_batch(batch) + bs = len(batch["img"]) + batch_idx = batch["batch_idx"] + gt_bbox, gt_class = [], [] + for i in range(bs): + gt_bbox.append(batch["bboxes"][batch_idx == i].to(batch_idx.device)) + gt_class.append(batch["cls"][batch_idx == i].to(device=batch_idx.device, dtype=torch.long)) + return batch diff --git a/ultralytics/models/rtdetr/val.py b/ultralytics/models/rtdetr/val.py new file mode 100644 index 0000000..88bb0ae --- /dev/null +++ b/ultralytics/models/rtdetr/val.py @@ -0,0 +1,135 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import torch + +from ultralytics.data import YOLODataset +from ultralytics.data.augment import Compose, Format, v8_transforms +from ultralytics.models.yolo.detect import DetectionValidator +from ultralytics.utils import colorstr, ops + +__all__ = ("RTDETRValidator",) # tuple or list + + +class RTDETRDataset(YOLODataset): + """ + Real-Time DEtection and TRacking (RT-DETR) dataset class extending the base YOLODataset class. + + This specialized dataset class is designed for use with the RT-DETR object detection model and is optimized for + real-time detection and tracking tasks. + """ + + def __init__(self, *args, data=None, **kwargs): + """Initialize the RTDETRDataset class by inheriting from the YOLODataset class.""" + super().__init__(*args, data=data, **kwargs) + + # NOTE: add stretch version load_image for RTDETR mosaic + def load_image(self, i, rect_mode=False): + """Loads 1 image from dataset index 'i', returns (im, resized hw).""" + return super().load_image(i=i, rect_mode=rect_mode) + + def build_transforms(self, hyp=None): + """Temporary, only for evaluation.""" + if self.augment: + hyp.mosaic = hyp.mosaic if self.augment and not self.rect else 0.0 + hyp.mixup = hyp.mixup if self.augment and not self.rect else 0.0 + transforms = v8_transforms(self, self.imgsz, hyp, stretch=True) + else: + # transforms = Compose([LetterBox(new_shape=(self.imgsz, self.imgsz), auto=False, scaleFill=True)]) + transforms = Compose([]) + transforms.append( + Format( + bbox_format="xywh", + normalize=True, + return_mask=self.use_segments, + return_keypoint=self.use_keypoints, + batch_idx=True, + mask_ratio=hyp.mask_ratio, + mask_overlap=hyp.overlap_mask, + ) + ) + return transforms + + +class RTDETRValidator(DetectionValidator): + """ + RTDETRValidator extends the DetectionValidator class to provide validation capabilities specifically tailored for + the RT-DETR (Real-Time DETR) object detection model. + + The class allows building of an RTDETR-specific dataset for validation, applies Non-maximum suppression for + post-processing, and updates evaluation metrics accordingly. + + Example: + ```python + from ultralytics.models.rtdetr import RTDETRValidator + + args = dict(model='rtdetr-l.pt', data='coco8.yaml') + validator = RTDETRValidator(args=args) + validator() + ``` + + Note: + For further details on the attributes and methods, refer to the parent DetectionValidator class. + """ + + def build_dataset(self, img_path, mode="val", batch=None): + """ + Build an RTDETR Dataset. + + Args: + img_path (str): Path to the folder containing images. + mode (str): `train` mode or `val` mode, users are able to customize different augmentations for each mode. + batch (int, optional): Size of batches, this is for `rect`. Defaults to None. + """ + return RTDETRDataset( + img_path=img_path, + imgsz=self.args.imgsz, + batch_size=batch, + augment=False, # no augmentation + hyp=self.args, + rect=False, # no rect + cache=self.args.cache or None, + prefix=colorstr(f"{mode}: "), + data=self.data, + ) + + def postprocess(self, preds): + """Apply Non-maximum suppression to prediction outputs.""" + if not isinstance(preds, (list, tuple)): # list for PyTorch inference but list[0] Tensor for export inference + preds = [preds, None] + + bs, _, nd = preds[0].shape + bboxes, scores = preds[0].split((4, nd - 4), dim=-1) + bboxes *= self.args.imgsz + outputs = [torch.zeros((0, 6), device=bboxes.device)] * bs + for i, bbox in enumerate(bboxes): # (300, 4) + bbox = ops.xywh2xyxy(bbox) + score, cls = scores[i].max(-1) # (300, ) + # Do not need threshold for evaluation as only got 300 boxes here + # idx = score > self.args.conf + pred = torch.cat([bbox, score[..., None], cls[..., None]], dim=-1) # filter + # Sort by confidence to correctly get internal metrics + pred = pred[score.argsort(descending=True)] + outputs[i] = pred # [idx] + + return outputs + + def _prepare_batch(self, si, batch): + """Prepares a batch for training or inference by applying transformations.""" + idx = batch["batch_idx"] == si + cls = batch["cls"][idx].squeeze(-1) + bbox = batch["bboxes"][idx] + ori_shape = batch["ori_shape"][si] + imgsz = batch["img"].shape[2:] + ratio_pad = batch["ratio_pad"][si] + if len(cls): + bbox = ops.xywh2xyxy(bbox) # target boxes + bbox[..., [0, 2]] *= ori_shape[1] # native-space pred + bbox[..., [1, 3]] *= ori_shape[0] # native-space pred + return dict(cls=cls, bbox=bbox, ori_shape=ori_shape, imgsz=imgsz, ratio_pad=ratio_pad) + + def _prepare_pred(self, pred, pbatch): + """Prepares and returns a batch with transformed bounding boxes and class labels.""" + predn = pred.clone() + predn[..., [0, 2]] *= pbatch["ori_shape"][1] / self.args.imgsz # native-space pred + predn[..., [1, 3]] *= pbatch["ori_shape"][0] / self.args.imgsz # native-space pred + return predn.float() diff --git a/ultralytics/models/sam/__init__.py b/ultralytics/models/sam/__init__.py new file mode 100644 index 0000000..8701fcc --- /dev/null +++ b/ultralytics/models/sam/__init__.py @@ -0,0 +1,6 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from .model import SAM +from .predict import Predictor + +__all__ = "SAM", "Predictor" # tuple or list diff --git a/ultralytics/models/sam/__pycache__/__init__.cpython-312.pyc b/ultralytics/models/sam/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..d2a7d8a Binary files /dev/null and b/ultralytics/models/sam/__pycache__/__init__.cpython-312.pyc differ diff --git a/ultralytics/models/sam/__pycache__/__init__.cpython-39.pyc b/ultralytics/models/sam/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..81a0c71 Binary files /dev/null and b/ultralytics/models/sam/__pycache__/__init__.cpython-39.pyc differ diff --git a/ultralytics/models/sam/__pycache__/amg.cpython-312.pyc b/ultralytics/models/sam/__pycache__/amg.cpython-312.pyc new file mode 100644 index 0000000..b934393 Binary files /dev/null and b/ultralytics/models/sam/__pycache__/amg.cpython-312.pyc differ diff --git a/ultralytics/models/sam/__pycache__/amg.cpython-39.pyc b/ultralytics/models/sam/__pycache__/amg.cpython-39.pyc new file mode 100644 index 0000000..6cf79c4 Binary files /dev/null and b/ultralytics/models/sam/__pycache__/amg.cpython-39.pyc differ diff --git a/ultralytics/models/sam/__pycache__/build.cpython-312.pyc b/ultralytics/models/sam/__pycache__/build.cpython-312.pyc new file mode 100644 index 0000000..38ed4bb Binary files /dev/null and b/ultralytics/models/sam/__pycache__/build.cpython-312.pyc differ diff --git a/ultralytics/models/sam/__pycache__/build.cpython-39.pyc b/ultralytics/models/sam/__pycache__/build.cpython-39.pyc new file mode 100644 index 0000000..43e49b1 Binary files /dev/null and b/ultralytics/models/sam/__pycache__/build.cpython-39.pyc differ diff --git a/ultralytics/models/sam/__pycache__/model.cpython-312.pyc b/ultralytics/models/sam/__pycache__/model.cpython-312.pyc new file mode 100644 index 0000000..d1d2998 Binary files /dev/null and b/ultralytics/models/sam/__pycache__/model.cpython-312.pyc differ diff --git a/ultralytics/models/sam/__pycache__/model.cpython-39.pyc b/ultralytics/models/sam/__pycache__/model.cpython-39.pyc new file mode 100644 index 0000000..1826afc Binary files /dev/null and b/ultralytics/models/sam/__pycache__/model.cpython-39.pyc differ diff --git a/ultralytics/models/sam/__pycache__/predict.cpython-312.pyc b/ultralytics/models/sam/__pycache__/predict.cpython-312.pyc new file mode 100644 index 0000000..e8874e1 Binary files /dev/null and b/ultralytics/models/sam/__pycache__/predict.cpython-312.pyc differ diff --git a/ultralytics/models/sam/__pycache__/predict.cpython-39.pyc b/ultralytics/models/sam/__pycache__/predict.cpython-39.pyc new file mode 100644 index 0000000..ea68a38 Binary files /dev/null and b/ultralytics/models/sam/__pycache__/predict.cpython-39.pyc differ diff --git a/ultralytics/models/sam/amg.py b/ultralytics/models/sam/amg.py new file mode 100644 index 0000000..128108f --- /dev/null +++ b/ultralytics/models/sam/amg.py @@ -0,0 +1,187 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import math +from itertools import product +from typing import Any, Generator, List, Tuple + +import numpy as np +import torch + + +def is_box_near_crop_edge( + boxes: torch.Tensor, crop_box: List[int], orig_box: List[int], atol: float = 20.0 +) -> torch.Tensor: + """Return a boolean tensor indicating if boxes are near the crop edge.""" + crop_box_torch = torch.as_tensor(crop_box, dtype=torch.float, device=boxes.device) + orig_box_torch = torch.as_tensor(orig_box, dtype=torch.float, device=boxes.device) + boxes = uncrop_boxes_xyxy(boxes, crop_box).float() + near_crop_edge = torch.isclose(boxes, crop_box_torch[None, :], atol=atol, rtol=0) + near_image_edge = torch.isclose(boxes, orig_box_torch[None, :], atol=atol, rtol=0) + near_crop_edge = torch.logical_and(near_crop_edge, ~near_image_edge) + return torch.any(near_crop_edge, dim=1) + + +def batch_iterator(batch_size: int, *args) -> Generator[List[Any], None, None]: + """Yield batches of data from the input arguments.""" + assert args and all(len(a) == len(args[0]) for a in args), "Batched iteration must have same-size inputs." + n_batches = len(args[0]) // batch_size + int(len(args[0]) % batch_size != 0) + for b in range(n_batches): + yield [arg[b * batch_size : (b + 1) * batch_size] for arg in args] + + +def calculate_stability_score(masks: torch.Tensor, mask_threshold: float, threshold_offset: float) -> torch.Tensor: + """ + Computes the stability score for a batch of masks. + + The stability score is the IoU between the binary masks obtained by thresholding the predicted mask logits at high + and low values. + + Notes: + - One mask is always contained inside the other. + - Save memory by preventing unnecessary cast to torch.int64 + """ + intersections = (masks > (mask_threshold + threshold_offset)).sum(-1, dtype=torch.int16).sum(-1, dtype=torch.int32) + unions = (masks > (mask_threshold - threshold_offset)).sum(-1, dtype=torch.int16).sum(-1, dtype=torch.int32) + return intersections / unions + + +def build_point_grid(n_per_side: int) -> np.ndarray: + """Generate a 2D grid of evenly spaced points in the range [0,1]x[0,1].""" + offset = 1 / (2 * n_per_side) + points_one_side = np.linspace(offset, 1 - offset, n_per_side) + points_x = np.tile(points_one_side[None, :], (n_per_side, 1)) + points_y = np.tile(points_one_side[:, None], (1, n_per_side)) + return np.stack([points_x, points_y], axis=-1).reshape(-1, 2) + + +def build_all_layer_point_grids(n_per_side: int, n_layers: int, scale_per_layer: int) -> List[np.ndarray]: + """Generate point grids for all crop layers.""" + return [build_point_grid(int(n_per_side / (scale_per_layer**i))) for i in range(n_layers + 1)] + + +def generate_crop_boxes( + im_size: Tuple[int, ...], n_layers: int, overlap_ratio: float +) -> Tuple[List[List[int]], List[int]]: + """ + Generates a list of crop boxes of different sizes. + + Each layer has (2**i)**2 boxes for the ith layer. + """ + crop_boxes, layer_idxs = [], [] + im_h, im_w = im_size + short_side = min(im_h, im_w) + + # Original image + crop_boxes.append([0, 0, im_w, im_h]) + layer_idxs.append(0) + + def crop_len(orig_len, n_crops, overlap): + """Crops bounding boxes to the size of the input image.""" + return int(math.ceil((overlap * (n_crops - 1) + orig_len) / n_crops)) + + for i_layer in range(n_layers): + n_crops_per_side = 2 ** (i_layer + 1) + overlap = int(overlap_ratio * short_side * (2 / n_crops_per_side)) + + crop_w = crop_len(im_w, n_crops_per_side, overlap) + crop_h = crop_len(im_h, n_crops_per_side, overlap) + + crop_box_x0 = [int((crop_w - overlap) * i) for i in range(n_crops_per_side)] + crop_box_y0 = [int((crop_h - overlap) * i) for i in range(n_crops_per_side)] + + # Crops in XYWH format + for x0, y0 in product(crop_box_x0, crop_box_y0): + box = [x0, y0, min(x0 + crop_w, im_w), min(y0 + crop_h, im_h)] + crop_boxes.append(box) + layer_idxs.append(i_layer + 1) + + return crop_boxes, layer_idxs + + +def uncrop_boxes_xyxy(boxes: torch.Tensor, crop_box: List[int]) -> torch.Tensor: + """Uncrop bounding boxes by adding the crop box offset.""" + x0, y0, _, _ = crop_box + offset = torch.tensor([[x0, y0, x0, y0]], device=boxes.device) + # Check if boxes has a channel dimension + if len(boxes.shape) == 3: + offset = offset.unsqueeze(1) + return boxes + offset + + +def uncrop_points(points: torch.Tensor, crop_box: List[int]) -> torch.Tensor: + """Uncrop points by adding the crop box offset.""" + x0, y0, _, _ = crop_box + offset = torch.tensor([[x0, y0]], device=points.device) + # Check if points has a channel dimension + if len(points.shape) == 3: + offset = offset.unsqueeze(1) + return points + offset + + +def uncrop_masks(masks: torch.Tensor, crop_box: List[int], orig_h: int, orig_w: int) -> torch.Tensor: + """Uncrop masks by padding them to the original image size.""" + x0, y0, x1, y1 = crop_box + if x0 == 0 and y0 == 0 and x1 == orig_w and y1 == orig_h: + return masks + # Coordinate transform masks + pad_x, pad_y = orig_w - (x1 - x0), orig_h - (y1 - y0) + pad = (x0, pad_x - x0, y0, pad_y - y0) + return torch.nn.functional.pad(masks, pad, value=0) + + +def remove_small_regions(mask: np.ndarray, area_thresh: float, mode: str) -> Tuple[np.ndarray, bool]: + """Remove small disconnected regions or holes in a mask, returning the mask and a modification indicator.""" + import cv2 # type: ignore + + assert mode in {"holes", "islands"} + correct_holes = mode == "holes" + working_mask = (correct_holes ^ mask).astype(np.uint8) + n_labels, regions, stats, _ = cv2.connectedComponentsWithStats(working_mask, 8) + sizes = stats[:, -1][1:] # Row 0 is background label + small_regions = [i + 1 for i, s in enumerate(sizes) if s < area_thresh] + if not small_regions: + return mask, False + fill_labels = [0] + small_regions + if not correct_holes: + # If every region is below threshold, keep largest + fill_labels = [i for i in range(n_labels) if i not in fill_labels] or [int(np.argmax(sizes)) + 1] + mask = np.isin(regions, fill_labels) + return mask, True + + +def batched_mask_to_box(masks: torch.Tensor) -> torch.Tensor: + """ + Calculates boxes in XYXY format around masks. + + Return [0,0,0,0] for an empty mask. For input shape C1xC2x...xHxW, the output shape is C1xC2x...x4. + """ + # torch.max below raises an error on empty inputs, just skip in this case + if torch.numel(masks) == 0: + return torch.zeros(*masks.shape[:-2], 4, device=masks.device) + + # Normalize shape to CxHxW + shape = masks.shape + h, w = shape[-2:] + masks = masks.flatten(0, -3) if len(shape) > 2 else masks.unsqueeze(0) + # Get top and bottom edges + in_height, _ = torch.max(masks, dim=-1) + in_height_coords = in_height * torch.arange(h, device=in_height.device)[None, :] + bottom_edges, _ = torch.max(in_height_coords, dim=-1) + in_height_coords = in_height_coords + h * (~in_height) + top_edges, _ = torch.min(in_height_coords, dim=-1) + + # Get left and right edges + in_width, _ = torch.max(masks, dim=-2) + in_width_coords = in_width * torch.arange(w, device=in_width.device)[None, :] + right_edges, _ = torch.max(in_width_coords, dim=-1) + in_width_coords = in_width_coords + w * (~in_width) + left_edges, _ = torch.min(in_width_coords, dim=-1) + + # If the mask is empty the right edge will be to the left of the left edge. + # Replace these boxes with [0, 0, 0, 0] + empty_filter = (right_edges < left_edges) | (bottom_edges < top_edges) + out = torch.stack([left_edges, top_edges, right_edges, bottom_edges], dim=-1) + out = out * (~empty_filter).unsqueeze(-1) + + # Return to original shape + return out.reshape(*shape[:-2], 4) if len(shape) > 2 else out[0] diff --git a/ultralytics/models/sam/build.py b/ultralytics/models/sam/build.py new file mode 100644 index 0000000..266587e --- /dev/null +++ b/ultralytics/models/sam/build.py @@ -0,0 +1,160 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +from functools import partial + +import torch + +from ultralytics.utils.downloads import attempt_download_asset +from .modules.decoders import MaskDecoder +from .modules.encoders import ImageEncoderViT, PromptEncoder +from .modules.sam import Sam +from .modules.tiny_encoder import TinyViT +from .modules.transformer import TwoWayTransformer + + +def build_sam_vit_h(checkpoint=None): + """Build and return a Segment Anything Model (SAM) h-size model.""" + return _build_sam( + encoder_embed_dim=1280, + encoder_depth=32, + encoder_num_heads=16, + encoder_global_attn_indexes=[7, 15, 23, 31], + checkpoint=checkpoint, + ) + + +def build_sam_vit_l(checkpoint=None): + """Build and return a Segment Anything Model (SAM) l-size model.""" + return _build_sam( + encoder_embed_dim=1024, + encoder_depth=24, + encoder_num_heads=16, + encoder_global_attn_indexes=[5, 11, 17, 23], + checkpoint=checkpoint, + ) + + +def build_sam_vit_b(checkpoint=None): + """Build and return a Segment Anything Model (SAM) b-size model.""" + return _build_sam( + encoder_embed_dim=768, + encoder_depth=12, + encoder_num_heads=12, + encoder_global_attn_indexes=[2, 5, 8, 11], + checkpoint=checkpoint, + ) + + +def build_mobile_sam(checkpoint=None): + """Build and return Mobile Segment Anything Model (Mobile-SAM).""" + return _build_sam( + encoder_embed_dim=[64, 128, 160, 320], + encoder_depth=[2, 2, 6, 2], + encoder_num_heads=[2, 4, 5, 10], + encoder_global_attn_indexes=None, + mobile_sam=True, + checkpoint=checkpoint, + ) + + +def _build_sam( + encoder_embed_dim, encoder_depth, encoder_num_heads, encoder_global_attn_indexes, checkpoint=None, mobile_sam=False +): + """Builds the selected SAM model architecture.""" + prompt_embed_dim = 256 + image_size = 1024 + vit_patch_size = 16 + image_embedding_size = image_size // vit_patch_size + image_encoder = ( + TinyViT( + img_size=1024, + in_chans=3, + num_classes=1000, + embed_dims=encoder_embed_dim, + depths=encoder_depth, + num_heads=encoder_num_heads, + window_sizes=[7, 7, 14, 7], + mlp_ratio=4.0, + drop_rate=0.0, + drop_path_rate=0.0, + use_checkpoint=False, + mbconv_expand_ratio=4.0, + local_conv_size=3, + layer_lr_decay=0.8, + ) + if mobile_sam + else ImageEncoderViT( + depth=encoder_depth, + embed_dim=encoder_embed_dim, + img_size=image_size, + mlp_ratio=4, + norm_layer=partial(torch.nn.LayerNorm, eps=1e-6), + num_heads=encoder_num_heads, + patch_size=vit_patch_size, + qkv_bias=True, + use_rel_pos=True, + global_attn_indexes=encoder_global_attn_indexes, + window_size=14, + out_chans=prompt_embed_dim, + ) + ) + sam = Sam( + image_encoder=image_encoder, + prompt_encoder=PromptEncoder( + embed_dim=prompt_embed_dim, + image_embedding_size=(image_embedding_size, image_embedding_size), + input_image_size=(image_size, image_size), + mask_in_chans=16, + ), + mask_decoder=MaskDecoder( + num_multimask_outputs=3, + transformer=TwoWayTransformer( + depth=2, + embedding_dim=prompt_embed_dim, + mlp_dim=2048, + num_heads=8, + ), + transformer_dim=prompt_embed_dim, + iou_head_depth=3, + iou_head_hidden_dim=256, + ), + pixel_mean=[123.675, 116.28, 103.53], + pixel_std=[58.395, 57.12, 57.375], + ) + if checkpoint is not None: + checkpoint = attempt_download_asset(checkpoint) + with open(checkpoint, "rb") as f: + state_dict = torch.load(f) + sam.load_state_dict(state_dict) + sam.eval() + # sam.load_state_dict(torch.load(checkpoint), strict=True) + # sam.eval() + return sam + + +sam_model_map = { + "sam_h.pt": build_sam_vit_h, + "sam_l.pt": build_sam_vit_l, + "sam_b.pt": build_sam_vit_b, + "mobile_sam.pt": build_mobile_sam, +} + + +def build_sam(ckpt="sam_b.pt"): + """Build a SAM model specified by ckpt.""" + model_builder = None + ckpt = str(ckpt) # to allow Path ckpt types + for k in sam_model_map.keys(): + if ckpt.endswith(k): + model_builder = sam_model_map.get(k) + + if not model_builder: + raise FileNotFoundError(f"{ckpt} is not a supported SAM model. Available models are: \n {sam_model_map.keys()}") + + return model_builder(ckpt) diff --git a/ultralytics/models/sam/model.py b/ultralytics/models/sam/model.py new file mode 100644 index 0000000..cb12bc7 --- /dev/null +++ b/ultralytics/models/sam/model.py @@ -0,0 +1,114 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +""" +SAM model interface. + +This module provides an interface to the Segment Anything Model (SAM) from Ultralytics, designed for real-time image +segmentation tasks. The SAM model allows for promptable segmentation with unparalleled versatility in image analysis, +and has been trained on the SA-1B dataset. It features zero-shot performance capabilities, enabling it to adapt to new +image distributions and tasks without prior knowledge. + +Key Features: + - Promptable segmentation + - Real-time performance + - Zero-shot transfer capabilities + - Trained on SA-1B dataset +""" + +from pathlib import Path + +from ultralytics.engine.model import Model +from ultralytics.utils.torch_utils import model_info +from .build import build_sam +from .predict import Predictor + + +class SAM(Model): + """ + SAM (Segment Anything Model) interface class. + + SAM is designed for promptable real-time image segmentation. It can be used with a variety of prompts such as + bounding boxes, points, or labels. The model has capabilities for zero-shot performance and is trained on the SA-1B + dataset. + """ + + def __init__(self, model="sam_b.pt") -> None: + """ + Initializes the SAM model with a pre-trained model file. + + Args: + model (str): Path to the pre-trained SAM model file. File should have a .pt or .pth extension. + + Raises: + NotImplementedError: If the model file extension is not .pt or .pth. + """ + if model and Path(model).suffix not in (".pt", ".pth"): + raise NotImplementedError("SAM prediction requires pre-trained *.pt or *.pth model.") + super().__init__(model=model, task="segment") + + def _load(self, weights: str, task=None): + """ + Loads the specified weights into the SAM model. + + Args: + weights (str): Path to the weights file. + task (str, optional): Task name. Defaults to None. + """ + self.model = build_sam(weights) + + def predict(self, source, stream=False, bboxes=None, points=None, labels=None, **kwargs): + """ + Performs segmentation prediction on the given image or video source. + + Args: + source (str): Path to the image or video file, or a PIL.Image object, or a numpy.ndarray object. + stream (bool, optional): If True, enables real-time streaming. Defaults to False. + bboxes (list, optional): List of bounding box coordinates for prompted segmentation. Defaults to None. + points (list, optional): List of points for prompted segmentation. Defaults to None. + labels (list, optional): List of labels for prompted segmentation. Defaults to None. + + Returns: + (list): The model predictions. + """ + overrides = dict(conf=0.25, task="segment", mode="predict", imgsz=1024) + kwargs.update(overrides) + prompts = dict(bboxes=bboxes, points=points, labels=labels) + return super().predict(source, stream, prompts=prompts, **kwargs) + + def __call__(self, source=None, stream=False, bboxes=None, points=None, labels=None, **kwargs): + """ + Alias for the 'predict' method. + + Args: + source (str): Path to the image or video file, or a PIL.Image object, or a numpy.ndarray object. + stream (bool, optional): If True, enables real-time streaming. Defaults to False. + bboxes (list, optional): List of bounding box coordinates for prompted segmentation. Defaults to None. + points (list, optional): List of points for prompted segmentation. Defaults to None. + labels (list, optional): List of labels for prompted segmentation. Defaults to None. + + Returns: + (list): The model predictions. + """ + return self.predict(source, stream, bboxes, points, labels, **kwargs) + + def info(self, detailed=False, verbose=True): + """ + Logs information about the SAM model. + + Args: + detailed (bool, optional): If True, displays detailed information about the model. Defaults to False. + verbose (bool, optional): If True, displays information on the console. Defaults to True. + + Returns: + (tuple): A tuple containing the model's information. + """ + return model_info(self.model, detailed=detailed, verbose=verbose) + + @property + def task_map(self): + """ + Provides a mapping from the 'segment' task to its corresponding 'Predictor'. + + Returns: + (dict): A dictionary mapping the 'segment' task to its corresponding 'Predictor'. + """ + return {"segment": {"predictor": Predictor}} diff --git a/ultralytics/models/sam/modules/__init__.py b/ultralytics/models/sam/modules/__init__.py new file mode 100644 index 0000000..9e68dc1 --- /dev/null +++ b/ultralytics/models/sam/modules/__init__.py @@ -0,0 +1 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license diff --git a/ultralytics/models/sam/modules/__pycache__/__init__.cpython-312.pyc b/ultralytics/models/sam/modules/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..5af7c4f Binary files /dev/null and b/ultralytics/models/sam/modules/__pycache__/__init__.cpython-312.pyc differ diff --git a/ultralytics/models/sam/modules/__pycache__/__init__.cpython-39.pyc b/ultralytics/models/sam/modules/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..f3e51cb Binary files /dev/null and b/ultralytics/models/sam/modules/__pycache__/__init__.cpython-39.pyc differ diff --git a/ultralytics/models/sam/modules/__pycache__/decoders.cpython-312.pyc b/ultralytics/models/sam/modules/__pycache__/decoders.cpython-312.pyc new file mode 100644 index 0000000..e2bc1a8 Binary files /dev/null and b/ultralytics/models/sam/modules/__pycache__/decoders.cpython-312.pyc differ diff --git a/ultralytics/models/sam/modules/__pycache__/decoders.cpython-39.pyc b/ultralytics/models/sam/modules/__pycache__/decoders.cpython-39.pyc new file mode 100644 index 0000000..bae68e2 Binary files /dev/null and b/ultralytics/models/sam/modules/__pycache__/decoders.cpython-39.pyc differ diff --git a/ultralytics/models/sam/modules/__pycache__/encoders.cpython-312.pyc b/ultralytics/models/sam/modules/__pycache__/encoders.cpython-312.pyc new file mode 100644 index 0000000..f3ad2cc Binary files /dev/null and b/ultralytics/models/sam/modules/__pycache__/encoders.cpython-312.pyc differ diff --git a/ultralytics/models/sam/modules/__pycache__/encoders.cpython-39.pyc b/ultralytics/models/sam/modules/__pycache__/encoders.cpython-39.pyc new file mode 100644 index 0000000..1a0e848 Binary files /dev/null and b/ultralytics/models/sam/modules/__pycache__/encoders.cpython-39.pyc differ diff --git a/ultralytics/models/sam/modules/__pycache__/sam.cpython-312.pyc b/ultralytics/models/sam/modules/__pycache__/sam.cpython-312.pyc new file mode 100644 index 0000000..dc46580 Binary files /dev/null and b/ultralytics/models/sam/modules/__pycache__/sam.cpython-312.pyc differ diff --git a/ultralytics/models/sam/modules/__pycache__/sam.cpython-39.pyc b/ultralytics/models/sam/modules/__pycache__/sam.cpython-39.pyc new file mode 100644 index 0000000..1b16234 Binary files /dev/null and b/ultralytics/models/sam/modules/__pycache__/sam.cpython-39.pyc differ diff --git a/ultralytics/models/sam/modules/__pycache__/tiny_encoder.cpython-312.pyc b/ultralytics/models/sam/modules/__pycache__/tiny_encoder.cpython-312.pyc new file mode 100644 index 0000000..0e80fea Binary files /dev/null and b/ultralytics/models/sam/modules/__pycache__/tiny_encoder.cpython-312.pyc differ diff --git a/ultralytics/models/sam/modules/__pycache__/tiny_encoder.cpython-39.pyc b/ultralytics/models/sam/modules/__pycache__/tiny_encoder.cpython-39.pyc new file mode 100644 index 0000000..4fb47c1 Binary files /dev/null and b/ultralytics/models/sam/modules/__pycache__/tiny_encoder.cpython-39.pyc differ diff --git a/ultralytics/models/sam/modules/__pycache__/transformer.cpython-312.pyc b/ultralytics/models/sam/modules/__pycache__/transformer.cpython-312.pyc new file mode 100644 index 0000000..8ad5fdc Binary files /dev/null and b/ultralytics/models/sam/modules/__pycache__/transformer.cpython-312.pyc differ diff --git a/ultralytics/models/sam/modules/__pycache__/transformer.cpython-39.pyc b/ultralytics/models/sam/modules/__pycache__/transformer.cpython-39.pyc new file mode 100644 index 0000000..9a69d56 Binary files /dev/null and b/ultralytics/models/sam/modules/__pycache__/transformer.cpython-39.pyc differ diff --git a/ultralytics/models/sam/modules/decoders.py b/ultralytics/models/sam/modules/decoders.py new file mode 100644 index 0000000..073b1ad --- /dev/null +++ b/ultralytics/models/sam/modules/decoders.py @@ -0,0 +1,190 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from typing import List, Tuple, Type + +import torch +from torch import nn +from torch.nn import functional as F + +from ultralytics.nn.modules import LayerNorm2d + + +class MaskDecoder(nn.Module): + """ + Decoder module for generating masks and their associated quality scores, using a transformer architecture to predict + masks given image and prompt embeddings. + + Attributes: + transformer_dim (int): Channel dimension for the transformer module. + transformer (nn.Module): The transformer module used for mask prediction. + num_multimask_outputs (int): Number of masks to predict for disambiguating masks. + iou_token (nn.Embedding): Embedding for the IoU token. + num_mask_tokens (int): Number of mask tokens. + mask_tokens (nn.Embedding): Embedding for the mask tokens. + output_upscaling (nn.Sequential): Neural network sequence for upscaling the output. + output_hypernetworks_mlps (nn.ModuleList): Hypernetwork MLPs for generating masks. + iou_prediction_head (nn.Module): MLP for predicting mask quality. + """ + + def __init__( + self, + *, + transformer_dim: int, + transformer: nn.Module, + num_multimask_outputs: int = 3, + activation: Type[nn.Module] = nn.GELU, + iou_head_depth: int = 3, + iou_head_hidden_dim: int = 256, + ) -> None: + """ + Predicts masks given an image and prompt embeddings, using a transformer architecture. + + Args: + transformer_dim (int): the channel dimension of the transformer module + transformer (nn.Module): the transformer used to predict masks + num_multimask_outputs (int): the number of masks to predict when disambiguating masks + activation (nn.Module): the type of activation to use when upscaling masks + iou_head_depth (int): the depth of the MLP used to predict mask quality + iou_head_hidden_dim (int): the hidden dimension of the MLP used to predict mask quality + """ + super().__init__() + self.transformer_dim = transformer_dim + self.transformer = transformer + + self.num_multimask_outputs = num_multimask_outputs + + self.iou_token = nn.Embedding(1, transformer_dim) + self.num_mask_tokens = num_multimask_outputs + 1 + self.mask_tokens = nn.Embedding(self.num_mask_tokens, transformer_dim) + + self.output_upscaling = nn.Sequential( + nn.ConvTranspose2d(transformer_dim, transformer_dim // 4, kernel_size=2, stride=2), + LayerNorm2d(transformer_dim // 4), + activation(), + nn.ConvTranspose2d(transformer_dim // 4, transformer_dim // 8, kernel_size=2, stride=2), + activation(), + ) + self.output_hypernetworks_mlps = nn.ModuleList( + [MLP(transformer_dim, transformer_dim, transformer_dim // 8, 3) for _ in range(self.num_mask_tokens)] + ) + + self.iou_prediction_head = MLP(transformer_dim, iou_head_hidden_dim, self.num_mask_tokens, iou_head_depth) + + def forward( + self, + image_embeddings: torch.Tensor, + image_pe: torch.Tensor, + sparse_prompt_embeddings: torch.Tensor, + dense_prompt_embeddings: torch.Tensor, + multimask_output: bool, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Predict masks given image and prompt embeddings. + + Args: + image_embeddings (torch.Tensor): the embeddings from the image encoder + image_pe (torch.Tensor): positional encoding with the shape of image_embeddings + sparse_prompt_embeddings (torch.Tensor): the embeddings of the points and boxes + dense_prompt_embeddings (torch.Tensor): the embeddings of the mask inputs + multimask_output (bool): Whether to return multiple masks or a single mask. + + Returns: + torch.Tensor: batched predicted masks + torch.Tensor: batched predictions of mask quality + """ + masks, iou_pred = self.predict_masks( + image_embeddings=image_embeddings, + image_pe=image_pe, + sparse_prompt_embeddings=sparse_prompt_embeddings, + dense_prompt_embeddings=dense_prompt_embeddings, + ) + + # Select the correct mask or masks for output + mask_slice = slice(1, None) if multimask_output else slice(0, 1) + masks = masks[:, mask_slice, :, :] + iou_pred = iou_pred[:, mask_slice] + + # Prepare output + return masks, iou_pred + + def predict_masks( + self, + image_embeddings: torch.Tensor, + image_pe: torch.Tensor, + sparse_prompt_embeddings: torch.Tensor, + dense_prompt_embeddings: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Predicts masks. + + See 'forward' for more details. + """ + # Concatenate output tokens + output_tokens = torch.cat([self.iou_token.weight, self.mask_tokens.weight], dim=0) + output_tokens = output_tokens.unsqueeze(0).expand(sparse_prompt_embeddings.shape[0], -1, -1) + tokens = torch.cat((output_tokens, sparse_prompt_embeddings), dim=1) + + # Expand per-image data in batch direction to be per-mask + src = torch.repeat_interleave(image_embeddings, tokens.shape[0], dim=0) + src = src + dense_prompt_embeddings + pos_src = torch.repeat_interleave(image_pe, tokens.shape[0], dim=0) + b, c, h, w = src.shape + + # Run the transformer + hs, src = self.transformer(src, pos_src, tokens) + iou_token_out = hs[:, 0, :] + mask_tokens_out = hs[:, 1 : (1 + self.num_mask_tokens), :] + + # Upscale mask embeddings and predict masks using the mask tokens + src = src.transpose(1, 2).view(b, c, h, w) + upscaled_embedding = self.output_upscaling(src) + hyper_in_list: List[torch.Tensor] = [ + self.output_hypernetworks_mlps[i](mask_tokens_out[:, i, :]) for i in range(self.num_mask_tokens) + ] + hyper_in = torch.stack(hyper_in_list, dim=1) + b, c, h, w = upscaled_embedding.shape + masks = (hyper_in @ upscaled_embedding.view(b, c, h * w)).view(b, -1, h, w) + + # Generate mask quality predictions + iou_pred = self.iou_prediction_head(iou_token_out) + + return masks, iou_pred + + +class MLP(nn.Module): + """ + MLP (Multi-Layer Perceptron) model lightly adapted from + https://github.com/facebookresearch/MaskFormer/blob/main/mask_former/modeling/transformer/transformer_predictor.py + """ + + def __init__( + self, + input_dim: int, + hidden_dim: int, + output_dim: int, + num_layers: int, + sigmoid_output: bool = False, + ) -> None: + """ + Initializes the MLP (Multi-Layer Perceptron) model. + + Args: + input_dim (int): The dimensionality of the input features. + hidden_dim (int): The dimensionality of the hidden layers. + output_dim (int): The dimensionality of the output layer. + num_layers (int): The number of hidden layers. + sigmoid_output (bool, optional): Apply a sigmoid activation to the output layer. Defaults to False. + """ + super().__init__() + self.num_layers = num_layers + h = [hidden_dim] * (num_layers - 1) + self.layers = nn.ModuleList(nn.Linear(n, k) for n, k in zip([input_dim] + h, h + [output_dim])) + self.sigmoid_output = sigmoid_output + + def forward(self, x): + """Executes feedforward within the neural network module and applies activation.""" + for i, layer in enumerate(self.layers): + x = F.relu(layer(x)) if i < self.num_layers - 1 else layer(x) + if self.sigmoid_output: + x = torch.sigmoid(x) + return x diff --git a/ultralytics/models/sam/modules/encoders.py b/ultralytics/models/sam/modules/encoders.py new file mode 100644 index 0000000..a51c347 --- /dev/null +++ b/ultralytics/models/sam/modules/encoders.py @@ -0,0 +1,603 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from typing import Any, Optional, Tuple, Type + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ultralytics.nn.modules import LayerNorm2d, MLPBlock + + +class ImageEncoderViT(nn.Module): + """ + An image encoder using Vision Transformer (ViT) architecture for encoding an image into a compact latent space. The + encoder takes an image, splits it into patches, and processes these patches through a series of transformer blocks. + The encoded patches are then processed through a neck to generate the final encoded representation. + + This class and its supporting functions below lightly adapted from the ViTDet backbone available at + https://github.com/facebookresearch/detectron2/blob/main/detectron2/modeling/backbone/vit.py. + + Attributes: + img_size (int): Dimension of input images, assumed to be square. + patch_embed (PatchEmbed): Module for patch embedding. + pos_embed (nn.Parameter, optional): Absolute positional embedding for patches. + blocks (nn.ModuleList): List of transformer blocks for processing patch embeddings. + neck (nn.Sequential): Neck module to further process the output. + """ + + def __init__( + self, + img_size: int = 1024, + patch_size: int = 16, + in_chans: int = 3, + embed_dim: int = 768, + depth: int = 12, + num_heads: int = 12, + mlp_ratio: float = 4.0, + out_chans: int = 256, + qkv_bias: bool = True, + norm_layer: Type[nn.Module] = nn.LayerNorm, + act_layer: Type[nn.Module] = nn.GELU, + use_abs_pos: bool = True, + use_rel_pos: bool = False, + rel_pos_zero_init: bool = True, + window_size: int = 0, + global_attn_indexes: Tuple[int, ...] = (), + ) -> None: + """ + Args: + img_size (int): Input image size. + patch_size (int): Patch size. + in_chans (int): Number of input image channels. + embed_dim (int): Patch embedding dimension. + depth (int): Depth of ViT. + num_heads (int): Number of attention heads in each ViT block. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool): If True, add a learnable bias to query, key, value. + norm_layer (nn.Module): Normalization layer. + act_layer (nn.Module): Activation layer. + use_abs_pos (bool): If True, use absolute positional embeddings. + use_rel_pos (bool): If True, add relative positional embeddings to the attention map. + rel_pos_zero_init (bool): If True, zero initialize relative positional parameters. + window_size (int): Window size for window attention blocks. + global_attn_indexes (list): Indexes for blocks using global attention. + """ + super().__init__() + self.img_size = img_size + + self.patch_embed = PatchEmbed( + kernel_size=(patch_size, patch_size), + stride=(patch_size, patch_size), + in_chans=in_chans, + embed_dim=embed_dim, + ) + + self.pos_embed: Optional[nn.Parameter] = None + if use_abs_pos: + # Initialize absolute positional embedding with pretrain image size. + self.pos_embed = nn.Parameter(torch.zeros(1, img_size // patch_size, img_size // patch_size, embed_dim)) + + self.blocks = nn.ModuleList() + for i in range(depth): + block = Block( + dim=embed_dim, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + norm_layer=norm_layer, + act_layer=act_layer, + use_rel_pos=use_rel_pos, + rel_pos_zero_init=rel_pos_zero_init, + window_size=window_size if i not in global_attn_indexes else 0, + input_size=(img_size // patch_size, img_size // patch_size), + ) + self.blocks.append(block) + + self.neck = nn.Sequential( + nn.Conv2d( + embed_dim, + out_chans, + kernel_size=1, + bias=False, + ), + LayerNorm2d(out_chans), + nn.Conv2d( + out_chans, + out_chans, + kernel_size=3, + padding=1, + bias=False, + ), + LayerNorm2d(out_chans), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Processes input through patch embedding, applies positional embedding if present, and passes through blocks + and neck. + """ + x = self.patch_embed(x) + if self.pos_embed is not None: + x = x + self.pos_embed + for blk in self.blocks: + x = blk(x) + return self.neck(x.permute(0, 3, 1, 2)) + + +class PromptEncoder(nn.Module): + """ + Encodes different types of prompts, including points, boxes, and masks, for input to SAM's mask decoder. The encoder + produces both sparse and dense embeddings for the input prompts. + + Attributes: + embed_dim (int): Dimension of the embeddings. + input_image_size (Tuple[int, int]): Size of the input image as (H, W). + image_embedding_size (Tuple[int, int]): Spatial size of the image embedding as (H, W). + pe_layer (PositionEmbeddingRandom): Module for random position embedding. + num_point_embeddings (int): Number of point embeddings for different types of points. + point_embeddings (nn.ModuleList): List of point embeddings. + not_a_point_embed (nn.Embedding): Embedding for points that are not a part of any label. + mask_input_size (Tuple[int, int]): Size of the input mask. + mask_downscaling (nn.Sequential): Neural network for downscaling the mask. + no_mask_embed (nn.Embedding): Embedding for cases where no mask is provided. + """ + + def __init__( + self, + embed_dim: int, + image_embedding_size: Tuple[int, int], + input_image_size: Tuple[int, int], + mask_in_chans: int, + activation: Type[nn.Module] = nn.GELU, + ) -> None: + """ + Encodes prompts for input to SAM's mask decoder. + + Args: + embed_dim (int): The prompts' embedding dimension + image_embedding_size (tuple(int, int)): The spatial size of the + image embedding, as (H, W). + input_image_size (int): The padded size of the image as input + to the image encoder, as (H, W). + mask_in_chans (int): The number of hidden channels used for + encoding input masks. + activation (nn.Module): The activation to use when encoding + input masks. + """ + super().__init__() + self.embed_dim = embed_dim + self.input_image_size = input_image_size + self.image_embedding_size = image_embedding_size + self.pe_layer = PositionEmbeddingRandom(embed_dim // 2) + + self.num_point_embeddings: int = 4 # pos/neg point + 2 box corners + point_embeddings = [nn.Embedding(1, embed_dim) for _ in range(self.num_point_embeddings)] + self.point_embeddings = nn.ModuleList(point_embeddings) + self.not_a_point_embed = nn.Embedding(1, embed_dim) + + self.mask_input_size = (4 * image_embedding_size[0], 4 * image_embedding_size[1]) + self.mask_downscaling = nn.Sequential( + nn.Conv2d(1, mask_in_chans // 4, kernel_size=2, stride=2), + LayerNorm2d(mask_in_chans // 4), + activation(), + nn.Conv2d(mask_in_chans // 4, mask_in_chans, kernel_size=2, stride=2), + LayerNorm2d(mask_in_chans), + activation(), + nn.Conv2d(mask_in_chans, embed_dim, kernel_size=1), + ) + self.no_mask_embed = nn.Embedding(1, embed_dim) + + def get_dense_pe(self) -> torch.Tensor: + """ + Returns the positional encoding used to encode point prompts, applied to a dense set of points the shape of the + image encoding. + + Returns: + torch.Tensor: Positional encoding with shape 1x(embed_dim)x(embedding_h)x(embedding_w) + """ + return self.pe_layer(self.image_embedding_size).unsqueeze(0) + + def _embed_points(self, points: torch.Tensor, labels: torch.Tensor, pad: bool) -> torch.Tensor: + """Embeds point prompts.""" + points = points + 0.5 # Shift to center of pixel + if pad: + padding_point = torch.zeros((points.shape[0], 1, 2), device=points.device) + padding_label = -torch.ones((labels.shape[0], 1), device=labels.device) + points = torch.cat([points, padding_point], dim=1) + labels = torch.cat([labels, padding_label], dim=1) + point_embedding = self.pe_layer.forward_with_coords(points, self.input_image_size) + point_embedding[labels == -1] = 0.0 + point_embedding[labels == -1] += self.not_a_point_embed.weight + point_embedding[labels == 0] += self.point_embeddings[0].weight + point_embedding[labels == 1] += self.point_embeddings[1].weight + return point_embedding + + def _embed_boxes(self, boxes: torch.Tensor) -> torch.Tensor: + """Embeds box prompts.""" + boxes = boxes + 0.5 # Shift to center of pixel + coords = boxes.reshape(-1, 2, 2) + corner_embedding = self.pe_layer.forward_with_coords(coords, self.input_image_size) + corner_embedding[:, 0, :] += self.point_embeddings[2].weight + corner_embedding[:, 1, :] += self.point_embeddings[3].weight + return corner_embedding + + def _embed_masks(self, masks: torch.Tensor) -> torch.Tensor: + """Embeds mask inputs.""" + return self.mask_downscaling(masks) + + def _get_batch_size( + self, + points: Optional[Tuple[torch.Tensor, torch.Tensor]], + boxes: Optional[torch.Tensor], + masks: Optional[torch.Tensor], + ) -> int: + """Gets the batch size of the output given the batch size of the input prompts.""" + if points is not None: + return points[0].shape[0] + elif boxes is not None: + return boxes.shape[0] + elif masks is not None: + return masks.shape[0] + else: + return 1 + + def _get_device(self) -> torch.device: + """Returns the device of the first point embedding's weight tensor.""" + return self.point_embeddings[0].weight.device + + def forward( + self, + points: Optional[Tuple[torch.Tensor, torch.Tensor]], + boxes: Optional[torch.Tensor], + masks: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Embeds different types of prompts, returning both sparse and dense embeddings. + + Args: + points (tuple(torch.Tensor, torch.Tensor), None): point coordinates and labels to embed. + boxes (torch.Tensor, None): boxes to embed + masks (torch.Tensor, None): masks to embed + + Returns: + torch.Tensor: sparse embeddings for the points and boxes, with shape BxNx(embed_dim), where N is determined + by the number of input points and boxes. + torch.Tensor: dense embeddings for the masks, in the shape Bx(embed_dim)x(embed_H)x(embed_W) + """ + bs = self._get_batch_size(points, boxes, masks) + sparse_embeddings = torch.empty((bs, 0, self.embed_dim), device=self._get_device()) + if points is not None: + coords, labels = points + point_embeddings = self._embed_points(coords, labels, pad=(boxes is None)) + sparse_embeddings = torch.cat([sparse_embeddings, point_embeddings], dim=1) + if boxes is not None: + box_embeddings = self._embed_boxes(boxes) + sparse_embeddings = torch.cat([sparse_embeddings, box_embeddings], dim=1) + + if masks is not None: + dense_embeddings = self._embed_masks(masks) + else: + dense_embeddings = self.no_mask_embed.weight.reshape(1, -1, 1, 1).expand( + bs, -1, self.image_embedding_size[0], self.image_embedding_size[1] + ) + + return sparse_embeddings, dense_embeddings + + +class PositionEmbeddingRandom(nn.Module): + """Positional encoding using random spatial frequencies.""" + + def __init__(self, num_pos_feats: int = 64, scale: Optional[float] = None) -> None: + """Initializes a position embedding using random spatial frequencies.""" + super().__init__() + if scale is None or scale <= 0.0: + scale = 1.0 + self.register_buffer("positional_encoding_gaussian_matrix", scale * torch.randn((2, num_pos_feats))) + + # Set non-deterministic for forward() error 'cumsum_cuda_kernel does not have a deterministic implementation' + torch.use_deterministic_algorithms(False) + torch.backends.cudnn.deterministic = False + + def _pe_encoding(self, coords: torch.Tensor) -> torch.Tensor: + """Positionally encode points that are normalized to [0,1].""" + # Assuming coords are in [0, 1]^2 square and have d_1 x ... x d_n x 2 shape + coords = 2 * coords - 1 + coords = coords @ self.positional_encoding_gaussian_matrix + coords = 2 * np.pi * coords + # Outputs d_1 x ... x d_n x C shape + return torch.cat([torch.sin(coords), torch.cos(coords)], dim=-1) + + def forward(self, size: Tuple[int, int]) -> torch.Tensor: + """Generate positional encoding for a grid of the specified size.""" + h, w = size + device: Any = self.positional_encoding_gaussian_matrix.device + grid = torch.ones((h, w), device=device, dtype=torch.float32) + y_embed = grid.cumsum(dim=0) - 0.5 + x_embed = grid.cumsum(dim=1) - 0.5 + y_embed = y_embed / h + x_embed = x_embed / w + + pe = self._pe_encoding(torch.stack([x_embed, y_embed], dim=-1)) + return pe.permute(2, 0, 1) # C x H x W + + def forward_with_coords(self, coords_input: torch.Tensor, image_size: Tuple[int, int]) -> torch.Tensor: + """Positionally encode points that are not normalized to [0,1].""" + coords = coords_input.clone() + coords[:, :, 0] = coords[:, :, 0] / image_size[1] + coords[:, :, 1] = coords[:, :, 1] / image_size[0] + return self._pe_encoding(coords.to(torch.float)) # B x N x C + + +class Block(nn.Module): + """Transformer blocks with support of window attention and residual propagation blocks.""" + + def __init__( + self, + dim: int, + num_heads: int, + mlp_ratio: float = 4.0, + qkv_bias: bool = True, + norm_layer: Type[nn.Module] = nn.LayerNorm, + act_layer: Type[nn.Module] = nn.GELU, + use_rel_pos: bool = False, + rel_pos_zero_init: bool = True, + window_size: int = 0, + input_size: Optional[Tuple[int, int]] = None, + ) -> None: + """ + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads in each ViT block. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool): If True, add a learnable bias to query, key, value. + norm_layer (nn.Module): Normalization layer. + act_layer (nn.Module): Activation layer. + use_rel_pos (bool): If True, add relative positional embeddings to the attention map. + rel_pos_zero_init (bool): If True, zero initialize relative positional parameters. + window_size (int): Window size for window attention blocks. If it equals 0, then + use global attention. + input_size (tuple(int, int), None): Input resolution for calculating the relative + positional parameter size. + """ + super().__init__() + self.norm1 = norm_layer(dim) + self.attn = Attention( + dim, + num_heads=num_heads, + qkv_bias=qkv_bias, + use_rel_pos=use_rel_pos, + rel_pos_zero_init=rel_pos_zero_init, + input_size=input_size if window_size == 0 else (window_size, window_size), + ) + + self.norm2 = norm_layer(dim) + self.mlp = MLPBlock(embedding_dim=dim, mlp_dim=int(dim * mlp_ratio), act=act_layer) + + self.window_size = window_size + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Executes a forward pass through the transformer block with window attention and non-overlapping windows.""" + shortcut = x + x = self.norm1(x) + # Window partition + if self.window_size > 0: + H, W = x.shape[1], x.shape[2] + x, pad_hw = window_partition(x, self.window_size) + + x = self.attn(x) + # Reverse window partition + if self.window_size > 0: + x = window_unpartition(x, self.window_size, pad_hw, (H, W)) + + x = shortcut + x + return x + self.mlp(self.norm2(x)) + + +class Attention(nn.Module): + """Multi-head Attention block with relative position embeddings.""" + + def __init__( + self, + dim: int, + num_heads: int = 8, + qkv_bias: bool = True, + use_rel_pos: bool = False, + rel_pos_zero_init: bool = True, + input_size: Optional[Tuple[int, int]] = None, + ) -> None: + """ + Initialize Attention module. + + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. + qkv_bias (bool): If True, add a learnable bias to query, key, value. + rel_pos_zero_init (bool): If True, zero initialize relative positional parameters. + input_size (tuple(int, int), None): Input resolution for calculating the relative + positional parameter size. + """ + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = head_dim**-0.5 + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.proj = nn.Linear(dim, dim) + + self.use_rel_pos = use_rel_pos + if self.use_rel_pos: + assert input_size is not None, "Input size must be provided if using relative positional encoding." + # Initialize relative positional embeddings + self.rel_pos_h = nn.Parameter(torch.zeros(2 * input_size[0] - 1, head_dim)) + self.rel_pos_w = nn.Parameter(torch.zeros(2 * input_size[1] - 1, head_dim)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Applies the forward operation including attention, normalization, MLP, and indexing within window limits.""" + B, H, W, _ = x.shape + # qkv with shape (3, B, nHead, H * W, C) + qkv = self.qkv(x).reshape(B, H * W, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) + # q, k, v with shape (B * nHead, H * W, C) + q, k, v = qkv.reshape(3, B * self.num_heads, H * W, -1).unbind(0) + + attn = (q * self.scale) @ k.transpose(-2, -1) + + if self.use_rel_pos: + attn = add_decomposed_rel_pos(attn, q, self.rel_pos_h, self.rel_pos_w, (H, W), (H, W)) + + attn = attn.softmax(dim=-1) + x = (attn @ v).view(B, self.num_heads, H, W, -1).permute(0, 2, 3, 1, 4).reshape(B, H, W, -1) + return self.proj(x) + + +def window_partition(x: torch.Tensor, window_size: int) -> Tuple[torch.Tensor, Tuple[int, int]]: + """ + Partition into non-overlapping windows with padding if needed. + Args: + x (tensor): input tokens with [B, H, W, C]. + window_size (int): window size. + + Returns: + windows: windows after partition with [B * num_windows, window_size, window_size, C]. + (Hp, Wp): padded height and width before partition + """ + B, H, W, C = x.shape + + pad_h = (window_size - H % window_size) % window_size + pad_w = (window_size - W % window_size) % window_size + if pad_h > 0 or pad_w > 0: + x = F.pad(x, (0, 0, 0, pad_w, 0, pad_h)) + Hp, Wp = H + pad_h, W + pad_w + + x = x.view(B, Hp // window_size, window_size, Wp // window_size, window_size, C) + windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C) + return windows, (Hp, Wp) + + +def window_unpartition( + windows: torch.Tensor, window_size: int, pad_hw: Tuple[int, int], hw: Tuple[int, int] +) -> torch.Tensor: + """ + Window unpartition into original sequences and removing padding. + + Args: + windows (tensor): input tokens with [B * num_windows, window_size, window_size, C]. + window_size (int): window size. + pad_hw (Tuple): padded height and width (Hp, Wp). + hw (Tuple): original height and width (H, W) before padding. + + Returns: + x: unpartitioned sequences with [B, H, W, C]. + """ + Hp, Wp = pad_hw + H, W = hw + B = windows.shape[0] // (Hp * Wp // window_size // window_size) + x = windows.view(B, Hp // window_size, Wp // window_size, window_size, window_size, -1) + x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, Hp, Wp, -1) + + if Hp > H or Wp > W: + x = x[:, :H, :W, :].contiguous() + return x + + +def get_rel_pos(q_size: int, k_size: int, rel_pos: torch.Tensor) -> torch.Tensor: + """ + Get relative positional embeddings according to the relative positions of query and key sizes. + + Args: + q_size (int): size of query q. + k_size (int): size of key k. + rel_pos (Tensor): relative position embeddings (L, C). + + Returns: + Extracted positional embeddings according to relative positions. + """ + max_rel_dist = int(2 * max(q_size, k_size) - 1) + # Interpolate rel pos if needed. + if rel_pos.shape[0] != max_rel_dist: + # Interpolate rel pos. + rel_pos_resized = F.interpolate( + rel_pos.reshape(1, rel_pos.shape[0], -1).permute(0, 2, 1), + size=max_rel_dist, + mode="linear", + ) + rel_pos_resized = rel_pos_resized.reshape(-1, max_rel_dist).permute(1, 0) + else: + rel_pos_resized = rel_pos + + # Scale the coords with short length if shapes for q and k are different. + q_coords = torch.arange(q_size)[:, None] * max(k_size / q_size, 1.0) + k_coords = torch.arange(k_size)[None, :] * max(q_size / k_size, 1.0) + relative_coords = (q_coords - k_coords) + (k_size - 1) * max(q_size / k_size, 1.0) + + return rel_pos_resized[relative_coords.long()] + + +def add_decomposed_rel_pos( + attn: torch.Tensor, + q: torch.Tensor, + rel_pos_h: torch.Tensor, + rel_pos_w: torch.Tensor, + q_size: Tuple[int, int], + k_size: Tuple[int, int], +) -> torch.Tensor: + """ + Calculate decomposed Relative Positional Embeddings from mvitv2 paper at + https://github.com/facebookresearch/mvit/blob/main/mvit/models/attention.py. + + Args: + attn (Tensor): attention map. + q (Tensor): query q in the attention layer with shape (B, q_h * q_w, C). + rel_pos_h (Tensor): relative position embeddings (Lh, C) for height axis. + rel_pos_w (Tensor): relative position embeddings (Lw, C) for width axis. + q_size (Tuple): spatial sequence size of query q with (q_h, q_w). + k_size (Tuple): spatial sequence size of key k with (k_h, k_w). + + Returns: + attn (Tensor): attention map with added relative positional embeddings. + """ + q_h, q_w = q_size + k_h, k_w = k_size + Rh = get_rel_pos(q_h, k_h, rel_pos_h) + Rw = get_rel_pos(q_w, k_w, rel_pos_w) + + B, _, dim = q.shape + r_q = q.reshape(B, q_h, q_w, dim) + rel_h = torch.einsum("bhwc,hkc->bhwk", r_q, Rh) + rel_w = torch.einsum("bhwc,wkc->bhwk", r_q, Rw) + + attn = (attn.view(B, q_h, q_w, k_h, k_w) + rel_h[:, :, :, :, None] + rel_w[:, :, :, None, :]).view( + B, q_h * q_w, k_h * k_w + ) + + return attn + + +class PatchEmbed(nn.Module): + """Image to Patch Embedding.""" + + def __init__( + self, + kernel_size: Tuple[int, int] = (16, 16), + stride: Tuple[int, int] = (16, 16), + padding: Tuple[int, int] = (0, 0), + in_chans: int = 3, + embed_dim: int = 768, + ) -> None: + """ + Initialize PatchEmbed module. + + Args: + kernel_size (Tuple): kernel size of the projection layer. + stride (Tuple): stride of the projection layer. + padding (Tuple): padding size of the projection layer. + in_chans (int): Number of input image channels. + embed_dim (int): Patch embedding dimension. + """ + super().__init__() + + self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=kernel_size, stride=stride, padding=padding) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Computes patch embedding by applying convolution and transposing resulting tensor.""" + return self.proj(x).permute(0, 2, 3, 1) # B C H W -> B H W C diff --git a/ultralytics/models/sam/modules/sam.py b/ultralytics/models/sam/modules/sam.py new file mode 100644 index 0000000..95d9bbe --- /dev/null +++ b/ultralytics/models/sam/modules/sam.py @@ -0,0 +1,65 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. + +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +from typing import List + +import torch +from torch import nn + +from .decoders import MaskDecoder +from .encoders import ImageEncoderViT, PromptEncoder + + +class Sam(nn.Module): + """ + Sam (Segment Anything Model) is designed for object segmentation tasks. It uses image encoders to generate image + embeddings, and prompt encoders to encode various types of input prompts. These embeddings are then used by the mask + decoder to predict object masks. + + Attributes: + mask_threshold (float): Threshold value for mask prediction. + image_format (str): Format of the input image, default is 'RGB'. + image_encoder (ImageEncoderViT): The backbone used to encode the image into embeddings. + prompt_encoder (PromptEncoder): Encodes various types of input prompts. + mask_decoder (MaskDecoder): Predicts object masks from the image and prompt embeddings. + pixel_mean (List[float]): Mean pixel values for image normalization. + pixel_std (List[float]): Standard deviation values for image normalization. + """ + + mask_threshold: float = 0.0 + image_format: str = "RGB" + + def __init__( + self, + image_encoder: ImageEncoderViT, + prompt_encoder: PromptEncoder, + mask_decoder: MaskDecoder, + pixel_mean: List[float] = (123.675, 116.28, 103.53), + pixel_std: List[float] = (58.395, 57.12, 57.375), + ) -> None: + """ + Initialize the Sam class to predict object masks from an image and input prompts. + + Note: + All forward() operations moved to SAMPredictor. + + Args: + image_encoder (ImageEncoderViT): The backbone used to encode the image into image embeddings. + prompt_encoder (PromptEncoder): Encodes various types of input prompts. + mask_decoder (MaskDecoder): Predicts masks from the image embeddings and encoded prompts. + pixel_mean (List[float], optional): Mean values for normalizing pixels in the input image. Defaults to + (123.675, 116.28, 103.53). + pixel_std (List[float], optional): Std values for normalizing pixels in the input image. Defaults to + (58.395, 57.12, 57.375). + """ + super().__init__() + self.image_encoder = image_encoder + self.prompt_encoder = prompt_encoder + self.mask_decoder = mask_decoder + self.register_buffer("pixel_mean", torch.Tensor(pixel_mean).view(-1, 1, 1), False) + self.register_buffer("pixel_std", torch.Tensor(pixel_std).view(-1, 1, 1), False) diff --git a/ultralytics/models/sam/modules/tiny_encoder.py b/ultralytics/models/sam/modules/tiny_encoder.py new file mode 100644 index 0000000..98f5ac0 --- /dev/null +++ b/ultralytics/models/sam/modules/tiny_encoder.py @@ -0,0 +1,742 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +# -------------------------------------------------------- +# TinyViT Model Architecture +# Copyright (c) 2022 Microsoft +# Adapted from LeViT and Swin Transformer +# LeViT: (https://github.com/facebookresearch/levit) +# Swin: (https://github.com/microsoft/swin-transformer) +# Build the TinyViT Model +# -------------------------------------------------------- + +import itertools +from typing import Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint + +from ultralytics.utils.instance import to_2tuple + + +class Conv2d_BN(torch.nn.Sequential): + """A sequential container that performs 2D convolution followed by batch normalization.""" + + def __init__(self, a, b, ks=1, stride=1, pad=0, dilation=1, groups=1, bn_weight_init=1): + """Initializes the MBConv model with given input channels, output channels, expansion ratio, activation, and + drop path. + """ + super().__init__() + self.add_module("c", torch.nn.Conv2d(a, b, ks, stride, pad, dilation, groups, bias=False)) + bn = torch.nn.BatchNorm2d(b) + torch.nn.init.constant_(bn.weight, bn_weight_init) + torch.nn.init.constant_(bn.bias, 0) + self.add_module("bn", bn) + + +class PatchEmbed(nn.Module): + """Embeds images into patches and projects them into a specified embedding dimension.""" + + def __init__(self, in_chans, embed_dim, resolution, activation): + """Initialize the PatchMerging class with specified input, output dimensions, resolution and activation + function. + """ + super().__init__() + img_size: Tuple[int, int] = to_2tuple(resolution) + self.patches_resolution = (img_size[0] // 4, img_size[1] // 4) + self.num_patches = self.patches_resolution[0] * self.patches_resolution[1] + self.in_chans = in_chans + self.embed_dim = embed_dim + n = embed_dim + self.seq = nn.Sequential( + Conv2d_BN(in_chans, n // 2, 3, 2, 1), + activation(), + Conv2d_BN(n // 2, n, 3, 2, 1), + ) + + def forward(self, x): + """Runs input tensor 'x' through the PatchMerging model's sequence of operations.""" + return self.seq(x) + + +class MBConv(nn.Module): + """Mobile Inverted Bottleneck Conv (MBConv) layer, part of the EfficientNet architecture.""" + + def __init__(self, in_chans, out_chans, expand_ratio, activation, drop_path): + """Initializes a convolutional layer with specified dimensions, input resolution, depth, and activation + function. + """ + super().__init__() + self.in_chans = in_chans + self.hidden_chans = int(in_chans * expand_ratio) + self.out_chans = out_chans + + self.conv1 = Conv2d_BN(in_chans, self.hidden_chans, ks=1) + self.act1 = activation() + + self.conv2 = Conv2d_BN(self.hidden_chans, self.hidden_chans, ks=3, stride=1, pad=1, groups=self.hidden_chans) + self.act2 = activation() + + self.conv3 = Conv2d_BN(self.hidden_chans, out_chans, ks=1, bn_weight_init=0.0) + self.act3 = activation() + + # NOTE: `DropPath` is needed only for training. + # self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.drop_path = nn.Identity() + + def forward(self, x): + """Implements the forward pass for the model architecture.""" + shortcut = x + x = self.conv1(x) + x = self.act1(x) + x = self.conv2(x) + x = self.act2(x) + x = self.conv3(x) + x = self.drop_path(x) + x += shortcut + return self.act3(x) + + +class PatchMerging(nn.Module): + """Merges neighboring patches in the feature map and projects to a new dimension.""" + + def __init__(self, input_resolution, dim, out_dim, activation): + """Initializes the ConvLayer with specific dimension, input resolution, depth, activation, drop path, and other + optional parameters. + """ + super().__init__() + + self.input_resolution = input_resolution + self.dim = dim + self.out_dim = out_dim + self.act = activation() + self.conv1 = Conv2d_BN(dim, out_dim, 1, 1, 0) + stride_c = 1 if out_dim in [320, 448, 576] else 2 + self.conv2 = Conv2d_BN(out_dim, out_dim, 3, stride_c, 1, groups=out_dim) + self.conv3 = Conv2d_BN(out_dim, out_dim, 1, 1, 0) + + def forward(self, x): + """Applies forward pass on the input utilizing convolution and activation layers, and returns the result.""" + if x.ndim == 3: + H, W = self.input_resolution + B = len(x) + # (B, C, H, W) + x = x.view(B, H, W, -1).permute(0, 3, 1, 2) + + x = self.conv1(x) + x = self.act(x) + + x = self.conv2(x) + x = self.act(x) + x = self.conv3(x) + return x.flatten(2).transpose(1, 2) + + +class ConvLayer(nn.Module): + """ + Convolutional Layer featuring multiple MobileNetV3-style inverted bottleneck convolutions (MBConv). + + Optionally applies downsample operations to the output, and provides support for gradient checkpointing. + """ + + def __init__( + self, + dim, + input_resolution, + depth, + activation, + drop_path=0.0, + downsample=None, + use_checkpoint=False, + out_dim=None, + conv_expand_ratio=4.0, + ): + """ + Initializes the ConvLayer with the given dimensions and settings. + + Args: + dim (int): The dimensionality of the input and output. + input_resolution (Tuple[int, int]): The resolution of the input image. + depth (int): The number of MBConv layers in the block. + activation (Callable): Activation function applied after each convolution. + drop_path (Union[float, List[float]]): Drop path rate. Single float or a list of floats for each MBConv. + downsample (Optional[Callable]): Function for downsampling the output. None to skip downsampling. + use_checkpoint (bool): Whether to use gradient checkpointing to save memory. + out_dim (Optional[int]): The dimensionality of the output. None means it will be the same as `dim`. + conv_expand_ratio (float): Expansion ratio for the MBConv layers. + """ + super().__init__() + self.dim = dim + self.input_resolution = input_resolution + self.depth = depth + self.use_checkpoint = use_checkpoint + + # Build blocks + self.blocks = nn.ModuleList( + [ + MBConv( + dim, + dim, + conv_expand_ratio, + activation, + drop_path[i] if isinstance(drop_path, list) else drop_path, + ) + for i in range(depth) + ] + ) + + # Patch merging layer + self.downsample = ( + None + if downsample is None + else downsample(input_resolution, dim=dim, out_dim=out_dim, activation=activation) + ) + + def forward(self, x): + """Processes the input through a series of convolutional layers and returns the activated output.""" + for blk in self.blocks: + x = checkpoint.checkpoint(blk, x) if self.use_checkpoint else blk(x) + return x if self.downsample is None else self.downsample(x) + + +class Mlp(nn.Module): + """ + Multi-layer Perceptron (MLP) for transformer architectures. + + This layer takes an input with in_features, applies layer normalization and two fully-connected layers. + """ + + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.0): + """Initializes Attention module with the given parameters including dimension, key_dim, number of heads, etc.""" + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.norm = nn.LayerNorm(in_features) + self.fc1 = nn.Linear(in_features, hidden_features) + self.fc2 = nn.Linear(hidden_features, out_features) + self.act = act_layer() + self.drop = nn.Dropout(drop) + + def forward(self, x): + """Applies operations on input x and returns modified x, runs downsample if not None.""" + x = self.norm(x) + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + return self.drop(x) + + +class Attention(torch.nn.Module): + """ + Multi-head attention module with support for spatial awareness, applying attention biases based on spatial + resolution. Implements trainable attention biases for each unique offset between spatial positions in the resolution + grid. + + Attributes: + ab (Tensor, optional): Cached attention biases for inference, deleted during training. + """ + + def __init__( + self, + dim, + key_dim, + num_heads=8, + attn_ratio=4, + resolution=(14, 14), + ): + """ + Initializes the Attention module. + + Args: + dim (int): The dimensionality of the input and output. + key_dim (int): The dimensionality of the keys and queries. + num_heads (int, optional): Number of attention heads. Default is 8. + attn_ratio (float, optional): Attention ratio, affecting the dimensions of the value vectors. Default is 4. + resolution (Tuple[int, int], optional): Spatial resolution of the input feature map. Default is (14, 14). + + Raises: + AssertionError: If `resolution` is not a tuple of length 2. + """ + super().__init__() + + assert isinstance(resolution, tuple) and len(resolution) == 2 + self.num_heads = num_heads + self.scale = key_dim**-0.5 + self.key_dim = key_dim + self.nh_kd = nh_kd = key_dim * num_heads + self.d = int(attn_ratio * key_dim) + self.dh = int(attn_ratio * key_dim) * num_heads + self.attn_ratio = attn_ratio + h = self.dh + nh_kd * 2 + + self.norm = nn.LayerNorm(dim) + self.qkv = nn.Linear(dim, h) + self.proj = nn.Linear(self.dh, dim) + + points = list(itertools.product(range(resolution[0]), range(resolution[1]))) + N = len(points) + attention_offsets = {} + idxs = [] + for p1 in points: + for p2 in points: + offset = (abs(p1[0] - p2[0]), abs(p1[1] - p2[1])) + if offset not in attention_offsets: + attention_offsets[offset] = len(attention_offsets) + idxs.append(attention_offsets[offset]) + self.attention_biases = torch.nn.Parameter(torch.zeros(num_heads, len(attention_offsets))) + self.register_buffer("attention_bias_idxs", torch.LongTensor(idxs).view(N, N), persistent=False) + + @torch.no_grad() + def train(self, mode=True): + """Sets the module in training mode and handles attribute 'ab' based on the mode.""" + super().train(mode) + if mode and hasattr(self, "ab"): + del self.ab + else: + self.ab = self.attention_biases[:, self.attention_bias_idxs] + + def forward(self, x): # x + """Performs forward pass over the input tensor 'x' by applying normalization and querying keys/values.""" + B, N, _ = x.shape # B, N, C + + # Normalization + x = self.norm(x) + + qkv = self.qkv(x) + # (B, N, num_heads, d) + q, k, v = qkv.view(B, N, self.num_heads, -1).split([self.key_dim, self.key_dim, self.d], dim=3) + # (B, num_heads, N, d) + q = q.permute(0, 2, 1, 3) + k = k.permute(0, 2, 1, 3) + v = v.permute(0, 2, 1, 3) + self.ab = self.ab.to(self.attention_biases.device) + + attn = (q @ k.transpose(-2, -1)) * self.scale + ( + self.attention_biases[:, self.attention_bias_idxs] if self.training else self.ab + ) + attn = attn.softmax(dim=-1) + x = (attn @ v).transpose(1, 2).reshape(B, N, self.dh) + return self.proj(x) + + +class TinyViTBlock(nn.Module): + """TinyViT Block that applies self-attention and a local convolution to the input.""" + + def __init__( + self, + dim, + input_resolution, + num_heads, + window_size=7, + mlp_ratio=4.0, + drop=0.0, + drop_path=0.0, + local_conv_size=3, + activation=nn.GELU, + ): + """ + Initializes the TinyViTBlock. + + Args: + dim (int): The dimensionality of the input and output. + input_resolution (Tuple[int, int]): Spatial resolution of the input feature map. + num_heads (int): Number of attention heads. + window_size (int, optional): Window size for attention. Default is 7. + mlp_ratio (float, optional): Ratio of mlp hidden dim to embedding dim. Default is 4. + drop (float, optional): Dropout rate. Default is 0. + drop_path (float, optional): Stochastic depth rate. Default is 0. + local_conv_size (int, optional): The kernel size of the local convolution. Default is 3. + activation (torch.nn, optional): Activation function for MLP. Default is nn.GELU. + + Raises: + AssertionError: If `window_size` is not greater than 0. + AssertionError: If `dim` is not divisible by `num_heads`. + """ + super().__init__() + self.dim = dim + self.input_resolution = input_resolution + self.num_heads = num_heads + assert window_size > 0, "window_size must be greater than 0" + self.window_size = window_size + self.mlp_ratio = mlp_ratio + + # NOTE: `DropPath` is needed only for training. + # self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.drop_path = nn.Identity() + + assert dim % num_heads == 0, "dim must be divisible by num_heads" + head_dim = dim // num_heads + + window_resolution = (window_size, window_size) + self.attn = Attention(dim, head_dim, num_heads, attn_ratio=1, resolution=window_resolution) + + mlp_hidden_dim = int(dim * mlp_ratio) + mlp_activation = activation + self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=mlp_activation, drop=drop) + + pad = local_conv_size // 2 + self.local_conv = Conv2d_BN(dim, dim, ks=local_conv_size, stride=1, pad=pad, groups=dim) + + def forward(self, x): + """Applies attention-based transformation or padding to input 'x' before passing it through a local + convolution. + """ + H, W = self.input_resolution + B, L, C = x.shape + assert L == H * W, "input feature has wrong size" + res_x = x + if H == self.window_size and W == self.window_size: + x = self.attn(x) + else: + x = x.view(B, H, W, C) + pad_b = (self.window_size - H % self.window_size) % self.window_size + pad_r = (self.window_size - W % self.window_size) % self.window_size + padding = pad_b > 0 or pad_r > 0 + + if padding: + x = F.pad(x, (0, 0, 0, pad_r, 0, pad_b)) + + pH, pW = H + pad_b, W + pad_r + nH = pH // self.window_size + nW = pW // self.window_size + # Window partition + x = ( + x.view(B, nH, self.window_size, nW, self.window_size, C) + .transpose(2, 3) + .reshape(B * nH * nW, self.window_size * self.window_size, C) + ) + x = self.attn(x) + # Window reverse + x = x.view(B, nH, nW, self.window_size, self.window_size, C).transpose(2, 3).reshape(B, pH, pW, C) + + if padding: + x = x[:, :H, :W].contiguous() + + x = x.view(B, L, C) + + x = res_x + self.drop_path(x) + + x = x.transpose(1, 2).reshape(B, C, H, W) + x = self.local_conv(x) + x = x.view(B, C, L).transpose(1, 2) + + return x + self.drop_path(self.mlp(x)) + + def extra_repr(self) -> str: + """Returns a formatted string representing the TinyViTBlock's parameters: dimension, input resolution, number of + attentions heads, window size, and MLP ratio. + """ + return ( + f"dim={self.dim}, input_resolution={self.input_resolution}, num_heads={self.num_heads}, " + f"window_size={self.window_size}, mlp_ratio={self.mlp_ratio}" + ) + + +class BasicLayer(nn.Module): + """A basic TinyViT layer for one stage in a TinyViT architecture.""" + + def __init__( + self, + dim, + input_resolution, + depth, + num_heads, + window_size, + mlp_ratio=4.0, + drop=0.0, + drop_path=0.0, + downsample=None, + use_checkpoint=False, + local_conv_size=3, + activation=nn.GELU, + out_dim=None, + ): + """ + Initializes the BasicLayer. + + Args: + dim (int): The dimensionality of the input and output. + input_resolution (Tuple[int, int]): Spatial resolution of the input feature map. + depth (int): Number of TinyViT blocks. + num_heads (int): Number of attention heads. + window_size (int): Local window size. + mlp_ratio (float, optional): Ratio of mlp hidden dim to embedding dim. Default is 4. + drop (float, optional): Dropout rate. Default is 0. + drop_path (float | tuple[float], optional): Stochastic depth rate. Default is 0. + downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default is None. + use_checkpoint (bool, optional): Whether to use checkpointing to save memory. Default is False. + local_conv_size (int, optional): Kernel size of the local convolution. Default is 3. + activation (torch.nn, optional): Activation function for MLP. Default is nn.GELU. + out_dim (int | None, optional): The output dimension of the layer. Default is None. + + Raises: + ValueError: If `drop_path` is a list of float but its length doesn't match `depth`. + """ + super().__init__() + self.dim = dim + self.input_resolution = input_resolution + self.depth = depth + self.use_checkpoint = use_checkpoint + + # Build blocks + self.blocks = nn.ModuleList( + [ + TinyViTBlock( + dim=dim, + input_resolution=input_resolution, + num_heads=num_heads, + window_size=window_size, + mlp_ratio=mlp_ratio, + drop=drop, + drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path, + local_conv_size=local_conv_size, + activation=activation, + ) + for i in range(depth) + ] + ) + + # Patch merging layer + self.downsample = ( + None + if downsample is None + else downsample(input_resolution, dim=dim, out_dim=out_dim, activation=activation) + ) + + def forward(self, x): + """Performs forward propagation on the input tensor and returns a normalized tensor.""" + for blk in self.blocks: + x = checkpoint.checkpoint(blk, x) if self.use_checkpoint else blk(x) + return x if self.downsample is None else self.downsample(x) + + def extra_repr(self) -> str: + """Returns a string representation of the extra_repr function with the layer's parameters.""" + return f"dim={self.dim}, input_resolution={self.input_resolution}, depth={self.depth}" + + +class LayerNorm2d(nn.Module): + """A PyTorch implementation of Layer Normalization in 2D.""" + + def __init__(self, num_channels: int, eps: float = 1e-6) -> None: + """Initialize LayerNorm2d with the number of channels and an optional epsilon.""" + super().__init__() + self.weight = nn.Parameter(torch.ones(num_channels)) + self.bias = nn.Parameter(torch.zeros(num_channels)) + self.eps = eps + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Perform a forward pass, normalizing the input tensor.""" + u = x.mean(1, keepdim=True) + s = (x - u).pow(2).mean(1, keepdim=True) + x = (x - u) / torch.sqrt(s + self.eps) + return self.weight[:, None, None] * x + self.bias[:, None, None] + + +class TinyViT(nn.Module): + """ + The TinyViT architecture for vision tasks. + + Attributes: + img_size (int): Input image size. + in_chans (int): Number of input channels. + num_classes (int): Number of classification classes. + embed_dims (List[int]): List of embedding dimensions for each layer. + depths (List[int]): List of depths for each layer. + num_heads (List[int]): List of number of attention heads for each layer. + window_sizes (List[int]): List of window sizes for each layer. + mlp_ratio (float): Ratio of MLP hidden dimension to embedding dimension. + drop_rate (float): Dropout rate for drop layers. + drop_path_rate (float): Drop path rate for stochastic depth. + use_checkpoint (bool): Use checkpointing for efficient memory usage. + mbconv_expand_ratio (float): Expansion ratio for MBConv layer. + local_conv_size (int): Local convolution kernel size. + layer_lr_decay (float): Layer-wise learning rate decay. + + Note: + This implementation is generalized to accept a list of depths, attention heads, + embedding dimensions and window sizes, which allows you to create a + "stack" of TinyViT models of varying configurations. + """ + + def __init__( + self, + img_size=224, + in_chans=3, + num_classes=1000, + embed_dims=[96, 192, 384, 768], + depths=[2, 2, 6, 2], + num_heads=[3, 6, 12, 24], + window_sizes=[7, 7, 14, 7], + mlp_ratio=4.0, + drop_rate=0.0, + drop_path_rate=0.1, + use_checkpoint=False, + mbconv_expand_ratio=4.0, + local_conv_size=3, + layer_lr_decay=1.0, + ): + """ + Initializes the TinyViT model. + + Args: + img_size (int, optional): The input image size. Defaults to 224. + in_chans (int, optional): Number of input channels. Defaults to 3. + num_classes (int, optional): Number of classification classes. Defaults to 1000. + embed_dims (List[int], optional): List of embedding dimensions for each layer. Defaults to [96, 192, 384, 768]. + depths (List[int], optional): List of depths for each layer. Defaults to [2, 2, 6, 2]. + num_heads (List[int], optional): List of number of attention heads for each layer. Defaults to [3, 6, 12, 24]. + window_sizes (List[int], optional): List of window sizes for each layer. Defaults to [7, 7, 14, 7]. + mlp_ratio (float, optional): Ratio of MLP hidden dimension to embedding dimension. Defaults to 4. + drop_rate (float, optional): Dropout rate. Defaults to 0. + drop_path_rate (float, optional): Drop path rate for stochastic depth. Defaults to 0.1. + use_checkpoint (bool, optional): Whether to use checkpointing for efficient memory usage. Defaults to False. + mbconv_expand_ratio (float, optional): Expansion ratio for MBConv layer. Defaults to 4.0. + local_conv_size (int, optional): Local convolution kernel size. Defaults to 3. + layer_lr_decay (float, optional): Layer-wise learning rate decay. Defaults to 1.0. + """ + super().__init__() + self.img_size = img_size + self.num_classes = num_classes + self.depths = depths + self.num_layers = len(depths) + self.mlp_ratio = mlp_ratio + + activation = nn.GELU + + self.patch_embed = PatchEmbed( + in_chans=in_chans, embed_dim=embed_dims[0], resolution=img_size, activation=activation + ) + + patches_resolution = self.patch_embed.patches_resolution + self.patches_resolution = patches_resolution + + # Stochastic depth + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))] # stochastic depth decay rule + + # Build layers + self.layers = nn.ModuleList() + for i_layer in range(self.num_layers): + kwargs = dict( + dim=embed_dims[i_layer], + input_resolution=( + patches_resolution[0] // (2 ** (i_layer - 1 if i_layer == 3 else i_layer)), + patches_resolution[1] // (2 ** (i_layer - 1 if i_layer == 3 else i_layer)), + ), + # input_resolution=(patches_resolution[0] // (2 ** i_layer), + # patches_resolution[1] // (2 ** i_layer)), + depth=depths[i_layer], + drop_path=dpr[sum(depths[:i_layer]) : sum(depths[: i_layer + 1])], + downsample=PatchMerging if (i_layer < self.num_layers - 1) else None, + use_checkpoint=use_checkpoint, + out_dim=embed_dims[min(i_layer + 1, len(embed_dims) - 1)], + activation=activation, + ) + if i_layer == 0: + layer = ConvLayer(conv_expand_ratio=mbconv_expand_ratio, **kwargs) + else: + layer = BasicLayer( + num_heads=num_heads[i_layer], + window_size=window_sizes[i_layer], + mlp_ratio=self.mlp_ratio, + drop=drop_rate, + local_conv_size=local_conv_size, + **kwargs, + ) + self.layers.append(layer) + + # Classifier head + self.norm_head = nn.LayerNorm(embed_dims[-1]) + self.head = nn.Linear(embed_dims[-1], num_classes) if num_classes > 0 else torch.nn.Identity() + + # Init weights + self.apply(self._init_weights) + self.set_layer_lr_decay(layer_lr_decay) + self.neck = nn.Sequential( + nn.Conv2d( + embed_dims[-1], + 256, + kernel_size=1, + bias=False, + ), + LayerNorm2d(256), + nn.Conv2d( + 256, + 256, + kernel_size=3, + padding=1, + bias=False, + ), + LayerNorm2d(256), + ) + + def set_layer_lr_decay(self, layer_lr_decay): + """Sets the learning rate decay for each layer in the TinyViT model.""" + decay_rate = layer_lr_decay + + # Layers -> blocks (depth) + depth = sum(self.depths) + lr_scales = [decay_rate ** (depth - i - 1) for i in range(depth)] + + def _set_lr_scale(m, scale): + """Sets the learning rate scale for each layer in the model based on the layer's depth.""" + for p in m.parameters(): + p.lr_scale = scale + + self.patch_embed.apply(lambda x: _set_lr_scale(x, lr_scales[0])) + i = 0 + for layer in self.layers: + for block in layer.blocks: + block.apply(lambda x: _set_lr_scale(x, lr_scales[i])) + i += 1 + if layer.downsample is not None: + layer.downsample.apply(lambda x: _set_lr_scale(x, lr_scales[i - 1])) + assert i == depth + for m in [self.norm_head, self.head]: + m.apply(lambda x: _set_lr_scale(x, lr_scales[-1])) + + for k, p in self.named_parameters(): + p.param_name = k + + def _check_lr_scale(m): + """Checks if the learning rate scale attribute is present in module's parameters.""" + for p in m.parameters(): + assert hasattr(p, "lr_scale"), p.param_name + + self.apply(_check_lr_scale) + + def _init_weights(self, m): + """Initializes weights for linear layers and layer normalization in the given module.""" + if isinstance(m, nn.Linear): + # NOTE: This initialization is needed only for training. + # trunc_normal_(m.weight, std=.02) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + @torch.jit.ignore + def no_weight_decay_keywords(self): + """Returns a dictionary of parameter names where weight decay should not be applied.""" + return {"attention_biases"} + + def forward_features(self, x): + """Runs the input through the model layers and returns the transformed output.""" + x = self.patch_embed(x) # x input is (N, C, H, W) + + x = self.layers[0](x) + start_i = 1 + + for i in range(start_i, len(self.layers)): + layer = self.layers[i] + x = layer(x) + B, _, C = x.shape + x = x.view(B, 64, 64, C) + x = x.permute(0, 3, 1, 2) + return self.neck(x) + + def forward(self, x): + """Executes a forward pass on the input tensor through the constructed model layers.""" + return self.forward_features(x) diff --git a/ultralytics/models/sam/modules/transformer.py b/ultralytics/models/sam/modules/transformer.py new file mode 100644 index 0000000..1ad0741 --- /dev/null +++ b/ultralytics/models/sam/modules/transformer.py @@ -0,0 +1,274 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import math +from typing import Tuple, Type + +import torch +from torch import Tensor, nn + +from ultralytics.nn.modules import MLPBlock + + +class TwoWayTransformer(nn.Module): + """ + A Two-Way Transformer module that enables the simultaneous attention to both image and query points. This class + serves as a specialized transformer decoder that attends to an input image using queries whose positional embedding + is supplied. This is particularly useful for tasks like object detection, image segmentation, and point cloud + processing. + + Attributes: + depth (int): The number of layers in the transformer. + embedding_dim (int): The channel dimension for the input embeddings. + num_heads (int): The number of heads for multihead attention. + mlp_dim (int): The internal channel dimension for the MLP block. + layers (nn.ModuleList): The list of TwoWayAttentionBlock layers that make up the transformer. + final_attn_token_to_image (Attention): The final attention layer applied from the queries to the image. + norm_final_attn (nn.LayerNorm): The layer normalization applied to the final queries. + """ + + def __init__( + self, + depth: int, + embedding_dim: int, + num_heads: int, + mlp_dim: int, + activation: Type[nn.Module] = nn.ReLU, + attention_downsample_rate: int = 2, + ) -> None: + """ + A transformer decoder that attends to an input image using queries whose positional embedding is supplied. + + Args: + depth (int): number of layers in the transformer + embedding_dim (int): the channel dimension for the input embeddings + num_heads (int): the number of heads for multihead attention. Must + divide embedding_dim + mlp_dim (int): the channel dimension internal to the MLP block + activation (nn.Module): the activation to use in the MLP block + """ + super().__init__() + self.depth = depth + self.embedding_dim = embedding_dim + self.num_heads = num_heads + self.mlp_dim = mlp_dim + self.layers = nn.ModuleList() + + for i in range(depth): + self.layers.append( + TwoWayAttentionBlock( + embedding_dim=embedding_dim, + num_heads=num_heads, + mlp_dim=mlp_dim, + activation=activation, + attention_downsample_rate=attention_downsample_rate, + skip_first_layer_pe=(i == 0), + ) + ) + + self.final_attn_token_to_image = Attention(embedding_dim, num_heads, downsample_rate=attention_downsample_rate) + self.norm_final_attn = nn.LayerNorm(embedding_dim) + + def forward( + self, + image_embedding: Tensor, + image_pe: Tensor, + point_embedding: Tensor, + ) -> Tuple[Tensor, Tensor]: + """ + Args: + image_embedding (torch.Tensor): image to attend to. Should be shape B x embedding_dim x h x w for any h and w. + image_pe (torch.Tensor): the positional encoding to add to the image. Must have same shape as image_embedding. + point_embedding (torch.Tensor): the embedding to add to the query points. + Must have shape B x N_points x embedding_dim for any N_points. + + Returns: + (torch.Tensor): the processed point_embedding + (torch.Tensor): the processed image_embedding + """ + # BxCxHxW -> BxHWxC == B x N_image_tokens x C + bs, c, h, w = image_embedding.shape + image_embedding = image_embedding.flatten(2).permute(0, 2, 1) + image_pe = image_pe.flatten(2).permute(0, 2, 1) + + # Prepare queries + queries = point_embedding + keys = image_embedding + + # Apply transformer blocks and final layernorm + for layer in self.layers: + queries, keys = layer( + queries=queries, + keys=keys, + query_pe=point_embedding, + key_pe=image_pe, + ) + + # Apply the final attention layer from the points to the image + q = queries + point_embedding + k = keys + image_pe + attn_out = self.final_attn_token_to_image(q=q, k=k, v=keys) + queries = queries + attn_out + queries = self.norm_final_attn(queries) + + return queries, keys + + +class TwoWayAttentionBlock(nn.Module): + """ + An attention block that performs both self-attention and cross-attention in two directions: queries to keys and + keys to queries. This block consists of four main layers: (1) self-attention on sparse inputs, (2) cross-attention + of sparse inputs to dense inputs, (3) an MLP block on sparse inputs, and (4) cross-attention of dense inputs to + sparse inputs. + + Attributes: + self_attn (Attention): The self-attention layer for the queries. + norm1 (nn.LayerNorm): Layer normalization following the first attention block. + cross_attn_token_to_image (Attention): Cross-attention layer from queries to keys. + norm2 (nn.LayerNorm): Layer normalization following the second attention block. + mlp (MLPBlock): MLP block that transforms the query embeddings. + norm3 (nn.LayerNorm): Layer normalization following the MLP block. + norm4 (nn.LayerNorm): Layer normalization following the third attention block. + cross_attn_image_to_token (Attention): Cross-attention layer from keys to queries. + skip_first_layer_pe (bool): Whether to skip the positional encoding in the first layer. + """ + + def __init__( + self, + embedding_dim: int, + num_heads: int, + mlp_dim: int = 2048, + activation: Type[nn.Module] = nn.ReLU, + attention_downsample_rate: int = 2, + skip_first_layer_pe: bool = False, + ) -> None: + """ + A transformer block with four layers: (1) self-attention of sparse inputs, (2) cross attention of sparse + inputs to dense inputs, (3) mlp block on sparse inputs, and (4) cross attention of dense inputs to sparse + inputs. + + Args: + embedding_dim (int): the channel dimension of the embeddings + num_heads (int): the number of heads in the attention layers + mlp_dim (int): the hidden dimension of the mlp block + activation (nn.Module): the activation of the mlp block + skip_first_layer_pe (bool): skip the PE on the first layer + """ + super().__init__() + self.self_attn = Attention(embedding_dim, num_heads) + self.norm1 = nn.LayerNorm(embedding_dim) + + self.cross_attn_token_to_image = Attention(embedding_dim, num_heads, downsample_rate=attention_downsample_rate) + self.norm2 = nn.LayerNorm(embedding_dim) + + self.mlp = MLPBlock(embedding_dim, mlp_dim, activation) + self.norm3 = nn.LayerNorm(embedding_dim) + + self.norm4 = nn.LayerNorm(embedding_dim) + self.cross_attn_image_to_token = Attention(embedding_dim, num_heads, downsample_rate=attention_downsample_rate) + + self.skip_first_layer_pe = skip_first_layer_pe + + def forward(self, queries: Tensor, keys: Tensor, query_pe: Tensor, key_pe: Tensor) -> Tuple[Tensor, Tensor]: + """Apply self-attention and cross-attention to queries and keys and return the processed embeddings.""" + + # Self attention block + if self.skip_first_layer_pe: + queries = self.self_attn(q=queries, k=queries, v=queries) + else: + q = queries + query_pe + attn_out = self.self_attn(q=q, k=q, v=queries) + queries = queries + attn_out + queries = self.norm1(queries) + + # Cross attention block, tokens attending to image embedding + q = queries + query_pe + k = keys + key_pe + attn_out = self.cross_attn_token_to_image(q=q, k=k, v=keys) + queries = queries + attn_out + queries = self.norm2(queries) + + # MLP block + mlp_out = self.mlp(queries) + queries = queries + mlp_out + queries = self.norm3(queries) + + # Cross attention block, image embedding attending to tokens + q = queries + query_pe + k = keys + key_pe + attn_out = self.cross_attn_image_to_token(q=k, k=q, v=queries) + keys = keys + attn_out + keys = self.norm4(keys) + + return queries, keys + + +class Attention(nn.Module): + """An attention layer that allows for downscaling the size of the embedding after projection to queries, keys, and + values. + """ + + def __init__( + self, + embedding_dim: int, + num_heads: int, + downsample_rate: int = 1, + ) -> None: + """ + Initializes the Attention model with the given dimensions and settings. + + Args: + embedding_dim (int): The dimensionality of the input embeddings. + num_heads (int): The number of attention heads. + downsample_rate (int, optional): The factor by which the internal dimensions are downsampled. Defaults to 1. + + Raises: + AssertionError: If 'num_heads' does not evenly divide the internal dimension (embedding_dim / downsample_rate). + """ + super().__init__() + self.embedding_dim = embedding_dim + self.internal_dim = embedding_dim // downsample_rate + self.num_heads = num_heads + assert self.internal_dim % num_heads == 0, "num_heads must divide embedding_dim." + + self.q_proj = nn.Linear(embedding_dim, self.internal_dim) + self.k_proj = nn.Linear(embedding_dim, self.internal_dim) + self.v_proj = nn.Linear(embedding_dim, self.internal_dim) + self.out_proj = nn.Linear(self.internal_dim, embedding_dim) + + @staticmethod + def _separate_heads(x: Tensor, num_heads: int) -> Tensor: + """Separate the input tensor into the specified number of attention heads.""" + b, n, c = x.shape + x = x.reshape(b, n, num_heads, c // num_heads) + return x.transpose(1, 2) # B x N_heads x N_tokens x C_per_head + + @staticmethod + def _recombine_heads(x: Tensor) -> Tensor: + """Recombine the separated attention heads into a single tensor.""" + b, n_heads, n_tokens, c_per_head = x.shape + x = x.transpose(1, 2) + return x.reshape(b, n_tokens, n_heads * c_per_head) # B x N_tokens x C + + def forward(self, q: Tensor, k: Tensor, v: Tensor) -> Tensor: + """Compute the attention output given the input query, key, and value tensors.""" + + # Input projections + q = self.q_proj(q) + k = self.k_proj(k) + v = self.v_proj(v) + + # Separate into heads + q = self._separate_heads(q, self.num_heads) + k = self._separate_heads(k, self.num_heads) + v = self._separate_heads(v, self.num_heads) + + # Attention + _, _, _, c_per_head = q.shape + attn = q @ k.permute(0, 1, 3, 2) # B x N_heads x N_tokens x N_tokens + attn = attn / math.sqrt(c_per_head) + attn = torch.softmax(attn, dim=-1) + + # Get output + out = attn @ v + out = self._recombine_heads(out) + return self.out_proj(out) diff --git a/ultralytics/models/sam/predict.py b/ultralytics/models/sam/predict.py new file mode 100644 index 0000000..63ca632 --- /dev/null +++ b/ultralytics/models/sam/predict.py @@ -0,0 +1,474 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +""" +Generate predictions using the Segment Anything Model (SAM). + +SAM is an advanced image segmentation model offering features like promptable segmentation and zero-shot performance. +This module contains the implementation of the prediction logic and auxiliary utilities required to perform segmentation +using SAM. It forms an integral part of the Ultralytics framework and is designed for high-performance, real-time image +segmentation tasks. +""" + +import numpy as np +import torch +import torch.nn.functional as F +import torchvision + +from ultralytics.data.augment import LetterBox +from ultralytics.engine.predictor import BasePredictor +from ultralytics.engine.results import Results +from ultralytics.utils import DEFAULT_CFG, ops +from ultralytics.utils.torch_utils import select_device +from .amg import ( + batch_iterator, + batched_mask_to_box, + build_all_layer_point_grids, + calculate_stability_score, + generate_crop_boxes, + is_box_near_crop_edge, + remove_small_regions, + uncrop_boxes_xyxy, + uncrop_masks, +) +from .build import build_sam + + +class Predictor(BasePredictor): + """ + Predictor class for the Segment Anything Model (SAM), extending BasePredictor. + + The class provides an interface for model inference tailored to image segmentation tasks. + With advanced architecture and promptable segmentation capabilities, it facilitates flexible and real-time + mask generation. The class is capable of working with various types of prompts such as bounding boxes, + points, and low-resolution masks. + + Attributes: + cfg (dict): Configuration dictionary specifying model and task-related parameters. + overrides (dict): Dictionary containing values that override the default configuration. + _callbacks (dict): Dictionary of user-defined callback functions to augment behavior. + args (namespace): Namespace to hold command-line arguments or other operational variables. + im (torch.Tensor): Preprocessed input image tensor. + features (torch.Tensor): Extracted image features used for inference. + prompts (dict): Collection of various prompt types, such as bounding boxes and points. + segment_all (bool): Flag to control whether to segment all objects in the image or only specified ones. + """ + + def __init__(self, cfg=DEFAULT_CFG, overrides=None, _callbacks=None): + """ + Initialize the Predictor with configuration, overrides, and callbacks. + + The method sets up the Predictor object and applies any configuration overrides or callbacks provided. It + initializes task-specific settings for SAM, such as retina_masks being set to True for optimal results. + + Args: + cfg (dict): Configuration dictionary. + overrides (dict, optional): Dictionary of values to override default configuration. + _callbacks (dict, optional): Dictionary of callback functions to customize behavior. + """ + if overrides is None: + overrides = {} + overrides.update(dict(task="segment", mode="predict", imgsz=1024)) + super().__init__(cfg, overrides, _callbacks) + self.args.retina_masks = True + self.im = None + self.features = None + self.prompts = {} + self.segment_all = False + + def preprocess(self, im): + """ + Preprocess the input image for model inference. + + The method prepares the input image by applying transformations and normalization. + It supports both torch.Tensor and list of np.ndarray as input formats. + + Args: + im (torch.Tensor | List[np.ndarray]): BCHW tensor format or list of HWC numpy arrays. + + Returns: + (torch.Tensor): The preprocessed image tensor. + """ + if self.im is not None: + return self.im + not_tensor = not isinstance(im, torch.Tensor) + if not_tensor: + im = np.stack(self.pre_transform(im)) + im = im[..., ::-1].transpose((0, 3, 1, 2)) + im = np.ascontiguousarray(im) + im = torch.from_numpy(im) + + im = im.to(self.device) + im = im.half() if self.model.fp16 else im.float() + if not_tensor: + im = (im - self.mean) / self.std + return im + + def pre_transform(self, im): + """ + Perform initial transformations on the input image for preprocessing. + + The method applies transformations such as resizing to prepare the image for further preprocessing. + Currently, batched inference is not supported; hence the list length should be 1. + + Args: + im (List[np.ndarray]): List containing images in HWC numpy array format. + + Returns: + (List[np.ndarray]): List of transformed images. + """ + assert len(im) == 1, "SAM model does not currently support batched inference" + letterbox = LetterBox(self.args.imgsz, auto=False, center=False) + return [letterbox(image=x) for x in im] + + def inference(self, im, bboxes=None, points=None, labels=None, masks=None, multimask_output=False, *args, **kwargs): + """ + Perform image segmentation inference based on the given input cues, using the currently loaded image. This + method leverages SAM's (Segment Anything Model) architecture consisting of image encoder, prompt encoder, and + mask decoder for real-time and promptable segmentation tasks. + + Args: + im (torch.Tensor): The preprocessed input image in tensor format, with shape (N, C, H, W). + bboxes (np.ndarray | List, optional): Bounding boxes with shape (N, 4), in XYXY format. + points (np.ndarray | List, optional): Points indicating object locations with shape (N, 2), in pixel coordinates. + labels (np.ndarray | List, optional): Labels for point prompts, shape (N, ). 1 for foreground and 0 for background. + masks (np.ndarray, optional): Low-resolution masks from previous predictions. Shape should be (N, H, W). For SAM, H=W=256. + multimask_output (bool, optional): Flag to return multiple masks. Helpful for ambiguous prompts. Defaults to False. + + Returns: + (tuple): Contains the following three elements. + - np.ndarray: The output masks in shape CxHxW, where C is the number of generated masks. + - np.ndarray: An array of length C containing quality scores predicted by the model for each mask. + - np.ndarray: Low-resolution logits of shape CxHxW for subsequent inference, where H=W=256. + """ + # Override prompts if any stored in self.prompts + bboxes = self.prompts.pop("bboxes", bboxes) + points = self.prompts.pop("points", points) + masks = self.prompts.pop("masks", masks) + + if all(i is None for i in [bboxes, points, masks]): + return self.generate(im, *args, **kwargs) + + return self.prompt_inference(im, bboxes, points, labels, masks, multimask_output) + + def prompt_inference(self, im, bboxes=None, points=None, labels=None, masks=None, multimask_output=False): + """ + Internal function for image segmentation inference based on cues like bounding boxes, points, and masks. + Leverages SAM's specialized architecture for prompt-based, real-time segmentation. + + Args: + im (torch.Tensor): The preprocessed input image in tensor format, with shape (N, C, H, W). + bboxes (np.ndarray | List, optional): Bounding boxes with shape (N, 4), in XYXY format. + points (np.ndarray | List, optional): Points indicating object locations with shape (N, 2), in pixel coordinates. + labels (np.ndarray | List, optional): Labels for point prompts, shape (N, ). 1 for foreground and 0 for background. + masks (np.ndarray, optional): Low-resolution masks from previous predictions. Shape should be (N, H, W). For SAM, H=W=256. + multimask_output (bool, optional): Flag to return multiple masks. Helpful for ambiguous prompts. Defaults to False. + + Returns: + (tuple): Contains the following three elements. + - np.ndarray: The output masks in shape CxHxW, where C is the number of generated masks. + - np.ndarray: An array of length C containing quality scores predicted by the model for each mask. + - np.ndarray: Low-resolution logits of shape CxHxW for subsequent inference, where H=W=256. + """ + features = self.model.image_encoder(im) if self.features is None else self.features + + src_shape, dst_shape = self.batch[1][0].shape[:2], im.shape[2:] + r = 1.0 if self.segment_all else min(dst_shape[0] / src_shape[0], dst_shape[1] / src_shape[1]) + # Transform input prompts + if points is not None: + points = torch.as_tensor(points, dtype=torch.float32, device=self.device) + points = points[None] if points.ndim == 1 else points + # Assuming labels are all positive if users don't pass labels. + if labels is None: + labels = np.ones(points.shape[0]) + labels = torch.as_tensor(labels, dtype=torch.int32, device=self.device) + points *= r + # (N, 2) --> (N, 1, 2), (N, ) --> (N, 1) + points, labels = points[:, None, :], labels[:, None] + if bboxes is not None: + bboxes = torch.as_tensor(bboxes, dtype=torch.float32, device=self.device) + bboxes = bboxes[None] if bboxes.ndim == 1 else bboxes + bboxes *= r + if masks is not None: + masks = torch.as_tensor(masks, dtype=torch.float32, device=self.device).unsqueeze(1) + + points = (points, labels) if points is not None else None + # Embed prompts + sparse_embeddings, dense_embeddings = self.model.prompt_encoder(points=points, boxes=bboxes, masks=masks) + + # Predict masks + pred_masks, pred_scores = self.model.mask_decoder( + image_embeddings=features, + image_pe=self.model.prompt_encoder.get_dense_pe(), + sparse_prompt_embeddings=sparse_embeddings, + dense_prompt_embeddings=dense_embeddings, + multimask_output=multimask_output, + ) + + # (N, d, H, W) --> (N*d, H, W), (N, d) --> (N*d, ) + # `d` could be 1 or 3 depends on `multimask_output`. + return pred_masks.flatten(0, 1), pred_scores.flatten(0, 1) + + def generate( + self, + im, + crop_n_layers=0, + crop_overlap_ratio=512 / 1500, + crop_downscale_factor=1, + point_grids=None, + points_stride=32, + points_batch_size=64, + conf_thres=0.88, + stability_score_thresh=0.95, + stability_score_offset=0.95, + crop_nms_thresh=0.7, + ): + """ + Perform image segmentation using the Segment Anything Model (SAM). + + This function segments an entire image into constituent parts by leveraging SAM's advanced architecture + and real-time performance capabilities. It can optionally work on image crops for finer segmentation. + + Args: + im (torch.Tensor): Input tensor representing the preprocessed image with dimensions (N, C, H, W). + crop_n_layers (int): Specifies the number of layers for additional mask predictions on image crops. + Each layer produces 2**i_layer number of image crops. + crop_overlap_ratio (float): Determines the extent of overlap between crops. Scaled down in subsequent layers. + crop_downscale_factor (int): Scaling factor for the number of sampled points-per-side in each layer. + point_grids (list[np.ndarray], optional): Custom grids for point sampling normalized to [0,1]. + Used in the nth crop layer. + points_stride (int, optional): Number of points to sample along each side of the image. + Exclusive with 'point_grids'. + points_batch_size (int): Batch size for the number of points processed simultaneously. + conf_thres (float): Confidence threshold [0,1] for filtering based on the model's mask quality prediction. + stability_score_thresh (float): Stability threshold [0,1] for mask filtering based on mask stability. + stability_score_offset (float): Offset value for calculating stability score. + crop_nms_thresh (float): IoU cutoff for Non-Maximum Suppression (NMS) to remove duplicate masks between crops. + + Returns: + (tuple): A tuple containing segmented masks, confidence scores, and bounding boxes. + """ + self.segment_all = True + ih, iw = im.shape[2:] + crop_regions, layer_idxs = generate_crop_boxes((ih, iw), crop_n_layers, crop_overlap_ratio) + if point_grids is None: + point_grids = build_all_layer_point_grids(points_stride, crop_n_layers, crop_downscale_factor) + pred_masks, pred_scores, pred_bboxes, region_areas = [], [], [], [] + for crop_region, layer_idx in zip(crop_regions, layer_idxs): + x1, y1, x2, y2 = crop_region + w, h = x2 - x1, y2 - y1 + area = torch.tensor(w * h, device=im.device) + points_scale = np.array([[w, h]]) # w, h + # Crop image and interpolate to input size + crop_im = F.interpolate(im[..., y1:y2, x1:x2], (ih, iw), mode="bilinear", align_corners=False) + # (num_points, 2) + points_for_image = point_grids[layer_idx] * points_scale + crop_masks, crop_scores, crop_bboxes = [], [], [] + for (points,) in batch_iterator(points_batch_size, points_for_image): + pred_mask, pred_score = self.prompt_inference(crop_im, points=points, multimask_output=True) + # Interpolate predicted masks to input size + pred_mask = F.interpolate(pred_mask[None], (h, w), mode="bilinear", align_corners=False)[0] + idx = pred_score > conf_thres + pred_mask, pred_score = pred_mask[idx], pred_score[idx] + + stability_score = calculate_stability_score( + pred_mask, self.model.mask_threshold, stability_score_offset + ) + idx = stability_score > stability_score_thresh + pred_mask, pred_score = pred_mask[idx], pred_score[idx] + # Bool type is much more memory-efficient. + pred_mask = pred_mask > self.model.mask_threshold + # (N, 4) + pred_bbox = batched_mask_to_box(pred_mask).float() + keep_mask = ~is_box_near_crop_edge(pred_bbox, crop_region, [0, 0, iw, ih]) + if not torch.all(keep_mask): + pred_bbox, pred_mask, pred_score = pred_bbox[keep_mask], pred_mask[keep_mask], pred_score[keep_mask] + + crop_masks.append(pred_mask) + crop_bboxes.append(pred_bbox) + crop_scores.append(pred_score) + + # Do nms within this crop + crop_masks = torch.cat(crop_masks) + crop_bboxes = torch.cat(crop_bboxes) + crop_scores = torch.cat(crop_scores) + keep = torchvision.ops.nms(crop_bboxes, crop_scores, self.args.iou) # NMS + crop_bboxes = uncrop_boxes_xyxy(crop_bboxes[keep], crop_region) + crop_masks = uncrop_masks(crop_masks[keep], crop_region, ih, iw) + crop_scores = crop_scores[keep] + + pred_masks.append(crop_masks) + pred_bboxes.append(crop_bboxes) + pred_scores.append(crop_scores) + region_areas.append(area.expand(len(crop_masks))) + + pred_masks = torch.cat(pred_masks) + pred_bboxes = torch.cat(pred_bboxes) + pred_scores = torch.cat(pred_scores) + region_areas = torch.cat(region_areas) + + # Remove duplicate masks between crops + if len(crop_regions) > 1: + scores = 1 / region_areas + keep = torchvision.ops.nms(pred_bboxes, scores, crop_nms_thresh) + pred_masks, pred_bboxes, pred_scores = pred_masks[keep], pred_bboxes[keep], pred_scores[keep] + + return pred_masks, pred_scores, pred_bboxes + + def setup_model(self, model, verbose=True): + """ + Initializes the Segment Anything Model (SAM) for inference. + + This method sets up the SAM model by allocating it to the appropriate device and initializing the necessary + parameters for image normalization and other Ultralytics compatibility settings. + + Args: + model (torch.nn.Module): A pre-trained SAM model. If None, a model will be built based on configuration. + verbose (bool): If True, prints selected device information. + + Attributes: + model (torch.nn.Module): The SAM model allocated to the chosen device for inference. + device (torch.device): The device to which the model and tensors are allocated. + mean (torch.Tensor): The mean values for image normalization. + std (torch.Tensor): The standard deviation values for image normalization. + """ + device = select_device(self.args.device, verbose=verbose) + if model is None: + model = build_sam(self.args.model) + model.eval() + self.model = model.to(device) + self.device = device + self.mean = torch.tensor([123.675, 116.28, 103.53]).view(-1, 1, 1).to(device) + self.std = torch.tensor([58.395, 57.12, 57.375]).view(-1, 1, 1).to(device) + + # Ultralytics compatibility settings + self.model.pt = False + self.model.triton = False + self.model.stride = 32 + self.model.fp16 = False + self.done_warmup = True + + def postprocess(self, preds, img, orig_imgs): + """ + Post-processes SAM's inference outputs to generate object detection masks and bounding boxes. + + The method scales masks and boxes to the original image size and applies a threshold to the mask predictions. The + SAM model uses advanced architecture and promptable segmentation tasks to achieve real-time performance. + + Args: + preds (tuple): The output from SAM model inference, containing masks, scores, and optional bounding boxes. + img (torch.Tensor): The processed input image tensor. + orig_imgs (list | torch.Tensor): The original, unprocessed images. + + Returns: + (list): List of Results objects containing detection masks, bounding boxes, and other metadata. + """ + # (N, 1, H, W), (N, 1) + pred_masks, pred_scores = preds[:2] + pred_bboxes = preds[2] if self.segment_all else None + names = dict(enumerate(str(i) for i in range(len(pred_masks)))) + + if not isinstance(orig_imgs, list): # input images are a torch.Tensor, not a list + orig_imgs = ops.convert_torch2numpy_batch(orig_imgs) + + results = [] + for i, masks in enumerate([pred_masks]): + orig_img = orig_imgs[i] + if pred_bboxes is not None: + pred_bboxes = ops.scale_boxes(img.shape[2:], pred_bboxes.float(), orig_img.shape, padding=False) + cls = torch.arange(len(pred_masks), dtype=torch.int32, device=pred_masks.device) + pred_bboxes = torch.cat([pred_bboxes, pred_scores[:, None], cls[:, None]], dim=-1) + + masks = ops.scale_masks(masks[None].float(), orig_img.shape[:2], padding=False)[0] + masks = masks > self.model.mask_threshold # to bool + img_path = self.batch[0][i] + results.append(Results(orig_img, path=img_path, names=names, masks=masks, boxes=pred_bboxes)) + # Reset segment-all mode. + self.segment_all = False + return results + + def setup_source(self, source): + """ + Sets up the data source for inference. + + This method configures the data source from which images will be fetched for inference. The source could be a + directory, a video file, or other types of image data sources. + + Args: + source (str | Path): The path to the image data source for inference. + """ + if source is not None: + super().setup_source(source) + + def set_image(self, image): + """ + Preprocesses and sets a single image for inference. + + This function sets up the model if not already initialized, configures the data source to the specified image, + and preprocesses the image for feature extraction. Only one image can be set at a time. + + Args: + image (str | np.ndarray): Image file path as a string, or a np.ndarray image read by cv2. + + Raises: + AssertionError: If more than one image is set. + """ + if self.model is None: + model = build_sam(self.args.model) + self.setup_model(model) + self.setup_source(image) + assert len(self.dataset) == 1, "`set_image` only supports setting one image!" + for batch in self.dataset: + im = self.preprocess(batch[1]) + self.features = self.model.image_encoder(im) + self.im = im + break + + def set_prompts(self, prompts): + """Set prompts in advance.""" + self.prompts = prompts + + def reset_image(self): + """Resets the image and its features to None.""" + self.im = None + self.features = None + + @staticmethod + def remove_small_regions(masks, min_area=0, nms_thresh=0.7): + """ + Perform post-processing on segmentation masks generated by the Segment Anything Model (SAM). Specifically, this + function removes small disconnected regions and holes from the input masks, and then performs Non-Maximum + Suppression (NMS) to eliminate any newly created duplicate boxes. + + Args: + masks (torch.Tensor): A tensor containing the masks to be processed. Shape should be (N, H, W), where N is + the number of masks, H is height, and W is width. + min_area (int): The minimum area below which disconnected regions and holes will be removed. Defaults to 0. + nms_thresh (float): The IoU threshold for the NMS algorithm. Defaults to 0.7. + + Returns: + (tuple([torch.Tensor, List[int]])): + - new_masks (torch.Tensor): The processed masks with small regions removed. Shape is (N, H, W). + - keep (List[int]): The indices of the remaining masks post-NMS, which can be used to filter the boxes. + """ + if len(masks) == 0: + return masks + + # Filter small disconnected regions and holes + new_masks = [] + scores = [] + for mask in masks: + mask = mask.cpu().numpy().astype(np.uint8) + mask, changed = remove_small_regions(mask, min_area, mode="holes") + unchanged = not changed + mask, changed = remove_small_regions(mask, min_area, mode="islands") + unchanged = unchanged and not changed + + new_masks.append(torch.as_tensor(mask).unsqueeze(0)) + # Give score=0 to changed masks and 1 to unchanged masks so NMS prefers masks not needing postprocessing + scores.append(float(unchanged)) + + # Recalculate boxes and remove any new duplicates + new_masks = torch.cat(new_masks, dim=0) + boxes = batched_mask_to_box(new_masks) + keep = torchvision.ops.nms(boxes.float(), torch.as_tensor(scores), nms_thresh) + + return new_masks[keep].to(device=masks.device, dtype=masks.dtype), keep diff --git a/ultralytics/models/utils/__init__.py b/ultralytics/models/utils/__init__.py new file mode 100644 index 0000000..9e68dc1 --- /dev/null +++ b/ultralytics/models/utils/__init__.py @@ -0,0 +1 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license diff --git a/ultralytics/models/utils/loss.py b/ultralytics/models/utils/loss.py new file mode 100644 index 0000000..ac48775 --- /dev/null +++ b/ultralytics/models/utils/loss.py @@ -0,0 +1,345 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ultralytics.utils.loss import FocalLoss, VarifocalLoss +from ultralytics.utils.metrics import bbox_iou +from .ops import HungarianMatcher + + +class DETRLoss(nn.Module): + """ + DETR (DEtection TRansformer) Loss class. This class calculates and returns the different loss components for the + DETR object detection model. It computes classification loss, bounding box loss, GIoU loss, and optionally auxiliary + losses. + + Attributes: + nc (int): The number of classes. + loss_gain (dict): Coefficients for different loss components. + aux_loss (bool): Whether to compute auxiliary losses. + use_fl (bool): Use FocalLoss or not. + use_vfl (bool): Use VarifocalLoss or not. + use_uni_match (bool): Whether to use a fixed layer to assign labels for the auxiliary branch. + uni_match_ind (int): The fixed indices of a layer to use if `use_uni_match` is True. + matcher (HungarianMatcher): Object to compute matching cost and indices. + fl (FocalLoss or None): Focal Loss object if `use_fl` is True, otherwise None. + vfl (VarifocalLoss or None): Varifocal Loss object if `use_vfl` is True, otherwise None. + device (torch.device): Device on which tensors are stored. + """ + + def __init__( + self, nc=80, loss_gain=None, aux_loss=True, use_fl=True, use_vfl=False, use_uni_match=False, uni_match_ind=0 + ): + """ + DETR loss function. + + Args: + nc (int): The number of classes. + loss_gain (dict): The coefficient of loss. + aux_loss (bool): If 'aux_loss = True', loss at each decoder layer are to be used. + use_vfl (bool): Use VarifocalLoss or not. + use_uni_match (bool): Whether to use a fixed layer to assign labels for auxiliary branch. + uni_match_ind (int): The fixed indices of a layer. + """ + super().__init__() + + if loss_gain is None: + loss_gain = {"class": 1, "bbox": 5, "giou": 2, "no_object": 0.1, "mask": 1, "dice": 1} + self.nc = nc + self.matcher = HungarianMatcher(cost_gain={"class": 2, "bbox": 5, "giou": 2}) + self.loss_gain = loss_gain + self.aux_loss = aux_loss + self.fl = FocalLoss() if use_fl else None + self.vfl = VarifocalLoss() if use_vfl else None + + self.use_uni_match = use_uni_match + self.uni_match_ind = uni_match_ind + self.device = None + + def _get_loss_class(self, pred_scores, targets, gt_scores, num_gts, postfix=""): + """Computes the classification loss based on predictions, target values, and ground truth scores.""" + # Logits: [b, query, num_classes], gt_class: list[[n, 1]] + name_class = f"loss_class{postfix}" + bs, nq = pred_scores.shape[:2] + # one_hot = F.one_hot(targets, self.nc + 1)[..., :-1] # (bs, num_queries, num_classes) + one_hot = torch.zeros((bs, nq, self.nc + 1), dtype=torch.int64, device=targets.device) + one_hot.scatter_(2, targets.unsqueeze(-1), 1) + one_hot = one_hot[..., :-1] + gt_scores = gt_scores.view(bs, nq, 1) * one_hot + + if self.fl: + if num_gts and self.vfl: + loss_cls = self.vfl(pred_scores, gt_scores, one_hot) + else: + loss_cls = self.fl(pred_scores, one_hot.float()) + loss_cls /= max(num_gts, 1) / nq + else: + loss_cls = nn.BCEWithLogitsLoss(reduction="none")(pred_scores, gt_scores).mean(1).sum() # YOLO CLS loss + + return {name_class: loss_cls.squeeze() * self.loss_gain["class"]} + + def _get_loss_bbox(self, pred_bboxes, gt_bboxes, postfix=""): + """Calculates and returns the bounding box loss and GIoU loss for the predicted and ground truth bounding + boxes. + """ + # Boxes: [b, query, 4], gt_bbox: list[[n, 4]] + name_bbox = f"loss_bbox{postfix}" + name_giou = f"loss_giou{postfix}" + + loss = {} + if len(gt_bboxes) == 0: + loss[name_bbox] = torch.tensor(0.0, device=self.device) + loss[name_giou] = torch.tensor(0.0, device=self.device) + return loss + + loss[name_bbox] = self.loss_gain["bbox"] * F.l1_loss(pred_bboxes, gt_bboxes, reduction="sum") / len(gt_bboxes) + loss[name_giou] = 1.0 - bbox_iou(pred_bboxes, gt_bboxes, xywh=True, GIoU=True) + loss[name_giou] = loss[name_giou].sum() / len(gt_bboxes) + loss[name_giou] = self.loss_gain["giou"] * loss[name_giou] + return {k: v.squeeze() for k, v in loss.items()} + + # This function is for future RT-DETR Segment models + # def _get_loss_mask(self, masks, gt_mask, match_indices, postfix=''): + # # masks: [b, query, h, w], gt_mask: list[[n, H, W]] + # name_mask = f'loss_mask{postfix}' + # name_dice = f'loss_dice{postfix}' + # + # loss = {} + # if sum(len(a) for a in gt_mask) == 0: + # loss[name_mask] = torch.tensor(0., device=self.device) + # loss[name_dice] = torch.tensor(0., device=self.device) + # return loss + # + # num_gts = len(gt_mask) + # src_masks, target_masks = self._get_assigned_bboxes(masks, gt_mask, match_indices) + # src_masks = F.interpolate(src_masks.unsqueeze(0), size=target_masks.shape[-2:], mode='bilinear')[0] + # # TODO: torch does not have `sigmoid_focal_loss`, but it's not urgent since we don't use mask branch for now. + # loss[name_mask] = self.loss_gain['mask'] * F.sigmoid_focal_loss(src_masks, target_masks, + # torch.tensor([num_gts], dtype=torch.float32)) + # loss[name_dice] = self.loss_gain['dice'] * self._dice_loss(src_masks, target_masks, num_gts) + # return loss + + # This function is for future RT-DETR Segment models + # @staticmethod + # def _dice_loss(inputs, targets, num_gts): + # inputs = F.sigmoid(inputs).flatten(1) + # targets = targets.flatten(1) + # numerator = 2 * (inputs * targets).sum(1) + # denominator = inputs.sum(-1) + targets.sum(-1) + # loss = 1 - (numerator + 1) / (denominator + 1) + # return loss.sum() / num_gts + + def _get_loss_aux( + self, + pred_bboxes, + pred_scores, + gt_bboxes, + gt_cls, + gt_groups, + match_indices=None, + postfix="", + masks=None, + gt_mask=None, + ): + """Get auxiliary losses.""" + # NOTE: loss class, bbox, giou, mask, dice + loss = torch.zeros(5 if masks is not None else 3, device=pred_bboxes.device) + if match_indices is None and self.use_uni_match: + match_indices = self.matcher( + pred_bboxes[self.uni_match_ind], + pred_scores[self.uni_match_ind], + gt_bboxes, + gt_cls, + gt_groups, + masks=masks[self.uni_match_ind] if masks is not None else None, + gt_mask=gt_mask, + ) + for i, (aux_bboxes, aux_scores) in enumerate(zip(pred_bboxes, pred_scores)): + aux_masks = masks[i] if masks is not None else None + loss_ = self._get_loss( + aux_bboxes, + aux_scores, + gt_bboxes, + gt_cls, + gt_groups, + masks=aux_masks, + gt_mask=gt_mask, + postfix=postfix, + match_indices=match_indices, + ) + loss[0] += loss_[f"loss_class{postfix}"] + loss[1] += loss_[f"loss_bbox{postfix}"] + loss[2] += loss_[f"loss_giou{postfix}"] + # if masks is not None and gt_mask is not None: + # loss_ = self._get_loss_mask(aux_masks, gt_mask, match_indices, postfix) + # loss[3] += loss_[f'loss_mask{postfix}'] + # loss[4] += loss_[f'loss_dice{postfix}'] + + loss = { + f"loss_class_aux{postfix}": loss[0], + f"loss_bbox_aux{postfix}": loss[1], + f"loss_giou_aux{postfix}": loss[2], + } + # if masks is not None and gt_mask is not None: + # loss[f'loss_mask_aux{postfix}'] = loss[3] + # loss[f'loss_dice_aux{postfix}'] = loss[4] + return loss + + @staticmethod + def _get_index(match_indices): + """Returns batch indices, source indices, and destination indices from provided match indices.""" + batch_idx = torch.cat([torch.full_like(src, i) for i, (src, _) in enumerate(match_indices)]) + src_idx = torch.cat([src for (src, _) in match_indices]) + dst_idx = torch.cat([dst for (_, dst) in match_indices]) + return (batch_idx, src_idx), dst_idx + + def _get_assigned_bboxes(self, pred_bboxes, gt_bboxes, match_indices): + """Assigns predicted bounding boxes to ground truth bounding boxes based on the match indices.""" + pred_assigned = torch.cat( + [ + t[i] if len(i) > 0 else torch.zeros(0, t.shape[-1], device=self.device) + for t, (i, _) in zip(pred_bboxes, match_indices) + ] + ) + gt_assigned = torch.cat( + [ + t[j] if len(j) > 0 else torch.zeros(0, t.shape[-1], device=self.device) + for t, (_, j) in zip(gt_bboxes, match_indices) + ] + ) + return pred_assigned, gt_assigned + + def _get_loss( + self, + pred_bboxes, + pred_scores, + gt_bboxes, + gt_cls, + gt_groups, + masks=None, + gt_mask=None, + postfix="", + match_indices=None, + ): + """Get losses.""" + if match_indices is None: + match_indices = self.matcher( + pred_bboxes, pred_scores, gt_bboxes, gt_cls, gt_groups, masks=masks, gt_mask=gt_mask + ) + + idx, gt_idx = self._get_index(match_indices) + pred_bboxes, gt_bboxes = pred_bboxes[idx], gt_bboxes[gt_idx] + + bs, nq = pred_scores.shape[:2] + targets = torch.full((bs, nq), self.nc, device=pred_scores.device, dtype=gt_cls.dtype) + targets[idx] = gt_cls[gt_idx] + + gt_scores = torch.zeros([bs, nq], device=pred_scores.device) + if len(gt_bboxes): + gt_scores[idx] = bbox_iou(pred_bboxes.detach(), gt_bboxes, xywh=True).squeeze(-1) + + loss = {} + loss.update(self._get_loss_class(pred_scores, targets, gt_scores, len(gt_bboxes), postfix)) + loss.update(self._get_loss_bbox(pred_bboxes, gt_bboxes, postfix)) + # if masks is not None and gt_mask is not None: + # loss.update(self._get_loss_mask(masks, gt_mask, match_indices, postfix)) + return loss + + def forward(self, pred_bboxes, pred_scores, batch, postfix="", **kwargs): + """ + Args: + pred_bboxes (torch.Tensor): [l, b, query, 4] + pred_scores (torch.Tensor): [l, b, query, num_classes] + batch (dict): A dict includes: + gt_cls (torch.Tensor) with shape [num_gts, ], + gt_bboxes (torch.Tensor): [num_gts, 4], + gt_groups (List(int)): a list of batch size length includes the number of gts of each image. + postfix (str): postfix of loss name. + """ + self.device = pred_bboxes.device + match_indices = kwargs.get("match_indices", None) + gt_cls, gt_bboxes, gt_groups = batch["cls"], batch["bboxes"], batch["gt_groups"] + + total_loss = self._get_loss( + pred_bboxes[-1], pred_scores[-1], gt_bboxes, gt_cls, gt_groups, postfix=postfix, match_indices=match_indices + ) + + if self.aux_loss: + total_loss.update( + self._get_loss_aux( + pred_bboxes[:-1], pred_scores[:-1], gt_bboxes, gt_cls, gt_groups, match_indices, postfix + ) + ) + + return total_loss + + +class RTDETRDetectionLoss(DETRLoss): + """ + Real-Time DeepTracker (RT-DETR) Detection Loss class that extends the DETRLoss. + + This class computes the detection loss for the RT-DETR model, which includes the standard detection loss as well as + an additional denoising training loss when provided with denoising metadata. + """ + + def forward(self, preds, batch, dn_bboxes=None, dn_scores=None, dn_meta=None): + """ + Forward pass to compute the detection loss. + + Args: + preds (tuple): Predicted bounding boxes and scores. + batch (dict): Batch data containing ground truth information. + dn_bboxes (torch.Tensor, optional): Denoising bounding boxes. Default is None. + dn_scores (torch.Tensor, optional): Denoising scores. Default is None. + dn_meta (dict, optional): Metadata for denoising. Default is None. + + Returns: + (dict): Dictionary containing the total loss and, if applicable, the denoising loss. + """ + pred_bboxes, pred_scores = preds + total_loss = super().forward(pred_bboxes, pred_scores, batch) + + # Check for denoising metadata to compute denoising training loss + if dn_meta is not None: + dn_pos_idx, dn_num_group = dn_meta["dn_pos_idx"], dn_meta["dn_num_group"] + assert len(batch["gt_groups"]) == len(dn_pos_idx) + + # Get the match indices for denoising + match_indices = self.get_dn_match_indices(dn_pos_idx, dn_num_group, batch["gt_groups"]) + + # Compute the denoising training loss + dn_loss = super().forward(dn_bboxes, dn_scores, batch, postfix="_dn", match_indices=match_indices) + total_loss.update(dn_loss) + else: + # If no denoising metadata is provided, set denoising loss to zero + total_loss.update({f"{k}_dn": torch.tensor(0.0, device=self.device) for k in total_loss.keys()}) + + return total_loss + + @staticmethod + def get_dn_match_indices(dn_pos_idx, dn_num_group, gt_groups): + """ + Get the match indices for denoising. + + Args: + dn_pos_idx (List[torch.Tensor]): List of tensors containing positive indices for denoising. + dn_num_group (int): Number of denoising groups. + gt_groups (List[int]): List of integers representing the number of ground truths for each image. + + Returns: + (List[tuple]): List of tuples containing matched indices for denoising. + """ + dn_match_indices = [] + idx_groups = torch.as_tensor([0, *gt_groups[:-1]]).cumsum_(0) + for i, num_gt in enumerate(gt_groups): + if num_gt > 0: + gt_idx = torch.arange(end=num_gt, dtype=torch.long) + idx_groups[i] + gt_idx = gt_idx.repeat(dn_num_group) + assert len(dn_pos_idx[i]) == len(gt_idx), "Expected the same length, " + f"but got {len(dn_pos_idx[i])} and {len(gt_idx)} respectively." + dn_match_indices.append((dn_pos_idx[i], gt_idx)) + else: + dn_match_indices.append((torch.zeros([0], dtype=torch.long), torch.zeros([0], dtype=torch.long))) + return dn_match_indices diff --git a/ultralytics/models/utils/ops.py b/ultralytics/models/utils/ops.py new file mode 100644 index 0000000..4f66fee --- /dev/null +++ b/ultralytics/models/utils/ops.py @@ -0,0 +1,263 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import torch +import torch.nn as nn +import torch.nn.functional as F +from scipy.optimize import linear_sum_assignment + +from ultralytics.utils.metrics import bbox_iou +from ultralytics.utils.ops import xywh2xyxy, xyxy2xywh + + +class HungarianMatcher(nn.Module): + """ + A module implementing the HungarianMatcher, which is a differentiable module to solve the assignment problem in an + end-to-end fashion. + + HungarianMatcher performs optimal assignment over the predicted and ground truth bounding boxes using a cost + function that considers classification scores, bounding box coordinates, and optionally, mask predictions. + + Attributes: + cost_gain (dict): Dictionary of cost coefficients: 'class', 'bbox', 'giou', 'mask', and 'dice'. + use_fl (bool): Indicates whether to use Focal Loss for the classification cost calculation. + with_mask (bool): Indicates whether the model makes mask predictions. + num_sample_points (int): The number of sample points used in mask cost calculation. + alpha (float): The alpha factor in Focal Loss calculation. + gamma (float): The gamma factor in Focal Loss calculation. + + Methods: + forward(pred_bboxes, pred_scores, gt_bboxes, gt_cls, gt_groups, masks=None, gt_mask=None): Computes the + assignment between predictions and ground truths for a batch. + _cost_mask(bs, num_gts, masks=None, gt_mask=None): Computes the mask cost and dice cost if masks are predicted. + """ + + def __init__(self, cost_gain=None, use_fl=True, with_mask=False, num_sample_points=12544, alpha=0.25, gamma=2.0): + """Initializes HungarianMatcher with cost coefficients, Focal Loss, mask prediction, sample points, and alpha + gamma factors. + """ + super().__init__() + if cost_gain is None: + cost_gain = {"class": 1, "bbox": 5, "giou": 2, "mask": 1, "dice": 1} + self.cost_gain = cost_gain + self.use_fl = use_fl + self.with_mask = with_mask + self.num_sample_points = num_sample_points + self.alpha = alpha + self.gamma = gamma + + def forward(self, pred_bboxes, pred_scores, gt_bboxes, gt_cls, gt_groups, masks=None, gt_mask=None): + """ + Forward pass for HungarianMatcher. This function computes costs based on prediction and ground truth + (classification cost, L1 cost between boxes and GIoU cost between boxes) and finds the optimal matching between + predictions and ground truth based on these costs. + + Args: + pred_bboxes (Tensor): Predicted bounding boxes with shape [batch_size, num_queries, 4]. + pred_scores (Tensor): Predicted scores with shape [batch_size, num_queries, num_classes]. + gt_cls (torch.Tensor): Ground truth classes with shape [num_gts, ]. + gt_bboxes (torch.Tensor): Ground truth bounding boxes with shape [num_gts, 4]. + gt_groups (List[int]): List of length equal to batch size, containing the number of ground truths for + each image. + masks (Tensor, optional): Predicted masks with shape [batch_size, num_queries, height, width]. + Defaults to None. + gt_mask (List[Tensor], optional): List of ground truth masks, each with shape [num_masks, Height, Width]. + Defaults to None. + + Returns: + (List[Tuple[Tensor, Tensor]]): A list of size batch_size, each element is a tuple (index_i, index_j), where: + - index_i is the tensor of indices of the selected predictions (in order) + - index_j is the tensor of indices of the corresponding selected ground truth targets (in order) + For each batch element, it holds: + len(index_i) = len(index_j) = min(num_queries, num_target_boxes) + """ + + bs, nq, nc = pred_scores.shape + + if sum(gt_groups) == 0: + return [(torch.tensor([], dtype=torch.long), torch.tensor([], dtype=torch.long)) for _ in range(bs)] + + # We flatten to compute the cost matrices in a batch + # [batch_size * num_queries, num_classes] + pred_scores = pred_scores.detach().view(-1, nc) + pred_scores = F.sigmoid(pred_scores) if self.use_fl else F.softmax(pred_scores, dim=-1) + # [batch_size * num_queries, 4] + pred_bboxes = pred_bboxes.detach().view(-1, 4) + + # Compute the classification cost + pred_scores = pred_scores[:, gt_cls] + if self.use_fl: + neg_cost_class = (1 - self.alpha) * (pred_scores**self.gamma) * (-(1 - pred_scores + 1e-8).log()) + pos_cost_class = self.alpha * ((1 - pred_scores) ** self.gamma) * (-(pred_scores + 1e-8).log()) + cost_class = pos_cost_class - neg_cost_class + else: + cost_class = -pred_scores + + # Compute the L1 cost between boxes + cost_bbox = (pred_bboxes.unsqueeze(1) - gt_bboxes.unsqueeze(0)).abs().sum(-1) # (bs*num_queries, num_gt) + + # Compute the GIoU cost between boxes, (bs*num_queries, num_gt) + cost_giou = 1.0 - bbox_iou(pred_bboxes.unsqueeze(1), gt_bboxes.unsqueeze(0), xywh=True, GIoU=True).squeeze(-1) + + # Final cost matrix + C = ( + self.cost_gain["class"] * cost_class + + self.cost_gain["bbox"] * cost_bbox + + self.cost_gain["giou"] * cost_giou + ) + # Compute the mask cost and dice cost + if self.with_mask: + C += self._cost_mask(bs, gt_groups, masks, gt_mask) + + # Set invalid values (NaNs and infinities) to 0 (fixes ValueError: matrix contains invalid numeric entries) + C[C.isnan() | C.isinf()] = 0.0 + + C = C.view(bs, nq, -1).cpu() + indices = [linear_sum_assignment(c[i]) for i, c in enumerate(C.split(gt_groups, -1))] + gt_groups = torch.as_tensor([0, *gt_groups[:-1]]).cumsum_(0) # (idx for queries, idx for gt) + return [ + (torch.tensor(i, dtype=torch.long), torch.tensor(j, dtype=torch.long) + gt_groups[k]) + for k, (i, j) in enumerate(indices) + ] + + # This function is for future RT-DETR Segment models + # def _cost_mask(self, bs, num_gts, masks=None, gt_mask=None): + # assert masks is not None and gt_mask is not None, 'Make sure the input has `mask` and `gt_mask`' + # # all masks share the same set of points for efficient matching + # sample_points = torch.rand([bs, 1, self.num_sample_points, 2]) + # sample_points = 2.0 * sample_points - 1.0 + # + # out_mask = F.grid_sample(masks.detach(), sample_points, align_corners=False).squeeze(-2) + # out_mask = out_mask.flatten(0, 1) + # + # tgt_mask = torch.cat(gt_mask).unsqueeze(1) + # sample_points = torch.cat([a.repeat(b, 1, 1, 1) for a, b in zip(sample_points, num_gts) if b > 0]) + # tgt_mask = F.grid_sample(tgt_mask, sample_points, align_corners=False).squeeze([1, 2]) + # + # with torch.cuda.amp.autocast(False): + # # binary cross entropy cost + # pos_cost_mask = F.binary_cross_entropy_with_logits(out_mask, torch.ones_like(out_mask), reduction='none') + # neg_cost_mask = F.binary_cross_entropy_with_logits(out_mask, torch.zeros_like(out_mask), reduction='none') + # cost_mask = torch.matmul(pos_cost_mask, tgt_mask.T) + torch.matmul(neg_cost_mask, 1 - tgt_mask.T) + # cost_mask /= self.num_sample_points + # + # # dice cost + # out_mask = F.sigmoid(out_mask) + # numerator = 2 * torch.matmul(out_mask, tgt_mask.T) + # denominator = out_mask.sum(-1, keepdim=True) + tgt_mask.sum(-1).unsqueeze(0) + # cost_dice = 1 - (numerator + 1) / (denominator + 1) + # + # C = self.cost_gain['mask'] * cost_mask + self.cost_gain['dice'] * cost_dice + # return C + + +def get_cdn_group( + batch, num_classes, num_queries, class_embed, num_dn=100, cls_noise_ratio=0.5, box_noise_scale=1.0, training=False +): + """ + Get contrastive denoising training group. This function creates a contrastive denoising training group with positive + and negative samples from the ground truths (gt). It applies noise to the class labels and bounding box coordinates, + and returns the modified labels, bounding boxes, attention mask and meta information. + + Args: + batch (dict): A dict that includes 'gt_cls' (torch.Tensor with shape [num_gts, ]), 'gt_bboxes' + (torch.Tensor with shape [num_gts, 4]), 'gt_groups' (List(int)) which is a list of batch size length + indicating the number of gts of each image. + num_classes (int): Number of classes. + num_queries (int): Number of queries. + class_embed (torch.Tensor): Embedding weights to map class labels to embedding space. + num_dn (int, optional): Number of denoising. Defaults to 100. + cls_noise_ratio (float, optional): Noise ratio for class labels. Defaults to 0.5. + box_noise_scale (float, optional): Noise scale for bounding box coordinates. Defaults to 1.0. + training (bool, optional): If it's in training mode. Defaults to False. + + Returns: + (Tuple[Optional[Tensor], Optional[Tensor], Optional[Tensor], Optional[Dict]]): The modified class embeddings, + bounding boxes, attention mask and meta information for denoising. If not in training mode or 'num_dn' + is less than or equal to 0, the function returns None for all elements in the tuple. + """ + + if (not training) or num_dn <= 0: + return None, None, None, None + gt_groups = batch["gt_groups"] + total_num = sum(gt_groups) + max_nums = max(gt_groups) + if max_nums == 0: + return None, None, None, None + + num_group = num_dn // max_nums + num_group = 1 if num_group == 0 else num_group + # Pad gt to max_num of a batch + bs = len(gt_groups) + gt_cls = batch["cls"] # (bs*num, ) + gt_bbox = batch["bboxes"] # bs*num, 4 + b_idx = batch["batch_idx"] + + # Each group has positive and negative queries. + dn_cls = gt_cls.repeat(2 * num_group) # (2*num_group*bs*num, ) + dn_bbox = gt_bbox.repeat(2 * num_group, 1) # 2*num_group*bs*num, 4 + dn_b_idx = b_idx.repeat(2 * num_group).view(-1) # (2*num_group*bs*num, ) + + # Positive and negative mask + # (bs*num*num_group, ), the second total_num*num_group part as negative samples + neg_idx = torch.arange(total_num * num_group, dtype=torch.long, device=gt_bbox.device) + num_group * total_num + + if cls_noise_ratio > 0: + # Half of bbox prob + mask = torch.rand(dn_cls.shape) < (cls_noise_ratio * 0.5) + idx = torch.nonzero(mask).squeeze(-1) + # Randomly put a new one here + new_label = torch.randint_like(idx, 0, num_classes, dtype=dn_cls.dtype, device=dn_cls.device) + dn_cls[idx] = new_label + + if box_noise_scale > 0: + known_bbox = xywh2xyxy(dn_bbox) + + diff = (dn_bbox[..., 2:] * 0.5).repeat(1, 2) * box_noise_scale # 2*num_group*bs*num, 4 + + rand_sign = torch.randint_like(dn_bbox, 0, 2) * 2.0 - 1.0 + rand_part = torch.rand_like(dn_bbox) + rand_part[neg_idx] += 1.0 + rand_part *= rand_sign + known_bbox += rand_part * diff + known_bbox.clip_(min=0.0, max=1.0) + dn_bbox = xyxy2xywh(known_bbox) + dn_bbox = torch.logit(dn_bbox, eps=1e-6) # inverse sigmoid + + num_dn = int(max_nums * 2 * num_group) # total denoising queries + # class_embed = torch.cat([class_embed, torch.zeros([1, class_embed.shape[-1]], device=class_embed.device)]) + dn_cls_embed = class_embed[dn_cls] # bs*num * 2 * num_group, 256 + padding_cls = torch.zeros(bs, num_dn, dn_cls_embed.shape[-1], device=gt_cls.device) + padding_bbox = torch.zeros(bs, num_dn, 4, device=gt_bbox.device) + + map_indices = torch.cat([torch.tensor(range(num), dtype=torch.long) for num in gt_groups]) + pos_idx = torch.stack([map_indices + max_nums * i for i in range(num_group)], dim=0) + + map_indices = torch.cat([map_indices + max_nums * i for i in range(2 * num_group)]) + padding_cls[(dn_b_idx, map_indices)] = dn_cls_embed + padding_bbox[(dn_b_idx, map_indices)] = dn_bbox + + tgt_size = num_dn + num_queries + attn_mask = torch.zeros([tgt_size, tgt_size], dtype=torch.bool) + # Match query cannot see the reconstruct + attn_mask[num_dn:, :num_dn] = True + # Reconstruct cannot see each other + for i in range(num_group): + if i == 0: + attn_mask[max_nums * 2 * i : max_nums * 2 * (i + 1), max_nums * 2 * (i + 1) : num_dn] = True + if i == num_group - 1: + attn_mask[max_nums * 2 * i : max_nums * 2 * (i + 1), : max_nums * i * 2] = True + else: + attn_mask[max_nums * 2 * i : max_nums * 2 * (i + 1), max_nums * 2 * (i + 1) : num_dn] = True + attn_mask[max_nums * 2 * i : max_nums * 2 * (i + 1), : max_nums * 2 * i] = True + dn_meta = { + "dn_pos_idx": [p.reshape(-1) for p in pos_idx.cpu().split(list(gt_groups), dim=1)], + "dn_num_group": num_group, + "dn_num_split": [num_dn, num_queries], + } + + return ( + padding_cls.to(class_embed.device), + padding_bbox.to(class_embed.device), + attn_mask.to(class_embed.device), + dn_meta, + ) diff --git a/ultralytics/models/yolo/__init__.py b/ultralytics/models/yolo/__init__.py new file mode 100644 index 0000000..7b1a597 --- /dev/null +++ b/ultralytics/models/yolo/__init__.py @@ -0,0 +1,7 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from ultralytics.models.yolo import classify, detect, obb, pose, segment + +from .model import YOLO, YOLOWorld + +__all__ = "classify", "segment", "detect", "pose", "obb", "YOLO", "YOLOWorld" diff --git a/ultralytics/models/yolo/__pycache__/__init__.cpython-312.pyc b/ultralytics/models/yolo/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..7ae5fc9 Binary files /dev/null and b/ultralytics/models/yolo/__pycache__/__init__.cpython-312.pyc differ diff --git a/ultralytics/models/yolo/__pycache__/__init__.cpython-39.pyc b/ultralytics/models/yolo/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..f8d9866 Binary files /dev/null and b/ultralytics/models/yolo/__pycache__/__init__.cpython-39.pyc differ diff --git a/ultralytics/models/yolo/__pycache__/model.cpython-312.pyc b/ultralytics/models/yolo/__pycache__/model.cpython-312.pyc new file mode 100644 index 0000000..afdc537 Binary files /dev/null and b/ultralytics/models/yolo/__pycache__/model.cpython-312.pyc differ diff --git a/ultralytics/models/yolo/__pycache__/model.cpython-39.pyc b/ultralytics/models/yolo/__pycache__/model.cpython-39.pyc new file mode 100644 index 0000000..f489232 Binary files /dev/null and b/ultralytics/models/yolo/__pycache__/model.cpython-39.pyc differ diff --git a/ultralytics/models/yolo/classify/__init__.py b/ultralytics/models/yolo/classify/__init__.py new file mode 100644 index 0000000..ca92f89 --- /dev/null +++ b/ultralytics/models/yolo/classify/__init__.py @@ -0,0 +1,7 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from ultralytics.models.yolo.classify.predict import ClassificationPredictor +from ultralytics.models.yolo.classify.train import ClassificationTrainer +from ultralytics.models.yolo.classify.val import ClassificationValidator + +__all__ = "ClassificationPredictor", "ClassificationTrainer", "ClassificationValidator" diff --git a/ultralytics/models/yolo/classify/__pycache__/__init__.cpython-312.pyc b/ultralytics/models/yolo/classify/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..f001b5c Binary files /dev/null and b/ultralytics/models/yolo/classify/__pycache__/__init__.cpython-312.pyc differ diff --git a/ultralytics/models/yolo/classify/__pycache__/__init__.cpython-39.pyc b/ultralytics/models/yolo/classify/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..419dfac Binary files /dev/null and b/ultralytics/models/yolo/classify/__pycache__/__init__.cpython-39.pyc differ diff --git a/ultralytics/models/yolo/classify/__pycache__/predict.cpython-312.pyc b/ultralytics/models/yolo/classify/__pycache__/predict.cpython-312.pyc new file mode 100644 index 0000000..ba029b4 Binary files /dev/null and b/ultralytics/models/yolo/classify/__pycache__/predict.cpython-312.pyc differ diff --git a/ultralytics/models/yolo/classify/__pycache__/predict.cpython-39.pyc b/ultralytics/models/yolo/classify/__pycache__/predict.cpython-39.pyc new file mode 100644 index 0000000..2d90a86 Binary files /dev/null and b/ultralytics/models/yolo/classify/__pycache__/predict.cpython-39.pyc differ diff --git a/ultralytics/models/yolo/classify/__pycache__/train.cpython-312.pyc b/ultralytics/models/yolo/classify/__pycache__/train.cpython-312.pyc new file mode 100644 index 0000000..1a06333 Binary files /dev/null and b/ultralytics/models/yolo/classify/__pycache__/train.cpython-312.pyc differ diff --git a/ultralytics/models/yolo/classify/__pycache__/train.cpython-39.pyc b/ultralytics/models/yolo/classify/__pycache__/train.cpython-39.pyc new file mode 100644 index 0000000..8078fc1 Binary files /dev/null and b/ultralytics/models/yolo/classify/__pycache__/train.cpython-39.pyc differ diff --git a/ultralytics/models/yolo/classify/__pycache__/val.cpython-312.pyc b/ultralytics/models/yolo/classify/__pycache__/val.cpython-312.pyc new file mode 100644 index 0000000..ec15934 Binary files /dev/null and b/ultralytics/models/yolo/classify/__pycache__/val.cpython-312.pyc differ diff --git a/ultralytics/models/yolo/classify/__pycache__/val.cpython-39.pyc b/ultralytics/models/yolo/classify/__pycache__/val.cpython-39.pyc new file mode 100644 index 0000000..63b50a9 Binary files /dev/null and b/ultralytics/models/yolo/classify/__pycache__/val.cpython-39.pyc differ diff --git a/ultralytics/models/yolo/classify/predict.py b/ultralytics/models/yolo/classify/predict.py new file mode 100644 index 0000000..853ef04 --- /dev/null +++ b/ultralytics/models/yolo/classify/predict.py @@ -0,0 +1,61 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import cv2 +import torch +from PIL import Image + +from ultralytics.engine.predictor import BasePredictor +from ultralytics.engine.results import Results +from ultralytics.utils import DEFAULT_CFG, ops + + +class ClassificationPredictor(BasePredictor): + """ + A class extending the BasePredictor class for prediction based on a classification model. + + Notes: + - Torchvision classification models can also be passed to the 'model' argument, i.e. model='resnet18'. + + Example: + ```python + from ultralytics.utils import ASSETS + from ultralytics.models.yolo.classify import ClassificationPredictor + + args = dict(model='yolov8n-cls.pt', source=ASSETS) + predictor = ClassificationPredictor(overrides=args) + predictor.predict_cli() + ``` + """ + + def __init__(self, cfg=DEFAULT_CFG, overrides=None, _callbacks=None): + """Initializes ClassificationPredictor setting the task to 'classify'.""" + super().__init__(cfg, overrides, _callbacks) + self.args.task = "classify" + self._legacy_transform_name = "ultralytics.yolo.data.augment.ToTensor" + + def preprocess(self, img): + """Converts input image to model-compatible data type.""" + if not isinstance(img, torch.Tensor): + is_legacy_transform = any( + self._legacy_transform_name in str(transform) for transform in self.transforms.transforms + ) + if is_legacy_transform: # to handle legacy transforms + img = torch.stack([self.transforms(im) for im in img], dim=0) + else: + img = torch.stack( + [self.transforms(Image.fromarray(cv2.cvtColor(im, cv2.COLOR_BGR2RGB))) for im in img], dim=0 + ) + img = (img if isinstance(img, torch.Tensor) else torch.from_numpy(img)).to(self.model.device) + return img.half() if self.model.fp16 else img.float() # uint8 to fp16/32 + + def postprocess(self, preds, img, orig_imgs): + """Post-processes predictions to return Results objects.""" + if not isinstance(orig_imgs, list): # input images are a torch.Tensor, not a list + orig_imgs = ops.convert_torch2numpy_batch(orig_imgs) + + results = [] + for i, pred in enumerate(preds): + orig_img = orig_imgs[i] + img_path = self.batch[0][i] + results.append(Results(orig_img, path=img_path, names=self.model.names, probs=pred)) + return results diff --git a/ultralytics/models/yolo/classify/train.py b/ultralytics/models/yolo/classify/train.py new file mode 100644 index 0000000..42c6554 --- /dev/null +++ b/ultralytics/models/yolo/classify/train.py @@ -0,0 +1,158 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import torch +import torchvision + +from ultralytics.data import ClassificationDataset, build_dataloader +from ultralytics.engine.trainer import BaseTrainer +from ultralytics.models import yolo +from ultralytics.nn.tasks import ClassificationModel, attempt_load_one_weight +from ultralytics.utils import DEFAULT_CFG, LOGGER, RANK, colorstr +from ultralytics.utils.plotting import plot_images, plot_results +from ultralytics.utils.torch_utils import is_parallel, strip_optimizer, torch_distributed_zero_first + + +class ClassificationTrainer(BaseTrainer): + """ + A class extending the BaseTrainer class for training based on a classification model. + + Notes: + - Torchvision classification models can also be passed to the 'model' argument, i.e. model='resnet18'. + + Example: + ```python + from ultralytics.models.yolo.classify import ClassificationTrainer + + args = dict(model='yolov8n-cls.pt', data='imagenet10', epochs=3) + trainer = ClassificationTrainer(overrides=args) + trainer.train() + ``` + """ + + def __init__(self, cfg=DEFAULT_CFG, overrides=None, _callbacks=None): + """Initialize a ClassificationTrainer object with optional configuration overrides and callbacks.""" + if overrides is None: + overrides = {} + overrides["task"] = "classify" + if overrides.get("imgsz") is None: + overrides["imgsz"] = 224 + super().__init__(cfg, overrides, _callbacks) + + def set_model_attributes(self): + """Set the YOLO model's class names from the loaded dataset.""" + self.model.names = self.data["names"] + + def get_model(self, cfg=None, weights=None, verbose=True): + """Returns a modified PyTorch model configured for training YOLO.""" + model = ClassificationModel(cfg, nc=self.data["nc"], verbose=verbose and RANK == -1) + if weights: + model.load(weights) + + for m in model.modules(): + if not self.args.pretrained and hasattr(m, "reset_parameters"): + m.reset_parameters() + if isinstance(m, torch.nn.Dropout) and self.args.dropout: + m.p = self.args.dropout # set dropout + for p in model.parameters(): + p.requires_grad = True # for training + return model + + def setup_model(self): + """Load, create or download model for any task.""" + if isinstance(self.model, torch.nn.Module): # if model is loaded beforehand. No setup needed + return + + model, ckpt = str(self.model), None + # Load a YOLO model locally, from torchvision, or from Ultralytics assets + if model.endswith(".pt"): + self.model, ckpt = attempt_load_one_weight(model, device="cpu") + for p in self.model.parameters(): + p.requires_grad = True # for training + elif model.split(".")[-1] in ("yaml", "yml"): + self.model = self.get_model(cfg=model) + elif model in torchvision.models.__dict__: + self.model = torchvision.models.__dict__[model](weights="IMAGENET1K_V1" if self.args.pretrained else None) + else: + raise FileNotFoundError(f"ERROR: model={model} not found locally or online. Please check model name.") + ClassificationModel.reshape_outputs(self.model, self.data["nc"]) + + return ckpt + + def build_dataset(self, img_path, mode="train", batch=None): + """Creates a ClassificationDataset instance given an image path, and mode (train/test etc.).""" + return ClassificationDataset(root=img_path, args=self.args, augment=mode == "train", prefix=mode) + + def get_dataloader(self, dataset_path, batch_size=16, rank=0, mode="train"): + """Returns PyTorch DataLoader with transforms to preprocess images for inference.""" + with torch_distributed_zero_first(rank): # init dataset *.cache only once if DDP + dataset = self.build_dataset(dataset_path, mode) + + loader = build_dataloader(dataset, batch_size, self.args.workers, rank=rank) + # Attach inference transforms + if mode != "train": + if is_parallel(self.model): + self.model.module.transforms = loader.dataset.torch_transforms + else: + self.model.transforms = loader.dataset.torch_transforms + return loader + + def preprocess_batch(self, batch): + """Preprocesses a batch of images and classes.""" + batch["img"] = batch["img"].to(self.device) + batch["cls"] = batch["cls"].to(self.device) + return batch + + def progress_string(self): + """Returns a formatted string showing training progress.""" + return ("\n" + "%11s" * (4 + len(self.loss_names))) % ( + "Epoch", + "GPU_mem", + *self.loss_names, + "Instances", + "Size", + ) + + def get_validator(self): + """Returns an instance of ClassificationValidator for validation.""" + self.loss_names = ["loss"] + return yolo.classify.ClassificationValidator(self.test_loader, self.save_dir, _callbacks=self.callbacks) + + def label_loss_items(self, loss_items=None, prefix="train"): + """ + Returns a loss dict with labelled training loss items tensor. + + Not needed for classification but necessary for segmentation & detection + """ + keys = [f"{prefix}/{x}" for x in self.loss_names] + if loss_items is None: + return keys + loss_items = [round(float(loss_items), 5)] + return dict(zip(keys, loss_items)) + + def plot_metrics(self): + """Plots metrics from a CSV file.""" + plot_results(file=self.csv, classify=True, on_plot=self.on_plot) # save results.png + + def final_eval(self): + """Evaluate trained model and save validation results.""" + for f in self.last, self.best: + if f.exists(): + strip_optimizer(f) # strip optimizers + if f is self.best: + LOGGER.info(f"\nValidating {f}...") + self.validator.args.data = self.args.data + self.validator.args.plots = self.args.plots + self.metrics = self.validator(model=f) + self.metrics.pop("fitness", None) + self.run_callbacks("on_fit_epoch_end") + LOGGER.info(f"Results saved to {colorstr('bold', self.save_dir)}") + + def plot_training_samples(self, batch, ni): + """Plots training samples with their annotations.""" + plot_images( + images=batch["img"], + batch_idx=torch.arange(len(batch["img"])), + cls=batch["cls"].view(-1), # warning: use .view(), not .squeeze() for Classify models + fname=self.save_dir / f"train_batch{ni}.jpg", + on_plot=self.on_plot, + ) diff --git a/ultralytics/models/yolo/classify/val.py b/ultralytics/models/yolo/classify/val.py new file mode 100644 index 0000000..de3cff2 --- /dev/null +++ b/ultralytics/models/yolo/classify/val.py @@ -0,0 +1,113 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import torch + +from ultralytics.data import ClassificationDataset, build_dataloader +from ultralytics.engine.validator import BaseValidator +from ultralytics.utils import LOGGER +from ultralytics.utils.metrics import ClassifyMetrics, ConfusionMatrix +from ultralytics.utils.plotting import plot_images + + +class ClassificationValidator(BaseValidator): + """ + A class extending the BaseValidator class for validation based on a classification model. + + Notes: + - Torchvision classification models can also be passed to the 'model' argument, i.e. model='resnet18'. + + Example: + ```python + from ultralytics.models.yolo.classify import ClassificationValidator + + args = dict(model='yolov8n-cls.pt', data='imagenet10') + validator = ClassificationValidator(args=args) + validator() + ``` + """ + + def __init__(self, dataloader=None, save_dir=None, pbar=None, args=None, _callbacks=None): + """Initializes ClassificationValidator instance with args, dataloader, save_dir, and progress bar.""" + super().__init__(dataloader, save_dir, pbar, args, _callbacks) + self.targets = None + self.pred = None + self.args.task = "classify" + self.metrics = ClassifyMetrics() + + def get_desc(self): + """Returns a formatted string summarizing classification metrics.""" + return ("%22s" + "%11s" * 2) % ("classes", "top1_acc", "top5_acc") + + def init_metrics(self, model): + """Initialize confusion matrix, class names, and top-1 and top-5 accuracy.""" + self.names = model.names + self.nc = len(model.names) + self.confusion_matrix = ConfusionMatrix(nc=self.nc, conf=self.args.conf, task="classify") + self.pred = [] + self.targets = [] + + def preprocess(self, batch): + """Preprocesses input batch and returns it.""" + batch["img"] = batch["img"].to(self.device, non_blocking=True) + batch["img"] = batch["img"].half() if self.args.half else batch["img"].float() + batch["cls"] = batch["cls"].to(self.device) + return batch + + def update_metrics(self, preds, batch): + """Updates running metrics with model predictions and batch targets.""" + n5 = min(len(self.names), 5) + self.pred.append(preds.argsort(1, descending=True)[:, :n5]) + self.targets.append(batch["cls"]) + + def finalize_metrics(self, *args, **kwargs): + """Finalizes metrics of the model such as confusion_matrix and speed.""" + self.confusion_matrix.process_cls_preds(self.pred, self.targets) + if self.args.plots: + for normalize in True, False: + self.confusion_matrix.plot( + save_dir=self.save_dir, names=self.names.values(), normalize=normalize, on_plot=self.on_plot + ) + self.metrics.speed = self.speed + self.metrics.confusion_matrix = self.confusion_matrix + self.metrics.save_dir = self.save_dir + + def get_stats(self): + """Returns a dictionary of metrics obtained by processing targets and predictions.""" + self.metrics.process(self.targets, self.pred) + return self.metrics.results_dict + + def build_dataset(self, img_path): + """Creates and returns a ClassificationDataset instance using given image path and preprocessing parameters.""" + return ClassificationDataset(root=img_path, args=self.args, augment=False, prefix=self.args.split) + + def get_dataloader(self, dataset_path, batch_size): + """Builds and returns a data loader for classification tasks with given parameters.""" + dataset = self.build_dataset(dataset_path) + return build_dataloader(dataset, batch_size, self.args.workers, rank=-1) + + def print_results(self): + """Prints evaluation metrics for YOLO object detection model.""" + pf = "%22s" + "%11.3g" * len(self.metrics.keys) # print format + LOGGER.info(pf % ("all", self.metrics.top1, self.metrics.top5)) + + def plot_val_samples(self, batch, ni): + """Plot validation image samples.""" + plot_images( + images=batch["img"], + batch_idx=torch.arange(len(batch["img"])), + cls=batch["cls"].view(-1), # warning: use .view(), not .squeeze() for Classify models + fname=self.save_dir / f"val_batch{ni}_labels.jpg", + names=self.names, + on_plot=self.on_plot, + ) + + def plot_predictions(self, batch, preds, ni): + """Plots predicted bounding boxes on input images and saves the result.""" + plot_images( + batch["img"], + batch_idx=torch.arange(len(batch["img"])), + cls=torch.argmax(preds, dim=1), + fname=self.save_dir / f"val_batch{ni}_pred.jpg", + names=self.names, + on_plot=self.on_plot, + ) # pred diff --git a/ultralytics/models/yolo/detect/__init__.py b/ultralytics/models/yolo/detect/__init__.py new file mode 100644 index 0000000..5f3e62c --- /dev/null +++ b/ultralytics/models/yolo/detect/__init__.py @@ -0,0 +1,7 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from .predict import DetectionPredictor +from .train import DetectionTrainer +from .val import DetectionValidator + +__all__ = "DetectionPredictor", "DetectionTrainer", "DetectionValidator" diff --git a/ultralytics/models/yolo/detect/__pycache__/__init__.cpython-312.pyc b/ultralytics/models/yolo/detect/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..a9a96f6 Binary files /dev/null and b/ultralytics/models/yolo/detect/__pycache__/__init__.cpython-312.pyc differ diff --git a/ultralytics/models/yolo/detect/__pycache__/__init__.cpython-39.pyc b/ultralytics/models/yolo/detect/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..aa695ec Binary files /dev/null and b/ultralytics/models/yolo/detect/__pycache__/__init__.cpython-39.pyc differ diff --git a/ultralytics/models/yolo/detect/__pycache__/predict.cpython-312.pyc b/ultralytics/models/yolo/detect/__pycache__/predict.cpython-312.pyc new file mode 100644 index 0000000..6f78b2e Binary files /dev/null and b/ultralytics/models/yolo/detect/__pycache__/predict.cpython-312.pyc differ diff --git a/ultralytics/models/yolo/detect/__pycache__/predict.cpython-39.pyc b/ultralytics/models/yolo/detect/__pycache__/predict.cpython-39.pyc new file mode 100644 index 0000000..25c3247 Binary files /dev/null and b/ultralytics/models/yolo/detect/__pycache__/predict.cpython-39.pyc differ diff --git a/ultralytics/models/yolo/detect/__pycache__/train.cpython-312.pyc b/ultralytics/models/yolo/detect/__pycache__/train.cpython-312.pyc new file mode 100644 index 0000000..6db7aad Binary files /dev/null and b/ultralytics/models/yolo/detect/__pycache__/train.cpython-312.pyc differ diff --git a/ultralytics/models/yolo/detect/__pycache__/train.cpython-39.pyc b/ultralytics/models/yolo/detect/__pycache__/train.cpython-39.pyc new file mode 100644 index 0000000..c3a0b6f Binary files /dev/null and b/ultralytics/models/yolo/detect/__pycache__/train.cpython-39.pyc differ diff --git a/ultralytics/models/yolo/detect/__pycache__/val.cpython-312.pyc b/ultralytics/models/yolo/detect/__pycache__/val.cpython-312.pyc new file mode 100644 index 0000000..c2b58d5 Binary files /dev/null and b/ultralytics/models/yolo/detect/__pycache__/val.cpython-312.pyc differ diff --git a/ultralytics/models/yolo/detect/__pycache__/val.cpython-39.pyc b/ultralytics/models/yolo/detect/__pycache__/val.cpython-39.pyc new file mode 100644 index 0000000..fa187f3 Binary files /dev/null and b/ultralytics/models/yolo/detect/__pycache__/val.cpython-39.pyc differ diff --git a/ultralytics/models/yolo/detect/predict.py b/ultralytics/models/yolo/detect/predict.py new file mode 100644 index 0000000..3a0c628 --- /dev/null +++ b/ultralytics/models/yolo/detect/predict.py @@ -0,0 +1,43 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from ultralytics.engine.predictor import BasePredictor +from ultralytics.engine.results import Results +from ultralytics.utils import ops + + +class DetectionPredictor(BasePredictor): + """ + A class extending the BasePredictor class for prediction based on a detection model. + + Example: + ```python + from ultralytics.utils import ASSETS + from ultralytics.models.yolo.detect import DetectionPredictor + + args = dict(model='yolov8n.pt', source=ASSETS) + predictor = DetectionPredictor(overrides=args) + predictor.predict_cli() + ``` + """ + + def postprocess(self, preds, img, orig_imgs): + """Post-processes predictions and returns a list of Results objects.""" + preds = ops.non_max_suppression( + preds, + self.args.conf, + self.args.iou, + agnostic=self.args.agnostic_nms, + max_det=self.args.max_det, + classes=self.args.classes, + ) + + if not isinstance(orig_imgs, list): # input images are a torch.Tensor, not a list + orig_imgs = ops.convert_torch2numpy_batch(orig_imgs) + + results = [] + for i, pred in enumerate(preds): + orig_img = orig_imgs[i] + pred[:, :4] = ops.scale_boxes(img.shape[2:], pred[:, :4], orig_img.shape) + img_path = self.batch[0][i] + results.append(Results(orig_img, path=img_path, names=self.model.names, boxes=pred)) + return results diff --git a/ultralytics/models/yolo/detect/train.py b/ultralytics/models/yolo/detect/train.py new file mode 100644 index 0000000..3326512 --- /dev/null +++ b/ultralytics/models/yolo/detect/train.py @@ -0,0 +1,143 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import math +import random +from copy import copy + +import numpy as np +import torch.nn as nn + +from ultralytics.data import build_dataloader, build_yolo_dataset +from ultralytics.engine.trainer import BaseTrainer +from ultralytics.models import yolo +from ultralytics.nn.tasks import DetectionModel +from ultralytics.utils import LOGGER, RANK +from ultralytics.utils.plotting import plot_images, plot_labels, plot_results +from ultralytics.utils.torch_utils import de_parallel, torch_distributed_zero_first + + +class DetectionTrainer(BaseTrainer): + """ + A class extending the BaseTrainer class for training based on a detection model. + + Example: + ```python + from ultralytics.models.yolo.detect import DetectionTrainer + + args = dict(model='yolov8n.pt', data='coco8.yaml', epochs=3) + trainer = DetectionTrainer(overrides=args) + trainer.train() + ``` + """ + + def build_dataset(self, img_path, mode="train", batch=None): + """ + Build YOLO Dataset. + + Args: + img_path (str): Path to the folder containing images. + mode (str): `train` mode or `val` mode, users are able to customize different augmentations for each mode. + batch (int, optional): Size of batches, this is for `rect`. Defaults to None. + """ + gs = max(int(de_parallel(self.model).stride.max() if self.model else 0), 32) + return build_yolo_dataset(self.args, img_path, batch, self.data, mode=mode, rect=mode == "val", stride=gs) + + def get_dataloader(self, dataset_path, batch_size=16, rank=0, mode="train"): + """Construct and return dataloader.""" + assert mode in ["train", "val"] + with torch_distributed_zero_first(rank): # init dataset *.cache only once if DDP + dataset = self.build_dataset(dataset_path, mode, batch_size) + shuffle = mode == "train" + if getattr(dataset, "rect", False) and shuffle: + LOGGER.warning("WARNING ⚠️ 'rect=True' is incompatible with DataLoader shuffle, setting shuffle=False") + shuffle = False + workers = self.args.workers if mode == "train" else self.args.workers * 2 + return build_dataloader(dataset, batch_size, workers, shuffle, rank) # return dataloader + + def preprocess_batch(self, batch): + """Preprocesses a batch of images by scaling and converting to float.""" + batch["img"] = batch["img"].to(self.device, non_blocking=True).float() / 255 + if self.args.multi_scale: + imgs = batch["img"] + sz = ( + random.randrange(self.args.imgsz * 0.5, self.args.imgsz * 1.5 + self.stride) + // self.stride + * self.stride + ) # size + sf = sz / max(imgs.shape[2:]) # scale factor + if sf != 1: + ns = [ + math.ceil(x * sf / self.stride) * self.stride for x in imgs.shape[2:] + ] # new shape (stretched to gs-multiple) + imgs = nn.functional.interpolate(imgs, size=ns, mode="bilinear", align_corners=False) + batch["img"] = imgs + return batch + + def set_model_attributes(self): + """Nl = de_parallel(self.model).model[-1].nl # number of detection layers (to scale hyps).""" + # self.args.box *= 3 / nl # scale to layers + # self.args.cls *= self.data["nc"] / 80 * 3 / nl # scale to classes and layers + # self.args.cls *= (self.args.imgsz / 640) ** 2 * 3 / nl # scale to image size and layers + self.model.nc = self.data["nc"] # attach number of classes to model + self.model.names = self.data["names"] # attach class names to model + self.model.args = self.args # attach hyperparameters to model + # TODO: self.model.class_weights = labels_to_class_weights(dataset.labels, nc).to(device) * nc + + def get_model(self, cfg=None, weights=None, verbose=True): + """Return a YOLO detection model.""" + model = DetectionModel(cfg, nc=self.data["nc"], verbose=verbose and RANK == -1) + if weights: + model.load(weights) + return model + + def get_validator(self): + """Returns a DetectionValidator for YOLO model validation.""" + self.loss_names = "box_loss", "cls_loss", "dfl_loss" + return yolo.detect.DetectionValidator( + self.test_loader, save_dir=self.save_dir, args=copy(self.args), _callbacks=self.callbacks + ) + + def label_loss_items(self, loss_items=None, prefix="train"): + """ + Returns a loss dict with labelled training loss items tensor. + + Not needed for classification but necessary for segmentation & detection + """ + keys = [f"{prefix}/{x}" for x in self.loss_names] + if loss_items is not None: + loss_items = [round(float(x), 5) for x in loss_items] # convert tensors to 5 decimal place floats + return dict(zip(keys, loss_items)) + else: + return keys + + def progress_string(self): + """Returns a formatted string of training progress with epoch, GPU memory, loss, instances and size.""" + return ("\n" + "%11s" * (4 + len(self.loss_names))) % ( + "Epoch", + "GPU_mem", + *self.loss_names, + "Instances", + "Size", + ) + + def plot_training_samples(self, batch, ni): + """Plots training samples with their annotations.""" + plot_images( + images=batch["img"], + batch_idx=batch["batch_idx"], + cls=batch["cls"].squeeze(-1), + bboxes=batch["bboxes"], + paths=batch["im_file"], + fname=self.save_dir / f"train_batch{ni}.jpg", + on_plot=self.on_plot, + ) + + def plot_metrics(self): + """Plots metrics from a CSV file.""" + plot_results(file=self.csv, on_plot=self.on_plot) # save results.png + + def plot_training_labels(self): + """Create a labeled training plot of the YOLO model.""" + boxes = np.concatenate([lb["bboxes"] for lb in self.train_loader.dataset.labels], 0) + cls = np.concatenate([lb["cls"] for lb in self.train_loader.dataset.labels], 0) + plot_labels(boxes, cls.squeeze(), names=self.data["names"], save_dir=self.save_dir, on_plot=self.on_plot) diff --git a/ultralytics/models/yolo/detect/val.py b/ultralytics/models/yolo/detect/val.py new file mode 100644 index 0000000..5550ec3 --- /dev/null +++ b/ultralytics/models/yolo/detect/val.py @@ -0,0 +1,299 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import os +from pathlib import Path + +import numpy as np +import torch + +from ultralytics.data import build_dataloader, build_yolo_dataset, converter +from ultralytics.engine.validator import BaseValidator +from ultralytics.utils import LOGGER, ops +from ultralytics.utils.checks import check_requirements +from ultralytics.utils.metrics import ConfusionMatrix, DetMetrics, box_iou +from ultralytics.utils.plotting import output_to_target, plot_images + + +class DetectionValidator(BaseValidator): + """ + A class extending the BaseValidator class for validation based on a detection model. + + Example: + ```python + from ultralytics.models.yolo.detect import DetectionValidator + + args = dict(model='yolov8n.pt', data='coco8.yaml') + validator = DetectionValidator(args=args) + validator() + ``` + """ + + def __init__(self, dataloader=None, save_dir=None, pbar=None, args=None, _callbacks=None): + """Initialize detection model with necessary variables and settings.""" + super().__init__(dataloader, save_dir, pbar, args, _callbacks) + self.nt_per_class = None + self.is_coco = False + self.class_map = None + self.args.task = "detect" + self.metrics = DetMetrics(save_dir=self.save_dir, on_plot=self.on_plot) + self.iouv = torch.linspace(0.5, 0.95, 10) # IoU vector for mAP@0.5:0.95 + self.niou = self.iouv.numel() + self.lb = [] # for autolabelling + + def preprocess(self, batch): + """Preprocesses batch of images for YOLO training.""" + batch["img"] = batch["img"].to(self.device, non_blocking=True) + batch["img"] = (batch["img"].half() if self.args.half else batch["img"].float()) / 255 + for k in ["batch_idx", "cls", "bboxes"]: + batch[k] = batch[k].to(self.device) + + if self.args.save_hybrid: + height, width = batch["img"].shape[2:] + nb = len(batch["img"]) + bboxes = batch["bboxes"] * torch.tensor((width, height, width, height), device=self.device) + self.lb = ( + [ + torch.cat([batch["cls"][batch["batch_idx"] == i], bboxes[batch["batch_idx"] == i]], dim=-1) + for i in range(nb) + ] + if self.args.save_hybrid + else [] + ) # for autolabelling + + return batch + + def init_metrics(self, model): + """Initialize evaluation metrics for YOLO.""" + val = self.data.get(self.args.split, "") # validation path + self.is_coco = isinstance(val, str) and "coco" in val and val.endswith(f"{os.sep}val2017.txt") # is COCO + self.class_map = converter.coco80_to_coco91_class() if self.is_coco else list(range(1000)) + self.args.save_json |= self.is_coco # run on final val if training COCO + self.names = model.names + self.nc = len(model.names) + self.metrics.names = self.names + self.metrics.plot = self.args.plots + self.confusion_matrix = ConfusionMatrix(nc=self.nc, conf=self.args.conf) + self.seen = 0 + self.jdict = [] + self.stats = dict(tp=[], conf=[], pred_cls=[], target_cls=[]) + + def get_desc(self): + """Return a formatted string summarizing class metrics of YOLO model.""" + return ("%22s" + "%11s" * 6) % ("Class", "Images", "Instances", "Box(P", "R", "mAP50", "mAP50-95)") + + def postprocess(self, preds): + """Apply Non-maximum suppression to prediction outputs.""" + return ops.non_max_suppression( + preds, + self.args.conf, + self.args.iou, + labels=self.lb, + multi_label=True, + agnostic=self.args.single_cls, + max_det=self.args.max_det, + ) + + def _prepare_batch(self, si, batch): + """Prepares a batch of images and annotations for validation.""" + idx = batch["batch_idx"] == si + cls = batch["cls"][idx].squeeze(-1) + bbox = batch["bboxes"][idx] + ori_shape = batch["ori_shape"][si] + imgsz = batch["img"].shape[2:] + ratio_pad = batch["ratio_pad"][si] + if len(cls): + bbox = ops.xywh2xyxy(bbox) * torch.tensor(imgsz, device=self.device)[[1, 0, 1, 0]] # target boxes + ops.scale_boxes(imgsz, bbox, ori_shape, ratio_pad=ratio_pad) # native-space labels + return dict(cls=cls, bbox=bbox, ori_shape=ori_shape, imgsz=imgsz, ratio_pad=ratio_pad) + + def _prepare_pred(self, pred, pbatch): + """Prepares a batch of images and annotations for validation.""" + predn = pred.clone() + ops.scale_boxes( + pbatch["imgsz"], predn[:, :4], pbatch["ori_shape"], ratio_pad=pbatch["ratio_pad"] + ) # native-space pred + return predn + + def update_metrics(self, preds, batch): + """Metrics.""" + for si, pred in enumerate(preds): + self.seen += 1 + npr = len(pred) + stat = dict( + conf=torch.zeros(0, device=self.device), + pred_cls=torch.zeros(0, device=self.device), + tp=torch.zeros(npr, self.niou, dtype=torch.bool, device=self.device), + ) + pbatch = self._prepare_batch(si, batch) + cls, bbox = pbatch.pop("cls"), pbatch.pop("bbox") + nl = len(cls) + stat["target_cls"] = cls + if npr == 0: + if nl: + for k in self.stats.keys(): + self.stats[k].append(stat[k]) + if self.args.plots: + self.confusion_matrix.process_batch(detections=None, gt_bboxes=bbox, gt_cls=cls) + continue + + # Predictions + if self.args.single_cls: + pred[:, 5] = 0 + predn = self._prepare_pred(pred, pbatch) + stat["conf"] = predn[:, 4] + stat["pred_cls"] = predn[:, 5] + + # Evaluate + if nl: + stat["tp"] = self._process_batch(predn, bbox, cls) + if self.args.plots: + self.confusion_matrix.process_batch(predn, bbox, cls) + for k in self.stats.keys(): + self.stats[k].append(stat[k]) + + # Save + if self.args.save_json: + self.pred_to_json(predn, batch["im_file"][si]) + if self.args.save_txt: + file = self.save_dir / "labels" / f'{Path(batch["im_file"][si]).stem}.txt' + self.save_one_txt(predn, self.args.save_conf, pbatch["ori_shape"], file) + + def finalize_metrics(self, *args, **kwargs): + """Set final values for metrics speed and confusion matrix.""" + self.metrics.speed = self.speed + self.metrics.confusion_matrix = self.confusion_matrix + + def get_stats(self): + """Returns metrics statistics and results dictionary.""" + stats = {k: torch.cat(v, 0).cpu().numpy() for k, v in self.stats.items()} # to numpy + if len(stats) and stats["tp"].any(): + self.metrics.process(**stats) + self.nt_per_class = np.bincount( + stats["target_cls"].astype(int), minlength=self.nc + ) # number of targets per class + return self.metrics.results_dict + + def print_results(self): + """Prints training/validation set metrics per class.""" + pf = "%22s" + "%11i" * 2 + "%11.3g" * len(self.metrics.keys) # print format + LOGGER.info(pf % ("all", self.seen, self.nt_per_class.sum(), *self.metrics.mean_results())) + if self.nt_per_class.sum() == 0: + LOGGER.warning(f"WARNING ⚠️ no labels found in {self.args.task} set, can not compute metrics without labels") + + # Print results per class + if self.args.verbose and not self.training and self.nc > 1 and len(self.stats): + for i, c in enumerate(self.metrics.ap_class_index): + LOGGER.info(pf % (self.names[c], self.seen, self.nt_per_class[c], *self.metrics.class_result(i))) + + if self.args.plots: + for normalize in True, False: + self.confusion_matrix.plot( + save_dir=self.save_dir, names=self.names.values(), normalize=normalize, on_plot=self.on_plot + ) + + def _process_batch(self, detections, gt_bboxes, gt_cls): + """ + Return correct prediction matrix. + + Args: + detections (torch.Tensor): Tensor of shape [N, 6] representing detections. + Each detection is of the format: x1, y1, x2, y2, conf, class. + labels (torch.Tensor): Tensor of shape [M, 5] representing labels. + Each label is of the format: class, x1, y1, x2, y2. + + Returns: + (torch.Tensor): Correct prediction matrix of shape [N, 10] for 10 IoU levels. + """ + iou = box_iou(gt_bboxes, detections[:, :4]) + return self.match_predictions(detections[:, 5], gt_cls, iou) + + def build_dataset(self, img_path, mode="val", batch=None): + """ + Build YOLO Dataset. + + Args: + img_path (str): Path to the folder containing images. + mode (str): `train` mode or `val` mode, users are able to customize different augmentations for each mode. + batch (int, optional): Size of batches, this is for `rect`. Defaults to None. + """ + return build_yolo_dataset(self.args, img_path, batch, self.data, mode=mode, stride=self.stride) + + def get_dataloader(self, dataset_path, batch_size): + """Construct and return dataloader.""" + dataset = self.build_dataset(dataset_path, batch=batch_size, mode="val") + return build_dataloader(dataset, batch_size, self.args.workers, shuffle=False, rank=-1) # return dataloader + + def plot_val_samples(self, batch, ni): + """Plot validation image samples.""" + plot_images( + batch["img"], + batch["batch_idx"], + batch["cls"].squeeze(-1), + batch["bboxes"], + paths=batch["im_file"], + fname=self.save_dir / f"val_batch{ni}_labels.jpg", + names=self.names, + on_plot=self.on_plot, + ) + + def plot_predictions(self, batch, preds, ni): + """Plots predicted bounding boxes on input images and saves the result.""" + plot_images( + batch["img"], + *output_to_target(preds, max_det=self.args.max_det), + paths=batch["im_file"], + fname=self.save_dir / f"val_batch{ni}_pred.jpg", + names=self.names, + on_plot=self.on_plot, + ) # pred + + def save_one_txt(self, predn, save_conf, shape, file): + """Save YOLO detections to a txt file in normalized coordinates in a specific format.""" + gn = torch.tensor(shape)[[1, 0, 1, 0]] # normalization gain whwh + for *xyxy, conf, cls in predn.tolist(): + xywh = (ops.xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh + line = (cls, *xywh, conf) if save_conf else (cls, *xywh) # label format + with open(file, "a") as f: + f.write(("%g " * len(line)).rstrip() % line + "\n") + + def pred_to_json(self, predn, filename): + """Serialize YOLO predictions to COCO json format.""" + stem = Path(filename).stem + image_id = int(stem) if stem.isnumeric() else stem + box = ops.xyxy2xywh(predn[:, :4]) # xywh + box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner + for p, b in zip(predn.tolist(), box.tolist()): + self.jdict.append( + { + "image_id": image_id, + "category_id": self.class_map[int(p[5])], + "bbox": [round(x, 3) for x in b], + "score": round(p[4], 5), + } + ) + + def eval_json(self, stats): + """Evaluates YOLO output in JSON format and returns performance statistics.""" + if self.args.save_json and self.is_coco and len(self.jdict): + anno_json = self.data["path"] / "annotations/instances_val2017.json" # annotations + pred_json = self.save_dir / "predictions.json" # predictions + LOGGER.info(f"\nEvaluating pycocotools mAP using {pred_json} and {anno_json}...") + try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb + check_requirements("pycocotools>=2.0.6") + from pycocotools.coco import COCO # noqa + from pycocotools.cocoeval import COCOeval # noqa + + for x in anno_json, pred_json: + assert x.is_file(), f"{x} file not found" + anno = COCO(str(anno_json)) # init annotations api + pred = anno.loadRes(str(pred_json)) # init predictions api (must pass string, not Path) + eval = COCOeval(anno, pred, "bbox") + if self.is_coco: + eval.params.imgIds = [int(Path(x).stem) for x in self.dataloader.dataset.im_files] # images to eval + eval.evaluate() + eval.accumulate() + eval.summarize() + stats[self.metrics.keys[-1]], stats[self.metrics.keys[-2]] = eval.stats[:2] # update mAP50-95 and mAP50 + except Exception as e: + LOGGER.warning(f"pycocotools unable to run: {e}") + return stats diff --git a/ultralytics/models/yolo/model.py b/ultralytics/models/yolo/model.py new file mode 100644 index 0000000..f10dc97 --- /dev/null +++ b/ultralytics/models/yolo/model.py @@ -0,0 +1,111 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from pathlib import Path + +from ultralytics.engine.model import Model +from ultralytics.models import yolo +from ultralytics.nn.tasks import ClassificationModel, DetectionModel, OBBModel, PoseModel, SegmentationModel, WorldModel +from ultralytics.utils import yaml_load, ROOT + + +class YOLO(Model): + """YOLO (You Only Look Once) object detection model.""" + + def __init__(self, model="yolov8n.pt", task=None, verbose=False): + """Initialize YOLO model, switching to YOLOWorld if model filename contains '-world'.""" + path = Path(model) + if "-world" in path.stem and path.suffix in {".pt", ".yaml", ".yml"}: # if YOLOWorld PyTorch model + new_instance = YOLOWorld(path) + self.__class__ = type(new_instance) + self.__dict__ = new_instance.__dict__ + elif "yolov10" in path.stem: + from ultralytics import YOLOv10 + new_instance = YOLOv10(path) + self.__class__ = type(new_instance) + self.__dict__ = new_instance.__dict__ + else: + # Continue with default YOLO initialization + super().__init__(model=model, task=task, verbose=verbose) + + @property + def task_map(self): + """Map head to model, trainer, validator, and predictor classes.""" + return { + "classify": { + "model": ClassificationModel, + "trainer": yolo.classify.ClassificationTrainer, + "validator": yolo.classify.ClassificationValidator, + "predictor": yolo.classify.ClassificationPredictor, + }, + "detect": { + "model": DetectionModel, + "trainer": yolo.detect.DetectionTrainer, + "validator": yolo.detect.DetectionValidator, + "predictor": yolo.detect.DetectionPredictor, + }, + "segment": { + "model": SegmentationModel, + "trainer": yolo.segment.SegmentationTrainer, + "validator": yolo.segment.SegmentationValidator, + "predictor": yolo.segment.SegmentationPredictor, + }, + "pose": { + "model": PoseModel, + "trainer": yolo.pose.PoseTrainer, + "validator": yolo.pose.PoseValidator, + "predictor": yolo.pose.PosePredictor, + }, + "obb": { + "model": OBBModel, + "trainer": yolo.obb.OBBTrainer, + "validator": yolo.obb.OBBValidator, + "predictor": yolo.obb.OBBPredictor, + }, + } + + +class YOLOWorld(Model): + """YOLO-World object detection model.""" + + def __init__(self, model="yolov8s-world.pt") -> None: + """ + Initializes the YOLOv8-World model with the given pre-trained model file. Supports *.pt and *.yaml formats. + + Args: + model (str | Path): Path to the pre-trained model. Defaults to 'yolov8s-world.pt'. + """ + super().__init__(model=model, task="detect") + + # Assign default COCO class names when there are no custom names + if not hasattr(self.model, "names"): + self.model.names = yaml_load(ROOT / "cfg/datasets/coco8.yaml").get("names") + + @property + def task_map(self): + """Map head to model, validator, and predictor classes.""" + return { + "detect": { + "model": WorldModel, + "validator": yolo.detect.DetectionValidator, + "predictor": yolo.detect.DetectionPredictor, + } + } + + def set_classes(self, classes): + """ + Set classes. + + Args: + classes (List(str)): A list of categories i.e ["person"]. + """ + self.model.set_classes(classes) + # Remove background if it's given + background = " " + if background in classes: + classes.remove(background) + self.model.names = classes + + # Reset method class names + # self.predictor = None # reset predictor otherwise old names remain + if self.predictor: + self.predictor.model.names = classes diff --git a/ultralytics/models/yolo/obb/__init__.py b/ultralytics/models/yolo/obb/__init__.py new file mode 100644 index 0000000..f60349a --- /dev/null +++ b/ultralytics/models/yolo/obb/__init__.py @@ -0,0 +1,7 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from .predict import OBBPredictor +from .train import OBBTrainer +from .val import OBBValidator + +__all__ = "OBBPredictor", "OBBTrainer", "OBBValidator" diff --git a/ultralytics/models/yolo/obb/__pycache__/__init__.cpython-312.pyc b/ultralytics/models/yolo/obb/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..921dfcc Binary files /dev/null and b/ultralytics/models/yolo/obb/__pycache__/__init__.cpython-312.pyc differ diff --git a/ultralytics/models/yolo/obb/__pycache__/__init__.cpython-39.pyc b/ultralytics/models/yolo/obb/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..56eb79a Binary files /dev/null and b/ultralytics/models/yolo/obb/__pycache__/__init__.cpython-39.pyc differ diff --git a/ultralytics/models/yolo/obb/__pycache__/predict.cpython-312.pyc b/ultralytics/models/yolo/obb/__pycache__/predict.cpython-312.pyc new file mode 100644 index 0000000..baecc6f Binary files /dev/null and b/ultralytics/models/yolo/obb/__pycache__/predict.cpython-312.pyc differ diff --git a/ultralytics/models/yolo/obb/__pycache__/predict.cpython-39.pyc b/ultralytics/models/yolo/obb/__pycache__/predict.cpython-39.pyc new file mode 100644 index 0000000..51d5ac8 Binary files /dev/null and b/ultralytics/models/yolo/obb/__pycache__/predict.cpython-39.pyc differ diff --git a/ultralytics/models/yolo/obb/__pycache__/train.cpython-312.pyc b/ultralytics/models/yolo/obb/__pycache__/train.cpython-312.pyc new file mode 100644 index 0000000..3e59675 Binary files /dev/null and b/ultralytics/models/yolo/obb/__pycache__/train.cpython-312.pyc differ diff --git a/ultralytics/models/yolo/obb/__pycache__/train.cpython-39.pyc b/ultralytics/models/yolo/obb/__pycache__/train.cpython-39.pyc new file mode 100644 index 0000000..9c2cb08 Binary files /dev/null and b/ultralytics/models/yolo/obb/__pycache__/train.cpython-39.pyc differ diff --git a/ultralytics/models/yolo/obb/__pycache__/val.cpython-312.pyc b/ultralytics/models/yolo/obb/__pycache__/val.cpython-312.pyc new file mode 100644 index 0000000..74971df Binary files /dev/null and b/ultralytics/models/yolo/obb/__pycache__/val.cpython-312.pyc differ diff --git a/ultralytics/models/yolo/obb/__pycache__/val.cpython-39.pyc b/ultralytics/models/yolo/obb/__pycache__/val.cpython-39.pyc new file mode 100644 index 0000000..c9ca611 Binary files /dev/null and b/ultralytics/models/yolo/obb/__pycache__/val.cpython-39.pyc differ diff --git a/ultralytics/models/yolo/obb/predict.py b/ultralytics/models/yolo/obb/predict.py new file mode 100644 index 0000000..bb8d4d3 --- /dev/null +++ b/ultralytics/models/yolo/obb/predict.py @@ -0,0 +1,53 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import torch + +from ultralytics.engine.results import Results +from ultralytics.models.yolo.detect.predict import DetectionPredictor +from ultralytics.utils import DEFAULT_CFG, ops + + +class OBBPredictor(DetectionPredictor): + """ + A class extending the DetectionPredictor class for prediction based on an Oriented Bounding Box (OBB) model. + + Example: + ```python + from ultralytics.utils import ASSETS + from ultralytics.models.yolo.obb import OBBPredictor + + args = dict(model='yolov8n-obb.pt', source=ASSETS) + predictor = OBBPredictor(overrides=args) + predictor.predict_cli() + ``` + """ + + def __init__(self, cfg=DEFAULT_CFG, overrides=None, _callbacks=None): + """Initializes OBBPredictor with optional model and data configuration overrides.""" + super().__init__(cfg, overrides, _callbacks) + self.args.task = "obb" + + def postprocess(self, preds, img, orig_imgs): + """Post-processes predictions and returns a list of Results objects.""" + preds = ops.non_max_suppression( + preds, + self.args.conf, + self.args.iou, + agnostic=self.args.agnostic_nms, + max_det=self.args.max_det, + nc=len(self.model.names), + classes=self.args.classes, + rotated=True, + ) + + if not isinstance(orig_imgs, list): # input images are a torch.Tensor, not a list + orig_imgs = ops.convert_torch2numpy_batch(orig_imgs) + + results = [] + for pred, orig_img, img_path in zip(preds, orig_imgs, self.batch[0]): + rboxes = ops.regularize_rboxes(torch.cat([pred[:, :4], pred[:, -1:]], dim=-1)) + rboxes[:, :4] = ops.scale_boxes(img.shape[2:], rboxes[:, :4], orig_img.shape, xywh=True) + # xywh, r, conf, cls + obb = torch.cat([rboxes, pred[:, 4:6]], dim=-1) + results.append(Results(orig_img, path=img_path, names=self.model.names, obb=obb)) + return results diff --git a/ultralytics/models/yolo/obb/train.py b/ultralytics/models/yolo/obb/train.py new file mode 100644 index 0000000..40a35a9 --- /dev/null +++ b/ultralytics/models/yolo/obb/train.py @@ -0,0 +1,42 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from copy import copy + +from ultralytics.models import yolo +from ultralytics.nn.tasks import OBBModel +from ultralytics.utils import DEFAULT_CFG, RANK + + +class OBBTrainer(yolo.detect.DetectionTrainer): + """ + A class extending the DetectionTrainer class for training based on an Oriented Bounding Box (OBB) model. + + Example: + ```python + from ultralytics.models.yolo.obb import OBBTrainer + + args = dict(model='yolov8n-obb.pt', data='dota8.yaml', epochs=3) + trainer = OBBTrainer(overrides=args) + trainer.train() + ``` + """ + + def __init__(self, cfg=DEFAULT_CFG, overrides=None, _callbacks=None): + """Initialize a OBBTrainer object with given arguments.""" + if overrides is None: + overrides = {} + overrides["task"] = "obb" + super().__init__(cfg, overrides, _callbacks) + + def get_model(self, cfg=None, weights=None, verbose=True): + """Return OBBModel initialized with specified config and weights.""" + model = OBBModel(cfg, ch=3, nc=self.data["nc"], verbose=verbose and RANK == -1) + if weights: + model.load(weights) + + return model + + def get_validator(self): + """Return an instance of OBBValidator for validation of YOLO model.""" + self.loss_names = "box_loss", "cls_loss", "dfl_loss" + return yolo.obb.OBBValidator(self.test_loader, save_dir=self.save_dir, args=copy(self.args)) diff --git a/ultralytics/models/yolo/obb/val.py b/ultralytics/models/yolo/obb/val.py new file mode 100644 index 0000000..c440fe2 --- /dev/null +++ b/ultralytics/models/yolo/obb/val.py @@ -0,0 +1,185 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from pathlib import Path + +import torch + +from ultralytics.models.yolo.detect import DetectionValidator +from ultralytics.utils import LOGGER, ops +from ultralytics.utils.metrics import OBBMetrics, batch_probiou +from ultralytics.utils.plotting import output_to_rotated_target, plot_images + + +class OBBValidator(DetectionValidator): + """ + A class extending the DetectionValidator class for validation based on an Oriented Bounding Box (OBB) model. + + Example: + ```python + from ultralytics.models.yolo.obb import OBBValidator + + args = dict(model='yolov8n-obb.pt', data='dota8.yaml') + validator = OBBValidator(args=args) + validator(model=args['model']) + ``` + """ + + def __init__(self, dataloader=None, save_dir=None, pbar=None, args=None, _callbacks=None): + """Initialize OBBValidator and set task to 'obb', metrics to OBBMetrics.""" + super().__init__(dataloader, save_dir, pbar, args, _callbacks) + self.args.task = "obb" + self.metrics = OBBMetrics(save_dir=self.save_dir, plot=True, on_plot=self.on_plot) + + def init_metrics(self, model): + """Initialize evaluation metrics for YOLO.""" + super().init_metrics(model) + val = self.data.get(self.args.split, "") # validation path + self.is_dota = isinstance(val, str) and "DOTA" in val # is COCO + + def postprocess(self, preds): + """Apply Non-maximum suppression to prediction outputs.""" + return ops.non_max_suppression( + preds, + self.args.conf, + self.args.iou, + labels=self.lb, + nc=self.nc, + multi_label=True, + agnostic=self.args.single_cls, + max_det=self.args.max_det, + rotated=True, + ) + + def _process_batch(self, detections, gt_bboxes, gt_cls): + """ + Return correct prediction matrix. + + Args: + detections (torch.Tensor): Tensor of shape [N, 7] representing detections. + Each detection is of the format: x1, y1, x2, y2, conf, class, angle. + gt_bboxes (torch.Tensor): Tensor of shape [M, 5] representing rotated boxes. + Each box is of the format: x1, y1, x2, y2, angle. + labels (torch.Tensor): Tensor of shape [M] representing labels. + + Returns: + (torch.Tensor): Correct prediction matrix of shape [N, 10] for 10 IoU levels. + """ + iou = batch_probiou(gt_bboxes, torch.cat([detections[:, :4], detections[:, -1:]], dim=-1)) + return self.match_predictions(detections[:, 5], gt_cls, iou) + + def _prepare_batch(self, si, batch): + """Prepares and returns a batch for OBB validation.""" + idx = batch["batch_idx"] == si + cls = batch["cls"][idx].squeeze(-1) + bbox = batch["bboxes"][idx] + ori_shape = batch["ori_shape"][si] + imgsz = batch["img"].shape[2:] + ratio_pad = batch["ratio_pad"][si] + if len(cls): + bbox[..., :4].mul_(torch.tensor(imgsz, device=self.device)[[1, 0, 1, 0]]) # target boxes + ops.scale_boxes(imgsz, bbox, ori_shape, ratio_pad=ratio_pad, xywh=True) # native-space labels + return dict(cls=cls, bbox=bbox, ori_shape=ori_shape, imgsz=imgsz, ratio_pad=ratio_pad) + + def _prepare_pred(self, pred, pbatch): + """Prepares and returns a batch for OBB validation with scaled and padded bounding boxes.""" + predn = pred.clone() + ops.scale_boxes( + pbatch["imgsz"], predn[:, :4], pbatch["ori_shape"], ratio_pad=pbatch["ratio_pad"], xywh=True + ) # native-space pred + return predn + + def plot_predictions(self, batch, preds, ni): + """Plots predicted bounding boxes on input images and saves the result.""" + plot_images( + batch["img"], + *output_to_rotated_target(preds, max_det=self.args.max_det), + paths=batch["im_file"], + fname=self.save_dir / f"val_batch{ni}_pred.jpg", + names=self.names, + on_plot=self.on_plot, + ) # pred + + def pred_to_json(self, predn, filename): + """Serialize YOLO predictions to COCO json format.""" + stem = Path(filename).stem + image_id = int(stem) if stem.isnumeric() else stem + rbox = torch.cat([predn[:, :4], predn[:, -1:]], dim=-1) + poly = ops.xywhr2xyxyxyxy(rbox).view(-1, 8) + for i, (r, b) in enumerate(zip(rbox.tolist(), poly.tolist())): + self.jdict.append( + { + "image_id": image_id, + "category_id": self.class_map[int(predn[i, 5].item())], + "score": round(predn[i, 4].item(), 5), + "rbox": [round(x, 3) for x in r], + "poly": [round(x, 3) for x in b], + } + ) + + def save_one_txt(self, predn, save_conf, shape, file): + """Save YOLO detections to a txt file in normalized coordinates in a specific format.""" + gn = torch.tensor(shape)[[1, 0]] # normalization gain whwh + for *xywh, conf, cls, angle in predn.tolist(): + xywha = torch.tensor([*xywh, angle]).view(1, 5) + xyxyxyxy = (ops.xywhr2xyxyxyxy(xywha) / gn).view(-1).tolist() # normalized xywh + line = (cls, *xyxyxyxy, conf) if save_conf else (cls, *xyxyxyxy) # label format + with open(file, "a") as f: + f.write(("%g " * len(line)).rstrip() % line + "\n") + + def eval_json(self, stats): + """Evaluates YOLO output in JSON format and returns performance statistics.""" + if self.args.save_json and self.is_dota and len(self.jdict): + import json + import re + from collections import defaultdict + + pred_json = self.save_dir / "predictions.json" # predictions + pred_txt = self.save_dir / "predictions_txt" # predictions + pred_txt.mkdir(parents=True, exist_ok=True) + data = json.load(open(pred_json)) + # Save split results + LOGGER.info(f"Saving predictions with DOTA format to {pred_txt}...") + for d in data: + image_id = d["image_id"] + score = d["score"] + classname = self.names[d["category_id"]].replace(" ", "-") + p = d["poly"] + + with open(f'{pred_txt / f"Task1_{classname}"}.txt', "a") as f: + f.writelines(f"{image_id} {score} {p[0]} {p[1]} {p[2]} {p[3]} {p[4]} {p[5]} {p[6]} {p[7]}\n") + # Save merged results, this could result slightly lower map than using official merging script, + # because of the probiou calculation. + pred_merged_txt = self.save_dir / "predictions_merged_txt" # predictions + pred_merged_txt.mkdir(parents=True, exist_ok=True) + merged_results = defaultdict(list) + LOGGER.info(f"Saving merged predictions with DOTA format to {pred_merged_txt}...") + for d in data: + image_id = d["image_id"].split("__")[0] + pattern = re.compile(r"\d+___\d+") + x, y = (int(c) for c in re.findall(pattern, d["image_id"])[0].split("___")) + bbox, score, cls = d["rbox"], d["score"], d["category_id"] + bbox[0] += x + bbox[1] += y + bbox.extend([score, cls]) + merged_results[image_id].append(bbox) + for image_id, bbox in merged_results.items(): + bbox = torch.tensor(bbox) + max_wh = torch.max(bbox[:, :2]).item() * 2 + c = bbox[:, 6:7] * max_wh # classes + scores = bbox[:, 5] # scores + b = bbox[:, :5].clone() + b[:, :2] += c + # 0.3 could get results close to the ones from official merging script, even slightly better. + i = ops.nms_rotated(b, scores, 0.3) + bbox = bbox[i] + + b = ops.xywhr2xyxyxyxy(bbox[:, :5]).view(-1, 8) + for x in torch.cat([b, bbox[:, 5:7]], dim=-1).tolist(): + classname = self.names[int(x[-1])].replace(" ", "-") + p = [round(i, 3) for i in x[:-2]] # poly + score = round(x[-2], 3) + + with open(f'{pred_merged_txt / f"Task1_{classname}"}.txt', "a") as f: + f.writelines(f"{image_id} {score} {p[0]} {p[1]} {p[2]} {p[3]} {p[4]} {p[5]} {p[6]} {p[7]}\n") + + return stats diff --git a/ultralytics/models/yolo/pose/__init__.py b/ultralytics/models/yolo/pose/__init__.py new file mode 100644 index 0000000..d566943 --- /dev/null +++ b/ultralytics/models/yolo/pose/__init__.py @@ -0,0 +1,7 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from .predict import PosePredictor +from .train import PoseTrainer +from .val import PoseValidator + +__all__ = "PoseTrainer", "PoseValidator", "PosePredictor" diff --git a/ultralytics/models/yolo/pose/__pycache__/__init__.cpython-312.pyc b/ultralytics/models/yolo/pose/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..bb6e5f0 Binary files /dev/null and b/ultralytics/models/yolo/pose/__pycache__/__init__.cpython-312.pyc differ diff --git a/ultralytics/models/yolo/pose/__pycache__/__init__.cpython-39.pyc b/ultralytics/models/yolo/pose/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..1db6261 Binary files /dev/null and b/ultralytics/models/yolo/pose/__pycache__/__init__.cpython-39.pyc differ diff --git a/ultralytics/models/yolo/pose/__pycache__/predict.cpython-312.pyc b/ultralytics/models/yolo/pose/__pycache__/predict.cpython-312.pyc new file mode 100644 index 0000000..901b7c3 Binary files /dev/null and b/ultralytics/models/yolo/pose/__pycache__/predict.cpython-312.pyc differ diff --git a/ultralytics/models/yolo/pose/__pycache__/predict.cpython-39.pyc b/ultralytics/models/yolo/pose/__pycache__/predict.cpython-39.pyc new file mode 100644 index 0000000..de16763 Binary files /dev/null and b/ultralytics/models/yolo/pose/__pycache__/predict.cpython-39.pyc differ diff --git a/ultralytics/models/yolo/pose/__pycache__/train.cpython-312.pyc b/ultralytics/models/yolo/pose/__pycache__/train.cpython-312.pyc new file mode 100644 index 0000000..7289f01 Binary files /dev/null and b/ultralytics/models/yolo/pose/__pycache__/train.cpython-312.pyc differ diff --git a/ultralytics/models/yolo/pose/__pycache__/train.cpython-39.pyc b/ultralytics/models/yolo/pose/__pycache__/train.cpython-39.pyc new file mode 100644 index 0000000..e7268de Binary files /dev/null and b/ultralytics/models/yolo/pose/__pycache__/train.cpython-39.pyc differ diff --git a/ultralytics/models/yolo/pose/__pycache__/val.cpython-312.pyc b/ultralytics/models/yolo/pose/__pycache__/val.cpython-312.pyc new file mode 100644 index 0000000..59d7a76 Binary files /dev/null and b/ultralytics/models/yolo/pose/__pycache__/val.cpython-312.pyc differ diff --git a/ultralytics/models/yolo/pose/__pycache__/val.cpython-39.pyc b/ultralytics/models/yolo/pose/__pycache__/val.cpython-39.pyc new file mode 100644 index 0000000..a331001 Binary files /dev/null and b/ultralytics/models/yolo/pose/__pycache__/val.cpython-39.pyc differ diff --git a/ultralytics/models/yolo/pose/predict.py b/ultralytics/models/yolo/pose/predict.py new file mode 100644 index 0000000..7c55709 --- /dev/null +++ b/ultralytics/models/yolo/pose/predict.py @@ -0,0 +1,58 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from ultralytics.engine.results import Results +from ultralytics.models.yolo.detect.predict import DetectionPredictor +from ultralytics.utils import DEFAULT_CFG, LOGGER, ops + + +class PosePredictor(DetectionPredictor): + """ + A class extending the DetectionPredictor class for prediction based on a pose model. + + Example: + ```python + from ultralytics.utils import ASSETS + from ultralytics.models.yolo.pose import PosePredictor + + args = dict(model='yolov8n-pose.pt', source=ASSETS) + predictor = PosePredictor(overrides=args) + predictor.predict_cli() + ``` + """ + + def __init__(self, cfg=DEFAULT_CFG, overrides=None, _callbacks=None): + """Initializes PosePredictor, sets task to 'pose' and logs a warning for using 'mps' as device.""" + super().__init__(cfg, overrides, _callbacks) + self.args.task = "pose" + if isinstance(self.args.device, str) and self.args.device.lower() == "mps": + LOGGER.warning( + "WARNING ⚠️ Apple MPS known Pose bug. Recommend 'device=cpu' for Pose models. " + "See https://github.com/ultralytics/ultralytics/issues/4031." + ) + + def postprocess(self, preds, img, orig_imgs): + """Return detection results for a given input image or list of images.""" + preds = ops.non_max_suppression( + preds, + self.args.conf, + self.args.iou, + agnostic=self.args.agnostic_nms, + max_det=self.args.max_det, + classes=self.args.classes, + nc=len(self.model.names), + ) + + if not isinstance(orig_imgs, list): # input images are a torch.Tensor, not a list + orig_imgs = ops.convert_torch2numpy_batch(orig_imgs) + + results = [] + for i, pred in enumerate(preds): + orig_img = orig_imgs[i] + pred[:, :4] = ops.scale_boxes(img.shape[2:], pred[:, :4], orig_img.shape).round() + pred_kpts = pred[:, 6:].view(len(pred), *self.model.kpt_shape) if len(pred) else pred[:, 6:] + pred_kpts = ops.scale_coords(img.shape[2:], pred_kpts, orig_img.shape) + img_path = self.batch[0][i] + results.append( + Results(orig_img, path=img_path, names=self.model.names, boxes=pred[:, :6], keypoints=pred_kpts) + ) + return results diff --git a/ultralytics/models/yolo/pose/train.py b/ultralytics/models/yolo/pose/train.py new file mode 100644 index 0000000..f5229e5 --- /dev/null +++ b/ultralytics/models/yolo/pose/train.py @@ -0,0 +1,79 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from copy import copy + +from ultralytics.models import yolo +from ultralytics.nn.tasks import PoseModel +from ultralytics.utils import DEFAULT_CFG, LOGGER +from ultralytics.utils.plotting import plot_images, plot_results + + +class PoseTrainer(yolo.detect.DetectionTrainer): + """ + A class extending the DetectionTrainer class for training based on a pose model. + + Example: + ```python + from ultralytics.models.yolo.pose import PoseTrainer + + args = dict(model='yolov8n-pose.pt', data='coco8-pose.yaml', epochs=3) + trainer = PoseTrainer(overrides=args) + trainer.train() + ``` + """ + + def __init__(self, cfg=DEFAULT_CFG, overrides=None, _callbacks=None): + """Initialize a PoseTrainer object with specified configurations and overrides.""" + if overrides is None: + overrides = {} + overrides["task"] = "pose" + super().__init__(cfg, overrides, _callbacks) + + if isinstance(self.args.device, str) and self.args.device.lower() == "mps": + LOGGER.warning( + "WARNING ⚠️ Apple MPS known Pose bug. Recommend 'device=cpu' for Pose models. " + "See https://github.com/ultralytics/ultralytics/issues/4031." + ) + + def get_model(self, cfg=None, weights=None, verbose=True): + """Get pose estimation model with specified configuration and weights.""" + model = PoseModel(cfg, ch=3, nc=self.data["nc"], data_kpt_shape=self.data["kpt_shape"], verbose=verbose) + if weights: + model.load(weights) + + return model + + def set_model_attributes(self): + """Sets keypoints shape attribute of PoseModel.""" + super().set_model_attributes() + self.model.kpt_shape = self.data["kpt_shape"] + + def get_validator(self): + """Returns an instance of the PoseValidator class for validation.""" + self.loss_names = "box_loss", "pose_loss", "kobj_loss", "cls_loss", "dfl_loss" + return yolo.pose.PoseValidator( + self.test_loader, save_dir=self.save_dir, args=copy(self.args), _callbacks=self.callbacks + ) + + def plot_training_samples(self, batch, ni): + """Plot a batch of training samples with annotated class labels, bounding boxes, and keypoints.""" + images = batch["img"] + kpts = batch["keypoints"] + cls = batch["cls"].squeeze(-1) + bboxes = batch["bboxes"] + paths = batch["im_file"] + batch_idx = batch["batch_idx"] + plot_images( + images, + batch_idx, + cls, + bboxes, + kpts=kpts, + paths=paths, + fname=self.save_dir / f"train_batch{ni}.jpg", + on_plot=self.on_plot, + ) + + def plot_metrics(self): + """Plots training/val metrics.""" + plot_results(file=self.csv, pose=True, on_plot=self.on_plot) # save results.png diff --git a/ultralytics/models/yolo/pose/val.py b/ultralytics/models/yolo/pose/val.py new file mode 100644 index 0000000..8405686 --- /dev/null +++ b/ultralytics/models/yolo/pose/val.py @@ -0,0 +1,248 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from pathlib import Path + +import numpy as np +import torch + +from ultralytics.models.yolo.detect import DetectionValidator +from ultralytics.utils import LOGGER, ops +from ultralytics.utils.checks import check_requirements +from ultralytics.utils.metrics import OKS_SIGMA, PoseMetrics, box_iou, kpt_iou +from ultralytics.utils.plotting import output_to_target, plot_images + + +class PoseValidator(DetectionValidator): + """ + A class extending the DetectionValidator class for validation based on a pose model. + + Example: + ```python + from ultralytics.models.yolo.pose import PoseValidator + + args = dict(model='yolov8n-pose.pt', data='coco8-pose.yaml') + validator = PoseValidator(args=args) + validator() + ``` + """ + + def __init__(self, dataloader=None, save_dir=None, pbar=None, args=None, _callbacks=None): + """Initialize a 'PoseValidator' object with custom parameters and assigned attributes.""" + super().__init__(dataloader, save_dir, pbar, args, _callbacks) + self.sigma = None + self.kpt_shape = None + self.args.task = "pose" + self.metrics = PoseMetrics(save_dir=self.save_dir, on_plot=self.on_plot) + if isinstance(self.args.device, str) and self.args.device.lower() == "mps": + LOGGER.warning( + "WARNING ⚠️ Apple MPS known Pose bug. Recommend 'device=cpu' for Pose models. " + "See https://github.com/ultralytics/ultralytics/issues/4031." + ) + + def preprocess(self, batch): + """Preprocesses the batch by converting the 'keypoints' data into a float and moving it to the device.""" + batch = super().preprocess(batch) + batch["keypoints"] = batch["keypoints"].to(self.device).float() + return batch + + def get_desc(self): + """Returns description of evaluation metrics in string format.""" + return ("%22s" + "%11s" * 10) % ( + "Class", + "Images", + "Instances", + "Box(P", + "R", + "mAP50", + "mAP50-95)", + "Pose(P", + "R", + "mAP50", + "mAP50-95)", + ) + + def postprocess(self, preds): + """Apply non-maximum suppression and return detections with high confidence scores.""" + return ops.non_max_suppression( + preds, + self.args.conf, + self.args.iou, + labels=self.lb, + multi_label=True, + agnostic=self.args.single_cls, + max_det=self.args.max_det, + nc=self.nc, + ) + + def init_metrics(self, model): + """Initiate pose estimation metrics for YOLO model.""" + super().init_metrics(model) + self.kpt_shape = self.data["kpt_shape"] + is_pose = self.kpt_shape == [17, 3] + nkpt = self.kpt_shape[0] + self.sigma = OKS_SIGMA if is_pose else np.ones(nkpt) / nkpt + self.stats = dict(tp_p=[], tp=[], conf=[], pred_cls=[], target_cls=[]) + + def _prepare_batch(self, si, batch): + """Prepares a batch for processing by converting keypoints to float and moving to device.""" + pbatch = super()._prepare_batch(si, batch) + kpts = batch["keypoints"][batch["batch_idx"] == si] + h, w = pbatch["imgsz"] + kpts = kpts.clone() + kpts[..., 0] *= w + kpts[..., 1] *= h + kpts = ops.scale_coords(pbatch["imgsz"], kpts, pbatch["ori_shape"], ratio_pad=pbatch["ratio_pad"]) + pbatch["kpts"] = kpts + return pbatch + + def _prepare_pred(self, pred, pbatch): + """Prepares and scales keypoints in a batch for pose processing.""" + predn = super()._prepare_pred(pred, pbatch) + nk = pbatch["kpts"].shape[1] + pred_kpts = predn[:, 6:].view(len(predn), nk, -1) + ops.scale_coords(pbatch["imgsz"], pred_kpts, pbatch["ori_shape"], ratio_pad=pbatch["ratio_pad"]) + return predn, pred_kpts + + def update_metrics(self, preds, batch): + """Metrics.""" + for si, pred in enumerate(preds): + self.seen += 1 + npr = len(pred) + stat = dict( + conf=torch.zeros(0, device=self.device), + pred_cls=torch.zeros(0, device=self.device), + tp=torch.zeros(npr, self.niou, dtype=torch.bool, device=self.device), + tp_p=torch.zeros(npr, self.niou, dtype=torch.bool, device=self.device), + ) + pbatch = self._prepare_batch(si, batch) + cls, bbox = pbatch.pop("cls"), pbatch.pop("bbox") + nl = len(cls) + stat["target_cls"] = cls + if npr == 0: + if nl: + for k in self.stats.keys(): + self.stats[k].append(stat[k]) + if self.args.plots: + self.confusion_matrix.process_batch(detections=None, gt_bboxes=bbox, gt_cls=cls) + continue + + # Predictions + if self.args.single_cls: + pred[:, 5] = 0 + predn, pred_kpts = self._prepare_pred(pred, pbatch) + stat["conf"] = predn[:, 4] + stat["pred_cls"] = predn[:, 5] + + # Evaluate + if nl: + stat["tp"] = self._process_batch(predn, bbox, cls) + stat["tp_p"] = self._process_batch(predn, bbox, cls, pred_kpts, pbatch["kpts"]) + if self.args.plots: + self.confusion_matrix.process_batch(predn, bbox, cls) + + for k in self.stats.keys(): + self.stats[k].append(stat[k]) + + # Save + if self.args.save_json: + self.pred_to_json(predn, batch["im_file"][si]) + # if self.args.save_txt: + # save_one_txt(predn, save_conf, shape, file=save_dir / 'labels' / f'{path.stem}.txt') + + def _process_batch(self, detections, gt_bboxes, gt_cls, pred_kpts=None, gt_kpts=None): + """ + Return correct prediction matrix. + + Args: + detections (torch.Tensor): Tensor of shape [N, 6] representing detections. + Each detection is of the format: x1, y1, x2, y2, conf, class. + labels (torch.Tensor): Tensor of shape [M, 5] representing labels. + Each label is of the format: class, x1, y1, x2, y2. + pred_kpts (torch.Tensor, optional): Tensor of shape [N, 51] representing predicted keypoints. + 51 corresponds to 17 keypoints each with 3 values. + gt_kpts (torch.Tensor, optional): Tensor of shape [N, 51] representing ground truth keypoints. + + Returns: + torch.Tensor: Correct prediction matrix of shape [N, 10] for 10 IoU levels. + """ + if pred_kpts is not None and gt_kpts is not None: + # `0.53` is from https://github.com/jin-s13/xtcocoapi/blob/master/xtcocotools/cocoeval.py#L384 + area = ops.xyxy2xywh(gt_bboxes)[:, 2:].prod(1) * 0.53 + iou = kpt_iou(gt_kpts, pred_kpts, sigma=self.sigma, area=area) + else: # boxes + iou = box_iou(gt_bboxes, detections[:, :4]) + + return self.match_predictions(detections[:, 5], gt_cls, iou) + + def plot_val_samples(self, batch, ni): + """Plots and saves validation set samples with predicted bounding boxes and keypoints.""" + plot_images( + batch["img"], + batch["batch_idx"], + batch["cls"].squeeze(-1), + batch["bboxes"], + kpts=batch["keypoints"], + paths=batch["im_file"], + fname=self.save_dir / f"val_batch{ni}_labels.jpg", + names=self.names, + on_plot=self.on_plot, + ) + + def plot_predictions(self, batch, preds, ni): + """Plots predictions for YOLO model.""" + pred_kpts = torch.cat([p[:, 6:].view(-1, *self.kpt_shape) for p in preds], 0) + plot_images( + batch["img"], + *output_to_target(preds, max_det=self.args.max_det), + kpts=pred_kpts, + paths=batch["im_file"], + fname=self.save_dir / f"val_batch{ni}_pred.jpg", + names=self.names, + on_plot=self.on_plot, + ) # pred + + def pred_to_json(self, predn, filename): + """Converts YOLO predictions to COCO JSON format.""" + stem = Path(filename).stem + image_id = int(stem) if stem.isnumeric() else stem + box = ops.xyxy2xywh(predn[:, :4]) # xywh + box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner + for p, b in zip(predn.tolist(), box.tolist()): + self.jdict.append( + { + "image_id": image_id, + "category_id": self.class_map[int(p[5])], + "bbox": [round(x, 3) for x in b], + "keypoints": p[6:], + "score": round(p[4], 5), + } + ) + + def eval_json(self, stats): + """Evaluates object detection model using COCO JSON format.""" + if self.args.save_json and self.is_coco and len(self.jdict): + anno_json = self.data["path"] / "annotations/person_keypoints_val2017.json" # annotations + pred_json = self.save_dir / "predictions.json" # predictions + LOGGER.info(f"\nEvaluating pycocotools mAP using {pred_json} and {anno_json}...") + try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb + check_requirements("pycocotools>=2.0.6") + from pycocotools.coco import COCO # noqa + from pycocotools.cocoeval import COCOeval # noqa + + for x in anno_json, pred_json: + assert x.is_file(), f"{x} file not found" + anno = COCO(str(anno_json)) # init annotations api + pred = anno.loadRes(str(pred_json)) # init predictions api (must pass string, not Path) + for i, eval in enumerate([COCOeval(anno, pred, "bbox"), COCOeval(anno, pred, "keypoints")]): + if self.is_coco: + eval.params.imgIds = [int(Path(x).stem) for x in self.dataloader.dataset.im_files] # im to eval + eval.evaluate() + eval.accumulate() + eval.summarize() + idx = i * 4 + 2 + stats[self.metrics.keys[idx + 1]], stats[self.metrics.keys[idx]] = eval.stats[ + :2 + ] # update mAP50-95 and mAP50 + except Exception as e: + LOGGER.warning(f"pycocotools unable to run: {e}") + return stats diff --git a/ultralytics/models/yolo/segment/__init__.py b/ultralytics/models/yolo/segment/__init__.py new file mode 100644 index 0000000..ec1ac79 --- /dev/null +++ b/ultralytics/models/yolo/segment/__init__.py @@ -0,0 +1,7 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from .predict import SegmentationPredictor +from .train import SegmentationTrainer +from .val import SegmentationValidator + +__all__ = "SegmentationPredictor", "SegmentationTrainer", "SegmentationValidator" diff --git a/ultralytics/models/yolo/segment/__pycache__/__init__.cpython-312.pyc b/ultralytics/models/yolo/segment/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..c19ecf4 Binary files /dev/null and b/ultralytics/models/yolo/segment/__pycache__/__init__.cpython-312.pyc differ diff --git a/ultralytics/models/yolo/segment/__pycache__/__init__.cpython-39.pyc b/ultralytics/models/yolo/segment/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..05ab95f Binary files /dev/null and b/ultralytics/models/yolo/segment/__pycache__/__init__.cpython-39.pyc differ diff --git a/ultralytics/models/yolo/segment/__pycache__/predict.cpython-312.pyc b/ultralytics/models/yolo/segment/__pycache__/predict.cpython-312.pyc new file mode 100644 index 0000000..45d023d Binary files /dev/null and b/ultralytics/models/yolo/segment/__pycache__/predict.cpython-312.pyc differ diff --git a/ultralytics/models/yolo/segment/__pycache__/predict.cpython-39.pyc b/ultralytics/models/yolo/segment/__pycache__/predict.cpython-39.pyc new file mode 100644 index 0000000..135cf37 Binary files /dev/null and b/ultralytics/models/yolo/segment/__pycache__/predict.cpython-39.pyc differ diff --git a/ultralytics/models/yolo/segment/__pycache__/train.cpython-312.pyc b/ultralytics/models/yolo/segment/__pycache__/train.cpython-312.pyc new file mode 100644 index 0000000..a237882 Binary files /dev/null and b/ultralytics/models/yolo/segment/__pycache__/train.cpython-312.pyc differ diff --git a/ultralytics/models/yolo/segment/__pycache__/train.cpython-39.pyc b/ultralytics/models/yolo/segment/__pycache__/train.cpython-39.pyc new file mode 100644 index 0000000..44c61db Binary files /dev/null and b/ultralytics/models/yolo/segment/__pycache__/train.cpython-39.pyc differ diff --git a/ultralytics/models/yolo/segment/__pycache__/val.cpython-312.pyc b/ultralytics/models/yolo/segment/__pycache__/val.cpython-312.pyc new file mode 100644 index 0000000..d53c480 Binary files /dev/null and b/ultralytics/models/yolo/segment/__pycache__/val.cpython-312.pyc differ diff --git a/ultralytics/models/yolo/segment/__pycache__/val.cpython-39.pyc b/ultralytics/models/yolo/segment/__pycache__/val.cpython-39.pyc new file mode 100644 index 0000000..b5e60d5 Binary files /dev/null and b/ultralytics/models/yolo/segment/__pycache__/val.cpython-39.pyc differ diff --git a/ultralytics/models/yolo/segment/predict.py b/ultralytics/models/yolo/segment/predict.py new file mode 100644 index 0000000..9d7015f --- /dev/null +++ b/ultralytics/models/yolo/segment/predict.py @@ -0,0 +1,57 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from ultralytics.engine.results import Results +from ultralytics.models.yolo.detect.predict import DetectionPredictor +from ultralytics.utils import DEFAULT_CFG, ops + + +class SegmentationPredictor(DetectionPredictor): + """ + A class extending the DetectionPredictor class for prediction based on a segmentation model. + + Example: + ```python + from ultralytics.utils import ASSETS + from ultralytics.models.yolo.segment import SegmentationPredictor + + args = dict(model='yolov8n-seg.pt', source=ASSETS) + predictor = SegmentationPredictor(overrides=args) + predictor.predict_cli() + ``` + """ + + def __init__(self, cfg=DEFAULT_CFG, overrides=None, _callbacks=None): + """Initializes the SegmentationPredictor with the provided configuration, overrides, and callbacks.""" + super().__init__(cfg, overrides, _callbacks) + self.args.task = "segment" + + def postprocess(self, preds, img, orig_imgs): + """Applies non-max suppression and processes detections for each image in an input batch.""" + p = ops.non_max_suppression( + preds[0], + self.args.conf, + self.args.iou, + agnostic=self.args.agnostic_nms, + max_det=self.args.max_det, + nc=len(self.model.names), + classes=self.args.classes, + ) + + if not isinstance(orig_imgs, list): # input images are a torch.Tensor, not a list + orig_imgs = ops.convert_torch2numpy_batch(orig_imgs) + + results = [] + proto = preds[1][-1] if isinstance(preds[1], tuple) else preds[1] # tuple if PyTorch model or array if exported + for i, pred in enumerate(p): + orig_img = orig_imgs[i] + img_path = self.batch[0][i] + if not len(pred): # save empty boxes + masks = None + elif self.args.retina_masks: + pred[:, :4] = ops.scale_boxes(img.shape[2:], pred[:, :4], orig_img.shape) + masks = ops.process_mask_native(proto[i], pred[:, 6:], pred[:, :4], orig_img.shape[:2]) # HWC + else: + masks = ops.process_mask(proto[i], pred[:, 6:], pred[:, :4], img.shape[2:], upsample=True) # HWC + pred[:, :4] = ops.scale_boxes(img.shape[2:], pred[:, :4], orig_img.shape) + results.append(Results(orig_img, path=img_path, names=self.model.names, boxes=pred[:, :6], masks=masks)) + return results diff --git a/ultralytics/models/yolo/segment/train.py b/ultralytics/models/yolo/segment/train.py new file mode 100644 index 0000000..126baf2 --- /dev/null +++ b/ultralytics/models/yolo/segment/train.py @@ -0,0 +1,62 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from copy import copy + +from ultralytics.models import yolo +from ultralytics.nn.tasks import SegmentationModel +from ultralytics.utils import DEFAULT_CFG, RANK +from ultralytics.utils.plotting import plot_images, plot_results + + +class SegmentationTrainer(yolo.detect.DetectionTrainer): + """ + A class extending the DetectionTrainer class for training based on a segmentation model. + + Example: + ```python + from ultralytics.models.yolo.segment import SegmentationTrainer + + args = dict(model='yolov8n-seg.pt', data='coco8-seg.yaml', epochs=3) + trainer = SegmentationTrainer(overrides=args) + trainer.train() + ``` + """ + + def __init__(self, cfg=DEFAULT_CFG, overrides=None, _callbacks=None): + """Initialize a SegmentationTrainer object with given arguments.""" + if overrides is None: + overrides = {} + overrides["task"] = "segment" + super().__init__(cfg, overrides, _callbacks) + + def get_model(self, cfg=None, weights=None, verbose=True): + """Return SegmentationModel initialized with specified config and weights.""" + model = SegmentationModel(cfg, ch=3, nc=self.data["nc"], verbose=verbose and RANK == -1) + if weights: + model.load(weights) + + return model + + def get_validator(self): + """Return an instance of SegmentationValidator for validation of YOLO model.""" + self.loss_names = "box_loss", "seg_loss", "cls_loss", "dfl_loss" + return yolo.segment.SegmentationValidator( + self.test_loader, save_dir=self.save_dir, args=copy(self.args), _callbacks=self.callbacks + ) + + def plot_training_samples(self, batch, ni): + """Creates a plot of training sample images with labels and box coordinates.""" + plot_images( + batch["img"], + batch["batch_idx"], + batch["cls"].squeeze(-1), + batch["bboxes"], + masks=batch["masks"], + paths=batch["im_file"], + fname=self.save_dir / f"train_batch{ni}.jpg", + on_plot=self.on_plot, + ) + + def plot_metrics(self): + """Plots training/val metrics.""" + plot_results(file=self.csv, segment=True, on_plot=self.on_plot) # save results.png diff --git a/ultralytics/models/yolo/segment/val.py b/ultralytics/models/yolo/segment/val.py new file mode 100644 index 0000000..94757c4 --- /dev/null +++ b/ultralytics/models/yolo/segment/val.py @@ -0,0 +1,277 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from multiprocessing.pool import ThreadPool +from pathlib import Path + +import numpy as np +import torch +import torch.nn.functional as F + +from ultralytics.models.yolo.detect import DetectionValidator +from ultralytics.utils import LOGGER, NUM_THREADS, ops +from ultralytics.utils.checks import check_requirements +from ultralytics.utils.metrics import SegmentMetrics, box_iou, mask_iou +from ultralytics.utils.plotting import output_to_target, plot_images + + +class SegmentationValidator(DetectionValidator): + """ + A class extending the DetectionValidator class for validation based on a segmentation model. + + Example: + ```python + from ultralytics.models.yolo.segment import SegmentationValidator + + args = dict(model='yolov8n-seg.pt', data='coco8-seg.yaml') + validator = SegmentationValidator(args=args) + validator() + ``` + """ + + def __init__(self, dataloader=None, save_dir=None, pbar=None, args=None, _callbacks=None): + """Initialize SegmentationValidator and set task to 'segment', metrics to SegmentMetrics.""" + super().__init__(dataloader, save_dir, pbar, args, _callbacks) + self.plot_masks = None + self.process = None + self.args.task = "segment" + self.metrics = SegmentMetrics(save_dir=self.save_dir, on_plot=self.on_plot) + + def preprocess(self, batch): + """Preprocesses batch by converting masks to float and sending to device.""" + batch = super().preprocess(batch) + batch["masks"] = batch["masks"].to(self.device).float() + return batch + + def init_metrics(self, model): + """Initialize metrics and select mask processing function based on save_json flag.""" + super().init_metrics(model) + self.plot_masks = [] + if self.args.save_json: + check_requirements("pycocotools>=2.0.6") + self.process = ops.process_mask_upsample # more accurate + else: + self.process = ops.process_mask # faster + self.stats = dict(tp_m=[], tp=[], conf=[], pred_cls=[], target_cls=[]) + + def get_desc(self): + """Return a formatted description of evaluation metrics.""" + return ("%22s" + "%11s" * 10) % ( + "Class", + "Images", + "Instances", + "Box(P", + "R", + "mAP50", + "mAP50-95)", + "Mask(P", + "R", + "mAP50", + "mAP50-95)", + ) + + def postprocess(self, preds): + """Post-processes YOLO predictions and returns output detections with proto.""" + p = ops.non_max_suppression( + preds[0], + self.args.conf, + self.args.iou, + labels=self.lb, + multi_label=True, + agnostic=self.args.single_cls, + max_det=self.args.max_det, + nc=self.nc, + ) + proto = preds[1][-1] if len(preds[1]) == 3 else preds[1] # second output is len 3 if pt, but only 1 if exported + return p, proto + + def _prepare_batch(self, si, batch): + """Prepares a batch for training or inference by processing images and targets.""" + prepared_batch = super()._prepare_batch(si, batch) + midx = [si] if self.args.overlap_mask else batch["batch_idx"] == si + prepared_batch["masks"] = batch["masks"][midx] + return prepared_batch + + def _prepare_pred(self, pred, pbatch, proto): + """Prepares a batch for training or inference by processing images and targets.""" + predn = super()._prepare_pred(pred, pbatch) + pred_masks = self.process(proto, pred[:, 6:], pred[:, :4], shape=pbatch["imgsz"]) + return predn, pred_masks + + def update_metrics(self, preds, batch): + """Metrics.""" + for si, (pred, proto) in enumerate(zip(preds[0], preds[1])): + self.seen += 1 + npr = len(pred) + stat = dict( + conf=torch.zeros(0, device=self.device), + pred_cls=torch.zeros(0, device=self.device), + tp=torch.zeros(npr, self.niou, dtype=torch.bool, device=self.device), + tp_m=torch.zeros(npr, self.niou, dtype=torch.bool, device=self.device), + ) + pbatch = self._prepare_batch(si, batch) + cls, bbox = pbatch.pop("cls"), pbatch.pop("bbox") + nl = len(cls) + stat["target_cls"] = cls + if npr == 0: + if nl: + for k in self.stats.keys(): + self.stats[k].append(stat[k]) + if self.args.plots: + self.confusion_matrix.process_batch(detections=None, gt_bboxes=bbox, gt_cls=cls) + continue + + # Masks + gt_masks = pbatch.pop("masks") + # Predictions + if self.args.single_cls: + pred[:, 5] = 0 + predn, pred_masks = self._prepare_pred(pred, pbatch, proto) + stat["conf"] = predn[:, 4] + stat["pred_cls"] = predn[:, 5] + + # Evaluate + if nl: + stat["tp"] = self._process_batch(predn, bbox, cls) + stat["tp_m"] = self._process_batch( + predn, bbox, cls, pred_masks, gt_masks, self.args.overlap_mask, masks=True + ) + if self.args.plots: + self.confusion_matrix.process_batch(predn, bbox, cls) + + for k in self.stats.keys(): + self.stats[k].append(stat[k]) + + pred_masks = torch.as_tensor(pred_masks, dtype=torch.uint8) + if self.args.plots and self.batch_i < 3: + self.plot_masks.append(pred_masks[:15].cpu()) # filter top 15 to plot + + # Save + if self.args.save_json: + pred_masks = ops.scale_image( + pred_masks.permute(1, 2, 0).contiguous().cpu().numpy(), + pbatch["ori_shape"], + ratio_pad=batch["ratio_pad"][si], + ) + self.pred_to_json(predn, batch["im_file"][si], pred_masks) + # if self.args.save_txt: + # save_one_txt(predn, save_conf, shape, file=save_dir / 'labels' / f'{path.stem}.txt') + + def finalize_metrics(self, *args, **kwargs): + """Sets speed and confusion matrix for evaluation metrics.""" + self.metrics.speed = self.speed + self.metrics.confusion_matrix = self.confusion_matrix + + def _process_batch(self, detections, gt_bboxes, gt_cls, pred_masks=None, gt_masks=None, overlap=False, masks=False): + """ + Return correct prediction matrix. + + Args: + detections (array[N, 6]), x1, y1, x2, y2, conf, class + labels (array[M, 5]), class, x1, y1, x2, y2 + + Returns: + correct (array[N, 10]), for 10 IoU levels + """ + if masks: + if overlap: + nl = len(gt_cls) + index = torch.arange(nl, device=gt_masks.device).view(nl, 1, 1) + 1 + gt_masks = gt_masks.repeat(nl, 1, 1) # shape(1,640,640) -> (n,640,640) + gt_masks = torch.where(gt_masks == index, 1.0, 0.0) + if gt_masks.shape[1:] != pred_masks.shape[1:]: + gt_masks = F.interpolate(gt_masks[None], pred_masks.shape[1:], mode="bilinear", align_corners=False)[0] + gt_masks = gt_masks.gt_(0.5) + iou = mask_iou(gt_masks.view(gt_masks.shape[0], -1), pred_masks.view(pred_masks.shape[0], -1)) + else: # boxes + iou = box_iou(gt_bboxes, detections[:, :4]) + + return self.match_predictions(detections[:, 5], gt_cls, iou) + + def plot_val_samples(self, batch, ni): + """Plots validation samples with bounding box labels.""" + plot_images( + batch["img"], + batch["batch_idx"], + batch["cls"].squeeze(-1), + batch["bboxes"], + masks=batch["masks"], + paths=batch["im_file"], + fname=self.save_dir / f"val_batch{ni}_labels.jpg", + names=self.names, + on_plot=self.on_plot, + ) + + def plot_predictions(self, batch, preds, ni): + """Plots batch predictions with masks and bounding boxes.""" + plot_images( + batch["img"], + *output_to_target(preds[0], max_det=15), # not set to self.args.max_det due to slow plotting speed + torch.cat(self.plot_masks, dim=0) if len(self.plot_masks) else self.plot_masks, + paths=batch["im_file"], + fname=self.save_dir / f"val_batch{ni}_pred.jpg", + names=self.names, + on_plot=self.on_plot, + ) # pred + self.plot_masks.clear() + + def pred_to_json(self, predn, filename, pred_masks): + """ + Save one JSON result. + + Examples: + >>> result = {"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236} + """ + from pycocotools.mask import encode # noqa + + def single_encode(x): + """Encode predicted masks as RLE and append results to jdict.""" + rle = encode(np.asarray(x[:, :, None], order="F", dtype="uint8"))[0] + rle["counts"] = rle["counts"].decode("utf-8") + return rle + + stem = Path(filename).stem + image_id = int(stem) if stem.isnumeric() else stem + box = ops.xyxy2xywh(predn[:, :4]) # xywh + box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner + pred_masks = np.transpose(pred_masks, (2, 0, 1)) + with ThreadPool(NUM_THREADS) as pool: + rles = pool.map(single_encode, pred_masks) + for i, (p, b) in enumerate(zip(predn.tolist(), box.tolist())): + self.jdict.append( + { + "image_id": image_id, + "category_id": self.class_map[int(p[5])], + "bbox": [round(x, 3) for x in b], + "score": round(p[4], 5), + "segmentation": rles[i], + } + ) + + def eval_json(self, stats): + """Return COCO-style object detection evaluation metrics.""" + if self.args.save_json and self.is_coco and len(self.jdict): + anno_json = self.data["path"] / "annotations/instances_val2017.json" # annotations + pred_json = self.save_dir / "predictions.json" # predictions + LOGGER.info(f"\nEvaluating pycocotools mAP using {pred_json} and {anno_json}...") + try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb + check_requirements("pycocotools>=2.0.6") + from pycocotools.coco import COCO # noqa + from pycocotools.cocoeval import COCOeval # noqa + + for x in anno_json, pred_json: + assert x.is_file(), f"{x} file not found" + anno = COCO(str(anno_json)) # init annotations api + pred = anno.loadRes(str(pred_json)) # init predictions api (must pass string, not Path) + for i, eval in enumerate([COCOeval(anno, pred, "bbox"), COCOeval(anno, pred, "segm")]): + if self.is_coco: + eval.params.imgIds = [int(Path(x).stem) for x in self.dataloader.dataset.im_files] # im to eval + eval.evaluate() + eval.accumulate() + eval.summarize() + idx = i * 4 + 2 + stats[self.metrics.keys[idx + 1]], stats[self.metrics.keys[idx]] = eval.stats[ + :2 + ] # update mAP50-95 and mAP50 + except Exception as e: + LOGGER.warning(f"pycocotools unable to run: {e}") + return stats diff --git a/ultralytics/models/yolov10/__init__.py b/ultralytics/models/yolov10/__init__.py new file mode 100644 index 0000000..97f137f --- /dev/null +++ b/ultralytics/models/yolov10/__init__.py @@ -0,0 +1,5 @@ +from .model import YOLOv10 +from .predict import YOLOv10DetectionPredictor +from .val import YOLOv10DetectionValidator + +__all__ = "YOLOv10DetectionPredictor", "YOLOv10DetectionValidator", "YOLOv10" diff --git a/ultralytics/models/yolov10/__pycache__/__init__.cpython-312.pyc b/ultralytics/models/yolov10/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..6458114 Binary files /dev/null and b/ultralytics/models/yolov10/__pycache__/__init__.cpython-312.pyc differ diff --git a/ultralytics/models/yolov10/__pycache__/__init__.cpython-39.pyc b/ultralytics/models/yolov10/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..54fffc5 Binary files /dev/null and b/ultralytics/models/yolov10/__pycache__/__init__.cpython-39.pyc differ diff --git a/ultralytics/models/yolov10/__pycache__/card.cpython-312.pyc b/ultralytics/models/yolov10/__pycache__/card.cpython-312.pyc new file mode 100644 index 0000000..05627f5 Binary files /dev/null and b/ultralytics/models/yolov10/__pycache__/card.cpython-312.pyc differ diff --git a/ultralytics/models/yolov10/__pycache__/card.cpython-39.pyc b/ultralytics/models/yolov10/__pycache__/card.cpython-39.pyc new file mode 100644 index 0000000..45addf0 Binary files /dev/null and b/ultralytics/models/yolov10/__pycache__/card.cpython-39.pyc differ diff --git a/ultralytics/models/yolov10/__pycache__/model.cpython-312.pyc b/ultralytics/models/yolov10/__pycache__/model.cpython-312.pyc new file mode 100644 index 0000000..cb48aff Binary files /dev/null and b/ultralytics/models/yolov10/__pycache__/model.cpython-312.pyc differ diff --git a/ultralytics/models/yolov10/__pycache__/model.cpython-39.pyc b/ultralytics/models/yolov10/__pycache__/model.cpython-39.pyc new file mode 100644 index 0000000..402cd0b Binary files /dev/null and b/ultralytics/models/yolov10/__pycache__/model.cpython-39.pyc differ diff --git a/ultralytics/models/yolov10/__pycache__/predict.cpython-312.pyc b/ultralytics/models/yolov10/__pycache__/predict.cpython-312.pyc new file mode 100644 index 0000000..54cc277 Binary files /dev/null and b/ultralytics/models/yolov10/__pycache__/predict.cpython-312.pyc differ diff --git a/ultralytics/models/yolov10/__pycache__/predict.cpython-39.pyc b/ultralytics/models/yolov10/__pycache__/predict.cpython-39.pyc new file mode 100644 index 0000000..b0b5942 Binary files /dev/null and b/ultralytics/models/yolov10/__pycache__/predict.cpython-39.pyc differ diff --git a/ultralytics/models/yolov10/__pycache__/train.cpython-312.pyc b/ultralytics/models/yolov10/__pycache__/train.cpython-312.pyc new file mode 100644 index 0000000..778f84f Binary files /dev/null and b/ultralytics/models/yolov10/__pycache__/train.cpython-312.pyc differ diff --git a/ultralytics/models/yolov10/__pycache__/train.cpython-39.pyc b/ultralytics/models/yolov10/__pycache__/train.cpython-39.pyc new file mode 100644 index 0000000..a0e1e89 Binary files /dev/null and b/ultralytics/models/yolov10/__pycache__/train.cpython-39.pyc differ diff --git a/ultralytics/models/yolov10/__pycache__/val.cpython-312.pyc b/ultralytics/models/yolov10/__pycache__/val.cpython-312.pyc new file mode 100644 index 0000000..b762622 Binary files /dev/null and b/ultralytics/models/yolov10/__pycache__/val.cpython-312.pyc differ diff --git a/ultralytics/models/yolov10/__pycache__/val.cpython-39.pyc b/ultralytics/models/yolov10/__pycache__/val.cpython-39.pyc new file mode 100644 index 0000000..24e02c3 Binary files /dev/null and b/ultralytics/models/yolov10/__pycache__/val.cpython-39.pyc differ diff --git a/ultralytics/models/yolov10/card.py b/ultralytics/models/yolov10/card.py new file mode 100644 index 0000000..fc2405c --- /dev/null +++ b/ultralytics/models/yolov10/card.py @@ -0,0 +1,64 @@ +card_template_text = """ +--- +license: agpl-3.0 +library_name: ultralytics +repo_url: https://github.com/THU-MIG/yolov10 +tags: +- object-detection +- computer-vision +- yolov10 +datasets: +- detection-datasets/coco +inference: false +--- + +### Model Description +[YOLOv10: Real-Time End-to-End Object Detection](https://arxiv.org/abs/2405.14458v1) + +- arXiv: https://arxiv.org/abs/2405.14458v1 +- github: https://github.com/THU-MIG/yolov10 + +### Installation +``` +pip install git+https://github.com/THU-MIG/yolov10.git +``` + +### Training and validation +```python +from ultralytics import YOLOv10 + +model = YOLOv10.from_pretrained('jameslahm/yolov10n') +# Training +model.train(...) +# after training, one can push to the hub +model.push_to_hub("your-hf-username/yolov10-finetuned") + +# Validation +model.val(...) +``` + +### Inference + +Here's an end-to-end example showcasing inference on a cats image: + +```python +from ultralytics import YOLOv10 + +model = YOLOv10.from_pretrained('jameslahm/yolov10n') +source = 'http://images.cocodataset.org/val2017/000000039769.jpg' +model.predict(source=source, save=True) +``` +which shows: + +![image/png](https://cdn-uploads.huggingface.co/production/uploads/628ece6054698ce61d1e7be3/tBwAsKcQA_96HCYQp7BRr.png) + +### BibTeX Entry and Citation Info +``` +@article{wang2024yolov10, + title={YOLOv10: Real-Time End-to-End Object Detection}, + author={Wang, Ao and Chen, Hui and Liu, Lihao and Chen, Kai and Lin, Zijia and Han, Jungong and Ding, Guiguang}, + journal={arXiv preprint arXiv:2405.14458}, + year={2024} +} +``` +""".strip() \ No newline at end of file diff --git a/ultralytics/models/yolov10/model.py b/ultralytics/models/yolov10/model.py new file mode 100644 index 0000000..09592c8 --- /dev/null +++ b/ultralytics/models/yolov10/model.py @@ -0,0 +1,36 @@ +from ultralytics.engine.model import Model +from ultralytics.nn.tasks import YOLOv10DetectionModel +from .val import YOLOv10DetectionValidator +from .predict import YOLOv10DetectionPredictor +from .train import YOLOv10DetectionTrainer + +from huggingface_hub import PyTorchModelHubMixin +from .card import card_template_text + +class YOLOv10(Model, PyTorchModelHubMixin, model_card_template=card_template_text): + + def __init__(self, model="yolov10n.pt", task=None, verbose=False, + names=None): + super().__init__(model=model, task=task, verbose=verbose) + if names is not None: + setattr(self.model, 'names', names) + + def push_to_hub(self, repo_name, **kwargs): + config = kwargs.get('config', {}) + config['names'] = self.names + config['model'] = self.model.yaml['yaml_file'] + config['task'] = self.task + kwargs['config'] = config + super().push_to_hub(repo_name, **kwargs) + + @property + def task_map(self): + """Map head to model, trainer, validator, and predictor classes.""" + return { + "detect": { + "model": YOLOv10DetectionModel, + "trainer": YOLOv10DetectionTrainer, + "validator": YOLOv10DetectionValidator, + "predictor": YOLOv10DetectionPredictor, + }, + } \ No newline at end of file diff --git a/ultralytics/models/yolov10/predict.py b/ultralytics/models/yolov10/predict.py new file mode 100644 index 0000000..77644e9 --- /dev/null +++ b/ultralytics/models/yolov10/predict.py @@ -0,0 +1,38 @@ +from ultralytics.models.yolo.detect import DetectionPredictor +import torch +from ultralytics.utils import ops +from ultralytics.engine.results import Results + + +class YOLOv10DetectionPredictor(DetectionPredictor): + def postprocess(self, preds, img, orig_imgs): + if isinstance(preds, dict): + preds = preds["one2one"] + + if isinstance(preds, (list, tuple)): + preds = preds[0] + + if preds.shape[-1] == 6: + pass + else: + preds = preds.transpose(-1, -2) + bboxes, scores, labels = ops.v10postprocess(preds, self.args.max_det, preds.shape[-1]-4) + bboxes = ops.xywh2xyxy(bboxes) + preds = torch.cat([bboxes, scores.unsqueeze(-1), labels.unsqueeze(-1)], dim=-1) + + mask = preds[..., 4] > self.args.conf + if self.args.classes is not None: + mask = mask & (preds[..., 5:6] == torch.tensor(self.args.classes, device=preds.device).unsqueeze(0)).any(2) + + preds = [p[mask[idx]] for idx, p in enumerate(preds)] + + if not isinstance(orig_imgs, list): # input images are a torch.Tensor, not a list + orig_imgs = ops.convert_torch2numpy_batch(orig_imgs) + + results = [] + for i, pred in enumerate(preds): + orig_img = orig_imgs[i] + pred[:, :4] = ops.scale_boxes(img.shape[2:], pred[:, :4], orig_img.shape) + img_path = self.batch[0][i] + results.append(Results(orig_img, path=img_path, names=self.model.names, boxes=pred)) + return results diff --git a/ultralytics/models/yolov10/train.py b/ultralytics/models/yolov10/train.py new file mode 100644 index 0000000..7305bca --- /dev/null +++ b/ultralytics/models/yolov10/train.py @@ -0,0 +1,20 @@ +from ultralytics.models.yolo.detect import DetectionTrainer +from .val import YOLOv10DetectionValidator +from .model import YOLOv10DetectionModel +from copy import copy +from ultralytics.utils import RANK + +class YOLOv10DetectionTrainer(DetectionTrainer): + def get_validator(self): + """Returns a DetectionValidator for YOLO model validation.""" + self.loss_names = "box_om", "cls_om", "dfl_om", "box_oo", "cls_oo", "dfl_oo", + return YOLOv10DetectionValidator( + self.test_loader, save_dir=self.save_dir, args=copy(self.args), _callbacks=self.callbacks + ) + + def get_model(self, cfg=None, weights=None, verbose=True): + """Return a YOLO detection model.""" + model = YOLOv10DetectionModel(cfg, nc=self.data["nc"], verbose=verbose and RANK == -1) + if weights: + model.load(weights) + return model diff --git a/ultralytics/models/yolov10/val.py b/ultralytics/models/yolov10/val.py new file mode 100644 index 0000000..19a019c --- /dev/null +++ b/ultralytics/models/yolov10/val.py @@ -0,0 +1,24 @@ +from ultralytics.models.yolo.detect import DetectionValidator +from ultralytics.utils import ops +import torch + +class YOLOv10DetectionValidator(DetectionValidator): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.args.save_json |= self.is_coco + + def postprocess(self, preds): + if isinstance(preds, dict): + preds = preds["one2one"] + + if isinstance(preds, (list, tuple)): + preds = preds[0] + + # Acknowledgement: Thanks to sanha9999 in #190 and #181! + if preds.shape[-1] == 6: + return preds + else: + preds = preds.transpose(-1, -2) + boxes, scores, labels = ops.v10postprocess(preds, self.args.max_det, self.nc) + bboxes = ops.xywh2xyxy(boxes) + return torch.cat([bboxes, scores.unsqueeze(-1), labels.unsqueeze(-1)], dim=-1) \ No newline at end of file diff --git a/ultralytics/nn/__init__.py b/ultralytics/nn/__init__.py new file mode 100644 index 0000000..6905d34 --- /dev/null +++ b/ultralytics/nn/__init__.py @@ -0,0 +1,29 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from .tasks import ( + BaseModel, + ClassificationModel, + DetectionModel, + SegmentationModel, + attempt_load_one_weight, + attempt_load_weights, + guess_model_scale, + guess_model_task, + parse_model, + torch_safe_load, + yaml_model_load, +) + +__all__ = ( + "attempt_load_one_weight", + "attempt_load_weights", + "parse_model", + "yaml_model_load", + "guess_model_task", + "guess_model_scale", + "torch_safe_load", + "DetectionModel", + "SegmentationModel", + "ClassificationModel", + "BaseModel", +) diff --git a/ultralytics/nn/__pycache__/__init__.cpython-312.pyc b/ultralytics/nn/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..2798df6 Binary files /dev/null and b/ultralytics/nn/__pycache__/__init__.cpython-312.pyc differ diff --git a/ultralytics/nn/__pycache__/__init__.cpython-39.pyc b/ultralytics/nn/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..0bef79a Binary files /dev/null and b/ultralytics/nn/__pycache__/__init__.cpython-39.pyc differ diff --git a/ultralytics/nn/__pycache__/autobackend.cpython-312.pyc b/ultralytics/nn/__pycache__/autobackend.cpython-312.pyc new file mode 100644 index 0000000..8ebecda Binary files /dev/null and b/ultralytics/nn/__pycache__/autobackend.cpython-312.pyc differ diff --git a/ultralytics/nn/__pycache__/autobackend.cpython-39.pyc b/ultralytics/nn/__pycache__/autobackend.cpython-39.pyc new file mode 100644 index 0000000..3f6ef4e Binary files /dev/null and b/ultralytics/nn/__pycache__/autobackend.cpython-39.pyc differ diff --git a/ultralytics/nn/__pycache__/tasks.cpython-312.pyc b/ultralytics/nn/__pycache__/tasks.cpython-312.pyc new file mode 100644 index 0000000..a9df42e Binary files /dev/null and b/ultralytics/nn/__pycache__/tasks.cpython-312.pyc differ diff --git a/ultralytics/nn/__pycache__/tasks.cpython-39.pyc b/ultralytics/nn/__pycache__/tasks.cpython-39.pyc new file mode 100644 index 0000000..60201b2 Binary files /dev/null and b/ultralytics/nn/__pycache__/tasks.cpython-39.pyc differ diff --git a/ultralytics/nn/autobackend.py b/ultralytics/nn/autobackend.py new file mode 100644 index 0000000..abd255c --- /dev/null +++ b/ultralytics/nn/autobackend.py @@ -0,0 +1,620 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import ast +import contextlib +import json +import platform +import zipfile +from collections import OrderedDict, namedtuple +from pathlib import Path + +import cv2 +import numpy as np +import torch +import torch.nn as nn +from PIL import Image + +from ultralytics.utils import ARM64, LINUX, LOGGER, ROOT, yaml_load +from ultralytics.utils.checks import check_requirements, check_suffix, check_version, check_yaml +from ultralytics.utils.downloads import attempt_download_asset, is_url + + +def check_class_names(names): + """ + Check class names. + + Map imagenet class codes to human-readable names if required. Convert lists to dicts. + """ + if isinstance(names, list): # names is a list + names = dict(enumerate(names)) # convert to dict + if isinstance(names, dict): + # Convert 1) string keys to int, i.e. '0' to 0, and non-string values to strings, i.e. True to 'True' + names = {int(k): str(v) for k, v in names.items()} + n = len(names) + if max(names.keys()) >= n: + raise KeyError( + f"{n}-class dataset requires class indices 0-{n - 1}, but you have invalid class indices " + f"{min(names.keys())}-{max(names.keys())} defined in your dataset YAML." + ) + if isinstance(names[0], str) and names[0].startswith("n0"): # imagenet class codes, i.e. 'n01440764' + names_map = yaml_load(ROOT / "cfg/datasets/ImageNet.yaml")["map"] # human-readable names + names = {k: names_map[v] for k, v in names.items()} + return names + + +def default_class_names(data=None): + """Applies default class names to an input YAML file or returns numerical class names.""" + if data: + with contextlib.suppress(Exception): + return yaml_load(check_yaml(data))["names"] + return {i: f"class{i}" for i in range(999)} # return default if above errors + + +class AutoBackend(nn.Module): + """ + Handles dynamic backend selection for running inference using Ultralytics YOLO models. + + The AutoBackend class is designed to provide an abstraction layer for various inference engines. It supports a wide + range of formats, each with specific naming conventions as outlined below: + + Supported Formats and Naming Conventions: + | Format | File Suffix | + |-----------------------|------------------| + | PyTorch | *.pt | + | TorchScript | *.torchscript | + | ONNX Runtime | *.onnx | + | ONNX OpenCV DNN | *.onnx (dnn=True)| + | OpenVINO | *openvino_model/ | + | CoreML | *.mlpackage | + | TensorRT | *.engine | + | TensorFlow SavedModel | *_saved_model | + | TensorFlow GraphDef | *.pb | + | TensorFlow Lite | *.tflite | + | TensorFlow Edge TPU | *_edgetpu.tflite | + | PaddlePaddle | *_paddle_model | + | NCNN | *_ncnn_model | + + This class offers dynamic backend switching capabilities based on the input model format, making it easier to deploy + models across various platforms. + """ + + @torch.no_grad() + def __init__( + self, + weights="yolov8n.pt", + device=torch.device("cpu"), + dnn=False, + data=None, + fp16=False, + batch=1, + fuse=True, + verbose=True, + ): + """ + Initialize the AutoBackend for inference. + + Args: + weights (str): Path to the model weights file. Defaults to 'yolov8n.pt'. + device (torch.device): Device to run the model on. Defaults to CPU. + dnn (bool): Use OpenCV DNN module for ONNX inference. Defaults to False. + data (str | Path | optional): Path to the additional data.yaml file containing class names. Optional. + fp16 (bool): Enable half-precision inference. Supported only on specific backends. Defaults to False. + batch (int): Batch-size to assume for inference. + fuse (bool): Fuse Conv2D + BatchNorm layers for optimization. Defaults to True. + verbose (bool): Enable verbose logging. Defaults to True. + """ + super().__init__() + w = str(weights[0] if isinstance(weights, list) else weights) + nn_module = isinstance(weights, torch.nn.Module) + ( + pt, + jit, + onnx, + xml, + engine, + coreml, + saved_model, + pb, + tflite, + edgetpu, + tfjs, + paddle, + ncnn, + triton, + ) = self._model_type(w) + fp16 &= pt or jit or onnx or xml or engine or nn_module or triton # FP16 + nhwc = coreml or saved_model or pb or tflite or edgetpu # BHWC formats (vs torch BCWH) + stride = 32 # default stride + model, metadata = None, None + + # Set device + cuda = torch.cuda.is_available() and device.type != "cpu" # use CUDA + if cuda and not any([nn_module, pt, jit, engine, onnx]): # GPU dataloader formats + device = torch.device("cpu") + cuda = False + + # Download if not local + if not (pt or triton or nn_module): + w = attempt_download_asset(w) + + # In-memory PyTorch model + if nn_module: + model = weights.to(device) + model = model.fuse(verbose=verbose) if fuse else model + if hasattr(model, "kpt_shape"): + kpt_shape = model.kpt_shape # pose-only + stride = max(int(model.stride.max()), 32) # model stride + names = model.module.names if hasattr(model, "module") else model.names # get class names + model.half() if fp16 else model.float() + self.model = model # explicitly assign for to(), cpu(), cuda(), half() + pt = True + + # PyTorch + elif pt: + from ultralytics.nn.tasks import attempt_load_weights + + model = attempt_load_weights( + weights if isinstance(weights, list) else w, device=device, inplace=True, fuse=fuse + ) + if hasattr(model, "kpt_shape"): + kpt_shape = model.kpt_shape # pose-only + stride = max(int(model.stride.max()), 32) # model stride + names = model.module.names if hasattr(model, "module") else model.names # get class names + model.half() if fp16 else model.float() + self.model = model # explicitly assign for to(), cpu(), cuda(), half() + + # TorchScript + elif jit: + LOGGER.info(f"Loading {w} for TorchScript inference...") + extra_files = {"config.txt": ""} # model metadata + model = torch.jit.load(w, _extra_files=extra_files, map_location=device) + model.half() if fp16 else model.float() + if extra_files["config.txt"]: # load metadata dict + metadata = json.loads(extra_files["config.txt"], object_hook=lambda x: dict(x.items())) + + # ONNX OpenCV DNN + elif dnn: + LOGGER.info(f"Loading {w} for ONNX OpenCV DNN inference...") + check_requirements("opencv-python>=4.5.4") + net = cv2.dnn.readNetFromONNX(w) + + # ONNX Runtime + elif onnx: + LOGGER.info(f"Loading {w} for ONNX Runtime inference...") + check_requirements(("onnx", "onnxruntime-gpu" if cuda else "onnxruntime")) + import onnxruntime + + providers = ["CUDAExecutionProvider", "CPUExecutionProvider"] if cuda else ["CPUExecutionProvider"] + session = onnxruntime.InferenceSession(w, providers=providers) + output_names = [x.name for x in session.get_outputs()] + metadata = session.get_modelmeta().custom_metadata_map + + # OpenVINO + elif xml: + LOGGER.info(f"Loading {w} for OpenVINO inference...") + check_requirements("openvino>=2024.0.0") + import openvino as ov + + core = ov.Core() + w = Path(w) + if not w.is_file(): # if not *.xml + w = next(w.glob("*.xml")) # get *.xml file from *_openvino_model dir + ov_model = core.read_model(model=str(w), weights=w.with_suffix(".bin")) + if ov_model.get_parameters()[0].get_layout().empty: + ov_model.get_parameters()[0].set_layout(ov.Layout("NCHW")) + + # OpenVINO inference modes are 'LATENCY', 'THROUGHPUT' (not recommended), or 'CUMULATIVE_THROUGHPUT' + inference_mode = "CUMULATIVE_THROUGHPUT" if batch > 1 else "LATENCY" + LOGGER.info(f"Using OpenVINO {inference_mode} mode for batch={batch} inference...") + ov_compiled_model = core.compile_model( + ov_model, + device_name="AUTO", # AUTO selects best available device, do not modify + config={"PERFORMANCE_HINT": inference_mode}, + ) + input_name = ov_compiled_model.input().get_any_name() + metadata = w.parent / "metadata.yaml" + + # TensorRT + elif engine: + LOGGER.info(f"Loading {w} for TensorRT inference...") + try: + import tensorrt as trt # noqa https://developer.nvidia.com/nvidia-tensorrt-download + except ImportError: + if LINUX: + check_requirements("nvidia-tensorrt", cmds="-U --index-url https://pypi.ngc.nvidia.com") + import tensorrt as trt # noqa + check_version(trt.__version__, "7.0.0", hard=True) # require tensorrt>=7.0.0 + if device.type == "cpu": + device = torch.device("cuda:0") + Binding = namedtuple("Binding", ("name", "dtype", "shape", "data", "ptr")) + logger = trt.Logger(trt.Logger.INFO) + # Read file + with open(w, "rb") as f, trt.Runtime(logger) as runtime: + meta_len = int.from_bytes(f.read(4), byteorder="little") # read metadata length + metadata = json.loads(f.read(meta_len).decode("utf-8")) # read metadata + model = runtime.deserialize_cuda_engine(f.read()) # read engine + context = model.create_execution_context() + bindings = OrderedDict() + output_names = [] + fp16 = False # default updated below + dynamic = False + for i in range(model.num_bindings): + name = model.get_binding_name(i) + dtype = trt.nptype(model.get_binding_dtype(i)) + if model.binding_is_input(i): + if -1 in tuple(model.get_binding_shape(i)): # dynamic + dynamic = True + context.set_binding_shape(i, tuple(model.get_profile_shape(0, i)[2])) + if dtype == np.float16: + fp16 = True + else: # output + output_names.append(name) + shape = tuple(context.get_binding_shape(i)) + im = torch.from_numpy(np.empty(shape, dtype=dtype)).to(device) + bindings[name] = Binding(name, dtype, shape, im, int(im.data_ptr())) + binding_addrs = OrderedDict((n, d.ptr) for n, d in bindings.items()) + batch_size = bindings["images"].shape[0] # if dynamic, this is instead max batch size + + # CoreML + elif coreml: + LOGGER.info(f"Loading {w} for CoreML inference...") + import coremltools as ct + + model = ct.models.MLModel(w) + metadata = dict(model.user_defined_metadata) + + # TF SavedModel + elif saved_model: + LOGGER.info(f"Loading {w} for TensorFlow SavedModel inference...") + import tensorflow as tf + + keras = False # assume TF1 saved_model + model = tf.keras.models.load_model(w) if keras else tf.saved_model.load(w) + metadata = Path(w) / "metadata.yaml" + + # TF GraphDef + elif pb: # https://www.tensorflow.org/guide/migrate#a_graphpb_or_graphpbtxt + LOGGER.info(f"Loading {w} for TensorFlow GraphDef inference...") + import tensorflow as tf + + from ultralytics.engine.exporter import gd_outputs + + def wrap_frozen_graph(gd, inputs, outputs): + """Wrap frozen graphs for deployment.""" + x = tf.compat.v1.wrap_function(lambda: tf.compat.v1.import_graph_def(gd, name=""), []) # wrapped + ge = x.graph.as_graph_element + return x.prune(tf.nest.map_structure(ge, inputs), tf.nest.map_structure(ge, outputs)) + + gd = tf.Graph().as_graph_def() # TF GraphDef + with open(w, "rb") as f: + gd.ParseFromString(f.read()) + frozen_func = wrap_frozen_graph(gd, inputs="x:0", outputs=gd_outputs(gd)) + + # TFLite or TFLite Edge TPU + elif tflite or edgetpu: # https://www.tensorflow.org/lite/guide/python#install_tensorflow_lite_for_python + try: # https://coral.ai/docs/edgetpu/tflite-python/#update-existing-tf-lite-code-for-the-edge-tpu + from tflite_runtime.interpreter import Interpreter, load_delegate + except ImportError: + import tensorflow as tf + + Interpreter, load_delegate = tf.lite.Interpreter, tf.lite.experimental.load_delegate + if edgetpu: # TF Edge TPU https://coral.ai/software/#edgetpu-runtime + LOGGER.info(f"Loading {w} for TensorFlow Lite Edge TPU inference...") + delegate = {"Linux": "libedgetpu.so.1", "Darwin": "libedgetpu.1.dylib", "Windows": "edgetpu.dll"}[ + platform.system() + ] + interpreter = Interpreter(model_path=w, experimental_delegates=[load_delegate(delegate)]) + else: # TFLite + LOGGER.info(f"Loading {w} for TensorFlow Lite inference...") + interpreter = Interpreter(model_path=w) # load TFLite model + interpreter.allocate_tensors() # allocate + input_details = interpreter.get_input_details() # inputs + output_details = interpreter.get_output_details() # outputs + # Load metadata + with contextlib.suppress(zipfile.BadZipFile): + with zipfile.ZipFile(w, "r") as model: + meta_file = model.namelist()[0] + metadata = ast.literal_eval(model.read(meta_file).decode("utf-8")) + + # TF.js + elif tfjs: + raise NotImplementedError("YOLOv8 TF.js inference is not currently supported.") + + # PaddlePaddle + elif paddle: + LOGGER.info(f"Loading {w} for PaddlePaddle inference...") + check_requirements("paddlepaddle-gpu" if cuda else "paddlepaddle") + import paddle.inference as pdi # noqa + + w = Path(w) + if not w.is_file(): # if not *.pdmodel + w = next(w.rglob("*.pdmodel")) # get *.pdmodel file from *_paddle_model dir + config = pdi.Config(str(w), str(w.with_suffix(".pdiparams"))) + if cuda: + config.enable_use_gpu(memory_pool_init_size_mb=2048, device_id=0) + predictor = pdi.create_predictor(config) + input_handle = predictor.get_input_handle(predictor.get_input_names()[0]) + output_names = predictor.get_output_names() + metadata = w.parents[1] / "metadata.yaml" + + # NCNN + elif ncnn: + LOGGER.info(f"Loading {w} for NCNN inference...") + check_requirements("git+https://github.com/Tencent/ncnn.git" if ARM64 else "ncnn") # requires NCNN + import ncnn as pyncnn + + net = pyncnn.Net() + net.opt.use_vulkan_compute = cuda + w = Path(w) + if not w.is_file(): # if not *.param + w = next(w.glob("*.param")) # get *.param file from *_ncnn_model dir + net.load_param(str(w)) + net.load_model(str(w.with_suffix(".bin"))) + metadata = w.parent / "metadata.yaml" + + # NVIDIA Triton Inference Server + elif triton: + check_requirements("tritonclient[all]") + from ultralytics.utils.triton import TritonRemoteModel + + model = TritonRemoteModel(w) + + # Any other format (unsupported) + else: + from ultralytics.engine.exporter import export_formats + + raise TypeError( + f"model='{w}' is not a supported model format. " + f"See https://docs.ultralytics.com/modes/predict for help.\n\n{export_formats()}" + ) + + # Load external metadata YAML + if isinstance(metadata, (str, Path)) and Path(metadata).exists(): + metadata = yaml_load(metadata) + if metadata: + for k, v in metadata.items(): + if k in ("stride", "batch"): + metadata[k] = int(v) + elif k in ("imgsz", "names", "kpt_shape") and isinstance(v, str): + metadata[k] = eval(v) + stride = metadata["stride"] + task = metadata["task"] + batch = metadata["batch"] + imgsz = metadata["imgsz"] + names = metadata["names"] + kpt_shape = metadata.get("kpt_shape") + elif not (pt or triton or nn_module): + LOGGER.warning(f"WARNING ⚠️ Metadata not found for 'model={weights}'") + + # Check names + if "names" not in locals(): # names missing + names = default_class_names(data) + names = check_class_names(names) + + # Disable gradients + if pt: + for p in model.parameters(): + p.requires_grad = False + + self.__dict__.update(locals()) # assign all variables to self + + def forward(self, im, augment=False, visualize=False, embed=None): + """ + Runs inference on the YOLOv8 MultiBackend model. + + Args: + im (torch.Tensor): The image tensor to perform inference on. + augment (bool): whether to perform data augmentation during inference, defaults to False + visualize (bool): whether to visualize the output predictions, defaults to False + embed (list, optional): A list of feature vectors/embeddings to return. + + Returns: + (tuple): Tuple containing the raw output tensor, and processed output for visualization (if visualize=True) + """ + b, ch, h, w = im.shape # batch, channel, height, width + if self.fp16 and im.dtype != torch.float16: + im = im.half() # to FP16 + if self.nhwc: + im = im.permute(0, 2, 3, 1) # torch BCHW to numpy BHWC shape(1,320,192,3) + + # PyTorch + if self.pt or self.nn_module: + y = self.model(im, augment=augment, visualize=visualize, embed=embed) + + # TorchScript + elif self.jit: + y = self.model(im) + + # ONNX OpenCV DNN + elif self.dnn: + im = im.cpu().numpy() # torch to numpy + self.net.setInput(im) + y = self.net.forward() + + # ONNX Runtime + elif self.onnx: + im = im.cpu().numpy() # torch to numpy + y = self.session.run(self.output_names, {self.session.get_inputs()[0].name: im}) + + # OpenVINO + elif self.xml: + im = im.cpu().numpy() # FP32 + + if self.inference_mode in {"THROUGHPUT", "CUMULATIVE_THROUGHPUT"}: # optimized for larger batch-sizes + n = im.shape[0] # number of images in batch + results = [None] * n # preallocate list with None to match the number of images + + def callback(request, userdata): + """Places result in preallocated list using userdata index.""" + results[userdata] = request.results + + # Create AsyncInferQueue, set the callback and start asynchronous inference for each input image + async_queue = self.ov.runtime.AsyncInferQueue(self.ov_compiled_model) + async_queue.set_callback(callback) + for i in range(n): + # Start async inference with userdata=i to specify the position in results list + async_queue.start_async(inputs={self.input_name: im[i : i + 1]}, userdata=i) # keep image as BCHW + async_queue.wait_all() # wait for all inference requests to complete + y = np.concatenate([list(r.values())[0] for r in results]) + + else: # inference_mode = "LATENCY", optimized for fastest first result at batch-size 1 + y = list(self.ov_compiled_model(im).values()) + + # TensorRT + elif self.engine: + if self.dynamic and im.shape != self.bindings["images"].shape: + i = self.model.get_binding_index("images") + self.context.set_binding_shape(i, im.shape) # reshape if dynamic + self.bindings["images"] = self.bindings["images"]._replace(shape=im.shape) + for name in self.output_names: + i = self.model.get_binding_index(name) + self.bindings[name].data.resize_(tuple(self.context.get_binding_shape(i))) + s = self.bindings["images"].shape + assert im.shape == s, f"input size {im.shape} {'>' if self.dynamic else 'not equal to'} max model size {s}" + self.binding_addrs["images"] = int(im.data_ptr()) + self.context.execute_v2(list(self.binding_addrs.values())) + y = [self.bindings[x].data for x in sorted(self.output_names)] + + # CoreML + elif self.coreml: + im = im[0].cpu().numpy() + im_pil = Image.fromarray((im * 255).astype("uint8")) + # im = im.resize((192, 320), Image.BILINEAR) + y = self.model.predict({"image": im_pil}) # coordinates are xywh normalized + if "confidence" in y: + raise TypeError( + "Ultralytics only supports inference of non-pipelined CoreML models exported with " + f"'nms=False', but 'model={w}' has an NMS pipeline created by an 'nms=True' export." + ) + # TODO: CoreML NMS inference handling + # from ultralytics.utils.ops import xywh2xyxy + # box = xywh2xyxy(y['coordinates'] * [[w, h, w, h]]) # xyxy pixels + # conf, cls = y['confidence'].max(1), y['confidence'].argmax(1).astype(np.float32) + # y = np.concatenate((box, conf.reshape(-1, 1), cls.reshape(-1, 1)), 1) + elif len(y) == 1: # classification model + y = list(y.values()) + elif len(y) == 2: # segmentation model + y = list(reversed(y.values())) # reversed for segmentation models (pred, proto) + + # PaddlePaddle + elif self.paddle: + im = im.cpu().numpy().astype(np.float32) + self.input_handle.copy_from_cpu(im) + self.predictor.run() + y = [self.predictor.get_output_handle(x).copy_to_cpu() for x in self.output_names] + + # NCNN + elif self.ncnn: + mat_in = self.pyncnn.Mat(im[0].cpu().numpy()) + with self.net.create_extractor() as ex: + ex.input(self.net.input_names()[0], mat_in) + y = [np.array(ex.extract(x)[1])[None] for x in self.net.output_names()] + + # NVIDIA Triton Inference Server + elif self.triton: + im = im.cpu().numpy() # torch to numpy + y = self.model(im) + + # TensorFlow (SavedModel, GraphDef, Lite, Edge TPU) + else: + im = im.cpu().numpy() + if self.saved_model: # SavedModel + y = self.model(im, training=False) if self.keras else self.model(im) + if not isinstance(y, list): + y = [y] + elif self.pb: # GraphDef + y = self.frozen_func(x=self.tf.constant(im)) + if len(y) == 2 and len(self.names) == 999: # segments and names not defined + ip, ib = (0, 1) if len(y[0].shape) == 4 else (1, 0) # index of protos, boxes + nc = y[ib].shape[1] - y[ip].shape[3] - 4 # y = (1, 160, 160, 32), (1, 116, 8400) + self.names = {i: f"class{i}" for i in range(nc)} + else: # Lite or Edge TPU + details = self.input_details[0] + integer = details["dtype"] in (np.int8, np.int16) # is TFLite quantized int8 or int16 model + if integer: + scale, zero_point = details["quantization"] + im = (im / scale + zero_point).astype(details["dtype"]) # de-scale + self.interpreter.set_tensor(details["index"], im) + self.interpreter.invoke() + y = [] + for output in self.output_details: + x = self.interpreter.get_tensor(output["index"]) + if integer: + scale, zero_point = output["quantization"] + x = (x.astype(np.float32) - zero_point) * scale # re-scale + if x.ndim > 2: # if task is not classification + # Denormalize xywh by image size. See https://github.com/ultralytics/ultralytics/pull/1695 + # xywh are normalized in TFLite/EdgeTPU to mitigate quantization error of integer models + x[:, [0, 2]] *= w + x[:, [1, 3]] *= h + y.append(x) + # TF segment fixes: export is reversed vs ONNX export and protos are transposed + if len(y) == 2: # segment with (det, proto) output order reversed + if len(y[1].shape) != 4: + y = list(reversed(y)) # should be y = (1, 116, 8400), (1, 160, 160, 32) + y[1] = np.transpose(y[1], (0, 3, 1, 2)) # should be y = (1, 116, 8400), (1, 32, 160, 160) + y = [x if isinstance(x, np.ndarray) else x.numpy() for x in y] + + # for x in y: + # print(type(x), len(x)) if isinstance(x, (list, tuple)) else print(type(x), x.shape) # debug shapes + if isinstance(y, (list, tuple)): + return self.from_numpy(y[0]) if len(y) == 1 else [self.from_numpy(x) for x in y] + else: + return self.from_numpy(y) + + def from_numpy(self, x): + """ + Convert a numpy array to a tensor. + + Args: + x (np.ndarray): The array to be converted. + + Returns: + (torch.Tensor): The converted tensor + """ + return torch.tensor(x).to(self.device) if isinstance(x, np.ndarray) else x + + def warmup(self, imgsz=(1, 3, 640, 640)): + """ + Warm up the model by running one forward pass with a dummy input. + + Args: + imgsz (tuple): The shape of the dummy input tensor in the format (batch_size, channels, height, width) + """ + warmup_types = self.pt, self.jit, self.onnx, self.engine, self.saved_model, self.pb, self.triton, self.nn_module + if any(warmup_types) and (self.device.type != "cpu" or self.triton): + im = torch.empty(*imgsz, dtype=torch.half if self.fp16 else torch.float, device=self.device) # input + for _ in range(2 if self.jit else 1): + self.forward(im) # warmup + + @staticmethod + def _model_type(p="path/to/model.pt"): + """ + This function takes a path to a model file and returns the model type. Possibles types are pt, jit, onnx, xml, + engine, coreml, saved_model, pb, tflite, edgetpu, tfjs, ncnn or paddle. + + Args: + p: path to the model file. Defaults to path/to/model.pt + + Examples: + >>> model = AutoBackend(weights="path/to/model.onnx") + >>> model_type = model._model_type() # returns "onnx" + """ + from ultralytics.engine.exporter import export_formats + + sf = list(export_formats().Suffix) # export suffixes + if not is_url(p) and not isinstance(p, str): + check_suffix(p, sf) # checks + name = Path(p).name + types = [s in name for s in sf] + types[5] |= name.endswith(".mlmodel") # retain support for older Apple CoreML *.mlmodel formats + types[8] &= not types[9] # tflite &= not edgetpu + if any(types): + triton = False + else: + from urllib.parse import urlsplit + + url = urlsplit(p) + triton = bool(url.netloc) and bool(url.path) and url.scheme in {"http", "grpc"} + + return types + [triton] diff --git a/ultralytics/nn/modules/__init__.py b/ultralytics/nn/modules/__init__.py new file mode 100644 index 0000000..7f4c4fe --- /dev/null +++ b/ultralytics/nn/modules/__init__.py @@ -0,0 +1,147 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +""" +Ultralytics modules. + +Example: + Visualize a module with Netron. + ```python + from ultralytics.nn.modules import * + import torch + import os + + x = torch.ones(1, 128, 40, 40) + m = Conv(128, 128) + f = f'{m._get_name()}.onnx' + torch.onnx.export(m, x, f) + os.system(f'onnxslim {f} {f} && open {f}') # pip install onnxslim + ``` +""" + +from .block import ( + C1, + C2, + C3, + C3TR, + DFL, + SPP, + SPPF, + Bottleneck, + BottleneckCSP, + C2f, + C2fAttn, + ImagePoolingAttn, + C3Ghost, + C3x, + GhostBottleneck, + HGBlock, + HGStem, + Proto, + RepC3, + ResNetLayer, + ContrastiveHead, + BNContrastiveHead, + RepNCSPELAN4, + ADown, + SPPELAN, + CBFuse, + CBLinear, + Silence, + PSA, + C2fCIB, + SCDown, + RepVGGDW +) +from .conv import ( + CBAM, + ChannelAttention, + Concat, + Conv, + Conv2, + ConvTranspose, + DWConv, + DWConvTranspose2d, + Focus, + GhostConv, + LightConv, + RepConv, + SpatialAttention, +) +from .head import OBB, Classify, Detect, Pose, RTDETRDecoder, Segment, WorldDetect, v10Detect +from .transformer import ( + AIFI, + MLP, + DeformableTransformerDecoder, + DeformableTransformerDecoderLayer, + LayerNorm2d, + MLPBlock, + MSDeformAttn, + TransformerBlock, + TransformerEncoderLayer, + TransformerLayer, +) + +__all__ = ( + "Conv", + "Conv2", + "LightConv", + "RepConv", + "DWConv", + "DWConvTranspose2d", + "ConvTranspose", + "Focus", + "GhostConv", + "ChannelAttention", + "SpatialAttention", + "CBAM", + "Concat", + "TransformerLayer", + "TransformerBlock", + "MLPBlock", + "LayerNorm2d", + "DFL", + "HGBlock", + "HGStem", + "SPP", + "SPPF", + "C1", + "C2", + "C3", + "C2f", + "C2fAttn", + "C3x", + "C3TR", + "C3Ghost", + "GhostBottleneck", + "Bottleneck", + "BottleneckCSP", + "Proto", + "Detect", + "Segment", + "Pose", + "Classify", + "TransformerEncoderLayer", + "RepC3", + "RTDETRDecoder", + "AIFI", + "DeformableTransformerDecoder", + "DeformableTransformerDecoderLayer", + "MSDeformAttn", + "MLP", + "ResNetLayer", + "OBB", + "WorldDetect", + "ImagePoolingAttn", + "ContrastiveHead", + "BNContrastiveHead", + "RepNCSPELAN4", + "ADown", + "SPPELAN", + "CBFuse", + "CBLinear", + "Silence", + "PSA", + "C2fCIB", + "SCDown", + "RepVGGDW", + "v10Detect" +) diff --git a/ultralytics/nn/modules/__pycache__/__init__.cpython-312.pyc b/ultralytics/nn/modules/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..2e183d9 Binary files /dev/null and b/ultralytics/nn/modules/__pycache__/__init__.cpython-312.pyc differ diff --git a/ultralytics/nn/modules/__pycache__/__init__.cpython-39.pyc b/ultralytics/nn/modules/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..8afb5eb Binary files /dev/null and b/ultralytics/nn/modules/__pycache__/__init__.cpython-39.pyc differ diff --git a/ultralytics/nn/modules/__pycache__/block.cpython-312.pyc b/ultralytics/nn/modules/__pycache__/block.cpython-312.pyc new file mode 100644 index 0000000..ae56bcd Binary files /dev/null and b/ultralytics/nn/modules/__pycache__/block.cpython-312.pyc differ diff --git a/ultralytics/nn/modules/__pycache__/block.cpython-39.pyc b/ultralytics/nn/modules/__pycache__/block.cpython-39.pyc new file mode 100644 index 0000000..56ce422 Binary files /dev/null and b/ultralytics/nn/modules/__pycache__/block.cpython-39.pyc differ diff --git a/ultralytics/nn/modules/__pycache__/conv.cpython-312.pyc b/ultralytics/nn/modules/__pycache__/conv.cpython-312.pyc new file mode 100644 index 0000000..0dcddcb Binary files /dev/null and b/ultralytics/nn/modules/__pycache__/conv.cpython-312.pyc differ diff --git a/ultralytics/nn/modules/__pycache__/conv.cpython-39.pyc b/ultralytics/nn/modules/__pycache__/conv.cpython-39.pyc new file mode 100644 index 0000000..c967af1 Binary files /dev/null and b/ultralytics/nn/modules/__pycache__/conv.cpython-39.pyc differ diff --git a/ultralytics/nn/modules/__pycache__/head.cpython-312.pyc b/ultralytics/nn/modules/__pycache__/head.cpython-312.pyc new file mode 100644 index 0000000..19309c7 Binary files /dev/null and b/ultralytics/nn/modules/__pycache__/head.cpython-312.pyc differ diff --git a/ultralytics/nn/modules/__pycache__/head.cpython-39.pyc b/ultralytics/nn/modules/__pycache__/head.cpython-39.pyc new file mode 100644 index 0000000..35edea3 Binary files /dev/null and b/ultralytics/nn/modules/__pycache__/head.cpython-39.pyc differ diff --git a/ultralytics/nn/modules/__pycache__/transformer.cpython-312.pyc b/ultralytics/nn/modules/__pycache__/transformer.cpython-312.pyc new file mode 100644 index 0000000..bba38c6 Binary files /dev/null and b/ultralytics/nn/modules/__pycache__/transformer.cpython-312.pyc differ diff --git a/ultralytics/nn/modules/__pycache__/transformer.cpython-39.pyc b/ultralytics/nn/modules/__pycache__/transformer.cpython-39.pyc new file mode 100644 index 0000000..565233c Binary files /dev/null and b/ultralytics/nn/modules/__pycache__/transformer.cpython-39.pyc differ diff --git a/ultralytics/nn/modules/__pycache__/utils.cpython-312.pyc b/ultralytics/nn/modules/__pycache__/utils.cpython-312.pyc new file mode 100644 index 0000000..9fd5563 Binary files /dev/null and b/ultralytics/nn/modules/__pycache__/utils.cpython-312.pyc differ diff --git a/ultralytics/nn/modules/__pycache__/utils.cpython-39.pyc b/ultralytics/nn/modules/__pycache__/utils.cpython-39.pyc new file mode 100644 index 0000000..51ac323 Binary files /dev/null and b/ultralytics/nn/modules/__pycache__/utils.cpython-39.pyc differ diff --git a/ultralytics/nn/modules/block.py b/ultralytics/nn/modules/block.py new file mode 100644 index 0000000..d11c16e --- /dev/null +++ b/ultralytics/nn/modules/block.py @@ -0,0 +1,827 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +"""Block modules.""" + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .conv import Conv, DWConv, GhostConv, LightConv, RepConv, autopad +from .transformer import TransformerBlock +from ultralytics.utils.torch_utils import fuse_conv_and_bn + +__all__ = ( + "DFL", + "HGBlock", + "HGStem", + "SPP", + "SPPF", + "C1", + "C2", + "C3", + "C2f", + "C2fAttn", + "ImagePoolingAttn", + "ContrastiveHead", + "BNContrastiveHead", + "C3x", + "C3TR", + "C3Ghost", + "GhostBottleneck", + "Bottleneck", + "BottleneckCSP", + "Proto", + "RepC3", + "ResNetLayer", + "RepNCSPELAN4", + "ADown", + "SPPELAN", + "CBFuse", + "CBLinear", + "Silence", +) + + +class DFL(nn.Module): + """ + Integral module of Distribution Focal Loss (DFL). + + Proposed in Generalized Focal Loss https://ieeexplore.ieee.org/document/9792391 + """ + + def __init__(self, c1=16): + """Initialize a convolutional layer with a given number of input channels.""" + super().__init__() + self.conv = nn.Conv2d(c1, 1, 1, bias=False).requires_grad_(False) + x = torch.arange(c1, dtype=torch.float) + self.conv.weight.data[:] = nn.Parameter(x.view(1, c1, 1, 1)) + self.c1 = c1 + + def forward(self, x): + """Applies a transformer layer on input tensor 'x' and returns a tensor.""" + b, _, a = x.shape # batch, channels, anchors + return self.conv(x.view(b, 4, self.c1, a).transpose(2, 1).softmax(1)).view(b, 4, a) + # return self.conv(x.view(b, self.c1, 4, a).softmax(1)).view(b, 4, a) + + +class Proto(nn.Module): + """YOLOv8 mask Proto module for segmentation models.""" + + def __init__(self, c1, c_=256, c2=32): + """ + Initializes the YOLOv8 mask Proto module with specified number of protos and masks. + + Input arguments are ch_in, number of protos, number of masks. + """ + super().__init__() + self.cv1 = Conv(c1, c_, k=3) + self.upsample = nn.ConvTranspose2d(c_, c_, 2, 2, 0, bias=True) # nn.Upsample(scale_factor=2, mode='nearest') + self.cv2 = Conv(c_, c_, k=3) + self.cv3 = Conv(c_, c2) + + def forward(self, x): + """Performs a forward pass through layers using an upsampled input image.""" + return self.cv3(self.cv2(self.upsample(self.cv1(x)))) + + +class HGStem(nn.Module): + """ + StemBlock of PPHGNetV2 with 5 convolutions and one maxpool2d. + + https://github.com/PaddlePaddle/PaddleDetection/blob/develop/ppdet/modeling/backbones/hgnet_v2.py + """ + + def __init__(self, c1, cm, c2): + """Initialize the SPP layer with input/output channels and specified kernel sizes for max pooling.""" + super().__init__() + self.stem1 = Conv(c1, cm, 3, 2, act=nn.ReLU()) + self.stem2a = Conv(cm, cm // 2, 2, 1, 0, act=nn.ReLU()) + self.stem2b = Conv(cm // 2, cm, 2, 1, 0, act=nn.ReLU()) + self.stem3 = Conv(cm * 2, cm, 3, 2, act=nn.ReLU()) + self.stem4 = Conv(cm, c2, 1, 1, act=nn.ReLU()) + self.pool = nn.MaxPool2d(kernel_size=2, stride=1, padding=0, ceil_mode=True) + + def forward(self, x): + """Forward pass of a PPHGNetV2 backbone layer.""" + x = self.stem1(x) + x = F.pad(x, [0, 1, 0, 1]) + x2 = self.stem2a(x) + x2 = F.pad(x2, [0, 1, 0, 1]) + x2 = self.stem2b(x2) + x1 = self.pool(x) + x = torch.cat([x1, x2], dim=1) + x = self.stem3(x) + x = self.stem4(x) + return x + + +class HGBlock(nn.Module): + """ + HG_Block of PPHGNetV2 with 2 convolutions and LightConv. + + https://github.com/PaddlePaddle/PaddleDetection/blob/develop/ppdet/modeling/backbones/hgnet_v2.py + """ + + def __init__(self, c1, cm, c2, k=3, n=6, lightconv=False, shortcut=False, act=nn.ReLU()): + """Initializes a CSP Bottleneck with 1 convolution using specified input and output channels.""" + super().__init__() + block = LightConv if lightconv else Conv + self.m = nn.ModuleList(block(c1 if i == 0 else cm, cm, k=k, act=act) for i in range(n)) + self.sc = Conv(c1 + n * cm, c2 // 2, 1, 1, act=act) # squeeze conv + self.ec = Conv(c2 // 2, c2, 1, 1, act=act) # excitation conv + self.add = shortcut and c1 == c2 + + def forward(self, x): + """Forward pass of a PPHGNetV2 backbone layer.""" + y = [x] + y.extend(m(y[-1]) for m in self.m) + y = self.ec(self.sc(torch.cat(y, 1))) + return y + x if self.add else y + + +class SPP(nn.Module): + """Spatial Pyramid Pooling (SPP) layer https://arxiv.org/abs/1406.4729.""" + + def __init__(self, c1, c2, k=(5, 9, 13)): + """Initialize the SPP layer with input/output channels and pooling kernel sizes.""" + super().__init__() + c_ = c1 // 2 # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = Conv(c_ * (len(k) + 1), c2, 1, 1) + self.m = nn.ModuleList([nn.MaxPool2d(kernel_size=x, stride=1, padding=x // 2) for x in k]) + + def forward(self, x): + """Forward pass of the SPP layer, performing spatial pyramid pooling.""" + x = self.cv1(x) + return self.cv2(torch.cat([x] + [m(x) for m in self.m], 1)) + + +class SPPF(nn.Module): + """Spatial Pyramid Pooling - Fast (SPPF) layer for YOLOv5 by Glenn Jocher.""" + + def __init__(self, c1, c2, k=5): + """ + Initializes the SPPF layer with given input/output channels and kernel size. + + This module is equivalent to SPP(k=(5, 9, 13)). + """ + super().__init__() + c_ = c1 // 2 # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = Conv(c_ * 4, c2, 1, 1) + self.m = nn.MaxPool2d(kernel_size=k, stride=1, padding=k // 2) + + def forward(self, x): + """Forward pass through Ghost Convolution block.""" + x = self.cv1(x) + y1 = self.m(x) + y2 = self.m(y1) + return self.cv2(torch.cat((x, y1, y2, self.m(y2)), 1)) + + +class C1(nn.Module): + """CSP Bottleneck with 1 convolution.""" + + def __init__(self, c1, c2, n=1): + """Initializes the CSP Bottleneck with configurations for 1 convolution with arguments ch_in, ch_out, number.""" + super().__init__() + self.cv1 = Conv(c1, c2, 1, 1) + self.m = nn.Sequential(*(Conv(c2, c2, 3) for _ in range(n))) + + def forward(self, x): + """Applies cross-convolutions to input in the C3 module.""" + y = self.cv1(x) + return self.m(y) + y + + +class C2(nn.Module): + """CSP Bottleneck with 2 convolutions.""" + + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): + """Initializes the CSP Bottleneck with 2 convolutions module with arguments ch_in, ch_out, number, shortcut, + groups, expansion. + """ + super().__init__() + self.c = int(c2 * e) # hidden channels + self.cv1 = Conv(c1, 2 * self.c, 1, 1) + self.cv2 = Conv(2 * self.c, c2, 1) # optional act=FReLU(c2) + # self.attention = ChannelAttention(2 * self.c) # or SpatialAttention() + self.m = nn.Sequential(*(Bottleneck(self.c, self.c, shortcut, g, k=((3, 3), (3, 3)), e=1.0) for _ in range(n))) + + def forward(self, x): + """Forward pass through the CSP bottleneck with 2 convolutions.""" + a, b = self.cv1(x).chunk(2, 1) + return self.cv2(torch.cat((self.m(a), b), 1)) + + +class C2f(nn.Module): + """Faster Implementation of CSP Bottleneck with 2 convolutions.""" + + def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5): + """Initialize CSP bottleneck layer with two convolutions with arguments ch_in, ch_out, number, shortcut, groups, + expansion. + """ + super().__init__() + self.c = int(c2 * e) # hidden channels + self.cv1 = Conv(c1, 2 * self.c, 1, 1) + self.cv2 = Conv((2 + n) * self.c, c2, 1) # optional act=FReLU(c2) + self.m = nn.ModuleList(Bottleneck(self.c, self.c, shortcut, g, k=((3, 3), (3, 3)), e=1.0) for _ in range(n)) + + def forward(self, x): + """Forward pass through C2f layer.""" + y = list(self.cv1(x).chunk(2, 1)) + y.extend(m(y[-1]) for m in self.m) + return self.cv2(torch.cat(y, 1)) + + def forward_split(self, x): + """Forward pass using split() instead of chunk().""" + y = list(self.cv1(x).split((self.c, self.c), 1)) + y.extend(m(y[-1]) for m in self.m) + return self.cv2(torch.cat(y, 1)) + + +class C3(nn.Module): + """CSP Bottleneck with 3 convolutions.""" + + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): + """Initialize the CSP Bottleneck with given channels, number, shortcut, groups, and expansion values.""" + super().__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = Conv(c1, c_, 1, 1) + self.cv3 = Conv(2 * c_, c2, 1) # optional act=FReLU(c2) + self.m = nn.Sequential(*(Bottleneck(c_, c_, shortcut, g, k=((1, 1), (3, 3)), e=1.0) for _ in range(n))) + + def forward(self, x): + """Forward pass through the CSP bottleneck with 2 convolutions.""" + return self.cv3(torch.cat((self.m(self.cv1(x)), self.cv2(x)), 1)) + + +class C3x(C3): + """C3 module with cross-convolutions.""" + + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): + """Initialize C3TR instance and set default parameters.""" + super().__init__(c1, c2, n, shortcut, g, e) + self.c_ = int(c2 * e) + self.m = nn.Sequential(*(Bottleneck(self.c_, self.c_, shortcut, g, k=((1, 3), (3, 1)), e=1) for _ in range(n))) + + +class RepC3(nn.Module): + """Rep C3.""" + + def __init__(self, c1, c2, n=3, e=1.0): + """Initialize CSP Bottleneck with a single convolution using input channels, output channels, and number.""" + super().__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = Conv(c1, c2, 1, 1) + self.cv2 = Conv(c1, c2, 1, 1) + self.m = nn.Sequential(*[RepConv(c_, c_) for _ in range(n)]) + self.cv3 = Conv(c_, c2, 1, 1) if c_ != c2 else nn.Identity() + + def forward(self, x): + """Forward pass of RT-DETR neck layer.""" + return self.cv3(self.m(self.cv1(x)) + self.cv2(x)) + + +class C3TR(C3): + """C3 module with TransformerBlock().""" + + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): + """Initialize C3Ghost module with GhostBottleneck().""" + super().__init__(c1, c2, n, shortcut, g, e) + c_ = int(c2 * e) + self.m = TransformerBlock(c_, c_, 4, n) + + +class C3Ghost(C3): + """C3 module with GhostBottleneck().""" + + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): + """Initialize 'SPP' module with various pooling sizes for spatial pyramid pooling.""" + super().__init__(c1, c2, n, shortcut, g, e) + c_ = int(c2 * e) # hidden channels + self.m = nn.Sequential(*(GhostBottleneck(c_, c_) for _ in range(n))) + + +class GhostBottleneck(nn.Module): + """Ghost Bottleneck https://github.com/huawei-noah/ghostnet.""" + + def __init__(self, c1, c2, k=3, s=1): + """Initializes GhostBottleneck module with arguments ch_in, ch_out, kernel, stride.""" + super().__init__() + c_ = c2 // 2 + self.conv = nn.Sequential( + GhostConv(c1, c_, 1, 1), # pw + DWConv(c_, c_, k, s, act=False) if s == 2 else nn.Identity(), # dw + GhostConv(c_, c2, 1, 1, act=False), # pw-linear + ) + self.shortcut = ( + nn.Sequential(DWConv(c1, c1, k, s, act=False), Conv(c1, c2, 1, 1, act=False)) if s == 2 else nn.Identity() + ) + + def forward(self, x): + """Applies skip connection and concatenation to input tensor.""" + return self.conv(x) + self.shortcut(x) + + +class Bottleneck(nn.Module): + """Standard bottleneck.""" + + def __init__(self, c1, c2, shortcut=True, g=1, k=(3, 3), e=0.5): + """Initializes a bottleneck module with given input/output channels, shortcut option, group, kernels, and + expansion. + """ + super().__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = Conv(c1, c_, k[0], 1) + self.cv2 = Conv(c_, c2, k[1], 1, g=g) + self.add = shortcut and c1 == c2 + + def forward(self, x): + """'forward()' applies the YOLO FPN to input data.""" + return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x)) + + +class BottleneckCSP(nn.Module): + """CSP Bottleneck https://github.com/WongKinYiu/CrossStagePartialNetworks.""" + + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): + """Initializes the CSP Bottleneck given arguments for ch_in, ch_out, number, shortcut, groups, expansion.""" + super().__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = nn.Conv2d(c1, c_, 1, 1, bias=False) + self.cv3 = nn.Conv2d(c_, c_, 1, 1, bias=False) + self.cv4 = Conv(2 * c_, c2, 1, 1) + self.bn = nn.BatchNorm2d(2 * c_) # applied to cat(cv2, cv3) + self.act = nn.SiLU() + self.m = nn.Sequential(*(Bottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n))) + + def forward(self, x): + """Applies a CSP bottleneck with 3 convolutions.""" + y1 = self.cv3(self.m(self.cv1(x))) + y2 = self.cv2(x) + return self.cv4(self.act(self.bn(torch.cat((y1, y2), 1)))) + + +class ResNetBlock(nn.Module): + """ResNet block with standard convolution layers.""" + + def __init__(self, c1, c2, s=1, e=4): + """Initialize convolution with given parameters.""" + super().__init__() + c3 = e * c2 + self.cv1 = Conv(c1, c2, k=1, s=1, act=True) + self.cv2 = Conv(c2, c2, k=3, s=s, p=1, act=True) + self.cv3 = Conv(c2, c3, k=1, act=False) + self.shortcut = nn.Sequential(Conv(c1, c3, k=1, s=s, act=False)) if s != 1 or c1 != c3 else nn.Identity() + + def forward(self, x): + """Forward pass through the ResNet block.""" + return F.relu(self.cv3(self.cv2(self.cv1(x))) + self.shortcut(x)) + + +class ResNetLayer(nn.Module): + """ResNet layer with multiple ResNet blocks.""" + + def __init__(self, c1, c2, s=1, is_first=False, n=1, e=4): + """Initializes the ResNetLayer given arguments.""" + super().__init__() + self.is_first = is_first + + if self.is_first: + self.layer = nn.Sequential( + Conv(c1, c2, k=7, s=2, p=3, act=True), nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + ) + else: + blocks = [ResNetBlock(c1, c2, s, e=e)] + blocks.extend([ResNetBlock(e * c2, c2, 1, e=e) for _ in range(n - 1)]) + self.layer = nn.Sequential(*blocks) + + def forward(self, x): + """Forward pass through the ResNet layer.""" + return self.layer(x) + + +class MaxSigmoidAttnBlock(nn.Module): + """Max Sigmoid attention block.""" + + def __init__(self, c1, c2, nh=1, ec=128, gc=512, scale=False): + """Initializes MaxSigmoidAttnBlock with specified arguments.""" + super().__init__() + self.nh = nh + self.hc = c2 // nh + self.ec = Conv(c1, ec, k=1, act=False) if c1 != ec else None + self.gl = nn.Linear(gc, ec) + self.bias = nn.Parameter(torch.zeros(nh)) + self.proj_conv = Conv(c1, c2, k=3, s=1, act=False) + self.scale = nn.Parameter(torch.ones(1, nh, 1, 1)) if scale else 1.0 + + def forward(self, x, guide): + """Forward process.""" + bs, _, h, w = x.shape + + guide = self.gl(guide) + guide = guide.view(bs, -1, self.nh, self.hc) + embed = self.ec(x) if self.ec is not None else x + embed = embed.view(bs, self.nh, self.hc, h, w) + + aw = torch.einsum("bmchw,bnmc->bmhwn", embed, guide) + aw = aw.max(dim=-1)[0] + aw = aw / (self.hc**0.5) + aw = aw + self.bias[None, :, None, None] + aw = aw.sigmoid() * self.scale + + x = self.proj_conv(x) + x = x.view(bs, self.nh, -1, h, w) + x = x * aw.unsqueeze(2) + return x.view(bs, -1, h, w) + + +class C2fAttn(nn.Module): + """C2f module with an additional attn module.""" + + def __init__(self, c1, c2, n=1, ec=128, nh=1, gc=512, shortcut=False, g=1, e=0.5): + """Initialize CSP bottleneck layer with two convolutions with arguments ch_in, ch_out, number, shortcut, groups, + expansion. + """ + super().__init__() + self.c = int(c2 * e) # hidden channels + self.cv1 = Conv(c1, 2 * self.c, 1, 1) + self.cv2 = Conv((3 + n) * self.c, c2, 1) # optional act=FReLU(c2) + self.m = nn.ModuleList(Bottleneck(self.c, self.c, shortcut, g, k=((3, 3), (3, 3)), e=1.0) for _ in range(n)) + self.attn = MaxSigmoidAttnBlock(self.c, self.c, gc=gc, ec=ec, nh=nh) + + def forward(self, x, guide): + """Forward pass through C2f layer.""" + y = list(self.cv1(x).chunk(2, 1)) + y.extend(m(y[-1]) for m in self.m) + y.append(self.attn(y[-1], guide)) + return self.cv2(torch.cat(y, 1)) + + def forward_split(self, x, guide): + """Forward pass using split() instead of chunk().""" + y = list(self.cv1(x).split((self.c, self.c), 1)) + y.extend(m(y[-1]) for m in self.m) + y.append(self.attn(y[-1], guide)) + return self.cv2(torch.cat(y, 1)) + + +class ImagePoolingAttn(nn.Module): + """ImagePoolingAttn: Enhance the text embeddings with image-aware information.""" + + def __init__(self, ec=256, ch=(), ct=512, nh=8, k=3, scale=False): + """Initializes ImagePoolingAttn with specified arguments.""" + super().__init__() + + nf = len(ch) + self.query = nn.Sequential(nn.LayerNorm(ct), nn.Linear(ct, ec)) + self.key = nn.Sequential(nn.LayerNorm(ec), nn.Linear(ec, ec)) + self.value = nn.Sequential(nn.LayerNorm(ec), nn.Linear(ec, ec)) + self.proj = nn.Linear(ec, ct) + self.scale = nn.Parameter(torch.tensor([0.0]), requires_grad=True) if scale else 1.0 + self.projections = nn.ModuleList([nn.Conv2d(in_channels, ec, kernel_size=1) for in_channels in ch]) + self.im_pools = nn.ModuleList([nn.AdaptiveMaxPool2d((k, k)) for _ in range(nf)]) + self.ec = ec + self.nh = nh + self.nf = nf + self.hc = ec // nh + self.k = k + + def forward(self, x, text): + """Executes attention mechanism on input tensor x and guide tensor.""" + bs = x[0].shape[0] + assert len(x) == self.nf + num_patches = self.k**2 + x = [pool(proj(x)).view(bs, -1, num_patches) for (x, proj, pool) in zip(x, self.projections, self.im_pools)] + x = torch.cat(x, dim=-1).transpose(1, 2) + q = self.query(text) + k = self.key(x) + v = self.value(x) + + # q = q.reshape(1, text.shape[1], self.nh, self.hc).repeat(bs, 1, 1, 1) + q = q.reshape(bs, -1, self.nh, self.hc) + k = k.reshape(bs, -1, self.nh, self.hc) + v = v.reshape(bs, -1, self.nh, self.hc) + + aw = torch.einsum("bnmc,bkmc->bmnk", q, k) + aw = aw / (self.hc**0.5) + aw = F.softmax(aw, dim=-1) + + x = torch.einsum("bmnk,bkmc->bnmc", aw, v) + x = self.proj(x.reshape(bs, -1, self.ec)) + return x * self.scale + text + + +class ContrastiveHead(nn.Module): + """Contrastive Head for YOLO-World compute the region-text scores according to the similarity between image and text + features. + """ + + def __init__(self): + """Initializes ContrastiveHead with specified region-text similarity parameters.""" + super().__init__() + self.bias = nn.Parameter(torch.zeros([])) + self.logit_scale = nn.Parameter(torch.ones([]) * torch.tensor(1 / 0.07).log()) + + def forward(self, x, w): + """Forward function of contrastive learning.""" + x = F.normalize(x, dim=1, p=2) + w = F.normalize(w, dim=-1, p=2) + x = torch.einsum("bchw,bkc->bkhw", x, w) + return x * self.logit_scale.exp() + self.bias + + +class BNContrastiveHead(nn.Module): + """ + Batch Norm Contrastive Head for YOLO-World using batch norm instead of l2-normalization. + + Args: + embed_dims (int): Embed dimensions of text and image features. + """ + + def __init__(self, embed_dims: int): + """Initialize ContrastiveHead with region-text similarity parameters.""" + super().__init__() + self.norm = nn.BatchNorm2d(embed_dims) + self.bias = nn.Parameter(torch.zeros([])) + # use -1.0 is more stable + self.logit_scale = nn.Parameter(-1.0 * torch.ones([])) + + def forward(self, x, w): + """Forward function of contrastive learning.""" + x = self.norm(x) + w = F.normalize(w, dim=-1, p=2) + x = torch.einsum("bchw,bkc->bkhw", x, w) + return x * self.logit_scale.exp() + self.bias + + +class RepBottleneck(nn.Module): + """Rep bottleneck.""" + + def __init__(self, c1, c2, shortcut=True, g=1, k=(3, 3), e=0.5): + """Initializes a RepBottleneck module with customizable in/out channels, shortcut option, groups and expansion + ratio. + """ + super().__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = RepConv(c1, c_, k[0], 1) + self.cv2 = Conv(c_, c2, k[1], 1, g=g) + self.add = shortcut and c1 == c2 + + def forward(self, x): + """Forward pass through RepBottleneck layer.""" + return x + self.cv2(self.cv1(x)) if self.add else self.cv2(self.cv1(x)) + + +class RepCSP(nn.Module): + """Rep CSP Bottleneck with 3 convolutions.""" + + def __init__(self, c1, c2, n=1, shortcut=True, g=1, e=0.5): + """Initializes RepCSP layer with given channels, repetitions, shortcut, groups and expansion ratio.""" + super().__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = Conv(c1, c_, 1, 1) + self.cv2 = Conv(c1, c_, 1, 1) + self.cv3 = Conv(2 * c_, c2, 1) # optional act=FReLU(c2) + self.m = nn.Sequential(*(RepBottleneck(c_, c_, shortcut, g, e=1.0) for _ in range(n))) + + def forward(self, x): + """Forward pass through RepCSP layer.""" + return self.cv3(torch.cat((self.m(self.cv1(x)), self.cv2(x)), 1)) + + +class RepNCSPELAN4(nn.Module): + """CSP-ELAN.""" + + def __init__(self, c1, c2, c3, c4, n=1): + """Initializes CSP-ELAN layer with specified channel sizes, repetitions, and convolutions.""" + super().__init__() + self.c = c3 // 2 + self.cv1 = Conv(c1, c3, 1, 1) + self.cv2 = nn.Sequential(RepCSP(c3 // 2, c4, n), Conv(c4, c4, 3, 1)) + self.cv3 = nn.Sequential(RepCSP(c4, c4, n), Conv(c4, c4, 3, 1)) + self.cv4 = Conv(c3 + (2 * c4), c2, 1, 1) + + def forward(self, x): + """Forward pass through RepNCSPELAN4 layer.""" + y = list(self.cv1(x).chunk(2, 1)) + y.extend((m(y[-1])) for m in [self.cv2, self.cv3]) + return self.cv4(torch.cat(y, 1)) + + def forward_split(self, x): + """Forward pass using split() instead of chunk().""" + y = list(self.cv1(x).split((self.c, self.c), 1)) + y.extend(m(y[-1]) for m in [self.cv2, self.cv3]) + return self.cv4(torch.cat(y, 1)) + + +class ADown(nn.Module): + """ADown.""" + + def __init__(self, c1, c2): + """Initializes ADown module with convolution layers to downsample input from channels c1 to c2.""" + super().__init__() + self.c = c2 // 2 + self.cv1 = Conv(c1 // 2, self.c, 3, 2, 1) + self.cv2 = Conv(c1 // 2, self.c, 1, 1, 0) + + def forward(self, x): + """Forward pass through ADown layer.""" + x = torch.nn.functional.avg_pool2d(x, 2, 1, 0, False, True) + x1, x2 = x.chunk(2, 1) + x1 = self.cv1(x1) + x2 = torch.nn.functional.max_pool2d(x2, 3, 2, 1) + x2 = self.cv2(x2) + return torch.cat((x1, x2), 1) + + +class SPPELAN(nn.Module): + """SPP-ELAN.""" + + def __init__(self, c1, c2, c3, k=5): + """Initializes SPP-ELAN block with convolution and max pooling layers for spatial pyramid pooling.""" + super().__init__() + self.c = c3 + self.cv1 = Conv(c1, c3, 1, 1) + self.cv2 = nn.MaxPool2d(kernel_size=k, stride=1, padding=k // 2) + self.cv3 = nn.MaxPool2d(kernel_size=k, stride=1, padding=k // 2) + self.cv4 = nn.MaxPool2d(kernel_size=k, stride=1, padding=k // 2) + self.cv5 = Conv(4 * c3, c2, 1, 1) + + def forward(self, x): + """Forward pass through SPPELAN layer.""" + y = [self.cv1(x)] + y.extend(m(y[-1]) for m in [self.cv2, self.cv3, self.cv4]) + return self.cv5(torch.cat(y, 1)) + + +class Silence(nn.Module): + """Silence.""" + + def __init__(self): + """Initializes the Silence module.""" + super(Silence, self).__init__() + + def forward(self, x): + """Forward pass through Silence layer.""" + return x + + +class CBLinear(nn.Module): + """CBLinear.""" + + def __init__(self, c1, c2s, k=1, s=1, p=None, g=1): + """Initializes the CBLinear module, passing inputs unchanged.""" + super(CBLinear, self).__init__() + self.c2s = c2s + self.conv = nn.Conv2d(c1, sum(c2s), k, s, autopad(k, p), groups=g, bias=True) + + def forward(self, x): + """Forward pass through CBLinear layer.""" + outs = self.conv(x).split(self.c2s, dim=1) + return outs + + +class CBFuse(nn.Module): + """CBFuse.""" + + def __init__(self, idx): + """Initializes CBFuse module with layer index for selective feature fusion.""" + super(CBFuse, self).__init__() + self.idx = idx + + def forward(self, xs): + """Forward pass through CBFuse layer.""" + target_size = xs[-1].shape[2:] + res = [F.interpolate(x[self.idx[i]], size=target_size, mode="nearest") for i, x in enumerate(xs[:-1])] + out = torch.sum(torch.stack(res + xs[-1:]), dim=0) + return out + + +class RepVGGDW(torch.nn.Module): + def __init__(self, ed) -> None: + super().__init__() + self.conv = Conv(ed, ed, 7, 1, 3, g=ed, act=False) + self.conv1 = Conv(ed, ed, 3, 1, 1, g=ed, act=False) + self.dim = ed + self.act = nn.SiLU() + + def forward(self, x): + return self.act(self.conv(x) + self.conv1(x)) + + def forward_fuse(self, x): + return self.act(self.conv(x)) + + @torch.no_grad() + def fuse(self): + conv = fuse_conv_and_bn(self.conv.conv, self.conv.bn) + conv1 = fuse_conv_and_bn(self.conv1.conv, self.conv1.bn) + + conv_w = conv.weight + conv_b = conv.bias + conv1_w = conv1.weight + conv1_b = conv1.bias + + conv1_w = torch.nn.functional.pad(conv1_w, [2,2,2,2]) + + final_conv_w = conv_w + conv1_w + final_conv_b = conv_b + conv1_b + + conv.weight.data.copy_(final_conv_w) + conv.bias.data.copy_(final_conv_b) + + self.conv = conv + del self.conv1 + +class CIB(nn.Module): + """Standard bottleneck.""" + + def __init__(self, c1, c2, shortcut=True, e=0.5, lk=False): + """Initializes a bottleneck module with given input/output channels, shortcut option, group, kernels, and + expansion. + """ + super().__init__() + c_ = int(c2 * e) # hidden channels + self.cv1 = nn.Sequential( + Conv(c1, c1, 3, g=c1), + Conv(c1, 2 * c_, 1), + Conv(2 * c_, 2 * c_, 3, g=2 * c_) if not lk else RepVGGDW(2 * c_), + Conv(2 * c_, c2, 1), + Conv(c2, c2, 3, g=c2), + ) + + self.add = shortcut and c1 == c2 + + def forward(self, x): + """'forward()' applies the YOLO FPN to input data.""" + return x + self.cv1(x) if self.add else self.cv1(x) + +class C2fCIB(C2f): + """Faster Implementation of CSP Bottleneck with 2 convolutions.""" + + def __init__(self, c1, c2, n=1, shortcut=False, lk=False, g=1, e=0.5): + """Initialize CSP bottleneck layer with two convolutions with arguments ch_in, ch_out, number, shortcut, groups, + expansion. + """ + super().__init__(c1, c2, n, shortcut, g, e) + self.m = nn.ModuleList(CIB(self.c, self.c, shortcut, e=1.0, lk=lk) for _ in range(n)) + + +class Attention(nn.Module): + def __init__(self, dim, num_heads=8, + attn_ratio=0.5): + super().__init__() + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.key_dim = int(self.head_dim * attn_ratio) + self.scale = self.key_dim ** -0.5 + nh_kd = nh_kd = self.key_dim * num_heads + h = dim + nh_kd * 2 + self.qkv = Conv(dim, h, 1, act=False) + self.proj = Conv(dim, dim, 1, act=False) + self.pe = Conv(dim, dim, 3, 1, g=dim, act=False) + + def forward(self, x): + B, C, H, W = x.shape + N = H * W + qkv = self.qkv(x) + q, k, v = qkv.view(B, self.num_heads, self.key_dim*2 + self.head_dim, N).split([self.key_dim, self.key_dim, self.head_dim], dim=2) + + attn = ( + (q.transpose(-2, -1) @ k) * self.scale + ) + attn = attn.softmax(dim=-1) + x = (v @ attn.transpose(-2, -1)).view(B, C, H, W) + self.pe(v.reshape(B, C, H, W)) + x = self.proj(x) + return x + +class PSA(nn.Module): + + def __init__(self, c1, c2, e=0.5): + super().__init__() + assert(c1 == c2) + self.c = int(c1 * e) + self.cv1 = Conv(c1, 2 * self.c, 1, 1) + self.cv2 = Conv(2 * self.c, c1, 1) + + self.attn = Attention(self.c, attn_ratio=0.5, num_heads=self.c // 64) + self.ffn = nn.Sequential( + Conv(self.c, self.c*2, 1), + Conv(self.c*2, self.c, 1, act=False) + ) + + def forward(self, x): + a, b = self.cv1(x).split((self.c, self.c), dim=1) + b = b + self.attn(b) + b = b + self.ffn(b) + return self.cv2(torch.cat((a, b), 1)) + +class SCDown(nn.Module): + def __init__(self, c1, c2, k, s): + super().__init__() + self.cv1 = Conv(c1, c2, 1, 1) + self.cv2 = Conv(c2, c2, k=k, s=s, g=c2, act=False) + + def forward(self, x): + return self.cv2(self.cv1(x)) \ No newline at end of file diff --git a/ultralytics/nn/modules/conv.py b/ultralytics/nn/modules/conv.py new file mode 100644 index 0000000..399c422 --- /dev/null +++ b/ultralytics/nn/modules/conv.py @@ -0,0 +1,333 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +"""Convolution modules.""" + +import math + +import numpy as np +import torch +import torch.nn as nn + +__all__ = ( + "Conv", + "Conv2", + "LightConv", + "DWConv", + "DWConvTranspose2d", + "ConvTranspose", + "Focus", + "GhostConv", + "ChannelAttention", + "SpatialAttention", + "CBAM", + "Concat", + "RepConv", +) + + +def autopad(k, p=None, d=1): # kernel, padding, dilation + """Pad to 'same' shape outputs.""" + if d > 1: + k = d * (k - 1) + 1 if isinstance(k, int) else [d * (x - 1) + 1 for x in k] # actual kernel-size + if p is None: + p = k // 2 if isinstance(k, int) else [x // 2 for x in k] # auto-pad + return p + + +class Conv(nn.Module): + """Standard convolution with args(ch_in, ch_out, kernel, stride, padding, groups, dilation, activation).""" + + default_act = nn.SiLU() # default activation + + def __init__(self, c1, c2, k=1, s=1, p=None, g=1, d=1, act=True): + """Initialize Conv layer with given arguments including activation.""" + super().__init__() + self.conv = nn.Conv2d(c1, c2, k, s, autopad(k, p, d), groups=g, dilation=d, bias=False) + self.bn = nn.BatchNorm2d(c2) + self.act = self.default_act if act is True else act if isinstance(act, nn.Module) else nn.Identity() + + def forward(self, x): + """Apply convolution, batch normalization and activation to input tensor.""" + return self.act(self.bn(self.conv(x))) + + def forward_fuse(self, x): + """Perform transposed convolution of 2D data.""" + return self.act(self.conv(x)) + + +class Conv2(Conv): + """Simplified RepConv module with Conv fusing.""" + + def __init__(self, c1, c2, k=3, s=1, p=None, g=1, d=1, act=True): + """Initialize Conv layer with given arguments including activation.""" + super().__init__(c1, c2, k, s, p, g=g, d=d, act=act) + self.cv2 = nn.Conv2d(c1, c2, 1, s, autopad(1, p, d), groups=g, dilation=d, bias=False) # add 1x1 conv + + def forward(self, x): + """Apply convolution, batch normalization and activation to input tensor.""" + return self.act(self.bn(self.conv(x) + self.cv2(x))) + + def forward_fuse(self, x): + """Apply fused convolution, batch normalization and activation to input tensor.""" + return self.act(self.bn(self.conv(x))) + + def fuse_convs(self): + """Fuse parallel convolutions.""" + w = torch.zeros_like(self.conv.weight.data) + i = [x // 2 for x in w.shape[2:]] + w[:, :, i[0] : i[0] + 1, i[1] : i[1] + 1] = self.cv2.weight.data.clone() + self.conv.weight.data += w + self.__delattr__("cv2") + self.forward = self.forward_fuse + + +class LightConv(nn.Module): + """ + Light convolution with args(ch_in, ch_out, kernel). + + https://github.com/PaddlePaddle/PaddleDetection/blob/develop/ppdet/modeling/backbones/hgnet_v2.py + """ + + def __init__(self, c1, c2, k=1, act=nn.ReLU()): + """Initialize Conv layer with given arguments including activation.""" + super().__init__() + self.conv1 = Conv(c1, c2, 1, act=False) + self.conv2 = DWConv(c2, c2, k, act=act) + + def forward(self, x): + """Apply 2 convolutions to input tensor.""" + return self.conv2(self.conv1(x)) + + +class DWConv(Conv): + """Depth-wise convolution.""" + + def __init__(self, c1, c2, k=1, s=1, d=1, act=True): # ch_in, ch_out, kernel, stride, dilation, activation + """Initialize Depth-wise convolution with given parameters.""" + super().__init__(c1, c2, k, s, g=math.gcd(c1, c2), d=d, act=act) + + +class DWConvTranspose2d(nn.ConvTranspose2d): + """Depth-wise transpose convolution.""" + + def __init__(self, c1, c2, k=1, s=1, p1=0, p2=0): # ch_in, ch_out, kernel, stride, padding, padding_out + """Initialize DWConvTranspose2d class with given parameters.""" + super().__init__(c1, c2, k, s, p1, p2, groups=math.gcd(c1, c2)) + + +class ConvTranspose(nn.Module): + """Convolution transpose 2d layer.""" + + default_act = nn.SiLU() # default activation + + def __init__(self, c1, c2, k=2, s=2, p=0, bn=True, act=True): + """Initialize ConvTranspose2d layer with batch normalization and activation function.""" + super().__init__() + self.conv_transpose = nn.ConvTranspose2d(c1, c2, k, s, p, bias=not bn) + self.bn = nn.BatchNorm2d(c2) if bn else nn.Identity() + self.act = self.default_act if act is True else act if isinstance(act, nn.Module) else nn.Identity() + + def forward(self, x): + """Applies transposed convolutions, batch normalization and activation to input.""" + return self.act(self.bn(self.conv_transpose(x))) + + def forward_fuse(self, x): + """Applies activation and convolution transpose operation to input.""" + return self.act(self.conv_transpose(x)) + + +class Focus(nn.Module): + """Focus wh information into c-space.""" + + def __init__(self, c1, c2, k=1, s=1, p=None, g=1, act=True): + """Initializes Focus object with user defined channel, convolution, padding, group and activation values.""" + super().__init__() + self.conv = Conv(c1 * 4, c2, k, s, p, g, act=act) + # self.contract = Contract(gain=2) + + def forward(self, x): + """ + Applies convolution to concatenated tensor and returns the output. + + Input shape is (b,c,w,h) and output shape is (b,4c,w/2,h/2). + """ + return self.conv(torch.cat((x[..., ::2, ::2], x[..., 1::2, ::2], x[..., ::2, 1::2], x[..., 1::2, 1::2]), 1)) + # return self.conv(self.contract(x)) + + +class GhostConv(nn.Module): + """Ghost Convolution https://github.com/huawei-noah/ghostnet.""" + + def __init__(self, c1, c2, k=1, s=1, g=1, act=True): + """Initializes the GhostConv object with input channels, output channels, kernel size, stride, groups and + activation. + """ + super().__init__() + c_ = c2 // 2 # hidden channels + self.cv1 = Conv(c1, c_, k, s, None, g, act=act) + self.cv2 = Conv(c_, c_, 5, 1, None, c_, act=act) + + def forward(self, x): + """Forward propagation through a Ghost Bottleneck layer with skip connection.""" + y = self.cv1(x) + return torch.cat((y, self.cv2(y)), 1) + + +class RepConv(nn.Module): + """ + RepConv is a basic rep-style block, including training and deploy status. + + This module is used in RT-DETR. + Based on https://github.com/DingXiaoH/RepVGG/blob/main/repvgg.py + """ + + default_act = nn.SiLU() # default activation + + def __init__(self, c1, c2, k=3, s=1, p=1, g=1, d=1, act=True, bn=False, deploy=False): + """Initializes Light Convolution layer with inputs, outputs & optional activation function.""" + super().__init__() + assert k == 3 and p == 1 + self.g = g + self.c1 = c1 + self.c2 = c2 + self.act = self.default_act if act is True else act if isinstance(act, nn.Module) else nn.Identity() + + self.bn = nn.BatchNorm2d(num_features=c1) if bn and c2 == c1 and s == 1 else None + self.conv1 = Conv(c1, c2, k, s, p=p, g=g, act=False) + self.conv2 = Conv(c1, c2, 1, s, p=(p - k // 2), g=g, act=False) + + def forward_fuse(self, x): + """Forward process.""" + return self.act(self.conv(x)) + + def forward(self, x): + """Forward process.""" + id_out = 0 if self.bn is None else self.bn(x) + return self.act(self.conv1(x) + self.conv2(x) + id_out) + + def get_equivalent_kernel_bias(self): + """Returns equivalent kernel and bias by adding 3x3 kernel, 1x1 kernel and identity kernel with their biases.""" + kernel3x3, bias3x3 = self._fuse_bn_tensor(self.conv1) + kernel1x1, bias1x1 = self._fuse_bn_tensor(self.conv2) + kernelid, biasid = self._fuse_bn_tensor(self.bn) + return kernel3x3 + self._pad_1x1_to_3x3_tensor(kernel1x1) + kernelid, bias3x3 + bias1x1 + biasid + + def _pad_1x1_to_3x3_tensor(self, kernel1x1): + """Pads a 1x1 tensor to a 3x3 tensor.""" + if kernel1x1 is None: + return 0 + else: + return torch.nn.functional.pad(kernel1x1, [1, 1, 1, 1]) + + def _fuse_bn_tensor(self, branch): + """Generates appropriate kernels and biases for convolution by fusing branches of the neural network.""" + if branch is None: + return 0, 0 + if isinstance(branch, Conv): + kernel = branch.conv.weight + running_mean = branch.bn.running_mean + running_var = branch.bn.running_var + gamma = branch.bn.weight + beta = branch.bn.bias + eps = branch.bn.eps + elif isinstance(branch, nn.BatchNorm2d): + if not hasattr(self, "id_tensor"): + input_dim = self.c1 // self.g + kernel_value = np.zeros((self.c1, input_dim, 3, 3), dtype=np.float32) + for i in range(self.c1): + kernel_value[i, i % input_dim, 1, 1] = 1 + self.id_tensor = torch.from_numpy(kernel_value).to(branch.weight.device) + kernel = self.id_tensor + running_mean = branch.running_mean + running_var = branch.running_var + gamma = branch.weight + beta = branch.bias + eps = branch.eps + std = (running_var + eps).sqrt() + t = (gamma / std).reshape(-1, 1, 1, 1) + return kernel * t, beta - running_mean * gamma / std + + def fuse_convs(self): + """Combines two convolution layers into a single layer and removes unused attributes from the class.""" + if hasattr(self, "conv"): + return + kernel, bias = self.get_equivalent_kernel_bias() + self.conv = nn.Conv2d( + in_channels=self.conv1.conv.in_channels, + out_channels=self.conv1.conv.out_channels, + kernel_size=self.conv1.conv.kernel_size, + stride=self.conv1.conv.stride, + padding=self.conv1.conv.padding, + dilation=self.conv1.conv.dilation, + groups=self.conv1.conv.groups, + bias=True, + ).requires_grad_(False) + self.conv.weight.data = kernel + self.conv.bias.data = bias + for para in self.parameters(): + para.detach_() + self.__delattr__("conv1") + self.__delattr__("conv2") + if hasattr(self, "nm"): + self.__delattr__("nm") + if hasattr(self, "bn"): + self.__delattr__("bn") + if hasattr(self, "id_tensor"): + self.__delattr__("id_tensor") + + +class ChannelAttention(nn.Module): + """Channel-attention module https://github.com/open-mmlab/mmdetection/tree/v3.0.0rc1/configs/rtmdet.""" + + def __init__(self, channels: int) -> None: + """Initializes the class and sets the basic configurations and instance variables required.""" + super().__init__() + self.pool = nn.AdaptiveAvgPool2d(1) + self.fc = nn.Conv2d(channels, channels, 1, 1, 0, bias=True) + self.act = nn.Sigmoid() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Applies forward pass using activation on convolutions of the input, optionally using batch normalization.""" + return x * self.act(self.fc(self.pool(x))) + + +class SpatialAttention(nn.Module): + """Spatial-attention module.""" + + def __init__(self, kernel_size=7): + """Initialize Spatial-attention module with kernel size argument.""" + super().__init__() + assert kernel_size in (3, 7), "kernel size must be 3 or 7" + padding = 3 if kernel_size == 7 else 1 + self.cv1 = nn.Conv2d(2, 1, kernel_size, padding=padding, bias=False) + self.act = nn.Sigmoid() + + def forward(self, x): + """Apply channel and spatial attention on input for feature recalibration.""" + return x * self.act(self.cv1(torch.cat([torch.mean(x, 1, keepdim=True), torch.max(x, 1, keepdim=True)[0]], 1))) + + +class CBAM(nn.Module): + """Convolutional Block Attention Module.""" + + def __init__(self, c1, kernel_size=7): + """Initialize CBAM with given input channel (c1) and kernel size.""" + super().__init__() + self.channel_attention = ChannelAttention(c1) + self.spatial_attention = SpatialAttention(kernel_size) + + def forward(self, x): + """Applies the forward pass through C1 module.""" + return self.spatial_attention(self.channel_attention(x)) + + +class Concat(nn.Module): + """Concatenate a list of tensors along dimension.""" + + def __init__(self, dimension=1): + """Concatenates a list of tensors along a specified dimension.""" + super().__init__() + self.d = dimension + + def forward(self, x): + """Forward pass for the YOLOv8 mask Proto module.""" + return torch.cat(x, self.d) diff --git a/ultralytics/nn/modules/head.py b/ultralytics/nn/modules/head.py new file mode 100644 index 0000000..a9c5d9e --- /dev/null +++ b/ultralytics/nn/modules/head.py @@ -0,0 +1,535 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +"""Model head modules.""" + +import math + +import torch +import torch.nn as nn +from torch.nn.init import constant_, xavier_uniform_ + +from ultralytics.utils.tal import TORCH_1_10, dist2bbox, dist2rbox, make_anchors +from .block import DFL, Proto, ContrastiveHead, BNContrastiveHead +from .conv import Conv +from .transformer import MLP, DeformableTransformerDecoder, DeformableTransformerDecoderLayer +from .utils import bias_init_with_prob, linear_init +import copy +from ultralytics.utils import ops + +__all__ = "Detect", "Segment", "Pose", "Classify", "OBB", "RTDETRDecoder" + + +class Detect(nn.Module): + """YOLOv8 Detect head for detection models.""" + + dynamic = False # force grid reconstruction + export = False # export mode + shape = None + anchors = torch.empty(0) # init + strides = torch.empty(0) # init + + def __init__(self, nc=80, ch=()): + """Initializes the YOLOv8 detection layer with specified number of classes and channels.""" + super().__init__() + self.nc = nc # number of classes + self.nl = len(ch) # number of detection layers + self.reg_max = 16 # DFL channels (ch[0] // 16 to scale 4/8/12/16/20 for n/s/m/l/x) + self.no = nc + self.reg_max * 4 # number of outputs per anchor + self.stride = torch.zeros(self.nl) # strides computed during build + c2, c3 = max((16, ch[0] // 4, self.reg_max * 4)), max(ch[0], min(self.nc, 100)) # channels + self.cv2 = nn.ModuleList( + nn.Sequential(Conv(x, c2, 3), Conv(c2, c2, 3), nn.Conv2d(c2, 4 * self.reg_max, 1)) for x in ch + ) + self.cv3 = nn.ModuleList(nn.Sequential(Conv(x, c3, 3), Conv(c3, c3, 3), nn.Conv2d(c3, self.nc, 1)) for x in ch) + self.dfl = DFL(self.reg_max) if self.reg_max > 1 else nn.Identity() + + def inference(self, x): + # Inference path + shape = x[0].shape # BCHW + x_cat = torch.cat([xi.view(shape[0], self.no, -1) for xi in x], 2) + if self.dynamic or self.shape != shape: + self.anchors, self.strides = (x.transpose(0, 1) for x in make_anchors(x, self.stride, 0.5)) + self.shape = shape + + if self.export and self.format in ("saved_model", "pb", "tflite", "edgetpu", "tfjs"): # avoid TF FlexSplitV ops + box = x_cat[:, : self.reg_max * 4] + cls = x_cat[:, self.reg_max * 4 :] + else: + box, cls = x_cat.split((self.reg_max * 4, self.nc), 1) + + if self.export and self.format in ("tflite", "edgetpu"): + # Precompute normalization factor to increase numerical stability + # See https://github.com/ultralytics/ultralytics/issues/7371 + grid_h = shape[2] + grid_w = shape[3] + grid_size = torch.tensor([grid_w, grid_h, grid_w, grid_h], device=box.device).reshape(1, 4, 1) + norm = self.strides / (self.stride[0] * grid_size) + dbox = self.decode_bboxes(self.dfl(box) * norm, self.anchors.unsqueeze(0) * norm[:, :2]) + else: + dbox = self.decode_bboxes(self.dfl(box), self.anchors.unsqueeze(0)) * self.strides + + y = torch.cat((dbox, cls.sigmoid()), 1) + return y if self.export else (y, x) + + def forward_feat(self, x, cv2, cv3): + y = [] + for i in range(self.nl): + y.append(torch.cat((cv2[i](x[i]), cv3[i](x[i])), 1)) + return y + + def forward(self, x): + """Concatenates and returns predicted bounding boxes and class probabilities.""" + y = self.forward_feat(x, self.cv2, self.cv3) + + if self.training: + return y + + return self.inference(y) + + def bias_init(self): + """Initialize Detect() biases, WARNING: requires stride availability.""" + m = self # self.model[-1] # Detect() module + # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1 + # ncf = math.log(0.6 / (m.nc - 0.999999)) if cf is None else torch.log(cf / cf.sum()) # nominal class frequency + for a, b, s in zip(m.cv2, m.cv3, m.stride): # from + a[-1].bias.data[:] = 1.0 # box + b[-1].bias.data[: m.nc] = math.log(5 / m.nc / (640 / s) ** 2) # cls (.01 objects, 80 classes, 640 img) + + def decode_bboxes(self, bboxes, anchors): + """Decode bounding boxes.""" + if self.export: + return dist2bbox(bboxes, anchors, xywh=False, dim=1) + return dist2bbox(bboxes, anchors, xywh=True, dim=1) + + +class Segment(Detect): + """YOLOv8 Segment head for segmentation models.""" + + def __init__(self, nc=80, nm=32, npr=256, ch=()): + """Initialize the YOLO model attributes such as the number of masks, prototypes, and the convolution layers.""" + super().__init__(nc, ch) + self.nm = nm # number of masks + self.npr = npr # number of protos + self.proto = Proto(ch[0], self.npr, self.nm) # protos + self.detect = Detect.forward + + c4 = max(ch[0] // 4, self.nm) + self.cv4 = nn.ModuleList(nn.Sequential(Conv(x, c4, 3), Conv(c4, c4, 3), nn.Conv2d(c4, self.nm, 1)) for x in ch) + + def forward(self, x): + """Return model outputs and mask coefficients if training, otherwise return outputs and mask coefficients.""" + p = self.proto(x[0]) # mask protos + bs = p.shape[0] # batch size + + mc = torch.cat([self.cv4[i](x[i]).view(bs, self.nm, -1) for i in range(self.nl)], 2) # mask coefficients + x = self.detect(self, x) + if self.training: + return x, mc, p + return (torch.cat([x, mc], 1), p) if self.export else (torch.cat([x[0], mc], 1), (x[1], mc, p)) + + +class OBB(Detect): + """YOLOv8 OBB detection head for detection with rotation models.""" + + def __init__(self, nc=80, ne=1, ch=()): + """Initialize OBB with number of classes `nc` and layer channels `ch`.""" + super().__init__(nc, ch) + self.ne = ne # number of extra parameters + self.detect = Detect.forward + + c4 = max(ch[0] // 4, self.ne) + self.cv4 = nn.ModuleList(nn.Sequential(Conv(x, c4, 3), Conv(c4, c4, 3), nn.Conv2d(c4, self.ne, 1)) for x in ch) + + def forward(self, x): + """Concatenates and returns predicted bounding boxes and class probabilities.""" + bs = x[0].shape[0] # batch size + angle = torch.cat([self.cv4[i](x[i]).view(bs, self.ne, -1) for i in range(self.nl)], 2) # OBB theta logits + # NOTE: set `angle` as an attribute so that `decode_bboxes` could use it. + angle = (angle.sigmoid() - 0.25) * math.pi # [-pi/4, 3pi/4] + # angle = angle.sigmoid() * math.pi / 2 # [0, pi/2] + if not self.training: + self.angle = angle + x = self.detect(self, x) + if self.training: + return x, angle + return torch.cat([x, angle], 1) if self.export else (torch.cat([x[0], angle], 1), (x[1], angle)) + + def decode_bboxes(self, bboxes, anchors): + """Decode rotated bounding boxes.""" + return dist2rbox(bboxes, self.angle, anchors, dim=1) + + +class Pose(Detect): + """YOLOv8 Pose head for keypoints models.""" + + def __init__(self, nc=80, kpt_shape=(17, 3), ch=()): + """Initialize YOLO network with default parameters and Convolutional Layers.""" + super().__init__(nc, ch) + self.kpt_shape = kpt_shape # number of keypoints, number of dims (2 for x,y or 3 for x,y,visible) + self.nk = kpt_shape[0] * kpt_shape[1] # number of keypoints total + self.detect = Detect.forward + + c4 = max(ch[0] // 4, self.nk) + self.cv4 = nn.ModuleList(nn.Sequential(Conv(x, c4, 3), Conv(c4, c4, 3), nn.Conv2d(c4, self.nk, 1)) for x in ch) + + def forward(self, x): + """Perform forward pass through YOLO model and return predictions.""" + bs = x[0].shape[0] # batch size + kpt = torch.cat([self.cv4[i](x[i]).view(bs, self.nk, -1) for i in range(self.nl)], -1) # (bs, 17*3, h*w) + x = self.detect(self, x) + if self.training: + return x, kpt + pred_kpt = self.kpts_decode(bs, kpt) + return torch.cat([x, pred_kpt], 1) if self.export else (torch.cat([x[0], pred_kpt], 1), (x[1], kpt)) + + def kpts_decode(self, bs, kpts): + """Decodes keypoints.""" + ndim = self.kpt_shape[1] + if self.export: # required for TFLite export to avoid 'PLACEHOLDER_FOR_GREATER_OP_CODES' bug + y = kpts.view(bs, *self.kpt_shape, -1) + a = (y[:, :, :2] * 2.0 + (self.anchors - 0.5)) * self.strides + if ndim == 3: + a = torch.cat((a, y[:, :, 2:3].sigmoid()), 2) + return a.view(bs, self.nk, -1) + else: + y = kpts.clone() + if ndim == 3: + y[:, 2::3] = y[:, 2::3].sigmoid() # sigmoid (WARNING: inplace .sigmoid_() Apple MPS bug) + y[:, 0::ndim] = (y[:, 0::ndim] * 2.0 + (self.anchors[0] - 0.5)) * self.strides + y[:, 1::ndim] = (y[:, 1::ndim] * 2.0 + (self.anchors[1] - 0.5)) * self.strides + return y + + +class Classify(nn.Module): + """YOLOv8 classification head, i.e. x(b,c1,20,20) to x(b,c2).""" + + def __init__(self, c1, c2, k=1, s=1, p=None, g=1): + """Initializes YOLOv8 classification head with specified input and output channels, kernel size, stride, + padding, and groups. + """ + super().__init__() + c_ = 1280 # efficientnet_b0 size + self.conv = Conv(c1, c_, k, s, p, g) + self.pool = nn.AdaptiveAvgPool2d(1) # to x(b,c_,1,1) + self.drop = nn.Dropout(p=0.0, inplace=True) + self.linear = nn.Linear(c_, c2) # to x(b,c2) + + def forward(self, x): + """Performs a forward pass of the YOLO model on input image data.""" + if isinstance(x, list): + x = torch.cat(x, 1) + x = self.linear(self.drop(self.pool(self.conv(x)).flatten(1))) + return x if self.training else x.softmax(1) + + +class WorldDetect(Detect): + def __init__(self, nc=80, embed=512, with_bn=False, ch=()): + """Initialize YOLOv8 detection layer with nc classes and layer channels ch.""" + super().__init__(nc, ch) + c3 = max(ch[0], min(self.nc, 100)) + self.cv3 = nn.ModuleList(nn.Sequential(Conv(x, c3, 3), Conv(c3, c3, 3), nn.Conv2d(c3, embed, 1)) for x in ch) + self.cv4 = nn.ModuleList(BNContrastiveHead(embed) if with_bn else ContrastiveHead() for _ in ch) + + def forward(self, x, text): + """Concatenates and returns predicted bounding boxes and class probabilities.""" + for i in range(self.nl): + x[i] = torch.cat((self.cv2[i](x[i]), self.cv4[i](self.cv3[i](x[i]), text)), 1) + if self.training: + return x + + # Inference path + shape = x[0].shape # BCHW + x_cat = torch.cat([xi.view(shape[0], self.nc + self.reg_max * 4, -1) for xi in x], 2) + if self.dynamic or self.shape != shape: + self.anchors, self.strides = (x.transpose(0, 1) for x in make_anchors(x, self.stride, 0.5)) + self.shape = shape + + if self.export and self.format in ("saved_model", "pb", "tflite", "edgetpu", "tfjs"): # avoid TF FlexSplitV ops + box = x_cat[:, : self.reg_max * 4] + cls = x_cat[:, self.reg_max * 4 :] + else: + box, cls = x_cat.split((self.reg_max * 4, self.nc), 1) + + if self.export and self.format in ("tflite", "edgetpu"): + # Precompute normalization factor to increase numerical stability + # See https://github.com/ultralytics/ultralytics/issues/7371 + grid_h = shape[2] + grid_w = shape[3] + grid_size = torch.tensor([grid_w, grid_h, grid_w, grid_h], device=box.device).reshape(1, 4, 1) + norm = self.strides / (self.stride[0] * grid_size) + dbox = self.decode_bboxes(self.dfl(box) * norm, self.anchors.unsqueeze(0) * norm[:, :2]) + else: + dbox = self.decode_bboxes(self.dfl(box), self.anchors.unsqueeze(0)) * self.strides + + y = torch.cat((dbox, cls.sigmoid()), 1) + return y if self.export else (y, x) + + +class RTDETRDecoder(nn.Module): + """ + Real-Time Deformable Transformer Decoder (RTDETRDecoder) module for object detection. + + This decoder module utilizes Transformer architecture along with deformable convolutions to predict bounding boxes + and class labels for objects in an image. It integrates features from multiple layers and runs through a series of + Transformer decoder layers to output the final predictions. + """ + + export = False # export mode + + def __init__( + self, + nc=80, + ch=(512, 1024, 2048), + hd=256, # hidden dim + nq=300, # num queries + ndp=4, # num decoder points + nh=8, # num head + ndl=6, # num decoder layers + d_ffn=1024, # dim of feedforward + dropout=0.0, + act=nn.ReLU(), + eval_idx=-1, + # Training args + nd=100, # num denoising + label_noise_ratio=0.5, + box_noise_scale=1.0, + learnt_init_query=False, + ): + """ + Initializes the RTDETRDecoder module with the given parameters. + + Args: + nc (int): Number of classes. Default is 80. + ch (tuple): Channels in the backbone feature maps. Default is (512, 1024, 2048). + hd (int): Dimension of hidden layers. Default is 256. + nq (int): Number of query points. Default is 300. + ndp (int): Number of decoder points. Default is 4. + nh (int): Number of heads in multi-head attention. Default is 8. + ndl (int): Number of decoder layers. Default is 6. + d_ffn (int): Dimension of the feed-forward networks. Default is 1024. + dropout (float): Dropout rate. Default is 0. + act (nn.Module): Activation function. Default is nn.ReLU. + eval_idx (int): Evaluation index. Default is -1. + nd (int): Number of denoising. Default is 100. + label_noise_ratio (float): Label noise ratio. Default is 0.5. + box_noise_scale (float): Box noise scale. Default is 1.0. + learnt_init_query (bool): Whether to learn initial query embeddings. Default is False. + """ + super().__init__() + self.hidden_dim = hd + self.nhead = nh + self.nl = len(ch) # num level + self.nc = nc + self.num_queries = nq + self.num_decoder_layers = ndl + + # Backbone feature projection + self.input_proj = nn.ModuleList(nn.Sequential(nn.Conv2d(x, hd, 1, bias=False), nn.BatchNorm2d(hd)) for x in ch) + # NOTE: simplified version but it's not consistent with .pt weights. + # self.input_proj = nn.ModuleList(Conv(x, hd, act=False) for x in ch) + + # Transformer module + decoder_layer = DeformableTransformerDecoderLayer(hd, nh, d_ffn, dropout, act, self.nl, ndp) + self.decoder = DeformableTransformerDecoder(hd, decoder_layer, ndl, eval_idx) + + # Denoising part + self.denoising_class_embed = nn.Embedding(nc, hd) + self.num_denoising = nd + self.label_noise_ratio = label_noise_ratio + self.box_noise_scale = box_noise_scale + + # Decoder embedding + self.learnt_init_query = learnt_init_query + if learnt_init_query: + self.tgt_embed = nn.Embedding(nq, hd) + self.query_pos_head = MLP(4, 2 * hd, hd, num_layers=2) + + # Encoder head + self.enc_output = nn.Sequential(nn.Linear(hd, hd), nn.LayerNorm(hd)) + self.enc_score_head = nn.Linear(hd, nc) + self.enc_bbox_head = MLP(hd, hd, 4, num_layers=3) + + # Decoder head + self.dec_score_head = nn.ModuleList([nn.Linear(hd, nc) for _ in range(ndl)]) + self.dec_bbox_head = nn.ModuleList([MLP(hd, hd, 4, num_layers=3) for _ in range(ndl)]) + + self._reset_parameters() + + def forward(self, x, batch=None): + """Runs the forward pass of the module, returning bounding box and classification scores for the input.""" + from ultralytics.models.utils.ops import get_cdn_group + + # Input projection and embedding + feats, shapes = self._get_encoder_input(x) + + # Prepare denoising training + dn_embed, dn_bbox, attn_mask, dn_meta = get_cdn_group( + batch, + self.nc, + self.num_queries, + self.denoising_class_embed.weight, + self.num_denoising, + self.label_noise_ratio, + self.box_noise_scale, + self.training, + ) + + embed, refer_bbox, enc_bboxes, enc_scores = self._get_decoder_input(feats, shapes, dn_embed, dn_bbox) + + # Decoder + dec_bboxes, dec_scores = self.decoder( + embed, + refer_bbox, + feats, + shapes, + self.dec_bbox_head, + self.dec_score_head, + self.query_pos_head, + attn_mask=attn_mask, + ) + x = dec_bboxes, dec_scores, enc_bboxes, enc_scores, dn_meta + if self.training: + return x + # (bs, 300, 4+nc) + y = torch.cat((dec_bboxes.squeeze(0), dec_scores.squeeze(0).sigmoid()), -1) + return y if self.export else (y, x) + + def _generate_anchors(self, shapes, grid_size=0.05, dtype=torch.float32, device="cpu", eps=1e-2): + """Generates anchor bounding boxes for given shapes with specific grid size and validates them.""" + anchors = [] + for i, (h, w) in enumerate(shapes): + sy = torch.arange(end=h, dtype=dtype, device=device) + sx = torch.arange(end=w, dtype=dtype, device=device) + grid_y, grid_x = torch.meshgrid(sy, sx, indexing="ij") if TORCH_1_10 else torch.meshgrid(sy, sx) + grid_xy = torch.stack([grid_x, grid_y], -1) # (h, w, 2) + + valid_WH = torch.tensor([w, h], dtype=dtype, device=device) + grid_xy = (grid_xy.unsqueeze(0) + 0.5) / valid_WH # (1, h, w, 2) + wh = torch.ones_like(grid_xy, dtype=dtype, device=device) * grid_size * (2.0**i) + anchors.append(torch.cat([grid_xy, wh], -1).view(-1, h * w, 4)) # (1, h*w, 4) + + anchors = torch.cat(anchors, 1) # (1, h*w*nl, 4) + valid_mask = ((anchors > eps) & (anchors < 1 - eps)).all(-1, keepdim=True) # 1, h*w*nl, 1 + anchors = torch.log(anchors / (1 - anchors)) + anchors = anchors.masked_fill(~valid_mask, float("inf")) + return anchors, valid_mask + + def _get_encoder_input(self, x): + """Processes and returns encoder inputs by getting projection features from input and concatenating them.""" + # Get projection features + x = [self.input_proj[i](feat) for i, feat in enumerate(x)] + # Get encoder inputs + feats = [] + shapes = [] + for feat in x: + h, w = feat.shape[2:] + # [b, c, h, w] -> [b, h*w, c] + feats.append(feat.flatten(2).permute(0, 2, 1)) + # [nl, 2] + shapes.append([h, w]) + + # [b, h*w, c] + feats = torch.cat(feats, 1) + return feats, shapes + + def _get_decoder_input(self, feats, shapes, dn_embed=None, dn_bbox=None): + """Generates and prepares the input required for the decoder from the provided features and shapes.""" + bs = feats.shape[0] + # Prepare input for decoder + anchors, valid_mask = self._generate_anchors(shapes, dtype=feats.dtype, device=feats.device) + features = self.enc_output(valid_mask * feats) # bs, h*w, 256 + + enc_outputs_scores = self.enc_score_head(features) # (bs, h*w, nc) + + # Query selection + # (bs, num_queries) + topk_ind = torch.topk(enc_outputs_scores.max(-1).values, self.num_queries, dim=1).indices.view(-1) + # (bs, num_queries) + batch_ind = torch.arange(end=bs, dtype=topk_ind.dtype).unsqueeze(-1).repeat(1, self.num_queries).view(-1) + + # (bs, num_queries, 256) + top_k_features = features[batch_ind, topk_ind].view(bs, self.num_queries, -1) + # (bs, num_queries, 4) + top_k_anchors = anchors[:, topk_ind].view(bs, self.num_queries, -1) + + # Dynamic anchors + static content + refer_bbox = self.enc_bbox_head(top_k_features) + top_k_anchors + + enc_bboxes = refer_bbox.sigmoid() + if dn_bbox is not None: + refer_bbox = torch.cat([dn_bbox, refer_bbox], 1) + enc_scores = enc_outputs_scores[batch_ind, topk_ind].view(bs, self.num_queries, -1) + + embeddings = self.tgt_embed.weight.unsqueeze(0).repeat(bs, 1, 1) if self.learnt_init_query else top_k_features + if self.training: + refer_bbox = refer_bbox.detach() + if not self.learnt_init_query: + embeddings = embeddings.detach() + if dn_embed is not None: + embeddings = torch.cat([dn_embed, embeddings], 1) + + return embeddings, refer_bbox, enc_bboxes, enc_scores + + # TODO + def _reset_parameters(self): + """Initializes or resets the parameters of the model's various components with predefined weights and biases.""" + # Class and bbox head init + bias_cls = bias_init_with_prob(0.01) / 80 * self.nc + # NOTE: the weight initialization in `linear_init` would cause NaN when training with custom datasets. + # linear_init(self.enc_score_head) + constant_(self.enc_score_head.bias, bias_cls) + constant_(self.enc_bbox_head.layers[-1].weight, 0.0) + constant_(self.enc_bbox_head.layers[-1].bias, 0.0) + for cls_, reg_ in zip(self.dec_score_head, self.dec_bbox_head): + # linear_init(cls_) + constant_(cls_.bias, bias_cls) + constant_(reg_.layers[-1].weight, 0.0) + constant_(reg_.layers[-1].bias, 0.0) + + linear_init(self.enc_output[0]) + xavier_uniform_(self.enc_output[0].weight) + if self.learnt_init_query: + xavier_uniform_(self.tgt_embed.weight) + xavier_uniform_(self.query_pos_head.layers[0].weight) + xavier_uniform_(self.query_pos_head.layers[1].weight) + for layer in self.input_proj: + xavier_uniform_(layer[0].weight) + +class v10Detect(Detect): + + max_det = 300 + + def __init__(self, nc=80, ch=()): + super().__init__(nc, ch) + c3 = max(ch[0], min(self.nc, 100)) # channels + self.cv3 = nn.ModuleList(nn.Sequential(nn.Sequential(Conv(x, x, 3, g=x), Conv(x, c3, 1)), \ + nn.Sequential(Conv(c3, c3, 3, g=c3), Conv(c3, c3, 1)), \ + nn.Conv2d(c3, self.nc, 1)) for i, x in enumerate(ch)) + + self.one2one_cv2 = copy.deepcopy(self.cv2) + self.one2one_cv3 = copy.deepcopy(self.cv3) + + def forward(self, x): + one2one = self.forward_feat([xi.detach() for xi in x], self.one2one_cv2, self.one2one_cv3) + if not self.export: + one2many = super().forward(x) + + if not self.training: + one2one = self.inference(one2one) + if not self.export: + return {"one2many": one2many, "one2one": one2one} + else: + assert(self.max_det != -1) + boxes, scores, labels = ops.v10postprocess(one2one.permute(0, 2, 1), self.max_det, self.nc) + return torch.cat([boxes, scores.unsqueeze(-1), labels.unsqueeze(-1).to(boxes.dtype)], dim=-1) + else: + return {"one2many": one2many, "one2one": one2one} + + def bias_init(self): + super().bias_init() + """Initialize Detect() biases, WARNING: requires stride availability.""" + m = self # self.model[-1] # Detect() module + # cf = torch.bincount(torch.tensor(np.concatenate(dataset.labels, 0)[:, 0]).long(), minlength=nc) + 1 + # ncf = math.log(0.6 / (m.nc - 0.999999)) if cf is None else torch.log(cf / cf.sum()) # nominal class frequency + for a, b, s in zip(m.one2one_cv2, m.one2one_cv3, m.stride): # from + a[-1].bias.data[:] = 1.0 # box + b[-1].bias.data[: m.nc] = math.log(5 / m.nc / (640 / s) ** 2) # cls (.01 objects, 80 classes, 640 img) diff --git a/ultralytics/nn/modules/transformer.py b/ultralytics/nn/modules/transformer.py new file mode 100644 index 0000000..062c609 --- /dev/null +++ b/ultralytics/nn/modules/transformer.py @@ -0,0 +1,426 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +"""Transformer modules.""" + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn.init import constant_, xavier_uniform_ + +from .conv import Conv +from .utils import _get_clones, inverse_sigmoid, multi_scale_deformable_attn_pytorch + +__all__ = ( + "TransformerEncoderLayer", + "TransformerLayer", + "TransformerBlock", + "MLPBlock", + "LayerNorm2d", + "AIFI", + "DeformableTransformerDecoder", + "DeformableTransformerDecoderLayer", + "MSDeformAttn", + "MLP", +) + + +class TransformerEncoderLayer(nn.Module): + """Defines a single layer of the transformer encoder.""" + + def __init__(self, c1, cm=2048, num_heads=8, dropout=0.0, act=nn.GELU(), normalize_before=False): + """Initialize the TransformerEncoderLayer with specified parameters.""" + super().__init__() + from ...utils.torch_utils import TORCH_1_9 + + if not TORCH_1_9: + raise ModuleNotFoundError( + "TransformerEncoderLayer() requires torch>=1.9 to use nn.MultiheadAttention(batch_first=True)." + ) + self.ma = nn.MultiheadAttention(c1, num_heads, dropout=dropout, batch_first=True) + # Implementation of Feedforward model + self.fc1 = nn.Linear(c1, cm) + self.fc2 = nn.Linear(cm, c1) + + self.norm1 = nn.LayerNorm(c1) + self.norm2 = nn.LayerNorm(c1) + self.dropout = nn.Dropout(dropout) + self.dropout1 = nn.Dropout(dropout) + self.dropout2 = nn.Dropout(dropout) + + self.act = act + self.normalize_before = normalize_before + + @staticmethod + def with_pos_embed(tensor, pos=None): + """Add position embeddings to the tensor if provided.""" + return tensor if pos is None else tensor + pos + + def forward_post(self, src, src_mask=None, src_key_padding_mask=None, pos=None): + """Performs forward pass with post-normalization.""" + q = k = self.with_pos_embed(src, pos) + src2 = self.ma(q, k, value=src, attn_mask=src_mask, key_padding_mask=src_key_padding_mask)[0] + src = src + self.dropout1(src2) + src = self.norm1(src) + src2 = self.fc2(self.dropout(self.act(self.fc1(src)))) + src = src + self.dropout2(src2) + return self.norm2(src) + + def forward_pre(self, src, src_mask=None, src_key_padding_mask=None, pos=None): + """Performs forward pass with pre-normalization.""" + src2 = self.norm1(src) + q = k = self.with_pos_embed(src2, pos) + src2 = self.ma(q, k, value=src2, attn_mask=src_mask, key_padding_mask=src_key_padding_mask)[0] + src = src + self.dropout1(src2) + src2 = self.norm2(src) + src2 = self.fc2(self.dropout(self.act(self.fc1(src2)))) + return src + self.dropout2(src2) + + def forward(self, src, src_mask=None, src_key_padding_mask=None, pos=None): + """Forward propagates the input through the encoder module.""" + if self.normalize_before: + return self.forward_pre(src, src_mask, src_key_padding_mask, pos) + return self.forward_post(src, src_mask, src_key_padding_mask, pos) + + +class AIFI(TransformerEncoderLayer): + """Defines the AIFI transformer layer.""" + + def __init__(self, c1, cm=2048, num_heads=8, dropout=0, act=nn.GELU(), normalize_before=False): + """Initialize the AIFI instance with specified parameters.""" + super().__init__(c1, cm, num_heads, dropout, act, normalize_before) + + def forward(self, x): + """Forward pass for the AIFI transformer layer.""" + c, h, w = x.shape[1:] + pos_embed = self.build_2d_sincos_position_embedding(w, h, c) + # Flatten [B, C, H, W] to [B, HxW, C] + x = super().forward(x.flatten(2).permute(0, 2, 1), pos=pos_embed.to(device=x.device, dtype=x.dtype)) + return x.permute(0, 2, 1).view([-1, c, h, w]).contiguous() + + @staticmethod + def build_2d_sincos_position_embedding(w, h, embed_dim=256, temperature=10000.0): + """Builds 2D sine-cosine position embedding.""" + assert embed_dim % 4 == 0, "Embed dimension must be divisible by 4 for 2D sin-cos position embedding" + grid_w = torch.arange(w, dtype=torch.float32) + grid_h = torch.arange(h, dtype=torch.float32) + grid_w, grid_h = torch.meshgrid(grid_w, grid_h, indexing="ij") + pos_dim = embed_dim // 4 + omega = torch.arange(pos_dim, dtype=torch.float32) / pos_dim + omega = 1.0 / (temperature**omega) + + out_w = grid_w.flatten()[..., None] @ omega[None] + out_h = grid_h.flatten()[..., None] @ omega[None] + + return torch.cat([torch.sin(out_w), torch.cos(out_w), torch.sin(out_h), torch.cos(out_h)], 1)[None] + + +class TransformerLayer(nn.Module): + """Transformer layer https://arxiv.org/abs/2010.11929 (LayerNorm layers removed for better performance).""" + + def __init__(self, c, num_heads): + """Initializes a self-attention mechanism using linear transformations and multi-head attention.""" + super().__init__() + self.q = nn.Linear(c, c, bias=False) + self.k = nn.Linear(c, c, bias=False) + self.v = nn.Linear(c, c, bias=False) + self.ma = nn.MultiheadAttention(embed_dim=c, num_heads=num_heads) + self.fc1 = nn.Linear(c, c, bias=False) + self.fc2 = nn.Linear(c, c, bias=False) + + def forward(self, x): + """Apply a transformer block to the input x and return the output.""" + x = self.ma(self.q(x), self.k(x), self.v(x))[0] + x + return self.fc2(self.fc1(x)) + x + + +class TransformerBlock(nn.Module): + """Vision Transformer https://arxiv.org/abs/2010.11929.""" + + def __init__(self, c1, c2, num_heads, num_layers): + """Initialize a Transformer module with position embedding and specified number of heads and layers.""" + super().__init__() + self.conv = None + if c1 != c2: + self.conv = Conv(c1, c2) + self.linear = nn.Linear(c2, c2) # learnable position embedding + self.tr = nn.Sequential(*(TransformerLayer(c2, num_heads) for _ in range(num_layers))) + self.c2 = c2 + + def forward(self, x): + """Forward propagates the input through the bottleneck module.""" + if self.conv is not None: + x = self.conv(x) + b, _, w, h = x.shape + p = x.flatten(2).permute(2, 0, 1) + return self.tr(p + self.linear(p)).permute(1, 2, 0).reshape(b, self.c2, w, h) + + +class MLPBlock(nn.Module): + """Implements a single block of a multi-layer perceptron.""" + + def __init__(self, embedding_dim, mlp_dim, act=nn.GELU): + """Initialize the MLPBlock with specified embedding dimension, MLP dimension, and activation function.""" + super().__init__() + self.lin1 = nn.Linear(embedding_dim, mlp_dim) + self.lin2 = nn.Linear(mlp_dim, embedding_dim) + self.act = act() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Forward pass for the MLPBlock.""" + return self.lin2(self.act(self.lin1(x))) + + +class MLP(nn.Module): + """Implements a simple multi-layer perceptron (also called FFN).""" + + def __init__(self, input_dim, hidden_dim, output_dim, num_layers): + """Initialize the MLP with specified input, hidden, output dimensions and number of layers.""" + super().__init__() + self.num_layers = num_layers + h = [hidden_dim] * (num_layers - 1) + self.layers = nn.ModuleList(nn.Linear(n, k) for n, k in zip([input_dim] + h, h + [output_dim])) + + def forward(self, x): + """Forward pass for the entire MLP.""" + for i, layer in enumerate(self.layers): + x = F.relu(layer(x)) if i < self.num_layers - 1 else layer(x) + return x + + +class LayerNorm2d(nn.Module): + """ + 2D Layer Normalization module inspired by Detectron2 and ConvNeXt implementations. + + Original implementations in + https://github.com/facebookresearch/detectron2/blob/main/detectron2/layers/batch_norm.py + and + https://github.com/facebookresearch/ConvNeXt/blob/main/models/convnext.py. + """ + + def __init__(self, num_channels, eps=1e-6): + """Initialize LayerNorm2d with the given parameters.""" + super().__init__() + self.weight = nn.Parameter(torch.ones(num_channels)) + self.bias = nn.Parameter(torch.zeros(num_channels)) + self.eps = eps + + def forward(self, x): + """Perform forward pass for 2D layer normalization.""" + u = x.mean(1, keepdim=True) + s = (x - u).pow(2).mean(1, keepdim=True) + x = (x - u) / torch.sqrt(s + self.eps) + return self.weight[:, None, None] * x + self.bias[:, None, None] + + +class MSDeformAttn(nn.Module): + """ + Multiscale Deformable Attention Module based on Deformable-DETR and PaddleDetection implementations. + + https://github.com/fundamentalvision/Deformable-DETR/blob/main/models/ops/modules/ms_deform_attn.py + """ + + def __init__(self, d_model=256, n_levels=4, n_heads=8, n_points=4): + """Initialize MSDeformAttn with the given parameters.""" + super().__init__() + if d_model % n_heads != 0: + raise ValueError(f"d_model must be divisible by n_heads, but got {d_model} and {n_heads}") + _d_per_head = d_model // n_heads + # Better to set _d_per_head to a power of 2 which is more efficient in a CUDA implementation + assert _d_per_head * n_heads == d_model, "`d_model` must be divisible by `n_heads`" + + self.im2col_step = 64 + + self.d_model = d_model + self.n_levels = n_levels + self.n_heads = n_heads + self.n_points = n_points + + self.sampling_offsets = nn.Linear(d_model, n_heads * n_levels * n_points * 2) + self.attention_weights = nn.Linear(d_model, n_heads * n_levels * n_points) + self.value_proj = nn.Linear(d_model, d_model) + self.output_proj = nn.Linear(d_model, d_model) + + self._reset_parameters() + + def _reset_parameters(self): + """Reset module parameters.""" + constant_(self.sampling_offsets.weight.data, 0.0) + thetas = torch.arange(self.n_heads, dtype=torch.float32) * (2.0 * math.pi / self.n_heads) + grid_init = torch.stack([thetas.cos(), thetas.sin()], -1) + grid_init = ( + (grid_init / grid_init.abs().max(-1, keepdim=True)[0]) + .view(self.n_heads, 1, 1, 2) + .repeat(1, self.n_levels, self.n_points, 1) + ) + for i in range(self.n_points): + grid_init[:, :, i, :] *= i + 1 + with torch.no_grad(): + self.sampling_offsets.bias = nn.Parameter(grid_init.view(-1)) + constant_(self.attention_weights.weight.data, 0.0) + constant_(self.attention_weights.bias.data, 0.0) + xavier_uniform_(self.value_proj.weight.data) + constant_(self.value_proj.bias.data, 0.0) + xavier_uniform_(self.output_proj.weight.data) + constant_(self.output_proj.bias.data, 0.0) + + def forward(self, query, refer_bbox, value, value_shapes, value_mask=None): + """ + Perform forward pass for multiscale deformable attention. + + https://github.com/PaddlePaddle/PaddleDetection/blob/develop/ppdet/modeling/transformers/deformable_transformer.py + + Args: + query (torch.Tensor): [bs, query_length, C] + refer_bbox (torch.Tensor): [bs, query_length, n_levels, 2], range in [0, 1], top-left (0,0), + bottom-right (1, 1), including padding area + value (torch.Tensor): [bs, value_length, C] + value_shapes (List): [n_levels, 2], [(H_0, W_0), (H_1, W_1), ..., (H_{L-1}, W_{L-1})] + value_mask (Tensor): [bs, value_length], True for non-padding elements, False for padding elements + + Returns: + output (Tensor): [bs, Length_{query}, C] + """ + bs, len_q = query.shape[:2] + len_v = value.shape[1] + assert sum(s[0] * s[1] for s in value_shapes) == len_v + + value = self.value_proj(value) + if value_mask is not None: + value = value.masked_fill(value_mask[..., None], float(0)) + value = value.view(bs, len_v, self.n_heads, self.d_model // self.n_heads) + sampling_offsets = self.sampling_offsets(query).view(bs, len_q, self.n_heads, self.n_levels, self.n_points, 2) + attention_weights = self.attention_weights(query).view(bs, len_q, self.n_heads, self.n_levels * self.n_points) + attention_weights = F.softmax(attention_weights, -1).view(bs, len_q, self.n_heads, self.n_levels, self.n_points) + # N, Len_q, n_heads, n_levels, n_points, 2 + num_points = refer_bbox.shape[-1] + if num_points == 2: + offset_normalizer = torch.as_tensor(value_shapes, dtype=query.dtype, device=query.device).flip(-1) + add = sampling_offsets / offset_normalizer[None, None, None, :, None, :] + sampling_locations = refer_bbox[:, :, None, :, None, :] + add + elif num_points == 4: + add = sampling_offsets / self.n_points * refer_bbox[:, :, None, :, None, 2:] * 0.5 + sampling_locations = refer_bbox[:, :, None, :, None, :2] + add + else: + raise ValueError(f"Last dim of reference_points must be 2 or 4, but got {num_points}.") + output = multi_scale_deformable_attn_pytorch(value, value_shapes, sampling_locations, attention_weights) + return self.output_proj(output) + + +class DeformableTransformerDecoderLayer(nn.Module): + """ + Deformable Transformer Decoder Layer inspired by PaddleDetection and Deformable-DETR implementations. + + https://github.com/PaddlePaddle/PaddleDetection/blob/develop/ppdet/modeling/transformers/deformable_transformer.py + https://github.com/fundamentalvision/Deformable-DETR/blob/main/models/deformable_transformer.py + """ + + def __init__(self, d_model=256, n_heads=8, d_ffn=1024, dropout=0.0, act=nn.ReLU(), n_levels=4, n_points=4): + """Initialize the DeformableTransformerDecoderLayer with the given parameters.""" + super().__init__() + + # Self attention + self.self_attn = nn.MultiheadAttention(d_model, n_heads, dropout=dropout) + self.dropout1 = nn.Dropout(dropout) + self.norm1 = nn.LayerNorm(d_model) + + # Cross attention + self.cross_attn = MSDeformAttn(d_model, n_levels, n_heads, n_points) + self.dropout2 = nn.Dropout(dropout) + self.norm2 = nn.LayerNorm(d_model) + + # FFN + self.linear1 = nn.Linear(d_model, d_ffn) + self.act = act + self.dropout3 = nn.Dropout(dropout) + self.linear2 = nn.Linear(d_ffn, d_model) + self.dropout4 = nn.Dropout(dropout) + self.norm3 = nn.LayerNorm(d_model) + + @staticmethod + def with_pos_embed(tensor, pos): + """Add positional embeddings to the input tensor, if provided.""" + return tensor if pos is None else tensor + pos + + def forward_ffn(self, tgt): + """Perform forward pass through the Feed-Forward Network part of the layer.""" + tgt2 = self.linear2(self.dropout3(self.act(self.linear1(tgt)))) + tgt = tgt + self.dropout4(tgt2) + return self.norm3(tgt) + + def forward(self, embed, refer_bbox, feats, shapes, padding_mask=None, attn_mask=None, query_pos=None): + """Perform the forward pass through the entire decoder layer.""" + + # Self attention + q = k = self.with_pos_embed(embed, query_pos) + tgt = self.self_attn(q.transpose(0, 1), k.transpose(0, 1), embed.transpose(0, 1), attn_mask=attn_mask)[ + 0 + ].transpose(0, 1) + embed = embed + self.dropout1(tgt) + embed = self.norm1(embed) + + # Cross attention + tgt = self.cross_attn( + self.with_pos_embed(embed, query_pos), refer_bbox.unsqueeze(2), feats, shapes, padding_mask + ) + embed = embed + self.dropout2(tgt) + embed = self.norm2(embed) + + # FFN + return self.forward_ffn(embed) + + +class DeformableTransformerDecoder(nn.Module): + """ + Implementation of Deformable Transformer Decoder based on PaddleDetection. + + https://github.com/PaddlePaddle/PaddleDetection/blob/develop/ppdet/modeling/transformers/deformable_transformer.py + """ + + def __init__(self, hidden_dim, decoder_layer, num_layers, eval_idx=-1): + """Initialize the DeformableTransformerDecoder with the given parameters.""" + super().__init__() + self.layers = _get_clones(decoder_layer, num_layers) + self.num_layers = num_layers + self.hidden_dim = hidden_dim + self.eval_idx = eval_idx if eval_idx >= 0 else num_layers + eval_idx + + def forward( + self, + embed, # decoder embeddings + refer_bbox, # anchor + feats, # image features + shapes, # feature shapes + bbox_head, + score_head, + pos_mlp, + attn_mask=None, + padding_mask=None, + ): + """Perform the forward pass through the entire decoder.""" + output = embed + dec_bboxes = [] + dec_cls = [] + last_refined_bbox = None + refer_bbox = refer_bbox.sigmoid() + for i, layer in enumerate(self.layers): + output = layer(output, refer_bbox, feats, shapes, padding_mask, attn_mask, pos_mlp(refer_bbox)) + + bbox = bbox_head[i](output) + refined_bbox = torch.sigmoid(bbox + inverse_sigmoid(refer_bbox)) + + if self.training: + dec_cls.append(score_head[i](output)) + if i == 0: + dec_bboxes.append(refined_bbox) + else: + dec_bboxes.append(torch.sigmoid(bbox + inverse_sigmoid(last_refined_bbox))) + elif i == self.eval_idx: + dec_cls.append(score_head[i](output)) + dec_bboxes.append(refined_bbox) + break + + last_refined_bbox = refined_bbox + refer_bbox = refined_bbox.detach() if self.training else refined_bbox + + return torch.stack(dec_bboxes), torch.stack(dec_cls) diff --git a/ultralytics/nn/modules/utils.py b/ultralytics/nn/modules/utils.py new file mode 100644 index 0000000..1512967 --- /dev/null +++ b/ultralytics/nn/modules/utils.py @@ -0,0 +1,85 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +"""Module utils.""" + +import copy +import math + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn.init import uniform_ + +__all__ = "multi_scale_deformable_attn_pytorch", "inverse_sigmoid" + + +def _get_clones(module, n): + """Create a list of cloned modules from the given module.""" + return nn.ModuleList([copy.deepcopy(module) for _ in range(n)]) + + +def bias_init_with_prob(prior_prob=0.01): + """Initialize conv/fc bias value according to a given probability value.""" + return float(-np.log((1 - prior_prob) / prior_prob)) # return bias_init + + +def linear_init(module): + """Initialize the weights and biases of a linear module.""" + bound = 1 / math.sqrt(module.weight.shape[0]) + uniform_(module.weight, -bound, bound) + if hasattr(module, "bias") and module.bias is not None: + uniform_(module.bias, -bound, bound) + + +def inverse_sigmoid(x, eps=1e-5): + """Calculate the inverse sigmoid function for a tensor.""" + x = x.clamp(min=0, max=1) + x1 = x.clamp(min=eps) + x2 = (1 - x).clamp(min=eps) + return torch.log(x1 / x2) + + +def multi_scale_deformable_attn_pytorch( + value: torch.Tensor, + value_spatial_shapes: torch.Tensor, + sampling_locations: torch.Tensor, + attention_weights: torch.Tensor, +) -> torch.Tensor: + """ + Multiscale deformable attention. + + https://github.com/IDEA-Research/detrex/blob/main/detrex/layers/multi_scale_deform_attn.py + """ + + bs, _, num_heads, embed_dims = value.shape + _, num_queries, num_heads, num_levels, num_points, _ = sampling_locations.shape + value_list = value.split([H_ * W_ for H_, W_ in value_spatial_shapes], dim=1) + sampling_grids = 2 * sampling_locations - 1 + sampling_value_list = [] + for level, (H_, W_) in enumerate(value_spatial_shapes): + # bs, H_*W_, num_heads, embed_dims -> + # bs, H_*W_, num_heads*embed_dims -> + # bs, num_heads*embed_dims, H_*W_ -> + # bs*num_heads, embed_dims, H_, W_ + value_l_ = value_list[level].flatten(2).transpose(1, 2).reshape(bs * num_heads, embed_dims, H_, W_) + # bs, num_queries, num_heads, num_points, 2 -> + # bs, num_heads, num_queries, num_points, 2 -> + # bs*num_heads, num_queries, num_points, 2 + sampling_grid_l_ = sampling_grids[:, :, :, level].transpose(1, 2).flatten(0, 1) + # bs*num_heads, embed_dims, num_queries, num_points + sampling_value_l_ = F.grid_sample( + value_l_, sampling_grid_l_, mode="bilinear", padding_mode="zeros", align_corners=False + ) + sampling_value_list.append(sampling_value_l_) + # (bs, num_queries, num_heads, num_levels, num_points) -> + # (bs, num_heads, num_queries, num_levels, num_points) -> + # (bs, num_heads, 1, num_queries, num_levels*num_points) + attention_weights = attention_weights.transpose(1, 2).reshape( + bs * num_heads, 1, num_queries, num_levels * num_points + ) + output = ( + (torch.stack(sampling_value_list, dim=-2).flatten(-2) * attention_weights) + .sum(-1) + .view(bs, num_heads * embed_dims, num_queries) + ) + return output.transpose(1, 2).contiguous() diff --git a/ultralytics/nn/tasks.py b/ultralytics/nn/tasks.py new file mode 100644 index 0000000..252f415 --- /dev/null +++ b/ultralytics/nn/tasks.py @@ -0,0 +1,1067 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import contextlib +from copy import deepcopy +from pathlib import Path + +import torch +import torch.nn as nn + +from ultralytics.nn.modules import ( + AIFI, + C1, + C2, + C3, + C3TR, + OBB, + SPP, + SPPF, + Bottleneck, + BottleneckCSP, + C2f, + C2fAttn, + ImagePoolingAttn, + C3Ghost, + C3x, + Classify, + Concat, + Conv, + Conv2, + ConvTranspose, + Detect, + DWConv, + DWConvTranspose2d, + Focus, + GhostBottleneck, + GhostConv, + HGBlock, + HGStem, + Pose, + RepC3, + RepConv, + ResNetLayer, + RTDETRDecoder, + Segment, + WorldDetect, + RepNCSPELAN4, + ADown, + SPPELAN, + CBFuse, + CBLinear, + Silence, + C2fCIB, + PSA, + SCDown, + RepVGGDW, + v10Detect +) +from ultralytics.utils import DEFAULT_CFG_DICT, DEFAULT_CFG_KEYS, LOGGER, colorstr, emojis, yaml_load +from ultralytics.utils.checks import check_requirements, check_suffix, check_yaml +from ultralytics.utils.loss import v8ClassificationLoss, v8DetectionLoss, v8OBBLoss, v8PoseLoss, v8SegmentationLoss, v10DetectLoss +from ultralytics.utils.plotting import feature_visualization +from ultralytics.utils.torch_utils import ( + fuse_conv_and_bn, + fuse_deconv_and_bn, + initialize_weights, + intersect_dicts, + make_divisible, + model_info, + scale_img, + time_sync, +) + +try: + import thop +except ImportError: + thop = None + + +class BaseModel(nn.Module): + """The BaseModel class serves as a base class for all the models in the Ultralytics YOLO family.""" + + def forward(self, x, *args, **kwargs): + """ + Forward pass of the model on a single scale. Wrapper for `_forward_once` method. + + Args: + x (torch.Tensor | dict): The input image tensor or a dict including image tensor and gt labels. + + Returns: + (torch.Tensor): The output of the network. + """ + if isinstance(x, dict): # for cases of training and validating while training. + return self.loss(x, *args, **kwargs) + return self.predict(x, *args, **kwargs) + + def predict(self, x, profile=False, visualize=False, augment=False, embed=None): + """ + Perform a forward pass through the network. + + Args: + x (torch.Tensor): The input tensor to the model. + profile (bool): Print the computation time of each layer if True, defaults to False. + visualize (bool): Save the feature maps of the model if True, defaults to False. + augment (bool): Augment image during prediction, defaults to False. + embed (list, optional): A list of feature vectors/embeddings to return. + + Returns: + (torch.Tensor): The last output of the model. + """ + if augment: + return self._predict_augment(x) + return self._predict_once(x, profile, visualize, embed) + + def _predict_once(self, x, profile=False, visualize=False, embed=None): + """ + Perform a forward pass through the network. + + Args: + x (torch.Tensor): The input tensor to the model. + profile (bool): Print the computation time of each layer if True, defaults to False. + visualize (bool): Save the feature maps of the model if True, defaults to False. + embed (list, optional): A list of feature vectors/embeddings to return. + + Returns: + (torch.Tensor): The last output of the model. + """ + y, dt, embeddings = [], [], [] # outputs + for m in self.model: + if m.f != -1: # if not from previous layer + x = y[m.f] if isinstance(m.f, int) else [x if j == -1 else y[j] for j in m.f] # from earlier layers + if profile: + self._profile_one_layer(m, x, dt) + x = m(x) # run + y.append(x if m.i in self.save else None) # save output + if visualize: + feature_visualization(x, m.type, m.i, save_dir=visualize) + if embed and m.i in embed: + embeddings.append(nn.functional.adaptive_avg_pool2d(x, (1, 1)).squeeze(-1).squeeze(-1)) # flatten + if m.i == max(embed): + return torch.unbind(torch.cat(embeddings, 1), dim=0) + return x + + def _predict_augment(self, x): + """Perform augmentations on input image x and return augmented inference.""" + LOGGER.warning( + f"WARNING ⚠️ {self.__class__.__name__} does not support augmented inference yet. " + f"Reverting to single-scale inference instead." + ) + return self._predict_once(x) + + def _profile_one_layer(self, m, x, dt): + """ + Profile the computation time and FLOPs of a single layer of the model on a given input. Appends the results to + the provided list. + + Args: + m (nn.Module): The layer to be profiled. + x (torch.Tensor): The input data to the layer. + dt (list): A list to store the computation time of the layer. + + Returns: + None + """ + c = m == self.model[-1] and isinstance(x, list) # is final layer list, copy input as inplace fix + flops = thop.profile(m, inputs=[x.copy() if c else x], verbose=False)[0] / 1e9 * 2 if thop else 0 # FLOPs + t = time_sync() + for _ in range(10): + m(x.copy() if c else x) + dt.append((time_sync() - t) * 100) + if m == self.model[0]: + LOGGER.info(f"{'time (ms)':>10s} {'GFLOPs':>10s} {'params':>10s} module") + LOGGER.info(f"{dt[-1]:10.2f} {flops:10.2f} {m.np:10.0f} {m.type}") + if c: + LOGGER.info(f"{sum(dt):10.2f} {'-':>10s} {'-':>10s} Total") + + def fuse(self, verbose=True): + """ + Fuse the `Conv2d()` and `BatchNorm2d()` layers of the model into a single layer, in order to improve the + computation efficiency. + + Returns: + (nn.Module): The fused model is returned. + """ + if not self.is_fused(): + for m in self.model.modules(): + if isinstance(m, (Conv, Conv2, DWConv)) and hasattr(m, "bn"): + if isinstance(m, Conv2): + m.fuse_convs() + m.conv = fuse_conv_and_bn(m.conv, m.bn) # update conv + delattr(m, "bn") # remove batchnorm + m.forward = m.forward_fuse # update forward + if isinstance(m, ConvTranspose) and hasattr(m, "bn"): + m.conv_transpose = fuse_deconv_and_bn(m.conv_transpose, m.bn) + delattr(m, "bn") # remove batchnorm + m.forward = m.forward_fuse # update forward + if isinstance(m, RepConv): + m.fuse_convs() + m.forward = m.forward_fuse # update forward + if isinstance(m, RepVGGDW): + m.fuse() + m.forward = m.forward_fuse + self.info(verbose=verbose) + + return self + + def is_fused(self, thresh=10): + """ + Check if the model has less than a certain threshold of BatchNorm layers. + + Args: + thresh (int, optional): The threshold number of BatchNorm layers. Default is 10. + + Returns: + (bool): True if the number of BatchNorm layers in the model is less than the threshold, False otherwise. + """ + bn = tuple(v for k, v in nn.__dict__.items() if "Norm" in k) # normalization layers, i.e. BatchNorm2d() + return sum(isinstance(v, bn) for v in self.modules()) < thresh # True if < 'thresh' BatchNorm layers in model + + def info(self, detailed=False, verbose=True, imgsz=640): + """ + Prints model information. + + Args: + detailed (bool): if True, prints out detailed information about the model. Defaults to False + verbose (bool): if True, prints out the model information. Defaults to False + imgsz (int): the size of the image that the model will be trained on. Defaults to 640 + """ + return model_info(self, detailed=detailed, verbose=verbose, imgsz=imgsz) + + def _apply(self, fn): + """ + Applies a function to all the tensors in the model that are not parameters or registered buffers. + + Args: + fn (function): the function to apply to the model + + Returns: + (BaseModel): An updated BaseModel object. + """ + self = super()._apply(fn) + m = self.model[-1] # Detect() + if isinstance(m, Detect): # includes all Detect subclasses like Segment, Pose, OBB, WorldDetect + m.stride = fn(m.stride) + m.anchors = fn(m.anchors) + m.strides = fn(m.strides) + return self + + def load(self, weights, verbose=True): + """ + Load the weights into the model. + + Args: + weights (dict | torch.nn.Module): The pre-trained weights to be loaded. + verbose (bool, optional): Whether to log the transfer progress. Defaults to True. + """ + model = weights["model"] if isinstance(weights, dict) else weights # torchvision models are not dicts + csd = model.float().state_dict() # checkpoint state_dict as FP32 + csd = intersect_dicts(csd, self.state_dict()) # intersect + self.load_state_dict(csd, strict=False) # load + if verbose: + LOGGER.info(f"Transferred {len(csd)}/{len(self.model.state_dict())} items from pretrained weights") + + def loss(self, batch, preds=None): + """ + Compute loss. + + Args: + batch (dict): Batch to compute loss on + preds (torch.Tensor | List[torch.Tensor]): Predictions. + """ + if not hasattr(self, "criterion"): + self.criterion = self.init_criterion() + + preds = self.forward(batch["img"]) if preds is None else preds + return self.criterion(preds, batch) + + def init_criterion(self): + """Initialize the loss criterion for the BaseModel.""" + raise NotImplementedError("compute_loss() needs to be implemented by task heads") + + +class DetectionModel(BaseModel): + """YOLOv8 detection model.""" + + def __init__(self, cfg="yolov8n.yaml", ch=3, nc=None, verbose=True): # model, input channels, number of classes + """Initialize the YOLOv8 detection model with the given config and parameters.""" + super().__init__() + self.yaml = cfg if isinstance(cfg, dict) else yaml_model_load(cfg) # cfg dict + + # Define model + ch = self.yaml["ch"] = self.yaml.get("ch", ch) # input channels + if nc and nc != self.yaml["nc"]: + LOGGER.info(f"Overriding model.yaml nc={self.yaml['nc']} with nc={nc}") + self.yaml["nc"] = nc # override YAML value + self.model, self.save = parse_model(deepcopy(self.yaml), ch=ch, verbose=verbose) # model, savelist + self.names = {i: f"{i}" for i in range(self.yaml["nc"])} # default names dict + self.inplace = self.yaml.get("inplace", True) + + # Build strides + m = self.model[-1] # Detect() + if isinstance(m, Detect): # includes all Detect subclasses like Segment, Pose, OBB, WorldDetect + s = 256 # 2x min stride + m.inplace = self.inplace + forward = lambda x: self.forward(x)[0] if isinstance(m, (Segment, Pose, OBB)) else self.forward(x) + if isinstance(m, v10Detect): + forward = lambda x: self.forward(x)["one2many"] + m.stride = torch.tensor([s / x.shape[-2] for x in forward(torch.zeros(1, ch, s, s))]) # forward + self.stride = m.stride + m.bias_init() # only run once + else: + self.stride = torch.Tensor([32]) # default stride for i.e. RTDETR + + # Init weights, biases + initialize_weights(self) + if verbose: + self.info() + LOGGER.info("") + + def _predict_augment(self, x): + """Perform augmentations on input image x and return augmented inference and train outputs.""" + img_size = x.shape[-2:] # height, width + s = [1, 0.83, 0.67] # scales + f = [None, 3, None] # flips (2-ud, 3-lr) + y = [] # outputs + for si, fi in zip(s, f): + xi = scale_img(x.flip(fi) if fi else x, si, gs=int(self.stride.max())) + yi = super().predict(xi) # forward + if isinstance(yi, dict): + yi = yi["one2one"] # yolov10 outputs + if isinstance(yi, (list, tuple)): + yi = yi[0] + yi = self._descale_pred(yi, fi, si, img_size) + y.append(yi) + y = self._clip_augmented(y) # clip augmented tails + return torch.cat(y, -1), None # augmented inference, train + + @staticmethod + def _descale_pred(p, flips, scale, img_size, dim=1): + """De-scale predictions following augmented inference (inverse operation).""" + p[:, :4] /= scale # de-scale + x, y, wh, cls = p.split((1, 1, 2, p.shape[dim] - 4), dim) + if flips == 2: + y = img_size[0] - y # de-flip ud + elif flips == 3: + x = img_size[1] - x # de-flip lr + return torch.cat((x, y, wh, cls), dim) + + def _clip_augmented(self, y): + """Clip YOLO augmented inference tails.""" + nl = self.model[-1].nl # number of detection layers (P3-P5) + g = sum(4**x for x in range(nl)) # grid points + e = 1 # exclude layer count + i = (y[0].shape[-1] // g) * sum(4**x for x in range(e)) # indices + y[0] = y[0][..., :-i] # large + i = (y[-1].shape[-1] // g) * sum(4 ** (nl - 1 - x) for x in range(e)) # indices + y[-1] = y[-1][..., i:] # small + return y + + def init_criterion(self): + """Initialize the loss criterion for the DetectionModel.""" + return v8DetectionLoss(self) + + +class OBBModel(DetectionModel): + """YOLOv8 Oriented Bounding Box (OBB) model.""" + + def __init__(self, cfg="yolov8n-obb.yaml", ch=3, nc=None, verbose=True): + """Initialize YOLOv8 OBB model with given config and parameters.""" + super().__init__(cfg=cfg, ch=ch, nc=nc, verbose=verbose) + + def init_criterion(self): + """Initialize the loss criterion for the model.""" + return v8OBBLoss(self) + + +class SegmentationModel(DetectionModel): + """YOLOv8 segmentation model.""" + + def __init__(self, cfg="yolov8n-seg.yaml", ch=3, nc=None, verbose=True): + """Initialize YOLOv8 segmentation model with given config and parameters.""" + super().__init__(cfg=cfg, ch=ch, nc=nc, verbose=verbose) + + def init_criterion(self): + """Initialize the loss criterion for the SegmentationModel.""" + return v8SegmentationLoss(self) + + +class PoseModel(DetectionModel): + """YOLOv8 pose model.""" + + def __init__(self, cfg="yolov8n-pose.yaml", ch=3, nc=None, data_kpt_shape=(None, None), verbose=True): + """Initialize YOLOv8 Pose model.""" + if not isinstance(cfg, dict): + cfg = yaml_model_load(cfg) # load model YAML + if any(data_kpt_shape) and list(data_kpt_shape) != list(cfg["kpt_shape"]): + LOGGER.info(f"Overriding model.yaml kpt_shape={cfg['kpt_shape']} with kpt_shape={data_kpt_shape}") + cfg["kpt_shape"] = data_kpt_shape + super().__init__(cfg=cfg, ch=ch, nc=nc, verbose=verbose) + + def init_criterion(self): + """Initialize the loss criterion for the PoseModel.""" + return v8PoseLoss(self) + + +class ClassificationModel(BaseModel): + """YOLOv8 classification model.""" + + def __init__(self, cfg="yolov8n-cls.yaml", ch=3, nc=None, verbose=True): + """Init ClassificationModel with YAML, channels, number of classes, verbose flag.""" + super().__init__() + self._from_yaml(cfg, ch, nc, verbose) + + def _from_yaml(self, cfg, ch, nc, verbose): + """Set YOLOv8 model configurations and define the model architecture.""" + self.yaml = cfg if isinstance(cfg, dict) else yaml_model_load(cfg) # cfg dict + + # Define model + ch = self.yaml["ch"] = self.yaml.get("ch", ch) # input channels + if nc and nc != self.yaml["nc"]: + LOGGER.info(f"Overriding model.yaml nc={self.yaml['nc']} with nc={nc}") + self.yaml["nc"] = nc # override YAML value + elif not nc and not self.yaml.get("nc", None): + raise ValueError("nc not specified. Must specify nc in model.yaml or function arguments.") + self.model, self.save = parse_model(deepcopy(self.yaml), ch=ch, verbose=verbose) # model, savelist + self.stride = torch.Tensor([1]) # no stride constraints + self.names = {i: f"{i}" for i in range(self.yaml["nc"])} # default names dict + self.info() + + @staticmethod + def reshape_outputs(model, nc): + """Update a TorchVision classification model to class count 'n' if required.""" + name, m = list((model.model if hasattr(model, "model") else model).named_children())[-1] # last module + if isinstance(m, Classify): # YOLO Classify() head + if m.linear.out_features != nc: + m.linear = nn.Linear(m.linear.in_features, nc) + elif isinstance(m, nn.Linear): # ResNet, EfficientNet + if m.out_features != nc: + setattr(model, name, nn.Linear(m.in_features, nc)) + elif isinstance(m, nn.Sequential): + types = [type(x) for x in m] + if nn.Linear in types: + i = types.index(nn.Linear) # nn.Linear index + if m[i].out_features != nc: + m[i] = nn.Linear(m[i].in_features, nc) + elif nn.Conv2d in types: + i = types.index(nn.Conv2d) # nn.Conv2d index + if m[i].out_channels != nc: + m[i] = nn.Conv2d(m[i].in_channels, nc, m[i].kernel_size, m[i].stride, bias=m[i].bias is not None) + + def init_criterion(self): + """Initialize the loss criterion for the ClassificationModel.""" + return v8ClassificationLoss() + + +class RTDETRDetectionModel(DetectionModel): + """ + RTDETR (Real-time DEtection and Tracking using Transformers) Detection Model class. + + This class is responsible for constructing the RTDETR architecture, defining loss functions, and facilitating both + the training and inference processes. RTDETR is an object detection and tracking model that extends from the + DetectionModel base class. + + Attributes: + cfg (str): The configuration file path or preset string. Default is 'rtdetr-l.yaml'. + ch (int): Number of input channels. Default is 3 (RGB). + nc (int, optional): Number of classes for object detection. Default is None. + verbose (bool): Specifies if summary statistics are shown during initialization. Default is True. + + Methods: + init_criterion: Initializes the criterion used for loss calculation. + loss: Computes and returns the loss during training. + predict: Performs a forward pass through the network and returns the output. + """ + + def __init__(self, cfg="rtdetr-l.yaml", ch=3, nc=None, verbose=True): + """ + Initialize the RTDETRDetectionModel. + + Args: + cfg (str): Configuration file name or path. + ch (int): Number of input channels. + nc (int, optional): Number of classes. Defaults to None. + verbose (bool, optional): Print additional information during initialization. Defaults to True. + """ + super().__init__(cfg=cfg, ch=ch, nc=nc, verbose=verbose) + + def init_criterion(self): + """Initialize the loss criterion for the RTDETRDetectionModel.""" + from ultralytics.models.utils.loss import RTDETRDetectionLoss + + return RTDETRDetectionLoss(nc=self.nc, use_vfl=True) + + def loss(self, batch, preds=None): + """ + Compute the loss for the given batch of data. + + Args: + batch (dict): Dictionary containing image and label data. + preds (torch.Tensor, optional): Precomputed model predictions. Defaults to None. + + Returns: + (tuple): A tuple containing the total loss and main three losses in a tensor. + """ + if not hasattr(self, "criterion"): + self.criterion = self.init_criterion() + + img = batch["img"] + # NOTE: preprocess gt_bbox and gt_labels to list. + bs = len(img) + batch_idx = batch["batch_idx"] + gt_groups = [(batch_idx == i).sum().item() for i in range(bs)] + targets = { + "cls": batch["cls"].to(img.device, dtype=torch.long).view(-1), + "bboxes": batch["bboxes"].to(device=img.device), + "batch_idx": batch_idx.to(img.device, dtype=torch.long).view(-1), + "gt_groups": gt_groups, + } + + preds = self.predict(img, batch=targets) if preds is None else preds + dec_bboxes, dec_scores, enc_bboxes, enc_scores, dn_meta = preds if self.training else preds[1] + if dn_meta is None: + dn_bboxes, dn_scores = None, None + else: + dn_bboxes, dec_bboxes = torch.split(dec_bboxes, dn_meta["dn_num_split"], dim=2) + dn_scores, dec_scores = torch.split(dec_scores, dn_meta["dn_num_split"], dim=2) + + dec_bboxes = torch.cat([enc_bboxes.unsqueeze(0), dec_bboxes]) # (7, bs, 300, 4) + dec_scores = torch.cat([enc_scores.unsqueeze(0), dec_scores]) + + loss = self.criterion( + (dec_bboxes, dec_scores), targets, dn_bboxes=dn_bboxes, dn_scores=dn_scores, dn_meta=dn_meta + ) + # NOTE: There are like 12 losses in RTDETR, backward with all losses but only show the main three losses. + return sum(loss.values()), torch.as_tensor( + [loss[k].detach() for k in ["loss_giou", "loss_class", "loss_bbox"]], device=img.device + ) + + def predict(self, x, profile=False, visualize=False, batch=None, augment=False, embed=None): + """ + Perform a forward pass through the model. + + Args: + x (torch.Tensor): The input tensor. + profile (bool, optional): If True, profile the computation time for each layer. Defaults to False. + visualize (bool, optional): If True, save feature maps for visualization. Defaults to False. + batch (dict, optional): Ground truth data for evaluation. Defaults to None. + augment (bool, optional): If True, perform data augmentation during inference. Defaults to False. + embed (list, optional): A list of feature vectors/embeddings to return. + + Returns: + (torch.Tensor): Model's output tensor. + """ + y, dt, embeddings = [], [], [] # outputs + for m in self.model[:-1]: # except the head part + if m.f != -1: # if not from previous layer + x = y[m.f] if isinstance(m.f, int) else [x if j == -1 else y[j] for j in m.f] # from earlier layers + if profile: + self._profile_one_layer(m, x, dt) + x = m(x) # run + y.append(x if m.i in self.save else None) # save output + if visualize: + feature_visualization(x, m.type, m.i, save_dir=visualize) + if embed and m.i in embed: + embeddings.append(nn.functional.adaptive_avg_pool2d(x, (1, 1)).squeeze(-1).squeeze(-1)) # flatten + if m.i == max(embed): + return torch.unbind(torch.cat(embeddings, 1), dim=0) + head = self.model[-1] + x = head([y[j] for j in head.f], batch) # head inference + return x + + +class WorldModel(DetectionModel): + """YOLOv8 World Model.""" + + def __init__(self, cfg="yolov8s-world.yaml", ch=3, nc=None, verbose=True): + """Initialize YOLOv8 world model with given config and parameters.""" + self.txt_feats = torch.randn(1, nc or 80, 512) # features placeholder + self.clip_model = None # CLIP model placeholder + super().__init__(cfg=cfg, ch=ch, nc=nc, verbose=verbose) + + def set_classes(self, text): + """Perform a forward pass with optional profiling, visualization, and embedding extraction.""" + try: + import clip + except ImportError: + check_requirements("git+https://github.com/openai/CLIP.git") + import clip + + if not getattr(self, "clip_model", None): # for backwards compatibility of models lacking clip_model attribute + self.clip_model = clip.load("ViT-B/32")[0] + device = next(self.clip_model.parameters()).device + text_token = clip.tokenize(text).to(device) + txt_feats = self.clip_model.encode_text(text_token).to(dtype=torch.float32) + txt_feats = txt_feats / txt_feats.norm(p=2, dim=-1, keepdim=True) + self.txt_feats = txt_feats.reshape(-1, len(text), txt_feats.shape[-1]).detach() + self.model[-1].nc = len(text) + + def init_criterion(self): + """Initialize the loss criterion for the model.""" + raise NotImplementedError + + def predict(self, x, profile=False, visualize=False, augment=False, embed=None): + """ + Perform a forward pass through the model. + + Args: + x (torch.Tensor): The input tensor. + profile (bool, optional): If True, profile the computation time for each layer. Defaults to False. + visualize (bool, optional): If True, save feature maps for visualization. Defaults to False. + augment (bool, optional): If True, perform data augmentation during inference. Defaults to False. + embed (list, optional): A list of feature vectors/embeddings to return. + + Returns: + (torch.Tensor): Model's output tensor. + """ + txt_feats = self.txt_feats.to(device=x.device, dtype=x.dtype) + if len(txt_feats) != len(x): + txt_feats = txt_feats.repeat(len(x), 1, 1) + ori_txt_feats = txt_feats.clone() + y, dt, embeddings = [], [], [] # outputs + for m in self.model: # except the head part + if m.f != -1: # if not from previous layer + x = y[m.f] if isinstance(m.f, int) else [x if j == -1 else y[j] for j in m.f] # from earlier layers + if profile: + self._profile_one_layer(m, x, dt) + if isinstance(m, C2fAttn): + x = m(x, txt_feats) + elif isinstance(m, WorldDetect): + x = m(x, ori_txt_feats) + elif isinstance(m, ImagePoolingAttn): + txt_feats = m(x, txt_feats) + else: + x = m(x) # run + + y.append(x if m.i in self.save else None) # save output + if visualize: + feature_visualization(x, m.type, m.i, save_dir=visualize) + if embed and m.i in embed: + embeddings.append(nn.functional.adaptive_avg_pool2d(x, (1, 1)).squeeze(-1).squeeze(-1)) # flatten + if m.i == max(embed): + return torch.unbind(torch.cat(embeddings, 1), dim=0) + return x + +class YOLOv10DetectionModel(DetectionModel): + def init_criterion(self): + return v10DetectLoss(self) + +class Ensemble(nn.ModuleList): + """Ensemble of models.""" + + def __init__(self): + """Initialize an ensemble of models.""" + super().__init__() + + def forward(self, x, augment=False, profile=False, visualize=False): + """Function generates the YOLO network's final layer.""" + y = [module(x, augment, profile, visualize)[0] for module in self] + # y = torch.stack(y).max(0)[0] # max ensemble + # y = torch.stack(y).mean(0) # mean ensemble + y = torch.cat(y, 2) # nms ensemble, y shape(B, HW, C) + return y, None # inference, train output + + +# Functions ------------------------------------------------------------------------------------------------------------ + + +@contextlib.contextmanager +def temporary_modules(modules=None): + """ + Context manager for temporarily adding or modifying modules in Python's module cache (`sys.modules`). + + This function can be used to change the module paths during runtime. It's useful when refactoring code, + where you've moved a module from one location to another, but you still want to support the old import + paths for backwards compatibility. + + Args: + modules (dict, optional): A dictionary mapping old module paths to new module paths. + + Example: + ```python + with temporary_modules({'old.module.path': 'new.module.path'}): + import old.module.path # this will now import new.module.path + ``` + + Note: + The changes are only in effect inside the context manager and are undone once the context manager exits. + Be aware that directly manipulating `sys.modules` can lead to unpredictable results, especially in larger + applications or libraries. Use this function with caution. + """ + if not modules: + modules = {} + + import importlib + import sys + + try: + # Set modules in sys.modules under their old name + for old, new in modules.items(): + sys.modules[old] = importlib.import_module(new) + + yield + finally: + # Remove the temporary module paths + for old in modules: + if old in sys.modules: + del sys.modules[old] + + +def torch_safe_load(weight): + """ + This function attempts to load a PyTorch model with the torch.load() function. If a ModuleNotFoundError is raised, + it catches the error, logs a warning message, and attempts to install the missing module via the + check_requirements() function. After installation, the function again attempts to load the model using torch.load(). + + Args: + weight (str): The file path of the PyTorch model. + + Returns: + (dict): The loaded PyTorch model. + """ + from ultralytics.utils.downloads import attempt_download_asset + + check_suffix(file=weight, suffix=".pt") + file = attempt_download_asset(weight) # search online if missing locally + try: + with temporary_modules( + { + "ultralytics.yolo.utils": "ultralytics.utils", + "ultralytics.yolo.v8": "ultralytics.models.yolo", + "ultralytics.yolo.data": "ultralytics.data", + } + ): # for legacy 8.0 Classify and Pose models + if torch.__version__ >= '2.6': + ckpt = torch.load(file, map_location="cpu", weights_only=False) + else: + ckpt = torch.load(file, map_location="cpu") + + + + except ModuleNotFoundError as e: # e.name is missing module name + if e.name == "models": + raise TypeError( + emojis( + f"ERROR ❌️ {weight} appears to be an Ultralytics YOLOv5 model originally trained " + f"with https://github.com/ultralytics/yolov5.\nThis model is NOT forwards compatible with " + f"YOLOv8 at https://github.com/ultralytics/ultralytics." + f"\nRecommend fixes are to train a new model using the latest 'ultralytics' package or to " + f"run a command with an official YOLOv8 model, i.e. 'yolo predict model=yolov8n.pt'" + ) + ) from e + LOGGER.warning( + f"WARNING ⚠️ {weight} appears to require '{e.name}', which is not in ultralytics requirements." + f"\nAutoInstall will run now for '{e.name}' but this feature will be removed in the future." + f"\nRecommend fixes are to train a new model using the latest 'ultralytics' package or to " + f"run a command with an official YOLOv8 model, i.e. 'yolo predict model=yolov8n.pt'" + ) + check_requirements(e.name) # install missing module + ckpt = torch.load(file, map_location="cpu") + + if not isinstance(ckpt, dict): + # File is likely a YOLO instance saved with i.e. torch.save(model, "saved_model.pt") + LOGGER.warning( + f"WARNING ⚠️ The file '{weight}' appears to be improperly saved or formatted. " + f"For optimal results, use model.save('filename.pt') to correctly save YOLO models." + ) + ckpt = {"model": ckpt.model} + + return ckpt, file # load + + +def attempt_load_weights(weights, device=None, inplace=True, fuse=False): + """Loads an ensemble of models weights=[a,b,c] or a single model weights=[a] or weights=a.""" + + ensemble = Ensemble() + for w in weights if isinstance(weights, list) else [weights]: + ckpt, w = torch_safe_load(w) # load ckpt + args = {**DEFAULT_CFG_DICT, **ckpt["train_args"]} if "train_args" in ckpt else None # combined args + model = (ckpt.get("ema") or ckpt["model"]).to(device).float() # FP32 model + + # Model compatibility updates + model.args = args # attach args to model + model.pt_path = w # attach *.pt file path to model + model.task = guess_model_task(model) + if not hasattr(model, "stride"): + model.stride = torch.tensor([32.0]) + + # Append + ensemble.append(model.fuse().eval() if fuse and hasattr(model, "fuse") else model.eval()) # model in eval mode + + # Module updates + for m in ensemble.modules(): + if hasattr(m, "inplace"): + m.inplace = inplace + elif isinstance(m, nn.Upsample) and not hasattr(m, "recompute_scale_factor"): + m.recompute_scale_factor = None # torch 1.11.0 compatibility + + # Return model + if len(ensemble) == 1: + return ensemble[-1] + + # Return ensemble + LOGGER.info(f"Ensemble created with {weights}\n") + for k in "names", "nc", "yaml": + setattr(ensemble, k, getattr(ensemble[0], k)) + ensemble.stride = ensemble[int(torch.argmax(torch.tensor([m.stride.max() for m in ensemble])))].stride + assert all(ensemble[0].nc == m.nc for m in ensemble), f"Models differ in class counts {[m.nc for m in ensemble]}" + return ensemble + + +def attempt_load_one_weight(weight, device=None, inplace=True, fuse=False): + """Loads a single model weights.""" + ckpt, weight = torch_safe_load(weight) # load ckpt + args = {**DEFAULT_CFG_DICT, **(ckpt.get("train_args", {}))} # combine model and default args, preferring model args + model = (ckpt.get("ema") or ckpt["model"]).to(device).float() # FP32 model + + # Model compatibility updates + model.args = {k: v for k, v in args.items() if k in DEFAULT_CFG_KEYS} # attach args to model + model.pt_path = weight # attach *.pt file path to model + model.task = guess_model_task(model) + if not hasattr(model, "stride"): + model.stride = torch.tensor([32.0]) + + model = model.fuse().eval() if fuse and hasattr(model, "fuse") else model.eval() # model in eval mode + + # Module updates + for m in model.modules(): + if hasattr(m, "inplace"): + m.inplace = inplace + elif isinstance(m, nn.Upsample) and not hasattr(m, "recompute_scale_factor"): + m.recompute_scale_factor = None # torch 1.11.0 compatibility + + # Return model and ckpt + return model, ckpt + + +def parse_model(d, ch, verbose=True): # model_dict, input_channels(3) + """Parse a YOLO model.yaml dictionary into a PyTorch model.""" + import ast + + # Args + max_channels = float("inf") + nc, act, scales = (d.get(x) for x in ("nc", "activation", "scales")) + depth, width, kpt_shape = (d.get(x, 1.0) for x in ("depth_multiple", "width_multiple", "kpt_shape")) + if scales: + scale = d.get("scale") + if not scale: + scale = tuple(scales.keys())[0] + LOGGER.warning(f"WARNING ⚠️ no model scale passed. Assuming scale='{scale}'.") + depth, width, max_channels = scales[scale] + + if act: + Conv.default_act = eval(act) # redefine default activation, i.e. Conv.default_act = nn.SiLU() + if verbose: + LOGGER.info(f"{colorstr('activation:')} {act}") # print + + if verbose: + LOGGER.info(f"\n{'':>3}{'from':>20}{'n':>3}{'params':>10} {'module':<45}{'arguments':<30}") + ch = [ch] + layers, save, c2 = [], [], ch[-1] # layers, savelist, ch out + for i, (f, n, m, args) in enumerate(d["backbone"] + d["head"]): # from, number, module, args + m = getattr(torch.nn, m[3:]) if "nn." in m else globals()[m] # get module + for j, a in enumerate(args): + if isinstance(a, str): + with contextlib.suppress(ValueError): + args[j] = locals()[a] if a in locals() else ast.literal_eval(a) + + n = n_ = max(round(n * depth), 1) if n > 1 else n # depth gain + if m in { + Classify, + Conv, + ConvTranspose, + GhostConv, + Bottleneck, + GhostBottleneck, + SPP, + SPPF, + DWConv, + Focus, + BottleneckCSP, + C1, + C2, + C2f, + RepNCSPELAN4, + ADown, + SPPELAN, + C2fAttn, + C3, + C3TR, + C3Ghost, + nn.ConvTranspose2d, + DWConvTranspose2d, + C3x, + RepC3, + PSA, + SCDown, + C2fCIB + }: + c1, c2 = ch[f], args[0] + if c2 != nc: # if c2 not equal to number of classes (i.e. for Classify() output) + c2 = make_divisible(min(c2, max_channels) * width, 8) + if m is C2fAttn: + args[1] = make_divisible(min(args[1], max_channels // 2) * width, 8) # embed channels + args[2] = int( + max(round(min(args[2], max_channels // 2 // 32)) * width, 1) if args[2] > 1 else args[2] + ) # num heads + + args = [c1, c2, *args[1:]] + if m in (BottleneckCSP, C1, C2, C2f, C2fAttn, C3, C3TR, C3Ghost, C3x, RepC3, C2fCIB): + args.insert(2, n) # number of repeats + n = 1 + elif m is AIFI: + args = [ch[f], *args] + elif m in {HGStem, HGBlock}: + c1, cm, c2 = ch[f], args[0], args[1] + args = [c1, cm, c2, *args[2:]] + if m is HGBlock: + args.insert(4, n) # number of repeats + n = 1 + elif m is ResNetLayer: + c2 = args[1] if args[3] else args[1] * 4 + elif m is nn.BatchNorm2d: + args = [ch[f]] + elif m is Concat: + c2 = sum(ch[x] for x in f) + elif m in {Detect, WorldDetect, Segment, Pose, OBB, ImagePoolingAttn, v10Detect}: + args.append([ch[x] for x in f]) + if m is Segment: + args[2] = make_divisible(min(args[2], max_channels) * width, 8) + elif m is RTDETRDecoder: # special case, channels arg must be passed in index 1 + args.insert(1, [ch[x] for x in f]) + elif m is CBLinear: + c2 = args[0] + c1 = ch[f] + args = [c1, c2, *args[1:]] + elif m is CBFuse: + c2 = ch[f[-1]] + else: + c2 = ch[f] + + m_ = nn.Sequential(*(m(*args) for _ in range(n))) if n > 1 else m(*args) # module + t = str(m)[8:-2].replace("__main__.", "") # module type + m.np = sum(x.numel() for x in m_.parameters()) # number params + m_.i, m_.f, m_.type = i, f, t # attach index, 'from' index, type + if verbose: + LOGGER.info(f"{i:>3}{str(f):>20}{n_:>3}{m.np:10.0f} {t:<45}{str(args):<30}") # print + save.extend(x % i for x in ([f] if isinstance(f, int) else f) if x != -1) # append to savelist + layers.append(m_) + if i == 0: + ch = [] + ch.append(c2) + return nn.Sequential(*layers), sorted(save) + + +def yaml_model_load(path): + """Load a YOLOv8 model from a YAML file.""" + import re + + path = Path(path) + if path.stem in (f"yolov{d}{x}6" for x in "nsmlx" for d in (5, 8)): + new_stem = re.sub(r"(\d+)([nslmx])6(.+)?$", r"\1\2-p6\3", path.stem) + LOGGER.warning(f"WARNING ⚠️ Ultralytics YOLO P6 models now use -p6 suffix. Renaming {path.stem} to {new_stem}.") + path = path.with_name(new_stem + path.suffix) + + if "v10" not in str(path): + unified_path = re.sub(r"(\d+)([nsblmx])(.+)?$", r"\1\3", str(path)) # i.e. yolov8x.yaml -> yolov8.yaml + else: + unified_path = path + yaml_file = check_yaml(unified_path, hard=False) or check_yaml(path) + d = yaml_load(yaml_file) # model dict + d["scale"] = guess_model_scale(path) + d["yaml_file"] = str(path) + return d + + +def guess_model_scale(model_path): + """ + Takes a path to a YOLO model's YAML file as input and extracts the size character of the model's scale. The function + uses regular expression matching to find the pattern of the model scale in the YAML file name, which is denoted by + n, s, m, l, or x. The function returns the size character of the model scale as a string. + + Args: + model_path (str | Path): The path to the YOLO model's YAML file. + + Returns: + (str): The size character of the model's scale, which can be n, s, m, l, or x. + """ + with contextlib.suppress(AttributeError): + import re + + return re.search(r"yolov\d+([nsblmx])", Path(model_path).stem).group(1) # n, s, m, l, or x + return "" + + +def guess_model_task(model): + """ + Guess the task of a PyTorch model from its architecture or configuration. + + Args: + model (nn.Module | dict): PyTorch model or model configuration in YAML format. + + Returns: + (str): Task of the model ('detect', 'segment', 'classify', 'pose'). + + Raises: + SyntaxError: If the task of the model could not be determined. + """ + + def cfg2task(cfg): + """Guess from YAML dictionary.""" + m = cfg["head"][-1][-2].lower() # output module name + if m in {"classify", "classifier", "cls", "fc"}: + return "classify" + if m == "detect" or m == "v10detect": + return "detect" + if m == "segment": + return "segment" + if m == "pose": + return "pose" + if m == "obb": + return "obb" + + # Guess from model cfg + if isinstance(model, dict): + with contextlib.suppress(Exception): + return cfg2task(model) + + # Guess from PyTorch model + if isinstance(model, nn.Module): # PyTorch model + for x in "model.args", "model.model.args", "model.model.model.args": + with contextlib.suppress(Exception): + return eval(x)["task"] + for x in "model.yaml", "model.model.yaml", "model.model.model.yaml": + with contextlib.suppress(Exception): + return cfg2task(eval(x)) + + for m in model.modules(): + if isinstance(m, Segment): + return "segment" + elif isinstance(m, Classify): + return "classify" + elif isinstance(m, Pose): + return "pose" + elif isinstance(m, OBB): + return "obb" + elif isinstance(m, (Detect, WorldDetect, v10Detect)): + return "detect" + + # Guess from model filename + if isinstance(model, (str, Path)): + model = Path(model) + if "-seg" in model.stem or "segment" in model.parts: + return "segment" + elif "-cls" in model.stem or "classify" in model.parts: + return "classify" + elif "-pose" in model.stem or "pose" in model.parts: + return "pose" + elif "-obb" in model.stem or "obb" in model.parts: + return "obb" + elif "detect" in model.parts: + return "detect" + + # Unable to determine task from model + LOGGER.warning( + "WARNING ⚠️ Unable to automatically guess model task, assuming 'task=detect'. " + "Explicitly define task for your model, i.e. 'task=detect', 'segment', 'classify','pose' or 'obb'." + ) + return "detect" # assume detect diff --git a/ultralytics/solutions/__init__.py b/ultralytics/solutions/__init__.py new file mode 100644 index 0000000..9e68dc1 --- /dev/null +++ b/ultralytics/solutions/__init__.py @@ -0,0 +1 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license diff --git a/ultralytics/solutions/ai_gym.py b/ultralytics/solutions/ai_gym.py new file mode 100644 index 0000000..b78cf59 --- /dev/null +++ b/ultralytics/solutions/ai_gym.py @@ -0,0 +1,150 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import cv2 + +from ultralytics.utils.checks import check_imshow +from ultralytics.utils.plotting import Annotator + + +class AIGym: + """A class to manage the gym steps of people in a real-time video stream based on their poses.""" + + def __init__(self): + """Initializes the AIGym with default values for Visual and Image parameters.""" + + # Image and line thickness + self.im0 = None + self.tf = None + + # Keypoints and count information + self.keypoints = None + self.poseup_angle = None + self.posedown_angle = None + self.threshold = 0.001 + + # Store stage, count and angle information + self.angle = None + self.count = None + self.stage = None + self.pose_type = "pushup" + self.kpts_to_check = None + + # Visual Information + self.view_img = False + self.annotator = None + + # Check if environment support imshow + self.env_check = check_imshow(warn=True) + + def set_args( + self, + kpts_to_check, + line_thickness=2, + view_img=False, + pose_up_angle=145.0, + pose_down_angle=90.0, + pose_type="pullup", + ): + """ + Configures the AIGym line_thickness, save image and view image parameters. + + Args: + kpts_to_check (list): 3 keypoints for counting + line_thickness (int): Line thickness for bounding boxes. + view_img (bool): display the im0 + pose_up_angle (float): Angle to set pose position up + pose_down_angle (float): Angle to set pose position down + pose_type (str): "pushup", "pullup" or "abworkout" + """ + self.kpts_to_check = kpts_to_check + self.tf = line_thickness + self.view_img = view_img + self.poseup_angle = pose_up_angle + self.posedown_angle = pose_down_angle + self.pose_type = pose_type + + def start_counting(self, im0, results, frame_count): + """ + Function used to count the gym steps. + + Args: + im0 (ndarray): Current frame from the video stream. + results (list): Pose estimation data + frame_count (int): store current frame count + """ + self.im0 = im0 + if frame_count == 1: + self.count = [0] * len(results[0]) + self.angle = [0] * len(results[0]) + self.stage = ["-" for _ in results[0]] + self.keypoints = results[0].keypoints.data + self.annotator = Annotator(im0, line_width=2) + + for ind, k in enumerate(reversed(self.keypoints)): + if self.pose_type in ["pushup", "pullup"]: + self.angle[ind] = self.annotator.estimate_pose_angle( + k[int(self.kpts_to_check[0])].cpu(), + k[int(self.kpts_to_check[1])].cpu(), + k[int(self.kpts_to_check[2])].cpu(), + ) + self.im0 = self.annotator.draw_specific_points(k, self.kpts_to_check, shape=(640, 640), radius=10) + + if self.pose_type == "abworkout": + self.angle[ind] = self.annotator.estimate_pose_angle( + k[int(self.kpts_to_check[0])].cpu(), + k[int(self.kpts_to_check[1])].cpu(), + k[int(self.kpts_to_check[2])].cpu(), + ) + self.im0 = self.annotator.draw_specific_points(k, self.kpts_to_check, shape=(640, 640), radius=10) + if self.angle[ind] > self.poseup_angle: + self.stage[ind] = "down" + if self.angle[ind] < self.posedown_angle and self.stage[ind] == "down": + self.stage[ind] = "up" + self.count[ind] += 1 + self.annotator.plot_angle_and_count_and_stage( + angle_text=self.angle[ind], + count_text=self.count[ind], + stage_text=self.stage[ind], + center_kpt=k[int(self.kpts_to_check[1])], + line_thickness=self.tf, + ) + + if self.pose_type == "pushup": + if self.angle[ind] > self.poseup_angle: + self.stage[ind] = "up" + if self.angle[ind] < self.posedown_angle and self.stage[ind] == "up": + self.stage[ind] = "down" + self.count[ind] += 1 + self.annotator.plot_angle_and_count_and_stage( + angle_text=self.angle[ind], + count_text=self.count[ind], + stage_text=self.stage[ind], + center_kpt=k[int(self.kpts_to_check[1])], + line_thickness=self.tf, + ) + if self.pose_type == "pullup": + if self.angle[ind] > self.poseup_angle: + self.stage[ind] = "down" + if self.angle[ind] < self.posedown_angle and self.stage[ind] == "down": + self.stage[ind] = "up" + self.count[ind] += 1 + self.annotator.plot_angle_and_count_and_stage( + angle_text=self.angle[ind], + count_text=self.count[ind], + stage_text=self.stage[ind], + center_kpt=k[int(self.kpts_to_check[1])], + line_thickness=self.tf, + ) + + self.annotator.kpts(k, shape=(640, 640), radius=1, kpt_line=True) + + if self.env_check and self.view_img: + cv2.imshow("Ultralytics YOLOv8 AI GYM", self.im0) + if cv2.waitKey(1) & 0xFF == ord("q"): + return + + return self.im0 + + +if __name__ == "__main__": + AIGym() diff --git a/ultralytics/solutions/distance_calculation.py b/ultralytics/solutions/distance_calculation.py new file mode 100644 index 0000000..f09209e --- /dev/null +++ b/ultralytics/solutions/distance_calculation.py @@ -0,0 +1,181 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import math + +import cv2 + +from ultralytics.utils.checks import check_imshow +from ultralytics.utils.plotting import Annotator, colors + + +class DistanceCalculation: + """A class to calculate distance between two objects in real-time video stream based on their tracks.""" + + def __init__(self): + """Initializes the distance calculation class with default values for Visual, Image, track and distance + parameters. + """ + + # Visual & im0 information + self.im0 = None + self.annotator = None + self.view_img = False + self.line_color = (255, 255, 0) + self.centroid_color = (255, 0, 255) + + # Predict/track information + self.clss = None + self.names = None + self.boxes = None + self.line_thickness = 2 + self.trk_ids = None + + # Distance calculation information + self.centroids = [] + self.pixel_per_meter = 10 + + # Mouse event + self.left_mouse_count = 0 + self.selected_boxes = {} + + # Check if environment support imshow + self.env_check = check_imshow(warn=True) + + def set_args( + self, + names, + pixels_per_meter=10, + view_img=False, + line_thickness=2, + line_color=(255, 255, 0), + centroid_color=(255, 0, 255), + ): + """ + Configures the distance calculation and display parameters. + + Args: + names (dict): object detection classes names + pixels_per_meter (int): Number of pixels in meter + view_img (bool): Flag indicating frame display + line_thickness (int): Line thickness for bounding boxes. + line_color (RGB): color of centroids line + centroid_color (RGB): colors of bbox centroids + """ + self.names = names + self.pixel_per_meter = pixels_per_meter + self.view_img = view_img + self.line_thickness = line_thickness + self.line_color = line_color + self.centroid_color = centroid_color + + def mouse_event_for_distance(self, event, x, y, flags, param): + """ + This function is designed to move region with mouse events in a real-time video stream. + + Args: + event (int): The type of mouse event (e.g., cv2.EVENT_MOUSEMOVE, cv2.EVENT_LBUTTONDOWN, etc.). + x (int): The x-coordinate of the mouse pointer. + y (int): The y-coordinate of the mouse pointer. + flags (int): Any flags associated with the event (e.g., cv2.EVENT_FLAG_CTRLKEY, + cv2.EVENT_FLAG_SHIFTKEY, etc.). + param (dict): Additional parameters you may want to pass to the function. + """ + global selected_boxes + global left_mouse_count + if event == cv2.EVENT_LBUTTONDOWN: + self.left_mouse_count += 1 + if self.left_mouse_count <= 2: + for box, track_id in zip(self.boxes, self.trk_ids): + if box[0] < x < box[2] and box[1] < y < box[3] and track_id not in self.selected_boxes: + self.selected_boxes[track_id] = [] + self.selected_boxes[track_id] = box + + if event == cv2.EVENT_RBUTTONDOWN: + self.selected_boxes = {} + self.left_mouse_count = 0 + + def extract_tracks(self, tracks): + """ + Extracts results from the provided data. + + Args: + tracks (list): List of tracks obtained from the object tracking process. + """ + self.boxes = tracks[0].boxes.xyxy.cpu() + self.clss = tracks[0].boxes.cls.cpu().tolist() + self.trk_ids = tracks[0].boxes.id.int().cpu().tolist() + + def calculate_centroid(self, box): + """ + Calculate the centroid of bounding box. + + Args: + box (list): Bounding box data + """ + return int((box[0] + box[2]) // 2), int((box[1] + box[3]) // 2) + + def calculate_distance(self, centroid1, centroid2): + """ + Calculate distance between two centroids. + + Args: + centroid1 (point): First bounding box data + centroid2 (point): Second bounding box data + """ + pixel_distance = math.sqrt((centroid1[0] - centroid2[0]) ** 2 + (centroid1[1] - centroid2[1]) ** 2) + return pixel_distance / self.pixel_per_meter, (pixel_distance / self.pixel_per_meter) * 1000 + + def start_process(self, im0, tracks): + """ + Calculate distance between two bounding boxes based on tracking data. + + Args: + im0 (nd array): Image + tracks (list): List of tracks obtained from the object tracking process. + """ + self.im0 = im0 + if tracks[0].boxes.id is None: + if self.view_img: + self.display_frames() + return + self.extract_tracks(tracks) + + self.annotator = Annotator(self.im0, line_width=2) + + for box, cls, track_id in zip(self.boxes, self.clss, self.trk_ids): + self.annotator.box_label(box, color=colors(int(cls), True), label=self.names[int(cls)]) + + if len(self.selected_boxes) == 2: + for trk_id, _ in self.selected_boxes.items(): + if trk_id == track_id: + self.selected_boxes[track_id] = box + + if len(self.selected_boxes) == 2: + for trk_id, box in self.selected_boxes.items(): + centroid = self.calculate_centroid(self.selected_boxes[trk_id]) + self.centroids.append(centroid) + + distance_m, distance_mm = self.calculate_distance(self.centroids[0], self.centroids[1]) + self.annotator.plot_distance_and_line( + distance_m, distance_mm, self.centroids, self.line_color, self.centroid_color + ) + + self.centroids = [] + + if self.view_img and self.env_check: + self.display_frames() + + return im0 + + def display_frames(self): + """Display frame.""" + cv2.namedWindow("Ultralytics Distance Estimation") + cv2.setMouseCallback("Ultralytics Distance Estimation", self.mouse_event_for_distance) + cv2.imshow("Ultralytics Distance Estimation", self.im0) + + if cv2.waitKey(1) & 0xFF == ord("q"): + return + + +if __name__ == "__main__": + DistanceCalculation() diff --git a/ultralytics/solutions/heatmap.py b/ultralytics/solutions/heatmap.py new file mode 100644 index 0000000..f70e62b --- /dev/null +++ b/ultralytics/solutions/heatmap.py @@ -0,0 +1,281 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from collections import defaultdict + +import cv2 +import numpy as np + +from ultralytics.utils.checks import check_imshow, check_requirements +from ultralytics.utils.plotting import Annotator + +check_requirements("shapely>=2.0.0") + +from shapely.geometry import LineString, Point, Polygon + + +class Heatmap: + """A class to draw heatmaps in real-time video stream based on their tracks.""" + + def __init__(self): + """Initializes the heatmap class with default values for Visual, Image, track, count and heatmap parameters.""" + + # Visual information + self.annotator = None + self.view_img = False + self.shape = "circle" + + # Image information + self.imw = None + self.imh = None + self.im0 = None + self.view_in_counts = True + self.view_out_counts = True + + # Heatmap colormap and heatmap np array + self.colormap = None + self.heatmap = None + self.heatmap_alpha = 0.5 + + # Predict/track information + self.boxes = None + self.track_ids = None + self.clss = None + self.track_history = defaultdict(list) + + # Region & Line Information + self.count_reg_pts = None + self.counting_region = None + self.line_dist_thresh = 15 + self.region_thickness = 5 + self.region_color = (255, 0, 255) + + # Object Counting Information + self.in_counts = 0 + self.out_counts = 0 + self.counting_list = [] + self.count_txt_thickness = 0 + self.count_txt_color = (0, 0, 0) + self.count_color = (255, 255, 255) + + # Decay factor + self.decay_factor = 0.99 + + # Check if environment support imshow + self.env_check = check_imshow(warn=True) + + def set_args( + self, + imw, + imh, + colormap=cv2.COLORMAP_JET, + heatmap_alpha=0.5, + view_img=False, + view_in_counts=True, + view_out_counts=True, + count_reg_pts=None, + count_txt_thickness=2, + count_txt_color=(0, 0, 0), + count_color=(255, 255, 255), + count_reg_color=(255, 0, 255), + region_thickness=5, + line_dist_thresh=15, + decay_factor=0.99, + shape="circle", + ): + """ + Configures the heatmap colormap, width, height and display parameters. + + Args: + colormap (cv2.COLORMAP): The colormap to be set. + imw (int): The width of the frame. + imh (int): The height of the frame. + heatmap_alpha (float): alpha value for heatmap display + view_img (bool): Flag indicating frame display + view_in_counts (bool): Flag to control whether to display the incounts on video stream. + view_out_counts (bool): Flag to control whether to display the outcounts on video stream. + count_reg_pts (list): Object counting region points + count_txt_thickness (int): Text thickness for object counting display + count_txt_color (RGB color): count text color value + count_color (RGB color): count text background color value + count_reg_color (RGB color): Color of object counting region + region_thickness (int): Object counting Region thickness + line_dist_thresh (int): Euclidean Distance threshold for line counter + decay_factor (float): value for removing heatmap area after object passed + shape (str): Heatmap shape, rect or circle shape supported + """ + self.imw = imw + self.imh = imh + self.heatmap_alpha = heatmap_alpha + self.view_img = view_img + self.view_in_counts = view_in_counts + self.view_out_counts = view_out_counts + self.colormap = colormap + + # Region and line selection + if count_reg_pts is not None: + if len(count_reg_pts) == 2: + print("Line Counter Initiated.") + self.count_reg_pts = count_reg_pts + self.counting_region = LineString(count_reg_pts) + + elif len(count_reg_pts) == 4: + print("Region Counter Initiated.") + self.count_reg_pts = count_reg_pts + self.counting_region = Polygon(self.count_reg_pts) + + else: + print("Region or line points Invalid, 2 or 4 points supported") + print("Using Line Counter Now") + self.counting_region = Polygon([(20, 400), (1260, 400)]) # dummy points + + # Heatmap new frame + self.heatmap = np.zeros((int(self.imh), int(self.imw)), dtype=np.float32) + + self.count_txt_thickness = count_txt_thickness + self.count_txt_color = count_txt_color + self.count_color = count_color + self.region_color = count_reg_color + self.region_thickness = region_thickness + self.decay_factor = decay_factor + self.line_dist_thresh = line_dist_thresh + self.shape = shape + + # shape of heatmap, if not selected + if self.shape not in ["circle", "rect"]: + print("Unknown shape value provided, 'circle' & 'rect' supported") + print("Using Circular shape now") + self.shape = "circle" + + def extract_results(self, tracks): + """ + Extracts results from the provided data. + + Args: + tracks (list): List of tracks obtained from the object tracking process. + """ + self.boxes = tracks[0].boxes.xyxy.cpu() + self.clss = tracks[0].boxes.cls.cpu().tolist() + self.track_ids = tracks[0].boxes.id.int().cpu().tolist() + + def generate_heatmap(self, im0, tracks): + """ + Generate heatmap based on tracking data. + + Args: + im0 (nd array): Image + tracks (list): List of tracks obtained from the object tracking process. + """ + self.im0 = im0 + if tracks[0].boxes.id is None: + self.heatmap = np.zeros((int(self.imh), int(self.imw)), dtype=np.float32) + if self.view_img and self.env_check: + self.display_frames() + return im0 + self.heatmap *= self.decay_factor # decay factor + self.extract_results(tracks) + self.annotator = Annotator(self.im0, self.count_txt_thickness, None) + + if self.count_reg_pts is not None: + # Draw counting region + if self.view_in_counts or self.view_out_counts: + self.annotator.draw_region( + reg_pts=self.count_reg_pts, color=self.region_color, thickness=self.region_thickness + ) + + for box, cls, track_id in zip(self.boxes, self.clss, self.track_ids): + if self.shape == "circle": + center = (int((box[0] + box[2]) // 2), int((box[1] + box[3]) // 2)) + radius = min(int(box[2]) - int(box[0]), int(box[3]) - int(box[1])) // 2 + + y, x = np.ogrid[0 : self.heatmap.shape[0], 0 : self.heatmap.shape[1]] + mask = (x - center[0]) ** 2 + (y - center[1]) ** 2 <= radius**2 + + self.heatmap[int(box[1]) : int(box[3]), int(box[0]) : int(box[2])] += ( + 2 * mask[int(box[1]) : int(box[3]), int(box[0]) : int(box[2])] + ) + + else: + self.heatmap[int(box[1]) : int(box[3]), int(box[0]) : int(box[2])] += 2 + + # Store tracking hist + track_line = self.track_history[track_id] + track_line.append((float((box[0] + box[2]) / 2), float((box[1] + box[3]) / 2))) + if len(track_line) > 30: + track_line.pop(0) + + # Count objects + if len(self.count_reg_pts) == 4: + if self.counting_region.contains(Point(track_line[-1])) and track_id not in self.counting_list: + self.counting_list.append(track_id) + if box[0] < self.counting_region.centroid.x: + self.out_counts += 1 + else: + self.in_counts += 1 + + elif len(self.count_reg_pts) == 2: + distance = Point(track_line[-1]).distance(self.counting_region) + if distance < self.line_dist_thresh and track_id not in self.counting_list: + self.counting_list.append(track_id) + if box[0] < self.counting_region.centroid.x: + self.out_counts += 1 + else: + self.in_counts += 1 + else: + for box, cls in zip(self.boxes, self.clss): + if self.shape == "circle": + center = (int((box[0] + box[2]) // 2), int((box[1] + box[3]) // 2)) + radius = min(int(box[2]) - int(box[0]), int(box[3]) - int(box[1])) // 2 + + y, x = np.ogrid[0 : self.heatmap.shape[0], 0 : self.heatmap.shape[1]] + mask = (x - center[0]) ** 2 + (y - center[1]) ** 2 <= radius**2 + + self.heatmap[int(box[1]) : int(box[3]), int(box[0]) : int(box[2])] += ( + 2 * mask[int(box[1]) : int(box[3]), int(box[0]) : int(box[2])] + ) + + else: + self.heatmap[int(box[1]) : int(box[3]), int(box[0]) : int(box[2])] += 2 + + # Normalize, apply colormap to heatmap and combine with original image + heatmap_normalized = cv2.normalize(self.heatmap, None, 0, 255, cv2.NORM_MINMAX) + heatmap_colored = cv2.applyColorMap(heatmap_normalized.astype(np.uint8), self.colormap) + + incount_label = f"In Count : {self.in_counts}" + outcount_label = f"OutCount : {self.out_counts}" + + # Display counts based on user choice + counts_label = None + if not self.view_in_counts and not self.view_out_counts: + counts_label = None + elif not self.view_in_counts: + counts_label = outcount_label + elif not self.view_out_counts: + counts_label = incount_label + else: + counts_label = f"{incount_label} {outcount_label}" + + if self.count_reg_pts is not None and counts_label is not None: + self.annotator.count_labels( + counts=counts_label, + count_txt_size=self.count_txt_thickness, + txt_color=self.count_txt_color, + color=self.count_color, + ) + + self.im0 = cv2.addWeighted(self.im0, 1 - self.heatmap_alpha, heatmap_colored, self.heatmap_alpha, 0) + + if self.env_check and self.view_img: + self.display_frames() + + return self.im0 + + def display_frames(self): + """Display frame.""" + cv2.imshow("Ultralytics Heatmap", self.im0) + + if cv2.waitKey(1) & 0xFF == ord("q"): + return + + +if __name__ == "__main__": + Heatmap() diff --git a/ultralytics/solutions/object_counter.py b/ultralytics/solutions/object_counter.py new file mode 100644 index 0000000..18f42c6 --- /dev/null +++ b/ultralytics/solutions/object_counter.py @@ -0,0 +1,278 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from collections import defaultdict + +import cv2 + +from ultralytics.utils.checks import check_imshow, check_requirements +from ultralytics.utils.plotting import Annotator, colors + +check_requirements("shapely>=2.0.0") + +from shapely.geometry import LineString, Point, Polygon + + +class ObjectCounter: + """A class to manage the counting of objects in a real-time video stream based on their tracks.""" + + def __init__(self): + """Initializes the Counter with default values for various tracking and counting parameters.""" + + # Mouse events + self.is_drawing = False + self.selected_point = None + + # Region & Line Information + self.reg_pts = [(20, 400), (1260, 400)] + self.line_dist_thresh = 15 + self.counting_region = None + self.region_color = (255, 0, 255) + self.region_thickness = 5 + + # Image and annotation Information + self.im0 = None + self.tf = None + self.view_img = False + self.view_in_counts = True + self.view_out_counts = True + + self.names = None # Classes names + self.annotator = None # Annotator + self.window_name = "Ultralytics YOLOv8 Object Counter" + + # Object counting Information + self.in_counts = 0 + self.out_counts = 0 + self.counting_dict = {} + self.count_txt_thickness = 0 + self.count_txt_color = (0, 0, 0) + self.count_color = (255, 255, 255) + + # Tracks info + self.track_history = defaultdict(list) + self.track_thickness = 2 + self.draw_tracks = False + self.track_color = (0, 255, 0) + + # Check if environment support imshow + self.env_check = check_imshow(warn=True) + + def set_args( + self, + classes_names, + reg_pts, + count_reg_color=(255, 0, 255), + line_thickness=2, + track_thickness=2, + view_img=False, + view_in_counts=True, + view_out_counts=True, + draw_tracks=False, + count_txt_thickness=2, + count_txt_color=(0, 0, 0), + count_color=(255, 255, 255), + track_color=(0, 255, 0), + region_thickness=5, + line_dist_thresh=15, + ): + """ + Configures the Counter's image, bounding box line thickness, and counting region points. + + Args: + line_thickness (int): Line thickness for bounding boxes. + view_img (bool): Flag to control whether to display the video stream. + view_in_counts (bool): Flag to control whether to display the incounts on video stream. + view_out_counts (bool): Flag to control whether to display the outcounts on video stream. + reg_pts (list): Initial list of points defining the counting region. + classes_names (dict): Classes names + track_thickness (int): Track thickness + draw_tracks (Bool): draw tracks + count_txt_thickness (int): Text thickness for object counting display + count_txt_color (RGB color): count text color value + count_color (RGB color): count text background color value + count_reg_color (RGB color): Color of object counting region + track_color (RGB color): color for tracks + region_thickness (int): Object counting Region thickness + line_dist_thresh (int): Euclidean Distance threshold for line counter + """ + self.tf = line_thickness + self.view_img = view_img + self.view_in_counts = view_in_counts + self.view_out_counts = view_out_counts + self.track_thickness = track_thickness + self.draw_tracks = draw_tracks + + # Region and line selection + if len(reg_pts) == 2: + print("Line Counter Initiated.") + self.reg_pts = reg_pts + self.counting_region = LineString(self.reg_pts) + elif len(reg_pts) >= 3: + print("Region Counter Initiated.") + self.reg_pts = reg_pts + self.counting_region = Polygon(self.reg_pts) + else: + print("Invalid Region points provided, region_points must be 2 for lines or >= 3 for polygons.") + print("Using Line Counter Now") + self.counting_region = LineString(self.reg_pts) + + self.names = classes_names + self.track_color = track_color + self.count_txt_thickness = count_txt_thickness + self.count_txt_color = count_txt_color + self.count_color = count_color + self.region_color = count_reg_color + self.region_thickness = region_thickness + self.line_dist_thresh = line_dist_thresh + + def mouse_event_for_region(self, event, x, y, flags, params): + """ + This function is designed to move region with mouse events in a real-time video stream. + + Args: + event (int): The type of mouse event (e.g., cv2.EVENT_MOUSEMOVE, cv2.EVENT_LBUTTONDOWN, etc.). + x (int): The x-coordinate of the mouse pointer. + y (int): The y-coordinate of the mouse pointer. + flags (int): Any flags associated with the event (e.g., cv2.EVENT_FLAG_CTRLKEY, + cv2.EVENT_FLAG_SHIFTKEY, etc.). + params (dict): Additional parameters you may want to pass to the function. + """ + if event == cv2.EVENT_LBUTTONDOWN: + for i, point in enumerate(self.reg_pts): + if ( + isinstance(point, (tuple, list)) + and len(point) >= 2 + and (abs(x - point[0]) < 10 and abs(y - point[1]) < 10) + ): + self.selected_point = i + self.is_drawing = True + break + + elif event == cv2.EVENT_MOUSEMOVE: + if self.is_drawing and self.selected_point is not None: + self.reg_pts[self.selected_point] = (x, y) + self.counting_region = Polygon(self.reg_pts) + + elif event == cv2.EVENT_LBUTTONUP: + self.is_drawing = False + self.selected_point = None + + def extract_and_process_tracks(self, tracks): + """Extracts and processes tracks for object counting in a video stream.""" + + # Annotator Init and region drawing + self.annotator = Annotator(self.im0, self.tf, self.names) + + if tracks[0].boxes.id is not None: + boxes = tracks[0].boxes.xyxy.cpu() + clss = tracks[0].boxes.cls.cpu().tolist() + track_ids = tracks[0].boxes.id.int().cpu().tolist() + + # Extract tracks + for box, track_id, cls in zip(boxes, track_ids, clss): + # Draw bounding box + self.annotator.box_label(box, label=f"{track_id}:{self.names[cls]}", color=colors(int(track_id), True)) + + # Draw Tracks + track_line = self.track_history[track_id] + track_line.append((float((box[0] + box[2]) / 2), float((box[1] + box[3]) / 2))) + if len(track_line) > 30: + track_line.pop(0) + + # Draw track trails + if self.draw_tracks: + self.annotator.draw_centroid_and_tracks( + track_line, color=self.track_color, track_thickness=self.track_thickness + ) + + prev_position = self.track_history[track_id][-2] if len(self.track_history[track_id]) > 1 else None + centroid = Point((box[:2] + box[2:]) / 2) + + # Count objects + if len(self.reg_pts) >= 3: # any polygon + is_inside = self.counting_region.contains(centroid) + current_position = "in" if is_inside else "out" + + if prev_position is not None: + if self.counting_dict[track_id] != current_position and is_inside: + self.in_counts += 1 + self.counting_dict[track_id] = "in" + elif self.counting_dict[track_id] != current_position and not is_inside: + self.out_counts += 1 + self.counting_dict[track_id] = "out" + else: + self.counting_dict[track_id] = current_position + + else: + self.counting_dict[track_id] = current_position + + elif len(self.reg_pts) == 2: + if prev_position is not None: + is_inside = (box[0] - prev_position[0]) * ( + self.counting_region.centroid.x - prev_position[0] + ) > 0 + current_position = "in" if is_inside else "out" + + if self.counting_dict[track_id] != current_position and is_inside: + self.in_counts += 1 + self.counting_dict[track_id] = "in" + elif self.counting_dict[track_id] != current_position and not is_inside: + self.out_counts += 1 + self.counting_dict[track_id] = "out" + else: + self.counting_dict[track_id] = current_position + else: + self.counting_dict[track_id] = None + + incount_label = f"In Count : {self.in_counts}" + outcount_label = f"OutCount : {self.out_counts}" + + # Display counts based on user choice + counts_label = None + if not self.view_in_counts and not self.view_out_counts: + counts_label = None + elif not self.view_in_counts: + counts_label = outcount_label + elif not self.view_out_counts: + counts_label = incount_label + else: + counts_label = f"{incount_label} {outcount_label}" + + if counts_label is not None: + self.annotator.count_labels( + counts=counts_label, + count_txt_size=self.count_txt_thickness, + txt_color=self.count_txt_color, + color=self.count_color, + ) + + def display_frames(self): + """Display frame.""" + if self.env_check: + self.annotator.draw_region(reg_pts=self.reg_pts, color=self.region_color, thickness=self.region_thickness) + cv2.namedWindow(self.window_name) + if len(self.reg_pts) == 4: # only add mouse event If user drawn region + cv2.setMouseCallback(self.window_name, self.mouse_event_for_region, {"region_points": self.reg_pts}) + cv2.imshow(self.window_name, self.im0) + # Break Window + if cv2.waitKey(1) & 0xFF == ord("q"): + return + + def start_counting(self, im0, tracks): + """ + Main function to start the object counting process. + + Args: + im0 (ndarray): Current frame from the video stream. + tracks (list): List of tracks obtained from the object tracking process. + """ + self.im0 = im0 # store image + self.extract_and_process_tracks(tracks) # draw region even if no objects + + if self.view_img: + self.display_frames() + return self.im0 + + +if __name__ == "__main__": + ObjectCounter() diff --git a/ultralytics/solutions/speed_estimation.py b/ultralytics/solutions/speed_estimation.py new file mode 100644 index 0000000..f3f1795 --- /dev/null +++ b/ultralytics/solutions/speed_estimation.py @@ -0,0 +1,198 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from collections import defaultdict +from time import time + +import cv2 +import numpy as np + +from ultralytics.utils.checks import check_imshow +from ultralytics.utils.plotting import Annotator, colors + + +class SpeedEstimator: + """A class to estimation speed of objects in real-time video stream based on their tracks.""" + + def __init__(self): + """Initializes the speed-estimator class with default values for Visual, Image, track and speed parameters.""" + + # Visual & im0 information + self.im0 = None + self.annotator = None + self.view_img = False + + # Region information + self.reg_pts = [(20, 400), (1260, 400)] + self.region_thickness = 3 + + # Predict/track information + self.clss = None + self.names = None + self.boxes = None + self.trk_ids = None + self.trk_pts = None + self.line_thickness = 2 + self.trk_history = defaultdict(list) + + # Speed estimator information + self.current_time = 0 + self.dist_data = {} + self.trk_idslist = [] + self.spdl_dist_thresh = 10 + self.trk_previous_times = {} + self.trk_previous_points = {} + + # Check if environment support imshow + self.env_check = check_imshow(warn=True) + + def set_args( + self, + reg_pts, + names, + view_img=False, + line_thickness=2, + region_thickness=5, + spdl_dist_thresh=10, + ): + """ + Configures the speed estimation and display parameters. + + Args: + reg_pts (list): Initial list of points defining the speed calculation region. + names (dict): object detection classes names + view_img (bool): Flag indicating frame display + line_thickness (int): Line thickness for bounding boxes. + region_thickness (int): Speed estimation region thickness + spdl_dist_thresh (int): Euclidean distance threshold for speed line + """ + if reg_pts is None: + print("Region points not provided, using default values") + else: + self.reg_pts = reg_pts + self.names = names + self.view_img = view_img + self.line_thickness = line_thickness + self.region_thickness = region_thickness + self.spdl_dist_thresh = spdl_dist_thresh + + def extract_tracks(self, tracks): + """ + Extracts results from the provided data. + + Args: + tracks (list): List of tracks obtained from the object tracking process. + """ + self.boxes = tracks[0].boxes.xyxy.cpu() + self.clss = tracks[0].boxes.cls.cpu().tolist() + self.trk_ids = tracks[0].boxes.id.int().cpu().tolist() + + def store_track_info(self, track_id, box): + """ + Store track data. + + Args: + track_id (int): object track id. + box (list): object bounding box data + """ + track = self.trk_history[track_id] + bbox_center = (float((box[0] + box[2]) / 2), float((box[1] + box[3]) / 2)) + track.append(bbox_center) + + if len(track) > 30: + track.pop(0) + + self.trk_pts = np.hstack(track).astype(np.int32).reshape((-1, 1, 2)) + return track + + def plot_box_and_track(self, track_id, box, cls, track): + """ + Plot track and bounding box. + + Args: + track_id (int): object track id. + box (list): object bounding box data + cls (str): object class name + track (list): tracking history for tracks path drawing + """ + speed_label = f"{int(self.dist_data[track_id])}km/ph" if track_id in self.dist_data else self.names[int(cls)] + bbox_color = colors(int(track_id)) if track_id in self.dist_data else (255, 0, 255) + + self.annotator.box_label(box, speed_label, bbox_color) + + cv2.polylines(self.im0, [self.trk_pts], isClosed=False, color=(0, 255, 0), thickness=1) + cv2.circle(self.im0, (int(track[-1][0]), int(track[-1][1])), 5, bbox_color, -1) + + def calculate_speed(self, trk_id, track): + """ + Calculation of object speed. + + Args: + trk_id (int): object track id. + track (list): tracking history for tracks path drawing + """ + + if not self.reg_pts[0][0] < track[-1][0] < self.reg_pts[1][0]: + return + if self.reg_pts[1][1] - self.spdl_dist_thresh < track[-1][1] < self.reg_pts[1][1] + self.spdl_dist_thresh: + direction = "known" + + elif self.reg_pts[0][1] - self.spdl_dist_thresh < track[-1][1] < self.reg_pts[0][1] + self.spdl_dist_thresh: + direction = "known" + + else: + direction = "unknown" + + if self.trk_previous_times[trk_id] != 0 and direction != "unknown" and trk_id not in self.trk_idslist: + self.trk_idslist.append(trk_id) + + time_difference = time() - self.trk_previous_times[trk_id] + if time_difference > 0: + dist_difference = np.abs(track[-1][1] - self.trk_previous_points[trk_id][1]) + speed = dist_difference / time_difference + self.dist_data[trk_id] = speed + + self.trk_previous_times[trk_id] = time() + self.trk_previous_points[trk_id] = track[-1] + + def estimate_speed(self, im0, tracks, region_color=(255, 0, 0)): + """ + Calculate object based on tracking data. + + Args: + im0 (nd array): Image + tracks (list): List of tracks obtained from the object tracking process. + region_color (tuple): Color to use when drawing regions. + """ + self.im0 = im0 + if tracks[0].boxes.id is None: + if self.view_img and self.env_check: + self.display_frames() + return im0 + self.extract_tracks(tracks) + + self.annotator = Annotator(self.im0, line_width=2) + self.annotator.draw_region(reg_pts=self.reg_pts, color=region_color, thickness=self.region_thickness) + + for box, trk_id, cls in zip(self.boxes, self.trk_ids, self.clss): + track = self.store_track_info(trk_id, box) + + if trk_id not in self.trk_previous_times: + self.trk_previous_times[trk_id] = 0 + + self.plot_box_and_track(trk_id, box, cls, track) + self.calculate_speed(trk_id, track) + + if self.view_img and self.env_check: + self.display_frames() + + return im0 + + def display_frames(self): + """Display frame.""" + cv2.imshow("Ultralytics Speed Estimation", self.im0) + if cv2.waitKey(1) & 0xFF == ord("q"): + return + + +if __name__ == "__main__": + SpeedEstimator() diff --git a/ultralytics/trackers/README.md b/ultralytics/trackers/README.md new file mode 100644 index 0000000..2cab3c0 --- /dev/null +++ b/ultralytics/trackers/README.md @@ -0,0 +1,321 @@ +# Multi-Object Tracking with Ultralytics YOLO + +YOLOv8 trackers visualization + +Object tracking in the realm of video analytics is a critical task that not only identifies the location and class of objects within the frame but also maintains a unique ID for each detected object as the video progresses. The applications are limitless—ranging from surveillance and security to real-time sports analytics. + +## Why Choose Ultralytics YOLO for Object Tracking? + +The output from Ultralytics trackers is consistent with standard object detection but has the added value of object IDs. This makes it easy to track objects in video streams and perform subsequent analytics. Here's why you should consider using Ultralytics YOLO for your object tracking needs: + +- **Efficiency:** Process video streams in real-time without compromising accuracy. +- **Flexibility:** Supports multiple tracking algorithms and configurations. +- **Ease of Use:** Simple Python API and CLI options for quick integration and deployment. +- **Customizability:** Easy to use with custom trained YOLO models, allowing integration into domain-specific applications. + +**Video Tutorial:** [Object Detection and Tracking with Ultralytics YOLOv8](https://www.youtube.com/embed/hHyHmOtmEgs?si=VNZtXmm45Nb9s-N-). + +## Features at a Glance + +Ultralytics YOLO extends its object detection features to provide robust and versatile object tracking: + +- **Real-Time Tracking:** Seamlessly track objects in high-frame-rate videos. +- **Multiple Tracker Support:** Choose from a variety of established tracking algorithms. +- **Customizable Tracker Configurations:** Tailor the tracking algorithm to meet specific requirements by adjusting various parameters. + +## Available Trackers + +Ultralytics YOLO supports the following tracking algorithms. They can be enabled by passing the relevant YAML configuration file such as `tracker=tracker_type.yaml`: + +- [BoT-SORT](https://github.com/NirAharon/BoT-SORT) - Use `botsort.yaml` to enable this tracker. +- [ByteTrack](https://github.com/ifzhang/ByteTrack) - Use `bytetrack.yaml` to enable this tracker. + +The default tracker is BoT-SORT. + +## Tracking + +To run the tracker on video streams, use a trained Detect, Segment or Pose model such as YOLOv8n, YOLOv8n-seg and YOLOv8n-pose. + +#### Python + +```python +from ultralytics import YOLO + +# Load an official or custom model +model = YOLO("yolov8n.pt") # Load an official Detect model +model = YOLO("yolov8n-seg.pt") # Load an official Segment model +model = YOLO("yolov8n-pose.pt") # Load an official Pose model +model = YOLO("path/to/best.pt") # Load a custom trained model + +# Perform tracking with the model +results = model.track( + source="https://youtu.be/LNwODJXcvt4", show=True +) # Tracking with default tracker +results = model.track( + source="https://youtu.be/LNwODJXcvt4", show=True, tracker="bytetrack.yaml" +) # Tracking with ByteTrack tracker +``` + +#### CLI + +```bash +# Perform tracking with various models using the command line interface +yolo track model=yolov8n.pt source="https://youtu.be/LNwODJXcvt4" # Official Detect model +yolo track model=yolov8n-seg.pt source="https://youtu.be/LNwODJXcvt4" # Official Segment model +yolo track model=yolov8n-pose.pt source="https://youtu.be/LNwODJXcvt4" # Official Pose model +yolo track model=path/to/best.pt source="https://youtu.be/LNwODJXcvt4" # Custom trained model + +# Track using ByteTrack tracker +yolo track model=path/to/best.pt tracker="bytetrack.yaml" +``` + +As can be seen in the above usage, tracking is available for all Detect, Segment and Pose models run on videos or streaming sources. + +## Configuration + +### Tracking Arguments + +Tracking configuration shares properties with Predict mode, such as `conf`, `iou`, and `show`. For further configurations, refer to the [Predict](https://docs.ultralytics.com/modes/predict/) model page. + +#### Python + +```python +from ultralytics import YOLO + +# Configure the tracking parameters and run the tracker +model = YOLO("yolov8n.pt") +results = model.track( + source="https://youtu.be/LNwODJXcvt4", conf=0.3, iou=0.5, show=True +) +``` + +#### CLI + +```bash +# Configure tracking parameters and run the tracker using the command line interface +yolo track model=yolov8n.pt source="https://youtu.be/LNwODJXcvt4" conf=0.3, iou=0.5 show +``` + +### Tracker Selection + +Ultralytics also allows you to use a modified tracker configuration file. To do this, simply make a copy of a tracker config file (for example, `custom_tracker.yaml`) from [ultralytics/cfg/trackers](https://github.com/ultralytics/ultralytics/tree/main/ultralytics/cfg/trackers) and modify any configurations (except the `tracker_type`) as per your needs. + +#### Python + +```python +from ultralytics import YOLO + +# Load the model and run the tracker with a custom configuration file +model = YOLO("yolov8n.pt") +results = model.track( + source="https://youtu.be/LNwODJXcvt4", tracker="custom_tracker.yaml" +) +``` + +#### CLI + +```bash +# Load the model and run the tracker with a custom configuration file using the command line interface +yolo track model=yolov8n.pt source="https://youtu.be/LNwODJXcvt4" tracker='custom_tracker.yaml' +``` + +For a comprehensive list of tracking arguments, refer to the [ultralytics/cfg/trackers](https://github.com/ultralytics/ultralytics/tree/main/ultralytics/cfg/trackers) page. + +## Python Examples + +### Persisting Tracks Loop + +Here is a Python script using OpenCV (`cv2`) and YOLOv8 to run object tracking on video frames. This script still assumes you have already installed the necessary packages (`opencv-python` and `ultralytics`). The `persist=True` argument tells the tracker than the current image or frame is the next in a sequence and to expect tracks from the previous image in the current image. + +#### Python + +```python +import cv2 +from ultralytics import YOLO + +# Load the YOLOv8 model +model = YOLO("yolov8n.pt") + +# Open the video file +video_path = "path/to/video.mp4" +cap = cv2.VideoCapture(video_path) + +# Loop through the video frames +while cap.isOpened(): + # Read a frame from the video + success, frame = cap.read() + + if success: + # Run YOLOv8 tracking on the frame, persisting tracks between frames + results = model.track(frame, persist=True) + + # Visualize the results on the frame + annotated_frame = results[0].plot() + + # Display the annotated frame + cv2.imshow("YOLOv8 Tracking", annotated_frame) + + # Break the loop if 'q' is pressed + if cv2.waitKey(1) & 0xFF == ord("q"): + break + else: + # Break the loop if the end of the video is reached + break + +# Release the video capture object and close the display window +cap.release() +cv2.destroyAllWindows() +``` + +Please note the change from `model(frame)` to `model.track(frame)`, which enables object tracking instead of simple detection. This modified script will run the tracker on each frame of the video, visualize the results, and display them in a window. The loop can be exited by pressing 'q'. + +### Plotting Tracks Over Time + +Visualizing object tracks over consecutive frames can provide valuable insights into the movement patterns and behavior of detected objects within a video. With Ultralytics YOLOv8, plotting these tracks is a seamless and efficient process. + +In the following example, we demonstrate how to utilize YOLOv8's tracking capabilities to plot the movement of detected objects across multiple video frames. This script involves opening a video file, reading it frame by frame, and utilizing the YOLO model to identify and track various objects. By retaining the center points of the detected bounding boxes and connecting them, we can draw lines that represent the paths followed by the tracked objects. + +#### Python + +```python +from collections import defaultdict + +import cv2 +import numpy as np + +from ultralytics import YOLO + +# Load the YOLOv8 model +model = YOLO("yolov8n.pt") + +# Open the video file +video_path = "path/to/video.mp4" +cap = cv2.VideoCapture(video_path) + +# Store the track history +track_history = defaultdict(lambda: []) + +# Loop through the video frames +while cap.isOpened(): + # Read a frame from the video + success, frame = cap.read() + + if success: + # Run YOLOv8 tracking on the frame, persisting tracks between frames + results = model.track(frame, persist=True) + + # Get the boxes and track IDs + boxes = results[0].boxes.xywh.cpu() + track_ids = results[0].boxes.id.int().cpu().tolist() + + # Visualize the results on the frame + annotated_frame = results[0].plot() + + # Plot the tracks + for box, track_id in zip(boxes, track_ids): + x, y, w, h = box + track = track_history[track_id] + track.append((float(x), float(y))) # x, y center point + if len(track) > 30: # retain 90 tracks for 90 frames + track.pop(0) + + # Draw the tracking lines + points = np.hstack(track).astype(np.int32).reshape((-1, 1, 2)) + cv2.polylines( + annotated_frame, + [points], + isClosed=False, + color=(230, 230, 230), + thickness=10, + ) + + # Display the annotated frame + cv2.imshow("YOLOv8 Tracking", annotated_frame) + + # Break the loop if 'q' is pressed + if cv2.waitKey(1) & 0xFF == ord("q"): + break + else: + # Break the loop if the end of the video is reached + break + +# Release the video capture object and close the display window +cap.release() +cv2.destroyAllWindows() +``` + +### Multithreaded Tracking + +Multithreaded tracking provides the capability to run object tracking on multiple video streams simultaneously. This is particularly useful when handling multiple video inputs, such as from multiple surveillance cameras, where concurrent processing can greatly enhance efficiency and performance. + +In the provided Python script, we make use of Python's `threading` module to run multiple instances of the tracker concurrently. Each thread is responsible for running the tracker on one video file, and all the threads run simultaneously in the background. + +To ensure that each thread receives the correct parameters (the video file and the model to use), we define a function `run_tracker_in_thread` that accepts these parameters and contains the main tracking loop. This function reads the video frame by frame, runs the tracker, and displays the results. + +Two different models are used in this example: `yolov8n.pt` and `yolov8n-seg.pt`, each tracking objects in a different video file. The video files are specified in `video_file1` and `video_file2`. + +The `daemon=True` parameter in `threading.Thread` means that these threads will be closed as soon as the main program finishes. We then start the threads with `start()` and use `join()` to make the main thread wait until both tracker threads have finished. + +Finally, after all threads have completed their task, the windows displaying the results are closed using `cv2.destroyAllWindows()`. + +#### Python + +```python +import threading + +import cv2 +from ultralytics import YOLO + + +def run_tracker_in_thread(filename, model): + video = cv2.VideoCapture(filename) + frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT)) + for _ in range(frames): + ret, frame = video.read() + if ret: + results = model.track(source=frame, persist=True) + res_plotted = results[0].plot() + cv2.imshow("p", res_plotted) + if cv2.waitKey(1) == ord("q"): + break + + +# Load the models +model1 = YOLO("yolov8n.pt") +model2 = YOLO("yolov8n-seg.pt") + +# Define the video files for the trackers +video_file1 = "path/to/video1.mp4" +video_file2 = "path/to/video2.mp4" + +# Create the tracker threads +tracker_thread1 = threading.Thread( + target=run_tracker_in_thread, args=(video_file1, model1), daemon=True +) +tracker_thread2 = threading.Thread( + target=run_tracker_in_thread, args=(video_file2, model2), daemon=True +) + +# Start the tracker threads +tracker_thread1.start() +tracker_thread2.start() + +# Wait for the tracker threads to finish +tracker_thread1.join() +tracker_thread2.join() + +# Clean up and close windows +cv2.destroyAllWindows() +``` + +This example can easily be extended to handle more video files and models by creating more threads and applying the same methodology. + +## Contribute New Trackers + +Are you proficient in multi-object tracking and have successfully implemented or adapted a tracking algorithm with Ultralytics YOLO? We invite you to contribute to our Trackers section in [ultralytics/cfg/trackers](https://github.com/ultralytics/ultralytics/tree/main/ultralytics/cfg/trackers)! Your real-world applications and solutions could be invaluable for users working on tracking tasks. + +By contributing to this section, you help expand the scope of tracking solutions available within the Ultralytics YOLO framework, adding another layer of functionality and utility for the community. + +To initiate your contribution, please refer to our [Contributing Guide](https://docs.ultralytics.com/help/contributing) for comprehensive instructions on submitting a Pull Request (PR) 🛠️. We are excited to see what you bring to the table! + +Together, let's enhance the tracking capabilities of the Ultralytics YOLO ecosystem 🙏! diff --git a/ultralytics/trackers/__init__.py b/ultralytics/trackers/__init__.py new file mode 100644 index 0000000..bf51b8d --- /dev/null +++ b/ultralytics/trackers/__init__.py @@ -0,0 +1,7 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from .bot_sort import BOTSORT +from .byte_tracker import BYTETracker +from .track import register_tracker + +__all__ = "register_tracker", "BOTSORT", "BYTETracker" # allow simpler import diff --git a/ultralytics/trackers/basetrack.py b/ultralytics/trackers/basetrack.py new file mode 100644 index 0000000..c900cac --- /dev/null +++ b/ultralytics/trackers/basetrack.py @@ -0,0 +1,105 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +"""This module defines the base classes and structures for object tracking in YOLO.""" + +from collections import OrderedDict + +import numpy as np + + +class TrackState: + """ + Enumeration class representing the possible states of an object being tracked. + + Attributes: + New (int): State when the object is newly detected. + Tracked (int): State when the object is successfully tracked in subsequent frames. + Lost (int): State when the object is no longer tracked. + Removed (int): State when the object is removed from tracking. + """ + + New = 0 + Tracked = 1 + Lost = 2 + Removed = 3 + + +class BaseTrack: + """ + Base class for object tracking, providing foundational attributes and methods. + + Attributes: + _count (int): Class-level counter for unique track IDs. + track_id (int): Unique identifier for the track. + is_activated (bool): Flag indicating whether the track is currently active. + state (TrackState): Current state of the track. + history (OrderedDict): Ordered history of the track's states. + features (list): List of features extracted from the object for tracking. + curr_feature (any): The current feature of the object being tracked. + score (float): The confidence score of the tracking. + start_frame (int): The frame number where tracking started. + frame_id (int): The most recent frame ID processed by the track. + time_since_update (int): Frames passed since the last update. + location (tuple): The location of the object in the context of multi-camera tracking. + + Methods: + end_frame: Returns the ID of the last frame where the object was tracked. + next_id: Increments and returns the next global track ID. + activate: Abstract method to activate the track. + predict: Abstract method to predict the next state of the track. + update: Abstract method to update the track with new data. + mark_lost: Marks the track as lost. + mark_removed: Marks the track as removed. + reset_id: Resets the global track ID counter. + """ + + _count = 0 + + def __init__(self): + """Initializes a new track with unique ID and foundational tracking attributes.""" + self.track_id = 0 + self.is_activated = False + self.state = TrackState.New + self.history = OrderedDict() + self.features = [] + self.curr_feature = None + self.score = 0 + self.start_frame = 0 + self.frame_id = 0 + self.time_since_update = 0 + self.location = (np.inf, np.inf) + + @property + def end_frame(self): + """Return the last frame ID of the track.""" + return self.frame_id + + @staticmethod + def next_id(): + """Increment and return the global track ID counter.""" + BaseTrack._count += 1 + return BaseTrack._count + + def activate(self, *args): + """Abstract method to activate the track with provided arguments.""" + raise NotImplementedError + + def predict(self): + """Abstract method to predict the next state of the track.""" + raise NotImplementedError + + def update(self, *args, **kwargs): + """Abstract method to update the track with new observations.""" + raise NotImplementedError + + def mark_lost(self): + """Mark the track as lost.""" + self.state = TrackState.Lost + + def mark_removed(self): + """Mark the track as removed.""" + self.state = TrackState.Removed + + @staticmethod + def reset_id(): + """Reset the global track ID counter.""" + BaseTrack._count = 0 diff --git a/ultralytics/trackers/bot_sort.py b/ultralytics/trackers/bot_sort.py new file mode 100644 index 0000000..31d5e1b --- /dev/null +++ b/ultralytics/trackers/bot_sort.py @@ -0,0 +1,200 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from collections import deque + +import numpy as np + +from .basetrack import TrackState +from .byte_tracker import BYTETracker, STrack +from .utils import matching +from .utils.gmc import GMC +from .utils.kalman_filter import KalmanFilterXYWH + + +class BOTrack(STrack): + """ + An extended version of the STrack class for YOLOv8, adding object tracking features. + + Attributes: + shared_kalman (KalmanFilterXYWH): A shared Kalman filter for all instances of BOTrack. + smooth_feat (np.ndarray): Smoothed feature vector. + curr_feat (np.ndarray): Current feature vector. + features (deque): A deque to store feature vectors with a maximum length defined by `feat_history`. + alpha (float): Smoothing factor for the exponential moving average of features. + mean (np.ndarray): The mean state of the Kalman filter. + covariance (np.ndarray): The covariance matrix of the Kalman filter. + + Methods: + update_features(feat): Update features vector and smooth it using exponential moving average. + predict(): Predicts the mean and covariance using Kalman filter. + re_activate(new_track, frame_id, new_id): Reactivates a track with updated features and optionally new ID. + update(new_track, frame_id): Update the YOLOv8 instance with new track and frame ID. + tlwh: Property that gets the current position in tlwh format `(top left x, top left y, width, height)`. + multi_predict(stracks): Predicts the mean and covariance of multiple object tracks using shared Kalman filter. + convert_coords(tlwh): Converts tlwh bounding box coordinates to xywh format. + tlwh_to_xywh(tlwh): Convert bounding box to xywh format `(center x, center y, width, height)`. + + Usage: + bo_track = BOTrack(tlwh, score, cls, feat) + bo_track.predict() + bo_track.update(new_track, frame_id) + """ + + shared_kalman = KalmanFilterXYWH() + + def __init__(self, tlwh, score, cls, feat=None, feat_history=50): + """Initialize YOLOv8 object with temporal parameters, such as feature history, alpha and current features.""" + super().__init__(tlwh, score, cls) + + self.smooth_feat = None + self.curr_feat = None + if feat is not None: + self.update_features(feat) + self.features = deque([], maxlen=feat_history) + self.alpha = 0.9 + + def update_features(self, feat): + """Update features vector and smooth it using exponential moving average.""" + feat /= np.linalg.norm(feat) + self.curr_feat = feat + if self.smooth_feat is None: + self.smooth_feat = feat + else: + self.smooth_feat = self.alpha * self.smooth_feat + (1 - self.alpha) * feat + self.features.append(feat) + self.smooth_feat /= np.linalg.norm(self.smooth_feat) + + def predict(self): + """Predicts the mean and covariance using Kalman filter.""" + mean_state = self.mean.copy() + if self.state != TrackState.Tracked: + mean_state[6] = 0 + mean_state[7] = 0 + + self.mean, self.covariance = self.kalman_filter.predict(mean_state, self.covariance) + + def re_activate(self, new_track, frame_id, new_id=False): + """Reactivates a track with updated features and optionally assigns a new ID.""" + if new_track.curr_feat is not None: + self.update_features(new_track.curr_feat) + super().re_activate(new_track, frame_id, new_id) + + def update(self, new_track, frame_id): + """Update the YOLOv8 instance with new track and frame ID.""" + if new_track.curr_feat is not None: + self.update_features(new_track.curr_feat) + super().update(new_track, frame_id) + + @property + def tlwh(self): + """Get current position in bounding box format `(top left x, top left y, width, height)`.""" + if self.mean is None: + return self._tlwh.copy() + ret = self.mean[:4].copy() + ret[:2] -= ret[2:] / 2 + return ret + + @staticmethod + def multi_predict(stracks): + """Predicts the mean and covariance of multiple object tracks using shared Kalman filter.""" + if len(stracks) <= 0: + return + multi_mean = np.asarray([st.mean.copy() for st in stracks]) + multi_covariance = np.asarray([st.covariance for st in stracks]) + for i, st in enumerate(stracks): + if st.state != TrackState.Tracked: + multi_mean[i][6] = 0 + multi_mean[i][7] = 0 + multi_mean, multi_covariance = BOTrack.shared_kalman.multi_predict(multi_mean, multi_covariance) + for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)): + stracks[i].mean = mean + stracks[i].covariance = cov + + def convert_coords(self, tlwh): + """Converts Top-Left-Width-Height bounding box coordinates to X-Y-Width-Height format.""" + return self.tlwh_to_xywh(tlwh) + + @staticmethod + def tlwh_to_xywh(tlwh): + """Convert bounding box to format `(center x, center y, width, height)`.""" + ret = np.asarray(tlwh).copy() + ret[:2] += ret[2:] / 2 + return ret + + +class BOTSORT(BYTETracker): + """ + An extended version of the BYTETracker class for YOLOv8, designed for object tracking with ReID and GMC algorithm. + + Attributes: + proximity_thresh (float): Threshold for spatial proximity (IoU) between tracks and detections. + appearance_thresh (float): Threshold for appearance similarity (ReID embeddings) between tracks and detections. + encoder (object): Object to handle ReID embeddings, set to None if ReID is not enabled. + gmc (GMC): An instance of the GMC algorithm for data association. + args (object): Parsed command-line arguments containing tracking parameters. + + Methods: + get_kalmanfilter(): Returns an instance of KalmanFilterXYWH for object tracking. + init_track(dets, scores, cls, img): Initialize track with detections, scores, and classes. + get_dists(tracks, detections): Get distances between tracks and detections using IoU and (optionally) ReID. + multi_predict(tracks): Predict and track multiple objects with YOLOv8 model. + + Usage: + bot_sort = BOTSORT(args, frame_rate) + bot_sort.init_track(dets, scores, cls, img) + bot_sort.multi_predict(tracks) + + Note: + The class is designed to work with the YOLOv8 object detection model and supports ReID only if enabled via args. + """ + + def __init__(self, args, frame_rate=30): + """Initialize YOLOv8 object with ReID module and GMC algorithm.""" + super().__init__(args, frame_rate) + # ReID module + self.proximity_thresh = args.proximity_thresh + self.appearance_thresh = args.appearance_thresh + + if args.with_reid: + # Haven't supported BoT-SORT(reid) yet + self.encoder = None + self.gmc = GMC(method=args.gmc_method) + + def get_kalmanfilter(self): + """Returns an instance of KalmanFilterXYWH for object tracking.""" + return KalmanFilterXYWH() + + def init_track(self, dets, scores, cls, img=None): + """Initialize track with detections, scores, and classes.""" + if len(dets) == 0: + return [] + if self.args.with_reid and self.encoder is not None: + features_keep = self.encoder.inference(img, dets) + return [BOTrack(xyxy, s, c, f) for (xyxy, s, c, f) in zip(dets, scores, cls, features_keep)] # detections + else: + return [BOTrack(xyxy, s, c) for (xyxy, s, c) in zip(dets, scores, cls)] # detections + + def get_dists(self, tracks, detections): + """Get distances between tracks and detections using IoU and (optionally) ReID embeddings.""" + dists = matching.iou_distance(tracks, detections) + dists_mask = dists > self.proximity_thresh + + # TODO: mot20 + # if not self.args.mot20: + dists = matching.fuse_score(dists, detections) + + if self.args.with_reid and self.encoder is not None: + emb_dists = matching.embedding_distance(tracks, detections) / 2.0 + emb_dists[emb_dists > self.appearance_thresh] = 1.0 + emb_dists[dists_mask] = 1.0 + dists = np.minimum(dists, emb_dists) + return dists + + def multi_predict(self, tracks): + """Predict and track multiple objects with YOLOv8 model.""" + BOTrack.multi_predict(tracks) + + def reset(self): + """Reset tracker.""" + super().reset() + self.gmc.reset_params() diff --git a/ultralytics/trackers/byte_tracker.py b/ultralytics/trackers/byte_tracker.py new file mode 100644 index 0000000..01cbca9 --- /dev/null +++ b/ultralytics/trackers/byte_tracker.py @@ -0,0 +1,444 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import numpy as np + +from .basetrack import BaseTrack, TrackState +from .utils import matching +from .utils.kalman_filter import KalmanFilterXYAH +from ..utils.ops import xywh2ltwh +from ..utils import LOGGER + + +class STrack(BaseTrack): + """ + Single object tracking representation that uses Kalman filtering for state estimation. + + This class is responsible for storing all the information regarding individual tracklets and performs state updates + and predictions based on Kalman filter. + + Attributes: + shared_kalman (KalmanFilterXYAH): Shared Kalman filter that is used across all STrack instances for prediction. + _tlwh (np.ndarray): Private attribute to store top-left corner coordinates and width and height of bounding box. + kalman_filter (KalmanFilterXYAH): Instance of Kalman filter used for this particular object track. + mean (np.ndarray): Mean state estimate vector. + covariance (np.ndarray): Covariance of state estimate. + is_activated (bool): Boolean flag indicating if the track has been activated. + score (float): Confidence score of the track. + tracklet_len (int): Length of the tracklet. + cls (any): Class label for the object. + idx (int): Index or identifier for the object. + frame_id (int): Current frame ID. + start_frame (int): Frame where the object was first detected. + + Methods: + predict(): Predict the next state of the object using Kalman filter. + multi_predict(stracks): Predict the next states for multiple tracks. + multi_gmc(stracks, H): Update multiple track states using a homography matrix. + activate(kalman_filter, frame_id): Activate a new tracklet. + re_activate(new_track, frame_id, new_id): Reactivate a previously lost tracklet. + update(new_track, frame_id): Update the state of a matched track. + convert_coords(tlwh): Convert bounding box to x-y-aspect-height format. + tlwh_to_xyah(tlwh): Convert tlwh bounding box to xyah format. + """ + + shared_kalman = KalmanFilterXYAH() + + def __init__(self, xywh, score, cls): + """Initialize new STrack instance.""" + super().__init__() + # xywh+idx or xywha+idx + assert len(xywh) in [5, 6], f"expected 5 or 6 values but got {len(xywh)}" + self._tlwh = np.asarray(xywh2ltwh(xywh[:4]), dtype=np.float32) + self.kalman_filter = None + self.mean, self.covariance = None, None + self.is_activated = False + + self.score = score + self.tracklet_len = 0 + self.cls = cls + self.idx = xywh[-1] + self.angle = xywh[4] if len(xywh) == 6 else None + + def predict(self): + """Predicts mean and covariance using Kalman filter.""" + mean_state = self.mean.copy() + if self.state != TrackState.Tracked: + mean_state[7] = 0 + self.mean, self.covariance = self.kalman_filter.predict(mean_state, self.covariance) + + @staticmethod + def multi_predict(stracks): + """Perform multi-object predictive tracking using Kalman filter for given stracks.""" + if len(stracks) <= 0: + return + multi_mean = np.asarray([st.mean.copy() for st in stracks]) + multi_covariance = np.asarray([st.covariance for st in stracks]) + for i, st in enumerate(stracks): + if st.state != TrackState.Tracked: + multi_mean[i][7] = 0 + multi_mean, multi_covariance = STrack.shared_kalman.multi_predict(multi_mean, multi_covariance) + for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)): + stracks[i].mean = mean + stracks[i].covariance = cov + + @staticmethod + def multi_gmc(stracks, H=np.eye(2, 3)): + """Update state tracks positions and covariances using a homography matrix.""" + if len(stracks) > 0: + multi_mean = np.asarray([st.mean.copy() for st in stracks]) + multi_covariance = np.asarray([st.covariance for st in stracks]) + + R = H[:2, :2] + R8x8 = np.kron(np.eye(4, dtype=float), R) + t = H[:2, 2] + + for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)): + mean = R8x8.dot(mean) + mean[:2] += t + cov = R8x8.dot(cov).dot(R8x8.transpose()) + + stracks[i].mean = mean + stracks[i].covariance = cov + + def activate(self, kalman_filter, frame_id): + """Start a new tracklet.""" + self.kalman_filter = kalman_filter + self.track_id = self.next_id() + self.mean, self.covariance = self.kalman_filter.initiate(self.convert_coords(self._tlwh)) + + self.tracklet_len = 0 + self.state = TrackState.Tracked + if frame_id == 1: + self.is_activated = True + self.frame_id = frame_id + self.start_frame = frame_id + + def re_activate(self, new_track, frame_id, new_id=False): + """Reactivates a previously lost track with a new detection.""" + self.mean, self.covariance = self.kalman_filter.update( + self.mean, self.covariance, self.convert_coords(new_track.tlwh) + ) + self.tracklet_len = 0 + self.state = TrackState.Tracked + self.is_activated = True + self.frame_id = frame_id + if new_id: + self.track_id = self.next_id() + self.score = new_track.score + self.cls = new_track.cls + self.angle = new_track.angle + self.idx = new_track.idx + + def update(self, new_track, frame_id): + """ + Update the state of a matched track. + + Args: + new_track (STrack): The new track containing updated information. + frame_id (int): The ID of the current frame. + """ + self.frame_id = frame_id + self.tracklet_len += 1 + + new_tlwh = new_track.tlwh + self.mean, self.covariance = self.kalman_filter.update( + self.mean, self.covariance, self.convert_coords(new_tlwh) + ) + self.state = TrackState.Tracked + self.is_activated = True + + self.score = new_track.score + self.cls = new_track.cls + self.angle = new_track.angle + self.idx = new_track.idx + + def convert_coords(self, tlwh): + """Convert a bounding box's top-left-width-height format to its x-y-aspect-height equivalent.""" + return self.tlwh_to_xyah(tlwh) + + @property + def tlwh(self): + """Get current position in bounding box format (top left x, top left y, width, height).""" + if self.mean is None: + return self._tlwh.copy() + ret = self.mean[:4].copy() + ret[2] *= ret[3] + ret[:2] -= ret[2:] / 2 + return ret + + @property + def xyxy(self): + """Convert bounding box to format (min x, min y, max x, max y), i.e., (top left, bottom right).""" + ret = self.tlwh.copy() + ret[2:] += ret[:2] + return ret + + @staticmethod + def tlwh_to_xyah(tlwh): + """Convert bounding box to format (center x, center y, aspect ratio, height), where the aspect ratio is width / + height. + """ + ret = np.asarray(tlwh).copy() + ret[:2] += ret[2:] / 2 + ret[2] /= ret[3] + return ret + + @property + def xywh(self): + """Get current position in bounding box format (center x, center y, width, height).""" + ret = np.asarray(self.tlwh).copy() + ret[:2] += ret[2:] / 2 + return ret + + @property + def xywha(self): + """Get current position in bounding box format (center x, center y, width, height, angle).""" + if self.angle is None: + LOGGER.warning("WARNING ⚠️ `angle` attr not found, returning `xywh` instead.") + return self.xywh + return np.concatenate([self.xywh, self.angle[None]]) + + @property + def result(self): + """Get current tracking results.""" + coords = self.xyxy if self.angle is None else self.xywha + return coords.tolist() + [self.track_id, self.score, self.cls, self.idx] + + def __repr__(self): + """Return a string representation of the BYTETracker object with start and end frames and track ID.""" + return f"OT_{self.track_id}_({self.start_frame}-{self.end_frame})" + + +class BYTETracker: + """ + BYTETracker: A tracking algorithm built on top of YOLOv8 for object detection and tracking. + + The class is responsible for initializing, updating, and managing the tracks for detected objects in a video + sequence. It maintains the state of tracked, lost, and removed tracks over frames, utilizes Kalman filtering for + predicting the new object locations, and performs data association. + + Attributes: + tracked_stracks (list[STrack]): List of successfully activated tracks. + lost_stracks (list[STrack]): List of lost tracks. + removed_stracks (list[STrack]): List of removed tracks. + frame_id (int): The current frame ID. + args (namespace): Command-line arguments. + max_time_lost (int): The maximum frames for a track to be considered as 'lost'. + kalman_filter (object): Kalman Filter object. + + Methods: + update(results, img=None): Updates object tracker with new detections. + get_kalmanfilter(): Returns a Kalman filter object for tracking bounding boxes. + init_track(dets, scores, cls, img=None): Initialize object tracking with detections. + get_dists(tracks, detections): Calculates the distance between tracks and detections. + multi_predict(tracks): Predicts the location of tracks. + reset_id(): Resets the ID counter of STrack. + joint_stracks(tlista, tlistb): Combines two lists of stracks. + sub_stracks(tlista, tlistb): Filters out the stracks present in the second list from the first list. + remove_duplicate_stracks(stracksa, stracksb): Removes duplicate stracks based on IoU. + """ + + def __init__(self, args, frame_rate=30): + """Initialize a YOLOv8 object to track objects with given arguments and frame rate.""" + self.tracked_stracks = [] # type: list[STrack] + self.lost_stracks = [] # type: list[STrack] + self.removed_stracks = [] # type: list[STrack] + + self.frame_id = 0 + self.args = args + self.max_time_lost = int(frame_rate / 30.0 * args.track_buffer) + self.kalman_filter = self.get_kalmanfilter() + self.reset_id() + + def update(self, results, img=None): + """Updates object tracker with new detections and returns tracked object bounding boxes.""" + self.frame_id += 1 + activated_stracks = [] + refind_stracks = [] + lost_stracks = [] + removed_stracks = [] + + scores = results.conf + bboxes = results.xywhr if hasattr(results, "xywhr") else results.xywh + # Add index + bboxes = np.concatenate([bboxes, np.arange(len(bboxes)).reshape(-1, 1)], axis=-1) + cls = results.cls + + remain_inds = scores > self.args.track_high_thresh + inds_low = scores > self.args.track_low_thresh + inds_high = scores < self.args.track_high_thresh + + inds_second = np.logical_and(inds_low, inds_high) + dets_second = bboxes[inds_second] + dets = bboxes[remain_inds] + scores_keep = scores[remain_inds] + scores_second = scores[inds_second] + cls_keep = cls[remain_inds] + cls_second = cls[inds_second] + + detections = self.init_track(dets, scores_keep, cls_keep, img) + # Add newly detected tracklets to tracked_stracks + unconfirmed = [] + tracked_stracks = [] # type: list[STrack] + for track in self.tracked_stracks: + if not track.is_activated: + unconfirmed.append(track) + else: + tracked_stracks.append(track) + # Step 2: First association, with high score detection boxes + strack_pool = self.joint_stracks(tracked_stracks, self.lost_stracks) + # Predict the current location with KF + self.multi_predict(strack_pool) + if hasattr(self, "gmc") and img is not None: + warp = self.gmc.apply(img, dets) + STrack.multi_gmc(strack_pool, warp) + STrack.multi_gmc(unconfirmed, warp) + + dists = self.get_dists(strack_pool, detections) + matches, u_track, u_detection = matching.linear_assignment(dists, thresh=self.args.match_thresh) + + for itracked, idet in matches: + track = strack_pool[itracked] + det = detections[idet] + if track.state == TrackState.Tracked: + track.update(det, self.frame_id) + activated_stracks.append(track) + else: + track.re_activate(det, self.frame_id, new_id=False) + refind_stracks.append(track) + # Step 3: Second association, with low score detection boxes association the untrack to the low score detections + detections_second = self.init_track(dets_second, scores_second, cls_second, img) + r_tracked_stracks = [strack_pool[i] for i in u_track if strack_pool[i].state == TrackState.Tracked] + # TODO + dists = matching.iou_distance(r_tracked_stracks, detections_second) + matches, u_track, u_detection_second = matching.linear_assignment(dists, thresh=0.5) + for itracked, idet in matches: + track = r_tracked_stracks[itracked] + det = detections_second[idet] + if track.state == TrackState.Tracked: + track.update(det, self.frame_id) + activated_stracks.append(track) + else: + track.re_activate(det, self.frame_id, new_id=False) + refind_stracks.append(track) + + for it in u_track: + track = r_tracked_stracks[it] + if track.state != TrackState.Lost: + track.mark_lost() + lost_stracks.append(track) + # Deal with unconfirmed tracks, usually tracks with only one beginning frame + detections = [detections[i] for i in u_detection] + dists = self.get_dists(unconfirmed, detections) + matches, u_unconfirmed, u_detection = matching.linear_assignment(dists, thresh=0.7) + for itracked, idet in matches: + unconfirmed[itracked].update(detections[idet], self.frame_id) + activated_stracks.append(unconfirmed[itracked]) + for it in u_unconfirmed: + track = unconfirmed[it] + track.mark_removed() + removed_stracks.append(track) + # Step 4: Init new stracks + for inew in u_detection: + track = detections[inew] + if track.score < self.args.new_track_thresh: + continue + track.activate(self.kalman_filter, self.frame_id) + activated_stracks.append(track) + # Step 5: Update state + for track in self.lost_stracks: + if self.frame_id - track.end_frame > self.max_time_lost: + track.mark_removed() + removed_stracks.append(track) + + self.tracked_stracks = [t for t in self.tracked_stracks if t.state == TrackState.Tracked] + self.tracked_stracks = self.joint_stracks(self.tracked_stracks, activated_stracks) + self.tracked_stracks = self.joint_stracks(self.tracked_stracks, refind_stracks) + self.lost_stracks = self.sub_stracks(self.lost_stracks, self.tracked_stracks) + self.lost_stracks.extend(lost_stracks) + self.lost_stracks = self.sub_stracks(self.lost_stracks, self.removed_stracks) + self.tracked_stracks, self.lost_stracks = self.remove_duplicate_stracks(self.tracked_stracks, self.lost_stracks) + self.removed_stracks.extend(removed_stracks) + if len(self.removed_stracks) > 1000: + self.removed_stracks = self.removed_stracks[-999:] # clip remove stracks to 1000 maximum + + return np.asarray([x.result for x in self.tracked_stracks if x.is_activated], dtype=np.float32) + + def get_kalmanfilter(self): + """Returns a Kalman filter object for tracking bounding boxes.""" + return KalmanFilterXYAH() + + def init_track(self, dets, scores, cls, img=None): + """Initialize object tracking with detections and scores using STrack algorithm.""" + return [STrack(xyxy, s, c) for (xyxy, s, c) in zip(dets, scores, cls)] if len(dets) else [] # detections + + def get_dists(self, tracks, detections): + """Calculates the distance between tracks and detections using IoU and fuses scores.""" + dists = matching.iou_distance(tracks, detections) + # TODO: mot20 + # if not self.args.mot20: + dists = matching.fuse_score(dists, detections) + return dists + + def multi_predict(self, tracks): + """Returns the predicted tracks using the YOLOv8 network.""" + STrack.multi_predict(tracks) + + @staticmethod + def reset_id(): + """Resets the ID counter of STrack.""" + STrack.reset_id() + + def reset(self): + """Reset tracker.""" + self.tracked_stracks = [] # type: list[STrack] + self.lost_stracks = [] # type: list[STrack] + self.removed_stracks = [] # type: list[STrack] + self.frame_id = 0 + self.kalman_filter = self.get_kalmanfilter() + self.reset_id() + + @staticmethod + def joint_stracks(tlista, tlistb): + """Combine two lists of stracks into a single one.""" + exists = {} + res = [] + for t in tlista: + exists[t.track_id] = 1 + res.append(t) + for t in tlistb: + tid = t.track_id + if not exists.get(tid, 0): + exists[tid] = 1 + res.append(t) + return res + + @staticmethod + def sub_stracks(tlista, tlistb): + """DEPRECATED CODE in https://github.com/ultralytics/ultralytics/pull/1890/ + stracks = {t.track_id: t for t in tlista} + for t in tlistb: + tid = t.track_id + if stracks.get(tid, 0): + del stracks[tid] + return list(stracks.values()) + """ + track_ids_b = {t.track_id for t in tlistb} + return [t for t in tlista if t.track_id not in track_ids_b] + + @staticmethod + def remove_duplicate_stracks(stracksa, stracksb): + """Remove duplicate stracks with non-maximum IoU distance.""" + pdist = matching.iou_distance(stracksa, stracksb) + pairs = np.where(pdist < 0.15) + dupa, dupb = [], [] + for p, q in zip(*pairs): + timep = stracksa[p].frame_id - stracksa[p].start_frame + timeq = stracksb[q].frame_id - stracksb[q].start_frame + if timep > timeq: + dupb.append(q) + else: + dupa.append(p) + resa = [t for i, t in enumerate(stracksa) if i not in dupa] + resb = [t for i, t in enumerate(stracksb) if i not in dupb] + return resa, resb diff --git a/ultralytics/trackers/track.py b/ultralytics/trackers/track.py new file mode 100644 index 0000000..7146a40 --- /dev/null +++ b/ultralytics/trackers/track.py @@ -0,0 +1,89 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from functools import partial +from pathlib import Path + +import torch + +from ultralytics.utils import IterableSimpleNamespace, yaml_load +from ultralytics.utils.checks import check_yaml +from .bot_sort import BOTSORT +from .byte_tracker import BYTETracker + +# A mapping of tracker types to corresponding tracker classes +TRACKER_MAP = {"bytetrack": BYTETracker, "botsort": BOTSORT} + + +def on_predict_start(predictor: object, persist: bool = False) -> None: + """ + Initialize trackers for object tracking during prediction. + + Args: + predictor (object): The predictor object to initialize trackers for. + persist (bool, optional): Whether to persist the trackers if they already exist. Defaults to False. + + Raises: + AssertionError: If the tracker_type is not 'bytetrack' or 'botsort'. + """ + if hasattr(predictor, "trackers") and persist: + return + + tracker = check_yaml(predictor.args.tracker) + cfg = IterableSimpleNamespace(**yaml_load(tracker)) + + if cfg.tracker_type not in ["bytetrack", "botsort"]: + raise AssertionError(f"Only 'bytetrack' and 'botsort' are supported for now, but got '{cfg.tracker_type}'") + + trackers = [] + for _ in range(predictor.dataset.bs): + tracker = TRACKER_MAP[cfg.tracker_type](args=cfg, frame_rate=30) + trackers.append(tracker) + if predictor.dataset.mode != "stream": # only need one tracker for other modes. + break + predictor.trackers = trackers + predictor.vid_path = [None] * predictor.dataset.bs # for determining when to reset tracker on new video + + +def on_predict_postprocess_end(predictor: object, persist: bool = False) -> None: + """ + Postprocess detected boxes and update with object tracking. + + Args: + predictor (object): The predictor object containing the predictions. + persist (bool, optional): Whether to persist the trackers if they already exist. Defaults to False. + """ + path, im0s = predictor.batch[:2] + + is_obb = predictor.args.task == "obb" + is_stream = predictor.dataset.mode == "stream" + for i in range(len(im0s)): + tracker = predictor.trackers[i if is_stream else 0] + vid_path = predictor.save_dir / Path(path[i]).name + if not persist and predictor.vid_path[i if is_stream else 0] != vid_path: + tracker.reset() + predictor.vid_path[i if is_stream else 0] = vid_path + + det = (predictor.results[i].obb if is_obb else predictor.results[i].boxes).cpu().numpy() + if len(det) == 0: + continue + tracks = tracker.update(det, im0s[i]) + if len(tracks) == 0: + continue + idx = tracks[:, -1].astype(int) + predictor.results[i] = predictor.results[i][idx] + + update_args = dict() + update_args["obb" if is_obb else "boxes"] = torch.as_tensor(tracks[:, :-1]) + predictor.results[i].update(**update_args) + + +def register_tracker(model: object, persist: bool) -> None: + """ + Register tracking callbacks to the model for object tracking during prediction. + + Args: + model (object): The model object to register tracking callbacks for. + persist (bool): Whether to persist the trackers if they already exist. + """ + model.add_callback("on_predict_start", partial(on_predict_start, persist=persist)) + model.add_callback("on_predict_postprocess_end", partial(on_predict_postprocess_end, persist=persist)) diff --git a/ultralytics/trackers/utils/__init__.py b/ultralytics/trackers/utils/__init__.py new file mode 100644 index 0000000..9e68dc1 --- /dev/null +++ b/ultralytics/trackers/utils/__init__.py @@ -0,0 +1 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license diff --git a/ultralytics/trackers/utils/gmc.py b/ultralytics/trackers/utils/gmc.py new file mode 100644 index 0000000..806f1b5 --- /dev/null +++ b/ultralytics/trackers/utils/gmc.py @@ -0,0 +1,363 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import copy + +import cv2 +import numpy as np + +from ultralytics.utils import LOGGER + + +class GMC: + """ + Generalized Motion Compensation (GMC) class for tracking and object detection in video frames. + + This class provides methods for tracking and detecting objects based on several tracking algorithms including ORB, + SIFT, ECC, and Sparse Optical Flow. It also supports downscaling of frames for computational efficiency. + + Attributes: + method (str): The method used for tracking. Options include 'orb', 'sift', 'ecc', 'sparseOptFlow', 'none'. + downscale (int): Factor by which to downscale the frames for processing. + prevFrame (np.ndarray): Stores the previous frame for tracking. + prevKeyPoints (list): Stores the keypoints from the previous frame. + prevDescriptors (np.ndarray): Stores the descriptors from the previous frame. + initializedFirstFrame (bool): Flag to indicate if the first frame has been processed. + + Methods: + __init__(self, method='sparseOptFlow', downscale=2): Initializes a GMC object with the specified method + and downscale factor. + apply(self, raw_frame, detections=None): Applies the chosen method to a raw frame and optionally uses + provided detections. + applyEcc(self, raw_frame, detections=None): Applies the ECC algorithm to a raw frame. + applyFeatures(self, raw_frame, detections=None): Applies feature-based methods like ORB or SIFT to a raw frame. + applySparseOptFlow(self, raw_frame, detections=None): Applies the Sparse Optical Flow method to a raw frame. + """ + + def __init__(self, method: str = "sparseOptFlow", downscale: int = 2) -> None: + """ + Initialize a video tracker with specified parameters. + + Args: + method (str): The method used for tracking. Options include 'orb', 'sift', 'ecc', 'sparseOptFlow', 'none'. + downscale (int): Downscale factor for processing frames. + """ + super().__init__() + + self.method = method + self.downscale = max(1, int(downscale)) + + if self.method == "orb": + self.detector = cv2.FastFeatureDetector_create(20) + self.extractor = cv2.ORB_create() + self.matcher = cv2.BFMatcher(cv2.NORM_HAMMING) + + elif self.method == "sift": + self.detector = cv2.SIFT_create(nOctaveLayers=3, contrastThreshold=0.02, edgeThreshold=20) + self.extractor = cv2.SIFT_create(nOctaveLayers=3, contrastThreshold=0.02, edgeThreshold=20) + self.matcher = cv2.BFMatcher(cv2.NORM_L2) + + elif self.method == "ecc": + number_of_iterations = 5000 + termination_eps = 1e-6 + self.warp_mode = cv2.MOTION_EUCLIDEAN + self.criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, number_of_iterations, termination_eps) + + elif self.method == "sparseOptFlow": + self.feature_params = dict( + maxCorners=1000, qualityLevel=0.01, minDistance=1, blockSize=3, useHarrisDetector=False, k=0.04 + ) + + elif self.method in {"none", "None", None}: + self.method = None + else: + raise ValueError(f"Error: Unknown GMC method:{method}") + + self.prevFrame = None + self.prevKeyPoints = None + self.prevDescriptors = None + self.initializedFirstFrame = False + + def apply(self, raw_frame: np.array, detections: list = None) -> np.array: + """ + Apply object detection on a raw frame using specified method. + + Args: + raw_frame (np.ndarray): The raw frame to be processed. + detections (list): List of detections to be used in the processing. + + Returns: + (np.ndarray): Processed frame. + + Examples: + >>> gmc = GMC() + >>> gmc.apply(np.array([[1, 2, 3], [4, 5, 6]])) + array([[1, 2, 3], + [4, 5, 6]]) + """ + if self.method in ["orb", "sift"]: + return self.applyFeatures(raw_frame, detections) + elif self.method == "ecc": + return self.applyEcc(raw_frame) + elif self.method == "sparseOptFlow": + return self.applySparseOptFlow(raw_frame) + else: + return np.eye(2, 3) + + def applyEcc(self, raw_frame: np.array) -> np.array: + """ + Apply ECC algorithm to a raw frame. + + Args: + raw_frame (np.ndarray): The raw frame to be processed. + + Returns: + (np.ndarray): Processed frame. + + Examples: + >>> gmc = GMC() + >>> gmc.applyEcc(np.array([[1, 2, 3], [4, 5, 6]])) + array([[1, 2, 3], + [4, 5, 6]]) + """ + height, width, _ = raw_frame.shape + frame = cv2.cvtColor(raw_frame, cv2.COLOR_BGR2GRAY) + H = np.eye(2, 3, dtype=np.float32) + + # Downscale image + if self.downscale > 1.0: + frame = cv2.GaussianBlur(frame, (3, 3), 1.5) + frame = cv2.resize(frame, (width // self.downscale, height // self.downscale)) + width = width // self.downscale + height = height // self.downscale + + # Handle first frame + if not self.initializedFirstFrame: + # Initialize data + self.prevFrame = frame.copy() + + # Initialization done + self.initializedFirstFrame = True + + return H + + # Run the ECC algorithm. The results are stored in warp_matrix. + # (cc, H) = cv2.findTransformECC(self.prevFrame, frame, H, self.warp_mode, self.criteria) + try: + (_, H) = cv2.findTransformECC(self.prevFrame, frame, H, self.warp_mode, self.criteria, None, 1) + except Exception as e: + LOGGER.warning(f"WARNING: find transform failed. Set warp as identity {e}") + + return H + + def applyFeatures(self, raw_frame: np.array, detections: list = None) -> np.array: + """ + Apply feature-based methods like ORB or SIFT to a raw frame. + + Args: + raw_frame (np.ndarray): The raw frame to be processed. + detections (list): List of detections to be used in the processing. + + Returns: + (np.ndarray): Processed frame. + + Examples: + >>> gmc = GMC() + >>> gmc.applyFeatures(np.array([[1, 2, 3], [4, 5, 6]])) + array([[1, 2, 3], + [4, 5, 6]]) + """ + height, width, _ = raw_frame.shape + frame = cv2.cvtColor(raw_frame, cv2.COLOR_BGR2GRAY) + H = np.eye(2, 3) + + # Downscale image + if self.downscale > 1.0: + frame = cv2.resize(frame, (width // self.downscale, height // self.downscale)) + width = width // self.downscale + height = height // self.downscale + + # Find the keypoints + mask = np.zeros_like(frame) + mask[int(0.02 * height) : int(0.98 * height), int(0.02 * width) : int(0.98 * width)] = 255 + if detections is not None: + for det in detections: + tlbr = (det[:4] / self.downscale).astype(np.int_) + mask[tlbr[1] : tlbr[3], tlbr[0] : tlbr[2]] = 0 + + keypoints = self.detector.detect(frame, mask) + + # Compute the descriptors + keypoints, descriptors = self.extractor.compute(frame, keypoints) + + # Handle first frame + if not self.initializedFirstFrame: + # Initialize data + self.prevFrame = frame.copy() + self.prevKeyPoints = copy.copy(keypoints) + self.prevDescriptors = copy.copy(descriptors) + + # Initialization done + self.initializedFirstFrame = True + + return H + + # Match descriptors + knnMatches = self.matcher.knnMatch(self.prevDescriptors, descriptors, 2) + + # Filter matches based on smallest spatial distance + matches = [] + spatialDistances = [] + + maxSpatialDistance = 0.25 * np.array([width, height]) + + # Handle empty matches case + if len(knnMatches) == 0: + # Store to next iteration + self.prevFrame = frame.copy() + self.prevKeyPoints = copy.copy(keypoints) + self.prevDescriptors = copy.copy(descriptors) + + return H + + for m, n in knnMatches: + if m.distance < 0.9 * n.distance: + prevKeyPointLocation = self.prevKeyPoints[m.queryIdx].pt + currKeyPointLocation = keypoints[m.trainIdx].pt + + spatialDistance = ( + prevKeyPointLocation[0] - currKeyPointLocation[0], + prevKeyPointLocation[1] - currKeyPointLocation[1], + ) + + if (np.abs(spatialDistance[0]) < maxSpatialDistance[0]) and ( + np.abs(spatialDistance[1]) < maxSpatialDistance[1] + ): + spatialDistances.append(spatialDistance) + matches.append(m) + + meanSpatialDistances = np.mean(spatialDistances, 0) + stdSpatialDistances = np.std(spatialDistances, 0) + + inliers = (spatialDistances - meanSpatialDistances) < 2.5 * stdSpatialDistances + + goodMatches = [] + prevPoints = [] + currPoints = [] + for i in range(len(matches)): + if inliers[i, 0] and inliers[i, 1]: + goodMatches.append(matches[i]) + prevPoints.append(self.prevKeyPoints[matches[i].queryIdx].pt) + currPoints.append(keypoints[matches[i].trainIdx].pt) + + prevPoints = np.array(prevPoints) + currPoints = np.array(currPoints) + + # Draw the keypoint matches on the output image + # if False: + # import matplotlib.pyplot as plt + # matches_img = np.hstack((self.prevFrame, frame)) + # matches_img = cv2.cvtColor(matches_img, cv2.COLOR_GRAY2BGR) + # W = self.prevFrame.shape[1] + # for m in goodMatches: + # prev_pt = np.array(self.prevKeyPoints[m.queryIdx].pt, dtype=np.int_) + # curr_pt = np.array(keypoints[m.trainIdx].pt, dtype=np.int_) + # curr_pt[0] += W + # color = np.random.randint(0, 255, 3) + # color = (int(color[0]), int(color[1]), int(color[2])) + # + # matches_img = cv2.line(matches_img, prev_pt, curr_pt, tuple(color), 1, cv2.LINE_AA) + # matches_img = cv2.circle(matches_img, prev_pt, 2, tuple(color), -1) + # matches_img = cv2.circle(matches_img, curr_pt, 2, tuple(color), -1) + # + # plt.figure() + # plt.imshow(matches_img) + # plt.show() + + # Find rigid matrix + if prevPoints.shape[0] > 4: + H, inliers = cv2.estimateAffinePartial2D(prevPoints, currPoints, cv2.RANSAC) + + # Handle downscale + if self.downscale > 1.0: + H[0, 2] *= self.downscale + H[1, 2] *= self.downscale + else: + LOGGER.warning("WARNING: not enough matching points") + + # Store to next iteration + self.prevFrame = frame.copy() + self.prevKeyPoints = copy.copy(keypoints) + self.prevDescriptors = copy.copy(descriptors) + + return H + + def applySparseOptFlow(self, raw_frame: np.array) -> np.array: + """ + Apply Sparse Optical Flow method to a raw frame. + + Args: + raw_frame (np.ndarray): The raw frame to be processed. + + Returns: + (np.ndarray): Processed frame. + + Examples: + >>> gmc = GMC() + >>> gmc.applySparseOptFlow(np.array([[1, 2, 3], [4, 5, 6]])) + array([[1, 2, 3], + [4, 5, 6]]) + """ + height, width, _ = raw_frame.shape + frame = cv2.cvtColor(raw_frame, cv2.COLOR_BGR2GRAY) + H = np.eye(2, 3) + + # Downscale image + if self.downscale > 1.0: + frame = cv2.resize(frame, (width // self.downscale, height // self.downscale)) + + # Find the keypoints + keypoints = cv2.goodFeaturesToTrack(frame, mask=None, **self.feature_params) + + # Handle first frame + if not self.initializedFirstFrame: + self.prevFrame = frame.copy() + self.prevKeyPoints = copy.copy(keypoints) + self.initializedFirstFrame = True + return H + + # Find correspondences + matchedKeypoints, status, _ = cv2.calcOpticalFlowPyrLK(self.prevFrame, frame, self.prevKeyPoints, None) + + # Leave good correspondences only + prevPoints = [] + currPoints = [] + + for i in range(len(status)): + if status[i]: + prevPoints.append(self.prevKeyPoints[i]) + currPoints.append(matchedKeypoints[i]) + + prevPoints = np.array(prevPoints) + currPoints = np.array(currPoints) + + # Find rigid matrix + if (prevPoints.shape[0] > 4) and (prevPoints.shape[0] == prevPoints.shape[0]): + H, _ = cv2.estimateAffinePartial2D(prevPoints, currPoints, cv2.RANSAC) + + if self.downscale > 1.0: + H[0, 2] *= self.downscale + H[1, 2] *= self.downscale + else: + LOGGER.warning("WARNING: not enough matching points") + + self.prevFrame = frame.copy() + self.prevKeyPoints = copy.copy(keypoints) + + return H + + def reset_params(self) -> None: + """Reset parameters.""" + self.prevFrame = None + self.prevKeyPoints = None + self.prevDescriptors = None + self.initializedFirstFrame = False diff --git a/ultralytics/trackers/utils/kalman_filter.py b/ultralytics/trackers/utils/kalman_filter.py new file mode 100644 index 0000000..4ae68be --- /dev/null +++ b/ultralytics/trackers/utils/kalman_filter.py @@ -0,0 +1,360 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import numpy as np +import scipy.linalg + + +class KalmanFilterXYAH: + """ + For bytetrack. A simple Kalman filter for tracking bounding boxes in image space. + + The 8-dimensional state space (x, y, a, h, vx, vy, va, vh) contains the bounding box center position (x, y), aspect + ratio a, height h, and their respective velocities. + + Object motion follows a constant velocity model. The bounding box location (x, y, a, h) is taken as direct + observation of the state space (linear observation model). + """ + + def __init__(self): + """Initialize Kalman filter model matrices with motion and observation uncertainty weights.""" + ndim, dt = 4, 1.0 + + # Create Kalman filter model matrices + self._motion_mat = np.eye(2 * ndim, 2 * ndim) + for i in range(ndim): + self._motion_mat[i, ndim + i] = dt + self._update_mat = np.eye(ndim, 2 * ndim) + + # Motion and observation uncertainty are chosen relative to the current state estimate. These weights control + # the amount of uncertainty in the model. + self._std_weight_position = 1.0 / 20 + self._std_weight_velocity = 1.0 / 160 + + def initiate(self, measurement: np.ndarray) -> tuple: + """ + Create track from unassociated measurement. + + Args: + measurement (ndarray): Bounding box coordinates (x, y, a, h) with center position (x, y), aspect ratio a, + and height h. + + Returns: + (tuple[ndarray, ndarray]): Returns the mean vector (8 dimensional) and covariance matrix (8x8 dimensional) of + the new track. Unobserved velocities are initialized to 0 mean. + """ + mean_pos = measurement + mean_vel = np.zeros_like(mean_pos) + mean = np.r_[mean_pos, mean_vel] + + std = [ + 2 * self._std_weight_position * measurement[3], + 2 * self._std_weight_position * measurement[3], + 1e-2, + 2 * self._std_weight_position * measurement[3], + 10 * self._std_weight_velocity * measurement[3], + 10 * self._std_weight_velocity * measurement[3], + 1e-5, + 10 * self._std_weight_velocity * measurement[3], + ] + covariance = np.diag(np.square(std)) + return mean, covariance + + def predict(self, mean: np.ndarray, covariance: np.ndarray) -> tuple: + """ + Run Kalman filter prediction step. + + Args: + mean (ndarray): The 8 dimensional mean vector of the object state at the previous time step. + covariance (ndarray): The 8x8 dimensional covariance matrix of the object state at the previous time step. + + Returns: + (tuple[ndarray, ndarray]): Returns the mean vector and covariance matrix of the predicted state. Unobserved + velocities are initialized to 0 mean. + """ + std_pos = [ + self._std_weight_position * mean[3], + self._std_weight_position * mean[3], + 1e-2, + self._std_weight_position * mean[3], + ] + std_vel = [ + self._std_weight_velocity * mean[3], + self._std_weight_velocity * mean[3], + 1e-5, + self._std_weight_velocity * mean[3], + ] + motion_cov = np.diag(np.square(np.r_[std_pos, std_vel])) + + mean = np.dot(mean, self._motion_mat.T) + covariance = np.linalg.multi_dot((self._motion_mat, covariance, self._motion_mat.T)) + motion_cov + + return mean, covariance + + def project(self, mean: np.ndarray, covariance: np.ndarray) -> tuple: + """ + Project state distribution to measurement space. + + Args: + mean (ndarray): The state's mean vector (8 dimensional array). + covariance (ndarray): The state's covariance matrix (8x8 dimensional). + + Returns: + (tuple[ndarray, ndarray]): Returns the projected mean and covariance matrix of the given state estimate. + """ + std = [ + self._std_weight_position * mean[3], + self._std_weight_position * mean[3], + 1e-1, + self._std_weight_position * mean[3], + ] + innovation_cov = np.diag(np.square(std)) + + mean = np.dot(self._update_mat, mean) + covariance = np.linalg.multi_dot((self._update_mat, covariance, self._update_mat.T)) + return mean, covariance + innovation_cov + + def multi_predict(self, mean: np.ndarray, covariance: np.ndarray) -> tuple: + """ + Run Kalman filter prediction step (Vectorized version). + + Args: + mean (ndarray): The Nx8 dimensional mean matrix of the object states at the previous time step. + covariance (ndarray): The Nx8x8 covariance matrix of the object states at the previous time step. + + Returns: + (tuple[ndarray, ndarray]): Returns the mean vector and covariance matrix of the predicted state. Unobserved + velocities are initialized to 0 mean. + """ + std_pos = [ + self._std_weight_position * mean[:, 3], + self._std_weight_position * mean[:, 3], + 1e-2 * np.ones_like(mean[:, 3]), + self._std_weight_position * mean[:, 3], + ] + std_vel = [ + self._std_weight_velocity * mean[:, 3], + self._std_weight_velocity * mean[:, 3], + 1e-5 * np.ones_like(mean[:, 3]), + self._std_weight_velocity * mean[:, 3], + ] + sqr = np.square(np.r_[std_pos, std_vel]).T + + motion_cov = [np.diag(sqr[i]) for i in range(len(mean))] + motion_cov = np.asarray(motion_cov) + + mean = np.dot(mean, self._motion_mat.T) + left = np.dot(self._motion_mat, covariance).transpose((1, 0, 2)) + covariance = np.dot(left, self._motion_mat.T) + motion_cov + + return mean, covariance + + def update(self, mean: np.ndarray, covariance: np.ndarray, measurement: np.ndarray) -> tuple: + """ + Run Kalman filter correction step. + + Args: + mean (ndarray): The predicted state's mean vector (8 dimensional). + covariance (ndarray): The state's covariance matrix (8x8 dimensional). + measurement (ndarray): The 4 dimensional measurement vector (x, y, a, h), where (x, y) is the center + position, a the aspect ratio, and h the height of the bounding box. + + Returns: + (tuple[ndarray, ndarray]): Returns the measurement-corrected state distribution. + """ + projected_mean, projected_cov = self.project(mean, covariance) + + chol_factor, lower = scipy.linalg.cho_factor(projected_cov, lower=True, check_finite=False) + kalman_gain = scipy.linalg.cho_solve( + (chol_factor, lower), np.dot(covariance, self._update_mat.T).T, check_finite=False + ).T + innovation = measurement - projected_mean + + new_mean = mean + np.dot(innovation, kalman_gain.T) + new_covariance = covariance - np.linalg.multi_dot((kalman_gain, projected_cov, kalman_gain.T)) + return new_mean, new_covariance + + def gating_distance( + self, + mean: np.ndarray, + covariance: np.ndarray, + measurements: np.ndarray, + only_position: bool = False, + metric: str = "maha", + ) -> np.ndarray: + """ + Compute gating distance between state distribution and measurements. A suitable distance threshold can be + obtained from `chi2inv95`. If `only_position` is False, the chi-square distribution has 4 degrees of freedom, + otherwise 2. + + Args: + mean (ndarray): Mean vector over the state distribution (8 dimensional). + covariance (ndarray): Covariance of the state distribution (8x8 dimensional). + measurements (ndarray): An Nx4 matrix of N measurements, each in format (x, y, a, h) where (x, y) + is the bounding box center position, a the aspect ratio, and h the height. + only_position (bool, optional): If True, distance computation is done with respect to the bounding box + center position only. Defaults to False. + metric (str, optional): The metric to use for calculating the distance. Options are 'gaussian' for the + squared Euclidean distance and 'maha' for the squared Mahalanobis distance. Defaults to 'maha'. + + Returns: + (np.ndarray): Returns an array of length N, where the i-th element contains the squared distance between + (mean, covariance) and `measurements[i]`. + """ + mean, covariance = self.project(mean, covariance) + if only_position: + mean, covariance = mean[:2], covariance[:2, :2] + measurements = measurements[:, :2] + + d = measurements - mean + if metric == "gaussian": + return np.sum(d * d, axis=1) + elif metric == "maha": + cholesky_factor = np.linalg.cholesky(covariance) + z = scipy.linalg.solve_triangular(cholesky_factor, d.T, lower=True, check_finite=False, overwrite_b=True) + return np.sum(z * z, axis=0) # square maha + else: + raise ValueError("Invalid distance metric") + + +class KalmanFilterXYWH(KalmanFilterXYAH): + """ + For BoT-SORT. A simple Kalman filter for tracking bounding boxes in image space. + + The 8-dimensional state space (x, y, w, h, vx, vy, vw, vh) contains the bounding box center position (x, y), width + w, height h, and their respective velocities. + + Object motion follows a constant velocity model. The bounding box location (x, y, w, h) is taken as direct + observation of the state space (linear observation model). + """ + + def initiate(self, measurement: np.ndarray) -> tuple: + """ + Create track from unassociated measurement. + + Args: + measurement (ndarray): Bounding box coordinates (x, y, w, h) with center position (x, y), width, and height. + + Returns: + (tuple[ndarray, ndarray]): Returns the mean vector (8 dimensional) and covariance matrix (8x8 dimensional) of + the new track. Unobserved velocities are initialized to 0 mean. + """ + mean_pos = measurement + mean_vel = np.zeros_like(mean_pos) + mean = np.r_[mean_pos, mean_vel] + + std = [ + 2 * self._std_weight_position * measurement[2], + 2 * self._std_weight_position * measurement[3], + 2 * self._std_weight_position * measurement[2], + 2 * self._std_weight_position * measurement[3], + 10 * self._std_weight_velocity * measurement[2], + 10 * self._std_weight_velocity * measurement[3], + 10 * self._std_weight_velocity * measurement[2], + 10 * self._std_weight_velocity * measurement[3], + ] + covariance = np.diag(np.square(std)) + return mean, covariance + + def predict(self, mean, covariance) -> tuple: + """ + Run Kalman filter prediction step. + + Args: + mean (ndarray): The 8 dimensional mean vector of the object state at the previous time step. + covariance (ndarray): The 8x8 dimensional covariance matrix of the object state at the previous time step. + + Returns: + (tuple[ndarray, ndarray]): Returns the mean vector and covariance matrix of the predicted state. Unobserved + velocities are initialized to 0 mean. + """ + std_pos = [ + self._std_weight_position * mean[2], + self._std_weight_position * mean[3], + self._std_weight_position * mean[2], + self._std_weight_position * mean[3], + ] + std_vel = [ + self._std_weight_velocity * mean[2], + self._std_weight_velocity * mean[3], + self._std_weight_velocity * mean[2], + self._std_weight_velocity * mean[3], + ] + motion_cov = np.diag(np.square(np.r_[std_pos, std_vel])) + + mean = np.dot(mean, self._motion_mat.T) + covariance = np.linalg.multi_dot((self._motion_mat, covariance, self._motion_mat.T)) + motion_cov + + return mean, covariance + + def project(self, mean, covariance) -> tuple: + """ + Project state distribution to measurement space. + + Args: + mean (ndarray): The state's mean vector (8 dimensional array). + covariance (ndarray): The state's covariance matrix (8x8 dimensional). + + Returns: + (tuple[ndarray, ndarray]): Returns the projected mean and covariance matrix of the given state estimate. + """ + std = [ + self._std_weight_position * mean[2], + self._std_weight_position * mean[3], + self._std_weight_position * mean[2], + self._std_weight_position * mean[3], + ] + innovation_cov = np.diag(np.square(std)) + + mean = np.dot(self._update_mat, mean) + covariance = np.linalg.multi_dot((self._update_mat, covariance, self._update_mat.T)) + return mean, covariance + innovation_cov + + def multi_predict(self, mean, covariance) -> tuple: + """ + Run Kalman filter prediction step (Vectorized version). + + Args: + mean (ndarray): The Nx8 dimensional mean matrix of the object states at the previous time step. + covariance (ndarray): The Nx8x8 covariance matrix of the object states at the previous time step. + + Returns: + (tuple[ndarray, ndarray]): Returns the mean vector and covariance matrix of the predicted state. Unobserved + velocities are initialized to 0 mean. + """ + std_pos = [ + self._std_weight_position * mean[:, 2], + self._std_weight_position * mean[:, 3], + self._std_weight_position * mean[:, 2], + self._std_weight_position * mean[:, 3], + ] + std_vel = [ + self._std_weight_velocity * mean[:, 2], + self._std_weight_velocity * mean[:, 3], + self._std_weight_velocity * mean[:, 2], + self._std_weight_velocity * mean[:, 3], + ] + sqr = np.square(np.r_[std_pos, std_vel]).T + + motion_cov = [np.diag(sqr[i]) for i in range(len(mean))] + motion_cov = np.asarray(motion_cov) + + mean = np.dot(mean, self._motion_mat.T) + left = np.dot(self._motion_mat, covariance).transpose((1, 0, 2)) + covariance = np.dot(left, self._motion_mat.T) + motion_cov + + return mean, covariance + + def update(self, mean, covariance, measurement) -> tuple: + """ + Run Kalman filter correction step. + + Args: + mean (ndarray): The predicted state's mean vector (8 dimensional). + covariance (ndarray): The state's covariance matrix (8x8 dimensional). + measurement (ndarray): The 4 dimensional measurement vector (x, y, w, h), where (x, y) is the center + position, w the width, and h the height of the bounding box. + + Returns: + (tuple[ndarray, ndarray]): Returns the measurement-corrected state distribution. + """ + return super().update(mean, covariance, measurement) diff --git a/ultralytics/trackers/utils/matching.py b/ultralytics/trackers/utils/matching.py new file mode 100644 index 0000000..fa72b8b --- /dev/null +++ b/ultralytics/trackers/utils/matching.py @@ -0,0 +1,138 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import numpy as np +import scipy +from scipy.spatial.distance import cdist + +from ultralytics.utils.metrics import bbox_ioa, batch_probiou + +try: + import lap # for linear_assignment + + assert lap.__version__ # verify package is not directory +except (ImportError, AssertionError, AttributeError): + from ultralytics.utils.checks import check_requirements + + check_requirements("lapx>=0.5.2") # update to lap package from https://github.com/rathaROG/lapx + import lap + + +def linear_assignment(cost_matrix: np.ndarray, thresh: float, use_lap: bool = True) -> tuple: + """ + Perform linear assignment using scipy or lap.lapjv. + + Args: + cost_matrix (np.ndarray): The matrix containing cost values for assignments. + thresh (float): Threshold for considering an assignment valid. + use_lap (bool, optional): Whether to use lap.lapjv. Defaults to True. + + Returns: + Tuple with: + - matched indices + - unmatched indices from 'a' + - unmatched indices from 'b' + """ + + if cost_matrix.size == 0: + return np.empty((0, 2), dtype=int), tuple(range(cost_matrix.shape[0])), tuple(range(cost_matrix.shape[1])) + + if use_lap: + # Use lap.lapjv + # https://github.com/gatagat/lap + _, x, y = lap.lapjv(cost_matrix, extend_cost=True, cost_limit=thresh) + matches = [[ix, mx] for ix, mx in enumerate(x) if mx >= 0] + unmatched_a = np.where(x < 0)[0] + unmatched_b = np.where(y < 0)[0] + else: + # Use scipy.optimize.linear_sum_assignment + # https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.linear_sum_assignment.html + x, y = scipy.optimize.linear_sum_assignment(cost_matrix) # row x, col y + matches = np.asarray([[x[i], y[i]] for i in range(len(x)) if cost_matrix[x[i], y[i]] <= thresh]) + if len(matches) == 0: + unmatched_a = list(np.arange(cost_matrix.shape[0])) + unmatched_b = list(np.arange(cost_matrix.shape[1])) + else: + unmatched_a = list(set(np.arange(cost_matrix.shape[0])) - set(matches[:, 0])) + unmatched_b = list(set(np.arange(cost_matrix.shape[1])) - set(matches[:, 1])) + + return matches, unmatched_a, unmatched_b + + +def iou_distance(atracks: list, btracks: list) -> np.ndarray: + """ + Compute cost based on Intersection over Union (IoU) between tracks. + + Args: + atracks (list[STrack] | list[np.ndarray]): List of tracks 'a' or bounding boxes. + btracks (list[STrack] | list[np.ndarray]): List of tracks 'b' or bounding boxes. + + Returns: + (np.ndarray): Cost matrix computed based on IoU. + """ + + if atracks and isinstance(atracks[0], np.ndarray) or btracks and isinstance(btracks[0], np.ndarray): + atlbrs = atracks + btlbrs = btracks + else: + atlbrs = [track.xywha if track.angle is not None else track.xyxy for track in atracks] + btlbrs = [track.xywha if track.angle is not None else track.xyxy for track in btracks] + + ious = np.zeros((len(atlbrs), len(btlbrs)), dtype=np.float32) + if len(atlbrs) and len(btlbrs): + if len(atlbrs[0]) == 5 and len(btlbrs[0]) == 5: + ious = batch_probiou( + np.ascontiguousarray(atlbrs, dtype=np.float32), + np.ascontiguousarray(btlbrs, dtype=np.float32), + ).numpy() + else: + ious = bbox_ioa( + np.ascontiguousarray(atlbrs, dtype=np.float32), + np.ascontiguousarray(btlbrs, dtype=np.float32), + iou=True, + ) + return 1 - ious # cost matrix + + +def embedding_distance(tracks: list, detections: list, metric: str = "cosine") -> np.ndarray: + """ + Compute distance between tracks and detections based on embeddings. + + Args: + tracks (list[STrack]): List of tracks. + detections (list[BaseTrack]): List of detections. + metric (str, optional): Metric for distance computation. Defaults to 'cosine'. + + Returns: + (np.ndarray): Cost matrix computed based on embeddings. + """ + + cost_matrix = np.zeros((len(tracks), len(detections)), dtype=np.float32) + if cost_matrix.size == 0: + return cost_matrix + det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float32) + # for i, track in enumerate(tracks): + # cost_matrix[i, :] = np.maximum(0.0, cdist(track.smooth_feat.reshape(1,-1), det_features, metric)) + track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float32) + cost_matrix = np.maximum(0.0, cdist(track_features, det_features, metric)) # Normalized features + return cost_matrix + + +def fuse_score(cost_matrix: np.ndarray, detections: list) -> np.ndarray: + """ + Fuses cost matrix with detection scores to produce a single similarity matrix. + + Args: + cost_matrix (np.ndarray): The matrix containing cost values for assignments. + detections (list[BaseTrack]): List of detections with scores. + + Returns: + (np.ndarray): Fused similarity matrix. + """ + + if cost_matrix.size == 0: + return cost_matrix + iou_sim = 1 - cost_matrix + det_scores = np.array([det.score for det in detections]) + det_scores = np.expand_dims(det_scores, axis=0).repeat(cost_matrix.shape[0], axis=0) + fuse_sim = iou_sim * det_scores + return 1 - fuse_sim # fuse_cost diff --git a/ultralytics/utils/__init__.py b/ultralytics/utils/__init__.py new file mode 100644 index 0000000..93347f5 --- /dev/null +++ b/ultralytics/utils/__init__.py @@ -0,0 +1,1049 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import contextlib +import inspect +import logging.config +import os +import platform +import re +import subprocess +import sys +import threading +import time +import urllib +import uuid +from pathlib import Path +from types import SimpleNamespace +from typing import Union + +import cv2 +import matplotlib.pyplot as plt +import numpy as np +import torch +import yaml +from tqdm import tqdm as tqdm_original + +from ultralytics import __version__ + +# PyTorch Multi-GPU DDP Constants +RANK = int(os.getenv("RANK", -1)) +LOCAL_RANK = int(os.getenv("LOCAL_RANK", -1)) # https://pytorch.org/docs/stable/elastic/run.html + +# Other Constants +FILE = Path(__file__).resolve() +ROOT = FILE.parents[1] # YOLO +ASSETS = ROOT / "assets" # default images +DEFAULT_CFG_PATH = ROOT / "cfg/default.yaml" +NUM_THREADS = min(8, max(1, os.cpu_count() - 1)) # number of YOLOv5 multiprocessing threads +AUTOINSTALL = str(os.getenv("YOLO_AUTOINSTALL", True)).lower() == "true" # global auto-install mode +VERBOSE = str(os.getenv("YOLO_VERBOSE", True)).lower() == "true" # global verbose mode +TQDM_BAR_FORMAT = "{l_bar}{bar:10}{r_bar}" if VERBOSE else None # tqdm bar format +LOGGING_NAME = "ultralytics" +MACOS, LINUX, WINDOWS = (platform.system() == x for x in ["Darwin", "Linux", "Windows"]) # environment booleans +ARM64 = platform.machine() in ("arm64", "aarch64") # ARM64 booleans +HELP_MSG = """ + Usage examples for running YOLOv8: + + 1. Install the ultralytics package: + + pip install ultralytics + + 2. Use the Python SDK: + + from ultralytics import YOLO + + # Load a model + model = YOLO('yolov8n.yaml') # build a new model from scratch + model = YOLO("yolov8n.pt") # load a pretrained model (recommended for training) + + # Use the model + results = model.train(data="coco128.yaml", epochs=3) # train the model + results = model.val() # evaluate model performance on the validation set + results = model('https://ultralytics.com/images/bus.jpg') # predict on an image + success = model.export(format='onnx') # export the model to ONNX format + + 3. Use the command line interface (CLI): + + YOLOv8 'yolo' CLI commands use the following syntax: + + yolo TASK MODE ARGS + + Where TASK (optional) is one of [detect, segment, classify] + MODE (required) is one of [train, val, predict, export] + ARGS (optional) are any number of custom 'arg=value' pairs like 'imgsz=320' that override defaults. + See all ARGS at https://docs.ultralytics.com/usage/cfg or with 'yolo cfg' + + - Train a detection model for 10 epochs with an initial learning_rate of 0.01 + yolo detect train data=coco128.yaml model=yolov8n.pt epochs=10 lr0=0.01 + + - Predict a YouTube video using a pretrained segmentation model at image size 320: + yolo segment predict model=yolov8n-seg.pt source='https://youtu.be/LNwODJXcvt4' imgsz=320 + + - Val a pretrained detection model at batch-size 1 and image size 640: + yolo detect val model=yolov8n.pt data=coco128.yaml batch=1 imgsz=640 + + - Export a YOLOv8n classification model to ONNX format at image size 224 by 128 (no TASK required) + yolo export model=yolov8n-cls.pt format=onnx imgsz=224,128 + + - Run special commands: + yolo help + yolo checks + yolo version + yolo settings + yolo copy-cfg + yolo cfg + + Docs: https://docs.ultralytics.com + Community: https://community.ultralytics.com + GitHub: https://github.com/ultralytics/ultralytics + """ + +# Settings +torch.set_printoptions(linewidth=320, precision=4, profile="default") +np.set_printoptions(linewidth=320, formatter={"float_kind": "{:11.5g}".format}) # format short g, %precision=5 +cv2.setNumThreads(0) # prevent OpenCV from multithreading (incompatible with PyTorch DataLoader) +os.environ["NUMEXPR_MAX_THREADS"] = str(NUM_THREADS) # NumExpr max threads +os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" # for deterministic training +os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" # suppress verbose TF compiler warnings in Colab + + +class TQDM(tqdm_original): + """ + Custom Ultralytics tqdm class with different default arguments. + + Args: + *args (list): Positional arguments passed to original tqdm. + **kwargs (any): Keyword arguments, with custom defaults applied. + """ + + def __init__(self, *args, **kwargs): + """ + Initialize custom Ultralytics tqdm class with different default arguments. + + Note these can still be overridden when calling TQDM. + """ + kwargs["disable"] = not VERBOSE or kwargs.get("disable", False) # logical 'and' with default value if passed + kwargs.setdefault("bar_format", TQDM_BAR_FORMAT) # override default value if passed + super().__init__(*args, **kwargs) + + +class SimpleClass: + """Ultralytics SimpleClass is a base class providing helpful string representation, error reporting, and attribute + access methods for easier debugging and usage. + """ + + def __str__(self): + """Return a human-readable string representation of the object.""" + attr = [] + for a in dir(self): + v = getattr(self, a) + if not callable(v) and not a.startswith("_"): + if isinstance(v, SimpleClass): + # Display only the module and class name for subclasses + s = f"{a}: {v.__module__}.{v.__class__.__name__} object" + else: + s = f"{a}: {repr(v)}" + attr.append(s) + return f"{self.__module__}.{self.__class__.__name__} object with attributes:\n\n" + "\n".join(attr) + + def __repr__(self): + """Return a machine-readable string representation of the object.""" + return self.__str__() + + def __getattr__(self, attr): + """Custom attribute access error message with helpful information.""" + name = self.__class__.__name__ + raise AttributeError(f"'{name}' object has no attribute '{attr}'. See valid attributes below.\n{self.__doc__}") + + +class IterableSimpleNamespace(SimpleNamespace): + """Ultralytics IterableSimpleNamespace is an extension class of SimpleNamespace that adds iterable functionality and + enables usage with dict() and for loops. + """ + + def __iter__(self): + """Return an iterator of key-value pairs from the namespace's attributes.""" + return iter(vars(self).items()) + + def __str__(self): + """Return a human-readable string representation of the object.""" + return "\n".join(f"{k}={v}" for k, v in vars(self).items()) + + def __getattr__(self, attr): + """Custom attribute access error message with helpful information.""" + name = self.__class__.__name__ + raise AttributeError( + f""" + '{name}' object has no attribute '{attr}'. This may be caused by a modified or out of date ultralytics + 'default.yaml' file.\nPlease update your code with 'pip install -U ultralytics' and if necessary replace + {DEFAULT_CFG_PATH} with the latest version from + https://github.com/ultralytics/ultralytics/blob/main/ultralytics/cfg/default.yaml + """ + ) + + def get(self, key, default=None): + """Return the value of the specified key if it exists; otherwise, return the default value.""" + return getattr(self, key, default) + + +def plt_settings(rcparams=None, backend="Agg"): + """ + Decorator to temporarily set rc parameters and the backend for a plotting function. + + Example: + decorator: @plt_settings({"font.size": 12}) + context manager: with plt_settings({"font.size": 12}): + + Args: + rcparams (dict): Dictionary of rc parameters to set. + backend (str, optional): Name of the backend to use. Defaults to 'Agg'. + + Returns: + (Callable): Decorated function with temporarily set rc parameters and backend. This decorator can be + applied to any function that needs to have specific matplotlib rc parameters and backend for its execution. + """ + + if rcparams is None: + rcparams = {"font.size": 11} + + def decorator(func): + """Decorator to apply temporary rc parameters and backend to a function.""" + + def wrapper(*args, **kwargs): + """Sets rc parameters and backend, calls the original function, and restores the settings.""" + original_backend = plt.get_backend() + if backend.lower() != original_backend.lower(): + plt.close("all") # auto-close()ing of figures upon backend switching is deprecated since 3.8 + plt.switch_backend(backend) + + with plt.rc_context(rcparams): + result = func(*args, **kwargs) + + if backend != original_backend: + plt.close("all") + plt.switch_backend(original_backend) + return result + + return wrapper + + return decorator + + +def set_logging(name=LOGGING_NAME, verbose=True): + """Sets up logging for the given name with UTF-8 encoding support.""" + level = logging.INFO if verbose and RANK in {-1, 0} else logging.ERROR # rank in world for Multi-GPU trainings + + # Configure the console (stdout) encoding to UTF-8 + formatter = logging.Formatter("%(message)s") # Default formatter + if WINDOWS and sys.stdout.encoding != "utf-8": + try: + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8") + elif hasattr(sys.stdout, "buffer"): + import io + + sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8") + else: + sys.stdout.encoding = "utf-8" + except Exception as e: + print(f"Creating custom formatter for non UTF-8 environments due to {e}") + + class CustomFormatter(logging.Formatter): + def format(self, record): + """Sets up logging with UTF-8 encoding and configurable verbosity.""" + return emojis(super().format(record)) + + formatter = CustomFormatter("%(message)s") # Use CustomFormatter to eliminate UTF-8 output as last recourse + + # Create and configure the StreamHandler + stream_handler = logging.StreamHandler(sys.stdout) + stream_handler.setFormatter(formatter) + stream_handler.setLevel(level) + + logger = logging.getLogger(name) + logger.setLevel(level) + logger.addHandler(stream_handler) + logger.propagate = False + return logger + + +# Set logger +LOGGER = set_logging(LOGGING_NAME, verbose=VERBOSE) # define globally (used in train.py, val.py, predict.py, etc.) +for logger in "sentry_sdk", "urllib3.connectionpool": + logging.getLogger(logger).setLevel(logging.CRITICAL + 1) + + +def emojis(string=""): + """Return platform-dependent emoji-safe version of string.""" + return string.encode().decode("ascii", "ignore") if WINDOWS else string + + +class ThreadingLocked: + """ + A decorator class for ensuring thread-safe execution of a function or method. This class can be used as a decorator + to make sure that if the decorated function is called from multiple threads, only one thread at a time will be able + to execute the function. + + Attributes: + lock (threading.Lock): A lock object used to manage access to the decorated function. + + Example: + ```python + from ultralytics.utils import ThreadingLocked + + @ThreadingLocked() + def my_function(): + # Your code here + pass + ``` + """ + + def __init__(self): + """Initializes the decorator class for thread-safe execution of a function or method.""" + self.lock = threading.Lock() + + def __call__(self, f): + """Run thread-safe execution of function or method.""" + from functools import wraps + + @wraps(f) + def decorated(*args, **kwargs): + """Applies thread-safety to the decorated function or method.""" + with self.lock: + return f(*args, **kwargs) + + return decorated + + +def yaml_save(file="data.yaml", data=None, header=""): + """ + Save YAML data to a file. + + Args: + file (str, optional): File name. Default is 'data.yaml'. + data (dict): Data to save in YAML format. + header (str, optional): YAML header to add. + + Returns: + (None): Data is saved to the specified file. + """ + if data is None: + data = {} + file = Path(file) + if not file.parent.exists(): + # Create parent directories if they don't exist + file.parent.mkdir(parents=True, exist_ok=True) + + # Convert Path objects to strings + valid_types = int, float, str, bool, list, tuple, dict, type(None) + for k, v in data.items(): + if not isinstance(v, valid_types): + data[k] = str(v) + + # Dump data to file in YAML format + with open(file, "w", errors="ignore", encoding="utf-8") as f: + if header: + f.write(header) + yaml.safe_dump(data, f, sort_keys=False, allow_unicode=True) + + +def yaml_load(file="data.yaml", append_filename=False): + """ + Load YAML data from a file. + + Args: + file (str, optional): File name. Default is 'data.yaml'. + append_filename (bool): Add the YAML filename to the YAML dictionary. Default is False. + + Returns: + (dict): YAML data and file name. + """ + assert Path(file).suffix in (".yaml", ".yml"), f"Attempting to load non-YAML file {file} with yaml_load()" + with open(file, errors="ignore", encoding="utf-8") as f: + s = f.read() # string + + # Remove special characters + if not s.isprintable(): + s = re.sub(r"[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD\U00010000-\U0010ffff]+", "", s) + + # Add YAML filename to dict and return + data = yaml.safe_load(s) or {} # always return a dict (yaml.safe_load() may return None for empty files) + if append_filename: + data["yaml_file"] = str(file) + return data + + +def yaml_print(yaml_file: Union[str, Path, dict]) -> None: + """ + Pretty prints a YAML file or a YAML-formatted dictionary. + + Args: + yaml_file: The file path of the YAML file or a YAML-formatted dictionary. + + Returns: + (None) + """ + yaml_dict = yaml_load(yaml_file) if isinstance(yaml_file, (str, Path)) else yaml_file + dump = yaml.dump(yaml_dict, sort_keys=False, allow_unicode=True) + LOGGER.info(f"Printing '{colorstr('bold', 'black', yaml_file)}'\n\n{dump}") + + +# Default configuration +DEFAULT_CFG_DICT = yaml_load(DEFAULT_CFG_PATH) +for k, v in DEFAULT_CFG_DICT.items(): + if isinstance(v, str) and v.lower() == "none": + DEFAULT_CFG_DICT[k] = None +DEFAULT_CFG_KEYS = DEFAULT_CFG_DICT.keys() +DEFAULT_CFG = IterableSimpleNamespace(**DEFAULT_CFG_DICT) + + +def is_ubuntu() -> bool: + """ + Check if the OS is Ubuntu. + + Returns: + (bool): True if OS is Ubuntu, False otherwise. + """ + with contextlib.suppress(FileNotFoundError): + with open("/etc/os-release") as f: + return "ID=ubuntu" in f.read() + return False + + +def is_colab(): + """ + Check if the current script is running inside a Google Colab notebook. + + Returns: + (bool): True if running inside a Colab notebook, False otherwise. + """ + return "COLAB_RELEASE_TAG" in os.environ or "COLAB_BACKEND_VERSION" in os.environ + + +def is_kaggle(): + """ + Check if the current script is running inside a Kaggle kernel. + + Returns: + (bool): True if running inside a Kaggle kernel, False otherwise. + """ + return os.environ.get("PWD") == "/kaggle/working" and os.environ.get("KAGGLE_URL_BASE") == "https://www.kaggle.com" + + +def is_jupyter(): + """ + Check if the current script is running inside a Jupyter Notebook. Verified on Colab, Jupyterlab, Kaggle, Paperspace. + + Returns: + (bool): True if running inside a Jupyter Notebook, False otherwise. + """ + with contextlib.suppress(Exception): + from IPython import get_ipython + + return get_ipython() is not None + return False + + +def is_docker() -> bool: + """ + Determine if the script is running inside a Docker container. + + Returns: + (bool): True if the script is running inside a Docker container, False otherwise. + """ + file = Path("/proc/self/cgroup") + if file.exists(): + with open(file) as f: + return "docker" in f.read() + else: + return False + + +def is_online() -> bool: + """ + Check internet connectivity by attempting to connect to a known online host. + + Returns: + (bool): True if connection is successful, False otherwise. + """ + import socket + + for host in "1.1.1.1", "8.8.8.8", "223.5.5.5": # Cloudflare, Google, AliDNS: + try: + test_connection = socket.create_connection(address=(host, 53), timeout=2) + except (socket.timeout, socket.gaierror, OSError): + continue + else: + # If the connection was successful, close it to avoid a ResourceWarning + test_connection.close() + return True + return False + + +ONLINE = is_online() + + +def is_pip_package(filepath: str = __name__) -> bool: + """ + Determines if the file at the given filepath is part of a pip package. + + Args: + filepath (str): The filepath to check. + + Returns: + (bool): True if the file is part of a pip package, False otherwise. + """ + import importlib.util + + # Get the spec for the module + spec = importlib.util.find_spec(filepath) + + # Return whether the spec is not None and the origin is not None (indicating it is a package) + return spec is not None and spec.origin is not None + + +def is_dir_writeable(dir_path: Union[str, Path]) -> bool: + """ + Check if a directory is writeable. + + Args: + dir_path (str | Path): The path to the directory. + + Returns: + (bool): True if the directory is writeable, False otherwise. + """ + return os.access(str(dir_path), os.W_OK) + + +def is_pytest_running(): + """ + Determines whether pytest is currently running or not. + + Returns: + (bool): True if pytest is running, False otherwise. + """ + return ("PYTEST_CURRENT_TEST" in os.environ) or ("pytest" in sys.modules) or ("pytest" in Path(sys.argv[0]).stem) + + +def is_github_action_running() -> bool: + """ + Determine if the current environment is a GitHub Actions runner. + + Returns: + (bool): True if the current environment is a GitHub Actions runner, False otherwise. + """ + return "GITHUB_ACTIONS" in os.environ and "GITHUB_WORKFLOW" in os.environ and "RUNNER_OS" in os.environ + + +def is_git_dir(): + """ + Determines whether the current file is part of a git repository. If the current file is not part of a git + repository, returns None. + + Returns: + (bool): True if current file is part of a git repository. + """ + return get_git_dir() is not None + + +def get_git_dir(): + """ + Determines whether the current file is part of a git repository and if so, returns the repository root directory. If + the current file is not part of a git repository, returns None. + + Returns: + (Path | None): Git root directory if found or None if not found. + """ + for d in Path(__file__).parents: + if (d / ".git").is_dir(): + return d + + +def get_git_origin_url(): + """ + Retrieves the origin URL of a git repository. + + Returns: + (str | None): The origin URL of the git repository or None if not git directory. + """ + if is_git_dir(): + with contextlib.suppress(subprocess.CalledProcessError): + origin = subprocess.check_output(["git", "config", "--get", "remote.origin.url"]) + return origin.decode().strip() + + +def get_git_branch(): + """ + Returns the current git branch name. If not in a git repository, returns None. + + Returns: + (str | None): The current git branch name or None if not a git directory. + """ + if is_git_dir(): + with contextlib.suppress(subprocess.CalledProcessError): + origin = subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]) + return origin.decode().strip() + + +def get_default_args(func): + """ + Returns a dictionary of default arguments for a function. + + Args: + func (callable): The function to inspect. + + Returns: + (dict): A dictionary where each key is a parameter name, and each value is the default value of that parameter. + """ + signature = inspect.signature(func) + return {k: v.default for k, v in signature.parameters.items() if v.default is not inspect.Parameter.empty} + + +def get_ubuntu_version(): + """ + Retrieve the Ubuntu version if the OS is Ubuntu. + + Returns: + (str): Ubuntu version or None if not an Ubuntu OS. + """ + if is_ubuntu(): + with contextlib.suppress(FileNotFoundError, AttributeError): + with open("/etc/os-release") as f: + return re.search(r'VERSION_ID="(\d+\.\d+)"', f.read())[1] + + +def get_user_config_dir(sub_dir="yolov10"): + """ + Return the appropriate config directory based on the environment operating system. + + Args: + sub_dir (str): The name of the subdirectory to create. + + Returns: + (Path): The path to the user config directory. + """ + if WINDOWS: + path = Path.home() / "AppData" / "Roaming" / sub_dir + elif MACOS: # macOS + path = Path.home() / "Library" / "Application Support" / sub_dir + elif LINUX: + path = Path.home() / ".config" / sub_dir + else: + raise ValueError(f"Unsupported operating system: {platform.system()}") + + # GCP and AWS lambda fix, only /tmp is writeable + if not is_dir_writeable(path.parent): + LOGGER.warning( + f"WARNING ⚠️ user config directory '{path}' is not writeable, defaulting to '/tmp' or CWD." + "Alternatively you can define a YOLO_CONFIG_DIR environment variable for this path." + ) + path = Path("/tmp") / sub_dir if is_dir_writeable("/tmp") else Path().cwd() / sub_dir + + # Create the subdirectory if it does not exist + path.mkdir(parents=True, exist_ok=True) + + return path + + +USER_CONFIG_DIR = Path(os.getenv("YOLO_CONFIG_DIR") or get_user_config_dir()) # Ultralytics settings dir +SETTINGS_YAML = USER_CONFIG_DIR / "settings.yaml" + + +def colorstr(*input): + """ + Colors a string based on the provided color and style arguments. Utilizes ANSI escape codes. + See https://en.wikipedia.org/wiki/ANSI_escape_code for more details. + + This function can be called in two ways: + - colorstr('color', 'style', 'your string') + - colorstr('your string') + + In the second form, 'blue' and 'bold' will be applied by default. + + Args: + *input (str): A sequence of strings where the first n-1 strings are color and style arguments, + and the last string is the one to be colored. + + Supported Colors and Styles: + Basic Colors: 'black', 'red', 'green', 'yellow', 'blue', 'magenta', 'cyan', 'white' + Bright Colors: 'bright_black', 'bright_red', 'bright_green', 'bright_yellow', + 'bright_blue', 'bright_magenta', 'bright_cyan', 'bright_white' + Misc: 'end', 'bold', 'underline' + + Returns: + (str): The input string wrapped with ANSI escape codes for the specified color and style. + + Examples: + >>> colorstr('blue', 'bold', 'hello world') + >>> '\033[34m\033[1mhello world\033[0m' + """ + *args, string = input if len(input) > 1 else ("blue", "bold", input[0]) # color arguments, string + colors = { + "black": "\033[30m", # basic colors + "red": "\033[31m", + "green": "\033[32m", + "yellow": "\033[33m", + "blue": "\033[34m", + "magenta": "\033[35m", + "cyan": "\033[36m", + "white": "\033[37m", + "bright_black": "\033[90m", # bright colors + "bright_red": "\033[91m", + "bright_green": "\033[92m", + "bright_yellow": "\033[93m", + "bright_blue": "\033[94m", + "bright_magenta": "\033[95m", + "bright_cyan": "\033[96m", + "bright_white": "\033[97m", + "end": "\033[0m", # misc + "bold": "\033[1m", + "underline": "\033[4m", + } + return "".join(colors[x] for x in args) + f"{string}" + colors["end"] + + +def remove_colorstr(input_string): + """ + Removes ANSI escape codes from a string, effectively un-coloring it. + + Args: + input_string (str): The string to remove color and style from. + + Returns: + (str): A new string with all ANSI escape codes removed. + + Examples: + >>> remove_colorstr(colorstr('blue', 'bold', 'hello world')) + >>> 'hello world' + """ + ansi_escape = re.compile(r"\x1B\[[0-9;]*[A-Za-z]") + return ansi_escape.sub("", input_string) + + +class TryExcept(contextlib.ContextDecorator): + """ + Ultralytics TryExcept class. Use as @TryExcept() decorator or 'with TryExcept():' context manager. + + Examples: + As a decorator: + >>> @TryExcept(msg="Error occurred in func", verbose=True) + >>> def func(): + >>> # Function logic here + >>> pass + + As a context manager: + >>> with TryExcept(msg="Error occurred in block", verbose=True): + >>> # Code block here + >>> pass + """ + + def __init__(self, msg="", verbose=True): + """Initialize TryExcept class with optional message and verbosity settings.""" + self.msg = msg + self.verbose = verbose + + def __enter__(self): + """Executes when entering TryExcept context, initializes instance.""" + pass + + def __exit__(self, exc_type, value, traceback): + """Defines behavior when exiting a 'with' block, prints error message if necessary.""" + if self.verbose and value: + print(emojis(f"{self.msg}{': ' if self.msg else ''}{value}")) + return True + + +class Retry(contextlib.ContextDecorator): + """ + Retry class for function execution with exponential backoff. + + Can be used as a decorator or a context manager to retry a function or block of code on exceptions, up to a + specified number of times with an exponentially increasing delay between retries. + + Examples: + Example usage as a decorator: + >>> @Retry(times=3, delay=2) + >>> def test_func(): + >>> # Replace with function logic that may raise exceptions + >>> return True + + Example usage as a context manager: + >>> with Retry(times=3, delay=2): + >>> # Replace with code block that may raise exceptions + >>> pass + """ + + def __init__(self, times=3, delay=2): + """Initialize Retry class with specified number of retries and delay.""" + self.times = times + self.delay = delay + self._attempts = 0 + + def __call__(self, func): + """Decorator implementation for Retry with exponential backoff.""" + + def wrapped_func(*args, **kwargs): + """Applies retries to the decorated function or method.""" + self._attempts = 0 + while self._attempts < self.times: + try: + return func(*args, **kwargs) + except Exception as e: + self._attempts += 1 + print(f"Retry {self._attempts}/{self.times} failed: {e}") + if self._attempts >= self.times: + raise e + time.sleep(self.delay * (2**self._attempts)) # exponential backoff delay + + return wrapped_func + + def __enter__(self): + """Enter the runtime context related to this object.""" + self._attempts = 0 + + def __exit__(self, exc_type, exc_value, traceback): + """Exit the runtime context related to this object with exponential backoff.""" + if exc_type is not None: + self._attempts += 1 + if self._attempts < self.times: + print(f"Retry {self._attempts}/{self.times} failed: {exc_value}") + time.sleep(self.delay * (2**self._attempts)) # exponential backoff delay + return True # Suppresses the exception and retries + return False # Re-raises the exception if retries are exhausted + + +def threaded(func): + """ + Multi-threads a target function by default and returns the thread or function result. + + Use as @threaded decorator. The function runs in a separate thread unless 'threaded=False' is passed. + """ + + def wrapper(*args, **kwargs): + """Multi-threads a given function based on 'threaded' kwarg and returns the thread or function result.""" + if kwargs.pop("threaded", True): # run in thread + thread = threading.Thread(target=func, args=args, kwargs=kwargs, daemon=True) + thread.start() + return thread + else: + return func(*args, **kwargs) + + return wrapper + + +def set_sentry(): + """ + Initialize the Sentry SDK for error tracking and reporting. Only used if sentry_sdk package is installed and + sync=True in settings. Run 'yolo settings' to see and update settings YAML file. + + Conditions required to send errors (ALL conditions must be met or no errors will be reported): + - sentry_sdk package is installed + - sync=True in YOLO settings + - pytest is not running + - running in a pip package installation + - running in a non-git directory + - running with rank -1 or 0 + - online environment + - CLI used to run package (checked with 'yolo' as the name of the main CLI command) + + The function also configures Sentry SDK to ignore KeyboardInterrupt and FileNotFoundError + exceptions and to exclude events with 'out of memory' in their exception message. + + Additionally, the function sets custom tags and user information for Sentry events. + """ + + def before_send(event, hint): + """ + Modify the event before sending it to Sentry based on specific exception types and messages. + + Args: + event (dict): The event dictionary containing information about the error. + hint (dict): A dictionary containing additional information about the error. + + Returns: + dict: The modified event or None if the event should not be sent to Sentry. + """ + if "exc_info" in hint: + exc_type, exc_value, tb = hint["exc_info"] + if exc_type in (KeyboardInterrupt, FileNotFoundError) or "out of memory" in str(exc_value): + return None # do not send event + + event["tags"] = { + "sys_argv": sys.argv[0], + "sys_argv_name": Path(sys.argv[0]).name, + "install": "git" if is_git_dir() else "pip" if is_pip_package() else "other", + "os": ENVIRONMENT, + } + return event + + if ( + SETTINGS["sync"] + and RANK in (-1, 0) + and Path(sys.argv[0]).name == "yolo" + and not TESTS_RUNNING + and ONLINE + and is_pip_package() + and not is_git_dir() + ): + # If sentry_sdk package is not installed then return and do not use Sentry + try: + import sentry_sdk # noqa + except ImportError: + return + + sentry_sdk.init( + dsn="https://5ff1556b71594bfea135ff0203a0d290@o4504521589325824.ingest.sentry.io/4504521592406016", + debug=False, + traces_sample_rate=1.0, + release=__version__, + environment="production", # 'dev' or 'production' + before_send=before_send, + ignore_errors=[KeyboardInterrupt, FileNotFoundError], + ) + sentry_sdk.set_user({"id": SETTINGS["uuid"]}) # SHA-256 anonymized UUID hash + + +class SettingsManager(dict): + """ + Manages Ultralytics settings stored in a YAML file. + + Args: + file (str | Path): Path to the Ultralytics settings YAML file. Default is USER_CONFIG_DIR / 'settings.yaml'. + version (str): Settings version. In case of local version mismatch, new default settings will be saved. + """ + + def __init__(self, file=SETTINGS_YAML, version="0.0.4"): + """Initialize the SettingsManager with default settings, load and validate current settings from the YAML + file. + """ + import copy + import hashlib + + from ultralytics.utils.checks import check_version + from ultralytics.utils.torch_utils import torch_distributed_zero_first + + git_dir = get_git_dir() + root = git_dir or Path() + datasets_root = (root.parent if git_dir and is_dir_writeable(root.parent) else root).resolve() + + self.file = Path(file) + self.version = version + self.defaults = { + "settings_version": version, + "datasets_dir": str(datasets_root / "datasets"), + "weights_dir": str(root / "weights"), + "runs_dir": str(root / "runs"), + "uuid": hashlib.sha256(str(uuid.getnode()).encode()).hexdigest(), + "sync": True, + "api_key": "", + "openai_api_key": "", + "clearml": True, # integrations + "comet": True, + "dvc": True, + "hub": True, + "mlflow": True, + "neptune": True, + "raytune": True, + "tensorboard": True, + "wandb": True, + } + + super().__init__(copy.deepcopy(self.defaults)) + + with torch_distributed_zero_first(RANK): + if not self.file.exists(): + self.save() + + self.load() + correct_keys = self.keys() == self.defaults.keys() + correct_types = all(type(a) is type(b) for a, b in zip(self.values(), self.defaults.values())) + correct_version = check_version(self["settings_version"], self.version) + help_msg = ( + f"\nView settings with 'yolo settings' or at '{self.file}'" + "\nUpdate settings with 'yolo settings key=value', i.e. 'yolo settings runs_dir=path/to/dir'. " + "For help see https://docs.ultralytics.com/quickstart/#ultralytics-settings." + ) + if not (correct_keys and correct_types and correct_version): + LOGGER.warning( + "WARNING ⚠️ Ultralytics settings reset to default values. This may be due to a possible problem " + f"with your settings or a recent ultralytics package update. {help_msg}" + ) + self.reset() + + if self.get("datasets_dir") == self.get("runs_dir"): + LOGGER.warning( + f"WARNING ⚠️ Ultralytics setting 'datasets_dir: {self.get('datasets_dir')}' " + f"must be different than 'runs_dir: {self.get('runs_dir')}'. " + f"Please change one to avoid possible issues during training. {help_msg}" + ) + + def load(self): + """Loads settings from the YAML file.""" + super().update(yaml_load(self.file)) + + def save(self): + """Saves the current settings to the YAML file.""" + yaml_save(self.file, dict(self)) + + def update(self, *args, **kwargs): + """Updates a setting value in the current settings.""" + super().update(*args, **kwargs) + self.save() + + def reset(self): + """Resets the settings to default and saves them.""" + self.clear() + self.update(self.defaults) + self.save() + + +def deprecation_warn(arg, new_arg, version=None): + """Issue a deprecation warning when a deprecated argument is used, suggesting an updated argument.""" + if not version: + version = float(__version__[:3]) + 0.2 # deprecate after 2nd major release + LOGGER.warning( + f"WARNING ⚠️ '{arg}' is deprecated and will be removed in 'ultralytics {version}' in the future. " + f"Please use '{new_arg}' instead." + ) + + +def clean_url(url): + """Strip auth from URL, i.e. https://url.com/file.txt?auth -> https://url.com/file.txt.""" + url = Path(url).as_posix().replace(":/", "://") # Pathlib turns :// -> :/, as_posix() for Windows + return urllib.parse.unquote(url).split("?")[0] # '%2F' to '/', split https://url.com/file.txt?auth + + +def url2file(url): + """Convert URL to filename, i.e. https://url.com/file.txt?auth -> file.txt.""" + return Path(clean_url(url)).name + + +# Run below code on utils init ------------------------------------------------------------------------------------ + +# Check first-install steps +PREFIX = colorstr("Ultralytics: ") +SETTINGS = SettingsManager() # initialize settings +DATASETS_DIR = Path(SETTINGS["datasets_dir"]) # global datasets directory +WEIGHTS_DIR = Path(SETTINGS["weights_dir"]) # global weights directory +RUNS_DIR = Path(SETTINGS["runs_dir"]) # global runs directory +ENVIRONMENT = ( + "Colab" + if is_colab() + else "Kaggle" + if is_kaggle() + else "Jupyter" + if is_jupyter() + else "Docker" + if is_docker() + else platform.system() +) +TESTS_RUNNING = is_pytest_running() or is_github_action_running() +set_sentry() + +# Apply monkey patches +from .patches import imread, imshow, imwrite, torch_save + +torch.save = torch_save +if WINDOWS: + # Apply cv2 patches for non-ASCII and non-UTF characters in image paths + cv2.imread, cv2.imwrite, cv2.imshow = imread, imwrite, imshow diff --git a/ultralytics/utils/__pycache__/__init__.cpython-312.pyc b/ultralytics/utils/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..5a45360 Binary files /dev/null and b/ultralytics/utils/__pycache__/__init__.cpython-312.pyc differ diff --git a/ultralytics/utils/__pycache__/__init__.cpython-39.pyc b/ultralytics/utils/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..30dadc2 Binary files /dev/null and b/ultralytics/utils/__pycache__/__init__.cpython-39.pyc differ diff --git a/ultralytics/utils/__pycache__/autobatch.cpython-312.pyc b/ultralytics/utils/__pycache__/autobatch.cpython-312.pyc new file mode 100644 index 0000000..ed761cc Binary files /dev/null and b/ultralytics/utils/__pycache__/autobatch.cpython-312.pyc differ diff --git a/ultralytics/utils/__pycache__/autobatch.cpython-39.pyc b/ultralytics/utils/__pycache__/autobatch.cpython-39.pyc new file mode 100644 index 0000000..0915af0 Binary files /dev/null and b/ultralytics/utils/__pycache__/autobatch.cpython-39.pyc differ diff --git a/ultralytics/utils/__pycache__/checks.cpython-312.pyc b/ultralytics/utils/__pycache__/checks.cpython-312.pyc new file mode 100644 index 0000000..feb62b2 Binary files /dev/null and b/ultralytics/utils/__pycache__/checks.cpython-312.pyc differ diff --git a/ultralytics/utils/__pycache__/checks.cpython-39.pyc b/ultralytics/utils/__pycache__/checks.cpython-39.pyc new file mode 100644 index 0000000..771b0dd Binary files /dev/null and b/ultralytics/utils/__pycache__/checks.cpython-39.pyc differ diff --git a/ultralytics/utils/__pycache__/dist.cpython-312.pyc b/ultralytics/utils/__pycache__/dist.cpython-312.pyc new file mode 100644 index 0000000..d4882b0 Binary files /dev/null and b/ultralytics/utils/__pycache__/dist.cpython-312.pyc differ diff --git a/ultralytics/utils/__pycache__/dist.cpython-39.pyc b/ultralytics/utils/__pycache__/dist.cpython-39.pyc new file mode 100644 index 0000000..ec3b3b5 Binary files /dev/null and b/ultralytics/utils/__pycache__/dist.cpython-39.pyc differ diff --git a/ultralytics/utils/__pycache__/downloads.cpython-312.pyc b/ultralytics/utils/__pycache__/downloads.cpython-312.pyc new file mode 100644 index 0000000..2cba1b4 Binary files /dev/null and b/ultralytics/utils/__pycache__/downloads.cpython-312.pyc differ diff --git a/ultralytics/utils/__pycache__/downloads.cpython-39.pyc b/ultralytics/utils/__pycache__/downloads.cpython-39.pyc new file mode 100644 index 0000000..bdac1b6 Binary files /dev/null and b/ultralytics/utils/__pycache__/downloads.cpython-39.pyc differ diff --git a/ultralytics/utils/__pycache__/files.cpython-312.pyc b/ultralytics/utils/__pycache__/files.cpython-312.pyc new file mode 100644 index 0000000..7216264 Binary files /dev/null and b/ultralytics/utils/__pycache__/files.cpython-312.pyc differ diff --git a/ultralytics/utils/__pycache__/files.cpython-39.pyc b/ultralytics/utils/__pycache__/files.cpython-39.pyc new file mode 100644 index 0000000..1b367bb Binary files /dev/null and b/ultralytics/utils/__pycache__/files.cpython-39.pyc differ diff --git a/ultralytics/utils/__pycache__/instance.cpython-312.pyc b/ultralytics/utils/__pycache__/instance.cpython-312.pyc new file mode 100644 index 0000000..8c42fe0 Binary files /dev/null and b/ultralytics/utils/__pycache__/instance.cpython-312.pyc differ diff --git a/ultralytics/utils/__pycache__/instance.cpython-39.pyc b/ultralytics/utils/__pycache__/instance.cpython-39.pyc new file mode 100644 index 0000000..5c8b014 Binary files /dev/null and b/ultralytics/utils/__pycache__/instance.cpython-39.pyc differ diff --git a/ultralytics/utils/__pycache__/loss.cpython-312.pyc b/ultralytics/utils/__pycache__/loss.cpython-312.pyc new file mode 100644 index 0000000..32ecb3f Binary files /dev/null and b/ultralytics/utils/__pycache__/loss.cpython-312.pyc differ diff --git a/ultralytics/utils/__pycache__/loss.cpython-39.pyc b/ultralytics/utils/__pycache__/loss.cpython-39.pyc new file mode 100644 index 0000000..e1d4160 Binary files /dev/null and b/ultralytics/utils/__pycache__/loss.cpython-39.pyc differ diff --git a/ultralytics/utils/__pycache__/metrics.cpython-312.pyc b/ultralytics/utils/__pycache__/metrics.cpython-312.pyc new file mode 100644 index 0000000..25928c5 Binary files /dev/null and b/ultralytics/utils/__pycache__/metrics.cpython-312.pyc differ diff --git a/ultralytics/utils/__pycache__/metrics.cpython-39.pyc b/ultralytics/utils/__pycache__/metrics.cpython-39.pyc new file mode 100644 index 0000000..2bb74a9 Binary files /dev/null and b/ultralytics/utils/__pycache__/metrics.cpython-39.pyc differ diff --git a/ultralytics/utils/__pycache__/ops.cpython-312.pyc b/ultralytics/utils/__pycache__/ops.cpython-312.pyc new file mode 100644 index 0000000..68fb216 Binary files /dev/null and b/ultralytics/utils/__pycache__/ops.cpython-312.pyc differ diff --git a/ultralytics/utils/__pycache__/ops.cpython-39.pyc b/ultralytics/utils/__pycache__/ops.cpython-39.pyc new file mode 100644 index 0000000..3fd77f6 Binary files /dev/null and b/ultralytics/utils/__pycache__/ops.cpython-39.pyc differ diff --git a/ultralytics/utils/__pycache__/patches.cpython-312.pyc b/ultralytics/utils/__pycache__/patches.cpython-312.pyc new file mode 100644 index 0000000..5f69eed Binary files /dev/null and b/ultralytics/utils/__pycache__/patches.cpython-312.pyc differ diff --git a/ultralytics/utils/__pycache__/patches.cpython-39.pyc b/ultralytics/utils/__pycache__/patches.cpython-39.pyc new file mode 100644 index 0000000..b456782 Binary files /dev/null and b/ultralytics/utils/__pycache__/patches.cpython-39.pyc differ diff --git a/ultralytics/utils/__pycache__/plotting.cpython-312.pyc b/ultralytics/utils/__pycache__/plotting.cpython-312.pyc new file mode 100644 index 0000000..a811d62 Binary files /dev/null and b/ultralytics/utils/__pycache__/plotting.cpython-312.pyc differ diff --git a/ultralytics/utils/__pycache__/plotting.cpython-39.pyc b/ultralytics/utils/__pycache__/plotting.cpython-39.pyc new file mode 100644 index 0000000..08c62be Binary files /dev/null and b/ultralytics/utils/__pycache__/plotting.cpython-39.pyc differ diff --git a/ultralytics/utils/__pycache__/tal.cpython-312.pyc b/ultralytics/utils/__pycache__/tal.cpython-312.pyc new file mode 100644 index 0000000..c430079 Binary files /dev/null and b/ultralytics/utils/__pycache__/tal.cpython-312.pyc differ diff --git a/ultralytics/utils/__pycache__/tal.cpython-39.pyc b/ultralytics/utils/__pycache__/tal.cpython-39.pyc new file mode 100644 index 0000000..a324052 Binary files /dev/null and b/ultralytics/utils/__pycache__/tal.cpython-39.pyc differ diff --git a/ultralytics/utils/__pycache__/torch_utils.cpython-312.pyc b/ultralytics/utils/__pycache__/torch_utils.cpython-312.pyc new file mode 100644 index 0000000..bef6c24 Binary files /dev/null and b/ultralytics/utils/__pycache__/torch_utils.cpython-312.pyc differ diff --git a/ultralytics/utils/__pycache__/torch_utils.cpython-39.pyc b/ultralytics/utils/__pycache__/torch_utils.cpython-39.pyc new file mode 100644 index 0000000..18adc00 Binary files /dev/null and b/ultralytics/utils/__pycache__/torch_utils.cpython-39.pyc differ diff --git a/ultralytics/utils/autobatch.py b/ultralytics/utils/autobatch.py new file mode 100644 index 0000000..daea14e --- /dev/null +++ b/ultralytics/utils/autobatch.py @@ -0,0 +1,88 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +"""Functions for estimating the best YOLO batch size to use a fraction of the available CUDA memory in PyTorch.""" + +from copy import deepcopy + +import numpy as np +import torch + +from ultralytics.utils import DEFAULT_CFG, LOGGER, colorstr +from ultralytics.utils.torch_utils import profile + + +def check_train_batch_size(model, imgsz=640, amp=True): + """ + Check YOLO training batch size using the autobatch() function. + + Args: + model (torch.nn.Module): YOLO model to check batch size for. + imgsz (int): Image size used for training. + amp (bool): If True, use automatic mixed precision (AMP) for training. + + Returns: + (int): Optimal batch size computed using the autobatch() function. + """ + + with torch.cuda.amp.autocast(amp): + return autobatch(deepcopy(model).train(), imgsz) # compute optimal batch size + + +def autobatch(model, imgsz=640, fraction=0.60, batch_size=DEFAULT_CFG.batch): + """ + Automatically estimate the best YOLO batch size to use a fraction of the available CUDA memory. + + Args: + model (torch.nn.module): YOLO model to compute batch size for. + imgsz (int, optional): The image size used as input for the YOLO model. Defaults to 640. + fraction (float, optional): The fraction of available CUDA memory to use. Defaults to 0.60. + batch_size (int, optional): The default batch size to use if an error is detected. Defaults to 16. + + Returns: + (int): The optimal batch size. + """ + + # Check device + prefix = colorstr("AutoBatch: ") + LOGGER.info(f"{prefix}Computing optimal batch size for imgsz={imgsz}") + device = next(model.parameters()).device # get model device + if device.type == "cpu": + LOGGER.info(f"{prefix}CUDA not detected, using default CPU batch-size {batch_size}") + return batch_size + if torch.backends.cudnn.benchmark: + LOGGER.info(f"{prefix} ⚠️ Requires torch.backends.cudnn.benchmark=False, using default batch-size {batch_size}") + return batch_size + + # Inspect CUDA memory + gb = 1 << 30 # bytes to GiB (1024 ** 3) + d = str(device).upper() # 'CUDA:0' + properties = torch.cuda.get_device_properties(device) # device properties + t = properties.total_memory / gb # GiB total + r = torch.cuda.memory_reserved(device) / gb # GiB reserved + a = torch.cuda.memory_allocated(device) / gb # GiB allocated + f = t - (r + a) # GiB free + LOGGER.info(f"{prefix}{d} ({properties.name}) {t:.2f}G total, {r:.2f}G reserved, {a:.2f}G allocated, {f:.2f}G free") + + # Profile batch sizes + batch_sizes = [1, 2, 4, 8, 16] + try: + img = [torch.empty(b, 3, imgsz, imgsz) for b in batch_sizes] + results = profile(img, model, n=3, device=device) + + # Fit a solution + y = [x[2] for x in results if x] # memory [2] + p = np.polyfit(batch_sizes[: len(y)], y, deg=1) # first degree polynomial fit + b = int((f * fraction - p[1]) / p[0]) # y intercept (optimal batch size) + if None in results: # some sizes failed + i = results.index(None) # first fail index + if b >= batch_sizes[i]: # y intercept above failure point + b = batch_sizes[max(i - 1, 0)] # select prior safe point + if b < 1 or b > 1024: # b outside of safe range + b = batch_size + LOGGER.info(f"{prefix}WARNING ⚠️ CUDA anomaly detected, using default batch-size {batch_size}.") + + fraction = (np.polyval(p, b) + r + a) / t # actual fraction predicted + LOGGER.info(f"{prefix}Using batch-size {b} for {d} {t * fraction:.2f}G/{t:.2f}G ({fraction * 100:.0f}%) ✅") + return b + except Exception as e: + LOGGER.warning(f"{prefix}WARNING ⚠️ error detected: {e}, using default batch-size {batch_size}.") + return batch_size diff --git a/ultralytics/utils/benchmarks.py b/ultralytics/utils/benchmarks.py new file mode 100644 index 0000000..0286990 --- /dev/null +++ b/ultralytics/utils/benchmarks.py @@ -0,0 +1,404 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +""" +Benchmark a YOLO model formats for speed and accuracy. + +Usage: + from ultralytics.utils.benchmarks import ProfileModels, benchmark + ProfileModels(['yolov8n.yaml', 'yolov8s.yaml']).profile() + benchmark(model='yolov8n.pt', imgsz=160) + +Format | `format=argument` | Model +--- | --- | --- +PyTorch | - | yolov8n.pt +TorchScript | `torchscript` | yolov8n.torchscript +ONNX | `onnx` | yolov8n.onnx +OpenVINO | `openvino` | yolov8n_openvino_model/ +TensorRT | `engine` | yolov8n.engine +CoreML | `coreml` | yolov8n.mlpackage +TensorFlow SavedModel | `saved_model` | yolov8n_saved_model/ +TensorFlow GraphDef | `pb` | yolov8n.pb +TensorFlow Lite | `tflite` | yolov8n.tflite +TensorFlow Edge TPU | `edgetpu` | yolov8n_edgetpu.tflite +TensorFlow.js | `tfjs` | yolov8n_web_model/ +PaddlePaddle | `paddle` | yolov8n_paddle_model/ +NCNN | `ncnn` | yolov8n_ncnn_model/ +""" + +import glob +import platform +import time +from pathlib import Path + +import numpy as np +import torch.cuda + +from ultralytics import YOLO, YOLOWorld +from ultralytics.cfg import TASK2DATA, TASK2METRIC +from ultralytics.engine.exporter import export_formats +from ultralytics.utils import ASSETS, LINUX, LOGGER, MACOS, TQDM, WEIGHTS_DIR +from ultralytics.utils.checks import IS_PYTHON_3_12, check_requirements, check_yolo +from ultralytics.utils.files import file_size +from ultralytics.utils.torch_utils import select_device + + +def benchmark( + model=WEIGHTS_DIR / "yolov8n.pt", data=None, imgsz=160, half=False, int8=False, device="cpu", verbose=False +): + """ + Benchmark a YOLO model across different formats for speed and accuracy. + + Args: + model (str | Path | optional): Path to the model file or directory. Default is + Path(SETTINGS['weights_dir']) / 'yolov8n.pt'. + data (str, optional): Dataset to evaluate on, inherited from TASK2DATA if not passed. Default is None. + imgsz (int, optional): Image size for the benchmark. Default is 160. + half (bool, optional): Use half-precision for the model if True. Default is False. + int8 (bool, optional): Use int8-precision for the model if True. Default is False. + device (str, optional): Device to run the benchmark on, either 'cpu' or 'cuda'. Default is 'cpu'. + verbose (bool | float | optional): If True or a float, assert benchmarks pass with given metric. + Default is False. + + Returns: + df (pandas.DataFrame): A pandas DataFrame with benchmark results for each format, including file size, + metric, and inference time. + + Example: + ```python + from ultralytics.utils.benchmarks import benchmark + + benchmark(model='yolov8n.pt', imgsz=640) + ``` + """ + + import pandas as pd + + pd.options.display.max_columns = 10 + pd.options.display.width = 120 + device = select_device(device, verbose=False) + if isinstance(model, (str, Path)): + model = YOLO(model) + + y = [] + t0 = time.time() + for i, (name, format, suffix, cpu, gpu) in export_formats().iterrows(): # index, (name, format, suffix, CPU, GPU) + emoji, filename = "❌", None # export defaults + try: + # Checks + if i == 9: # Edge TPU + assert LINUX, "Edge TPU export only supported on Linux" + elif i == 7: # TF GraphDef + assert model.task != "obb", "TensorFlow GraphDef not supported for OBB task" + elif i in {5, 10}: # CoreML and TF.js + assert MACOS or LINUX, "export only supported on macOS and Linux" + if i in {3, 5}: # CoreML and OpenVINO + assert not IS_PYTHON_3_12, "CoreML and OpenVINO not supported on Python 3.12" + if i in {6, 7, 8, 9, 10}: # All TF formats + assert not isinstance(model, YOLOWorld), "YOLOWorldv2 TensorFlow exports not supported by onnx2tf yet" + if i in {11}: # Paddle + assert not isinstance(model, YOLOWorld), "YOLOWorldv2 Paddle exports not supported yet" + if i in {12}: # NCNN + assert not isinstance(model, YOLOWorld), "YOLOWorldv2 NCNN exports not supported yet" + if "cpu" in device.type: + assert cpu, "inference not supported on CPU" + if "cuda" in device.type: + assert gpu, "inference not supported on GPU" + + # Export + if format == "-": + filename = model.ckpt_path or model.cfg + exported_model = model # PyTorch format + else: + filename = model.export(imgsz=imgsz, format=format, half=half, int8=int8, device=device, verbose=False) + exported_model = YOLO(filename, task=model.task) + assert suffix in str(filename), "export failed" + emoji = "❎" # indicates export succeeded + + # Predict + assert model.task != "pose" or i != 7, "GraphDef Pose inference is not supported" + assert i not in (9, 10), "inference not supported" # Edge TPU and TF.js are unsupported + assert i != 5 or platform.system() == "Darwin", "inference only supported on macOS>=10.13" # CoreML + exported_model.predict(ASSETS / "bus.jpg", imgsz=imgsz, device=device, half=half) + + # Validate + data = data or TASK2DATA[model.task] # task to dataset, i.e. coco8.yaml for task=detect + key = TASK2METRIC[model.task] # task to metric, i.e. metrics/mAP50-95(B) for task=detect + results = exported_model.val( + data=data, batch=1, imgsz=imgsz, plots=False, device=device, half=half, int8=int8, verbose=False + ) + metric, speed = results.results_dict[key], results.speed["inference"] + y.append([name, "✅", round(file_size(filename), 1), round(metric, 4), round(speed, 2)]) + except Exception as e: + if verbose: + assert type(e) is AssertionError, f"Benchmark failure for {name}: {e}" + LOGGER.warning(f"ERROR ❌️ Benchmark failure for {name}: {e}") + y.append([name, emoji, round(file_size(filename), 1), None, None]) # mAP, t_inference + + # Print results + check_yolo(device=device) # print system info + df = pd.DataFrame(y, columns=["Format", "Status❔", "Size (MB)", key, "Inference time (ms/im)"]) + + name = Path(model.ckpt_path).name + s = f"\nBenchmarks complete for {name} on {data} at imgsz={imgsz} ({time.time() - t0:.2f}s)\n{df}\n" + LOGGER.info(s) + with open("benchmarks.log", "a", errors="ignore", encoding="utf-8") as f: + f.write(s) + + if verbose and isinstance(verbose, float): + metrics = df[key].array # values to compare to floor + floor = verbose # minimum metric floor to pass, i.e. = 0.29 mAP for YOLOv5n + assert all(x > floor for x in metrics if pd.notna(x)), f"Benchmark failure: metric(s) < floor {floor}" + + return df + + +class ProfileModels: + """ + ProfileModels class for profiling different models on ONNX and TensorRT. + + This class profiles the performance of different models, returning results such as model speed and FLOPs. + + Attributes: + paths (list): Paths of the models to profile. + num_timed_runs (int): Number of timed runs for the profiling. Default is 100. + num_warmup_runs (int): Number of warmup runs before profiling. Default is 10. + min_time (float): Minimum number of seconds to profile for. Default is 60. + imgsz (int): Image size used in the models. Default is 640. + + Methods: + profile(): Profiles the models and prints the result. + + Example: + ```python + from ultralytics.utils.benchmarks import ProfileModels + + ProfileModels(['yolov8n.yaml', 'yolov8s.yaml'], imgsz=640).profile() + ``` + """ + + def __init__( + self, + paths: list, + num_timed_runs=100, + num_warmup_runs=10, + min_time=60, + imgsz=640, + half=True, + trt=True, + device=None, + ): + """ + Initialize the ProfileModels class for profiling models. + + Args: + paths (list): List of paths of the models to be profiled. + num_timed_runs (int, optional): Number of timed runs for the profiling. Default is 100. + num_warmup_runs (int, optional): Number of warmup runs before the actual profiling starts. Default is 10. + min_time (float, optional): Minimum time in seconds for profiling a model. Default is 60. + imgsz (int, optional): Size of the image used during profiling. Default is 640. + half (bool, optional): Flag to indicate whether to use half-precision floating point for profiling. + trt (bool, optional): Flag to indicate whether to profile using TensorRT. Default is True. + device (torch.device, optional): Device used for profiling. If None, it is determined automatically. + """ + self.paths = paths + self.num_timed_runs = num_timed_runs + self.num_warmup_runs = num_warmup_runs + self.min_time = min_time + self.imgsz = imgsz + self.half = half + self.trt = trt # run TensorRT profiling + self.device = device or torch.device(0 if torch.cuda.is_available() else "cpu") + + def profile(self): + """Logs the benchmarking results of a model, checks metrics against floor and returns the results.""" + files = self.get_files() + + if not files: + print("No matching *.pt or *.onnx files found.") + return + + table_rows = [] + output = [] + for file in files: + engine_file = file.with_suffix(".engine") + if file.suffix in (".pt", ".yaml", ".yml"): + model = YOLO(str(file)) + model.fuse() # to report correct params and GFLOPs in model.info() + model_info = model.info() + if self.trt and self.device.type != "cpu" and not engine_file.is_file(): + engine_file = model.export( + format="engine", half=self.half, imgsz=self.imgsz, device=self.device, verbose=False + ) + onnx_file = model.export( + format="onnx", half=self.half, imgsz=self.imgsz, simplify=True, device=self.device, verbose=False + ) + elif file.suffix == ".onnx": + model_info = self.get_onnx_model_info(file) + onnx_file = file + else: + continue + + t_engine = self.profile_tensorrt_model(str(engine_file)) + t_onnx = self.profile_onnx_model(str(onnx_file)) + table_rows.append(self.generate_table_row(file.stem, t_onnx, t_engine, model_info)) + output.append(self.generate_results_dict(file.stem, t_onnx, t_engine, model_info)) + + self.print_table(table_rows) + return output + + def get_files(self): + """Returns a list of paths for all relevant model files given by the user.""" + files = [] + for path in self.paths: + path = Path(path) + if path.is_dir(): + extensions = ["*.pt", "*.onnx", "*.yaml"] + files.extend([file for ext in extensions for file in glob.glob(str(path / ext))]) + elif path.suffix in {".pt", ".yaml", ".yml"}: # add non-existing + files.append(str(path)) + else: + files.extend(glob.glob(str(path))) + + print(f"Profiling: {sorted(files)}") + return [Path(file) for file in sorted(files)] + + def get_onnx_model_info(self, onnx_file: str): + """Retrieves the information including number of layers, parameters, gradients and FLOPs for an ONNX model + file. + """ + return 0.0, 0.0, 0.0, 0.0 # return (num_layers, num_params, num_gradients, num_flops) + + @staticmethod + def iterative_sigma_clipping(data, sigma=2, max_iters=3): + """Applies an iterative sigma clipping algorithm to the given data times number of iterations.""" + data = np.array(data) + for _ in range(max_iters): + mean, std = np.mean(data), np.std(data) + clipped_data = data[(data > mean - sigma * std) & (data < mean + sigma * std)] + if len(clipped_data) == len(data): + break + data = clipped_data + return data + + def profile_tensorrt_model(self, engine_file: str, eps: float = 1e-3): + """Profiles the TensorRT model, measuring average run time and standard deviation among runs.""" + if not self.trt or not Path(engine_file).is_file(): + return 0.0, 0.0 + + # Model and input + model = YOLO(engine_file) + input_data = np.random.rand(self.imgsz, self.imgsz, 3).astype(np.float32) # must be FP32 + + # Warmup runs + elapsed = 0.0 + for _ in range(3): + start_time = time.time() + for _ in range(self.num_warmup_runs): + model(input_data, imgsz=self.imgsz, verbose=False) + elapsed = time.time() - start_time + + # Compute number of runs as higher of min_time or num_timed_runs + num_runs = max(round(self.min_time / (elapsed + eps) * self.num_warmup_runs), self.num_timed_runs * 50) + + # Timed runs + run_times = [] + for _ in TQDM(range(num_runs), desc=engine_file): + results = model(input_data, imgsz=self.imgsz, verbose=False) + run_times.append(results[0].speed["inference"]) # Convert to milliseconds + + run_times = self.iterative_sigma_clipping(np.array(run_times), sigma=2, max_iters=3) # sigma clipping + return np.mean(run_times), np.std(run_times) + + def profile_onnx_model(self, onnx_file: str, eps: float = 1e-3): + """Profiles an ONNX model by executing it multiple times and returns the mean and standard deviation of run + times. + """ + check_requirements("onnxruntime") + import onnxruntime as ort + + # Session with either 'TensorrtExecutionProvider', 'CUDAExecutionProvider', 'CPUExecutionProvider' + sess_options = ort.SessionOptions() + sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL + sess_options.intra_op_num_threads = 8 # Limit the number of threads + sess = ort.InferenceSession(onnx_file, sess_options, providers=["CPUExecutionProvider"]) + + input_tensor = sess.get_inputs()[0] + input_type = input_tensor.type + dynamic = not all(isinstance(dim, int) and dim >= 0 for dim in input_tensor.shape) # dynamic input shape + input_shape = (1, 3, self.imgsz, self.imgsz) if dynamic else input_tensor.shape + + # Mapping ONNX datatype to numpy datatype + if "float16" in input_type: + input_dtype = np.float16 + elif "float" in input_type: + input_dtype = np.float32 + elif "double" in input_type: + input_dtype = np.float64 + elif "int64" in input_type: + input_dtype = np.int64 + elif "int32" in input_type: + input_dtype = np.int32 + else: + raise ValueError(f"Unsupported ONNX datatype {input_type}") + + input_data = np.random.rand(*input_shape).astype(input_dtype) + input_name = input_tensor.name + output_name = sess.get_outputs()[0].name + + # Warmup runs + elapsed = 0.0 + for _ in range(3): + start_time = time.time() + for _ in range(self.num_warmup_runs): + sess.run([output_name], {input_name: input_data}) + elapsed = time.time() - start_time + + # Compute number of runs as higher of min_time or num_timed_runs + num_runs = max(round(self.min_time / (elapsed + eps) * self.num_warmup_runs), self.num_timed_runs) + + # Timed runs + run_times = [] + for _ in TQDM(range(num_runs), desc=onnx_file): + start_time = time.time() + sess.run([output_name], {input_name: input_data}) + run_times.append((time.time() - start_time) * 1000) # Convert to milliseconds + + run_times = self.iterative_sigma_clipping(np.array(run_times), sigma=2, max_iters=5) # sigma clipping + return np.mean(run_times), np.std(run_times) + + def generate_table_row(self, model_name, t_onnx, t_engine, model_info): + """Generates a formatted string for a table row that includes model performance and metric details.""" + layers, params, gradients, flops = model_info + return ( + f"| {model_name:18s} | {self.imgsz} | - | {t_onnx[0]:.2f} ± {t_onnx[1]:.2f} ms | {t_engine[0]:.2f} ± " + f"{t_engine[1]:.2f} ms | {params / 1e6:.1f} | {flops:.1f} |" + ) + + @staticmethod + def generate_results_dict(model_name, t_onnx, t_engine, model_info): + """Generates a dictionary of model details including name, parameters, GFLOPS and speed metrics.""" + layers, params, gradients, flops = model_info + return { + "model/name": model_name, + "model/parameters": params, + "model/GFLOPs": round(flops, 3), + "model/speed_ONNX(ms)": round(t_onnx[0], 3), + "model/speed_TensorRT(ms)": round(t_engine[0], 3), + } + + @staticmethod + def print_table(table_rows): + """Formats and prints a comparison table for different models with given statistics and performance data.""" + gpu = torch.cuda.get_device_name(0) if torch.cuda.is_available() else "GPU" + header = ( + f"| Model | size
(pixels) | mAPval
50-95 | Speed
CPU ONNX
(ms) | " + f"Speed
{gpu} TensorRT
(ms) | params
(M) | FLOPs
(B) |" + ) + separator = ( + "|-------------|---------------------|--------------------|------------------------------|" + "-----------------------------------|------------------|-----------------|" + ) + + print(f"\n\n{header}") + print(separator) + for row in table_rows: + print(row) diff --git a/ultralytics/utils/callbacks/__init__.py b/ultralytics/utils/callbacks/__init__.py new file mode 100644 index 0000000..116babe --- /dev/null +++ b/ultralytics/utils/callbacks/__init__.py @@ -0,0 +1,5 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from .base import add_integration_callbacks, default_callbacks, get_default_callbacks + +__all__ = "add_integration_callbacks", "default_callbacks", "get_default_callbacks" diff --git a/ultralytics/utils/callbacks/__pycache__/__init__.cpython-312.pyc b/ultralytics/utils/callbacks/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..5b3c740 Binary files /dev/null and b/ultralytics/utils/callbacks/__pycache__/__init__.cpython-312.pyc differ diff --git a/ultralytics/utils/callbacks/__pycache__/__init__.cpython-39.pyc b/ultralytics/utils/callbacks/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..f4edc7a Binary files /dev/null and b/ultralytics/utils/callbacks/__pycache__/__init__.cpython-39.pyc differ diff --git a/ultralytics/utils/callbacks/__pycache__/base.cpython-312.pyc b/ultralytics/utils/callbacks/__pycache__/base.cpython-312.pyc new file mode 100644 index 0000000..d7e9f2b Binary files /dev/null and b/ultralytics/utils/callbacks/__pycache__/base.cpython-312.pyc differ diff --git a/ultralytics/utils/callbacks/__pycache__/base.cpython-39.pyc b/ultralytics/utils/callbacks/__pycache__/base.cpython-39.pyc new file mode 100644 index 0000000..acf88c8 Binary files /dev/null and b/ultralytics/utils/callbacks/__pycache__/base.cpython-39.pyc differ diff --git a/ultralytics/utils/callbacks/__pycache__/hub.cpython-312.pyc b/ultralytics/utils/callbacks/__pycache__/hub.cpython-312.pyc new file mode 100644 index 0000000..b7c103c Binary files /dev/null and b/ultralytics/utils/callbacks/__pycache__/hub.cpython-312.pyc differ diff --git a/ultralytics/utils/callbacks/__pycache__/hub.cpython-39.pyc b/ultralytics/utils/callbacks/__pycache__/hub.cpython-39.pyc new file mode 100644 index 0000000..4dcbe86 Binary files /dev/null and b/ultralytics/utils/callbacks/__pycache__/hub.cpython-39.pyc differ diff --git a/ultralytics/utils/callbacks/base.py b/ultralytics/utils/callbacks/base.py new file mode 100644 index 0000000..d015457 --- /dev/null +++ b/ultralytics/utils/callbacks/base.py @@ -0,0 +1,219 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +"""Base callbacks.""" + +from collections import defaultdict +from copy import deepcopy + + +# Trainer callbacks ---------------------------------------------------------------------------------------------------- + + +def on_pretrain_routine_start(trainer): + """Called before the pretraining routine starts.""" + pass + + +def on_pretrain_routine_end(trainer): + """Called after the pretraining routine ends.""" + pass + + +def on_train_start(trainer): + """Called when the training starts.""" + pass + + +def on_train_epoch_start(trainer): + """Called at the start of each training epoch.""" + pass + + +def on_train_batch_start(trainer): + """Called at the start of each training batch.""" + pass + + +def optimizer_step(trainer): + """Called when the optimizer takes a step.""" + pass + + +def on_before_zero_grad(trainer): + """Called before the gradients are set to zero.""" + pass + + +def on_train_batch_end(trainer): + """Called at the end of each training batch.""" + pass + + +def on_train_epoch_end(trainer): + """Called at the end of each training epoch.""" + pass + + +def on_fit_epoch_end(trainer): + """Called at the end of each fit epoch (train + val).""" + pass + + +def on_model_save(trainer): + """Called when the model is saved.""" + pass + + +def on_train_end(trainer): + """Called when the training ends.""" + pass + + +def on_params_update(trainer): + """Called when the model parameters are updated.""" + pass + + +def teardown(trainer): + """Called during the teardown of the training process.""" + pass + + +# Validator callbacks -------------------------------------------------------------------------------------------------- + + +def on_val_start(validator): + """Called when the validation starts.""" + pass + + +def on_val_batch_start(validator): + """Called at the start of each validation batch.""" + pass + + +def on_val_batch_end(validator): + """Called at the end of each validation batch.""" + pass + + +def on_val_end(validator): + """Called when the validation ends.""" + pass + + +# Predictor callbacks -------------------------------------------------------------------------------------------------- + + +def on_predict_start(predictor): + """Called when the prediction starts.""" + pass + + +def on_predict_batch_start(predictor): + """Called at the start of each prediction batch.""" + pass + + +def on_predict_batch_end(predictor): + """Called at the end of each prediction batch.""" + pass + + +def on_predict_postprocess_end(predictor): + """Called after the post-processing of the prediction ends.""" + pass + + +def on_predict_end(predictor): + """Called when the prediction ends.""" + pass + + +# Exporter callbacks --------------------------------------------------------------------------------------------------- + + +def on_export_start(exporter): + """Called when the model export starts.""" + pass + + +def on_export_end(exporter): + """Called when the model export ends.""" + pass + + +default_callbacks = { + # Run in trainer + "on_pretrain_routine_start": [on_pretrain_routine_start], + "on_pretrain_routine_end": [on_pretrain_routine_end], + "on_train_start": [on_train_start], + "on_train_epoch_start": [on_train_epoch_start], + "on_train_batch_start": [on_train_batch_start], + "optimizer_step": [optimizer_step], + "on_before_zero_grad": [on_before_zero_grad], + "on_train_batch_end": [on_train_batch_end], + "on_train_epoch_end": [on_train_epoch_end], + "on_fit_epoch_end": [on_fit_epoch_end], # fit = train + val + "on_model_save": [on_model_save], + "on_train_end": [on_train_end], + "on_params_update": [on_params_update], + "teardown": [teardown], + # Run in validator + "on_val_start": [on_val_start], + "on_val_batch_start": [on_val_batch_start], + "on_val_batch_end": [on_val_batch_end], + "on_val_end": [on_val_end], + # Run in predictor + "on_predict_start": [on_predict_start], + "on_predict_batch_start": [on_predict_batch_start], + "on_predict_postprocess_end": [on_predict_postprocess_end], + "on_predict_batch_end": [on_predict_batch_end], + "on_predict_end": [on_predict_end], + # Run in exporter + "on_export_start": [on_export_start], + "on_export_end": [on_export_end], +} + + +def get_default_callbacks(): + """ + Return a copy of the default_callbacks dictionary with lists as default values. + + Returns: + (defaultdict): A defaultdict with keys from default_callbacks and empty lists as default values. + """ + return defaultdict(list, deepcopy(default_callbacks)) + + +def add_integration_callbacks(instance): + """ + Add integration callbacks from various sources to the instance's callbacks. + + Args: + instance (Trainer, Predictor, Validator, Exporter): An object with a 'callbacks' attribute that is a dictionary + of callback lists. + """ + + # Load HUB callbacks + from .hub import callbacks as hub_cb + + callbacks_list = [hub_cb] + + # Load training callbacks + if "Trainer" in instance.__class__.__name__: + from .clearml import callbacks as clear_cb + from .comet import callbacks as comet_cb + from .dvc import callbacks as dvc_cb + from .mlflow import callbacks as mlflow_cb + from .neptune import callbacks as neptune_cb + from .raytune import callbacks as tune_cb + from .tensorboard import callbacks as tb_cb + from .wb import callbacks as wb_cb + + callbacks_list.extend([clear_cb, comet_cb, dvc_cb, mlflow_cb, neptune_cb, tune_cb, tb_cb, wb_cb]) + + # Add the callbacks to the callbacks dictionary + for callbacks in callbacks_list: + for k, v in callbacks.items(): + if v not in instance.callbacks[k]: + instance.callbacks[k].append(v) diff --git a/ultralytics/utils/callbacks/clearml.py b/ultralytics/utils/callbacks/clearml.py new file mode 100644 index 0000000..a030fc5 --- /dev/null +++ b/ultralytics/utils/callbacks/clearml.py @@ -0,0 +1,152 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from ultralytics.utils import LOGGER, SETTINGS, TESTS_RUNNING + +try: + assert not TESTS_RUNNING # do not log pytest + assert SETTINGS["clearml"] is True # verify integration is enabled + import clearml + from clearml import Task + from clearml.binding.frameworks.pytorch_bind import PatchPyTorchModelIO + from clearml.binding.matplotlib_bind import PatchedMatplotlib + + assert hasattr(clearml, "__version__") # verify package is not directory + +except (ImportError, AssertionError): + clearml = None + + +def _log_debug_samples(files, title="Debug Samples") -> None: + """ + Log files (images) as debug samples in the ClearML task. + + Args: + files (list): A list of file paths in PosixPath format. + title (str): A title that groups together images with the same values. + """ + import re + + if task := Task.current_task(): + for f in files: + if f.exists(): + it = re.search(r"_batch(\d+)", f.name) + iteration = int(it.groups()[0]) if it else 0 + task.get_logger().report_image( + title=title, series=f.name.replace(it.group(), ""), local_path=str(f), iteration=iteration + ) + + +def _log_plot(title, plot_path) -> None: + """ + Log an image as a plot in the plot section of ClearML. + + Args: + title (str): The title of the plot. + plot_path (str): The path to the saved image file. + """ + import matplotlib.image as mpimg + import matplotlib.pyplot as plt + + img = mpimg.imread(plot_path) + fig = plt.figure() + ax = fig.add_axes([0, 0, 1, 1], frameon=False, aspect="auto", xticks=[], yticks=[]) # no ticks + ax.imshow(img) + + Task.current_task().get_logger().report_matplotlib_figure( + title=title, series="", figure=fig, report_interactive=False + ) + + +def on_pretrain_routine_start(trainer): + """Runs at start of pretraining routine; initializes and connects/ logs task to ClearML.""" + try: + if task := Task.current_task(): + # Make sure the automatic pytorch and matplotlib bindings are disabled! + # We are logging these plots and model files manually in the integration + PatchPyTorchModelIO.update_current_task(None) + PatchedMatplotlib.update_current_task(None) + else: + task = Task.init( + project_name=trainer.args.project or "YOLOv8", + task_name=trainer.args.name, + tags=["YOLOv8"], + output_uri=True, + reuse_last_task_id=False, + auto_connect_frameworks={"pytorch": False, "matplotlib": False}, + ) + LOGGER.warning( + "ClearML Initialized a new task. If you want to run remotely, " + "please add clearml-init and connect your arguments before initializing YOLO." + ) + task.connect(vars(trainer.args), name="General") + except Exception as e: + LOGGER.warning(f"WARNING ⚠️ ClearML installed but not initialized correctly, not logging this run. {e}") + + +def on_train_epoch_end(trainer): + """Logs debug samples for the first epoch of YOLO training and report current training progress.""" + if task := Task.current_task(): + # Log debug samples + if trainer.epoch == 1: + _log_debug_samples(sorted(trainer.save_dir.glob("train_batch*.jpg")), "Mosaic") + # Report the current training progress + for k, v in trainer.label_loss_items(trainer.tloss, prefix="train").items(): + task.get_logger().report_scalar("train", k, v, iteration=trainer.epoch) + for k, v in trainer.lr.items(): + task.get_logger().report_scalar("lr", k, v, iteration=trainer.epoch) + + +def on_fit_epoch_end(trainer): + """Reports model information to logger at the end of an epoch.""" + if task := Task.current_task(): + # You should have access to the validation bboxes under jdict + task.get_logger().report_scalar( + title="Epoch Time", series="Epoch Time", value=trainer.epoch_time, iteration=trainer.epoch + ) + for k, v in trainer.metrics.items(): + task.get_logger().report_scalar("val", k, v, iteration=trainer.epoch) + if trainer.epoch == 0: + from ultralytics.utils.torch_utils import model_info_for_loggers + + for k, v in model_info_for_loggers(trainer).items(): + task.get_logger().report_single_value(k, v) + + +def on_val_end(validator): + """Logs validation results including labels and predictions.""" + if Task.current_task(): + # Log val_labels and val_pred + _log_debug_samples(sorted(validator.save_dir.glob("val*.jpg")), "Validation") + + +def on_train_end(trainer): + """Logs final model and its name on training completion.""" + if task := Task.current_task(): + # Log final results, CM matrix + PR plots + files = [ + "results.png", + "confusion_matrix.png", + "confusion_matrix_normalized.png", + *(f"{x}_curve.png" for x in ("F1", "PR", "P", "R")), + ] + files = [(trainer.save_dir / f) for f in files if (trainer.save_dir / f).exists()] # filter + for f in files: + _log_plot(title=f.stem, plot_path=f) + # Report final metrics + for k, v in trainer.validator.metrics.results_dict.items(): + task.get_logger().report_single_value(k, v) + # Log the final model + task.update_output_model(model_path=str(trainer.best), model_name=trainer.args.name, auto_delete_file=False) + + +callbacks = ( + { + "on_pretrain_routine_start": on_pretrain_routine_start, + "on_train_epoch_end": on_train_epoch_end, + "on_fit_epoch_end": on_fit_epoch_end, + "on_val_end": on_val_end, + "on_train_end": on_train_end, + } + if clearml + else {} +) diff --git a/ultralytics/utils/callbacks/comet.py b/ultralytics/utils/callbacks/comet.py new file mode 100644 index 0000000..1c5f585 --- /dev/null +++ b/ultralytics/utils/callbacks/comet.py @@ -0,0 +1,375 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from ultralytics.utils import LOGGER, RANK, SETTINGS, TESTS_RUNNING, ops + +try: + assert not TESTS_RUNNING # do not log pytest + assert SETTINGS["comet"] is True # verify integration is enabled + import comet_ml + + assert hasattr(comet_ml, "__version__") # verify package is not directory + + import os + from pathlib import Path + + # Ensures certain logging functions only run for supported tasks + COMET_SUPPORTED_TASKS = ["detect"] + + # Names of plots created by YOLOv8 that are logged to Comet + EVALUATION_PLOT_NAMES = "F1_curve", "P_curve", "R_curve", "PR_curve", "confusion_matrix" + LABEL_PLOT_NAMES = "labels", "labels_correlogram" + + _comet_image_prediction_count = 0 + +except (ImportError, AssertionError): + comet_ml = None + + +def _get_comet_mode(): + """Returns the mode of comet set in the environment variables, defaults to 'online' if not set.""" + return os.getenv("COMET_MODE", "online") + + +def _get_comet_model_name(): + """Returns the model name for Comet from the environment variable 'COMET_MODEL_NAME' or defaults to 'YOLOv8'.""" + return os.getenv("COMET_MODEL_NAME", "YOLOv8") + + +def _get_eval_batch_logging_interval(): + """Get the evaluation batch logging interval from environment variable or use default value 1.""" + return int(os.getenv("COMET_EVAL_BATCH_LOGGING_INTERVAL", 1)) + + +def _get_max_image_predictions_to_log(): + """Get the maximum number of image predictions to log from the environment variables.""" + return int(os.getenv("COMET_MAX_IMAGE_PREDICTIONS", 100)) + + +def _scale_confidence_score(score): + """Scales the given confidence score by a factor specified in an environment variable.""" + scale = float(os.getenv("COMET_MAX_CONFIDENCE_SCORE", 100.0)) + return score * scale + + +def _should_log_confusion_matrix(): + """Determines if the confusion matrix should be logged based on the environment variable settings.""" + return os.getenv("COMET_EVAL_LOG_CONFUSION_MATRIX", "false").lower() == "true" + + +def _should_log_image_predictions(): + """Determines whether to log image predictions based on a specified environment variable.""" + return os.getenv("COMET_EVAL_LOG_IMAGE_PREDICTIONS", "true").lower() == "true" + + +def _get_experiment_type(mode, project_name): + """Return an experiment based on mode and project name.""" + if mode == "offline": + return comet_ml.OfflineExperiment(project_name=project_name) + + return comet_ml.Experiment(project_name=project_name) + + +def _create_experiment(args): + """Ensures that the experiment object is only created in a single process during distributed training.""" + if RANK not in (-1, 0): + return + try: + comet_mode = _get_comet_mode() + _project_name = os.getenv("COMET_PROJECT_NAME", args.project) + experiment = _get_experiment_type(comet_mode, _project_name) + experiment.log_parameters(vars(args)) + experiment.log_others( + { + "eval_batch_logging_interval": _get_eval_batch_logging_interval(), + "log_confusion_matrix_on_eval": _should_log_confusion_matrix(), + "log_image_predictions": _should_log_image_predictions(), + "max_image_predictions": _get_max_image_predictions_to_log(), + } + ) + experiment.log_other("Created from", "yolov8") + + except Exception as e: + LOGGER.warning(f"WARNING ⚠️ Comet installed but not initialized correctly, not logging this run. {e}") + + +def _fetch_trainer_metadata(trainer): + """Returns metadata for YOLO training including epoch and asset saving status.""" + curr_epoch = trainer.epoch + 1 + + train_num_steps_per_epoch = len(trainer.train_loader.dataset) // trainer.batch_size + curr_step = curr_epoch * train_num_steps_per_epoch + final_epoch = curr_epoch == trainer.epochs + + save = trainer.args.save + save_period = trainer.args.save_period + save_interval = curr_epoch % save_period == 0 + save_assets = save and save_period > 0 and save_interval and not final_epoch + + return dict(curr_epoch=curr_epoch, curr_step=curr_step, save_assets=save_assets, final_epoch=final_epoch) + + +def _scale_bounding_box_to_original_image_shape(box, resized_image_shape, original_image_shape, ratio_pad): + """ + YOLOv8 resizes images during training and the label values are normalized based on this resized shape. + + This function rescales the bounding box labels to the original image shape. + """ + + resized_image_height, resized_image_width = resized_image_shape + + # Convert normalized xywh format predictions to xyxy in resized scale format + box = ops.xywhn2xyxy(box, h=resized_image_height, w=resized_image_width) + # Scale box predictions from resized image scale back to original image scale + box = ops.scale_boxes(resized_image_shape, box, original_image_shape, ratio_pad) + # Convert bounding box format from xyxy to xywh for Comet logging + box = ops.xyxy2xywh(box) + # Adjust xy center to correspond top-left corner + box[:2] -= box[2:] / 2 + box = box.tolist() + + return box + + +def _format_ground_truth_annotations_for_detection(img_idx, image_path, batch, class_name_map=None): + """Format ground truth annotations for detection.""" + indices = batch["batch_idx"] == img_idx + bboxes = batch["bboxes"][indices] + if len(bboxes) == 0: + LOGGER.debug(f"COMET WARNING: Image: {image_path} has no bounding boxes labels") + return None + + cls_labels = batch["cls"][indices].squeeze(1).tolist() + if class_name_map: + cls_labels = [str(class_name_map[label]) for label in cls_labels] + + original_image_shape = batch["ori_shape"][img_idx] + resized_image_shape = batch["resized_shape"][img_idx] + ratio_pad = batch["ratio_pad"][img_idx] + + data = [] + for box, label in zip(bboxes, cls_labels): + box = _scale_bounding_box_to_original_image_shape(box, resized_image_shape, original_image_shape, ratio_pad) + data.append( + { + "boxes": [box], + "label": f"gt_{label}", + "score": _scale_confidence_score(1.0), + } + ) + + return {"name": "ground_truth", "data": data} + + +def _format_prediction_annotations_for_detection(image_path, metadata, class_label_map=None): + """Format YOLO predictions for object detection visualization.""" + stem = image_path.stem + image_id = int(stem) if stem.isnumeric() else stem + + predictions = metadata.get(image_id) + if not predictions: + LOGGER.debug(f"COMET WARNING: Image: {image_path} has no bounding boxes predictions") + return None + + data = [] + for prediction in predictions: + boxes = prediction["bbox"] + score = _scale_confidence_score(prediction["score"]) + cls_label = prediction["category_id"] + if class_label_map: + cls_label = str(class_label_map[cls_label]) + + data.append({"boxes": [boxes], "label": cls_label, "score": score}) + + return {"name": "prediction", "data": data} + + +def _fetch_annotations(img_idx, image_path, batch, prediction_metadata_map, class_label_map): + """Join the ground truth and prediction annotations if they exist.""" + ground_truth_annotations = _format_ground_truth_annotations_for_detection( + img_idx, image_path, batch, class_label_map + ) + prediction_annotations = _format_prediction_annotations_for_detection( + image_path, prediction_metadata_map, class_label_map + ) + + annotations = [ + annotation for annotation in [ground_truth_annotations, prediction_annotations] if annotation is not None + ] + return [annotations] if annotations else None + + +def _create_prediction_metadata_map(model_predictions): + """Create metadata map for model predictions by groupings them based on image ID.""" + pred_metadata_map = {} + for prediction in model_predictions: + pred_metadata_map.setdefault(prediction["image_id"], []) + pred_metadata_map[prediction["image_id"]].append(prediction) + + return pred_metadata_map + + +def _log_confusion_matrix(experiment, trainer, curr_step, curr_epoch): + """Log the confusion matrix to Comet experiment.""" + conf_mat = trainer.validator.confusion_matrix.matrix + names = list(trainer.data["names"].values()) + ["background"] + experiment.log_confusion_matrix( + matrix=conf_mat, labels=names, max_categories=len(names), epoch=curr_epoch, step=curr_step + ) + + +def _log_images(experiment, image_paths, curr_step, annotations=None): + """Logs images to the experiment with optional annotations.""" + if annotations: + for image_path, annotation in zip(image_paths, annotations): + experiment.log_image(image_path, name=image_path.stem, step=curr_step, annotations=annotation) + + else: + for image_path in image_paths: + experiment.log_image(image_path, name=image_path.stem, step=curr_step) + + +def _log_image_predictions(experiment, validator, curr_step): + """Logs predicted boxes for a single image during training.""" + global _comet_image_prediction_count + + task = validator.args.task + if task not in COMET_SUPPORTED_TASKS: + return + + jdict = validator.jdict + if not jdict: + return + + predictions_metadata_map = _create_prediction_metadata_map(jdict) + dataloader = validator.dataloader + class_label_map = validator.names + + batch_logging_interval = _get_eval_batch_logging_interval() + max_image_predictions = _get_max_image_predictions_to_log() + + for batch_idx, batch in enumerate(dataloader): + if (batch_idx + 1) % batch_logging_interval != 0: + continue + + image_paths = batch["im_file"] + for img_idx, image_path in enumerate(image_paths): + if _comet_image_prediction_count >= max_image_predictions: + return + + image_path = Path(image_path) + annotations = _fetch_annotations( + img_idx, + image_path, + batch, + predictions_metadata_map, + class_label_map, + ) + _log_images( + experiment, + [image_path], + curr_step, + annotations=annotations, + ) + _comet_image_prediction_count += 1 + + +def _log_plots(experiment, trainer): + """Logs evaluation plots and label plots for the experiment.""" + plot_filenames = [trainer.save_dir / f"{plots}.png" for plots in EVALUATION_PLOT_NAMES] + _log_images(experiment, plot_filenames, None) + + label_plot_filenames = [trainer.save_dir / f"{labels}.jpg" for labels in LABEL_PLOT_NAMES] + _log_images(experiment, label_plot_filenames, None) + + +def _log_model(experiment, trainer): + """Log the best-trained model to Comet.ml.""" + model_name = _get_comet_model_name() + experiment.log_model(model_name, file_or_folder=str(trainer.best), file_name="best.pt", overwrite=True) + + +def on_pretrain_routine_start(trainer): + """Creates or resumes a CometML experiment at the start of a YOLO pre-training routine.""" + experiment = comet_ml.get_global_experiment() + is_alive = getattr(experiment, "alive", False) + if not experiment or not is_alive: + _create_experiment(trainer.args) + + +def on_train_epoch_end(trainer): + """Log metrics and save batch images at the end of training epochs.""" + experiment = comet_ml.get_global_experiment() + if not experiment: + return + + metadata = _fetch_trainer_metadata(trainer) + curr_epoch = metadata["curr_epoch"] + curr_step = metadata["curr_step"] + + experiment.log_metrics(trainer.label_loss_items(trainer.tloss, prefix="train"), step=curr_step, epoch=curr_epoch) + + if curr_epoch == 1: + _log_images(experiment, trainer.save_dir.glob("train_batch*.jpg"), curr_step) + + +def on_fit_epoch_end(trainer): + """Logs model assets at the end of each epoch.""" + experiment = comet_ml.get_global_experiment() + if not experiment: + return + + metadata = _fetch_trainer_metadata(trainer) + curr_epoch = metadata["curr_epoch"] + curr_step = metadata["curr_step"] + save_assets = metadata["save_assets"] + + experiment.log_metrics(trainer.metrics, step=curr_step, epoch=curr_epoch) + experiment.log_metrics(trainer.lr, step=curr_step, epoch=curr_epoch) + if curr_epoch == 1: + from ultralytics.utils.torch_utils import model_info_for_loggers + + experiment.log_metrics(model_info_for_loggers(trainer), step=curr_step, epoch=curr_epoch) + + if not save_assets: + return + + _log_model(experiment, trainer) + if _should_log_confusion_matrix(): + _log_confusion_matrix(experiment, trainer, curr_step, curr_epoch) + if _should_log_image_predictions(): + _log_image_predictions(experiment, trainer.validator, curr_step) + + +def on_train_end(trainer): + """Perform operations at the end of training.""" + experiment = comet_ml.get_global_experiment() + if not experiment: + return + + metadata = _fetch_trainer_metadata(trainer) + curr_epoch = metadata["curr_epoch"] + curr_step = metadata["curr_step"] + plots = trainer.args.plots + + _log_model(experiment, trainer) + if plots: + _log_plots(experiment, trainer) + + _log_confusion_matrix(experiment, trainer, curr_step, curr_epoch) + _log_image_predictions(experiment, trainer.validator, curr_step) + experiment.end() + + global _comet_image_prediction_count + _comet_image_prediction_count = 0 + + +callbacks = ( + { + "on_pretrain_routine_start": on_pretrain_routine_start, + "on_train_epoch_end": on_train_epoch_end, + "on_fit_epoch_end": on_fit_epoch_end, + "on_train_end": on_train_end, + } + if comet_ml + else {} +) diff --git a/ultralytics/utils/callbacks/dvc.py b/ultralytics/utils/callbacks/dvc.py new file mode 100644 index 0000000..ab51dc5 --- /dev/null +++ b/ultralytics/utils/callbacks/dvc.py @@ -0,0 +1,145 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from ultralytics.utils import LOGGER, SETTINGS, TESTS_RUNNING, checks + +try: + assert not TESTS_RUNNING # do not log pytest + assert SETTINGS["dvc"] is True # verify integration is enabled + import dvclive + + assert checks.check_version("dvclive", "2.11.0", verbose=True) + + import os + import re + from pathlib import Path + + # DVCLive logger instance + live = None + _processed_plots = {} + + # `on_fit_epoch_end` is called on final validation (probably need to be fixed) for now this is the way we + # distinguish final evaluation of the best model vs last epoch validation + _training_epoch = False + +except (ImportError, AssertionError, TypeError): + dvclive = None + + +def _log_images(path, prefix=""): + """Logs images at specified path with an optional prefix using DVCLive.""" + if live: + name = path.name + + # Group images by batch to enable sliders in UI + if m := re.search(r"_batch(\d+)", name): + ni = m[1] + new_stem = re.sub(r"_batch(\d+)", "_batch", path.stem) + name = (Path(new_stem) / ni).with_suffix(path.suffix) + + live.log_image(os.path.join(prefix, name), path) + + +def _log_plots(plots, prefix=""): + """Logs plot images for training progress if they have not been previously processed.""" + for name, params in plots.items(): + timestamp = params["timestamp"] + if _processed_plots.get(name) != timestamp: + _log_images(name, prefix) + _processed_plots[name] = timestamp + + +def _log_confusion_matrix(validator): + """Logs the confusion matrix for the given validator using DVCLive.""" + targets = [] + preds = [] + matrix = validator.confusion_matrix.matrix + names = list(validator.names.values()) + if validator.confusion_matrix.task == "detect": + names += ["background"] + + for ti, pred in enumerate(matrix.T.astype(int)): + for pi, num in enumerate(pred): + targets.extend([names[ti]] * num) + preds.extend([names[pi]] * num) + + live.log_sklearn_plot("confusion_matrix", targets, preds, name="cf.json", normalized=True) + + +def on_pretrain_routine_start(trainer): + """Initializes DVCLive logger for training metadata during pre-training routine.""" + try: + global live + live = dvclive.Live(save_dvc_exp=True, cache_images=True) + LOGGER.info("DVCLive is detected and auto logging is enabled (run 'yolo settings dvc=False' to disable).") + except Exception as e: + LOGGER.warning(f"WARNING ⚠️ DVCLive installed but not initialized correctly, not logging this run. {e}") + + +def on_pretrain_routine_end(trainer): + """Logs plots related to the training process at the end of the pretraining routine.""" + _log_plots(trainer.plots, "train") + + +def on_train_start(trainer): + """Logs the training parameters if DVCLive logging is active.""" + if live: + live.log_params(trainer.args) + + +def on_train_epoch_start(trainer): + """Sets the global variable _training_epoch value to True at the start of training each epoch.""" + global _training_epoch + _training_epoch = True + + +def on_fit_epoch_end(trainer): + """Logs training metrics and model info, and advances to next step on the end of each fit epoch.""" + global _training_epoch + if live and _training_epoch: + all_metrics = {**trainer.label_loss_items(trainer.tloss, prefix="train"), **trainer.metrics, **trainer.lr} + for metric, value in all_metrics.items(): + live.log_metric(metric, value) + + if trainer.epoch == 0: + from ultralytics.utils.torch_utils import model_info_for_loggers + + for metric, value in model_info_for_loggers(trainer).items(): + live.log_metric(metric, value, plot=False) + + _log_plots(trainer.plots, "train") + _log_plots(trainer.validator.plots, "val") + + live.next_step() + _training_epoch = False + + +def on_train_end(trainer): + """Logs the best metrics, plots, and confusion matrix at the end of training if DVCLive is active.""" + if live: + # At the end log the best metrics. It runs validator on the best model internally. + all_metrics = {**trainer.label_loss_items(trainer.tloss, prefix="train"), **trainer.metrics, **trainer.lr} + for metric, value in all_metrics.items(): + live.log_metric(metric, value, plot=False) + + _log_plots(trainer.plots, "val") + _log_plots(trainer.validator.plots, "val") + _log_confusion_matrix(trainer.validator) + + if trainer.best.exists(): + live.log_artifact(trainer.best, copy=True, type="model") + + live.end() + + +callbacks = ( + { + "on_pretrain_routine_start": on_pretrain_routine_start, + "on_pretrain_routine_end": on_pretrain_routine_end, + "on_train_start": on_train_start, + "on_train_epoch_start": on_train_epoch_start, + "on_fit_epoch_end": on_fit_epoch_end, + "on_train_end": on_train_end, + } + if dvclive + else {} +) diff --git a/ultralytics/utils/callbacks/hub.py b/ultralytics/utils/callbacks/hub.py new file mode 100644 index 0000000..cdb42b9 --- /dev/null +++ b/ultralytics/utils/callbacks/hub.py @@ -0,0 +1,108 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import json +from time import time + +from ultralytics.hub.utils import HUB_WEB_ROOT, PREFIX, events +from ultralytics.utils import LOGGER, SETTINGS + + +def on_pretrain_routine_end(trainer): + """Logs info before starting timer for upload rate limit.""" + session = getattr(trainer, "hub_session", None) + if session: + # Start timer for upload rate limit + session.timers = { + "metrics": time(), + "ckpt": time(), + } # start timer on session.rate_limit + + +def on_fit_epoch_end(trainer): + """Uploads training progress metrics at the end of each epoch.""" + session = getattr(trainer, "hub_session", None) + if session: + # Upload metrics after val end + all_plots = { + **trainer.label_loss_items(trainer.tloss, prefix="train"), + **trainer.metrics, + } + if trainer.epoch == 0: + from ultralytics.utils.torch_utils import model_info_for_loggers + + all_plots = {**all_plots, **model_info_for_loggers(trainer)} + + session.metrics_queue[trainer.epoch] = json.dumps(all_plots) + + # If any metrics fail to upload, add them to the queue to attempt uploading again. + if session.metrics_upload_failed_queue: + session.metrics_queue.update(session.metrics_upload_failed_queue) + + if time() - session.timers["metrics"] > session.rate_limits["metrics"]: + session.upload_metrics() + session.timers["metrics"] = time() # reset timer + session.metrics_queue = {} # reset queue + + +def on_model_save(trainer): + """Saves checkpoints to Ultralytics HUB with rate limiting.""" + session = getattr(trainer, "hub_session", None) + if session: + # Upload checkpoints with rate limiting + is_best = trainer.best_fitness == trainer.fitness + if time() - session.timers["ckpt"] > session.rate_limits["ckpt"]: + LOGGER.info(f"{PREFIX}Uploading checkpoint {HUB_WEB_ROOT}/models/{session.model.id}") + session.upload_model(trainer.epoch, trainer.last, is_best) + session.timers["ckpt"] = time() # reset timer + + +def on_train_end(trainer): + """Upload final model and metrics to Ultralytics HUB at the end of training.""" + session = getattr(trainer, "hub_session", None) + if session: + # Upload final model and metrics with exponential standoff + LOGGER.info(f"{PREFIX}Syncing final model...") + session.upload_model( + trainer.epoch, + trainer.best, + map=trainer.metrics.get("metrics/mAP50-95(B)", 0), + final=True, + ) + session.alive = False # stop heartbeats + LOGGER.info(f"{PREFIX}Done ✅\n" f"{PREFIX}View model at {session.model_url} 🚀") + + +def on_train_start(trainer): + """Run events on train start.""" + events(trainer.args) + + +def on_val_start(validator): + """Runs events on validation start.""" + events(validator.args) + + +def on_predict_start(predictor): + """Run events on predict start.""" + events(predictor.args) + + +def on_export_start(exporter): + """Run events on export start.""" + events(exporter.args) + + +callbacks = ( + { + "on_pretrain_routine_end": on_pretrain_routine_end, + "on_fit_epoch_end": on_fit_epoch_end, + "on_model_save": on_model_save, + "on_train_end": on_train_end, + "on_train_start": on_train_start, + "on_val_start": on_val_start, + "on_predict_start": on_predict_start, + "on_export_start": on_export_start, + } + if SETTINGS["hub"] is True + else {} +) # verify enabled diff --git a/ultralytics/utils/callbacks/mlflow.py b/ultralytics/utils/callbacks/mlflow.py new file mode 100644 index 0000000..e554620 --- /dev/null +++ b/ultralytics/utils/callbacks/mlflow.py @@ -0,0 +1,133 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +""" +MLflow Logging for Ultralytics YOLO. + +This module enables MLflow logging for Ultralytics YOLO. It logs metrics, parameters, and model artifacts. +For setting up, a tracking URI should be specified. The logging can be customized using environment variables. + +Commands: + 1. To set a project name: + `export MLFLOW_EXPERIMENT_NAME=` or use the project= argument + + 2. To set a run name: + `export MLFLOW_RUN=` or use the name= argument + + 3. To start a local MLflow server: + mlflow server --backend-store-uri runs/mlflow + It will by default start a local server at http://127.0.0.1:5000. + To specify a different URI, set the MLFLOW_TRACKING_URI environment variable. + + 4. To kill all running MLflow server instances: + ps aux | grep 'mlflow' | grep -v 'grep' | awk '{print $2}' | xargs kill -9 +""" + +from ultralytics.utils import LOGGER, RUNS_DIR, SETTINGS, TESTS_RUNNING, colorstr + +try: + import os + + assert not TESTS_RUNNING or "test_mlflow" in os.environ.get("PYTEST_CURRENT_TEST", "") # do not log pytest + assert SETTINGS["mlflow"] is True # verify integration is enabled + import mlflow + + assert hasattr(mlflow, "__version__") # verify package is not directory + from pathlib import Path + + PREFIX = colorstr("MLflow: ") + SANITIZE = lambda x: {k.replace("(", "").replace(")", ""): float(v) for k, v in x.items()} + +except (ImportError, AssertionError): + mlflow = None + + +def on_pretrain_routine_end(trainer): + """ + Log training parameters to MLflow at the end of the pretraining routine. + + This function sets up MLflow logging based on environment variables and trainer arguments. It sets the tracking URI, + experiment name, and run name, then starts the MLflow run if not already active. It finally logs the parameters + from the trainer. + + Args: + trainer (ultralytics.engine.trainer.BaseTrainer): The training object with arguments and parameters to log. + + Global: + mlflow: The imported mlflow module to use for logging. + + Environment Variables: + MLFLOW_TRACKING_URI: The URI for MLflow tracking. If not set, defaults to 'runs/mlflow'. + MLFLOW_EXPERIMENT_NAME: The name of the MLflow experiment. If not set, defaults to trainer.args.project. + MLFLOW_RUN: The name of the MLflow run. If not set, defaults to trainer.args.name. + MLFLOW_KEEP_RUN_ACTIVE: Boolean indicating whether to keep the MLflow run active after the end of the training phase. + """ + global mlflow + + uri = os.environ.get("MLFLOW_TRACKING_URI") or str(RUNS_DIR / "mlflow") + LOGGER.debug(f"{PREFIX} tracking uri: {uri}") + mlflow.set_tracking_uri(uri) + + # Set experiment and run names + experiment_name = os.environ.get("MLFLOW_EXPERIMENT_NAME") or trainer.args.project or "/Shared/YOLOv8" + run_name = os.environ.get("MLFLOW_RUN") or trainer.args.name + mlflow.set_experiment(experiment_name) + + mlflow.autolog() + try: + active_run = mlflow.active_run() or mlflow.start_run(run_name=run_name) + LOGGER.info(f"{PREFIX}logging run_id({active_run.info.run_id}) to {uri}") + if Path(uri).is_dir(): + LOGGER.info(f"{PREFIX}view at http://127.0.0.1:5000 with 'mlflow server --backend-store-uri {uri}'") + LOGGER.info(f"{PREFIX}disable with 'yolo settings mlflow=False'") + mlflow.log_params(dict(trainer.args)) + except Exception as e: + LOGGER.warning(f"{PREFIX}WARNING ⚠️ Failed to initialize: {e}\n" f"{PREFIX}WARNING ⚠️ Not tracking this run") + + +def on_train_epoch_end(trainer): + """Log training metrics at the end of each train epoch to MLflow.""" + if mlflow: + mlflow.log_metrics( + metrics={ + **SANITIZE(trainer.lr), + **SANITIZE(trainer.label_loss_items(trainer.tloss, prefix="train")), + }, + step=trainer.epoch, + ) + + +def on_fit_epoch_end(trainer): + """Log training metrics at the end of each fit epoch to MLflow.""" + if mlflow: + mlflow.log_metrics(metrics=SANITIZE(trainer.metrics), step=trainer.epoch) + + +def on_train_end(trainer): + """Log model artifacts at the end of the training.""" + if mlflow: + mlflow.log_artifact(str(trainer.best.parent)) # log save_dir/weights directory with best.pt and last.pt + for f in trainer.save_dir.glob("*"): # log all other files in save_dir + if f.suffix in {".png", ".jpg", ".csv", ".pt", ".yaml"}: + mlflow.log_artifact(str(f)) + keep_run_active = os.environ.get("MLFLOW_KEEP_RUN_ACTIVE", "False").lower() in ("true") + if keep_run_active: + LOGGER.info(f"{PREFIX}mlflow run still alive, remember to close it using mlflow.end_run()") + else: + mlflow.end_run() + LOGGER.debug(f"{PREFIX}mlflow run ended") + + LOGGER.info( + f"{PREFIX}results logged to {mlflow.get_tracking_uri()}\n" + f"{PREFIX}disable with 'yolo settings mlflow=False'" + ) + + +callbacks = ( + { + "on_pretrain_routine_end": on_pretrain_routine_end, + "on_train_epoch_end": on_train_epoch_end, + "on_fit_epoch_end": on_fit_epoch_end, + "on_train_end": on_train_end, + } + if mlflow + else {} +) diff --git a/ultralytics/utils/callbacks/neptune.py b/ultralytics/utils/callbacks/neptune.py new file mode 100644 index 0000000..6be8a82 --- /dev/null +++ b/ultralytics/utils/callbacks/neptune.py @@ -0,0 +1,112 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from ultralytics.utils import LOGGER, SETTINGS, TESTS_RUNNING + +try: + assert not TESTS_RUNNING # do not log pytest + assert SETTINGS["neptune"] is True # verify integration is enabled + import neptune + from neptune.types import File + + assert hasattr(neptune, "__version__") + + run = None # NeptuneAI experiment logger instance + +except (ImportError, AssertionError): + neptune = None + + +def _log_scalars(scalars, step=0): + """Log scalars to the NeptuneAI experiment logger.""" + if run: + for k, v in scalars.items(): + run[k].append(value=v, step=step) + + +def _log_images(imgs_dict, group=""): + """Log scalars to the NeptuneAI experiment logger.""" + if run: + for k, v in imgs_dict.items(): + run[f"{group}/{k}"].upload(File(v)) + + +def _log_plot(title, plot_path): + """ + Log plots to the NeptuneAI experiment logger. + + Args: + title (str): Title of the plot. + plot_path (PosixPath | str): Path to the saved image file. + """ + import matplotlib.image as mpimg + import matplotlib.pyplot as plt + + img = mpimg.imread(plot_path) + fig = plt.figure() + ax = fig.add_axes([0, 0, 1, 1], frameon=False, aspect="auto", xticks=[], yticks=[]) # no ticks + ax.imshow(img) + run[f"Plots/{title}"].upload(fig) + + +def on_pretrain_routine_start(trainer): + """Callback function called before the training routine starts.""" + try: + global run + run = neptune.init_run(project=trainer.args.project or "YOLOv8", name=trainer.args.name, tags=["YOLOv8"]) + run["Configuration/Hyperparameters"] = {k: "" if v is None else v for k, v in vars(trainer.args).items()} + except Exception as e: + LOGGER.warning(f"WARNING ⚠️ NeptuneAI installed but not initialized correctly, not logging this run. {e}") + + +def on_train_epoch_end(trainer): + """Callback function called at end of each training epoch.""" + _log_scalars(trainer.label_loss_items(trainer.tloss, prefix="train"), trainer.epoch + 1) + _log_scalars(trainer.lr, trainer.epoch + 1) + if trainer.epoch == 1: + _log_images({f.stem: str(f) for f in trainer.save_dir.glob("train_batch*.jpg")}, "Mosaic") + + +def on_fit_epoch_end(trainer): + """Callback function called at end of each fit (train+val) epoch.""" + if run and trainer.epoch == 0: + from ultralytics.utils.torch_utils import model_info_for_loggers + + run["Configuration/Model"] = model_info_for_loggers(trainer) + _log_scalars(trainer.metrics, trainer.epoch + 1) + + +def on_val_end(validator): + """Callback function called at end of each validation.""" + if run: + # Log val_labels and val_pred + _log_images({f.stem: str(f) for f in validator.save_dir.glob("val*.jpg")}, "Validation") + + +def on_train_end(trainer): + """Callback function called at end of training.""" + if run: + # Log final results, CM matrix + PR plots + files = [ + "results.png", + "confusion_matrix.png", + "confusion_matrix_normalized.png", + *(f"{x}_curve.png" for x in ("F1", "PR", "P", "R")), + ] + files = [(trainer.save_dir / f) for f in files if (trainer.save_dir / f).exists()] # filter + for f in files: + _log_plot(title=f.stem, plot_path=f) + # Log the final model + run[f"weights/{trainer.args.name or trainer.args.task}/{trainer.best.name}"].upload(File(str(trainer.best))) + + +callbacks = ( + { + "on_pretrain_routine_start": on_pretrain_routine_start, + "on_train_epoch_end": on_train_epoch_end, + "on_fit_epoch_end": on_fit_epoch_end, + "on_val_end": on_val_end, + "on_train_end": on_train_end, + } + if neptune + else {} +) diff --git a/ultralytics/utils/callbacks/raytune.py b/ultralytics/utils/callbacks/raytune.py new file mode 100644 index 0000000..f269455 --- /dev/null +++ b/ultralytics/utils/callbacks/raytune.py @@ -0,0 +1,29 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from ultralytics.utils import SETTINGS + +try: + assert SETTINGS["raytune"] is True # verify integration is enabled + import ray + from ray import tune + from ray.air import session + +except (ImportError, AssertionError): + tune = None + + +def on_fit_epoch_end(trainer): + """Sends training metrics to Ray Tune at end of each epoch.""" + if ray.tune.is_session_enabled(): + metrics = trainer.metrics + metrics["epoch"] = trainer.epoch + session.report(metrics) + + +callbacks = ( + { + "on_fit_epoch_end": on_fit_epoch_end, + } + if tune + else {} +) diff --git a/ultralytics/utils/callbacks/tensorboard.py b/ultralytics/utils/callbacks/tensorboard.py new file mode 100644 index 0000000..59024ee --- /dev/null +++ b/ultralytics/utils/callbacks/tensorboard.py @@ -0,0 +1,106 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +import contextlib + +from ultralytics.utils import LOGGER, SETTINGS, TESTS_RUNNING, colorstr + +try: + # WARNING: do not move SummaryWriter import due to protobuf bug https://github.com/ultralytics/ultralytics/pull/4674 + from torch.utils.tensorboard import SummaryWriter + + assert not TESTS_RUNNING # do not log pytest + assert SETTINGS["tensorboard"] is True # verify integration is enabled + WRITER = None # TensorBoard SummaryWriter instance + PREFIX = colorstr("TensorBoard: ") + + # Imports below only required if TensorBoard enabled + import warnings + from copy import deepcopy + from ultralytics.utils.torch_utils import de_parallel, torch + +except (ImportError, AssertionError, TypeError, AttributeError): + # TypeError for handling 'Descriptors cannot not be created directly.' protobuf errors in Windows + # AttributeError: module 'tensorflow' has no attribute 'io' if 'tensorflow' not installed + SummaryWriter = None + + +def _log_scalars(scalars, step=0): + """Logs scalar values to TensorBoard.""" + if WRITER: + for k, v in scalars.items(): + WRITER.add_scalar(k, v, step) + + +def _log_tensorboard_graph(trainer): + """Log model graph to TensorBoard.""" + + # Input image + imgsz = trainer.args.imgsz + imgsz = (imgsz, imgsz) if isinstance(imgsz, int) else imgsz + p = next(trainer.model.parameters()) # for device, type + im = torch.zeros((1, 3, *imgsz), device=p.device, dtype=p.dtype) # input image (must be zeros, not empty) + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=UserWarning) # suppress jit trace warning + warnings.simplefilter("ignore", category=torch.jit.TracerWarning) # suppress jit trace warning + + # Try simple method first (YOLO) + with contextlib.suppress(Exception): + trainer.model.eval() # place in .eval() mode to avoid BatchNorm statistics changes + WRITER.add_graph(torch.jit.trace(de_parallel(trainer.model), im, strict=False), []) + LOGGER.info(f"{PREFIX}model graph visualization added ✅") + return + + # Fallback to TorchScript export steps (RTDETR) + try: + model = deepcopy(de_parallel(trainer.model)) + model.eval() + model = model.fuse(verbose=False) + for m in model.modules(): + if hasattr(m, "export"): # Detect, RTDETRDecoder (Segment and Pose use Detect base class) + m.export = True + m.format = "torchscript" + model(im) # dry run + WRITER.add_graph(torch.jit.trace(model, im, strict=False), []) + LOGGER.info(f"{PREFIX}model graph visualization added ✅") + except Exception as e: + LOGGER.warning(f"{PREFIX}WARNING ⚠️ TensorBoard graph visualization failure {e}") + + +def on_pretrain_routine_start(trainer): + """Initialize TensorBoard logging with SummaryWriter.""" + if SummaryWriter: + try: + global WRITER + WRITER = SummaryWriter(str(trainer.save_dir)) + LOGGER.info(f"{PREFIX}Start with 'tensorboard --logdir {trainer.save_dir}', view at http://localhost:6006/") + except Exception as e: + LOGGER.warning(f"{PREFIX}WARNING ⚠️ TensorBoard not initialized correctly, not logging this run. {e}") + + +def on_train_start(trainer): + """Log TensorBoard graph.""" + if WRITER: + _log_tensorboard_graph(trainer) + + +def on_train_epoch_end(trainer): + """Logs scalar statistics at the end of a training epoch.""" + _log_scalars(trainer.label_loss_items(trainer.tloss, prefix="train"), trainer.epoch + 1) + _log_scalars(trainer.lr, trainer.epoch + 1) + + +def on_fit_epoch_end(trainer): + """Logs epoch metrics at end of training epoch.""" + _log_scalars(trainer.metrics, trainer.epoch + 1) + + +callbacks = ( + { + "on_pretrain_routine_start": on_pretrain_routine_start, + "on_train_start": on_train_start, + "on_fit_epoch_end": on_fit_epoch_end, + "on_train_epoch_end": on_train_epoch_end, + } + if SummaryWriter + else {} +) diff --git a/ultralytics/utils/callbacks/wb.py b/ultralytics/utils/callbacks/wb.py new file mode 100644 index 0000000..25a1b64 --- /dev/null +++ b/ultralytics/utils/callbacks/wb.py @@ -0,0 +1,163 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from ultralytics.utils import SETTINGS, TESTS_RUNNING +from ultralytics.utils.torch_utils import model_info_for_loggers + +try: + assert not TESTS_RUNNING # do not log pytest + assert SETTINGS["wandb"] is True # verify integration is enabled + import wandb as wb + + assert hasattr(wb, "__version__") # verify package is not directory + + import numpy as np + import pandas as pd + + _processed_plots = {} + +except (ImportError, AssertionError): + wb = None + + +def _custom_table(x, y, classes, title="Precision Recall Curve", x_title="Recall", y_title="Precision"): + """ + Create and log a custom metric visualization to wandb.plot.pr_curve. + + This function crafts a custom metric visualization that mimics the behavior of wandb's default precision-recall + curve while allowing for enhanced customization. The visual metric is useful for monitoring model performance across + different classes. + + Args: + x (List): Values for the x-axis; expected to have length N. + y (List): Corresponding values for the y-axis; also expected to have length N. + classes (List): Labels identifying the class of each point; length N. + title (str, optional): Title for the plot; defaults to 'Precision Recall Curve'. + x_title (str, optional): Label for the x-axis; defaults to 'Recall'. + y_title (str, optional): Label for the y-axis; defaults to 'Precision'. + + Returns: + (wandb.Object): A wandb object suitable for logging, showcasing the crafted metric visualization. + """ + df = pd.DataFrame({"class": classes, "y": y, "x": x}).round(3) + fields = {"x": "x", "y": "y", "class": "class"} + string_fields = {"title": title, "x-axis-title": x_title, "y-axis-title": y_title} + return wb.plot_table( + "wandb/area-under-curve/v0", wb.Table(dataframe=df), fields=fields, string_fields=string_fields + ) + + +def _plot_curve( + x, + y, + names=None, + id="precision-recall", + title="Precision Recall Curve", + x_title="Recall", + y_title="Precision", + num_x=100, + only_mean=False, +): + """ + Log a metric curve visualization. + + This function generates a metric curve based on input data and logs the visualization to wandb. + The curve can represent aggregated data (mean) or individual class data, depending on the 'only_mean' flag. + + Args: + x (np.ndarray): Data points for the x-axis with length N. + y (np.ndarray): Corresponding data points for the y-axis with shape CxN, where C is the number of classes. + names (list, optional): Names of the classes corresponding to the y-axis data; length C. Defaults to []. + id (str, optional): Unique identifier for the logged data in wandb. Defaults to 'precision-recall'. + title (str, optional): Title for the visualization plot. Defaults to 'Precision Recall Curve'. + x_title (str, optional): Label for the x-axis. Defaults to 'Recall'. + y_title (str, optional): Label for the y-axis. Defaults to 'Precision'. + num_x (int, optional): Number of interpolated data points for visualization. Defaults to 100. + only_mean (bool, optional): Flag to indicate if only the mean curve should be plotted. Defaults to True. + + Note: + The function leverages the '_custom_table' function to generate the actual visualization. + """ + # Create new x + if names is None: + names = [] + x_new = np.linspace(x[0], x[-1], num_x).round(5) + + # Create arrays for logging + x_log = x_new.tolist() + y_log = np.interp(x_new, x, np.mean(y, axis=0)).round(3).tolist() + + if only_mean: + table = wb.Table(data=list(zip(x_log, y_log)), columns=[x_title, y_title]) + wb.run.log({title: wb.plot.line(table, x_title, y_title, title=title)}) + else: + classes = ["mean"] * len(x_log) + for i, yi in enumerate(y): + x_log.extend(x_new) # add new x + y_log.extend(np.interp(x_new, x, yi)) # interpolate y to new x + classes.extend([names[i]] * len(x_new)) # add class names + wb.log({id: _custom_table(x_log, y_log, classes, title, x_title, y_title)}, commit=False) + + +def _log_plots(plots, step): + """Logs plots from the input dictionary if they haven't been logged already at the specified step.""" + for name, params in plots.items(): + timestamp = params["timestamp"] + if _processed_plots.get(name) != timestamp: + wb.run.log({name.stem: wb.Image(str(name))}, step=step) + _processed_plots[name] = timestamp + + +def on_pretrain_routine_start(trainer): + """Initiate and start project if module is present.""" + wb.run or wb.init(project=trainer.args.project or "YOLOv8", name=trainer.args.name, config=vars(trainer.args)) + + +def on_fit_epoch_end(trainer): + """Logs training metrics and model information at the end of an epoch.""" + wb.run.log(trainer.metrics, step=trainer.epoch + 1) + _log_plots(trainer.plots, step=trainer.epoch + 1) + _log_plots(trainer.validator.plots, step=trainer.epoch + 1) + if trainer.epoch == 0: + wb.run.log(model_info_for_loggers(trainer), step=trainer.epoch + 1) + + +def on_train_epoch_end(trainer): + """Log metrics and save images at the end of each training epoch.""" + wb.run.log(trainer.label_loss_items(trainer.tloss, prefix="train"), step=trainer.epoch + 1) + wb.run.log(trainer.lr, step=trainer.epoch + 1) + if trainer.epoch == 1: + _log_plots(trainer.plots, step=trainer.epoch + 1) + + +def on_train_end(trainer): + """Save the best model as an artifact at end of training.""" + _log_plots(trainer.validator.plots, step=trainer.epoch + 1) + _log_plots(trainer.plots, step=trainer.epoch + 1) + art = wb.Artifact(type="model", name=f"run_{wb.run.id}_model") + if trainer.best.exists(): + art.add_file(trainer.best) + wb.run.log_artifact(art, aliases=["best"]) + for curve_name, curve_values in zip(trainer.validator.metrics.curves, trainer.validator.metrics.curves_results): + x, y, x_title, y_title = curve_values + _plot_curve( + x, + y, + names=list(trainer.validator.metrics.names.values()), + id=f"curves/{curve_name}", + title=curve_name, + x_title=x_title, + y_title=y_title, + ) + wb.run.finish() # required or run continues on dashboard + + +callbacks = ( + { + "on_pretrain_routine_start": on_pretrain_routine_start, + "on_train_epoch_end": on_train_epoch_end, + "on_fit_epoch_end": on_fit_epoch_end, + "on_train_end": on_train_end, + } + if wb + else {} +) diff --git a/ultralytics/utils/checks.py b/ultralytics/utils/checks.py new file mode 100644 index 0000000..c44ac0b --- /dev/null +++ b/ultralytics/utils/checks.py @@ -0,0 +1,731 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import contextlib +import glob +import inspect +import math +import os +import platform +import re +import shutil +import subprocess +import time +from importlib import metadata +from pathlib import Path +from typing import Optional + +import cv2 +import numpy as np +import requests +import torch +from matplotlib import font_manager + +from ultralytics.utils import ( + ASSETS, + AUTOINSTALL, + LINUX, + LOGGER, + ONLINE, + ROOT, + USER_CONFIG_DIR, + SimpleNamespace, + ThreadingLocked, + TryExcept, + clean_url, + colorstr, + downloads, + emojis, + is_colab, + is_docker, + is_github_action_running, + is_jupyter, + is_kaggle, + is_online, + is_pip_package, + url2file, +) + +PYTHON_VERSION = platform.python_version() + + +def parse_requirements(file_path=ROOT.parent / "requirements.txt", package=""): + """ + Parse a requirements.txt file, ignoring lines that start with '#' and any text after '#'. + + Args: + file_path (Path): Path to the requirements.txt file. + package (str, optional): Python package to use instead of requirements.txt file, i.e. package='ultralytics'. + + Returns: + (List[Dict[str, str]]): List of parsed requirements as dictionaries with `name` and `specifier` keys. + + Example: + ```python + from ultralytics.utils.checks import parse_requirements + + parse_requirements(package='ultralytics') + ``` + """ + + if package: + requires = [x for x in metadata.distribution(package).requires if "extra == " not in x] + else: + requires = Path(file_path).read_text().splitlines() + + requirements = [] + for line in requires: + line = line.strip() + if line and not line.startswith("#"): + line = line.split("#")[0].strip() # ignore inline comments + match = re.match(r"([a-zA-Z0-9-_]+)\s*([<>!=~]+.*)?", line) + if match: + requirements.append(SimpleNamespace(name=match[1], specifier=match[2].strip() if match[2] else "")) + + return requirements + + +def parse_version(version="0.0.0") -> tuple: + """ + Convert a version string to a tuple of integers, ignoring any extra non-numeric string attached to the version. This + function replaces deprecated 'pkg_resources.parse_version(v)'. + + Args: + version (str): Version string, i.e. '2.0.1+cpu' + + Returns: + (tuple): Tuple of integers representing the numeric part of the version and the extra string, i.e. (2, 0, 1) + """ + try: + return tuple(map(int, re.findall(r"\d+", version)[:3])) # '2.0.1+cpu' -> (2, 0, 1) + except Exception as e: + LOGGER.warning(f"WARNING ⚠️ failure for parse_version({version}), returning (0, 0, 0): {e}") + return 0, 0, 0 + + +def is_ascii(s) -> bool: + """ + Check if a string is composed of only ASCII characters. + + Args: + s (str): String to be checked. + + Returns: + (bool): True if the string is composed only of ASCII characters, False otherwise. + """ + # Convert list, tuple, None, etc. to string + s = str(s) + + # Check if the string is composed of only ASCII characters + return all(ord(c) < 128 for c in s) + + +def check_imgsz(imgsz, stride=32, min_dim=1, max_dim=2, floor=0): + """ + Verify image size is a multiple of the given stride in each dimension. If the image size is not a multiple of the + stride, update it to the nearest multiple of the stride that is greater than or equal to the given floor value. + + Args: + imgsz (int | cList[int]): Image size. + stride (int): Stride value. + min_dim (int): Minimum number of dimensions. + max_dim (int): Maximum number of dimensions. + floor (int): Minimum allowed value for image size. + + Returns: + (List[int]): Updated image size. + """ + # Convert stride to integer if it is a tensor + stride = int(stride.max() if isinstance(stride, torch.Tensor) else stride) + + # Convert image size to list if it is an integer + if isinstance(imgsz, int): + imgsz = [imgsz] + elif isinstance(imgsz, (list, tuple)): + imgsz = list(imgsz) + elif isinstance(imgsz, str): # i.e. '640' or '[640,640]' + imgsz = [int(imgsz)] if imgsz.isnumeric() else eval(imgsz) + else: + raise TypeError( + f"'imgsz={imgsz}' is of invalid type {type(imgsz).__name__}. " + f"Valid imgsz types are int i.e. 'imgsz=640' or list i.e. 'imgsz=[640,640]'" + ) + + # Apply max_dim + if len(imgsz) > max_dim: + msg = ( + "'train' and 'val' imgsz must be an integer, while 'predict' and 'export' imgsz may be a [h, w] list " + "or an integer, i.e. 'yolo export imgsz=640,480' or 'yolo export imgsz=640'" + ) + if max_dim != 1: + raise ValueError(f"imgsz={imgsz} is not a valid image size. {msg}") + LOGGER.warning(f"WARNING ⚠️ updating to 'imgsz={max(imgsz)}'. {msg}") + imgsz = [max(imgsz)] + # Make image size a multiple of the stride + sz = [max(math.ceil(x / stride) * stride, floor) for x in imgsz] + + # Print warning message if image size was updated + if sz != imgsz: + LOGGER.warning(f"WARNING ⚠️ imgsz={imgsz} must be multiple of max stride {stride}, updating to {sz}") + + # Add missing dimensions if necessary + sz = [sz[0], sz[0]] if min_dim == 2 and len(sz) == 1 else sz[0] if min_dim == 1 and len(sz) == 1 else sz + + return sz + + +def check_version( + current: str = "0.0.0", + required: str = "0.0.0", + name: str = "version", + hard: bool = False, + verbose: bool = False, + msg: str = "", +) -> bool: + """ + Check current version against the required version or range. + + Args: + current (str): Current version or package name to get version from. + required (str): Required version or range (in pip-style format). + name (str, optional): Name to be used in warning message. + hard (bool, optional): If True, raise an AssertionError if the requirement is not met. + verbose (bool, optional): If True, print warning message if requirement is not met. + msg (str, optional): Extra message to display if verbose. + + Returns: + (bool): True if requirement is met, False otherwise. + + Example: + ```python + # Check if current version is exactly 22.04 + check_version(current='22.04', required='==22.04') + + # Check if current version is greater than or equal to 22.04 + check_version(current='22.10', required='22.04') # assumes '>=' inequality if none passed + + # Check if current version is less than or equal to 22.04 + check_version(current='22.04', required='<=22.04') + + # Check if current version is between 20.04 (inclusive) and 22.04 (exclusive) + check_version(current='21.10', required='>20.04,<22.04') + ``` + """ + if not current: # if current is '' or None + LOGGER.warning(f"WARNING ⚠️ invalid check_version({current}, {required}) requested, please check values.") + return True + elif not current[0].isdigit(): # current is package name rather than version string, i.e. current='ultralytics' + try: + name = current # assigned package name to 'name' arg + current = metadata.version(current) # get version string from package name + except metadata.PackageNotFoundError as e: + if hard: + raise ModuleNotFoundError(emojis(f"WARNING ⚠️ {current} package is required but not installed")) from e + else: + return False + + if not required: # if required is '' or None + return True + + op = "" + version = "" + result = True + c = parse_version(current) # '1.2.3' -> (1, 2, 3) + for r in required.strip(",").split(","): + op, version = re.match(r"([^0-9]*)([\d.]+)", r).groups() # split '>=22.04' -> ('>=', '22.04') + v = parse_version(version) # '1.2.3' -> (1, 2, 3) + if op == "==" and c != v: + result = False + elif op == "!=" and c == v: + result = False + elif op in (">=", "") and not (c >= v): # if no constraint passed assume '>=required' + result = False + elif op == "<=" and not (c <= v): + result = False + elif op == ">" and not (c > v): + result = False + elif op == "<" and not (c < v): + result = False + if not result: + warning = f"WARNING ⚠️ {name}{op}{version} is required, but {name}=={current} is currently installed {msg}" + if hard: + raise ModuleNotFoundError(emojis(warning)) # assert version requirements met + if verbose: + LOGGER.warning(warning) + return result + + +def check_latest_pypi_version(package_name="ultralytics"): + """ + Returns the latest version of a PyPI package without downloading or installing it. + + Parameters: + package_name (str): The name of the package to find the latest version for. + + Returns: + (str): The latest version of the package. + """ + with contextlib.suppress(Exception): + requests.packages.urllib3.disable_warnings() # Disable the InsecureRequestWarning + response = requests.get(f"https://pypi.org/pypi/{package_name}/json", timeout=3) + if response.status_code == 200: + return response.json()["info"]["version"] + + +def check_pip_update_available(): + """ + Checks if a new version of the ultralytics package is available on PyPI. + + Returns: + (bool): True if an update is available, False otherwise. + """ + if ONLINE and is_pip_package(): + with contextlib.suppress(Exception): + from ultralytics import __version__ + + latest = check_latest_pypi_version() + if check_version(__version__, f"<{latest}"): # check if current version is < latest version + LOGGER.info( + f"New https://pypi.org/project/ultralytics/{latest} available 😃 " + f"Update with 'pip install -U ultralytics'" + ) + return True + return False + + +@ThreadingLocked() +def check_font(font="Arial.ttf"): + """ + Find font locally or download to user's configuration directory if it does not already exist. + + Args: + font (str): Path or name of font. + + Returns: + file (Path): Resolved font file path. + """ + name = Path(font).name + + # Check USER_CONFIG_DIR + file = USER_CONFIG_DIR / name + if file.exists(): + return file + + # Check system fonts + matches = [s for s in font_manager.findSystemFonts() if font in s] + if any(matches): + return matches[0] + + # Download to USER_CONFIG_DIR if missing + url = f"https://ultralytics.com/assets/{name}" + if downloads.is_url(url, check=True): + downloads.safe_download(url=url, file=file) + return file + + +def check_python(minimum: str = "3.8.0") -> bool: + """ + Check current python version against the required minimum version. + + Args: + minimum (str): Required minimum version of python. + + Returns: + (bool): Whether the installed Python version meets the minimum constraints. + """ + return check_version(PYTHON_VERSION, minimum, name="Python ", hard=True) + + +@TryExcept() +def check_requirements(requirements=ROOT.parent / "requirements.txt", exclude=(), install=True, cmds=""): + """ + Check if installed dependencies meet YOLOv8 requirements and attempt to auto-update if needed. + + Args: + requirements (Union[Path, str, List[str]]): Path to a requirements.txt file, a single package requirement as a + string, or a list of package requirements as strings. + exclude (Tuple[str]): Tuple of package names to exclude from checking. + install (bool): If True, attempt to auto-update packages that don't meet requirements. + cmds (str): Additional commands to pass to the pip install command when auto-updating. + + Example: + ```python + from ultralytics.utils.checks import check_requirements + + # Check a requirements.txt file + check_requirements('path/to/requirements.txt') + + # Check a single package + check_requirements('ultralytics>=8.0.0') + + # Check multiple packages + check_requirements(['numpy', 'ultralytics>=8.0.0']) + ``` + """ + + prefix = colorstr("red", "bold", "requirements:") + check_python() # check python version + check_torchvision() # check torch-torchvision compatibility + if isinstance(requirements, Path): # requirements.txt file + file = requirements.resolve() + assert file.exists(), f"{prefix} {file} not found, check failed." + requirements = [f"{x.name}{x.specifier}" for x in parse_requirements(file) if x.name not in exclude] + elif isinstance(requirements, str): + requirements = [requirements] + + pkgs = [] + for r in requirements: + r_stripped = r.split("/")[-1].replace(".git", "") # replace git+https://org/repo.git -> 'repo' + match = re.match(r"([a-zA-Z0-9-_]+)([<>!=~]+.*)?", r_stripped) + name, required = match[1], match[2].strip() if match[2] else "" + try: + assert check_version(metadata.version(name), required) # exception if requirements not met + except (AssertionError, metadata.PackageNotFoundError): + pkgs.append(r) + + s = " ".join(f'"{x}"' for x in pkgs) # console string + if s: + if install and AUTOINSTALL: # check environment variable + n = len(pkgs) # number of packages updates + LOGGER.info(f"{prefix} Ultralytics requirement{'s' * (n > 1)} {pkgs} not found, attempting AutoUpdate...") + try: + t = time.time() + assert is_online(), "AutoUpdate skipped (offline)" + LOGGER.info(subprocess.check_output(f"pip install --no-cache {s} {cmds}", shell=True).decode()) + dt = time.time() - t + LOGGER.info( + f"{prefix} AutoUpdate success ✅ {dt:.1f}s, installed {n} package{'s' * (n > 1)}: {pkgs}\n" + f"{prefix} ⚠️ {colorstr('bold', 'Restart runtime or rerun command for updates to take effect')}\n" + ) + except Exception as e: + LOGGER.warning(f"{prefix} ❌ {e}") + return False + else: + return False + + return True + + +def check_torchvision(): + """ + Checks the installed versions of PyTorch and Torchvision to ensure they're compatible. + + This function checks the installed versions of PyTorch and Torchvision, and warns if they're incompatible according + to the provided compatibility table based on: + https://github.com/pytorch/vision#installation. + + The compatibility table is a dictionary where the keys are PyTorch versions and the values are lists of compatible + Torchvision versions. + """ + + import torchvision + + # Compatibility table + compatibility_table = {"2.0": ["0.15"], "1.13": ["0.14"], "1.12": ["0.13"]} + + # Extract only the major and minor versions + v_torch = ".".join(torch.__version__.split("+")[0].split(".")[:2]) + v_torchvision = ".".join(torchvision.__version__.split("+")[0].split(".")[:2]) + + if v_torch in compatibility_table: + compatible_versions = compatibility_table[v_torch] + if all(v_torchvision != v for v in compatible_versions): + print( + f"WARNING ⚠️ torchvision=={v_torchvision} is incompatible with torch=={v_torch}.\n" + f"Run 'pip install torchvision=={compatible_versions[0]}' to fix torchvision or " + "'pip install -U torch torchvision' to update both.\n" + "For a full compatibility table see https://github.com/pytorch/vision#installation" + ) + + +def check_suffix(file="yolov8n.pt", suffix=".pt", msg=""): + """Check file(s) for acceptable suffix.""" + if file and suffix: + if isinstance(suffix, str): + suffix = (suffix,) + for f in file if isinstance(file, (list, tuple)) else [file]: + s = Path(f).suffix.lower().strip() # file suffix + if len(s): + assert s in suffix, f"{msg}{f} acceptable suffix is {suffix}, not {s}" + + +def check_yolov5u_filename(file: str, verbose: bool = True): + """Replace legacy YOLOv5 filenames with updated YOLOv5u filenames.""" + if "yolov3" in file or "yolov5" in file: + if "u.yaml" in file: + file = file.replace("u.yaml", ".yaml") # i.e. yolov5nu.yaml -> yolov5n.yaml + elif ".pt" in file and "u" not in file: + original_file = file + file = re.sub(r"(.*yolov5([nsmlx]))\.pt", "\\1u.pt", file) # i.e. yolov5n.pt -> yolov5nu.pt + file = re.sub(r"(.*yolov5([nsmlx])6)\.pt", "\\1u.pt", file) # i.e. yolov5n6.pt -> yolov5n6u.pt + file = re.sub(r"(.*yolov3(|-tiny|-spp))\.pt", "\\1u.pt", file) # i.e. yolov3-spp.pt -> yolov3-sppu.pt + if file != original_file and verbose: + LOGGER.info( + f"PRO TIP 💡 Replace 'model={original_file}' with new 'model={file}'.\nYOLOv5 'u' models are " + f"trained with https://github.com/ultralytics/ultralytics and feature improved performance vs " + f"standard YOLOv5 models trained with https://github.com/ultralytics/yolov5.\n" + ) + return file + + +def check_model_file_from_stem(model="yolov8n"): + """Return a model filename from a valid model stem.""" + if model and not Path(model).suffix and Path(model).stem in downloads.GITHUB_ASSETS_STEMS: + return Path(model).with_suffix(".pt") # add suffix, i.e. yolov8n -> yolov8n.pt + else: + return model + + +def check_file(file, suffix="", download=True, hard=True): + """Search/download file (if necessary) and return path.""" + check_suffix(file, suffix) # optional + file = str(file).strip() # convert to string and strip spaces + file = check_yolov5u_filename(file) # yolov5n -> yolov5nu + if ( + not file + or ("://" not in file and Path(file).exists()) # '://' check required in Windows Python<3.10 + or file.lower().startswith("grpc://") + ): # file exists or gRPC Triton images + return file + elif download and file.lower().startswith(("https://", "http://", "rtsp://", "rtmp://", "tcp://")): # download + url = file # warning: Pathlib turns :// -> :/ + file = url2file(file) # '%2F' to '/', split https://url.com/file.txt?auth + if Path(file).exists(): + LOGGER.info(f"Found {clean_url(url)} locally at {file}") # file already exists + else: + downloads.safe_download(url=url, file=file, unzip=False) + return file + else: # search + files = glob.glob(str(ROOT / "**" / file), recursive=True) or glob.glob(str(ROOT.parent / file)) # find file + if not files and hard: + raise FileNotFoundError(f"'{file}' does not exist") + elif len(files) > 1 and hard: + raise FileNotFoundError(f"Multiple files match '{file}', specify exact path: {files}") + return files[0] if len(files) else [] if hard else file # return file + + +def check_yaml(file, suffix=(".yaml", ".yml"), hard=True): + """Search/download YAML file (if necessary) and return path, checking suffix.""" + return check_file(file, suffix, hard=hard) + + +def check_is_path_safe(basedir, path): + """ + Check if the resolved path is under the intended directory to prevent path traversal. + + Args: + basedir (Path | str): The intended directory. + path (Path | str): The path to check. + + Returns: + (bool): True if the path is safe, False otherwise. + """ + base_dir_resolved = Path(basedir).resolve() + path_resolved = Path(path).resolve() + + return path_resolved.is_file() and path_resolved.parts[: len(base_dir_resolved.parts)] == base_dir_resolved.parts + + +def check_imshow(warn=False): + """Check if environment supports image displays.""" + try: + if LINUX: + assert "DISPLAY" in os.environ and not is_docker() and not is_colab() and not is_kaggle() + cv2.imshow("test", np.zeros((8, 8, 3), dtype=np.uint8)) # show a small 8-pixel image + cv2.waitKey(1) + cv2.destroyAllWindows() + cv2.waitKey(1) + return True + except Exception as e: + if warn: + LOGGER.warning(f"WARNING ⚠️ Environment does not support cv2.imshow() or PIL Image.show()\n{e}") + return False + + +def check_yolo(verbose=True, device=""): + """Return a human-readable YOLO software and hardware summary.""" + import psutil + + from ultralytics.utils.torch_utils import select_device + + if is_jupyter(): + if check_requirements("wandb", install=False): + os.system("pip uninstall -y wandb") # uninstall wandb: unwanted account creation prompt with infinite hang + if is_colab(): + shutil.rmtree("sample_data", ignore_errors=True) # remove colab /sample_data directory + + if verbose: + # System info + gib = 1 << 30 # bytes per GiB + ram = psutil.virtual_memory().total + total, used, free = shutil.disk_usage("/") + s = f"({os.cpu_count()} CPUs, {ram / gib:.1f} GB RAM, {(total - free) / gib:.1f}/{total / gib:.1f} GB disk)" + with contextlib.suppress(Exception): # clear display if ipython is installed + from IPython import display + + display.clear_output() + else: + s = "" + + select_device(device=device, newline=False) + LOGGER.info(f"Setup complete ✅ {s}") + + +def collect_system_info(): + """Collect and print relevant system information including OS, Python, RAM, CPU, and CUDA.""" + + import psutil + + from ultralytics.utils import ENVIRONMENT, is_git_dir + from ultralytics.utils.torch_utils import get_cpu_info + + ram_info = psutil.virtual_memory().total / (1024**3) # Convert bytes to GB + check_yolo() + LOGGER.info( + f"\n{'OS':<20}{platform.platform()}\n" + f"{'Environment':<20}{ENVIRONMENT}\n" + f"{'Python':<20}{PYTHON_VERSION}\n" + f"{'Install':<20}{'git' if is_git_dir() else 'pip' if is_pip_package() else 'other'}\n" + f"{'RAM':<20}{ram_info:.2f} GB\n" + f"{'CPU':<20}{get_cpu_info()}\n" + f"{'CUDA':<20}{torch.version.cuda if torch and torch.cuda.is_available() else None}\n" + ) + + for r in parse_requirements(package="ultralytics"): + try: + current = metadata.version(r.name) + is_met = "✅ " if check_version(current, str(r.specifier), hard=True) else "❌ " + except metadata.PackageNotFoundError: + current = "(not installed)" + is_met = "❌ " + LOGGER.info(f"{r.name:<20}{is_met}{current}{r.specifier}") + + if is_github_action_running(): + LOGGER.info( + f"\nRUNNER_OS: {os.getenv('RUNNER_OS')}\n" + f"GITHUB_EVENT_NAME: {os.getenv('GITHUB_EVENT_NAME')}\n" + f"GITHUB_WORKFLOW: {os.getenv('GITHUB_WORKFLOW')}\n" + f"GITHUB_ACTOR: {os.getenv('GITHUB_ACTOR')}\n" + f"GITHUB_REPOSITORY: {os.getenv('GITHUB_REPOSITORY')}\n" + f"GITHUB_REPOSITORY_OWNER: {os.getenv('GITHUB_REPOSITORY_OWNER')}\n" + ) + + +def check_amp(model): + """ + This function checks the PyTorch Automatic Mixed Precision (AMP) functionality of a YOLOv8 model. If the checks + fail, it means there are anomalies with AMP on the system that may cause NaN losses or zero-mAP results, so AMP will + be disabled during training. + + Args: + model (nn.Module): A YOLOv8 model instance. + + Example: + ```python + from ultralytics import YOLO + from ultralytics.utils.checks import check_amp + + model = YOLO('yolov8n.pt').model.cuda() + check_amp(model) + ``` + + Returns: + (bool): Returns True if the AMP functionality works correctly with YOLOv8 model, else False. + """ + device = next(model.parameters()).device # get model device + if device.type in ("cpu", "mps"): + return False # AMP only used on CUDA devices + + def amp_allclose(m, im): + """All close FP32 vs AMP results.""" + a = m(im, device=device, verbose=False)[0].boxes.data # FP32 inference + with torch.cuda.amp.autocast(True): + b = m(im, device=device, verbose=False)[0].boxes.data # AMP inference + del m + return a.shape == b.shape and torch.allclose(a, b.float(), atol=0.5) # close to 0.5 absolute tolerance + + im = ASSETS / "bus.jpg" # image to check + prefix = colorstr("AMP: ") + LOGGER.info(f"{prefix}running Automatic Mixed Precision (AMP) checks with YOLOv8n...") + warning_msg = "Setting 'amp=True'. If you experience zero-mAP or NaN losses you can disable AMP with amp=False." + try: + from ultralytics import YOLO + + assert amp_allclose(YOLO("yolov8n.pt"), im) + LOGGER.info(f"{prefix}checks passed ✅") + except ConnectionError: + LOGGER.warning(f"{prefix}checks skipped ⚠️, offline and unable to download YOLOv8n. {warning_msg}") + except (AttributeError, ModuleNotFoundError): + LOGGER.warning( + f"{prefix}checks skipped ⚠️. " + f"Unable to load YOLOv8n due to possible Ultralytics package modifications. {warning_msg}" + ) + except AssertionError: + LOGGER.warning( + f"{prefix}checks failed ❌. Anomalies were detected with AMP on your system that may lead to " + f"NaN losses or zero-mAP results, so AMP will be disabled during training." + ) + return False + return True + + +def git_describe(path=ROOT): # path must be a directory + """Return human-readable git description, i.e. v5.0-5-g3e25f1e https://git-scm.com/docs/git-describe.""" + with contextlib.suppress(Exception): + return subprocess.check_output(f"git -C {path} describe --tags --long --always", shell=True).decode()[:-1] + return "" + + +def print_args(args: Optional[dict] = None, show_file=True, show_func=False): + """Print function arguments (optional args dict).""" + + def strip_auth(v): + """Clean longer Ultralytics HUB URLs by stripping potential authentication information.""" + return clean_url(v) if (isinstance(v, str) and v.startswith("http") and len(v) > 100) else v + + x = inspect.currentframe().f_back # previous frame + file, _, func, _, _ = inspect.getframeinfo(x) + if args is None: # get args automatically + args, _, _, frm = inspect.getargvalues(x) + args = {k: v for k, v in frm.items() if k in args} + try: + file = Path(file).resolve().relative_to(ROOT).with_suffix("") + except ValueError: + file = Path(file).stem + s = (f"{file}: " if show_file else "") + (f"{func}: " if show_func else "") + LOGGER.info(colorstr(s) + ", ".join(f"{k}={strip_auth(v)}" for k, v in args.items())) + + +def cuda_device_count() -> int: + """ + Get the number of NVIDIA GPUs available in the environment. + + Returns: + (int): The number of NVIDIA GPUs available. + """ + try: + # Run the nvidia-smi command and capture its output + output = subprocess.check_output( + ["nvidia-smi", "--query-gpu=count", "--format=csv,noheader,nounits"], encoding="utf-8" + ) + + # Take the first line and strip any leading/trailing white space + first_line = output.strip().split("\n")[0] + + return int(first_line) + except (subprocess.CalledProcessError, FileNotFoundError, ValueError): + # If the command fails, nvidia-smi is not found, or output is not an integer, assume no GPUs are available + return 0 + + +def cuda_is_available() -> bool: + """ + Check if CUDA is available in the environment. + + Returns: + (bool): True if one or more NVIDIA GPUs are available, False otherwise. + """ + return cuda_device_count() > 0 + + +# Define constants +IS_PYTHON_3_12 = PYTHON_VERSION.startswith("3.12") diff --git a/ultralytics/utils/dist.py b/ultralytics/utils/dist.py new file mode 100644 index 0000000..b669e52 --- /dev/null +++ b/ultralytics/utils/dist.py @@ -0,0 +1,71 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import os +import shutil +import socket +import sys +import tempfile + +from . import USER_CONFIG_DIR +from .torch_utils import TORCH_1_9 + + +def find_free_network_port() -> int: + """ + Finds a free port on localhost. + + It is useful in single-node training when we don't want to connect to a real main node but have to set the + `MASTER_PORT` environment variable. + """ + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] # port + + +def generate_ddp_file(trainer): + """Generates a DDP file and returns its file name.""" + module, name = f"{trainer.__class__.__module__}.{trainer.__class__.__name__}".rsplit(".", 1) + + content = f""" +# Ultralytics Multi-GPU training temp file (should be automatically deleted after use) +overrides = {vars(trainer.args)} + +if __name__ == "__main__": + from {module} import {name} + from ultralytics.utils import DEFAULT_CFG_DICT + + cfg = DEFAULT_CFG_DICT.copy() + cfg.update(save_dir='') # handle the extra key 'save_dir' + trainer = {name}(cfg=cfg, overrides=overrides) + results = trainer.train() +""" + (USER_CONFIG_DIR / "DDP").mkdir(exist_ok=True) + with tempfile.NamedTemporaryFile( + prefix="_temp_", + suffix=f"{id(trainer)}.py", + mode="w+", + encoding="utf-8", + dir=USER_CONFIG_DIR / "DDP", + delete=False, + ) as file: + file.write(content) + return file.name + + +def generate_ddp_command(world_size, trainer): + """Generates and returns command for distributed training.""" + import __main__ # noqa local import to avoid https://github.com/Lightning-AI/lightning/issues/15218 + + if not trainer.resume: + shutil.rmtree(trainer.save_dir) # remove the save_dir + file = generate_ddp_file(trainer) + dist_cmd = "torch.distributed.run" if TORCH_1_9 else "torch.distributed.launch" + port = find_free_network_port() + cmd = [sys.executable, "-m", dist_cmd, "--nproc_per_node", f"{world_size}", "--master_port", f"{port}", file] + return cmd, file + + +def ddp_cleanup(trainer, file): + """Delete temp file if created.""" + if f"{id(trainer)}.py" in file: # if temp_file suffix in file + os.remove(file) diff --git a/ultralytics/utils/downloads.py b/ultralytics/utils/downloads.py new file mode 100644 index 0000000..6191ade --- /dev/null +++ b/ultralytics/utils/downloads.py @@ -0,0 +1,500 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import contextlib +import re +import shutil +import subprocess +from itertools import repeat +from multiprocessing.pool import ThreadPool +from pathlib import Path +from urllib import parse, request + +import requests +import torch + +from ultralytics.utils import LOGGER, TQDM, checks, clean_url, emojis, is_online, url2file + +# Define Ultralytics GitHub assets maintained at https://github.com/ultralytics/assets +GITHUB_ASSETS_REPO = "ultralytics/assets" +GITHUB_ASSETS_NAMES = ( + [f"yolov8{k}{suffix}.pt" for k in "nsmlx" for suffix in ("", "-cls", "-seg", "-pose", "-obb")] + + [f"yolov5{k}{resolution}u.pt" for k in "nsmlx" for resolution in ("", "6")] + + [f"yolov3{k}u.pt" for k in ("", "-spp", "-tiny")] + + [f"yolov8{k}-world.pt" for k in "smlx"] + + [f"yolov8{k}-worldv2.pt" for k in "smlx"] + + [f"yolov9{k}.pt" for k in "ce"] + + [f"yolo_nas_{k}.pt" for k in "sml"] + + [f"sam_{k}.pt" for k in "bl"] + + [f"FastSAM-{k}.pt" for k in "sx"] + + [f"rtdetr-{k}.pt" for k in "lx"] + + ["mobile_sam.pt"] + + ["calibration_image_sample_data_20x128x128x3_float32.npy.zip"] +) +GITHUB_ASSETS_STEMS = [Path(k).stem for k in GITHUB_ASSETS_NAMES] + + +def is_url(url, check=False): + """ + Validates if the given string is a URL and optionally checks if the URL exists online. + + Args: + url (str): The string to be validated as a URL. + check (bool, optional): If True, performs an additional check to see if the URL exists online. + Defaults to True. + + Returns: + (bool): Returns True for a valid URL. If 'check' is True, also returns True if the URL exists online. + Returns False otherwise. + + Example: + ```python + valid = is_url("https://www.example.com") + ``` + """ + with contextlib.suppress(Exception): + url = str(url) + result = parse.urlparse(url) + assert all([result.scheme, result.netloc]) # check if is url + if check: + with request.urlopen(url) as response: + return response.getcode() == 200 # check if exists online + return True + return False + + +def delete_dsstore(path, files_to_delete=(".DS_Store", "__MACOSX")): + """ + Deletes all ".DS_store" files under a specified directory. + + Args: + path (str, optional): The directory path where the ".DS_store" files should be deleted. + files_to_delete (tuple): The files to be deleted. + + Example: + ```python + from ultralytics.utils.downloads import delete_dsstore + + delete_dsstore('path/to/dir') + ``` + + Note: + ".DS_store" files are created by the Apple operating system and contain metadata about folders and files. They + are hidden system files and can cause issues when transferring files between different operating systems. + """ + for file in files_to_delete: + matches = list(Path(path).rglob(file)) + LOGGER.info(f"Deleting {file} files: {matches}") + for f in matches: + f.unlink() + + +def zip_directory(directory, compress=True, exclude=(".DS_Store", "__MACOSX"), progress=True): + """ + Zips the contents of a directory, excluding files containing strings in the exclude list. The resulting zip file is + named after the directory and placed alongside it. + + Args: + directory (str | Path): The path to the directory to be zipped. + compress (bool): Whether to compress the files while zipping. Default is True. + exclude (tuple, optional): A tuple of filename strings to be excluded. Defaults to ('.DS_Store', '__MACOSX'). + progress (bool, optional): Whether to display a progress bar. Defaults to True. + + Returns: + (Path): The path to the resulting zip file. + + Example: + ```python + from ultralytics.utils.downloads import zip_directory + + file = zip_directory('path/to/dir') + ``` + """ + from zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile + + delete_dsstore(directory) + directory = Path(directory) + if not directory.is_dir(): + raise FileNotFoundError(f"Directory '{directory}' does not exist.") + + # Unzip with progress bar + files_to_zip = [f for f in directory.rglob("*") if f.is_file() and all(x not in f.name for x in exclude)] + zip_file = directory.with_suffix(".zip") + compression = ZIP_DEFLATED if compress else ZIP_STORED + with ZipFile(zip_file, "w", compression) as f: + for file in TQDM(files_to_zip, desc=f"Zipping {directory} to {zip_file}...", unit="file", disable=not progress): + f.write(file, file.relative_to(directory)) + + return zip_file # return path to zip file + + +def unzip_file(file, path=None, exclude=(".DS_Store", "__MACOSX"), exist_ok=False, progress=True): + """ + Unzips a *.zip file to the specified path, excluding files containing strings in the exclude list. + + If the zipfile does not contain a single top-level directory, the function will create a new + directory with the same name as the zipfile (without the extension) to extract its contents. + If a path is not provided, the function will use the parent directory of the zipfile as the default path. + + Args: + file (str): The path to the zipfile to be extracted. + path (str, optional): The path to extract the zipfile to. Defaults to None. + exclude (tuple, optional): A tuple of filename strings to be excluded. Defaults to ('.DS_Store', '__MACOSX'). + exist_ok (bool, optional): Whether to overwrite existing contents if they exist. Defaults to False. + progress (bool, optional): Whether to display a progress bar. Defaults to True. + + Raises: + BadZipFile: If the provided file does not exist or is not a valid zipfile. + + Returns: + (Path): The path to the directory where the zipfile was extracted. + + Example: + ```python + from ultralytics.utils.downloads import unzip_file + + dir = unzip_file('path/to/file.zip') + ``` + """ + from zipfile import BadZipFile, ZipFile, is_zipfile + + if not (Path(file).exists() and is_zipfile(file)): + raise BadZipFile(f"File '{file}' does not exist or is a bad zip file.") + if path is None: + path = Path(file).parent # default path + + # Unzip the file contents + with ZipFile(file) as zipObj: + files = [f for f in zipObj.namelist() if all(x not in f for x in exclude)] + top_level_dirs = {Path(f).parts[0] for f in files} + + if len(top_level_dirs) > 1 or (len(files) > 1 and not files[0].endswith("/")): + # Zip has multiple files at top level + path = extract_path = Path(path) / Path(file).stem # i.e. ../datasets/coco8 + else: + # Zip has 1 top-level directory + extract_path = path # i.e. ../datasets + path = Path(path) / list(top_level_dirs)[0] # i.e. ../datasets/coco8 + + # Check if destination directory already exists and contains files + if path.exists() and any(path.iterdir()) and not exist_ok: + # If it exists and is not empty, return the path without unzipping + LOGGER.warning(f"WARNING ⚠️ Skipping {file} unzip as destination directory {path} is not empty.") + return path + + for f in TQDM(files, desc=f"Unzipping {file} to {Path(path).resolve()}...", unit="file", disable=not progress): + # Ensure the file is within the extract_path to avoid path traversal security vulnerability + if ".." in Path(f).parts: + LOGGER.warning(f"Potentially insecure file path: {f}, skipping extraction.") + continue + zipObj.extract(f, extract_path) + + return path # return unzip dir + + +def check_disk_space(url="https://ultralytics.com/assets/coco128.zip", path=Path.cwd(), sf=1.5, hard=True): + """ + Check if there is sufficient disk space to download and store a file. + + Args: + url (str, optional): The URL to the file. Defaults to 'https://ultralytics.com/assets/coco128.zip'. + path (str | Path, optional): The path or drive to check the available free space on. + sf (float, optional): Safety factor, the multiplier for the required free space. Defaults to 2.0. + hard (bool, optional): Whether to throw an error or not on insufficient disk space. Defaults to True. + + Returns: + (bool): True if there is sufficient disk space, False otherwise. + """ + try: + r = requests.head(url) # response + assert r.status_code < 400, f"URL error for {url}: {r.status_code} {r.reason}" # check response + except Exception: + return True # requests issue, default to True + + # Check file size + gib = 1 << 30 # bytes per GiB + data = int(r.headers.get("Content-Length", 0)) / gib # file size (GB) + total, used, free = (x / gib for x in shutil.disk_usage(path)) # bytes + + if data * sf < free: + return True # sufficient space + + # Insufficient space + text = ( + f"WARNING ⚠️ Insufficient free disk space {free:.1f} GB < {data * sf:.3f} GB required, " + f"Please free {data * sf - free:.1f} GB additional disk space and try again." + ) + if hard: + raise MemoryError(text) + LOGGER.warning(text) + return False + + +def get_google_drive_file_info(link): + """ + Retrieves the direct download link and filename for a shareable Google Drive file link. + + Args: + link (str): The shareable link of the Google Drive file. + + Returns: + (str): Direct download URL for the Google Drive file. + (str): Original filename of the Google Drive file. If filename extraction fails, returns None. + + Example: + ```python + from ultralytics.utils.downloads import get_google_drive_file_info + + link = "https://drive.google.com/file/d/1cqT-cJgANNrhIHCrEufUYhQ4RqiWG_lJ/view?usp=drive_link" + url, filename = get_google_drive_file_info(link) + ``` + """ + file_id = link.split("/d/")[1].split("/view")[0] + drive_url = f"https://drive.google.com/uc?export=download&id={file_id}" + filename = None + + # Start session + with requests.Session() as session: + response = session.get(drive_url, stream=True) + if "quota exceeded" in str(response.content.lower()): + raise ConnectionError( + emojis( + f"❌ Google Drive file download quota exceeded. " + f"Please try again later or download this file manually at {link}." + ) + ) + for k, v in response.cookies.items(): + if k.startswith("download_warning"): + drive_url += f"&confirm={v}" # v is token + cd = response.headers.get("content-disposition") + if cd: + filename = re.findall('filename="(.+)"', cd)[0] + return drive_url, filename + + +def safe_download( + url, + file=None, + dir=None, + unzip=True, + delete=False, + curl=False, + retry=3, + min_bytes=1e0, + exist_ok=False, + progress=True, +): + """ + Downloads files from a URL, with options for retrying, unzipping, and deleting the downloaded file. + + Args: + url (str): The URL of the file to be downloaded. + file (str, optional): The filename of the downloaded file. + If not provided, the file will be saved with the same name as the URL. + dir (str, optional): The directory to save the downloaded file. + If not provided, the file will be saved in the current working directory. + unzip (bool, optional): Whether to unzip the downloaded file. Default: True. + delete (bool, optional): Whether to delete the downloaded file after unzipping. Default: False. + curl (bool, optional): Whether to use curl command line tool for downloading. Default: False. + retry (int, optional): The number of times to retry the download in case of failure. Default: 3. + min_bytes (float, optional): The minimum number of bytes that the downloaded file should have, to be considered + a successful download. Default: 1E0. + exist_ok (bool, optional): Whether to overwrite existing contents during unzipping. Defaults to False. + progress (bool, optional): Whether to display a progress bar during the download. Default: True. + + Example: + ```python + from ultralytics.utils.downloads import safe_download + + link = "https://ultralytics.com/assets/bus.jpg" + path = safe_download(link) + ``` + """ + gdrive = url.startswith("https://drive.google.com/") # check if the URL is a Google Drive link + if gdrive: + url, file = get_google_drive_file_info(url) + + f = Path(dir or ".") / (file or url2file(url)) # URL converted to filename + if "://" not in str(url) and Path(url).is_file(): # URL exists ('://' check required in Windows Python<3.10) + f = Path(url) # filename + elif not f.is_file(): # URL and file do not exist + desc = f"Downloading {url if gdrive else clean_url(url)} to '{f}'" + LOGGER.info(f"{desc}...") + f.parent.mkdir(parents=True, exist_ok=True) # make directory if missing + check_disk_space(url, path=f.parent) + for i in range(retry + 1): + try: + if curl or i > 0: # curl download with retry, continue + s = "sS" * (not progress) # silent + r = subprocess.run(["curl", "-#", f"-{s}L", url, "-o", f, "--retry", "3", "-C", "-"]).returncode + assert r == 0, f"Curl return value {r}" + else: # urllib download + method = "torch" + if method == "torch": + torch.hub.download_url_to_file(url, f, progress=progress) + else: + with request.urlopen(url) as response, TQDM( + total=int(response.getheader("Content-Length", 0)), + desc=desc, + disable=not progress, + unit="B", + unit_scale=True, + unit_divisor=1024, + ) as pbar: + with open(f, "wb") as f_opened: + for data in response: + f_opened.write(data) + pbar.update(len(data)) + + if f.exists(): + if f.stat().st_size > min_bytes: + break # success + f.unlink() # remove partial downloads + except Exception as e: + if i == 0 and not is_online(): + raise ConnectionError(emojis(f"❌ Download failure for {url}. Environment is not online.")) from e + elif i >= retry: + raise ConnectionError(emojis(f"❌ Download failure for {url}. Retry limit reached.")) from e + LOGGER.warning(f"⚠️ Download failure, retrying {i + 1}/{retry} {url}...") + + if unzip and f.exists() and f.suffix in ("", ".zip", ".tar", ".gz"): + from zipfile import is_zipfile + + unzip_dir = (dir or f.parent).resolve() # unzip to dir if provided else unzip in place + if is_zipfile(f): + unzip_dir = unzip_file(file=f, path=unzip_dir, exist_ok=exist_ok, progress=progress) # unzip + elif f.suffix in (".tar", ".gz"): + LOGGER.info(f"Unzipping {f} to {unzip_dir}...") + subprocess.run(["tar", "xf" if f.suffix == ".tar" else "xfz", f, "--directory", unzip_dir], check=True) + if delete: + f.unlink() # remove zip + return unzip_dir + + +def get_github_assets(repo="ultralytics/assets", version="latest", retry=False): + """ + Retrieve the specified version's tag and assets from a GitHub repository. If the version is not specified, the + function fetches the latest release assets. + + Args: + repo (str, optional): The GitHub repository in the format 'owner/repo'. Defaults to 'ultralytics/assets'. + version (str, optional): The release version to fetch assets from. Defaults to 'latest'. + retry (bool, optional): Flag to retry the request in case of a failure. Defaults to False. + + Returns: + (tuple): A tuple containing the release tag and a list of asset names. + + Example: + ```python + tag, assets = get_github_assets(repo='ultralytics/assets', version='latest') + ``` + """ + + if version != "latest": + version = f"tags/{version}" # i.e. tags/v6.2 + url = f"https://api.github.com/repos/{repo}/releases/{version}" + r = requests.get(url) # github api + if r.status_code != 200 and r.reason != "rate limit exceeded" and retry: # failed and not 403 rate limit exceeded + r = requests.get(url) # try again + if r.status_code != 200: + LOGGER.warning(f"⚠️ GitHub assets check failure for {url}: {r.status_code} {r.reason}") + return "", [] + data = r.json() + return data["tag_name"], [x["name"] for x in data["assets"]] # tag, assets i.e. ['yolov8n.pt', 'yolov8s.pt', ...] + + +def attempt_download_asset(file, repo="ultralytics/assets", release="v8.1.0", **kwargs): + """ + Attempt to download a file from GitHub release assets if it is not found locally. The function checks for the file + locally first, then tries to download it from the specified GitHub repository release. + + Args: + file (str | Path): The filename or file path to be downloaded. + repo (str, optional): The GitHub repository in the format 'owner/repo'. Defaults to 'ultralytics/assets'. + release (str, optional): The specific release version to be downloaded. Defaults to 'v8.1.0'. + **kwargs (any): Additional keyword arguments for the download process. + + Returns: + (str): The path to the downloaded file. + + Example: + ```python + file_path = attempt_download_asset('yolov5s.pt', repo='ultralytics/assets', release='latest') + ``` + """ + from ultralytics.utils import SETTINGS # scoped for circular import + + # YOLOv3/5u updates + file = str(file) + file = checks.check_yolov5u_filename(file) + file = Path(file.strip().replace("'", "")) + if file.exists(): + return str(file) + elif (SETTINGS["weights_dir"] / file).exists(): + return str(SETTINGS["weights_dir"] / file) + else: + # URL specified + name = Path(parse.unquote(str(file))).name # decode '%2F' to '/' etc. + download_url = f"https://github.com/{repo}/releases/download" + if str(file).startswith(("http:/", "https:/")): # download + url = str(file).replace(":/", "://") # Pathlib turns :// -> :/ + file = url2file(name) # parse authentication https://url.com/file.txt?auth... + if Path(file).is_file(): + LOGGER.info(f"Found {clean_url(url)} locally at {file}") # file already exists + else: + safe_download(url=url, file=file, min_bytes=1e5, **kwargs) + + elif repo == GITHUB_ASSETS_REPO and name in GITHUB_ASSETS_NAMES: + safe_download(url=f"{download_url}/{release}/{name}", file=file, min_bytes=1e5, **kwargs) + + else: + tag, assets = get_github_assets(repo, release) + if not assets: + tag, assets = get_github_assets(repo) # latest release + if name in assets: + safe_download(url=f"{download_url}/{tag}/{name}", file=file, min_bytes=1e5, **kwargs) + + return str(file) + + +def download(url, dir=Path.cwd(), unzip=True, delete=False, curl=False, threads=1, retry=3, exist_ok=False): + """ + Downloads files from specified URLs to a given directory. Supports concurrent downloads if multiple threads are + specified. + + Args: + url (str | list): The URL or list of URLs of the files to be downloaded. + dir (Path, optional): The directory where the files will be saved. Defaults to the current working directory. + unzip (bool, optional): Flag to unzip the files after downloading. Defaults to True. + delete (bool, optional): Flag to delete the zip files after extraction. Defaults to False. + curl (bool, optional): Flag to use curl for downloading. Defaults to False. + threads (int, optional): Number of threads to use for concurrent downloads. Defaults to 1. + retry (int, optional): Number of retries in case of download failure. Defaults to 3. + exist_ok (bool, optional): Whether to overwrite existing contents during unzipping. Defaults to False. + + Example: + ```python + download('https://ultralytics.com/assets/example.zip', dir='path/to/dir', unzip=True) + ``` + """ + dir = Path(dir) + dir.mkdir(parents=True, exist_ok=True) # make directory + if threads > 1: + with ThreadPool(threads) as pool: + pool.map( + lambda x: safe_download( + url=x[0], + dir=x[1], + unzip=unzip, + delete=delete, + curl=curl, + retry=retry, + exist_ok=exist_ok, + progress=threads <= 1, + ), + zip(url, repeat(dir)), + ) + pool.close() + pool.join() + else: + for u in [url] if isinstance(url, (str, Path)) else url: + safe_download(url=u, dir=dir, unzip=unzip, delete=delete, curl=curl, retry=retry, exist_ok=exist_ok) diff --git a/ultralytics/utils/errors.py b/ultralytics/utils/errors.py new file mode 100644 index 0000000..86aee1d --- /dev/null +++ b/ultralytics/utils/errors.py @@ -0,0 +1,22 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from ultralytics.utils import emojis + + +class HUBModelError(Exception): + """ + Custom exception class for handling errors related to model fetching in Ultralytics YOLO. + + This exception is raised when a requested model is not found or cannot be retrieved. + The message is also processed to include emojis for better user experience. + + Attributes: + message (str): The error message displayed when the exception is raised. + + Note: + The message is automatically processed through the 'emojis' function from the 'ultralytics.utils' package. + """ + + def __init__(self, message="Model not found. Please check model URL and try again."): + """Create an exception for when a model is not found.""" + super().__init__(emojis(message)) diff --git a/ultralytics/utils/files.py b/ultralytics/utils/files.py new file mode 100644 index 0000000..719caca --- /dev/null +++ b/ultralytics/utils/files.py @@ -0,0 +1,188 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import contextlib +import glob +import os +import shutil +import tempfile +from contextlib import contextmanager +from datetime import datetime +from pathlib import Path + + +class WorkingDirectory(contextlib.ContextDecorator): + """Usage: @WorkingDirectory(dir) decorator or 'with WorkingDirectory(dir):' context manager.""" + + def __init__(self, new_dir): + """Sets the working directory to 'new_dir' upon instantiation.""" + self.dir = new_dir # new dir + self.cwd = Path.cwd().resolve() # current dir + + def __enter__(self): + """Changes the current directory to the specified directory.""" + os.chdir(self.dir) + + def __exit__(self, exc_type, exc_val, exc_tb): # noqa + """Restore the current working directory on context exit.""" + os.chdir(self.cwd) + + +@contextmanager +def spaces_in_path(path): + """ + Context manager to handle paths with spaces in their names. If a path contains spaces, it replaces them with + underscores, copies the file/directory to the new path, executes the context code block, then copies the + file/directory back to its original location. + + Args: + path (str | Path): The original path. + + Yields: + (Path): Temporary path with spaces replaced by underscores if spaces were present, otherwise the original path. + + Example: + ```python + with ultralytics.utils.files import spaces_in_path + + with spaces_in_path('/path/with spaces') as new_path: + # Your code here + ``` + """ + + # If path has spaces, replace them with underscores + if " " in str(path): + string = isinstance(path, str) # input type + path = Path(path) + + # Create a temporary directory and construct the new path + with tempfile.TemporaryDirectory() as tmp_dir: + tmp_path = Path(tmp_dir) / path.name.replace(" ", "_") + + # Copy file/directory + if path.is_dir(): + # tmp_path.mkdir(parents=True, exist_ok=True) + shutil.copytree(path, tmp_path) + elif path.is_file(): + tmp_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, tmp_path) + + try: + # Yield the temporary path + yield str(tmp_path) if string else tmp_path + + finally: + # Copy file/directory back + if tmp_path.is_dir(): + shutil.copytree(tmp_path, path, dirs_exist_ok=True) + elif tmp_path.is_file(): + shutil.copy2(tmp_path, path) # Copy back the file + + else: + # If there are no spaces, just yield the original path + yield path + + +def increment_path(path, exist_ok=False, sep="", mkdir=False): + """ + Increments a file or directory path, i.e. runs/exp --> runs/exp{sep}2, runs/exp{sep}3, ... etc. + + If the path exists and exist_ok is not set to True, the path will be incremented by appending a number and sep to + the end of the path. If the path is a file, the file extension will be preserved. If the path is a directory, the + number will be appended directly to the end of the path. If mkdir is set to True, the path will be created as a + directory if it does not already exist. + + Args: + path (str, pathlib.Path): Path to increment. + exist_ok (bool, optional): If True, the path will not be incremented and returned as-is. Defaults to False. + sep (str, optional): Separator to use between the path and the incrementation number. Defaults to ''. + mkdir (bool, optional): Create a directory if it does not exist. Defaults to False. + + Returns: + (pathlib.Path): Incremented path. + """ + path = Path(path) # os-agnostic + if path.exists() and not exist_ok: + path, suffix = (path.with_suffix(""), path.suffix) if path.is_file() else (path, "") + + # Method 1 + for n in range(2, 9999): + p = f"{path}{sep}{n}{suffix}" # increment path + if not os.path.exists(p): + break + path = Path(p) + + if mkdir: + path.mkdir(parents=True, exist_ok=True) # make directory + + return path + + +def file_age(path=__file__): + """Return days since last file update.""" + dt = datetime.now() - datetime.fromtimestamp(Path(path).stat().st_mtime) # delta + return dt.days # + dt.seconds / 86400 # fractional days + + +def file_date(path=__file__): + """Return human-readable file modification date, i.e. '2021-3-26'.""" + t = datetime.fromtimestamp(Path(path).stat().st_mtime) + return f"{t.year}-{t.month}-{t.day}" + + +def file_size(path): + """Return file/dir size (MB).""" + if isinstance(path, (str, Path)): + mb = 1 << 20 # bytes to MiB (1024 ** 2) + path = Path(path) + if path.is_file(): + return path.stat().st_size / mb + elif path.is_dir(): + return sum(f.stat().st_size for f in path.glob("**/*") if f.is_file()) / mb + return 0.0 + + +def get_latest_run(search_dir="."): + """Return path to most recent 'last.pt' in /runs (i.e. to --resume from).""" + last_list = glob.glob(f"{search_dir}/**/last*.pt", recursive=True) + return max(last_list, key=os.path.getctime) if last_list else "" + + +def update_models(model_names=("yolov8n.pt",), source_dir=Path("."), update_names=False): + """ + Updates and re-saves specified YOLO models in an 'updated_models' subdirectory. + + Args: + model_names (tuple, optional): Model filenames to update, defaults to ("yolov8n.pt"). + source_dir (Path, optional): Directory containing models and target subdirectory, defaults to current directory. + update_names (bool, optional): Update model names from a data YAML. + + Example: + ```python + from ultralytics.utils.files import update_models + + model_names = (f"rtdetr-{size}.pt" for size in "lx") + update_models(model_names) + ``` + """ + from ultralytics import YOLO + from ultralytics.nn.autobackend import default_class_names + + target_dir = source_dir / "updated_models" + target_dir.mkdir(parents=True, exist_ok=True) # Ensure target directory exists + + for model_name in model_names: + model_path = source_dir / model_name + print(f"Loading model from {model_path}") + + # Load model + model = YOLO(model_path) + model.half() + if update_names: # update model names from a dataset YAML + model.model.names = default_class_names("coco8.yaml") + + # Define new save path + save_path = target_dir / model_name + + # Save model using model.save() + print(f"Re-saving {model_name} model to {save_path}") + model.save(save_path, use_dill=False) diff --git a/ultralytics/utils/instance.py b/ultralytics/utils/instance.py new file mode 100644 index 0000000..4e9ef2c --- /dev/null +++ b/ultralytics/utils/instance.py @@ -0,0 +1,407 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from collections import abc +from itertools import repeat +from numbers import Number +from typing import List + +import numpy as np + +from .ops import ltwh2xywh, ltwh2xyxy, xywh2ltwh, xywh2xyxy, xyxy2ltwh, xyxy2xywh + + +def _ntuple(n): + """From PyTorch internals.""" + + def parse(x): + """Parse bounding boxes format between XYWH and LTWH.""" + return x if isinstance(x, abc.Iterable) else tuple(repeat(x, n)) + + return parse + + +to_2tuple = _ntuple(2) +to_4tuple = _ntuple(4) + +# `xyxy` means left top and right bottom +# `xywh` means center x, center y and width, height(YOLO format) +# `ltwh` means left top and width, height(COCO format) +_formats = ["xyxy", "xywh", "ltwh"] + +__all__ = ("Bboxes",) # tuple or list + + +class Bboxes: + """ + A class for handling bounding boxes. + + The class supports various bounding box formats like 'xyxy', 'xywh', and 'ltwh'. + Bounding box data should be provided in numpy arrays. + + Attributes: + bboxes (numpy.ndarray): The bounding boxes stored in a 2D numpy array. + format (str): The format of the bounding boxes ('xyxy', 'xywh', or 'ltwh'). + + Note: + This class does not handle normalization or denormalization of bounding boxes. + """ + + def __init__(self, bboxes, format="xyxy") -> None: + """Initializes the Bboxes class with bounding box data in a specified format.""" + assert format in _formats, f"Invalid bounding box format: {format}, format must be one of {_formats}" + bboxes = bboxes[None, :] if bboxes.ndim == 1 else bboxes + assert bboxes.ndim == 2 + assert bboxes.shape[1] == 4 + self.bboxes = bboxes + self.format = format + # self.normalized = normalized + + def convert(self, format): + """Converts bounding box format from one type to another.""" + assert format in _formats, f"Invalid bounding box format: {format}, format must be one of {_formats}" + if self.format == format: + return + elif self.format == "xyxy": + func = xyxy2xywh if format == "xywh" else xyxy2ltwh + elif self.format == "xywh": + func = xywh2xyxy if format == "xyxy" else xywh2ltwh + else: + func = ltwh2xyxy if format == "xyxy" else ltwh2xywh + self.bboxes = func(self.bboxes) + self.format = format + + def areas(self): + """Return box areas.""" + self.convert("xyxy") + return (self.bboxes[:, 2] - self.bboxes[:, 0]) * (self.bboxes[:, 3] - self.bboxes[:, 1]) + + # def denormalize(self, w, h): + # if not self.normalized: + # return + # assert (self.bboxes <= 1.0).all() + # self.bboxes[:, 0::2] *= w + # self.bboxes[:, 1::2] *= h + # self.normalized = False + # + # def normalize(self, w, h): + # if self.normalized: + # return + # assert (self.bboxes > 1.0).any() + # self.bboxes[:, 0::2] /= w + # self.bboxes[:, 1::2] /= h + # self.normalized = True + + def mul(self, scale): + """ + Args: + scale (tuple | list | int): the scale for four coords. + """ + if isinstance(scale, Number): + scale = to_4tuple(scale) + assert isinstance(scale, (tuple, list)) + assert len(scale) == 4 + self.bboxes[:, 0] *= scale[0] + self.bboxes[:, 1] *= scale[1] + self.bboxes[:, 2] *= scale[2] + self.bboxes[:, 3] *= scale[3] + + def add(self, offset): + """ + Args: + offset (tuple | list | int): the offset for four coords. + """ + if isinstance(offset, Number): + offset = to_4tuple(offset) + assert isinstance(offset, (tuple, list)) + assert len(offset) == 4 + self.bboxes[:, 0] += offset[0] + self.bboxes[:, 1] += offset[1] + self.bboxes[:, 2] += offset[2] + self.bboxes[:, 3] += offset[3] + + def __len__(self): + """Return the number of boxes.""" + return len(self.bboxes) + + @classmethod + def concatenate(cls, boxes_list: List["Bboxes"], axis=0) -> "Bboxes": + """ + Concatenate a list of Bboxes objects into a single Bboxes object. + + Args: + boxes_list (List[Bboxes]): A list of Bboxes objects to concatenate. + axis (int, optional): The axis along which to concatenate the bounding boxes. + Defaults to 0. + + Returns: + Bboxes: A new Bboxes object containing the concatenated bounding boxes. + + Note: + The input should be a list or tuple of Bboxes objects. + """ + assert isinstance(boxes_list, (list, tuple)) + if not boxes_list: + return cls(np.empty(0)) + assert all(isinstance(box, Bboxes) for box in boxes_list) + + if len(boxes_list) == 1: + return boxes_list[0] + return cls(np.concatenate([b.bboxes for b in boxes_list], axis=axis)) + + def __getitem__(self, index) -> "Bboxes": + """ + Retrieve a specific bounding box or a set of bounding boxes using indexing. + + Args: + index (int, slice, or np.ndarray): The index, slice, or boolean array to select + the desired bounding boxes. + + Returns: + Bboxes: A new Bboxes object containing the selected bounding boxes. + + Raises: + AssertionError: If the indexed bounding boxes do not form a 2-dimensional matrix. + + Note: + When using boolean indexing, make sure to provide a boolean array with the same + length as the number of bounding boxes. + """ + if isinstance(index, int): + return Bboxes(self.bboxes[index].view(1, -1)) + b = self.bboxes[index] + assert b.ndim == 2, f"Indexing on Bboxes with {index} failed to return a matrix!" + return Bboxes(b) + + +class Instances: + """ + Container for bounding boxes, segments, and keypoints of detected objects in an image. + + Attributes: + _bboxes (Bboxes): Internal object for handling bounding box operations. + keypoints (ndarray): keypoints(x, y, visible) with shape [N, 17, 3]. Default is None. + normalized (bool): Flag indicating whether the bounding box coordinates are normalized. + segments (ndarray): Segments array with shape [N, 1000, 2] after resampling. + + Args: + bboxes (ndarray): An array of bounding boxes with shape [N, 4]. + segments (list | ndarray, optional): A list or array of object segments. Default is None. + keypoints (ndarray, optional): An array of keypoints with shape [N, 17, 3]. Default is None. + bbox_format (str, optional): The format of bounding boxes ('xywh' or 'xyxy'). Default is 'xywh'. + normalized (bool, optional): Whether the bounding box coordinates are normalized. Default is True. + + Examples: + ```python + # Create an Instances object + instances = Instances( + bboxes=np.array([[10, 10, 30, 30], [20, 20, 40, 40]]), + segments=[np.array([[5, 5], [10, 10]]), np.array([[15, 15], [20, 20]])], + keypoints=np.array([[[5, 5, 1], [10, 10, 1]], [[15, 15, 1], [20, 20, 1]]]) + ) + ``` + + Note: + The bounding box format is either 'xywh' or 'xyxy', and is determined by the `bbox_format` argument. + This class does not perform input validation, and it assumes the inputs are well-formed. + """ + + def __init__(self, bboxes, segments=None, keypoints=None, bbox_format="xywh", normalized=True) -> None: + """ + Args: + bboxes (ndarray): bboxes with shape [N, 4]. + segments (list | ndarray): segments. + keypoints (ndarray): keypoints(x, y, visible) with shape [N, 17, 3]. + """ + self._bboxes = Bboxes(bboxes=bboxes, format=bbox_format) + self.keypoints = keypoints + self.normalized = normalized + self.segments = segments + + def convert_bbox(self, format): + """Convert bounding box format.""" + self._bboxes.convert(format=format) + + @property + def bbox_areas(self): + """Calculate the area of bounding boxes.""" + return self._bboxes.areas() + + def scale(self, scale_w, scale_h, bbox_only=False): + """This might be similar with denormalize func but without normalized sign.""" + self._bboxes.mul(scale=(scale_w, scale_h, scale_w, scale_h)) + if bbox_only: + return + self.segments[..., 0] *= scale_w + self.segments[..., 1] *= scale_h + if self.keypoints is not None: + self.keypoints[..., 0] *= scale_w + self.keypoints[..., 1] *= scale_h + + def denormalize(self, w, h): + """Denormalizes boxes, segments, and keypoints from normalized coordinates.""" + if not self.normalized: + return + self._bboxes.mul(scale=(w, h, w, h)) + self.segments[..., 0] *= w + self.segments[..., 1] *= h + if self.keypoints is not None: + self.keypoints[..., 0] *= w + self.keypoints[..., 1] *= h + self.normalized = False + + def normalize(self, w, h): + """Normalize bounding boxes, segments, and keypoints to image dimensions.""" + if self.normalized: + return + self._bboxes.mul(scale=(1 / w, 1 / h, 1 / w, 1 / h)) + self.segments[..., 0] /= w + self.segments[..., 1] /= h + if self.keypoints is not None: + self.keypoints[..., 0] /= w + self.keypoints[..., 1] /= h + self.normalized = True + + def add_padding(self, padw, padh): + """Handle rect and mosaic situation.""" + assert not self.normalized, "you should add padding with absolute coordinates." + self._bboxes.add(offset=(padw, padh, padw, padh)) + self.segments[..., 0] += padw + self.segments[..., 1] += padh + if self.keypoints is not None: + self.keypoints[..., 0] += padw + self.keypoints[..., 1] += padh + + def __getitem__(self, index) -> "Instances": + """ + Retrieve a specific instance or a set of instances using indexing. + + Args: + index (int, slice, or np.ndarray): The index, slice, or boolean array to select + the desired instances. + + Returns: + Instances: A new Instances object containing the selected bounding boxes, + segments, and keypoints if present. + + Note: + When using boolean indexing, make sure to provide a boolean array with the same + length as the number of instances. + """ + segments = self.segments[index] if len(self.segments) else self.segments + keypoints = self.keypoints[index] if self.keypoints is not None else None + bboxes = self.bboxes[index] + bbox_format = self._bboxes.format + return Instances( + bboxes=bboxes, + segments=segments, + keypoints=keypoints, + bbox_format=bbox_format, + normalized=self.normalized, + ) + + def flipud(self, h): + """Flips the coordinates of bounding boxes, segments, and keypoints vertically.""" + if self._bboxes.format == "xyxy": + y1 = self.bboxes[:, 1].copy() + y2 = self.bboxes[:, 3].copy() + self.bboxes[:, 1] = h - y2 + self.bboxes[:, 3] = h - y1 + else: + self.bboxes[:, 1] = h - self.bboxes[:, 1] + self.segments[..., 1] = h - self.segments[..., 1] + if self.keypoints is not None: + self.keypoints[..., 1] = h - self.keypoints[..., 1] + + def fliplr(self, w): + """Reverses the order of the bounding boxes and segments horizontally.""" + if self._bboxes.format == "xyxy": + x1 = self.bboxes[:, 0].copy() + x2 = self.bboxes[:, 2].copy() + self.bboxes[:, 0] = w - x2 + self.bboxes[:, 2] = w - x1 + else: + self.bboxes[:, 0] = w - self.bboxes[:, 0] + self.segments[..., 0] = w - self.segments[..., 0] + if self.keypoints is not None: + self.keypoints[..., 0] = w - self.keypoints[..., 0] + + def clip(self, w, h): + """Clips bounding boxes, segments, and keypoints values to stay within image boundaries.""" + ori_format = self._bboxes.format + self.convert_bbox(format="xyxy") + self.bboxes[:, [0, 2]] = self.bboxes[:, [0, 2]].clip(0, w) + self.bboxes[:, [1, 3]] = self.bboxes[:, [1, 3]].clip(0, h) + if ori_format != "xyxy": + self.convert_bbox(format=ori_format) + self.segments[..., 0] = self.segments[..., 0].clip(0, w) + self.segments[..., 1] = self.segments[..., 1].clip(0, h) + if self.keypoints is not None: + self.keypoints[..., 0] = self.keypoints[..., 0].clip(0, w) + self.keypoints[..., 1] = self.keypoints[..., 1].clip(0, h) + + def remove_zero_area_boxes(self): + """ + Remove zero-area boxes, i.e. after clipping some boxes may have zero width or height. + + This removes them. + """ + good = self.bbox_areas > 0 + if not all(good): + self._bboxes = self._bboxes[good] + if len(self.segments): + self.segments = self.segments[good] + if self.keypoints is not None: + self.keypoints = self.keypoints[good] + return good + + def update(self, bboxes, segments=None, keypoints=None): + """Updates instance variables.""" + self._bboxes = Bboxes(bboxes, format=self._bboxes.format) + if segments is not None: + self.segments = segments + if keypoints is not None: + self.keypoints = keypoints + + def __len__(self): + """Return the length of the instance list.""" + return len(self.bboxes) + + @classmethod + def concatenate(cls, instances_list: List["Instances"], axis=0) -> "Instances": + """ + Concatenates a list of Instances objects into a single Instances object. + + Args: + instances_list (List[Instances]): A list of Instances objects to concatenate. + axis (int, optional): The axis along which the arrays will be concatenated. Defaults to 0. + + Returns: + Instances: A new Instances object containing the concatenated bounding boxes, + segments, and keypoints if present. + + Note: + The `Instances` objects in the list should have the same properties, such as + the format of the bounding boxes, whether keypoints are present, and if the + coordinates are normalized. + """ + assert isinstance(instances_list, (list, tuple)) + if not instances_list: + return cls(np.empty(0)) + assert all(isinstance(instance, Instances) for instance in instances_list) + + if len(instances_list) == 1: + return instances_list[0] + + use_keypoint = instances_list[0].keypoints is not None + bbox_format = instances_list[0]._bboxes.format + normalized = instances_list[0].normalized + + cat_boxes = np.concatenate([ins.bboxes for ins in instances_list], axis=axis) + cat_segments = np.concatenate([b.segments for b in instances_list], axis=axis) + cat_keypoints = np.concatenate([b.keypoints for b in instances_list], axis=axis) if use_keypoint else None + return cls(cat_boxes, cat_segments, cat_keypoints, bbox_format, normalized) + + @property + def bboxes(self): + """Return bounding boxes.""" + return self._bboxes.bboxes diff --git a/ultralytics/utils/loss.py b/ultralytics/utils/loss.py new file mode 100644 index 0000000..d0ca9c3 --- /dev/null +++ b/ultralytics/utils/loss.py @@ -0,0 +1,727 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ultralytics.utils.metrics import OKS_SIGMA +from ultralytics.utils.ops import crop_mask, xywh2xyxy, xyxy2xywh +from ultralytics.utils.tal import RotatedTaskAlignedAssigner, TaskAlignedAssigner, dist2bbox, dist2rbox, make_anchors +from .metrics import bbox_iou, probiou +from .tal import bbox2dist + + +class VarifocalLoss(nn.Module): + """ + Varifocal loss by Zhang et al. + + https://arxiv.org/abs/2008.13367. + """ + + def __init__(self): + """Initialize the VarifocalLoss class.""" + super().__init__() + + @staticmethod + def forward(pred_score, gt_score, label, alpha=0.75, gamma=2.0): + """Computes varfocal loss.""" + weight = alpha * pred_score.sigmoid().pow(gamma) * (1 - label) + gt_score * label + with torch.cuda.amp.autocast(enabled=False): + loss = ( + (F.binary_cross_entropy_with_logits(pred_score.float(), gt_score.float(), reduction="none") * weight) + .mean(1) + .sum() + ) + return loss + + +class FocalLoss(nn.Module): + """Wraps focal loss around existing loss_fcn(), i.e. criteria = FocalLoss(nn.BCEWithLogitsLoss(), gamma=1.5).""" + + def __init__(self): + """Initializer for FocalLoss class with no parameters.""" + super().__init__() + + @staticmethod + def forward(pred, label, gamma=1.5, alpha=0.25): + """Calculates and updates confusion matrix for object detection/classification tasks.""" + loss = F.binary_cross_entropy_with_logits(pred, label, reduction="none") + # p_t = torch.exp(-loss) + # loss *= self.alpha * (1.000001 - p_t) ** self.gamma # non-zero power for gradient stability + + # TF implementation https://github.com/tensorflow/addons/blob/v0.7.1/tensorflow_addons/losses/focal_loss.py + pred_prob = pred.sigmoid() # prob from logits + p_t = label * pred_prob + (1 - label) * (1 - pred_prob) + modulating_factor = (1.0 - p_t) ** gamma + loss *= modulating_factor + if alpha > 0: + alpha_factor = label * alpha + (1 - label) * (1 - alpha) + loss *= alpha_factor + return loss.mean(1).sum() + + +class BboxLoss(nn.Module): + """Criterion class for computing training losses during training.""" + + def __init__(self, reg_max, use_dfl=False): + """Initialize the BboxLoss module with regularization maximum and DFL settings.""" + super().__init__() + self.reg_max = reg_max + self.use_dfl = use_dfl + + def forward(self, pred_dist, pred_bboxes, anchor_points, target_bboxes, target_scores, target_scores_sum, fg_mask): + """IoU loss.""" + weight = target_scores.sum(-1)[fg_mask].unsqueeze(-1) + iou = bbox_iou(pred_bboxes[fg_mask], target_bboxes[fg_mask], xywh=False, CIoU=True) + loss_iou = ((1.0 - iou) * weight).sum() / target_scores_sum + + # DFL loss + if self.use_dfl: + target_ltrb = bbox2dist(anchor_points, target_bboxes, self.reg_max) + loss_dfl = self._df_loss(pred_dist[fg_mask].view(-1, self.reg_max + 1), target_ltrb[fg_mask]) * weight + loss_dfl = loss_dfl.sum() / target_scores_sum + else: + loss_dfl = torch.tensor(0.0).to(pred_dist.device) + + return loss_iou, loss_dfl + + @staticmethod + def _df_loss(pred_dist, target): + """ + Return sum of left and right DFL losses. + + Distribution Focal Loss (DFL) proposed in Generalized Focal Loss + https://ieeexplore.ieee.org/document/9792391 + """ + tl = target.long() # target left + tr = tl + 1 # target right + wl = tr - target # weight left + wr = 1 - wl # weight right + return ( + F.cross_entropy(pred_dist, tl.view(-1), reduction="none").view(tl.shape) * wl + + F.cross_entropy(pred_dist, tr.view(-1), reduction="none").view(tl.shape) * wr + ).mean(-1, keepdim=True) + + +class RotatedBboxLoss(BboxLoss): + """Criterion class for computing training losses during training.""" + + def __init__(self, reg_max, use_dfl=False): + """Initialize the BboxLoss module with regularization maximum and DFL settings.""" + super().__init__(reg_max, use_dfl) + + def forward(self, pred_dist, pred_bboxes, anchor_points, target_bboxes, target_scores, target_scores_sum, fg_mask): + """IoU loss.""" + weight = target_scores.sum(-1)[fg_mask].unsqueeze(-1) + iou = probiou(pred_bboxes[fg_mask], target_bboxes[fg_mask]) + loss_iou = ((1.0 - iou) * weight).sum() / target_scores_sum + + # DFL loss + if self.use_dfl: + target_ltrb = bbox2dist(anchor_points, xywh2xyxy(target_bboxes[..., :4]), self.reg_max) + loss_dfl = self._df_loss(pred_dist[fg_mask].view(-1, self.reg_max + 1), target_ltrb[fg_mask]) * weight + loss_dfl = loss_dfl.sum() / target_scores_sum + else: + loss_dfl = torch.tensor(0.0).to(pred_dist.device) + + return loss_iou, loss_dfl + + +class KeypointLoss(nn.Module): + """Criterion class for computing training losses.""" + + def __init__(self, sigmas) -> None: + """Initialize the KeypointLoss class.""" + super().__init__() + self.sigmas = sigmas + + def forward(self, pred_kpts, gt_kpts, kpt_mask, area): + """Calculates keypoint loss factor and Euclidean distance loss for predicted and actual keypoints.""" + d = (pred_kpts[..., 0] - gt_kpts[..., 0]).pow(2) + (pred_kpts[..., 1] - gt_kpts[..., 1]).pow(2) + kpt_loss_factor = kpt_mask.shape[1] / (torch.sum(kpt_mask != 0, dim=1) + 1e-9) + # e = d / (2 * (area * self.sigmas) ** 2 + 1e-9) # from formula + e = d / ((2 * self.sigmas).pow(2) * (area + 1e-9) * 2) # from cocoeval + return (kpt_loss_factor.view(-1, 1) * ((1 - torch.exp(-e)) * kpt_mask)).mean() + + +class v8DetectionLoss: + """Criterion class for computing training losses.""" + + def __init__(self, model, tal_topk=10): # model must be de-paralleled + """Initializes v8DetectionLoss with the model, defining model-related properties and BCE loss function.""" + device = next(model.parameters()).device # get model device + h = model.args # hyperparameters + + m = model.model[-1] # Detect() module + self.bce = nn.BCEWithLogitsLoss(reduction="none") + self.hyp = h + self.stride = m.stride # model strides + self.nc = m.nc # number of classes + self.no = m.no + self.reg_max = m.reg_max + self.device = device + + self.use_dfl = m.reg_max > 1 + + self.assigner = TaskAlignedAssigner(topk=tal_topk, num_classes=self.nc, alpha=0.5, beta=6.0) + self.bbox_loss = BboxLoss(m.reg_max - 1, use_dfl=self.use_dfl).to(device) + self.proj = torch.arange(m.reg_max, dtype=torch.float, device=device) + + def preprocess(self, targets, batch_size, scale_tensor): + """Preprocesses the target counts and matches with the input batch size to output a tensor.""" + if targets.shape[0] == 0: + out = torch.zeros(batch_size, 0, 5, device=self.device) + else: + i = targets[:, 0] # image index + _, counts = i.unique(return_counts=True) + counts = counts.to(dtype=torch.int32) + out = torch.zeros(batch_size, counts.max(), 5, device=self.device) + for j in range(batch_size): + matches = i == j + n = matches.sum() + if n: + out[j, :n] = targets[matches, 1:] + out[..., 1:5] = xywh2xyxy(out[..., 1:5].mul_(scale_tensor)) + return out + + def bbox_decode(self, anchor_points, pred_dist): + """Decode predicted object bounding box coordinates from anchor points and distribution.""" + if self.use_dfl: + b, a, c = pred_dist.shape # batch, anchors, channels + pred_dist = pred_dist.view(b, a, 4, c // 4).softmax(3).matmul(self.proj.type(pred_dist.dtype)) + # pred_dist = pred_dist.view(b, a, c // 4, 4).transpose(2,3).softmax(3).matmul(self.proj.type(pred_dist.dtype)) + # pred_dist = (pred_dist.view(b, a, c // 4, 4).softmax(2) * self.proj.type(pred_dist.dtype).view(1, 1, -1, 1)).sum(2) + return dist2bbox(pred_dist, anchor_points, xywh=False) + + def __call__(self, preds, batch): + """Calculate the sum of the loss for box, cls and dfl multiplied by batch size.""" + loss = torch.zeros(3, device=self.device) # box, cls, dfl + feats = preds[1] if isinstance(preds, tuple) else preds + pred_distri, pred_scores = torch.cat([xi.view(feats[0].shape[0], self.no, -1) for xi in feats], 2).split( + (self.reg_max * 4, self.nc), 1 + ) + + pred_scores = pred_scores.permute(0, 2, 1).contiguous() + pred_distri = pred_distri.permute(0, 2, 1).contiguous() + + dtype = pred_scores.dtype + batch_size = pred_scores.shape[0] + imgsz = torch.tensor(feats[0].shape[2:], device=self.device, dtype=dtype) * self.stride[0] # image size (h,w) + anchor_points, stride_tensor = make_anchors(feats, self.stride, 0.5) + + # Targets + targets = torch.cat((batch["batch_idx"].view(-1, 1), batch["cls"].view(-1, 1), batch["bboxes"]), 1) + targets = self.preprocess(targets.to(self.device), batch_size, scale_tensor=imgsz[[1, 0, 1, 0]]) + gt_labels, gt_bboxes = targets.split((1, 4), 2) # cls, xyxy + mask_gt = gt_bboxes.sum(2, keepdim=True).gt_(0) + + # Pboxes + pred_bboxes = self.bbox_decode(anchor_points, pred_distri) # xyxy, (b, h*w, 4) + + _, target_bboxes, target_scores, fg_mask, _ = self.assigner( + pred_scores.detach().sigmoid(), + (pred_bboxes.detach() * stride_tensor).type(gt_bboxes.dtype), + anchor_points * stride_tensor, + gt_labels, + gt_bboxes, + mask_gt, + ) + + target_scores_sum = max(target_scores.sum(), 1) + + # Cls loss + # loss[1] = self.varifocal_loss(pred_scores, target_scores, target_labels) / target_scores_sum # VFL way + loss[1] = self.bce(pred_scores, target_scores.to(dtype)).sum() / target_scores_sum # BCE + + # Bbox loss + if fg_mask.sum(): + target_bboxes /= stride_tensor + loss[0], loss[2] = self.bbox_loss( + pred_distri, pred_bboxes, anchor_points, target_bboxes, target_scores, target_scores_sum, fg_mask + ) + + loss[0] *= self.hyp.box # box gain + loss[1] *= self.hyp.cls # cls gain + loss[2] *= self.hyp.dfl # dfl gain + + return loss.sum() * batch_size, loss.detach() # loss(box, cls, dfl) + + +class v8SegmentationLoss(v8DetectionLoss): + """Criterion class for computing training losses.""" + + def __init__(self, model): # model must be de-paralleled + """Initializes the v8SegmentationLoss class, taking a de-paralleled model as argument.""" + super().__init__(model) + self.overlap = model.args.overlap_mask + + def __call__(self, preds, batch): + """Calculate and return the loss for the YOLO model.""" + loss = torch.zeros(4, device=self.device) # box, cls, dfl + feats, pred_masks, proto = preds if len(preds) == 3 else preds[1] + batch_size, _, mask_h, mask_w = proto.shape # batch size, number of masks, mask height, mask width + pred_distri, pred_scores = torch.cat([xi.view(feats[0].shape[0], self.no, -1) for xi in feats], 2).split( + (self.reg_max * 4, self.nc), 1 + ) + + # B, grids, .. + pred_scores = pred_scores.permute(0, 2, 1).contiguous() + pred_distri = pred_distri.permute(0, 2, 1).contiguous() + pred_masks = pred_masks.permute(0, 2, 1).contiguous() + + dtype = pred_scores.dtype + imgsz = torch.tensor(feats[0].shape[2:], device=self.device, dtype=dtype) * self.stride[0] # image size (h,w) + anchor_points, stride_tensor = make_anchors(feats, self.stride, 0.5) + + # Targets + try: + batch_idx = batch["batch_idx"].view(-1, 1) + targets = torch.cat((batch_idx, batch["cls"].view(-1, 1), batch["bboxes"]), 1) + targets = self.preprocess(targets.to(self.device), batch_size, scale_tensor=imgsz[[1, 0, 1, 0]]) + gt_labels, gt_bboxes = targets.split((1, 4), 2) # cls, xyxy + mask_gt = gt_bboxes.sum(2, keepdim=True).gt_(0) + except RuntimeError as e: + raise TypeError( + "ERROR ❌ segment dataset incorrectly formatted or not a segment dataset.\n" + "This error can occur when incorrectly training a 'segment' model on a 'detect' dataset, " + "i.e. 'yolo train model=yolov8n-seg.pt data=coco8.yaml'.\nVerify your dataset is a " + "correctly formatted 'segment' dataset using 'data=coco8-seg.yaml' " + "as an example.\nSee https://docs.ultralytics.com/datasets/segment/ for help." + ) from e + + # Pboxes + pred_bboxes = self.bbox_decode(anchor_points, pred_distri) # xyxy, (b, h*w, 4) + + _, target_bboxes, target_scores, fg_mask, target_gt_idx = self.assigner( + pred_scores.detach().sigmoid(), + (pred_bboxes.detach() * stride_tensor).type(gt_bboxes.dtype), + anchor_points * stride_tensor, + gt_labels, + gt_bboxes, + mask_gt, + ) + + target_scores_sum = max(target_scores.sum(), 1) + + # Cls loss + # loss[1] = self.varifocal_loss(pred_scores, target_scores, target_labels) / target_scores_sum # VFL way + loss[2] = self.bce(pred_scores, target_scores.to(dtype)).sum() / target_scores_sum # BCE + + if fg_mask.sum(): + # Bbox loss + loss[0], loss[3] = self.bbox_loss( + pred_distri, + pred_bboxes, + anchor_points, + target_bboxes / stride_tensor, + target_scores, + target_scores_sum, + fg_mask, + ) + # Masks loss + masks = batch["masks"].to(self.device).float() + if tuple(masks.shape[-2:]) != (mask_h, mask_w): # downsample + masks = F.interpolate(masks[None], (mask_h, mask_w), mode="nearest")[0] + + loss[1] = self.calculate_segmentation_loss( + fg_mask, masks, target_gt_idx, target_bboxes, batch_idx, proto, pred_masks, imgsz, self.overlap + ) + + # WARNING: lines below prevent Multi-GPU DDP 'unused gradient' PyTorch errors, do not remove + else: + loss[1] += (proto * 0).sum() + (pred_masks * 0).sum() # inf sums may lead to nan loss + + loss[0] *= self.hyp.box # box gain + loss[1] *= self.hyp.box # seg gain + loss[2] *= self.hyp.cls # cls gain + loss[3] *= self.hyp.dfl # dfl gain + + return loss.sum() * batch_size, loss.detach() # loss(box, cls, dfl) + + @staticmethod + def single_mask_loss( + gt_mask: torch.Tensor, pred: torch.Tensor, proto: torch.Tensor, xyxy: torch.Tensor, area: torch.Tensor + ) -> torch.Tensor: + """ + Compute the instance segmentation loss for a single image. + + Args: + gt_mask (torch.Tensor): Ground truth mask of shape (n, H, W), where n is the number of objects. + pred (torch.Tensor): Predicted mask coefficients of shape (n, 32). + proto (torch.Tensor): Prototype masks of shape (32, H, W). + xyxy (torch.Tensor): Ground truth bounding boxes in xyxy format, normalized to [0, 1], of shape (n, 4). + area (torch.Tensor): Area of each ground truth bounding box of shape (n,). + + Returns: + (torch.Tensor): The calculated mask loss for a single image. + + Notes: + The function uses the equation pred_mask = torch.einsum('in,nhw->ihw', pred, proto) to produce the + predicted masks from the prototype masks and predicted mask coefficients. + """ + pred_mask = torch.einsum("in,nhw->ihw", pred, proto) # (n, 32) @ (32, 80, 80) -> (n, 80, 80) + loss = F.binary_cross_entropy_with_logits(pred_mask, gt_mask, reduction="none") + return (crop_mask(loss, xyxy).mean(dim=(1, 2)) / area).sum() + + def calculate_segmentation_loss( + self, + fg_mask: torch.Tensor, + masks: torch.Tensor, + target_gt_idx: torch.Tensor, + target_bboxes: torch.Tensor, + batch_idx: torch.Tensor, + proto: torch.Tensor, + pred_masks: torch.Tensor, + imgsz: torch.Tensor, + overlap: bool, + ) -> torch.Tensor: + """ + Calculate the loss for instance segmentation. + + Args: + fg_mask (torch.Tensor): A binary tensor of shape (BS, N_anchors) indicating which anchors are positive. + masks (torch.Tensor): Ground truth masks of shape (BS, H, W) if `overlap` is False, otherwise (BS, ?, H, W). + target_gt_idx (torch.Tensor): Indexes of ground truth objects for each anchor of shape (BS, N_anchors). + target_bboxes (torch.Tensor): Ground truth bounding boxes for each anchor of shape (BS, N_anchors, 4). + batch_idx (torch.Tensor): Batch indices of shape (N_labels_in_batch, 1). + proto (torch.Tensor): Prototype masks of shape (BS, 32, H, W). + pred_masks (torch.Tensor): Predicted masks for each anchor of shape (BS, N_anchors, 32). + imgsz (torch.Tensor): Size of the input image as a tensor of shape (2), i.e., (H, W). + overlap (bool): Whether the masks in `masks` tensor overlap. + + Returns: + (torch.Tensor): The calculated loss for instance segmentation. + + Notes: + The batch loss can be computed for improved speed at higher memory usage. + For example, pred_mask can be computed as follows: + pred_mask = torch.einsum('in,nhw->ihw', pred, proto) # (i, 32) @ (32, 160, 160) -> (i, 160, 160) + """ + _, _, mask_h, mask_w = proto.shape + loss = 0 + + # Normalize to 0-1 + target_bboxes_normalized = target_bboxes / imgsz[[1, 0, 1, 0]] + + # Areas of target bboxes + marea = xyxy2xywh(target_bboxes_normalized)[..., 2:].prod(2) + + # Normalize to mask size + mxyxy = target_bboxes_normalized * torch.tensor([mask_w, mask_h, mask_w, mask_h], device=proto.device) + + for i, single_i in enumerate(zip(fg_mask, target_gt_idx, pred_masks, proto, mxyxy, marea, masks)): + fg_mask_i, target_gt_idx_i, pred_masks_i, proto_i, mxyxy_i, marea_i, masks_i = single_i + if fg_mask_i.any(): + mask_idx = target_gt_idx_i[fg_mask_i] + if overlap: + gt_mask = masks_i == (mask_idx + 1).view(-1, 1, 1) + gt_mask = gt_mask.float() + else: + gt_mask = masks[batch_idx.view(-1) == i][mask_idx] + + loss += self.single_mask_loss( + gt_mask, pred_masks_i[fg_mask_i], proto_i, mxyxy_i[fg_mask_i], marea_i[fg_mask_i] + ) + + # WARNING: lines below prevents Multi-GPU DDP 'unused gradient' PyTorch errors, do not remove + else: + loss += (proto * 0).sum() + (pred_masks * 0).sum() # inf sums may lead to nan loss + + return loss / fg_mask.sum() + + +class v8PoseLoss(v8DetectionLoss): + """Criterion class for computing training losses.""" + + def __init__(self, model): # model must be de-paralleled + """Initializes v8PoseLoss with model, sets keypoint variables and declares a keypoint loss instance.""" + super().__init__(model) + self.kpt_shape = model.model[-1].kpt_shape + self.bce_pose = nn.BCEWithLogitsLoss() + is_pose = self.kpt_shape == [17, 3] + nkpt = self.kpt_shape[0] # number of keypoints + sigmas = torch.from_numpy(OKS_SIGMA).to(self.device) if is_pose else torch.ones(nkpt, device=self.device) / nkpt + self.keypoint_loss = KeypointLoss(sigmas=sigmas) + + def __call__(self, preds, batch): + """Calculate the total loss and detach it.""" + loss = torch.zeros(5, device=self.device) # box, cls, dfl, kpt_location, kpt_visibility + feats, pred_kpts = preds if isinstance(preds[0], list) else preds[1] + pred_distri, pred_scores = torch.cat([xi.view(feats[0].shape[0], self.no, -1) for xi in feats], 2).split( + (self.reg_max * 4, self.nc), 1 + ) + + # B, grids, .. + pred_scores = pred_scores.permute(0, 2, 1).contiguous() + pred_distri = pred_distri.permute(0, 2, 1).contiguous() + pred_kpts = pred_kpts.permute(0, 2, 1).contiguous() + + dtype = pred_scores.dtype + imgsz = torch.tensor(feats[0].shape[2:], device=self.device, dtype=dtype) * self.stride[0] # image size (h,w) + anchor_points, stride_tensor = make_anchors(feats, self.stride, 0.5) + + # Targets + batch_size = pred_scores.shape[0] + batch_idx = batch["batch_idx"].view(-1, 1) + targets = torch.cat((batch_idx, batch["cls"].view(-1, 1), batch["bboxes"]), 1) + targets = self.preprocess(targets.to(self.device), batch_size, scale_tensor=imgsz[[1, 0, 1, 0]]) + gt_labels, gt_bboxes = targets.split((1, 4), 2) # cls, xyxy + mask_gt = gt_bboxes.sum(2, keepdim=True).gt_(0) + + # Pboxes + pred_bboxes = self.bbox_decode(anchor_points, pred_distri) # xyxy, (b, h*w, 4) + pred_kpts = self.kpts_decode(anchor_points, pred_kpts.view(batch_size, -1, *self.kpt_shape)) # (b, h*w, 17, 3) + + _, target_bboxes, target_scores, fg_mask, target_gt_idx = self.assigner( + pred_scores.detach().sigmoid(), + (pred_bboxes.detach() * stride_tensor).type(gt_bboxes.dtype), + anchor_points * stride_tensor, + gt_labels, + gt_bboxes, + mask_gt, + ) + + target_scores_sum = max(target_scores.sum(), 1) + + # Cls loss + # loss[1] = self.varifocal_loss(pred_scores, target_scores, target_labels) / target_scores_sum # VFL way + loss[3] = self.bce(pred_scores, target_scores.to(dtype)).sum() / target_scores_sum # BCE + + # Bbox loss + if fg_mask.sum(): + target_bboxes /= stride_tensor + loss[0], loss[4] = self.bbox_loss( + pred_distri, pred_bboxes, anchor_points, target_bboxes, target_scores, target_scores_sum, fg_mask + ) + keypoints = batch["keypoints"].to(self.device).float().clone() + keypoints[..., 0] *= imgsz[1] + keypoints[..., 1] *= imgsz[0] + + loss[1], loss[2] = self.calculate_keypoints_loss( + fg_mask, target_gt_idx, keypoints, batch_idx, stride_tensor, target_bboxes, pred_kpts + ) + + loss[0] *= self.hyp.box # box gain + loss[1] *= self.hyp.pose # pose gain + loss[2] *= self.hyp.kobj # kobj gain + loss[3] *= self.hyp.cls # cls gain + loss[4] *= self.hyp.dfl # dfl gain + + return loss.sum() * batch_size, loss.detach() # loss(box, cls, dfl) + + @staticmethod + def kpts_decode(anchor_points, pred_kpts): + """Decodes predicted keypoints to image coordinates.""" + y = pred_kpts.clone() + y[..., :2] *= 2.0 + y[..., 0] += anchor_points[:, [0]] - 0.5 + y[..., 1] += anchor_points[:, [1]] - 0.5 + return y + + def calculate_keypoints_loss( + self, masks, target_gt_idx, keypoints, batch_idx, stride_tensor, target_bboxes, pred_kpts + ): + """ + Calculate the keypoints loss for the model. + + This function calculates the keypoints loss and keypoints object loss for a given batch. The keypoints loss is + based on the difference between the predicted keypoints and ground truth keypoints. The keypoints object loss is + a binary classification loss that classifies whether a keypoint is present or not. + + Args: + masks (torch.Tensor): Binary mask tensor indicating object presence, shape (BS, N_anchors). + target_gt_idx (torch.Tensor): Index tensor mapping anchors to ground truth objects, shape (BS, N_anchors). + keypoints (torch.Tensor): Ground truth keypoints, shape (N_kpts_in_batch, N_kpts_per_object, kpts_dim). + batch_idx (torch.Tensor): Batch index tensor for keypoints, shape (N_kpts_in_batch, 1). + stride_tensor (torch.Tensor): Stride tensor for anchors, shape (N_anchors, 1). + target_bboxes (torch.Tensor): Ground truth boxes in (x1, y1, x2, y2) format, shape (BS, N_anchors, 4). + pred_kpts (torch.Tensor): Predicted keypoints, shape (BS, N_anchors, N_kpts_per_object, kpts_dim). + + Returns: + (tuple): Returns a tuple containing: + - kpts_loss (torch.Tensor): The keypoints loss. + - kpts_obj_loss (torch.Tensor): The keypoints object loss. + """ + batch_idx = batch_idx.flatten() + batch_size = len(masks) + + # Find the maximum number of keypoints in a single image + max_kpts = torch.unique(batch_idx, return_counts=True)[1].max() + + # Create a tensor to hold batched keypoints + batched_keypoints = torch.zeros( + (batch_size, max_kpts, keypoints.shape[1], keypoints.shape[2]), device=keypoints.device + ) + + # TODO: any idea how to vectorize this? + # Fill batched_keypoints with keypoints based on batch_idx + for i in range(batch_size): + keypoints_i = keypoints[batch_idx == i] + batched_keypoints[i, : keypoints_i.shape[0]] = keypoints_i + + # Expand dimensions of target_gt_idx to match the shape of batched_keypoints + target_gt_idx_expanded = target_gt_idx.unsqueeze(-1).unsqueeze(-1) + + # Use target_gt_idx_expanded to select keypoints from batched_keypoints + selected_keypoints = batched_keypoints.gather( + 1, target_gt_idx_expanded.expand(-1, -1, keypoints.shape[1], keypoints.shape[2]) + ) + + # Divide coordinates by stride + selected_keypoints /= stride_tensor.view(1, -1, 1, 1) + + kpts_loss = 0 + kpts_obj_loss = 0 + + if masks.any(): + gt_kpt = selected_keypoints[masks] + area = xyxy2xywh(target_bboxes[masks])[:, 2:].prod(1, keepdim=True) + pred_kpt = pred_kpts[masks] + kpt_mask = gt_kpt[..., 2] != 0 if gt_kpt.shape[-1] == 3 else torch.full_like(gt_kpt[..., 0], True) + kpts_loss = self.keypoint_loss(pred_kpt, gt_kpt, kpt_mask, area) # pose loss + + if pred_kpt.shape[-1] == 3: + kpts_obj_loss = self.bce_pose(pred_kpt[..., 2], kpt_mask.float()) # keypoint obj loss + + return kpts_loss, kpts_obj_loss + + +class v8ClassificationLoss: + """Criterion class for computing training losses.""" + + def __call__(self, preds, batch): + """Compute the classification loss between predictions and true labels.""" + loss = torch.nn.functional.cross_entropy(preds, batch["cls"], reduction="mean") + loss_items = loss.detach() + return loss, loss_items + + +class v8OBBLoss(v8DetectionLoss): + def __init__(self, model): + """ + Initializes v8OBBLoss with model, assigner, and rotated bbox loss. + + Note model must be de-paralleled. + """ + super().__init__(model) + self.assigner = RotatedTaskAlignedAssigner(topk=10, num_classes=self.nc, alpha=0.5, beta=6.0) + self.bbox_loss = RotatedBboxLoss(self.reg_max - 1, use_dfl=self.use_dfl).to(self.device) + + def preprocess(self, targets, batch_size, scale_tensor): + """Preprocesses the target counts and matches with the input batch size to output a tensor.""" + if targets.shape[0] == 0: + out = torch.zeros(batch_size, 0, 6, device=self.device) + else: + i = targets[:, 0] # image index + _, counts = i.unique(return_counts=True) + counts = counts.to(dtype=torch.int32) + out = torch.zeros(batch_size, counts.max(), 6, device=self.device) + for j in range(batch_size): + matches = i == j + n = matches.sum() + if n: + bboxes = targets[matches, 2:] + bboxes[..., :4].mul_(scale_tensor) + out[j, :n] = torch.cat([targets[matches, 1:2], bboxes], dim=-1) + return out + + def __call__(self, preds, batch): + """Calculate and return the loss for the YOLO model.""" + loss = torch.zeros(3, device=self.device) # box, cls, dfl + feats, pred_angle = preds if isinstance(preds[0], list) else preds[1] + batch_size = pred_angle.shape[0] # batch size, number of masks, mask height, mask width + pred_distri, pred_scores = torch.cat([xi.view(feats[0].shape[0], self.no, -1) for xi in feats], 2).split( + (self.reg_max * 4, self.nc), 1 + ) + + # b, grids, .. + pred_scores = pred_scores.permute(0, 2, 1).contiguous() + pred_distri = pred_distri.permute(0, 2, 1).contiguous() + pred_angle = pred_angle.permute(0, 2, 1).contiguous() + + dtype = pred_scores.dtype + imgsz = torch.tensor(feats[0].shape[2:], device=self.device, dtype=dtype) * self.stride[0] # image size (h,w) + anchor_points, stride_tensor = make_anchors(feats, self.stride, 0.5) + + # targets + try: + batch_idx = batch["batch_idx"].view(-1, 1) + targets = torch.cat((batch_idx, batch["cls"].view(-1, 1), batch["bboxes"].view(-1, 5)), 1) + rw, rh = targets[:, 4] * imgsz[0].item(), targets[:, 5] * imgsz[1].item() + targets = targets[(rw >= 2) & (rh >= 2)] # filter rboxes of tiny size to stabilize training + targets = self.preprocess(targets.to(self.device), batch_size, scale_tensor=imgsz[[1, 0, 1, 0]]) + gt_labels, gt_bboxes = targets.split((1, 5), 2) # cls, xywhr + mask_gt = gt_bboxes.sum(2, keepdim=True).gt_(0) + except RuntimeError as e: + raise TypeError( + "ERROR ❌ OBB dataset incorrectly formatted or not a OBB dataset.\n" + "This error can occur when incorrectly training a 'OBB' model on a 'detect' dataset, " + "i.e. 'yolo train model=yolov8n-obb.pt data=dota8.yaml'.\nVerify your dataset is a " + "correctly formatted 'OBB' dataset using 'data=dota8.yaml' " + "as an example.\nSee https://docs.ultralytics.com/datasets/obb/ for help." + ) from e + + # Pboxes + pred_bboxes = self.bbox_decode(anchor_points, pred_distri, pred_angle) # xyxy, (b, h*w, 4) + + bboxes_for_assigner = pred_bboxes.clone().detach() + # Only the first four elements need to be scaled + bboxes_for_assigner[..., :4] *= stride_tensor + _, target_bboxes, target_scores, fg_mask, _ = self.assigner( + pred_scores.detach().sigmoid(), + bboxes_for_assigner.type(gt_bboxes.dtype), + anchor_points * stride_tensor, + gt_labels, + gt_bboxes, + mask_gt, + ) + + target_scores_sum = max(target_scores.sum(), 1) + + # Cls loss + # loss[1] = self.varifocal_loss(pred_scores, target_scores, target_labels) / target_scores_sum # VFL way + loss[1] = self.bce(pred_scores, target_scores.to(dtype)).sum() / target_scores_sum # BCE + + # Bbox loss + if fg_mask.sum(): + target_bboxes[..., :4] /= stride_tensor + loss[0], loss[2] = self.bbox_loss( + pred_distri, pred_bboxes, anchor_points, target_bboxes, target_scores, target_scores_sum, fg_mask + ) + else: + loss[0] += (pred_angle * 0).sum() + + loss[0] *= self.hyp.box # box gain + loss[1] *= self.hyp.cls # cls gain + loss[2] *= self.hyp.dfl # dfl gain + + return loss.sum() * batch_size, loss.detach() # loss(box, cls, dfl) + + def bbox_decode(self, anchor_points, pred_dist, pred_angle): + """ + Decode predicted object bounding box coordinates from anchor points and distribution. + + Args: + anchor_points (torch.Tensor): Anchor points, (h*w, 2). + pred_dist (torch.Tensor): Predicted rotated distance, (bs, h*w, 4). + pred_angle (torch.Tensor): Predicted angle, (bs, h*w, 1). + + Returns: + (torch.Tensor): Predicted rotated bounding boxes with angles, (bs, h*w, 5). + """ + if self.use_dfl: + b, a, c = pred_dist.shape # batch, anchors, channels + pred_dist = pred_dist.view(b, a, 4, c // 4).softmax(3).matmul(self.proj.type(pred_dist.dtype)) + return torch.cat((dist2rbox(pred_dist, pred_angle, anchor_points), pred_angle), dim=-1) + +class v10DetectLoss: + def __init__(self, model): + self.one2many = v8DetectionLoss(model, tal_topk=10) + self.one2one = v8DetectionLoss(model, tal_topk=1) + + def __call__(self, preds, batch): + one2many = preds["one2many"] + loss_one2many = self.one2many(one2many, batch) + one2one = preds["one2one"] + loss_one2one = self.one2one(one2one, batch) + return loss_one2many[0] + loss_one2one[0], torch.cat((loss_one2many[1], loss_one2one[1])) diff --git a/ultralytics/utils/metrics.py b/ultralytics/utils/metrics.py new file mode 100644 index 0000000..b598811 --- /dev/null +++ b/ultralytics/utils/metrics.py @@ -0,0 +1,1292 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +"""Model validation metrics.""" + +import math +import warnings +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np +import torch + +from ultralytics.utils import LOGGER, SimpleClass, TryExcept, plt_settings + +OKS_SIGMA = ( + np.array([0.26, 0.25, 0.25, 0.35, 0.35, 0.79, 0.79, 0.72, 0.72, 0.62, 0.62, 1.07, 1.07, 0.87, 0.87, 0.89, 0.89]) + / 10.0 +) + + +def bbox_ioa(box1, box2, iou=False, eps=1e-7): + """ + Calculate the intersection over box2 area given box1 and box2. Boxes are in x1y1x2y2 format. + + Args: + box1 (np.ndarray): A numpy array of shape (n, 4) representing n bounding boxes. + box2 (np.ndarray): A numpy array of shape (m, 4) representing m bounding boxes. + iou (bool): Calculate the standard IoU if True else return inter_area/box2_area. + eps (float, optional): A small value to avoid division by zero. Defaults to 1e-7. + + Returns: + (np.ndarray): A numpy array of shape (n, m) representing the intersection over box2 area. + """ + + # Get the coordinates of bounding boxes + b1_x1, b1_y1, b1_x2, b1_y2 = box1.T + b2_x1, b2_y1, b2_x2, b2_y2 = box2.T + + # Intersection area + inter_area = (np.minimum(b1_x2[:, None], b2_x2) - np.maximum(b1_x1[:, None], b2_x1)).clip(0) * ( + np.minimum(b1_y2[:, None], b2_y2) - np.maximum(b1_y1[:, None], b2_y1) + ).clip(0) + + # Box2 area + area = (b2_x2 - b2_x1) * (b2_y2 - b2_y1) + if iou: + box1_area = (b1_x2 - b1_x1) * (b1_y2 - b1_y1) + area = area + box1_area[:, None] - inter_area + + # Intersection over box2 area + return inter_area / (area + eps) + + +def box_iou(box1, box2, eps=1e-7): + """ + Calculate intersection-over-union (IoU) of boxes. Both sets of boxes are expected to be in (x1, y1, x2, y2) format. + Based on https://github.com/pytorch/vision/blob/master/torchvision/ops/boxes.py + + Args: + box1 (torch.Tensor): A tensor of shape (N, 4) representing N bounding boxes. + box2 (torch.Tensor): A tensor of shape (M, 4) representing M bounding boxes. + eps (float, optional): A small value to avoid division by zero. Defaults to 1e-7. + + Returns: + (torch.Tensor): An NxM tensor containing the pairwise IoU values for every element in box1 and box2. + """ + + # NOTE: need float32 to get accurate iou values + box1 = torch.as_tensor(box1, dtype=torch.float32) + box2 = torch.as_tensor(box2, dtype=torch.float32) + # inter(N,M) = (rb(N,M,2) - lt(N,M,2)).clamp(0).prod(2) + (a1, a2), (b1, b2) = box1.unsqueeze(1).chunk(2, 2), box2.unsqueeze(0).chunk(2, 2) + inter = (torch.min(a2, b2) - torch.max(a1, b1)).clamp_(0).prod(2) + + # IoU = inter / (area1 + area2 - inter) + return inter / ((a2 - a1).prod(2) + (b2 - b1).prod(2) - inter + eps) + + +def bbox_iou(box1, box2, xywh=True, GIoU=False, DIoU=False, CIoU=False, eps=1e-7): + """ + Calculate Intersection over Union (IoU) of box1(1, 4) to box2(n, 4). + + Args: + box1 (torch.Tensor): A tensor representing a single bounding box with shape (1, 4). + box2 (torch.Tensor): A tensor representing n bounding boxes with shape (n, 4). + xywh (bool, optional): If True, input boxes are in (x, y, w, h) format. If False, input boxes are in + (x1, y1, x2, y2) format. Defaults to True. + GIoU (bool, optional): If True, calculate Generalized IoU. Defaults to False. + DIoU (bool, optional): If True, calculate Distance IoU. Defaults to False. + CIoU (bool, optional): If True, calculate Complete IoU. Defaults to False. + eps (float, optional): A small value to avoid division by zero. Defaults to 1e-7. + + Returns: + (torch.Tensor): IoU, GIoU, DIoU, or CIoU values depending on the specified flags. + """ + + # Get the coordinates of bounding boxes + if xywh: # transform from xywh to xyxy + (x1, y1, w1, h1), (x2, y2, w2, h2) = box1.chunk(4, -1), box2.chunk(4, -1) + w1_, h1_, w2_, h2_ = w1 / 2, h1 / 2, w2 / 2, h2 / 2 + b1_x1, b1_x2, b1_y1, b1_y2 = x1 - w1_, x1 + w1_, y1 - h1_, y1 + h1_ + b2_x1, b2_x2, b2_y1, b2_y2 = x2 - w2_, x2 + w2_, y2 - h2_, y2 + h2_ + else: # x1, y1, x2, y2 = box1 + b1_x1, b1_y1, b1_x2, b1_y2 = box1.chunk(4, -1) + b2_x1, b2_y1, b2_x2, b2_y2 = box2.chunk(4, -1) + w1, h1 = b1_x2 - b1_x1, b1_y2 - b1_y1 + eps + w2, h2 = b2_x2 - b2_x1, b2_y2 - b2_y1 + eps + + # Intersection area + inter = (b1_x2.minimum(b2_x2) - b1_x1.maximum(b2_x1)).clamp_(0) * ( + b1_y2.minimum(b2_y2) - b1_y1.maximum(b2_y1) + ).clamp_(0) + + # Union Area + union = w1 * h1 + w2 * h2 - inter + eps + + # IoU + iou = inter / union + if CIoU or DIoU or GIoU: + cw = b1_x2.maximum(b2_x2) - b1_x1.minimum(b2_x1) # convex (smallest enclosing box) width + ch = b1_y2.maximum(b2_y2) - b1_y1.minimum(b2_y1) # convex height + if CIoU or DIoU: # Distance or Complete IoU https://arxiv.org/abs/1911.08287v1 + c2 = cw.pow(2) + ch.pow(2) + eps # convex diagonal squared + rho2 = ( + (b2_x1 + b2_x2 - b1_x1 - b1_x2).pow(2) + (b2_y1 + b2_y2 - b1_y1 - b1_y2).pow(2) + ) / 4 # center dist**2 + if CIoU: # https://github.com/Zzh-tju/DIoU-SSD-pytorch/blob/master/utils/box/box_utils.py#L47 + v = (4 / math.pi**2) * ((w2 / h2).atan() - (w1 / h1).atan()).pow(2) + with torch.no_grad(): + alpha = v / (v - iou + (1 + eps)) + return iou - (rho2 / c2 + v * alpha) # CIoU + return iou - rho2 / c2 # DIoU + c_area = cw * ch + eps # convex area + return iou - (c_area - union) / c_area # GIoU https://arxiv.org/pdf/1902.09630.pdf + return iou # IoU + + +def mask_iou(mask1, mask2, eps=1e-7): + """ + Calculate masks IoU. + + Args: + mask1 (torch.Tensor): A tensor of shape (N, n) where N is the number of ground truth objects and n is the + product of image width and height. + mask2 (torch.Tensor): A tensor of shape (M, n) where M is the number of predicted objects and n is the + product of image width and height. + eps (float, optional): A small value to avoid division by zero. Defaults to 1e-7. + + Returns: + (torch.Tensor): A tensor of shape (N, M) representing masks IoU. + """ + intersection = torch.matmul(mask1, mask2.T).clamp_(0) + union = (mask1.sum(1)[:, None] + mask2.sum(1)[None]) - intersection # (area1 + area2) - intersection + return intersection / (union + eps) + + +def kpt_iou(kpt1, kpt2, area, sigma, eps=1e-7): + """ + Calculate Object Keypoint Similarity (OKS). + + Args: + kpt1 (torch.Tensor): A tensor of shape (N, 17, 3) representing ground truth keypoints. + kpt2 (torch.Tensor): A tensor of shape (M, 17, 3) representing predicted keypoints. + area (torch.Tensor): A tensor of shape (N,) representing areas from ground truth. + sigma (list): A list containing 17 values representing keypoint scales. + eps (float, optional): A small value to avoid division by zero. Defaults to 1e-7. + + Returns: + (torch.Tensor): A tensor of shape (N, M) representing keypoint similarities. + """ + d = (kpt1[:, None, :, 0] - kpt2[..., 0]).pow(2) + (kpt1[:, None, :, 1] - kpt2[..., 1]).pow(2) # (N, M, 17) + sigma = torch.tensor(sigma, device=kpt1.device, dtype=kpt1.dtype) # (17, ) + kpt_mask = kpt1[..., 2] != 0 # (N, 17) + e = d / (2 * sigma).pow(2) / (area[:, None, None] + eps) / 2 # from cocoeval + # e = d / ((area[None, :, None] + eps) * sigma) ** 2 / 2 # from formula + return ((-e).exp() * kpt_mask[:, None]).sum(-1) / (kpt_mask.sum(-1)[:, None] + eps) + + +def _get_covariance_matrix(boxes): + """ + Generating covariance matrix from obbs. + + Args: + boxes (torch.Tensor): A tensor of shape (N, 5) representing rotated bounding boxes, with xywhr format. + + Returns: + (torch.Tensor): Covariance metrixs corresponding to original rotated bounding boxes. + """ + # Gaussian bounding boxes, ignore the center points (the first two columns) because they are not needed here. + gbbs = torch.cat((boxes[:, 2:4].pow(2) / 12, boxes[:, 4:]), dim=-1) + a, b, c = gbbs.split(1, dim=-1) + cos = c.cos() + sin = c.sin() + cos2 = cos.pow(2) + sin2 = sin.pow(2) + return a * cos2 + b * sin2, a * sin2 + b * cos2, (a - b) * cos * sin + + +def probiou(obb1, obb2, CIoU=False, eps=1e-7): + """ + Calculate the prob IoU between oriented bounding boxes, https://arxiv.org/pdf/2106.06072v1.pdf. + + Args: + obb1 (torch.Tensor): A tensor of shape (N, 5) representing ground truth obbs, with xywhr format. + obb2 (torch.Tensor): A tensor of shape (N, 5) representing predicted obbs, with xywhr format. + eps (float, optional): A small value to avoid division by zero. Defaults to 1e-7. + + Returns: + (torch.Tensor): A tensor of shape (N, ) representing obb similarities. + """ + x1, y1 = obb1[..., :2].split(1, dim=-1) + x2, y2 = obb2[..., :2].split(1, dim=-1) + a1, b1, c1 = _get_covariance_matrix(obb1) + a2, b2, c2 = _get_covariance_matrix(obb2) + + t1 = ( + ((a1 + a2) * (y1 - y2).pow(2) + (b1 + b2) * (x1 - x2).pow(2)) / ((a1 + a2) * (b1 + b2) - (c1 + c2).pow(2) + eps) + ) * 0.25 + t2 = (((c1 + c2) * (x2 - x1) * (y1 - y2)) / ((a1 + a2) * (b1 + b2) - (c1 + c2).pow(2) + eps)) * 0.5 + t3 = ( + ((a1 + a2) * (b1 + b2) - (c1 + c2).pow(2)) + / (4 * ((a1 * b1 - c1.pow(2)).clamp_(0) * (a2 * b2 - c2.pow(2)).clamp_(0)).sqrt() + eps) + + eps + ).log() * 0.5 + bd = (t1 + t2 + t3).clamp(eps, 100.0) + hd = (1.0 - (-bd).exp() + eps).sqrt() + iou = 1 - hd + if CIoU: # only include the wh aspect ratio part + w1, h1 = obb1[..., 2:4].split(1, dim=-1) + w2, h2 = obb2[..., 2:4].split(1, dim=-1) + v = (4 / math.pi**2) * ((w2 / h2).atan() - (w1 / h1).atan()).pow(2) + with torch.no_grad(): + alpha = v / (v - iou + (1 + eps)) + return iou - v * alpha # CIoU + return iou + + +def batch_probiou(obb1, obb2, eps=1e-7): + """ + Calculate the prob IoU between oriented bounding boxes, https://arxiv.org/pdf/2106.06072v1.pdf. + + Args: + obb1 (torch.Tensor | np.ndarray): A tensor of shape (N, 5) representing ground truth obbs, with xywhr format. + obb2 (torch.Tensor | np.ndarray): A tensor of shape (M, 5) representing predicted obbs, with xywhr format. + eps (float, optional): A small value to avoid division by zero. Defaults to 1e-7. + + Returns: + (torch.Tensor): A tensor of shape (N, M) representing obb similarities. + """ + obb1 = torch.from_numpy(obb1) if isinstance(obb1, np.ndarray) else obb1 + obb2 = torch.from_numpy(obb2) if isinstance(obb2, np.ndarray) else obb2 + + x1, y1 = obb1[..., :2].split(1, dim=-1) + x2, y2 = (x.squeeze(-1)[None] for x in obb2[..., :2].split(1, dim=-1)) + a1, b1, c1 = _get_covariance_matrix(obb1) + a2, b2, c2 = (x.squeeze(-1)[None] for x in _get_covariance_matrix(obb2)) + + t1 = ( + ((a1 + a2) * (y1 - y2).pow(2) + (b1 + b2) * (x1 - x2).pow(2)) / ((a1 + a2) * (b1 + b2) - (c1 + c2).pow(2) + eps) + ) * 0.25 + t2 = (((c1 + c2) * (x2 - x1) * (y1 - y2)) / ((a1 + a2) * (b1 + b2) - (c1 + c2).pow(2) + eps)) * 0.5 + t3 = ( + ((a1 + a2) * (b1 + b2) - (c1 + c2).pow(2)) + / (4 * ((a1 * b1 - c1.pow(2)).clamp_(0) * (a2 * b2 - c2.pow(2)).clamp_(0)).sqrt() + eps) + + eps + ).log() * 0.5 + bd = (t1 + t2 + t3).clamp(eps, 100.0) + hd = (1.0 - (-bd).exp() + eps).sqrt() + return 1 - hd + + +def smooth_BCE(eps=0.1): + """ + Computes smoothed positive and negative Binary Cross-Entropy targets. + + This function calculates positive and negative label smoothing BCE targets based on a given epsilon value. + For implementation details, refer to https://github.com/ultralytics/yolov3/issues/238#issuecomment-598028441. + + Args: + eps (float, optional): The epsilon value for label smoothing. Defaults to 0.1. + + Returns: + (tuple): A tuple containing the positive and negative label smoothing BCE targets. + """ + return 1.0 - 0.5 * eps, 0.5 * eps + + +class ConfusionMatrix: + """ + A class for calculating and updating a confusion matrix for object detection and classification tasks. + + Attributes: + task (str): The type of task, either 'detect' or 'classify'. + matrix (np.ndarray): The confusion matrix, with dimensions depending on the task. + nc (int): The number of classes. + conf (float): The confidence threshold for detections. + iou_thres (float): The Intersection over Union threshold. + """ + + def __init__(self, nc, conf=0.25, iou_thres=0.45, task="detect"): + """Initialize attributes for the YOLO model.""" + self.task = task + self.matrix = np.zeros((nc + 1, nc + 1)) if self.task == "detect" else np.zeros((nc, nc)) + self.nc = nc # number of classes + self.conf = 0.25 if conf in (None, 0.001) else conf # apply 0.25 if default val conf is passed + self.iou_thres = iou_thres + + def process_cls_preds(self, preds, targets): + """ + Update confusion matrix for classification task. + + Args: + preds (Array[N, min(nc,5)]): Predicted class labels. + targets (Array[N, 1]): Ground truth class labels. + """ + preds, targets = torch.cat(preds)[:, 0], torch.cat(targets) + for p, t in zip(preds.cpu().numpy(), targets.cpu().numpy()): + self.matrix[p][t] += 1 + + def process_batch(self, detections, gt_bboxes, gt_cls): + """ + Update confusion matrix for object detection task. + + Args: + detections (Array[N, 6] | Array[N, 7]): Detected bounding boxes and their associated information. + Each row should contain (x1, y1, x2, y2, conf, class) + or with an additional element `angle` when it's obb. + gt_bboxes (Array[M, 4]| Array[N, 5]): Ground truth bounding boxes with xyxy/xyxyr format. + gt_cls (Array[M]): The class labels. + """ + if gt_cls.shape[0] == 0: # Check if labels is empty + if detections is not None: + detections = detections[detections[:, 4] > self.conf] + detection_classes = detections[:, 5].int() + for dc in detection_classes: + self.matrix[dc, self.nc] += 1 # false positives + return + if detections is None: + gt_classes = gt_cls.int() + for gc in gt_classes: + self.matrix[self.nc, gc] += 1 # background FN + return + + detections = detections[detections[:, 4] > self.conf] + gt_classes = gt_cls.int() + detection_classes = detections[:, 5].int() + is_obb = detections.shape[1] == 7 and gt_bboxes.shape[1] == 5 # with additional `angle` dimension + iou = ( + batch_probiou(gt_bboxes, torch.cat([detections[:, :4], detections[:, -1:]], dim=-1)) + if is_obb + else box_iou(gt_bboxes, detections[:, :4]) + ) + + x = torch.where(iou > self.iou_thres) + if x[0].shape[0]: + matches = torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1).cpu().numpy() + if x[0].shape[0] > 1: + matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 1], return_index=True)[1]] + matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 0], return_index=True)[1]] + else: + matches = np.zeros((0, 3)) + + n = matches.shape[0] > 0 + m0, m1, _ = matches.transpose().astype(int) + for i, gc in enumerate(gt_classes): + j = m0 == i + if n and sum(j) == 1: + self.matrix[detection_classes[m1[j]], gc] += 1 # correct + else: + self.matrix[self.nc, gc] += 1 # true background + + if n: + for i, dc in enumerate(detection_classes): + if not any(m1 == i): + self.matrix[dc, self.nc] += 1 # predicted background + + def matrix(self): + """Returns the confusion matrix.""" + return self.matrix + + def tp_fp(self): + """Returns true positives and false positives.""" + tp = self.matrix.diagonal() # true positives + fp = self.matrix.sum(1) - tp # false positives + # fn = self.matrix.sum(0) - tp # false negatives (missed detections) + return (tp[:-1], fp[:-1]) if self.task == "detect" else (tp, fp) # remove background class if task=detect + + @TryExcept("WARNING ⚠️ ConfusionMatrix plot failure") + @plt_settings() + def plot(self, normalize=True, save_dir="", names=(), on_plot=None): + """ + Plot the confusion matrix using seaborn and save it to a file. + + Args: + normalize (bool): Whether to normalize the confusion matrix. + save_dir (str): Directory where the plot will be saved. + names (tuple): Names of classes, used as labels on the plot. + on_plot (func): An optional callback to pass plots path and data when they are rendered. + """ + import seaborn as sn + + array = self.matrix / ((self.matrix.sum(0).reshape(1, -1) + 1e-9) if normalize else 1) # normalize columns + array[array < 0.005] = np.nan # don't annotate (would appear as 0.00) + + fig, ax = plt.subplots(1, 1, figsize=(12, 9), tight_layout=True) + nc, nn = self.nc, len(names) # number of classes, names + sn.set(font_scale=1.0 if nc < 50 else 0.8) # for label size + labels = (0 < nn < 99) and (nn == nc) # apply names to ticklabels + ticklabels = (list(names) + ["background"]) if labels else "auto" + with warnings.catch_warnings(): + warnings.simplefilter("ignore") # suppress empty matrix RuntimeWarning: All-NaN slice encountered + sn.heatmap( + array, + ax=ax, + annot=nc < 30, + annot_kws={"size": 8}, + cmap="Blues", + fmt=".2f" if normalize else ".0f", + square=True, + vmin=0.0, + xticklabels=ticklabels, + yticklabels=ticklabels, + ).set_facecolor((1, 1, 1)) + title = "Confusion Matrix" + " Normalized" * normalize + ax.set_xlabel("True") + ax.set_ylabel("Predicted") + ax.set_title(title) + plot_fname = Path(save_dir) / f'{title.lower().replace(" ", "_")}.png' + fig.savefig(plot_fname, dpi=250) + plt.close(fig) + if on_plot: + on_plot(plot_fname) + + def print(self): + """Print the confusion matrix to the console.""" + for i in range(self.nc + 1): + LOGGER.info(" ".join(map(str, self.matrix[i]))) + + +def smooth(y, f=0.05): + """Box filter of fraction f.""" + nf = round(len(y) * f * 2) // 2 + 1 # number of filter elements (must be odd) + p = np.ones(nf // 2) # ones padding + yp = np.concatenate((p * y[0], y, p * y[-1]), 0) # y padded + return np.convolve(yp, np.ones(nf) / nf, mode="valid") # y-smoothed + + +@plt_settings() +def plot_pr_curve(px, py, ap, save_dir=Path("pr_curve.png"), names=(), on_plot=None): + """Plots a precision-recall curve.""" + fig, ax = plt.subplots(1, 1, figsize=(9, 6), tight_layout=True) + py = np.stack(py, axis=1) + + if 0 < len(names) < 21: # display per-class legend if < 21 classes + for i, y in enumerate(py.T): + ax.plot(px, y, linewidth=1, label=f"{names[i]} {ap[i, 0]:.3f}") # plot(recall, precision) + else: + ax.plot(px, py, linewidth=1, color="grey") # plot(recall, precision) + + ax.plot(px, py.mean(1), linewidth=3, color="blue", label="all classes %.3f mAP@0.5" % ap[:, 0].mean()) + ax.set_xlabel("Recall") + ax.set_ylabel("Precision") + ax.set_xlim(0, 1) + ax.set_ylim(0, 1) + ax.legend(bbox_to_anchor=(1.04, 1), loc="upper left") + ax.set_title("Precision-Recall Curve") + fig.savefig(save_dir, dpi=250) + plt.close(fig) + if on_plot: + on_plot(save_dir) + + +@plt_settings() +def plot_mc_curve(px, py, save_dir=Path("mc_curve.png"), names=(), xlabel="Confidence", ylabel="Metric", on_plot=None): + """Plots a metric-confidence curve.""" + fig, ax = plt.subplots(1, 1, figsize=(9, 6), tight_layout=True) + + if 0 < len(names) < 21: # display per-class legend if < 21 classes + for i, y in enumerate(py): + ax.plot(px, y, linewidth=1, label=f"{names[i]}") # plot(confidence, metric) + else: + ax.plot(px, py.T, linewidth=1, color="grey") # plot(confidence, metric) + + y = smooth(py.mean(0), 0.05) + ax.plot(px, y, linewidth=3, color="blue", label=f"all classes {y.max():.2f} at {px[y.argmax()]:.3f}") + ax.set_xlabel(xlabel) + ax.set_ylabel(ylabel) + ax.set_xlim(0, 1) + ax.set_ylim(0, 1) + ax.legend(bbox_to_anchor=(1.04, 1), loc="upper left") + ax.set_title(f"{ylabel}-Confidence Curve") + fig.savefig(save_dir, dpi=250) + plt.close(fig) + if on_plot: + on_plot(save_dir) + + +def compute_ap(recall, precision): + """ + Compute the average precision (AP) given the recall and precision curves. + + Args: + recall (list): The recall curve. + precision (list): The precision curve. + + Returns: + (float): Average precision. + (np.ndarray): Precision envelope curve. + (np.ndarray): Modified recall curve with sentinel values added at the beginning and end. + """ + + # Append sentinel values to beginning and end + mrec = np.concatenate(([0.0], recall, [1.0])) + mpre = np.concatenate(([1.0], precision, [0.0])) + + # Compute the precision envelope + mpre = np.flip(np.maximum.accumulate(np.flip(mpre))) + + # Integrate area under curve + method = "interp" # methods: 'continuous', 'interp' + if method == "interp": + x = np.linspace(0, 1, 101) # 101-point interp (COCO) + ap = np.trapz(np.interp(x, mrec, mpre), x) # integrate + else: # 'continuous' + i = np.where(mrec[1:] != mrec[:-1])[0] # points where x-axis (recall) changes + ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) # area under curve + + return ap, mpre, mrec + + +def ap_per_class( + tp, conf, pred_cls, target_cls, plot=False, on_plot=None, save_dir=Path(), names=(), eps=1e-16, prefix="" +): + """ + Computes the average precision per class for object detection evaluation. + + Args: + tp (np.ndarray): Binary array indicating whether the detection is correct (True) or not (False). + conf (np.ndarray): Array of confidence scores of the detections. + pred_cls (np.ndarray): Array of predicted classes of the detections. + target_cls (np.ndarray): Array of true classes of the detections. + plot (bool, optional): Whether to plot PR curves or not. Defaults to False. + on_plot (func, optional): A callback to pass plots path and data when they are rendered. Defaults to None. + save_dir (Path, optional): Directory to save the PR curves. Defaults to an empty path. + names (tuple, optional): Tuple of class names to plot PR curves. Defaults to an empty tuple. + eps (float, optional): A small value to avoid division by zero. Defaults to 1e-16. + prefix (str, optional): A prefix string for saving the plot files. Defaults to an empty string. + + Returns: + (tuple): A tuple of six arrays and one array of unique classes, where: + tp (np.ndarray): True positive counts at threshold given by max F1 metric for each class.Shape: (nc,). + fp (np.ndarray): False positive counts at threshold given by max F1 metric for each class. Shape: (nc,). + p (np.ndarray): Precision values at threshold given by max F1 metric for each class. Shape: (nc,). + r (np.ndarray): Recall values at threshold given by max F1 metric for each class. Shape: (nc,). + f1 (np.ndarray): F1-score values at threshold given by max F1 metric for each class. Shape: (nc,). + ap (np.ndarray): Average precision for each class at different IoU thresholds. Shape: (nc, 10). + unique_classes (np.ndarray): An array of unique classes that have data. Shape: (nc,). + p_curve (np.ndarray): Precision curves for each class. Shape: (nc, 1000). + r_curve (np.ndarray): Recall curves for each class. Shape: (nc, 1000). + f1_curve (np.ndarray): F1-score curves for each class. Shape: (nc, 1000). + x (np.ndarray): X-axis values for the curves. Shape: (1000,). + prec_values: Precision values at mAP@0.5 for each class. Shape: (nc, 1000). + """ + + # Sort by objectness + i = np.argsort(-conf) + tp, conf, pred_cls = tp[i], conf[i], pred_cls[i] + + # Find unique classes + unique_classes, nt = np.unique(target_cls, return_counts=True) + nc = unique_classes.shape[0] # number of classes, number of detections + + # Create Precision-Recall curve and compute AP for each class + x, prec_values = np.linspace(0, 1, 1000), [] + + # Average precision, precision and recall curves + ap, p_curve, r_curve = np.zeros((nc, tp.shape[1])), np.zeros((nc, 1000)), np.zeros((nc, 1000)) + for ci, c in enumerate(unique_classes): + i = pred_cls == c + n_l = nt[ci] # number of labels + n_p = i.sum() # number of predictions + if n_p == 0 or n_l == 0: + continue + + # Accumulate FPs and TPs + fpc = (1 - tp[i]).cumsum(0) + tpc = tp[i].cumsum(0) + + # Recall + recall = tpc / (n_l + eps) # recall curve + r_curve[ci] = np.interp(-x, -conf[i], recall[:, 0], left=0) # negative x, xp because xp decreases + + # Precision + precision = tpc / (tpc + fpc) # precision curve + p_curve[ci] = np.interp(-x, -conf[i], precision[:, 0], left=1) # p at pr_score + + # AP from recall-precision curve + for j in range(tp.shape[1]): + ap[ci, j], mpre, mrec = compute_ap(recall[:, j], precision[:, j]) + if plot and j == 0: + prec_values.append(np.interp(x, mrec, mpre)) # precision at mAP@0.5 + + prec_values = np.array(prec_values) # (nc, 1000) + + # Compute F1 (harmonic mean of precision and recall) + f1_curve = 2 * p_curve * r_curve / (p_curve + r_curve + eps) + names = [v for k, v in names.items() if k in unique_classes] # list: only classes that have data + names = dict(enumerate(names)) # to dict + if plot: + plot_pr_curve(x, prec_values, ap, save_dir / f"{prefix}PR_curve.png", names, on_plot=on_plot) + plot_mc_curve(x, f1_curve, save_dir / f"{prefix}F1_curve.png", names, ylabel="F1", on_plot=on_plot) + plot_mc_curve(x, p_curve, save_dir / f"{prefix}P_curve.png", names, ylabel="Precision", on_plot=on_plot) + plot_mc_curve(x, r_curve, save_dir / f"{prefix}R_curve.png", names, ylabel="Recall", on_plot=on_plot) + + i = smooth(f1_curve.mean(0), 0.1).argmax() # max F1 index + p, r, f1 = p_curve[:, i], r_curve[:, i], f1_curve[:, i] # max-F1 precision, recall, F1 values + tp = (r * nt).round() # true positives + fp = (tp / (p + eps) - tp).round() # false positives + return tp, fp, p, r, f1, ap, unique_classes.astype(int), p_curve, r_curve, f1_curve, x, prec_values + + +class Metric(SimpleClass): + """ + Class for computing evaluation metrics for YOLOv8 model. + + Attributes: + p (list): Precision for each class. Shape: (nc,). + r (list): Recall for each class. Shape: (nc,). + f1 (list): F1 score for each class. Shape: (nc,). + all_ap (list): AP scores for all classes and all IoU thresholds. Shape: (nc, 10). + ap_class_index (list): Index of class for each AP score. Shape: (nc,). + nc (int): Number of classes. + + Methods: + ap50(): AP at IoU threshold of 0.5 for all classes. Returns: List of AP scores. Shape: (nc,) or []. + ap(): AP at IoU thresholds from 0.5 to 0.95 for all classes. Returns: List of AP scores. Shape: (nc,) or []. + mp(): Mean precision of all classes. Returns: Float. + mr(): Mean recall of all classes. Returns: Float. + map50(): Mean AP at IoU threshold of 0.5 for all classes. Returns: Float. + map75(): Mean AP at IoU threshold of 0.75 for all classes. Returns: Float. + map(): Mean AP at IoU thresholds from 0.5 to 0.95 for all classes. Returns: Float. + mean_results(): Mean of results, returns mp, mr, map50, map. + class_result(i): Class-aware result, returns p[i], r[i], ap50[i], ap[i]. + maps(): mAP of each class. Returns: Array of mAP scores, shape: (nc,). + fitness(): Model fitness as a weighted combination of metrics. Returns: Float. + update(results): Update metric attributes with new evaluation results. + """ + + def __init__(self) -> None: + """Initializes a Metric instance for computing evaluation metrics for the YOLOv8 model.""" + self.p = [] # (nc, ) + self.r = [] # (nc, ) + self.f1 = [] # (nc, ) + self.all_ap = [] # (nc, 10) + self.ap_class_index = [] # (nc, ) + self.nc = 0 + + @property + def ap50(self): + """ + Returns the Average Precision (AP) at an IoU threshold of 0.5 for all classes. + + Returns: + (np.ndarray, list): Array of shape (nc,) with AP50 values per class, or an empty list if not available. + """ + return self.all_ap[:, 0] if len(self.all_ap) else [] + + @property + def ap(self): + """ + Returns the Average Precision (AP) at an IoU threshold of 0.5-0.95 for all classes. + + Returns: + (np.ndarray, list): Array of shape (nc,) with AP50-95 values per class, or an empty list if not available. + """ + return self.all_ap.mean(1) if len(self.all_ap) else [] + + @property + def mp(self): + """ + Returns the Mean Precision of all classes. + + Returns: + (float): The mean precision of all classes. + """ + return self.p.mean() if len(self.p) else 0.0 + + @property + def mr(self): + """ + Returns the Mean Recall of all classes. + + Returns: + (float): The mean recall of all classes. + """ + return self.r.mean() if len(self.r) else 0.0 + + @property + def map50(self): + """ + Returns the mean Average Precision (mAP) at an IoU threshold of 0.5. + + Returns: + (float): The mAP at an IoU threshold of 0.5. + """ + return self.all_ap[:, 0].mean() if len(self.all_ap) else 0.0 + + @property + def map75(self): + """ + Returns the mean Average Precision (mAP) at an IoU threshold of 0.75. + + Returns: + (float): The mAP at an IoU threshold of 0.75. + """ + return self.all_ap[:, 5].mean() if len(self.all_ap) else 0.0 + + @property + def map(self): + """ + Returns the mean Average Precision (mAP) over IoU thresholds of 0.5 - 0.95 in steps of 0.05. + + Returns: + (float): The mAP over IoU thresholds of 0.5 - 0.95 in steps of 0.05. + """ + return self.all_ap.mean() if len(self.all_ap) else 0.0 + + def mean_results(self): + """Mean of results, return mp, mr, map50, map.""" + return [self.mp, self.mr, self.map50, self.map] + + def class_result(self, i): + """Class-aware result, return p[i], r[i], ap50[i], ap[i].""" + return self.p[i], self.r[i], self.ap50[i], self.ap[i] + + @property + def maps(self): + """MAP of each class.""" + maps = np.zeros(self.nc) + self.map + for i, c in enumerate(self.ap_class_index): + maps[c] = self.ap[i] + return maps + + def fitness(self): + """Model fitness as a weighted combination of metrics.""" + w = [0.0, 0.0, 0.1, 0.9] # weights for [P, R, mAP@0.5, mAP@0.5:0.95] + return (np.array(self.mean_results()) * w).sum() + + def update(self, results): + """ + Updates the evaluation metrics of the model with a new set of results. + + Args: + results (tuple): A tuple containing the following evaluation metrics: + - p (list): Precision for each class. Shape: (nc,). + - r (list): Recall for each class. Shape: (nc,). + - f1 (list): F1 score for each class. Shape: (nc,). + - all_ap (list): AP scores for all classes and all IoU thresholds. Shape: (nc, 10). + - ap_class_index (list): Index of class for each AP score. Shape: (nc,). + + Side Effects: + Updates the class attributes `self.p`, `self.r`, `self.f1`, `self.all_ap`, and `self.ap_class_index` based + on the values provided in the `results` tuple. + """ + ( + self.p, + self.r, + self.f1, + self.all_ap, + self.ap_class_index, + self.p_curve, + self.r_curve, + self.f1_curve, + self.px, + self.prec_values, + ) = results + + @property + def curves(self): + """Returns a list of curves for accessing specific metrics curves.""" + return [] + + @property + def curves_results(self): + """Returns a list of curves for accessing specific metrics curves.""" + return [ + [self.px, self.prec_values, "Recall", "Precision"], + [self.px, self.f1_curve, "Confidence", "F1"], + [self.px, self.p_curve, "Confidence", "Precision"], + [self.px, self.r_curve, "Confidence", "Recall"], + ] + + +class DetMetrics(SimpleClass): + """ + This class is a utility class for computing detection metrics such as precision, recall, and mean average precision + (mAP) of an object detection model. + + Args: + save_dir (Path): A path to the directory where the output plots will be saved. Defaults to current directory. + plot (bool): A flag that indicates whether to plot precision-recall curves for each class. Defaults to False. + on_plot (func): An optional callback to pass plots path and data when they are rendered. Defaults to None. + names (tuple of str): A tuple of strings that represents the names of the classes. Defaults to an empty tuple. + + Attributes: + save_dir (Path): A path to the directory where the output plots will be saved. + plot (bool): A flag that indicates whether to plot the precision-recall curves for each class. + on_plot (func): An optional callback to pass plots path and data when they are rendered. + names (tuple of str): A tuple of strings that represents the names of the classes. + box (Metric): An instance of the Metric class for storing the results of the detection metrics. + speed (dict): A dictionary for storing the execution time of different parts of the detection process. + + Methods: + process(tp, conf, pred_cls, target_cls): Updates the metric results with the latest batch of predictions. + keys: Returns a list of keys for accessing the computed detection metrics. + mean_results: Returns a list of mean values for the computed detection metrics. + class_result(i): Returns a list of values for the computed detection metrics for a specific class. + maps: Returns a dictionary of mean average precision (mAP) values for different IoU thresholds. + fitness: Computes the fitness score based on the computed detection metrics. + ap_class_index: Returns a list of class indices sorted by their average precision (AP) values. + results_dict: Returns a dictionary that maps detection metric keys to their computed values. + curves: TODO + curves_results: TODO + """ + + def __init__(self, save_dir=Path("."), plot=False, on_plot=None, names=()) -> None: + """Initialize a DetMetrics instance with a save directory, plot flag, callback function, and class names.""" + self.save_dir = save_dir + self.plot = plot + self.on_plot = on_plot + self.names = names + self.box = Metric() + self.speed = {"preprocess": 0.0, "inference": 0.0, "loss": 0.0, "postprocess": 0.0} + self.task = "detect" + + def process(self, tp, conf, pred_cls, target_cls): + """Process predicted results for object detection and update metrics.""" + results = ap_per_class( + tp, + conf, + pred_cls, + target_cls, + plot=self.plot, + save_dir=self.save_dir, + names=self.names, + on_plot=self.on_plot, + )[2:] + self.box.nc = len(self.names) + self.box.update(results) + + @property + def keys(self): + """Returns a list of keys for accessing specific metrics.""" + return ["metrics/precision(B)", "metrics/recall(B)", "metrics/mAP50(B)", "metrics/mAP50-95(B)"] + + def mean_results(self): + """Calculate mean of detected objects & return precision, recall, mAP50, and mAP50-95.""" + return self.box.mean_results() + + def class_result(self, i): + """Return the result of evaluating the performance of an object detection model on a specific class.""" + return self.box.class_result(i) + + @property + def maps(self): + """Returns mean Average Precision (mAP) scores per class.""" + return self.box.maps + + @property + def fitness(self): + """Returns the fitness of box object.""" + return self.box.fitness() + + @property + def ap_class_index(self): + """Returns the average precision index per class.""" + return self.box.ap_class_index + + @property + def results_dict(self): + """Returns dictionary of computed performance metrics and statistics.""" + return dict(zip(self.keys + ["fitness"], self.mean_results() + [self.fitness])) + + @property + def curves(self): + """Returns a list of curves for accessing specific metrics curves.""" + return ["Precision-Recall(B)", "F1-Confidence(B)", "Precision-Confidence(B)", "Recall-Confidence(B)"] + + @property + def curves_results(self): + """Returns dictionary of computed performance metrics and statistics.""" + return self.box.curves_results + + +class SegmentMetrics(SimpleClass): + """ + Calculates and aggregates detection and segmentation metrics over a given set of classes. + + Args: + save_dir (Path): Path to the directory where the output plots should be saved. Default is the current directory. + plot (bool): Whether to save the detection and segmentation plots. Default is False. + on_plot (func): An optional callback to pass plots path and data when they are rendered. Defaults to None. + names (list): List of class names. Default is an empty list. + + Attributes: + save_dir (Path): Path to the directory where the output plots should be saved. + plot (bool): Whether to save the detection and segmentation plots. + on_plot (func): An optional callback to pass plots path and data when they are rendered. + names (list): List of class names. + box (Metric): An instance of the Metric class to calculate box detection metrics. + seg (Metric): An instance of the Metric class to calculate mask segmentation metrics. + speed (dict): Dictionary to store the time taken in different phases of inference. + + Methods: + process(tp_m, tp_b, conf, pred_cls, target_cls): Processes metrics over the given set of predictions. + mean_results(): Returns the mean of the detection and segmentation metrics over all the classes. + class_result(i): Returns the detection and segmentation metrics of class `i`. + maps: Returns the mean Average Precision (mAP) scores for IoU thresholds ranging from 0.50 to 0.95. + fitness: Returns the fitness scores, which are a single weighted combination of metrics. + ap_class_index: Returns the list of indices of classes used to compute Average Precision (AP). + results_dict: Returns the dictionary containing all the detection and segmentation metrics and fitness score. + """ + + def __init__(self, save_dir=Path("."), plot=False, on_plot=None, names=()) -> None: + """Initialize a SegmentMetrics instance with a save directory, plot flag, callback function, and class names.""" + self.save_dir = save_dir + self.plot = plot + self.on_plot = on_plot + self.names = names + self.box = Metric() + self.seg = Metric() + self.speed = {"preprocess": 0.0, "inference": 0.0, "loss": 0.0, "postprocess": 0.0} + self.task = "segment" + + def process(self, tp, tp_m, conf, pred_cls, target_cls): + """ + Processes the detection and segmentation metrics over the given set of predictions. + + Args: + tp (list): List of True Positive boxes. + tp_m (list): List of True Positive masks. + conf (list): List of confidence scores. + pred_cls (list): List of predicted classes. + target_cls (list): List of target classes. + """ + + results_mask = ap_per_class( + tp_m, + conf, + pred_cls, + target_cls, + plot=self.plot, + on_plot=self.on_plot, + save_dir=self.save_dir, + names=self.names, + prefix="Mask", + )[2:] + self.seg.nc = len(self.names) + self.seg.update(results_mask) + results_box = ap_per_class( + tp, + conf, + pred_cls, + target_cls, + plot=self.plot, + on_plot=self.on_plot, + save_dir=self.save_dir, + names=self.names, + prefix="Box", + )[2:] + self.box.nc = len(self.names) + self.box.update(results_box) + + @property + def keys(self): + """Returns a list of keys for accessing metrics.""" + return [ + "metrics/precision(B)", + "metrics/recall(B)", + "metrics/mAP50(B)", + "metrics/mAP50-95(B)", + "metrics/precision(M)", + "metrics/recall(M)", + "metrics/mAP50(M)", + "metrics/mAP50-95(M)", + ] + + def mean_results(self): + """Return the mean metrics for bounding box and segmentation results.""" + return self.box.mean_results() + self.seg.mean_results() + + def class_result(self, i): + """Returns classification results for a specified class index.""" + return self.box.class_result(i) + self.seg.class_result(i) + + @property + def maps(self): + """Returns mAP scores for object detection and semantic segmentation models.""" + return self.box.maps + self.seg.maps + + @property + def fitness(self): + """Get the fitness score for both segmentation and bounding box models.""" + return self.seg.fitness() + self.box.fitness() + + @property + def ap_class_index(self): + """Boxes and masks have the same ap_class_index.""" + return self.box.ap_class_index + + @property + def results_dict(self): + """Returns results of object detection model for evaluation.""" + return dict(zip(self.keys + ["fitness"], self.mean_results() + [self.fitness])) + + @property + def curves(self): + """Returns a list of curves for accessing specific metrics curves.""" + return [ + "Precision-Recall(B)", + "F1-Confidence(B)", + "Precision-Confidence(B)", + "Recall-Confidence(B)", + "Precision-Recall(M)", + "F1-Confidence(M)", + "Precision-Confidence(M)", + "Recall-Confidence(M)", + ] + + @property + def curves_results(self): + """Returns dictionary of computed performance metrics and statistics.""" + return self.box.curves_results + self.seg.curves_results + + +class PoseMetrics(SegmentMetrics): + """ + Calculates and aggregates detection and pose metrics over a given set of classes. + + Args: + save_dir (Path): Path to the directory where the output plots should be saved. Default is the current directory. + plot (bool): Whether to save the detection and segmentation plots. Default is False. + on_plot (func): An optional callback to pass plots path and data when they are rendered. Defaults to None. + names (list): List of class names. Default is an empty list. + + Attributes: + save_dir (Path): Path to the directory where the output plots should be saved. + plot (bool): Whether to save the detection and segmentation plots. + on_plot (func): An optional callback to pass plots path and data when they are rendered. + names (list): List of class names. + box (Metric): An instance of the Metric class to calculate box detection metrics. + pose (Metric): An instance of the Metric class to calculate mask segmentation metrics. + speed (dict): Dictionary to store the time taken in different phases of inference. + + Methods: + process(tp_m, tp_b, conf, pred_cls, target_cls): Processes metrics over the given set of predictions. + mean_results(): Returns the mean of the detection and segmentation metrics over all the classes. + class_result(i): Returns the detection and segmentation metrics of class `i`. + maps: Returns the mean Average Precision (mAP) scores for IoU thresholds ranging from 0.50 to 0.95. + fitness: Returns the fitness scores, which are a single weighted combination of metrics. + ap_class_index: Returns the list of indices of classes used to compute Average Precision (AP). + results_dict: Returns the dictionary containing all the detection and segmentation metrics and fitness score. + """ + + def __init__(self, save_dir=Path("."), plot=False, on_plot=None, names=()) -> None: + """Initialize the PoseMetrics class with directory path, class names, and plotting options.""" + super().__init__(save_dir, plot, names) + self.save_dir = save_dir + self.plot = plot + self.on_plot = on_plot + self.names = names + self.box = Metric() + self.pose = Metric() + self.speed = {"preprocess": 0.0, "inference": 0.0, "loss": 0.0, "postprocess": 0.0} + self.task = "pose" + + def process(self, tp, tp_p, conf, pred_cls, target_cls): + """ + Processes the detection and pose metrics over the given set of predictions. + + Args: + tp (list): List of True Positive boxes. + tp_p (list): List of True Positive keypoints. + conf (list): List of confidence scores. + pred_cls (list): List of predicted classes. + target_cls (list): List of target classes. + """ + + results_pose = ap_per_class( + tp_p, + conf, + pred_cls, + target_cls, + plot=self.plot, + on_plot=self.on_plot, + save_dir=self.save_dir, + names=self.names, + prefix="Pose", + )[2:] + self.pose.nc = len(self.names) + self.pose.update(results_pose) + results_box = ap_per_class( + tp, + conf, + pred_cls, + target_cls, + plot=self.plot, + on_plot=self.on_plot, + save_dir=self.save_dir, + names=self.names, + prefix="Box", + )[2:] + self.box.nc = len(self.names) + self.box.update(results_box) + + @property + def keys(self): + """Returns list of evaluation metric keys.""" + return [ + "metrics/precision(B)", + "metrics/recall(B)", + "metrics/mAP50(B)", + "metrics/mAP50-95(B)", + "metrics/precision(P)", + "metrics/recall(P)", + "metrics/mAP50(P)", + "metrics/mAP50-95(P)", + ] + + def mean_results(self): + """Return the mean results of box and pose.""" + return self.box.mean_results() + self.pose.mean_results() + + def class_result(self, i): + """Return the class-wise detection results for a specific class i.""" + return self.box.class_result(i) + self.pose.class_result(i) + + @property + def maps(self): + """Returns the mean average precision (mAP) per class for both box and pose detections.""" + return self.box.maps + self.pose.maps + + @property + def fitness(self): + """Computes classification metrics and speed using the `targets` and `pred` inputs.""" + return self.pose.fitness() + self.box.fitness() + + @property + def curves(self): + """Returns a list of curves for accessing specific metrics curves.""" + return [ + "Precision-Recall(B)", + "F1-Confidence(B)", + "Precision-Confidence(B)", + "Recall-Confidence(B)", + "Precision-Recall(P)", + "F1-Confidence(P)", + "Precision-Confidence(P)", + "Recall-Confidence(P)", + ] + + @property + def curves_results(self): + """Returns dictionary of computed performance metrics and statistics.""" + return self.box.curves_results + self.pose.curves_results + + +class ClassifyMetrics(SimpleClass): + """ + Class for computing classification metrics including top-1 and top-5 accuracy. + + Attributes: + top1 (float): The top-1 accuracy. + top5 (float): The top-5 accuracy. + speed (Dict[str, float]): A dictionary containing the time taken for each step in the pipeline. + + Properties: + fitness (float): The fitness of the model, which is equal to top-5 accuracy. + results_dict (Dict[str, Union[float, str]]): A dictionary containing the classification metrics and fitness. + keys (List[str]): A list of keys for the results_dict. + + Methods: + process(targets, pred): Processes the targets and predictions to compute classification metrics. + """ + + def __init__(self) -> None: + """Initialize a ClassifyMetrics instance.""" + self.top1 = 0 + self.top5 = 0 + self.speed = {"preprocess": 0.0, "inference": 0.0, "loss": 0.0, "postprocess": 0.0} + self.task = "classify" + + def process(self, targets, pred): + """Target classes and predicted classes.""" + pred, targets = torch.cat(pred), torch.cat(targets) + correct = (targets[:, None] == pred).float() + acc = torch.stack((correct[:, 0], correct.max(1).values), dim=1) # (top1, top5) accuracy + self.top1, self.top5 = acc.mean(0).tolist() + + @property + def fitness(self): + """Returns mean of top-1 and top-5 accuracies as fitness score.""" + return (self.top1 + self.top5) / 2 + + @property + def results_dict(self): + """Returns a dictionary with model's performance metrics and fitness score.""" + return dict(zip(self.keys + ["fitness"], [self.top1, self.top5, self.fitness])) + + @property + def keys(self): + """Returns a list of keys for the results_dict property.""" + return ["metrics/accuracy_top1", "metrics/accuracy_top5"] + + @property + def curves(self): + """Returns a list of curves for accessing specific metrics curves.""" + return [] + + @property + def curves_results(self): + """Returns a list of curves for accessing specific metrics curves.""" + return [] + + +class OBBMetrics(SimpleClass): + def __init__(self, save_dir=Path("."), plot=False, on_plot=None, names=()) -> None: + self.save_dir = save_dir + self.plot = plot + self.on_plot = on_plot + self.names = names + self.box = Metric() + self.speed = {"preprocess": 0.0, "inference": 0.0, "loss": 0.0, "postprocess": 0.0} + + def process(self, tp, conf, pred_cls, target_cls): + """Process predicted results for object detection and update metrics.""" + results = ap_per_class( + tp, + conf, + pred_cls, + target_cls, + plot=self.plot, + save_dir=self.save_dir, + names=self.names, + on_plot=self.on_plot, + )[2:] + self.box.nc = len(self.names) + self.box.update(results) + + @property + def keys(self): + """Returns a list of keys for accessing specific metrics.""" + return ["metrics/precision(B)", "metrics/recall(B)", "metrics/mAP50(B)", "metrics/mAP50-95(B)"] + + def mean_results(self): + """Calculate mean of detected objects & return precision, recall, mAP50, and mAP50-95.""" + return self.box.mean_results() + + def class_result(self, i): + """Return the result of evaluating the performance of an object detection model on a specific class.""" + return self.box.class_result(i) + + @property + def maps(self): + """Returns mean Average Precision (mAP) scores per class.""" + return self.box.maps + + @property + def fitness(self): + """Returns the fitness of box object.""" + return self.box.fitness() + + @property + def ap_class_index(self): + """Returns the average precision index per class.""" + return self.box.ap_class_index + + @property + def results_dict(self): + """Returns dictionary of computed performance metrics and statistics.""" + return dict(zip(self.keys + ["fitness"], self.mean_results() + [self.fitness])) + + @property + def curves(self): + """Returns a list of curves for accessing specific metrics curves.""" + return [] + + @property + def curves_results(self): + """Returns a list of curves for accessing specific metrics curves.""" + return [] diff --git a/ultralytics/utils/ops.py b/ultralytics/utils/ops.py new file mode 100644 index 0000000..edbb103 --- /dev/null +++ b/ultralytics/utils/ops.py @@ -0,0 +1,864 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import contextlib +import math +import re +import time + +import cv2 +import numpy as np +import torch +import torch.nn.functional as F +import torchvision + +from ultralytics.utils import LOGGER +from ultralytics.utils.metrics import batch_probiou + + +class Profile(contextlib.ContextDecorator): + """ + YOLOv8 Profile class. Use as a decorator with @Profile() or as a context manager with 'with Profile():'. + + Example: + ```python + from ultralytics.utils.ops import Profile + + with Profile(device=device) as dt: + pass # slow operation here + + print(dt) # prints "Elapsed time is 9.5367431640625e-07 s" + ``` + """ + + def __init__(self, t=0.0, device: torch.device = None): + """ + Initialize the Profile class. + + Args: + t (float): Initial time. Defaults to 0.0. + device (torch.device): Devices used for model inference. Defaults to None (cpu). + """ + self.t = t + self.device = device + self.cuda = bool(device and str(device).startswith("cuda")) + + def __enter__(self): + """Start timing.""" + self.start = self.time() + return self + + def __exit__(self, type, value, traceback): # noqa + """Stop timing.""" + self.dt = self.time() - self.start # delta-time + self.t += self.dt # accumulate dt + + def __str__(self): + """Returns a human-readable string representing the accumulated elapsed time in the profiler.""" + return f"Elapsed time is {self.t} s" + + def time(self): + """Get current time.""" + if self.cuda: + torch.cuda.synchronize(self.device) + return time.time() + + +def segment2box(segment, width=640, height=640): + """ + Convert 1 segment label to 1 box label, applying inside-image constraint, i.e. (xy1, xy2, ...) to (xyxy). + + Args: + segment (torch.Tensor): the segment label + width (int): the width of the image. Defaults to 640 + height (int): The height of the image. Defaults to 640 + + Returns: + (np.ndarray): the minimum and maximum x and y values of the segment. + """ + x, y = segment.T # segment xy + inside = (x >= 0) & (y >= 0) & (x <= width) & (y <= height) + x = x[inside] + y = y[inside] + return ( + np.array([x.min(), y.min(), x.max(), y.max()], dtype=segment.dtype) + if any(x) + else np.zeros(4, dtype=segment.dtype) + ) # xyxy + + +def scale_boxes(img1_shape, boxes, img0_shape, ratio_pad=None, padding=True, xywh=False): + """ + Rescales bounding boxes (in the format of xyxy by default) from the shape of the image they were originally + specified in (img1_shape) to the shape of a different image (img0_shape). + + Args: + img1_shape (tuple): The shape of the image that the bounding boxes are for, in the format of (height, width). + boxes (torch.Tensor): the bounding boxes of the objects in the image, in the format of (x1, y1, x2, y2) + img0_shape (tuple): the shape of the target image, in the format of (height, width). + ratio_pad (tuple): a tuple of (ratio, pad) for scaling the boxes. If not provided, the ratio and pad will be + calculated based on the size difference between the two images. + padding (bool): If True, assuming the boxes is based on image augmented by yolo style. If False then do regular + rescaling. + xywh (bool): The box format is xywh or not, default=False. + + Returns: + boxes (torch.Tensor): The scaled bounding boxes, in the format of (x1, y1, x2, y2) + """ + if ratio_pad is None: # calculate from img0_shape + gain = min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1]) # gain = old / new + pad = ( + round((img1_shape[1] - img0_shape[1] * gain) / 2 - 0.1), + round((img1_shape[0] - img0_shape[0] * gain) / 2 - 0.1), + ) # wh padding + else: + gain = ratio_pad[0][0] + pad = ratio_pad[1] + + if padding: + boxes[..., 0] -= pad[0] # x padding + boxes[..., 1] -= pad[1] # y padding + if not xywh: + boxes[..., 2] -= pad[0] # x padding + boxes[..., 3] -= pad[1] # y padding + boxes[..., :4] /= gain + return clip_boxes(boxes, img0_shape) + + +def make_divisible(x, divisor): + """ + Returns the nearest number that is divisible by the given divisor. + + Args: + x (int): The number to make divisible. + divisor (int | torch.Tensor): The divisor. + + Returns: + (int): The nearest number divisible by the divisor. + """ + if isinstance(divisor, torch.Tensor): + divisor = int(divisor.max()) # to int + return math.ceil(x / divisor) * divisor + + +def nms_rotated(boxes, scores, threshold=0.45): + """ + NMS for obbs, powered by probiou and fast-nms. + + Args: + boxes (torch.Tensor): (N, 5), xywhr. + scores (torch.Tensor): (N, ). + threshold (float): IoU threshold. + + Returns: + """ + if len(boxes) == 0: + return np.empty((0,), dtype=np.int8) + sorted_idx = torch.argsort(scores, descending=True) + boxes = boxes[sorted_idx] + ious = batch_probiou(boxes, boxes).triu_(diagonal=1) + pick = torch.nonzero(ious.max(dim=0)[0] < threshold).squeeze_(-1) + return sorted_idx[pick] + + +def non_max_suppression( + prediction, + conf_thres=0.25, + iou_thres=0.45, + classes=None, + agnostic=False, + multi_label=False, + labels=(), + max_det=300, + nc=0, # number of classes (optional) + max_time_img=0.05, + max_nms=30000, + max_wh=7680, + in_place=True, + rotated=False, +): + """ + Perform non-maximum suppression (NMS) on a set of boxes, with support for masks and multiple labels per box. + + Args: + prediction (torch.Tensor): A tensor of shape (batch_size, num_classes + 4 + num_masks, num_boxes) + containing the predicted boxes, classes, and masks. The tensor should be in the format + output by a model, such as YOLO. + conf_thres (float): The confidence threshold below which boxes will be filtered out. + Valid values are between 0.0 and 1.0. + iou_thres (float): The IoU threshold below which boxes will be filtered out during NMS. + Valid values are between 0.0 and 1.0. + classes (List[int]): A list of class indices to consider. If None, all classes will be considered. + agnostic (bool): If True, the model is agnostic to the number of classes, and all + classes will be considered as one. + multi_label (bool): If True, each box may have multiple labels. + labels (List[List[Union[int, float, torch.Tensor]]]): A list of lists, where each inner + list contains the apriori labels for a given image. The list should be in the format + output by a dataloader, with each label being a tuple of (class_index, x1, y1, x2, y2). + max_det (int): The maximum number of boxes to keep after NMS. + nc (int, optional): The number of classes output by the model. Any indices after this will be considered masks. + max_time_img (float): The maximum time (seconds) for processing one image. + max_nms (int): The maximum number of boxes into torchvision.ops.nms(). + max_wh (int): The maximum box width and height in pixels. + in_place (bool): If True, the input prediction tensor will be modified in place. + + Returns: + (List[torch.Tensor]): A list of length batch_size, where each element is a tensor of + shape (num_boxes, 6 + num_masks) containing the kept boxes, with columns + (x1, y1, x2, y2, confidence, class, mask1, mask2, ...). + """ + + # Checks + assert 0 <= conf_thres <= 1, f"Invalid Confidence threshold {conf_thres}, valid values are between 0.0 and 1.0" + assert 0 <= iou_thres <= 1, f"Invalid IoU {iou_thres}, valid values are between 0.0 and 1.0" + if isinstance(prediction, (list, tuple)): # YOLOv8 model in validation model, output = (inference_out, loss_out) + prediction = prediction[0] # select only inference output + + bs = prediction.shape[0] # batch size + nc = nc or (prediction.shape[1] - 4) # number of classes + nm = prediction.shape[1] - nc - 4 + mi = 4 + nc # mask start index + xc = prediction[:, 4:mi].amax(1) > conf_thres # candidates + + # Settings + # min_wh = 2 # (pixels) minimum box width and height + time_limit = 2.0 + max_time_img * bs # seconds to quit after + multi_label &= nc > 1 # multiple labels per box (adds 0.5ms/img) + + prediction = prediction.transpose(-1, -2) # shape(1,84,6300) to shape(1,6300,84) + if not rotated: + if in_place: + prediction[..., :4] = xywh2xyxy(prediction[..., :4]) # xywh to xyxy + else: + prediction = torch.cat((xywh2xyxy(prediction[..., :4]), prediction[..., 4:]), dim=-1) # xywh to xyxy + + t = time.time() + output = [torch.zeros((0, 6 + nm), device=prediction.device)] * bs + for xi, x in enumerate(prediction): # image index, image inference + # Apply constraints + # x[((x[:, 2:4] < min_wh) | (x[:, 2:4] > max_wh)).any(1), 4] = 0 # width-height + x = x[xc[xi]] # confidence + + # Cat apriori labels if autolabelling + if labels and len(labels[xi]) and not rotated: + lb = labels[xi] + v = torch.zeros((len(lb), nc + nm + 4), device=x.device) + v[:, :4] = xywh2xyxy(lb[:, 1:5]) # box + v[range(len(lb)), lb[:, 0].long() + 4] = 1.0 # cls + x = torch.cat((x, v), 0) + + # If none remain process next image + if not x.shape[0]: + continue + + # Detections matrix nx6 (xyxy, conf, cls) + box, cls, mask = x.split((4, nc, nm), 1) + + if multi_label: + i, j = torch.where(cls > conf_thres) + x = torch.cat((box[i], x[i, 4 + j, None], j[:, None].float(), mask[i]), 1) + else: # best class only + conf, j = cls.max(1, keepdim=True) + x = torch.cat((box, conf, j.float(), mask), 1)[conf.view(-1) > conf_thres] + + # Filter by class + if classes is not None: + x = x[(x[:, 5:6] == torch.tensor(classes, device=x.device)).any(1)] + + # Check shape + n = x.shape[0] # number of boxes + if not n: # no boxes + continue + if n > max_nms: # excess boxes + x = x[x[:, 4].argsort(descending=True)[:max_nms]] # sort by confidence and remove excess boxes + + # Batched NMS + c = x[:, 5:6] * (0 if agnostic else max_wh) # classes + scores = x[:, 4] # scores + if rotated: + boxes = torch.cat((x[:, :2] + c, x[:, 2:4], x[:, -1:]), dim=-1) # xywhr + i = nms_rotated(boxes, scores, iou_thres) + else: + boxes = x[:, :4] + c # boxes (offset by class) + i = torchvision.ops.nms(boxes, scores, iou_thres) # NMS + i = i[:max_det] # limit detections + + # # Experimental + # merge = False # use merge-NMS + # if merge and (1 < n < 3E3): # Merge NMS (boxes merged using weighted mean) + # # Update boxes as boxes(i,4) = weights(i,n) * boxes(n,4) + # from .metrics import box_iou + # iou = box_iou(boxes[i], boxes) > iou_thres # IoU matrix + # weights = iou * scores[None] # box weights + # x[i, :4] = torch.mm(weights, x[:, :4]).float() / weights.sum(1, keepdim=True) # merged boxes + # redundant = True # require redundant detections + # if redundant: + # i = i[iou.sum(1) > 1] # require redundancy + + output[xi] = x[i] + if (time.time() - t) > time_limit: + LOGGER.warning(f"WARNING ⚠️ NMS time limit {time_limit:.3f}s exceeded") + break # time limit exceeded + + return output + + +def clip_boxes(boxes, shape): + """ + Takes a list of bounding boxes and a shape (height, width) and clips the bounding boxes to the shape. + + Args: + boxes (torch.Tensor): the bounding boxes to clip + shape (tuple): the shape of the image + + Returns: + (torch.Tensor | numpy.ndarray): Clipped boxes + """ + if isinstance(boxes, torch.Tensor): # faster individually (WARNING: inplace .clamp_() Apple MPS bug) + boxes[..., 0] = boxes[..., 0].clamp(0, shape[1]) # x1 + boxes[..., 1] = boxes[..., 1].clamp(0, shape[0]) # y1 + boxes[..., 2] = boxes[..., 2].clamp(0, shape[1]) # x2 + boxes[..., 3] = boxes[..., 3].clamp(0, shape[0]) # y2 + else: # np.array (faster grouped) + boxes[..., [0, 2]] = boxes[..., [0, 2]].clip(0, shape[1]) # x1, x2 + boxes[..., [1, 3]] = boxes[..., [1, 3]].clip(0, shape[0]) # y1, y2 + return boxes + + +def clip_coords(coords, shape): + """ + Clip line coordinates to the image boundaries. + + Args: + coords (torch.Tensor | numpy.ndarray): A list of line coordinates. + shape (tuple): A tuple of integers representing the size of the image in the format (height, width). + + Returns: + (torch.Tensor | numpy.ndarray): Clipped coordinates + """ + if isinstance(coords, torch.Tensor): # faster individually (WARNING: inplace .clamp_() Apple MPS bug) + coords[..., 0] = coords[..., 0].clamp(0, shape[1]) # x + coords[..., 1] = coords[..., 1].clamp(0, shape[0]) # y + else: # np.array (faster grouped) + coords[..., 0] = coords[..., 0].clip(0, shape[1]) # x + coords[..., 1] = coords[..., 1].clip(0, shape[0]) # y + return coords + + +def scale_image(masks, im0_shape, ratio_pad=None): + """ + Takes a mask, and resizes it to the original image size. + + Args: + masks (np.ndarray): resized and padded masks/images, [h, w, num]/[h, w, 3]. + im0_shape (tuple): the original image shape + ratio_pad (tuple): the ratio of the padding to the original image. + + Returns: + masks (torch.Tensor): The masks that are being returned. + """ + # Rescale coordinates (xyxy) from im1_shape to im0_shape + im1_shape = masks.shape + if im1_shape[:2] == im0_shape[:2]: + return masks + if ratio_pad is None: # calculate from im0_shape + gain = min(im1_shape[0] / im0_shape[0], im1_shape[1] / im0_shape[1]) # gain = old / new + pad = (im1_shape[1] - im0_shape[1] * gain) / 2, (im1_shape[0] - im0_shape[0] * gain) / 2 # wh padding + else: + # gain = ratio_pad[0][0] + pad = ratio_pad[1] + top, left = int(pad[1]), int(pad[0]) # y, x + bottom, right = int(im1_shape[0] - pad[1]), int(im1_shape[1] - pad[0]) + + if len(masks.shape) < 2: + raise ValueError(f'"len of masks shape" should be 2 or 3, but got {len(masks.shape)}') + masks = masks[top:bottom, left:right] + masks = cv2.resize(masks, (im0_shape[1], im0_shape[0])) + if len(masks.shape) == 2: + masks = masks[:, :, None] + + return masks + + +def xyxy2xywh(x): + """ + Convert bounding box coordinates from (x1, y1, x2, y2) format to (x, y, width, height) format where (x1, y1) is the + top-left corner and (x2, y2) is the bottom-right corner. + + Args: + x (np.ndarray | torch.Tensor): The input bounding box coordinates in (x1, y1, x2, y2) format. + + Returns: + y (np.ndarray | torch.Tensor): The bounding box coordinates in (x, y, width, height) format. + """ + assert x.shape[-1] == 4, f"input shape last dimension expected 4 but input shape is {x.shape}" + y = torch.empty_like(x) if isinstance(x, torch.Tensor) else np.empty_like(x) # faster than clone/copy + y[..., 0] = (x[..., 0] + x[..., 2]) / 2 # x center + y[..., 1] = (x[..., 1] + x[..., 3]) / 2 # y center + y[..., 2] = x[..., 2] - x[..., 0] # width + y[..., 3] = x[..., 3] - x[..., 1] # height + return y + + +def xywh2xyxy(x): + """ + Convert bounding box coordinates from (x, y, width, height) format to (x1, y1, x2, y2) format where (x1, y1) is the + top-left corner and (x2, y2) is the bottom-right corner. + + Args: + x (np.ndarray | torch.Tensor): The input bounding box coordinates in (x, y, width, height) format. + + Returns: + y (np.ndarray | torch.Tensor): The bounding box coordinates in (x1, y1, x2, y2) format. + """ + assert x.shape[-1] == 4, f"input shape last dimension expected 4 but input shape is {x.shape}" + y = torch.empty_like(x) if isinstance(x, torch.Tensor) else np.empty_like(x) # faster than clone/copy + dw = x[..., 2] / 2 # half-width + dh = x[..., 3] / 2 # half-height + y[..., 0] = x[..., 0] - dw # top left x + y[..., 1] = x[..., 1] - dh # top left y + y[..., 2] = x[..., 0] + dw # bottom right x + y[..., 3] = x[..., 1] + dh # bottom right y + return y + + +def xywhn2xyxy(x, w=640, h=640, padw=0, padh=0): + """ + Convert normalized bounding box coordinates to pixel coordinates. + + Args: + x (np.ndarray | torch.Tensor): The bounding box coordinates. + w (int): Width of the image. Defaults to 640 + h (int): Height of the image. Defaults to 640 + padw (int): Padding width. Defaults to 0 + padh (int): Padding height. Defaults to 0 + Returns: + y (np.ndarray | torch.Tensor): The coordinates of the bounding box in the format [x1, y1, x2, y2] where + x1,y1 is the top-left corner, x2,y2 is the bottom-right corner of the bounding box. + """ + assert x.shape[-1] == 4, f"input shape last dimension expected 4 but input shape is {x.shape}" + y = torch.empty_like(x) if isinstance(x, torch.Tensor) else np.empty_like(x) # faster than clone/copy + y[..., 0] = w * (x[..., 0] - x[..., 2] / 2) + padw # top left x + y[..., 1] = h * (x[..., 1] - x[..., 3] / 2) + padh # top left y + y[..., 2] = w * (x[..., 0] + x[..., 2] / 2) + padw # bottom right x + y[..., 3] = h * (x[..., 1] + x[..., 3] / 2) + padh # bottom right y + return y + + +def xyxy2xywhn(x, w=640, h=640, clip=False, eps=0.0): + """ + Convert bounding box coordinates from (x1, y1, x2, y2) format to (x, y, width, height, normalized) format. x, y, + width and height are normalized to image dimensions. + + Args: + x (np.ndarray | torch.Tensor): The input bounding box coordinates in (x1, y1, x2, y2) format. + w (int): The width of the image. Defaults to 640 + h (int): The height of the image. Defaults to 640 + clip (bool): If True, the boxes will be clipped to the image boundaries. Defaults to False + eps (float): The minimum value of the box's width and height. Defaults to 0.0 + + Returns: + y (np.ndarray | torch.Tensor): The bounding box coordinates in (x, y, width, height, normalized) format + """ + if clip: + x = clip_boxes(x, (h - eps, w - eps)) + assert x.shape[-1] == 4, f"input shape last dimension expected 4 but input shape is {x.shape}" + y = torch.empty_like(x) if isinstance(x, torch.Tensor) else np.empty_like(x) # faster than clone/copy + y[..., 0] = ((x[..., 0] + x[..., 2]) / 2) / w # x center + y[..., 1] = ((x[..., 1] + x[..., 3]) / 2) / h # y center + y[..., 2] = (x[..., 2] - x[..., 0]) / w # width + y[..., 3] = (x[..., 3] - x[..., 1]) / h # height + return y + + +def xywh2ltwh(x): + """ + Convert the bounding box format from [x, y, w, h] to [x1, y1, w, h], where x1, y1 are the top-left coordinates. + + Args: + x (np.ndarray | torch.Tensor): The input tensor with the bounding box coordinates in the xywh format + + Returns: + y (np.ndarray | torch.Tensor): The bounding box coordinates in the xyltwh format + """ + y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x) + y[..., 0] = x[..., 0] - x[..., 2] / 2 # top left x + y[..., 1] = x[..., 1] - x[..., 3] / 2 # top left y + return y + + +def xyxy2ltwh(x): + """ + Convert nx4 bounding boxes from [x1, y1, x2, y2] to [x1, y1, w, h], where xy1=top-left, xy2=bottom-right. + + Args: + x (np.ndarray | torch.Tensor): The input tensor with the bounding boxes coordinates in the xyxy format + + Returns: + y (np.ndarray | torch.Tensor): The bounding box coordinates in the xyltwh format. + """ + y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x) + y[..., 2] = x[..., 2] - x[..., 0] # width + y[..., 3] = x[..., 3] - x[..., 1] # height + return y + + +def ltwh2xywh(x): + """ + Convert nx4 boxes from [x1, y1, w, h] to [x, y, w, h] where xy1=top-left, xy=center. + + Args: + x (torch.Tensor): the input tensor + + Returns: + y (np.ndarray | torch.Tensor): The bounding box coordinates in the xywh format. + """ + y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x) + y[..., 0] = x[..., 0] + x[..., 2] / 2 # center x + y[..., 1] = x[..., 1] + x[..., 3] / 2 # center y + return y + + +def xyxyxyxy2xywhr(corners): + """ + Convert batched Oriented Bounding Boxes (OBB) from [xy1, xy2, xy3, xy4] to [xywh, rotation]. Rotation values are + expected in degrees from 0 to 90. + + Args: + corners (numpy.ndarray | torch.Tensor): Input corners of shape (n, 8). + + Returns: + (numpy.ndarray | torch.Tensor): Converted data in [cx, cy, w, h, rotation] format of shape (n, 5). + """ + is_torch = isinstance(corners, torch.Tensor) + points = corners.cpu().numpy() if is_torch else corners + points = points.reshape(len(corners), -1, 2) + rboxes = [] + for pts in points: + # NOTE: Use cv2.minAreaRect to get accurate xywhr, + # especially some objects are cut off by augmentations in dataloader. + (x, y), (w, h), angle = cv2.minAreaRect(pts) + rboxes.append([x, y, w, h, angle / 180 * np.pi]) + return ( + torch.tensor(rboxes, device=corners.device, dtype=corners.dtype) + if is_torch + else np.asarray(rboxes, dtype=points.dtype) + ) # rboxes + + +def xywhr2xyxyxyxy(rboxes): + """ + Convert batched Oriented Bounding Boxes (OBB) from [xywh, rotation] to [xy1, xy2, xy3, xy4]. Rotation values should + be in degrees from 0 to 90. + + Args: + rboxes (numpy.ndarray | torch.Tensor): Boxes in [cx, cy, w, h, rotation] format of shape (n, 5) or (b, n, 5). + + Returns: + (numpy.ndarray | torch.Tensor): Converted corner points of shape (n, 4, 2) or (b, n, 4, 2). + """ + is_numpy = isinstance(rboxes, np.ndarray) + cos, sin = (np.cos, np.sin) if is_numpy else (torch.cos, torch.sin) + + ctr = rboxes[..., :2] + w, h, angle = (rboxes[..., i : i + 1] for i in range(2, 5)) + cos_value, sin_value = cos(angle), sin(angle) + vec1 = [w / 2 * cos_value, w / 2 * sin_value] + vec2 = [-h / 2 * sin_value, h / 2 * cos_value] + vec1 = np.concatenate(vec1, axis=-1) if is_numpy else torch.cat(vec1, dim=-1) + vec2 = np.concatenate(vec2, axis=-1) if is_numpy else torch.cat(vec2, dim=-1) + pt1 = ctr + vec1 + vec2 + pt2 = ctr + vec1 - vec2 + pt3 = ctr - vec1 - vec2 + pt4 = ctr - vec1 + vec2 + return np.stack([pt1, pt2, pt3, pt4], axis=-2) if is_numpy else torch.stack([pt1, pt2, pt3, pt4], dim=-2) + + +def ltwh2xyxy(x): + """ + It converts the bounding box from [x1, y1, w, h] to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right. + + Args: + x (np.ndarray | torch.Tensor): the input image + + Returns: + y (np.ndarray | torch.Tensor): the xyxy coordinates of the bounding boxes. + """ + y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x) + y[..., 2] = x[..., 2] + x[..., 0] # width + y[..., 3] = x[..., 3] + x[..., 1] # height + return y + + +def segments2boxes(segments): + """ + It converts segment labels to box labels, i.e. (cls, xy1, xy2, ...) to (cls, xywh) + + Args: + segments (list): list of segments, each segment is a list of points, each point is a list of x, y coordinates + + Returns: + (np.ndarray): the xywh coordinates of the bounding boxes. + """ + boxes = [] + for s in segments: + x, y = s.T # segment xy + boxes.append([x.min(), y.min(), x.max(), y.max()]) # cls, xyxy + return xyxy2xywh(np.array(boxes)) # cls, xywh + + +def resample_segments(segments, n=1000): + """ + Inputs a list of segments (n,2) and returns a list of segments (n,2) up-sampled to n points each. + + Args: + segments (list): a list of (n,2) arrays, where n is the number of points in the segment. + n (int): number of points to resample the segment to. Defaults to 1000 + + Returns: + segments (list): the resampled segments. + """ + for i, s in enumerate(segments): + s = np.concatenate((s, s[0:1, :]), axis=0) + x = np.linspace(0, len(s) - 1, n) + xp = np.arange(len(s)) + segments[i] = ( + np.concatenate([np.interp(x, xp, s[:, i]) for i in range(2)], dtype=np.float32).reshape(2, -1).T + ) # segment xy + return segments + + +def crop_mask(masks, boxes): + """ + It takes a mask and a bounding box, and returns a mask that is cropped to the bounding box. + + Args: + masks (torch.Tensor): [n, h, w] tensor of masks + boxes (torch.Tensor): [n, 4] tensor of bbox coordinates in relative point form + + Returns: + (torch.Tensor): The masks are being cropped to the bounding box. + """ + _, h, w = masks.shape + x1, y1, x2, y2 = torch.chunk(boxes[:, :, None], 4, 1) # x1 shape(n,1,1) + r = torch.arange(w, device=masks.device, dtype=x1.dtype)[None, None, :] # rows shape(1,1,w) + c = torch.arange(h, device=masks.device, dtype=x1.dtype)[None, :, None] # cols shape(1,h,1) + + return masks * ((r >= x1) * (r < x2) * (c >= y1) * (c < y2)) + + +def process_mask_upsample(protos, masks_in, bboxes, shape): + """ + Takes the output of the mask head, and applies the mask to the bounding boxes. This produces masks of higher quality + but is slower. + + Args: + protos (torch.Tensor): [mask_dim, mask_h, mask_w] + masks_in (torch.Tensor): [n, mask_dim], n is number of masks after nms + bboxes (torch.Tensor): [n, 4], n is number of masks after nms + shape (tuple): the size of the input image (h,w) + + Returns: + (torch.Tensor): The upsampled masks. + """ + c, mh, mw = protos.shape # CHW + masks = (masks_in @ protos.float().view(c, -1)).sigmoid().view(-1, mh, mw) + masks = F.interpolate(masks[None], shape, mode="bilinear", align_corners=False)[0] # CHW + masks = crop_mask(masks, bboxes) # CHW + return masks.gt_(0.5) + + +def process_mask(protos, masks_in, bboxes, shape, upsample=False): + """ + Apply masks to bounding boxes using the output of the mask head. + + Args: + protos (torch.Tensor): A tensor of shape [mask_dim, mask_h, mask_w]. + masks_in (torch.Tensor): A tensor of shape [n, mask_dim], where n is the number of masks after NMS. + bboxes (torch.Tensor): A tensor of shape [n, 4], where n is the number of masks after NMS. + shape (tuple): A tuple of integers representing the size of the input image in the format (h, w). + upsample (bool): A flag to indicate whether to upsample the mask to the original image size. Default is False. + + Returns: + (torch.Tensor): A binary mask tensor of shape [n, h, w], where n is the number of masks after NMS, and h and w + are the height and width of the input image. The mask is applied to the bounding boxes. + """ + + c, mh, mw = protos.shape # CHW + ih, iw = shape + masks = (masks_in @ protos.float().view(c, -1)).sigmoid().view(-1, mh, mw) # CHW + width_ratio = mw / iw + height_ratio = mh / ih + + downsampled_bboxes = bboxes.clone() + downsampled_bboxes[:, 0] *= width_ratio + downsampled_bboxes[:, 2] *= width_ratio + downsampled_bboxes[:, 3] *= height_ratio + downsampled_bboxes[:, 1] *= height_ratio + + masks = crop_mask(masks, downsampled_bboxes) # CHW + if upsample: + masks = F.interpolate(masks[None], shape, mode="bilinear", align_corners=False)[0] # CHW + return masks.gt_(0.5) + + +def process_mask_native(protos, masks_in, bboxes, shape): + """ + It takes the output of the mask head, and crops it after upsampling to the bounding boxes. + + Args: + protos (torch.Tensor): [mask_dim, mask_h, mask_w] + masks_in (torch.Tensor): [n, mask_dim], n is number of masks after nms + bboxes (torch.Tensor): [n, 4], n is number of masks after nms + shape (tuple): the size of the input image (h,w) + + Returns: + masks (torch.Tensor): The returned masks with dimensions [h, w, n] + """ + c, mh, mw = protos.shape # CHW + masks = (masks_in @ protos.float().view(c, -1)).sigmoid().view(-1, mh, mw) + masks = scale_masks(masks[None], shape)[0] # CHW + masks = crop_mask(masks, bboxes) # CHW + return masks.gt_(0.5) + + +def scale_masks(masks, shape, padding=True): + """ + Rescale segment masks to shape. + + Args: + masks (torch.Tensor): (N, C, H, W). + shape (tuple): Height and width. + padding (bool): If True, assuming the boxes is based on image augmented by yolo style. If False then do regular + rescaling. + """ + mh, mw = masks.shape[2:] + gain = min(mh / shape[0], mw / shape[1]) # gain = old / new + pad = [mw - shape[1] * gain, mh - shape[0] * gain] # wh padding + if padding: + pad[0] /= 2 + pad[1] /= 2 + top, left = (int(pad[1]), int(pad[0])) if padding else (0, 0) # y, x + bottom, right = (int(mh - pad[1]), int(mw - pad[0])) + masks = masks[..., top:bottom, left:right] + + masks = F.interpolate(masks, shape, mode="bilinear", align_corners=False) # NCHW + return masks + + +def scale_coords(img1_shape, coords, img0_shape, ratio_pad=None, normalize=False, padding=True): + """ + Rescale segment coordinates (xy) from img1_shape to img0_shape. + + Args: + img1_shape (tuple): The shape of the image that the coords are from. + coords (torch.Tensor): the coords to be scaled of shape n,2. + img0_shape (tuple): the shape of the image that the segmentation is being applied to. + ratio_pad (tuple): the ratio of the image size to the padded image size. + normalize (bool): If True, the coordinates will be normalized to the range [0, 1]. Defaults to False. + padding (bool): If True, assuming the boxes is based on image augmented by yolo style. If False then do regular + rescaling. + + Returns: + coords (torch.Tensor): The scaled coordinates. + """ + if ratio_pad is None: # calculate from img0_shape + gain = min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1]) # gain = old / new + pad = (img1_shape[1] - img0_shape[1] * gain) / 2, (img1_shape[0] - img0_shape[0] * gain) / 2 # wh padding + else: + gain = ratio_pad[0][0] + pad = ratio_pad[1] + + if padding: + coords[..., 0] -= pad[0] # x padding + coords[..., 1] -= pad[1] # y padding + coords[..., 0] /= gain + coords[..., 1] /= gain + coords = clip_coords(coords, img0_shape) + if normalize: + coords[..., 0] /= img0_shape[1] # width + coords[..., 1] /= img0_shape[0] # height + return coords + + +def regularize_rboxes(rboxes): + """ + Regularize rotated boxes in range [0, pi/2]. + + Args: + rboxes (torch.Tensor): (N, 5), xywhr. + + Returns: + (torch.Tensor): The regularized boxes. + """ + x, y, w, h, t = rboxes.unbind(dim=-1) + # Swap edge and angle if h >= w + w_ = torch.where(w > h, w, h) + h_ = torch.where(w > h, h, w) + t = torch.where(w > h, t, t + math.pi / 2) % math.pi + return torch.stack([x, y, w_, h_, t], dim=-1) # regularized boxes + + +def masks2segments(masks, strategy="largest"): + """ + It takes a list of masks(n,h,w) and returns a list of segments(n,xy) + + Args: + masks (torch.Tensor): the output of the model, which is a tensor of shape (batch_size, 160, 160) + strategy (str): 'concat' or 'largest'. Defaults to largest + + Returns: + segments (List): list of segment masks + """ + segments = [] + for x in masks.int().cpu().numpy().astype("uint8"): + c = cv2.findContours(x, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[0] + if c: + if strategy == "concat": # concatenate all segments + c = np.concatenate([x.reshape(-1, 2) for x in c]) + elif strategy == "largest": # select largest segment + c = np.array(c[np.array([len(x) for x in c]).argmax()]).reshape(-1, 2) + else: + c = np.zeros((0, 2)) # no segments found + segments.append(c.astype("float32")) + return segments + + +def convert_torch2numpy_batch(batch: torch.Tensor) -> np.ndarray: + """ + Convert a batch of FP32 torch tensors (0.0-1.0) to a NumPy uint8 array (0-255), changing from BCHW to BHWC layout. + + Args: + batch (torch.Tensor): Input tensor batch of shape (Batch, Channels, Height, Width) and dtype torch.float32. + + Returns: + (np.ndarray): Output NumPy array batch of shape (Batch, Height, Width, Channels) and dtype uint8. + """ + return (batch.permute(0, 2, 3, 1).contiguous() * 255).clamp(0, 255).to(torch.uint8).cpu().numpy() + + +def clean_str(s): + """ + Cleans a string by replacing special characters with underscore _ + + Args: + s (str): a string needing special characters replaced + + Returns: + (str): a string with special characters replaced by an underscore _ + """ + return re.sub(pattern="[|@#!¡·$€%&()=?¿^*;:,¨´><+]", repl="_", string=s) + +def v10postprocess(preds, max_det, nc=80): + assert(4 + nc == preds.shape[-1]) + boxes, scores = preds.split([4, nc], dim=-1) + max_scores = scores.amax(dim=-1) + max_scores, index = torch.topk(max_scores, max_det, dim=-1) + index = index.unsqueeze(-1) + boxes = torch.gather(boxes, dim=1, index=index.repeat(1, 1, boxes.shape[-1])) + scores = torch.gather(scores, dim=1, index=index.repeat(1, 1, scores.shape[-1])) + + scores, index = torch.topk(scores.flatten(1), max_det, dim=-1) + labels = index % nc + index = index // nc + boxes = boxes.gather(dim=1, index=index.unsqueeze(-1).repeat(1, 1, boxes.shape[-1])) + return boxes, scores, labels \ No newline at end of file diff --git a/ultralytics/utils/patches.py b/ultralytics/utils/patches.py new file mode 100644 index 0000000..d438407 --- /dev/null +++ b/ultralytics/utils/patches.py @@ -0,0 +1,88 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license +"""Monkey patches to update/extend functionality of existing functions.""" + +import time +from pathlib import Path + +import cv2 +import numpy as np +import torch + +# OpenCV Multilanguage-friendly functions ------------------------------------------------------------------------------ +_imshow = cv2.imshow # copy to avoid recursion errors + + +def imread(filename: str, flags: int = cv2.IMREAD_COLOR): + """ + Read an image from a file. + + Args: + filename (str): Path to the file to read. + flags (int, optional): Flag that can take values of cv2.IMREAD_*. Defaults to cv2.IMREAD_COLOR. + + Returns: + (np.ndarray): The read image. + """ + return cv2.imdecode(np.fromfile(filename, np.uint8), flags) + + +def imwrite(filename: str, img: np.ndarray, params=None): + """ + Write an image to a file. + + Args: + filename (str): Path to the file to write. + img (np.ndarray): Image to write. + params (list of ints, optional): Additional parameters. See OpenCV documentation. + + Returns: + (bool): True if the file was written, False otherwise. + """ + try: + cv2.imencode(Path(filename).suffix, img, params)[1].tofile(filename) + return True + except Exception: + return False + + +def imshow(winname: str, mat: np.ndarray): + """ + Displays an image in the specified window. + + Args: + winname (str): Name of the window. + mat (np.ndarray): Image to be shown. + """ + _imshow(winname.encode("unicode_escape").decode(), mat) + + +# PyTorch functions ---------------------------------------------------------------------------------------------------- +_torch_save = torch.save # copy to avoid recursion errors + + +def torch_save(*args, use_dill=True, **kwargs): + """ + Optionally use dill to serialize lambda functions where pickle does not, adding robustness with 3 retries and + exponential standoff in case of save failure. + + Args: + *args (tuple): Positional arguments to pass to torch.save. + use_dill (bool): Whether to try using dill for serialization if available. Defaults to True. + **kwargs (any): Keyword arguments to pass to torch.save. + """ + try: + assert use_dill + import dill as pickle + except (AssertionError, ImportError): + import pickle + + if "pickle_module" not in kwargs: + kwargs["pickle_module"] = pickle + + for i in range(4): # 3 retries + try: + return _torch_save(*args, **kwargs) + except RuntimeError as e: # unable to save, possibly waiting for device to flush or antivirus scan + if i == 3: + raise e + time.sleep((2**i) / 2) # exponential standoff: 0.5s, 1.0s, 2.0s diff --git a/ultralytics/utils/plotting.py b/ultralytics/utils/plotting.py new file mode 100644 index 0000000..d0215ba --- /dev/null +++ b/ultralytics/utils/plotting.py @@ -0,0 +1,1047 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import contextlib +import math +import warnings +from pathlib import Path + +import cv2 +import matplotlib.pyplot as plt +import numpy as np +import torch +from PIL import Image, ImageDraw, ImageFont +from PIL import __version__ as pil_version + +from ultralytics.utils import LOGGER, TryExcept, ops, plt_settings, threaded +from .checks import check_font, check_version, is_ascii +from .files import increment_path + + +class Colors: + """ + Ultralytics default color palette https://ultralytics.com/. + + This class provides methods to work with the Ultralytics color palette, including converting hex color codes to + RGB values. + + Attributes: + palette (list of tuple): List of RGB color values. + n (int): The number of colors in the palette. + pose_palette (np.ndarray): A specific color palette array with dtype np.uint8. + """ + + def __init__(self): + """Initialize colors as hex = matplotlib.colors.TABLEAU_COLORS.values().""" + hexs = ( + "FF3838", + "FF9D97", + "FF701F", + "FFB21D", + "CFD231", + "48F90A", + "92CC17", + "3DDB86", + "1A9334", + "00D4BB", + "2C99A8", + "00C2FF", + "344593", + "6473FF", + "0018EC", + "8438FF", + "520085", + "CB38FF", + "FF95C8", + "FF37C7", + ) + self.palette = [self.hex2rgb(f"#{c}") for c in hexs] + self.n = len(self.palette) + self.pose_palette = np.array( + [ + [255, 128, 0], + [255, 153, 51], + [255, 178, 102], + [230, 230, 0], + [255, 153, 255], + [153, 204, 255], + [255, 102, 255], + [255, 51, 255], + [102, 178, 255], + [51, 153, 255], + [255, 153, 153], + [255, 102, 102], + [255, 51, 51], + [153, 255, 153], + [102, 255, 102], + [51, 255, 51], + [0, 255, 0], + [0, 0, 255], + [255, 0, 0], + [255, 255, 255], + ], + dtype=np.uint8, + ) + + def __call__(self, i, bgr=False): + """Converts hex color codes to RGB values.""" + c = self.palette[int(i) % self.n] + return (c[2], c[1], c[0]) if bgr else c + + @staticmethod + def hex2rgb(h): + """Converts hex color codes to RGB values (i.e. default PIL order).""" + return tuple(int(h[1 + i : 1 + i + 2], 16) for i in (0, 2, 4)) + + +colors = Colors() # create instance for 'from utils.plots import colors' + + +class Annotator: + """ + Ultralytics Annotator for train/val mosaics and JPGs and predictions annotations. + + Attributes: + im (Image.Image or numpy array): The image to annotate. + pil (bool): Whether to use PIL or cv2 for drawing annotations. + font (ImageFont.truetype or ImageFont.load_default): Font used for text annotations. + lw (float): Line width for drawing. + skeleton (List[List[int]]): Skeleton structure for keypoints. + limb_color (List[int]): Color palette for limbs. + kpt_color (List[int]): Color palette for keypoints. + """ + + def __init__(self, im, line_width=None, font_size=None, font="Arial.ttf", pil=False, example="abc"): + """Initialize the Annotator class with image and line width along with color palette for keypoints and limbs.""" + non_ascii = not is_ascii(example) # non-latin labels, i.e. asian, arabic, cyrillic + input_is_pil = isinstance(im, Image.Image) + self.pil = pil or non_ascii or input_is_pil + self.lw = line_width or max(round(sum(im.size if input_is_pil else im.shape) / 2 * 0.003), 2) + if self.pil: # use PIL + self.im = im if input_is_pil else Image.fromarray(im) + self.draw = ImageDraw.Draw(self.im) + try: + font = check_font("Arial.Unicode.ttf" if non_ascii else font) + size = font_size or max(round(sum(self.im.size) / 2 * 0.035), 12) + self.font = ImageFont.truetype(str(font), size) + except Exception: + self.font = ImageFont.load_default() + # Deprecation fix for w, h = getsize(string) -> _, _, w, h = getbox(string) + if check_version(pil_version, "9.2.0"): + self.font.getsize = lambda x: self.font.getbbox(x)[2:4] # text width, height + else: # use cv2 + assert im.data.contiguous, "Image not contiguous. Apply np.ascontiguousarray(im) to Annotator input images." + self.im = im if im.flags.writeable else im.copy() + self.tf = max(self.lw - 1, 1) # font thickness + self.sf = self.lw / 3 # font scale + # Pose + self.skeleton = [ + [16, 14], + [14, 12], + [17, 15], + [15, 13], + [12, 13], + [6, 12], + [7, 13], + [6, 7], + [6, 8], + [7, 9], + [8, 10], + [9, 11], + [2, 3], + [1, 2], + [1, 3], + [2, 4], + [3, 5], + [4, 6], + [5, 7], + ] + + self.limb_color = colors.pose_palette[[9, 9, 9, 9, 7, 7, 7, 0, 0, 0, 0, 0, 16, 16, 16, 16, 16, 16, 16]] + self.kpt_color = colors.pose_palette[[16, 16, 16, 16, 16, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9]] + + def box_label(self, box, label="", color=(128, 128, 128), txt_color=(255, 255, 255), rotated=False): + """Add one xyxy box to image with label.""" + if isinstance(box, torch.Tensor): + box = box.tolist() + if self.pil or not is_ascii(label): + if rotated: + p1 = box[0] + # NOTE: PIL-version polygon needs tuple type. + self.draw.polygon([tuple(b) for b in box], width=self.lw, outline=color) + else: + p1 = (box[0], box[1]) + self.draw.rectangle(box, width=self.lw, outline=color) # box + if label: + w, h = self.font.getsize(label) # text width, height + outside = p1[1] - h >= 0 # label fits outside box + self.draw.rectangle( + (p1[0], p1[1] - h if outside else p1[1], p1[0] + w + 1, p1[1] + 1 if outside else p1[1] + h + 1), + fill=color, + ) + # self.draw.text((box[0], box[1]), label, fill=txt_color, font=self.font, anchor='ls') # for PIL>8.0 + self.draw.text((p1[0], p1[1] - h if outside else p1[1]), label, fill=txt_color, font=self.font) + else: # cv2 + if rotated: + p1 = [int(b) for b in box[0]] + # NOTE: cv2-version polylines needs np.asarray type. + cv2.polylines(self.im, [np.asarray(box, dtype=int)], True, color, self.lw) + else: + p1, p2 = (int(box[0]), int(box[1])), (int(box[2]), int(box[3])) + cv2.rectangle(self.im, p1, p2, color, thickness=self.lw, lineType=cv2.LINE_AA) + if label: + w, h = cv2.getTextSize(label, 0, fontScale=self.sf, thickness=self.tf)[0] # text width, height + outside = p1[1] - h >= 3 + p2 = p1[0] + w, p1[1] - h - 3 if outside else p1[1] + h + 3 + cv2.rectangle(self.im, p1, p2, color, -1, cv2.LINE_AA) # filled + cv2.putText( + self.im, + label, + (p1[0], p1[1] - 2 if outside else p1[1] + h + 2), + 0, + self.sf, + txt_color, + thickness=self.tf, + lineType=cv2.LINE_AA, + ) + + def masks(self, masks, colors, im_gpu, alpha=0.5, retina_masks=False): + """ + Plot masks on image. + + Args: + masks (tensor): Predicted masks on cuda, shape: [n, h, w] + colors (List[List[Int]]): Colors for predicted masks, [[r, g, b] * n] + im_gpu (tensor): Image is in cuda, shape: [3, h, w], range: [0, 1] + alpha (float): Mask transparency: 0.0 fully transparent, 1.0 opaque + retina_masks (bool): Whether to use high resolution masks or not. Defaults to False. + """ + if self.pil: + # Convert to numpy first + self.im = np.asarray(self.im).copy() + if len(masks) == 0: + self.im[:] = im_gpu.permute(1, 2, 0).contiguous().cpu().numpy() * 255 + if im_gpu.device != masks.device: + im_gpu = im_gpu.to(masks.device) + colors = torch.tensor(colors, device=masks.device, dtype=torch.float32) / 255.0 # shape(n,3) + colors = colors[:, None, None] # shape(n,1,1,3) + masks = masks.unsqueeze(3) # shape(n,h,w,1) + masks_color = masks * (colors * alpha) # shape(n,h,w,3) + + inv_alpha_masks = (1 - masks * alpha).cumprod(0) # shape(n,h,w,1) + mcs = masks_color.max(dim=0).values # shape(n,h,w,3) + + im_gpu = im_gpu.flip(dims=[0]) # flip channel + im_gpu = im_gpu.permute(1, 2, 0).contiguous() # shape(h,w,3) + im_gpu = im_gpu * inv_alpha_masks[-1] + mcs + im_mask = im_gpu * 255 + im_mask_np = im_mask.byte().cpu().numpy() + self.im[:] = im_mask_np if retina_masks else ops.scale_image(im_mask_np, self.im.shape) + if self.pil: + # Convert im back to PIL and update draw + self.fromarray(self.im) + + def kpts(self, kpts, shape=(640, 640), radius=5, kpt_line=True): + """ + Plot keypoints on the image. + + Args: + kpts (tensor): Predicted keypoints with shape [17, 3]. Each keypoint has (x, y, confidence). + shape (tuple): Image shape as a tuple (h, w), where h is the height and w is the width. + radius (int, optional): Radius of the drawn keypoints. Default is 5. + kpt_line (bool, optional): If True, the function will draw lines connecting keypoints + for human pose. Default is True. + + Note: + `kpt_line=True` currently only supports human pose plotting. + """ + if self.pil: + # Convert to numpy first + self.im = np.asarray(self.im).copy() + nkpt, ndim = kpts.shape + is_pose = nkpt == 17 and ndim in {2, 3} + kpt_line &= is_pose # `kpt_line=True` for now only supports human pose plotting + for i, k in enumerate(kpts): + color_k = [int(x) for x in self.kpt_color[i]] if is_pose else colors(i) + x_coord, y_coord = k[0], k[1] + if x_coord % shape[1] != 0 and y_coord % shape[0] != 0: + if len(k) == 3: + conf = k[2] + if conf < 0.5: + continue + cv2.circle(self.im, (int(x_coord), int(y_coord)), radius, color_k, -1, lineType=cv2.LINE_AA) + + if kpt_line: + ndim = kpts.shape[-1] + for i, sk in enumerate(self.skeleton): + pos1 = (int(kpts[(sk[0] - 1), 0]), int(kpts[(sk[0] - 1), 1])) + pos2 = (int(kpts[(sk[1] - 1), 0]), int(kpts[(sk[1] - 1), 1])) + if ndim == 3: + conf1 = kpts[(sk[0] - 1), 2] + conf2 = kpts[(sk[1] - 1), 2] + if conf1 < 0.5 or conf2 < 0.5: + continue + if pos1[0] % shape[1] == 0 or pos1[1] % shape[0] == 0 or pos1[0] < 0 or pos1[1] < 0: + continue + if pos2[0] % shape[1] == 0 or pos2[1] % shape[0] == 0 or pos2[0] < 0 or pos2[1] < 0: + continue + cv2.line(self.im, pos1, pos2, [int(x) for x in self.limb_color[i]], thickness=2, lineType=cv2.LINE_AA) + if self.pil: + # Convert im back to PIL and update draw + self.fromarray(self.im) + + def rectangle(self, xy, fill=None, outline=None, width=1): + """Add rectangle to image (PIL-only).""" + self.draw.rectangle(xy, fill, outline, width) + + def text(self, xy, text, txt_color=(255, 255, 255), anchor="top", box_style=False): + """Adds text to an image using PIL or cv2.""" + if anchor == "bottom": # start y from font bottom + w, h = self.font.getsize(text) # text width, height + xy[1] += 1 - h + if self.pil: + if box_style: + w, h = self.font.getsize(text) + self.draw.rectangle((xy[0], xy[1], xy[0] + w + 1, xy[1] + h + 1), fill=txt_color) + # Using `txt_color` for background and draw fg with white color + txt_color = (255, 255, 255) + if "\n" in text: + lines = text.split("\n") + _, h = self.font.getsize(text) + for line in lines: + self.draw.text(xy, line, fill=txt_color, font=self.font) + xy[1] += h + else: + self.draw.text(xy, text, fill=txt_color, font=self.font) + else: + if box_style: + w, h = cv2.getTextSize(text, 0, fontScale=self.sf, thickness=self.tf)[0] # text width, height + outside = xy[1] - h >= 3 + p2 = xy[0] + w, xy[1] - h - 3 if outside else xy[1] + h + 3 + cv2.rectangle(self.im, xy, p2, txt_color, -1, cv2.LINE_AA) # filled + # Using `txt_color` for background and draw fg with white color + txt_color = (255, 255, 255) + cv2.putText(self.im, text, xy, 0, self.sf, txt_color, thickness=self.tf, lineType=cv2.LINE_AA) + + def fromarray(self, im): + """Update self.im from a numpy array.""" + self.im = im if isinstance(im, Image.Image) else Image.fromarray(im) + self.draw = ImageDraw.Draw(self.im) + + def result(self): + """Return annotated image as array.""" + return np.asarray(self.im) + + def show(self, title=None): + """Show the annotated image.""" + Image.fromarray(np.asarray(self.im)[..., ::-1]).show(title) + + def save(self, filename="image.jpg"): + """Save the annotated image to 'filename'.""" + cv2.imwrite(filename, np.asarray(self.im)) + + def draw_region(self, reg_pts=None, color=(0, 255, 0), thickness=5): + """ + Draw region line. + + Args: + reg_pts (list): Region Points (for line 2 points, for region 4 points) + color (tuple): Region Color value + thickness (int): Region area thickness value + """ + cv2.polylines(self.im, [np.array(reg_pts, dtype=np.int32)], isClosed=True, color=color, thickness=thickness) + + def draw_centroid_and_tracks(self, track, color=(255, 0, 255), track_thickness=2): + """ + Draw centroid point and track trails. + + Args: + track (list): object tracking points for trails display + color (tuple): tracks line color + track_thickness (int): track line thickness value + """ + points = np.hstack(track).astype(np.int32).reshape((-1, 1, 2)) + cv2.polylines(self.im, [points], isClosed=False, color=color, thickness=track_thickness) + cv2.circle(self.im, (int(track[-1][0]), int(track[-1][1])), track_thickness * 2, color, -1) + + def count_labels(self, counts=0, count_txt_size=2, color=(255, 255, 255), txt_color=(0, 0, 0)): + """ + Plot counts for object counter. + + Args: + counts (int): objects counts value + count_txt_size (int): text size for counts display + color (tuple): background color of counts display + txt_color (tuple): text color of counts display + """ + self.tf = count_txt_size + tl = self.tf or round(0.002 * (self.im.shape[0] + self.im.shape[1]) / 2) + 1 + tf = max(tl - 1, 1) + + # Get text size for in_count and out_count + t_size_in = cv2.getTextSize(str(counts), 0, fontScale=tl / 2, thickness=tf)[0] + + # Calculate positions for counts label + text_width = t_size_in[0] + text_x = (self.im.shape[1] - text_width) // 2 # Center x-coordinate + text_y = t_size_in[1] + + # Create a rounded rectangle for in_count + cv2.rectangle( + self.im, (text_x - 5, text_y - 5), (text_x + text_width + 7, text_y + t_size_in[1] + 7), color, -1 + ) + cv2.putText( + self.im, str(counts), (text_x, text_y + t_size_in[1]), 0, tl / 2, txt_color, self.tf, lineType=cv2.LINE_AA + ) + + @staticmethod + def estimate_pose_angle(a, b, c): + """ + Calculate the pose angle for object. + + Args: + a (float) : The value of pose point a + b (float): The value of pose point b + c (float): The value o pose point c + + Returns: + angle (degree): Degree value of angle between three points + """ + a, b, c = np.array(a), np.array(b), np.array(c) + radians = np.arctan2(c[1] - b[1], c[0] - b[0]) - np.arctan2(a[1] - b[1], a[0] - b[0]) + angle = np.abs(radians * 180.0 / np.pi) + if angle > 180.0: + angle = 360 - angle + return angle + + def draw_specific_points(self, keypoints, indices=[2, 5, 7], shape=(640, 640), radius=2): + """ + Draw specific keypoints for gym steps counting. + + Args: + keypoints (list): list of keypoints data to be plotted + indices (list): keypoints ids list to be plotted + shape (tuple): imgsz for model inference + radius (int): Keypoint radius value + """ + for i, k in enumerate(keypoints): + if i in indices: + x_coord, y_coord = k[0], k[1] + if x_coord % shape[1] != 0 and y_coord % shape[0] != 0: + if len(k) == 3: + conf = k[2] + if conf < 0.5: + continue + cv2.circle(self.im, (int(x_coord), int(y_coord)), radius, (0, 255, 0), -1, lineType=cv2.LINE_AA) + return self.im + + def plot_angle_and_count_and_stage(self, angle_text, count_text, stage_text, center_kpt, line_thickness=2): + """ + Plot the pose angle, count value and step stage. + + Args: + angle_text (str): angle value for workout monitoring + count_text (str): counts value for workout monitoring + stage_text (str): stage decision for workout monitoring + center_kpt (int): centroid pose index for workout monitoring + line_thickness (int): thickness for text display + """ + angle_text, count_text, stage_text = (f" {angle_text:.2f}", f"Steps : {count_text}", f" {stage_text}") + font_scale = 0.6 + (line_thickness / 10.0) + + # Draw angle + (angle_text_width, angle_text_height), _ = cv2.getTextSize(angle_text, 0, font_scale, line_thickness) + angle_text_position = (int(center_kpt[0]), int(center_kpt[1])) + angle_background_position = (angle_text_position[0], angle_text_position[1] - angle_text_height - 5) + angle_background_size = (angle_text_width + 2 * 5, angle_text_height + 2 * 5 + (line_thickness * 2)) + cv2.rectangle( + self.im, + angle_background_position, + ( + angle_background_position[0] + angle_background_size[0], + angle_background_position[1] + angle_background_size[1], + ), + (255, 255, 255), + -1, + ) + cv2.putText(self.im, angle_text, angle_text_position, 0, font_scale, (0, 0, 0), line_thickness) + + # Draw Counts + (count_text_width, count_text_height), _ = cv2.getTextSize(count_text, 0, font_scale, line_thickness) + count_text_position = (angle_text_position[0], angle_text_position[1] + angle_text_height + 20) + count_background_position = ( + angle_background_position[0], + angle_background_position[1] + angle_background_size[1] + 5, + ) + count_background_size = (count_text_width + 10, count_text_height + 10 + (line_thickness * 2)) + + cv2.rectangle( + self.im, + count_background_position, + ( + count_background_position[0] + count_background_size[0], + count_background_position[1] + count_background_size[1], + ), + (255, 255, 255), + -1, + ) + cv2.putText(self.im, count_text, count_text_position, 0, font_scale, (0, 0, 0), line_thickness) + + # Draw Stage + (stage_text_width, stage_text_height), _ = cv2.getTextSize(stage_text, 0, font_scale, line_thickness) + stage_text_position = (int(center_kpt[0]), int(center_kpt[1]) + angle_text_height + count_text_height + 40) + stage_background_position = (stage_text_position[0], stage_text_position[1] - stage_text_height - 5) + stage_background_size = (stage_text_width + 10, stage_text_height + 10) + + cv2.rectangle( + self.im, + stage_background_position, + ( + stage_background_position[0] + stage_background_size[0], + stage_background_position[1] + stage_background_size[1], + ), + (255, 255, 255), + -1, + ) + cv2.putText(self.im, stage_text, stage_text_position, 0, font_scale, (0, 0, 0), line_thickness) + + def seg_bbox(self, mask, mask_color=(255, 0, 255), det_label=None, track_label=None): + """ + Function for drawing segmented object in bounding box shape. + + Args: + mask (list): masks data list for instance segmentation area plotting + mask_color (tuple): mask foreground color + det_label (str): Detection label text + track_label (str): Tracking label text + """ + cv2.polylines(self.im, [np.int32([mask])], isClosed=True, color=mask_color, thickness=2) + + label = f"Track ID: {track_label}" if track_label else det_label + text_size, _ = cv2.getTextSize(label, 0, 0.7, 1) + + cv2.rectangle( + self.im, + (int(mask[0][0]) - text_size[0] // 2 - 10, int(mask[0][1]) - text_size[1] - 10), + (int(mask[0][0]) + text_size[0] // 2 + 5, int(mask[0][1] + 5)), + mask_color, + -1, + ) + + cv2.putText( + self.im, label, (int(mask[0][0]) - text_size[0] // 2, int(mask[0][1]) - 5), 0, 0.7, (255, 255, 255), 2 + ) + + def plot_distance_and_line(self, distance_m, distance_mm, centroids, line_color, centroid_color): + """ + Plot the distance and line on frame. + + Args: + distance_m (float): Distance between two bbox centroids in meters. + distance_mm (float): Distance between two bbox centroids in millimeters. + centroids (list): Bounding box centroids data. + line_color (RGB): Distance line color. + centroid_color (RGB): Bounding box centroid color. + """ + (text_width_m, text_height_m), _ = cv2.getTextSize( + f"Distance M: {distance_m:.2f}m", cv2.FONT_HERSHEY_SIMPLEX, 0.8, 2 + ) + cv2.rectangle(self.im, (15, 25), (15 + text_width_m + 10, 25 + text_height_m + 20), (255, 255, 255), -1) + cv2.putText( + self.im, + f"Distance M: {distance_m:.2f}m", + (20, 50), + cv2.FONT_HERSHEY_SIMPLEX, + 0.8, + (0, 0, 0), + 2, + cv2.LINE_AA, + ) + + (text_width_mm, text_height_mm), _ = cv2.getTextSize( + f"Distance MM: {distance_mm:.2f}mm", cv2.FONT_HERSHEY_SIMPLEX, 0.8, 2 + ) + cv2.rectangle(self.im, (15, 75), (15 + text_width_mm + 10, 75 + text_height_mm + 20), (255, 255, 255), -1) + cv2.putText( + self.im, + f"Distance MM: {distance_mm:.2f}mm", + (20, 100), + cv2.FONT_HERSHEY_SIMPLEX, + 0.8, + (0, 0, 0), + 2, + cv2.LINE_AA, + ) + + cv2.line(self.im, centroids[0], centroids[1], line_color, 3) + cv2.circle(self.im, centroids[0], 6, centroid_color, -1) + cv2.circle(self.im, centroids[1], 6, centroid_color, -1) + + def visioneye(self, box, center_point, color=(235, 219, 11), pin_color=(255, 0, 255), thickness=2, pins_radius=10): + """ + Function for pinpoint human-vision eye mapping and plotting. + + Args: + box (list): Bounding box coordinates + center_point (tuple): center point for vision eye view + color (tuple): object centroid and line color value + pin_color (tuple): visioneye point color value + thickness (int): int value for line thickness + pins_radius (int): visioneye point radius value + """ + center_bbox = int((box[0] + box[2]) / 2), int((box[1] + box[3]) / 2) + cv2.circle(self.im, center_point, pins_radius, pin_color, -1) + cv2.circle(self.im, center_bbox, pins_radius, color, -1) + cv2.line(self.im, center_point, center_bbox, color, thickness) + + +@TryExcept() # known issue https://github.com/ultralytics/yolov5/issues/5395 +@plt_settings() +def plot_labels(boxes, cls, names=(), save_dir=Path(""), on_plot=None): + """Plot training labels including class histograms and box statistics.""" + import pandas as pd + import seaborn as sn + + # Filter matplotlib>=3.7.2 warning and Seaborn use_inf and is_categorical FutureWarnings + warnings.filterwarnings("ignore", category=UserWarning, message="The figure layout has changed to tight") + warnings.filterwarnings("ignore", category=FutureWarning) + + # Plot dataset labels + LOGGER.info(f"Plotting labels to {save_dir / 'labels.jpg'}... ") + nc = int(cls.max() + 1) # number of classes + boxes = boxes[:1000000] # limit to 1M boxes + x = pd.DataFrame(boxes, columns=["x", "y", "width", "height"]) + + # Seaborn correlogram + sn.pairplot(x, corner=True, diag_kind="auto", kind="hist", diag_kws=dict(bins=50), plot_kws=dict(pmax=0.9)) + plt.savefig(save_dir / "labels_correlogram.jpg", dpi=200) + plt.close() + + # Matplotlib labels + ax = plt.subplots(2, 2, figsize=(8, 8), tight_layout=True)[1].ravel() + y = ax[0].hist(cls, bins=np.linspace(0, nc, nc + 1) - 0.5, rwidth=0.8) + for i in range(nc): + y[2].patches[i].set_color([x / 255 for x in colors(i)]) + ax[0].set_ylabel("instances") + if 0 < len(names) < 30: + ax[0].set_xticks(range(len(names))) + ax[0].set_xticklabels(list(names.values()), rotation=90, fontsize=10) + else: + ax[0].set_xlabel("classes") + sn.histplot(x, x="x", y="y", ax=ax[2], bins=50, pmax=0.9) + sn.histplot(x, x="width", y="height", ax=ax[3], bins=50, pmax=0.9) + + # Rectangles + boxes[:, 0:2] = 0.5 # center + boxes = ops.xywh2xyxy(boxes) * 1000 + img = Image.fromarray(np.ones((1000, 1000, 3), dtype=np.uint8) * 255) + for cls, box in zip(cls[:500], boxes[:500]): + ImageDraw.Draw(img).rectangle(box, width=1, outline=colors(cls)) # plot + ax[1].imshow(img) + ax[1].axis("off") + + for a in [0, 1, 2, 3]: + for s in ["top", "right", "left", "bottom"]: + ax[a].spines[s].set_visible(False) + + fname = save_dir / "labels.jpg" + plt.savefig(fname, dpi=200) + plt.close() + if on_plot: + on_plot(fname) + + +def save_one_box(xyxy, im, file=Path("im.jpg"), gain=1.02, pad=10, square=False, BGR=False, save=True): + """ + Save image crop as {file} with crop size multiple {gain} and {pad} pixels. Save and/or return crop. + + This function takes a bounding box and an image, and then saves a cropped portion of the image according + to the bounding box. Optionally, the crop can be squared, and the function allows for gain and padding + adjustments to the bounding box. + + Args: + xyxy (torch.Tensor or list): A tensor or list representing the bounding box in xyxy format. + im (numpy.ndarray): The input image. + file (Path, optional): The path where the cropped image will be saved. Defaults to 'im.jpg'. + gain (float, optional): A multiplicative factor to increase the size of the bounding box. Defaults to 1.02. + pad (int, optional): The number of pixels to add to the width and height of the bounding box. Defaults to 10. + square (bool, optional): If True, the bounding box will be transformed into a square. Defaults to False. + BGR (bool, optional): If True, the image will be saved in BGR format, otherwise in RGB. Defaults to False. + save (bool, optional): If True, the cropped image will be saved to disk. Defaults to True. + + Returns: + (numpy.ndarray): The cropped image. + + Example: + ```python + from ultralytics.utils.plotting import save_one_box + + xyxy = [50, 50, 150, 150] + im = cv2.imread('image.jpg') + cropped_im = save_one_box(xyxy, im, file='cropped.jpg', square=True) + ``` + """ + + if not isinstance(xyxy, torch.Tensor): # may be list + xyxy = torch.stack(xyxy) + b = ops.xyxy2xywh(xyxy.view(-1, 4)) # boxes + if square: + b[:, 2:] = b[:, 2:].max(1)[0].unsqueeze(1) # attempt rectangle to square + b[:, 2:] = b[:, 2:] * gain + pad # box wh * gain + pad + xyxy = ops.xywh2xyxy(b).long() + xyxy = ops.clip_boxes(xyxy, im.shape) + crop = im[int(xyxy[0, 1]) : int(xyxy[0, 3]), int(xyxy[0, 0]) : int(xyxy[0, 2]), :: (1 if BGR else -1)] + if save: + file.parent.mkdir(parents=True, exist_ok=True) # make directory + f = str(increment_path(file).with_suffix(".jpg")) + # cv2.imwrite(f, crop) # save BGR, https://github.com/ultralytics/yolov5/issues/7007 chroma subsampling issue + Image.fromarray(crop[..., ::-1]).save(f, quality=95, subsampling=0) # save RGB + return crop + + +@threaded +def plot_images( + images, + batch_idx, + cls, + bboxes=np.zeros(0, dtype=np.float32), + confs=None, + masks=np.zeros(0, dtype=np.uint8), + kpts=np.zeros((0, 51), dtype=np.float32), + paths=None, + fname="images.jpg", + names=None, + on_plot=None, + max_subplots=16, + save=True, + conf_thres=0.25, +): + """Plot image grid with labels.""" + if isinstance(images, torch.Tensor): + images = images.cpu().float().numpy() + if isinstance(cls, torch.Tensor): + cls = cls.cpu().numpy() + if isinstance(bboxes, torch.Tensor): + bboxes = bboxes.cpu().numpy() + if isinstance(masks, torch.Tensor): + masks = masks.cpu().numpy().astype(int) + if isinstance(kpts, torch.Tensor): + kpts = kpts.cpu().numpy() + if isinstance(batch_idx, torch.Tensor): + batch_idx = batch_idx.cpu().numpy() + + max_size = 1920 # max image size + bs, _, h, w = images.shape # batch size, _, height, width + bs = min(bs, max_subplots) # limit plot images + ns = np.ceil(bs**0.5) # number of subplots (square) + if np.max(images[0]) <= 1: + images *= 255 # de-normalise (optional) + + # Build Image + mosaic = np.full((int(ns * h), int(ns * w), 3), 255, dtype=np.uint8) # init + for i in range(bs): + x, y = int(w * (i // ns)), int(h * (i % ns)) # block origin + mosaic[y : y + h, x : x + w, :] = images[i].transpose(1, 2, 0) + + # Resize (optional) + scale = max_size / ns / max(h, w) + if scale < 1: + h = math.ceil(scale * h) + w = math.ceil(scale * w) + mosaic = cv2.resize(mosaic, tuple(int(x * ns) for x in (w, h))) + + # Annotate + fs = int((h + w) * ns * 0.01) # font size + annotator = Annotator(mosaic, line_width=round(fs / 10), font_size=fs, pil=True, example=names) + for i in range(bs): + x, y = int(w * (i // ns)), int(h * (i % ns)) # block origin + annotator.rectangle([x, y, x + w, y + h], None, (255, 255, 255), width=2) # borders + if paths: + annotator.text((x + 5, y + 5), text=Path(paths[i]).name[:40], txt_color=(220, 220, 220)) # filenames + if len(cls) > 0: + idx = batch_idx == i + classes = cls[idx].astype("int") + labels = confs is None + + if len(bboxes): + boxes = bboxes[idx] + conf = confs[idx] if confs is not None else None # check for confidence presence (label vs pred) + is_obb = boxes.shape[-1] == 5 # xywhr + boxes = ops.xywhr2xyxyxyxy(boxes) if is_obb else ops.xywh2xyxy(boxes) + if len(boxes): + if boxes[:, :4].max() <= 1.1: # if normalized with tolerance 0.1 + boxes[..., 0::2] *= w # scale to pixels + boxes[..., 1::2] *= h + elif scale < 1: # absolute coords need scale if image scales + boxes[..., :4] *= scale + boxes[..., 0::2] += x + boxes[..., 1::2] += y + for j, box in enumerate(boxes.astype(np.int64).tolist()): + c = classes[j] + color = colors(c) + c = names.get(c, c) if names else c + if labels or conf[j] > conf_thres: + label = f"{c}" if labels else f"{c} {conf[j]:.1f}" + annotator.box_label(box, label, color=color, rotated=is_obb) + + elif len(classes): + for c in classes: + color = colors(c) + c = names.get(c, c) if names else c + annotator.text((x, y), f"{c}", txt_color=color, box_style=True) + + # Plot keypoints + if len(kpts): + kpts_ = kpts[idx].copy() + if len(kpts_): + if kpts_[..., 0].max() <= 1.01 or kpts_[..., 1].max() <= 1.01: # if normalized with tolerance .01 + kpts_[..., 0] *= w # scale to pixels + kpts_[..., 1] *= h + elif scale < 1: # absolute coords need scale if image scales + kpts_ *= scale + kpts_[..., 0] += x + kpts_[..., 1] += y + for j in range(len(kpts_)): + if labels or conf[j] > conf_thres: + annotator.kpts(kpts_[j]) + + # Plot masks + if len(masks): + if idx.shape[0] == masks.shape[0]: # overlap_masks=False + image_masks = masks[idx] + else: # overlap_masks=True + image_masks = masks[[i]] # (1, 640, 640) + nl = idx.sum() + index = np.arange(nl).reshape((nl, 1, 1)) + 1 + image_masks = np.repeat(image_masks, nl, axis=0) + image_masks = np.where(image_masks == index, 1.0, 0.0) + + im = np.asarray(annotator.im).copy() + for j in range(len(image_masks)): + if labels or conf[j] > conf_thres: + color = colors(classes[j]) + mh, mw = image_masks[j].shape + if mh != h or mw != w: + mask = image_masks[j].astype(np.uint8) + mask = cv2.resize(mask, (w, h)) + mask = mask.astype(bool) + else: + mask = image_masks[j].astype(bool) + with contextlib.suppress(Exception): + im[y : y + h, x : x + w, :][mask] = ( + im[y : y + h, x : x + w, :][mask] * 0.4 + np.array(color) * 0.6 + ) + annotator.fromarray(im) + if not save: + return np.asarray(annotator.im) + annotator.im.save(fname) # save + if on_plot: + on_plot(fname) + + +@plt_settings() +def plot_results(file="path/to/results.csv", dir="", segment=False, pose=False, classify=False, on_plot=None): + """ + Plot training results from a results CSV file. The function supports various types of data including segmentation, + pose estimation, and classification. Plots are saved as 'results.png' in the directory where the CSV is located. + + Args: + file (str, optional): Path to the CSV file containing the training results. Defaults to 'path/to/results.csv'. + dir (str, optional): Directory where the CSV file is located if 'file' is not provided. Defaults to ''. + segment (bool, optional): Flag to indicate if the data is for segmentation. Defaults to False. + pose (bool, optional): Flag to indicate if the data is for pose estimation. Defaults to False. + classify (bool, optional): Flag to indicate if the data is for classification. Defaults to False. + on_plot (callable, optional): Callback function to be executed after plotting. Takes filename as an argument. + Defaults to None. + + Example: + ```python + from ultralytics.utils.plotting import plot_results + + plot_results('path/to/results.csv', segment=True) + ``` + """ + import pandas as pd + from scipy.ndimage import gaussian_filter1d + + save_dir = Path(file).parent if file else Path(dir) + if classify: + fig, ax = plt.subplots(2, 2, figsize=(6, 6), tight_layout=True) + index = [1, 4, 2, 3] + elif segment: + fig, ax = plt.subplots(2, 8, figsize=(18, 6), tight_layout=True) + index = [1, 2, 3, 4, 5, 6, 9, 10, 13, 14, 15, 16, 7, 8, 11, 12] + elif pose: + fig, ax = plt.subplots(2, 9, figsize=(21, 6), tight_layout=True) + index = [1, 2, 3, 4, 5, 6, 7, 10, 11, 14, 15, 16, 17, 18, 8, 9, 12, 13] + else: + fig, ax = plt.subplots(2, 5, figsize=(12, 6), tight_layout=True) + index = [1, 2, 3, 4, 5, 8, 9, 10, 6, 7] + ax = ax.ravel() + files = list(save_dir.glob("results*.csv")) + assert len(files), f"No results.csv files found in {save_dir.resolve()}, nothing to plot." + for f in files: + try: + data = pd.read_csv(f) + s = [x.strip() for x in data.columns] + x = data.values[:, 0] + for i, j in enumerate(index): + y = data.values[:, j].astype("float") + # y[y == 0] = np.nan # don't show zero values + ax[i].plot(x, y, marker=".", label=f.stem, linewidth=2, markersize=8) # actual results + ax[i].plot(x, gaussian_filter1d(y, sigma=3), ":", label="smooth", linewidth=2) # smoothing line + ax[i].set_title(s[j], fontsize=12) + # if j in [8, 9, 10]: # share train and val loss y axes + # ax[i].get_shared_y_axes().join(ax[i], ax[i - 5]) + except Exception as e: + LOGGER.warning(f"WARNING: Plotting error for {f}: {e}") + ax[1].legend() + fname = save_dir / "results.png" + fig.savefig(fname, dpi=200) + plt.close() + if on_plot: + on_plot(fname) + + +def plt_color_scatter(v, f, bins=20, cmap="viridis", alpha=0.8, edgecolors="none"): + """ + Plots a scatter plot with points colored based on a 2D histogram. + + Args: + v (array-like): Values for the x-axis. + f (array-like): Values for the y-axis. + bins (int, optional): Number of bins for the histogram. Defaults to 20. + cmap (str, optional): Colormap for the scatter plot. Defaults to 'viridis'. + alpha (float, optional): Alpha for the scatter plot. Defaults to 0.8. + edgecolors (str, optional): Edge colors for the scatter plot. Defaults to 'none'. + + Examples: + >>> v = np.random.rand(100) + >>> f = np.random.rand(100) + >>> plt_color_scatter(v, f) + """ + + # Calculate 2D histogram and corresponding colors + hist, xedges, yedges = np.histogram2d(v, f, bins=bins) + colors = [ + hist[ + min(np.digitize(v[i], xedges, right=True) - 1, hist.shape[0] - 1), + min(np.digitize(f[i], yedges, right=True) - 1, hist.shape[1] - 1), + ] + for i in range(len(v)) + ] + + # Scatter plot + plt.scatter(v, f, c=colors, cmap=cmap, alpha=alpha, edgecolors=edgecolors) + + +def plot_tune_results(csv_file="tune_results.csv"): + """ + Plot the evolution results stored in an 'tune_results.csv' file. The function generates a scatter plot for each key + in the CSV, color-coded based on fitness scores. The best-performing configurations are highlighted on the plots. + + Args: + csv_file (str, optional): Path to the CSV file containing the tuning results. Defaults to 'tune_results.csv'. + + Examples: + >>> plot_tune_results('path/to/tune_results.csv') + """ + + import pandas as pd + from scipy.ndimage import gaussian_filter1d + + # Scatter plots for each hyperparameter + csv_file = Path(csv_file) + data = pd.read_csv(csv_file) + num_metrics_columns = 1 + keys = [x.strip() for x in data.columns][num_metrics_columns:] + x = data.values + fitness = x[:, 0] # fitness + j = np.argmax(fitness) # max fitness index + n = math.ceil(len(keys) ** 0.5) # columns and rows in plot + plt.figure(figsize=(10, 10), tight_layout=True) + for i, k in enumerate(keys): + v = x[:, i + num_metrics_columns] + mu = v[j] # best single result + plt.subplot(n, n, i + 1) + plt_color_scatter(v, fitness, cmap="viridis", alpha=0.8, edgecolors="none") + plt.plot(mu, fitness.max(), "k+", markersize=15) + plt.title(f"{k} = {mu:.3g}", fontdict={"size": 9}) # limit to 40 characters + plt.tick_params(axis="both", labelsize=8) # Set axis label size to 8 + if i % n != 0: + plt.yticks([]) + + file = csv_file.with_name("tune_scatter_plots.png") # filename + plt.savefig(file, dpi=200) + plt.close() + LOGGER.info(f"Saved {file}") + + # Fitness vs iteration + x = range(1, len(fitness) + 1) + plt.figure(figsize=(10, 6), tight_layout=True) + plt.plot(x, fitness, marker="o", linestyle="none", label="fitness") + plt.plot(x, gaussian_filter1d(fitness, sigma=3), ":", label="smoothed", linewidth=2) # smoothing line + plt.title("Fitness vs Iteration") + plt.xlabel("Iteration") + plt.ylabel("Fitness") + plt.grid(True) + plt.legend() + + file = csv_file.with_name("tune_fitness.png") # filename + plt.savefig(file, dpi=200) + plt.close() + LOGGER.info(f"Saved {file}") + + +def output_to_target(output, max_det=300): + """Convert model output to target format [batch_id, class_id, x, y, w, h, conf] for plotting.""" + targets = [] + for i, o in enumerate(output): + box, conf, cls = o[:max_det, :6].cpu().split((4, 1, 1), 1) + j = torch.full((conf.shape[0], 1), i) + targets.append(torch.cat((j, cls, ops.xyxy2xywh(box), conf), 1)) + targets = torch.cat(targets, 0).numpy() + return targets[:, 0], targets[:, 1], targets[:, 2:-1], targets[:, -1] + + +def output_to_rotated_target(output, max_det=300): + """Convert model output to target format [batch_id, class_id, x, y, w, h, conf] for plotting.""" + targets = [] + for i, o in enumerate(output): + box, conf, cls, angle = o[:max_det].cpu().split((4, 1, 1, 1), 1) + j = torch.full((conf.shape[0], 1), i) + targets.append(torch.cat((j, cls, box, angle, conf), 1)) + targets = torch.cat(targets, 0).numpy() + return targets[:, 0], targets[:, 1], targets[:, 2:-1], targets[:, -1] + + +def feature_visualization(x, module_type, stage, n=32, save_dir=Path("runs/detect/exp")): + """ + Visualize feature maps of a given model module during inference. + + Args: + x (torch.Tensor): Features to be visualized. + module_type (str): Module type. + stage (int): Module stage within the model. + n (int, optional): Maximum number of feature maps to plot. Defaults to 32. + save_dir (Path, optional): Directory to save results. Defaults to Path('runs/detect/exp'). + """ + for m in ["Detect", "Pose", "Segment"]: + if m in module_type: + return + _, channels, height, width = x.shape # batch, channels, height, width + if height > 1 and width > 1: + f = save_dir / f"stage{stage}_{module_type.split('.')[-1]}_features.png" # filename + + blocks = torch.chunk(x[0].cpu(), channels, dim=0) # select batch index 0, block by channels + n = min(n, channels) # number of plots + _, ax = plt.subplots(math.ceil(n / 8), 8, tight_layout=True) # 8 rows x n/8 cols + ax = ax.ravel() + plt.subplots_adjust(wspace=0.05, hspace=0.05) + for i in range(n): + ax[i].imshow(blocks[i].squeeze()) # cmap='gray' + ax[i].axis("off") + + LOGGER.info(f"Saving {f}... ({n}/{channels})") + plt.savefig(f, dpi=300, bbox_inches="tight") + plt.close() + np.save(str(f.with_suffix(".npy")), x[0].cpu().numpy()) # npy save diff --git a/ultralytics/utils/tal.py b/ultralytics/utils/tal.py new file mode 100644 index 0000000..b11c2b2 --- /dev/null +++ b/ultralytics/utils/tal.py @@ -0,0 +1,345 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import torch +import torch.nn as nn + +from .checks import check_version +from .metrics import bbox_iou, probiou +from .ops import xywhr2xyxyxyxy + +TORCH_1_10 = check_version(torch.__version__, "1.10.0") + + +class TaskAlignedAssigner(nn.Module): + """ + A task-aligned assigner for object detection. + + This class assigns ground-truth (gt) objects to anchors based on the task-aligned metric, which combines both + classification and localization information. + + Attributes: + topk (int): The number of top candidates to consider. + num_classes (int): The number of object classes. + alpha (float): The alpha parameter for the classification component of the task-aligned metric. + beta (float): The beta parameter for the localization component of the task-aligned metric. + eps (float): A small value to prevent division by zero. + """ + + def __init__(self, topk=13, num_classes=80, alpha=1.0, beta=6.0, eps=1e-9): + """Initialize a TaskAlignedAssigner object with customizable hyperparameters.""" + super().__init__() + self.topk = topk + self.num_classes = num_classes + self.bg_idx = num_classes + self.alpha = alpha + self.beta = beta + self.eps = eps + + @torch.no_grad() + def forward(self, pd_scores, pd_bboxes, anc_points, gt_labels, gt_bboxes, mask_gt): + """ + Compute the task-aligned assignment. Reference code is available at + https://github.com/Nioolek/PPYOLOE_pytorch/blob/master/ppyoloe/assigner/tal_assigner.py. + + Args: + pd_scores (Tensor): shape(bs, num_total_anchors, num_classes) + pd_bboxes (Tensor): shape(bs, num_total_anchors, 4) + anc_points (Tensor): shape(num_total_anchors, 2) + gt_labels (Tensor): shape(bs, n_max_boxes, 1) + gt_bboxes (Tensor): shape(bs, n_max_boxes, 4) + mask_gt (Tensor): shape(bs, n_max_boxes, 1) + + Returns: + target_labels (Tensor): shape(bs, num_total_anchors) + target_bboxes (Tensor): shape(bs, num_total_anchors, 4) + target_scores (Tensor): shape(bs, num_total_anchors, num_classes) + fg_mask (Tensor): shape(bs, num_total_anchors) + target_gt_idx (Tensor): shape(bs, num_total_anchors) + """ + self.bs = pd_scores.shape[0] + self.n_max_boxes = gt_bboxes.shape[1] + + if self.n_max_boxes == 0: + device = gt_bboxes.device + return ( + torch.full_like(pd_scores[..., 0], self.bg_idx).to(device), + torch.zeros_like(pd_bboxes).to(device), + torch.zeros_like(pd_scores).to(device), + torch.zeros_like(pd_scores[..., 0]).to(device), + torch.zeros_like(pd_scores[..., 0]).to(device), + ) + + mask_pos, align_metric, overlaps = self.get_pos_mask( + pd_scores, pd_bboxes, gt_labels, gt_bboxes, anc_points, mask_gt + ) + + target_gt_idx, fg_mask, mask_pos = self.select_highest_overlaps(mask_pos, overlaps, self.n_max_boxes) + + # Assigned target + target_labels, target_bboxes, target_scores = self.get_targets(gt_labels, gt_bboxes, target_gt_idx, fg_mask) + + # Normalize + align_metric *= mask_pos + pos_align_metrics = align_metric.amax(dim=-1, keepdim=True) # b, max_num_obj + pos_overlaps = (overlaps * mask_pos).amax(dim=-1, keepdim=True) # b, max_num_obj + norm_align_metric = (align_metric * pos_overlaps / (pos_align_metrics + self.eps)).amax(-2).unsqueeze(-1) + target_scores = target_scores * norm_align_metric + + return target_labels, target_bboxes, target_scores, fg_mask.bool(), target_gt_idx + + def get_pos_mask(self, pd_scores, pd_bboxes, gt_labels, gt_bboxes, anc_points, mask_gt): + """Get in_gts mask, (b, max_num_obj, h*w).""" + mask_in_gts = self.select_candidates_in_gts(anc_points, gt_bboxes) + # Get anchor_align metric, (b, max_num_obj, h*w) + align_metric, overlaps = self.get_box_metrics(pd_scores, pd_bboxes, gt_labels, gt_bboxes, mask_in_gts * mask_gt) + # Get topk_metric mask, (b, max_num_obj, h*w) + mask_topk = self.select_topk_candidates(align_metric, topk_mask=mask_gt.expand(-1, -1, self.topk).bool()) + # Merge all mask to a final mask, (b, max_num_obj, h*w) + mask_pos = mask_topk * mask_in_gts * mask_gt + + return mask_pos, align_metric, overlaps + + def get_box_metrics(self, pd_scores, pd_bboxes, gt_labels, gt_bboxes, mask_gt): + """Compute alignment metric given predicted and ground truth bounding boxes.""" + na = pd_bboxes.shape[-2] + mask_gt = mask_gt.bool() # b, max_num_obj, h*w + overlaps = torch.zeros([self.bs, self.n_max_boxes, na], dtype=pd_bboxes.dtype, device=pd_bboxes.device) + bbox_scores = torch.zeros([self.bs, self.n_max_boxes, na], dtype=pd_scores.dtype, device=pd_scores.device) + + ind = torch.zeros([2, self.bs, self.n_max_boxes], dtype=torch.long) # 2, b, max_num_obj + ind[0] = torch.arange(end=self.bs).view(-1, 1).expand(-1, self.n_max_boxes) # b, max_num_obj + ind[1] = gt_labels.squeeze(-1) # b, max_num_obj + # Get the scores of each grid for each gt cls + bbox_scores[mask_gt] = pd_scores[ind[0], :, ind[1]][mask_gt] # b, max_num_obj, h*w + + # (b, max_num_obj, 1, 4), (b, 1, h*w, 4) + pd_boxes = pd_bboxes.unsqueeze(1).expand(-1, self.n_max_boxes, -1, -1)[mask_gt] + gt_boxes = gt_bboxes.unsqueeze(2).expand(-1, -1, na, -1)[mask_gt] + overlaps[mask_gt] = self.iou_calculation(gt_boxes, pd_boxes) + + align_metric = bbox_scores.pow(self.alpha) * overlaps.pow(self.beta) + return align_metric, overlaps + + def iou_calculation(self, gt_bboxes, pd_bboxes): + """IoU calculation for horizontal bounding boxes.""" + return bbox_iou(gt_bboxes, pd_bboxes, xywh=False, CIoU=True).squeeze(-1).clamp_(0) + + def select_topk_candidates(self, metrics, largest=True, topk_mask=None): + """ + Select the top-k candidates based on the given metrics. + + Args: + metrics (Tensor): A tensor of shape (b, max_num_obj, h*w), where b is the batch size, + max_num_obj is the maximum number of objects, and h*w represents the + total number of anchor points. + largest (bool): If True, select the largest values; otherwise, select the smallest values. + topk_mask (Tensor): An optional boolean tensor of shape (b, max_num_obj, topk), where + topk is the number of top candidates to consider. If not provided, + the top-k values are automatically computed based on the given metrics. + + Returns: + (Tensor): A tensor of shape (b, max_num_obj, h*w) containing the selected top-k candidates. + """ + + # (b, max_num_obj, topk) + topk_metrics, topk_idxs = torch.topk(metrics, self.topk, dim=-1, largest=largest) + if topk_mask is None: + topk_mask = (topk_metrics.max(-1, keepdim=True)[0] > self.eps).expand_as(topk_idxs) + # (b, max_num_obj, topk) + topk_idxs.masked_fill_(~topk_mask, 0) + + # (b, max_num_obj, topk, h*w) -> (b, max_num_obj, h*w) + count_tensor = torch.zeros(metrics.shape, dtype=torch.int8, device=topk_idxs.device) + ones = torch.ones_like(topk_idxs[:, :, :1], dtype=torch.int8, device=topk_idxs.device) + for k in range(self.topk): + # Expand topk_idxs for each value of k and add 1 at the specified positions + count_tensor.scatter_add_(-1, topk_idxs[:, :, k : k + 1], ones) + # count_tensor.scatter_add_(-1, topk_idxs, torch.ones_like(topk_idxs, dtype=torch.int8, device=topk_idxs.device)) + # Filter invalid bboxes + count_tensor.masked_fill_(count_tensor > 1, 0) + + return count_tensor.to(metrics.dtype) + + def get_targets(self, gt_labels, gt_bboxes, target_gt_idx, fg_mask): + """ + Compute target labels, target bounding boxes, and target scores for the positive anchor points. + + Args: + gt_labels (Tensor): Ground truth labels of shape (b, max_num_obj, 1), where b is the + batch size and max_num_obj is the maximum number of objects. + gt_bboxes (Tensor): Ground truth bounding boxes of shape (b, max_num_obj, 4). + target_gt_idx (Tensor): Indices of the assigned ground truth objects for positive + anchor points, with shape (b, h*w), where h*w is the total + number of anchor points. + fg_mask (Tensor): A boolean tensor of shape (b, h*w) indicating the positive + (foreground) anchor points. + + Returns: + (Tuple[Tensor, Tensor, Tensor]): A tuple containing the following tensors: + - target_labels (Tensor): Shape (b, h*w), containing the target labels for + positive anchor points. + - target_bboxes (Tensor): Shape (b, h*w, 4), containing the target bounding boxes + for positive anchor points. + - target_scores (Tensor): Shape (b, h*w, num_classes), containing the target scores + for positive anchor points, where num_classes is the number + of object classes. + """ + + # Assigned target labels, (b, 1) + batch_ind = torch.arange(end=self.bs, dtype=torch.int64, device=gt_labels.device)[..., None] + target_gt_idx = target_gt_idx + batch_ind * self.n_max_boxes # (b, h*w) + target_labels = gt_labels.long().flatten()[target_gt_idx] # (b, h*w) + + # Assigned target boxes, (b, max_num_obj, 4) -> (b, h*w, 4) + target_bboxes = gt_bboxes.view(-1, gt_bboxes.shape[-1])[target_gt_idx] + + # Assigned target scores + target_labels.clamp_(0) + + # 10x faster than F.one_hot() + target_scores = torch.zeros( + (target_labels.shape[0], target_labels.shape[1], self.num_classes), + dtype=torch.int64, + device=target_labels.device, + ) # (b, h*w, 80) + target_scores.scatter_(2, target_labels.unsqueeze(-1), 1) + + fg_scores_mask = fg_mask[:, :, None].repeat(1, 1, self.num_classes) # (b, h*w, 80) + target_scores = torch.where(fg_scores_mask > 0, target_scores, 0) + + return target_labels, target_bboxes, target_scores + + @staticmethod + def select_candidates_in_gts(xy_centers, gt_bboxes, eps=1e-9): + """ + Select the positive anchor center in gt. + + Args: + xy_centers (Tensor): shape(h*w, 2) + gt_bboxes (Tensor): shape(b, n_boxes, 4) + + Returns: + (Tensor): shape(b, n_boxes, h*w) + """ + n_anchors = xy_centers.shape[0] + bs, n_boxes, _ = gt_bboxes.shape + lt, rb = gt_bboxes.view(-1, 1, 4).chunk(2, 2) # left-top, right-bottom + bbox_deltas = torch.cat((xy_centers[None] - lt, rb - xy_centers[None]), dim=2).view(bs, n_boxes, n_anchors, -1) + # return (bbox_deltas.min(3)[0] > eps).to(gt_bboxes.dtype) + return bbox_deltas.amin(3).gt_(eps) + + @staticmethod + def select_highest_overlaps(mask_pos, overlaps, n_max_boxes): + """ + If an anchor box is assigned to multiple gts, the one with the highest IoU will be selected. + + Args: + mask_pos (Tensor): shape(b, n_max_boxes, h*w) + overlaps (Tensor): shape(b, n_max_boxes, h*w) + + Returns: + target_gt_idx (Tensor): shape(b, h*w) + fg_mask (Tensor): shape(b, h*w) + mask_pos (Tensor): shape(b, n_max_boxes, h*w) + """ + # (b, n_max_boxes, h*w) -> (b, h*w) + fg_mask = mask_pos.sum(-2) + if fg_mask.max() > 1: # one anchor is assigned to multiple gt_bboxes + mask_multi_gts = (fg_mask.unsqueeze(1) > 1).expand(-1, n_max_boxes, -1) # (b, n_max_boxes, h*w) + max_overlaps_idx = overlaps.argmax(1) # (b, h*w) + + is_max_overlaps = torch.zeros(mask_pos.shape, dtype=mask_pos.dtype, device=mask_pos.device) + is_max_overlaps.scatter_(1, max_overlaps_idx.unsqueeze(1), 1) + + mask_pos = torch.where(mask_multi_gts, is_max_overlaps, mask_pos).float() # (b, n_max_boxes, h*w) + fg_mask = mask_pos.sum(-2) + # Find each grid serve which gt(index) + target_gt_idx = mask_pos.argmax(-2) # (b, h*w) + return target_gt_idx, fg_mask, mask_pos + + +class RotatedTaskAlignedAssigner(TaskAlignedAssigner): + def iou_calculation(self, gt_bboxes, pd_bboxes): + """IoU calculation for rotated bounding boxes.""" + return probiou(gt_bboxes, pd_bboxes).squeeze(-1).clamp_(0) + + @staticmethod + def select_candidates_in_gts(xy_centers, gt_bboxes): + """ + Select the positive anchor center in gt for rotated bounding boxes. + + Args: + xy_centers (Tensor): shape(h*w, 2) + gt_bboxes (Tensor): shape(b, n_boxes, 5) + + Returns: + (Tensor): shape(b, n_boxes, h*w) + """ + # (b, n_boxes, 5) --> (b, n_boxes, 4, 2) + corners = xywhr2xyxyxyxy(gt_bboxes) + # (b, n_boxes, 1, 2) + a, b, _, d = corners.split(1, dim=-2) + ab = b - a + ad = d - a + + # (b, n_boxes, h*w, 2) + ap = xy_centers - a + norm_ab = (ab * ab).sum(dim=-1) + norm_ad = (ad * ad).sum(dim=-1) + ap_dot_ab = (ap * ab).sum(dim=-1) + ap_dot_ad = (ap * ad).sum(dim=-1) + return (ap_dot_ab >= 0) & (ap_dot_ab <= norm_ab) & (ap_dot_ad >= 0) & (ap_dot_ad <= norm_ad) # is_in_box + + +def make_anchors(feats, strides, grid_cell_offset=0.5): + """Generate anchors from features.""" + anchor_points, stride_tensor = [], [] + assert feats is not None + dtype, device = feats[0].dtype, feats[0].device + for i, stride in enumerate(strides): + _, _, h, w = feats[i].shape + sx = torch.arange(end=w, device=device, dtype=dtype) + grid_cell_offset # shift x + sy = torch.arange(end=h, device=device, dtype=dtype) + grid_cell_offset # shift y + sy, sx = torch.meshgrid(sy, sx, indexing="ij") if TORCH_1_10 else torch.meshgrid(sy, sx) + anchor_points.append(torch.stack((sx, sy), -1).view(-1, 2)) + stride_tensor.append(torch.full((h * w, 1), stride, dtype=dtype, device=device)) + return torch.cat(anchor_points), torch.cat(stride_tensor) + + +def dist2bbox(distance, anchor_points, xywh=True, dim=-1): + """Transform distance(ltrb) to box(xywh or xyxy).""" + assert(distance.shape[dim] == 4) + lt, rb = distance.split([2, 2], dim) + x1y1 = anchor_points - lt + x2y2 = anchor_points + rb + if xywh: + c_xy = (x1y1 + x2y2) / 2 + wh = x2y2 - x1y1 + return torch.cat((c_xy, wh), dim) # xywh bbox + return torch.cat((x1y1, x2y2), dim) # xyxy bbox + + +def bbox2dist(anchor_points, bbox, reg_max): + """Transform bbox(xyxy) to dist(ltrb).""" + x1y1, x2y2 = bbox.chunk(2, -1) + return torch.cat((anchor_points - x1y1, x2y2 - anchor_points), -1).clamp_(0, reg_max - 0.01) # dist (lt, rb) + + +def dist2rbox(pred_dist, pred_angle, anchor_points, dim=-1): + """ + Decode predicted object bounding box coordinates from anchor points and distribution. + + Args: + pred_dist (torch.Tensor): Predicted rotated distance, (bs, h*w, 4). + pred_angle (torch.Tensor): Predicted angle, (bs, h*w, 1). + anchor_points (torch.Tensor): Anchor points, (h*w, 2). + Returns: + (torch.Tensor): Predicted rotated bounding boxes, (bs, h*w, 4). + """ + lt, rb = pred_dist.split(2, dim=dim) + cos, sin = torch.cos(pred_angle), torch.sin(pred_angle) + # (bs, h*w, 1) + xf, yf = ((rb - lt) / 2).split(1, dim=dim) + x, y = xf * cos - yf * sin, xf * sin + yf * cos + xy = torch.cat([x, y], dim=dim) + anchor_points + return torch.cat([xy, lt + rb], dim=dim) diff --git a/ultralytics/utils/torch_utils.py b/ultralytics/utils/torch_utils.py new file mode 100644 index 0000000..d476e1f --- /dev/null +++ b/ultralytics/utils/torch_utils.py @@ -0,0 +1,610 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import math +import os +import random +import time +from contextlib import contextmanager +from copy import deepcopy +from pathlib import Path +from typing import Union + +import numpy as np +import torch +import torch.distributed as dist +import torch.nn as nn +import torch.nn.functional as F +import torchvision + +from ultralytics.utils import DEFAULT_CFG_DICT, DEFAULT_CFG_KEYS, LOGGER, __version__ +from ultralytics.utils.checks import PYTHON_VERSION, check_version + +try: + import thop +except ImportError: + thop = None + +# Version checks (all default to version>=min_version) +TORCH_1_9 = check_version(torch.__version__, "1.9.0") +TORCH_1_13 = check_version(torch.__version__, "1.13.0") +TORCH_2_0 = check_version(torch.__version__, "2.0.0") +TORCHVISION_0_10 = check_version(torchvision.__version__, "0.10.0") +TORCHVISION_0_11 = check_version(torchvision.__version__, "0.11.0") +TORCHVISION_0_13 = check_version(torchvision.__version__, "0.13.0") + + +@contextmanager +def torch_distributed_zero_first(local_rank: int): + """Decorator to make all processes in distributed training wait for each local_master to do something.""" + initialized = torch.distributed.is_available() and torch.distributed.is_initialized() + if initialized and local_rank not in (-1, 0): + dist.barrier(device_ids=[local_rank]) + yield + if initialized and local_rank == 0: + dist.barrier(device_ids=[0]) + + +def smart_inference_mode(): + """Applies torch.inference_mode() decorator if torch>=1.9.0 else torch.no_grad() decorator.""" + + def decorate(fn): + """Applies appropriate torch decorator for inference mode based on torch version.""" + if TORCH_1_9 and torch.is_inference_mode_enabled(): + return fn # already in inference_mode, act as a pass-through + else: + return (torch.inference_mode if TORCH_1_9 else torch.no_grad)()(fn) + + return decorate + + +def get_cpu_info(): + """Return a string with system CPU information, i.e. 'Apple M2'.""" + import cpuinfo # pip install py-cpuinfo + + k = "brand_raw", "hardware_raw", "arch_string_raw" # info keys sorted by preference (not all keys always available) + info = cpuinfo.get_cpu_info() # info dict + string = info.get(k[0] if k[0] in info else k[1] if k[1] in info else k[2], "unknown") + return string.replace("(R)", "").replace("CPU ", "").replace("@ ", "") + + +def select_device(device="", batch=0, newline=False, verbose=True): + """ + Selects the appropriate PyTorch device based on the provided arguments. + + The function takes a string specifying the device or a torch.device object and returns a torch.device object + representing the selected device. The function also validates the number of available devices and raises an + exception if the requested device(s) are not available. + + Args: + device (str | torch.device, optional): Device string or torch.device object. + Options are 'None', 'cpu', or 'cuda', or '0' or '0,1,2,3'. Defaults to an empty string, which auto-selects + the first available GPU, or CPU if no GPU is available. + batch (int, optional): Batch size being used in your model. Defaults to 0. + newline (bool, optional): If True, adds a newline at the end of the log string. Defaults to False. + verbose (bool, optional): If True, logs the device information. Defaults to True. + + Returns: + (torch.device): Selected device. + + Raises: + ValueError: If the specified device is not available or if the batch size is not a multiple of the number of + devices when using multiple GPUs. + + Examples: + >>> select_device('cuda:0') + device(type='cuda', index=0) + + >>> select_device('cpu') + device(type='cpu') + + Note: + Sets the 'CUDA_VISIBLE_DEVICES' environment variable for specifying which GPUs to use. + """ + + if isinstance(device, torch.device): + return device + + s = f"Ultralytics YOLOv{__version__} 🚀 Python-{PYTHON_VERSION} torch-{torch.__version__} " + device = str(device).lower() + for remove in "cuda:", "none", "(", ")", "[", "]", "'", " ": + device = device.replace(remove, "") # to string, 'cuda:0' -> '0' and '(0, 1)' -> '0,1' + cpu = device == "cpu" + mps = device in ("mps", "mps:0") # Apple Metal Performance Shaders (MPS) + if cpu or mps: + os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # force torch.cuda.is_available() = False + elif device: # non-cpu device requested + if device == "cuda": + device = "0" + visible = os.environ.get("CUDA_VISIBLE_DEVICES", None) + os.environ["CUDA_VISIBLE_DEVICES"] = device # set environment variable - must be before assert is_available() + if not (torch.cuda.is_available() and torch.cuda.device_count() >= len(device.split(","))): + LOGGER.info(s) + install = ( + "See https://pytorch.org/get-started/locally/ for up-to-date torch install instructions if no " + "CUDA devices are seen by torch.\n" + if torch.cuda.device_count() == 0 + else "" + ) + raise ValueError( + f"Invalid CUDA 'device={device}' requested." + f" Use 'device=cpu' or pass valid CUDA device(s) if available," + f" i.e. 'device=0' or 'device=0,1,2,3' for Multi-GPU.\n" + f"\ntorch.cuda.is_available(): {torch.cuda.is_available()}" + f"\ntorch.cuda.device_count(): {torch.cuda.device_count()}" + f"\nos.environ['CUDA_VISIBLE_DEVICES']: {visible}\n" + f"{install}" + ) + + if not cpu and not mps and torch.cuda.is_available(): # prefer GPU if available + devices = device.split(",") if device else "0" # range(torch.cuda.device_count()) # i.e. 0,1,6,7 + n = len(devices) # device count + if n > 1 and batch > 0 and batch % n != 0: # check batch_size is divisible by device_count + raise ValueError( + f"'batch={batch}' must be a multiple of GPU count {n}. Try 'batch={batch // n * n}' or " + f"'batch={batch // n * n + n}', the nearest batch sizes evenly divisible by {n}." + ) + space = " " * (len(s) + 1) + for i, d in enumerate(devices): + p = torch.cuda.get_device_properties(i) + s += f"{'' if i == 0 else space}CUDA:{d} ({p.name}, {p.total_memory / (1 << 20):.0f}MiB)\n" # bytes to MB + arg = "cuda:0" + elif mps and TORCH_2_0 and torch.backends.mps.is_available(): + # Prefer MPS if available + s += f"MPS ({get_cpu_info()})\n" + arg = "mps" + else: # revert to CPU + s += f"CPU ({get_cpu_info()})\n" + arg = "cpu" + + if verbose: + LOGGER.info(s if newline else s.rstrip()) + return torch.device(arg) + + +def time_sync(): + """PyTorch-accurate time.""" + if torch.cuda.is_available(): + torch.cuda.synchronize() + return time.time() + + +def fuse_conv_and_bn(conv, bn): + """Fuse Conv2d() and BatchNorm2d() layers https://tehnokv.com/posts/fusing-batchnorm-and-conv/.""" + fusedconv = ( + nn.Conv2d( + conv.in_channels, + conv.out_channels, + kernel_size=conv.kernel_size, + stride=conv.stride, + padding=conv.padding, + dilation=conv.dilation, + groups=conv.groups, + bias=True, + ) + .requires_grad_(False) + .to(conv.weight.device) + ) + + # Prepare filters + w_conv = conv.weight.clone().view(conv.out_channels, -1) + w_bn = torch.diag(bn.weight.div(torch.sqrt(bn.eps + bn.running_var))) + fusedconv.weight.copy_(torch.mm(w_bn, w_conv).view(fusedconv.weight.shape)) + + # Prepare spatial bias + b_conv = torch.zeros(conv.weight.shape[0], device=conv.weight.device) if conv.bias is None else conv.bias + b_bn = bn.bias - bn.weight.mul(bn.running_mean).div(torch.sqrt(bn.running_var + bn.eps)) + fusedconv.bias.copy_(torch.mm(w_bn, b_conv.reshape(-1, 1)).reshape(-1) + b_bn) + + return fusedconv + + +def fuse_deconv_and_bn(deconv, bn): + """Fuse ConvTranspose2d() and BatchNorm2d() layers.""" + fuseddconv = ( + nn.ConvTranspose2d( + deconv.in_channels, + deconv.out_channels, + kernel_size=deconv.kernel_size, + stride=deconv.stride, + padding=deconv.padding, + output_padding=deconv.output_padding, + dilation=deconv.dilation, + groups=deconv.groups, + bias=True, + ) + .requires_grad_(False) + .to(deconv.weight.device) + ) + + # Prepare filters + w_deconv = deconv.weight.clone().view(deconv.out_channels, -1) + w_bn = torch.diag(bn.weight.div(torch.sqrt(bn.eps + bn.running_var))) + fuseddconv.weight.copy_(torch.mm(w_bn, w_deconv).view(fuseddconv.weight.shape)) + + # Prepare spatial bias + b_conv = torch.zeros(deconv.weight.shape[1], device=deconv.weight.device) if deconv.bias is None else deconv.bias + b_bn = bn.bias - bn.weight.mul(bn.running_mean).div(torch.sqrt(bn.running_var + bn.eps)) + fuseddconv.bias.copy_(torch.mm(w_bn, b_conv.reshape(-1, 1)).reshape(-1) + b_bn) + + return fuseddconv + + +def model_info(model, detailed=False, verbose=True, imgsz=640): + """ + Model information. + + imgsz may be int or list, i.e. imgsz=640 or imgsz=[640, 320]. + """ + if not verbose: + return + n_p = get_num_params(model) # number of parameters + n_g = get_num_gradients(model) # number of gradients + n_l = len(list(model.modules())) # number of layers + if detailed: + LOGGER.info( + f"{'layer':>5} {'name':>40} {'gradient':>9} {'parameters':>12} {'shape':>20} {'mu':>10} {'sigma':>10}" + ) + for i, (name, p) in enumerate(model.named_parameters()): + name = name.replace("module_list.", "") + LOGGER.info( + "%5g %40s %9s %12g %20s %10.3g %10.3g %10s" + % (i, name, p.requires_grad, p.numel(), list(p.shape), p.mean(), p.std(), p.dtype) + ) + + flops = get_flops(model, imgsz) + fused = " (fused)" if getattr(model, "is_fused", lambda: False)() else "" + fs = f", {flops:.1f} GFLOPs" if flops else "" + yaml_file = getattr(model, "yaml_file", "") or getattr(model, "yaml", {}).get("yaml_file", "") + model_name = Path(yaml_file).stem.replace("yolo", "YOLO") or "Model" + LOGGER.info(f"{model_name} summary{fused}: {n_l} layers, {n_p} parameters, {n_g} gradients{fs}") + return n_l, n_p, n_g, flops + + +def get_num_params(model): + """Return the total number of parameters in a YOLO model.""" + return sum(x.numel() for x in model.parameters()) + + +def get_num_gradients(model): + """Return the total number of parameters with gradients in a YOLO model.""" + return sum(x.numel() for x in model.parameters() if x.requires_grad) + + +def model_info_for_loggers(trainer): + """ + Return model info dict with useful model information. + + Example: + YOLOv8n info for loggers + ```python + results = {'model/parameters': 3151904, + 'model/GFLOPs': 8.746, + 'model/speed_ONNX(ms)': 41.244, + 'model/speed_TensorRT(ms)': 3.211, + 'model/speed_PyTorch(ms)': 18.755} + ``` + """ + if trainer.args.profile: # profile ONNX and TensorRT times + from ultralytics.utils.benchmarks import ProfileModels + + results = ProfileModels([trainer.last], device=trainer.device).profile()[0] + results.pop("model/name") + else: # only return PyTorch times from most recent validation + results = { + "model/parameters": get_num_params(trainer.model), + "model/GFLOPs": round(get_flops(trainer.model), 3), + } + results["model/speed_PyTorch(ms)"] = round(trainer.validator.speed["inference"], 3) + return results + + +def get_flops(model, imgsz=640): + """Return a YOLO model's FLOPs.""" + if not thop: + return 0.0 # if not installed return 0.0 GFLOPs + + try: + model = de_parallel(model) + p = next(model.parameters()) + if not isinstance(imgsz, list): + imgsz = [imgsz, imgsz] # expand if int/float + try: + # Use stride size for input tensor + # stride = max(int(model.stride.max()), 32) if hasattr(model, "stride") else 32 # max stride + # im = torch.empty((1, p.shape[1], stride, stride), device=p.device) # input image in BCHW format + # flops = thop.profile(deepcopy(model), inputs=[im], verbose=False)[0] / 1e9 * 2 # stride GFLOPs + # return flops * imgsz[0] / stride * imgsz[1] / stride # imgsz GFLOPs + raise Exception + except Exception: + # Use actual image size for input tensor (i.e. required for RTDETR models) + im = torch.empty((1, p.shape[1], *imgsz), device=p.device) # input image in BCHW format + return thop.profile(deepcopy(model), inputs=[im], verbose=False)[0] / 1e9 * 2 # imgsz GFLOPs + except Exception: + return 0.0 + + +def get_flops_with_torch_profiler(model, imgsz=640): + """Compute model FLOPs (thop alternative).""" + if TORCH_2_0: + model = de_parallel(model) + p = next(model.parameters()) + stride = (max(int(model.stride.max()), 32) if hasattr(model, "stride") else 32) * 2 # max stride + im = torch.zeros((1, p.shape[1], stride, stride), device=p.device) # input image in BCHW format + with torch.profiler.profile(with_flops=True) as prof: + model(im) + flops = sum(x.flops for x in prof.key_averages()) / 1e9 + imgsz = imgsz if isinstance(imgsz, list) else [imgsz, imgsz] # expand if int/float + flops = flops * imgsz[0] / stride * imgsz[1] / stride # 640x640 GFLOPs + return flops + return 0 + + +def initialize_weights(model): + """Initialize model weights to random values.""" + for m in model.modules(): + t = type(m) + if t is nn.Conv2d: + pass # nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') + elif t is nn.BatchNorm2d: + m.eps = 1e-3 + m.momentum = 0.03 + elif t in [nn.Hardswish, nn.LeakyReLU, nn.ReLU, nn.ReLU6, nn.SiLU]: + m.inplace = True + + +def scale_img(img, ratio=1.0, same_shape=False, gs=32): + """Scales and pads an image tensor of shape img(bs,3,y,x) based on given ratio and grid size gs, optionally + retaining the original shape. + """ + if ratio == 1.0: + return img + h, w = img.shape[2:] + s = (int(h * ratio), int(w * ratio)) # new size + img = F.interpolate(img, size=s, mode="bilinear", align_corners=False) # resize + if not same_shape: # pad/crop img + h, w = (math.ceil(x * ratio / gs) * gs for x in (h, w)) + return F.pad(img, [0, w - s[1], 0, h - s[0]], value=0.447) # value = imagenet mean + + +def make_divisible(x, divisor): + """Returns nearest x divisible by divisor.""" + if isinstance(divisor, torch.Tensor): + divisor = int(divisor.max()) # to int + return math.ceil(x / divisor) * divisor + + +def copy_attr(a, b, include=(), exclude=()): + """Copies attributes from object 'b' to object 'a', with options to include/exclude certain attributes.""" + for k, v in b.__dict__.items(): + if (len(include) and k not in include) or k.startswith("_") or k in exclude: + continue + else: + setattr(a, k, v) + + +def get_latest_opset(): + """Return second-most (for maturity) recently supported ONNX opset by this version of torch.""" + return max(int(k[14:]) for k in vars(torch.onnx) if "symbolic_opset" in k) - 1 # opset + + +def intersect_dicts(da, db, exclude=()): + """Returns a dictionary of intersecting keys with matching shapes, excluding 'exclude' keys, using da values.""" + return {k: v for k, v in da.items() if k in db and all(x not in k for x in exclude) and v.shape == db[k].shape} + + +def is_parallel(model): + """Returns True if model is of type DP or DDP.""" + return isinstance(model, (nn.parallel.DataParallel, nn.parallel.DistributedDataParallel)) + + +def de_parallel(model): + """De-parallelize a model: returns single-GPU model if model is of type DP or DDP.""" + return model.module if is_parallel(model) else model + + +def one_cycle(y1=0.0, y2=1.0, steps=100): + """Returns a lambda function for sinusoidal ramp from y1 to y2 https://arxiv.org/pdf/1812.01187.pdf.""" + return lambda x: max((1 - math.cos(x * math.pi / steps)) / 2, 0) * (y2 - y1) + y1 + + +def init_seeds(seed=0, deterministic=False): + """Initialize random number generator (RNG) seeds https://pytorch.org/docs/stable/notes/randomness.html.""" + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) # for Multi-GPU, exception safe + # torch.backends.cudnn.benchmark = True # AutoBatch problem https://github.com/ultralytics/yolov5/issues/9287 + if deterministic: + if TORCH_2_0: + torch.use_deterministic_algorithms(True, warn_only=True) # warn if deterministic is not possible + torch.backends.cudnn.deterministic = True + os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" + os.environ["PYTHONHASHSEED"] = str(seed) + else: + LOGGER.warning("WARNING ⚠️ Upgrade to torch>=2.0.0 for deterministic training.") + else: + torch.use_deterministic_algorithms(False) + torch.backends.cudnn.deterministic = False + + +class ModelEMA: + """Updated Exponential Moving Average (EMA) from https://github.com/rwightman/pytorch-image-models + Keeps a moving average of everything in the model state_dict (parameters and buffers) + For EMA details see https://www.tensorflow.org/api_docs/python/tf/train/ExponentialMovingAverage + To disable EMA set the `enabled` attribute to `False`. + """ + + def __init__(self, model, decay=0.9999, tau=2000, updates=0): + """Create EMA.""" + self.ema = deepcopy(de_parallel(model)).eval() # FP32 EMA + self.updates = updates # number of EMA updates + self.decay = lambda x: decay * (1 - math.exp(-x / tau)) # decay exponential ramp (to help early epochs) + for p in self.ema.parameters(): + p.requires_grad_(False) + self.enabled = True + + def update(self, model): + """Update EMA parameters.""" + if self.enabled: + self.updates += 1 + d = self.decay(self.updates) + + msd = de_parallel(model).state_dict() # model state_dict + for k, v in self.ema.state_dict().items(): + if v.dtype.is_floating_point: # true for FP16 and FP32 + v *= d + v += (1 - d) * msd[k].detach() + # assert v.dtype == msd[k].dtype == torch.float32, f'{k}: EMA {v.dtype}, model {msd[k].dtype}' + + def update_attr(self, model, include=(), exclude=("process_group", "reducer")): + """Updates attributes and saves stripped model with optimizer removed.""" + if self.enabled: + copy_attr(self.ema, model, include, exclude) + + +def strip_optimizer(f: Union[str, Path] = "best.pt", s: str = "") -> None: + """ + Strip optimizer from 'f' to finalize training, optionally save as 's'. + + Args: + f (str): file path to model to strip the optimizer from. Default is 'best.pt'. + s (str): file path to save the model with stripped optimizer to. If not provided, 'f' will be overwritten. + + Returns: + None + + Example: + ```python + from pathlib import Path + from ultralytics.utils.torch_utils import strip_optimizer + + for f in Path('path/to/weights').rglob('*.pt'): + strip_optimizer(f) + ``` + """ + x = torch.load(f, map_location=torch.device("cpu")) + if "model" not in x: + LOGGER.info(f"Skipping {f}, not a valid Ultralytics model.") + return + + if hasattr(x["model"], "args"): + x["model"].args = dict(x["model"].args) # convert from IterableSimpleNamespace to dict + args = {**DEFAULT_CFG_DICT, **x["train_args"]} if "train_args" in x else None # combine args + if x.get("ema"): + x["model"] = x["ema"] # replace model with ema + for k in "optimizer", "best_fitness", "ema", "updates": # keys + x[k] = None + x["epoch"] = -1 + x["model"].half() # to FP16 + for p in x["model"].parameters(): + p.requires_grad = False + x["train_args"] = {k: v for k, v in args.items() if k in DEFAULT_CFG_KEYS} # strip non-default keys + # x['model'].args = x['train_args'] + torch.save(x, s or f) + mb = os.path.getsize(s or f) / 1e6 # file size + LOGGER.info(f"Optimizer stripped from {f},{f' saved as {s},' if s else ''} {mb:.1f}MB") + + +def profile(input, ops, n=10, device=None): + """ + Ultralytics speed, memory and FLOPs profiler. + + Example: + ```python + from ultralytics.utils.torch_utils import profile + + input = torch.randn(16, 3, 640, 640) + m1 = lambda x: x * torch.sigmoid(x) + m2 = nn.SiLU() + profile(input, [m1, m2], n=100) # profile over 100 iterations + ``` + """ + results = [] + if not isinstance(device, torch.device): + device = select_device(device) + LOGGER.info( + f"{'Params':>12s}{'GFLOPs':>12s}{'GPU_mem (GB)':>14s}{'forward (ms)':>14s}{'backward (ms)':>14s}" + f"{'input':>24s}{'output':>24s}" + ) + + for x in input if isinstance(input, list) else [input]: + x = x.to(device) + x.requires_grad = True + for m in ops if isinstance(ops, list) else [ops]: + m = m.to(device) if hasattr(m, "to") else m # device + m = m.half() if hasattr(m, "half") and isinstance(x, torch.Tensor) and x.dtype is torch.float16 else m + tf, tb, t = 0, 0, [0, 0, 0] # dt forward, backward + try: + flops = thop.profile(m, inputs=[x], verbose=False)[0] / 1e9 * 2 if thop else 0 # GFLOPs + except Exception: + flops = 0 + + try: + for _ in range(n): + t[0] = time_sync() + y = m(x) + t[1] = time_sync() + try: + (sum(yi.sum() for yi in y) if isinstance(y, list) else y).sum().backward() + t[2] = time_sync() + except Exception: # no backward method + # print(e) # for debug + t[2] = float("nan") + tf += (t[1] - t[0]) * 1000 / n # ms per op forward + tb += (t[2] - t[1]) * 1000 / n # ms per op backward + mem = torch.cuda.memory_reserved() / 1e9 if torch.cuda.is_available() else 0 # (GB) + s_in, s_out = (tuple(x.shape) if isinstance(x, torch.Tensor) else "list" for x in (x, y)) # shapes + p = sum(x.numel() for x in m.parameters()) if isinstance(m, nn.Module) else 0 # parameters + LOGGER.info(f"{p:12}{flops:12.4g}{mem:>14.3f}{tf:14.4g}{tb:14.4g}{str(s_in):>24s}{str(s_out):>24s}") + results.append([p, flops, mem, tf, tb, s_in, s_out]) + except Exception as e: + LOGGER.info(e) + results.append(None) + torch.cuda.empty_cache() + return results + + +class EarlyStopping: + """Early stopping class that stops training when a specified number of epochs have passed without improvement.""" + + def __init__(self, patience=50): + """ + Initialize early stopping object. + + Args: + patience (int, optional): Number of epochs to wait after fitness stops improving before stopping. + """ + self.best_fitness = 0.0 # i.e. mAP + self.best_epoch = 0 + self.patience = patience or float("inf") # epochs to wait after fitness stops improving to stop + self.possible_stop = False # possible stop may occur next epoch + + def __call__(self, epoch, fitness): + """ + Check whether to stop training. + + Args: + epoch (int): Current epoch of training + fitness (float): Fitness value of current epoch + + Returns: + (bool): True if training should stop, False otherwise + """ + if fitness is None: # check if fitness=None (happens when val=False) + return False + + if fitness >= self.best_fitness: # >= 0 to allow for early zero-fitness stage of training + self.best_epoch = epoch + self.best_fitness = fitness + delta = epoch - self.best_epoch # epochs without improvement + self.possible_stop = delta >= (self.patience - 1) # possible stop may occur next epoch + stop = delta >= self.patience # stop training if patience exceeded + if stop: + LOGGER.info( + f"Stopping training early as no improvement observed in last {self.patience} epochs. " + f"Best results observed at epoch {self.best_epoch}, best model saved as best.pt.\n" + f"To update EarlyStopping(patience={self.patience}) pass a new patience value, " + f"i.e. `patience=300` or use `patience=0` to disable EarlyStopping." + ) + return stop diff --git a/ultralytics/utils/triton.py b/ultralytics/utils/triton.py new file mode 100644 index 0000000..3f873a6 --- /dev/null +++ b/ultralytics/utils/triton.py @@ -0,0 +1,92 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +from typing import List +from urllib.parse import urlsplit + +import numpy as np + + +class TritonRemoteModel: + """ + Client for interacting with a remote Triton Inference Server model. + + Attributes: + endpoint (str): The name of the model on the Triton server. + url (str): The URL of the Triton server. + triton_client: The Triton client (either HTTP or gRPC). + InferInput: The input class for the Triton client. + InferRequestedOutput: The output request class for the Triton client. + input_formats (List[str]): The data types of the model inputs. + np_input_formats (List[type]): The numpy data types of the model inputs. + input_names (List[str]): The names of the model inputs. + output_names (List[str]): The names of the model outputs. + """ + + def __init__(self, url: str, endpoint: str = "", scheme: str = ""): + """ + Initialize the TritonRemoteModel. + + Arguments may be provided individually or parsed from a collective 'url' argument of the form + ://// + + Args: + url (str): The URL of the Triton server. + endpoint (str): The name of the model on the Triton server. + scheme (str): The communication scheme ('http' or 'grpc'). + """ + if not endpoint and not scheme: # Parse all args from URL string + splits = urlsplit(url) + endpoint = splits.path.strip("/").split("/")[0] + scheme = splits.scheme + url = splits.netloc + + self.endpoint = endpoint + self.url = url + + # Choose the Triton client based on the communication scheme + if scheme == "http": + import tritonclient.http as client # noqa + + self.triton_client = client.InferenceServerClient(url=self.url, verbose=False, ssl=False) + config = self.triton_client.get_model_config(endpoint) + else: + import tritonclient.grpc as client # noqa + + self.triton_client = client.InferenceServerClient(url=self.url, verbose=False, ssl=False) + config = self.triton_client.get_model_config(endpoint, as_json=True)["config"] + + # Sort output names alphabetically, i.e. 'output0', 'output1', etc. + config["output"] = sorted(config["output"], key=lambda x: x.get("name")) + + # Define model attributes + type_map = {"TYPE_FP32": np.float32, "TYPE_FP16": np.float16, "TYPE_UINT8": np.uint8} + self.InferRequestedOutput = client.InferRequestedOutput + self.InferInput = client.InferInput + self.input_formats = [x["data_type"] for x in config["input"]] + self.np_input_formats = [type_map[x] for x in self.input_formats] + self.input_names = [x["name"] for x in config["input"]] + self.output_names = [x["name"] for x in config["output"]] + + def __call__(self, *inputs: np.ndarray) -> List[np.ndarray]: + """ + Call the model with the given inputs. + + Args: + *inputs (List[np.ndarray]): Input data to the model. + + Returns: + (List[np.ndarray]): Model outputs. + """ + infer_inputs = [] + input_format = inputs[0].dtype + for i, x in enumerate(inputs): + if x.dtype != self.np_input_formats[i]: + x = x.astype(self.np_input_formats[i]) + infer_input = self.InferInput(self.input_names[i], [*x.shape], self.input_formats[i].replace("TYPE_", "")) + infer_input.set_data_from_numpy(x) + infer_inputs.append(infer_input) + + infer_outputs = [self.InferRequestedOutput(output_name) for output_name in self.output_names] + outputs = self.triton_client.infer(model_name=self.endpoint, inputs=infer_inputs, outputs=infer_outputs) + + return [outputs.as_numpy(output_name).astype(input_format) for output_name in self.output_names] diff --git a/ultralytics/utils/tuner.py b/ultralytics/utils/tuner.py new file mode 100644 index 0000000..305c60a --- /dev/null +++ b/ultralytics/utils/tuner.py @@ -0,0 +1,148 @@ +# Ultralytics YOLO 🚀, AGPL-3.0 license + +import subprocess + +from ultralytics.cfg import TASK2DATA, TASK2METRIC, get_save_dir +from ultralytics.utils import DEFAULT_CFG, DEFAULT_CFG_DICT, LOGGER, NUM_THREADS, checks + + +def run_ray_tune( + model, space: dict = None, grace_period: int = 10, gpu_per_trial: int = None, max_samples: int = 10, **train_args +): + """ + Runs hyperparameter tuning using Ray Tune. + + Args: + model (YOLO): Model to run the tuner on. + space (dict, optional): The hyperparameter search space. Defaults to None. + grace_period (int, optional): The grace period in epochs of the ASHA scheduler. Defaults to 10. + gpu_per_trial (int, optional): The number of GPUs to allocate per trial. Defaults to None. + max_samples (int, optional): The maximum number of trials to run. Defaults to 10. + train_args (dict, optional): Additional arguments to pass to the `train()` method. Defaults to {}. + + Returns: + (dict): A dictionary containing the results of the hyperparameter search. + + Example: + ```python + from ultralytics import YOLO + + # Load a YOLOv8n model + model = YOLO('yolov8n.pt') + + # Start tuning hyperparameters for YOLOv8n training on the COCO8 dataset + result_grid = model.tune(data='coco8.yaml', use_ray=True) + ``` + """ + + LOGGER.info("💡 Learn about RayTune at https://docs.ultralytics.com/integrations/ray-tune") + if train_args is None: + train_args = {} + + try: + subprocess.run("pip install ray[tune]<=2.9.3".split(), check=True) # do not add single quotes here + + import ray + from ray import tune + from ray.air import RunConfig + from ray.air.integrations.wandb import WandbLoggerCallback + from ray.tune.schedulers import ASHAScheduler + except ImportError: + raise ModuleNotFoundError('Ray Tune required but not found. To install run: pip install "ray[tune]<=2.9.3"') + + try: + import wandb + + assert hasattr(wandb, "__version__") + except (ImportError, AssertionError): + wandb = False + + checks.check_version(ray.__version__, "<=2.9.3", "ray") + default_space = { + # 'optimizer': tune.choice(['SGD', 'Adam', 'AdamW', 'NAdam', 'RAdam', 'RMSProp']), + "lr0": tune.uniform(1e-5, 1e-1), + "lrf": tune.uniform(0.01, 1.0), # final OneCycleLR learning rate (lr0 * lrf) + "momentum": tune.uniform(0.6, 0.98), # SGD momentum/Adam beta1 + "weight_decay": tune.uniform(0.0, 0.001), # optimizer weight decay 5e-4 + "warmup_epochs": tune.uniform(0.0, 5.0), # warmup epochs (fractions ok) + "warmup_momentum": tune.uniform(0.0, 0.95), # warmup initial momentum + "box": tune.uniform(0.02, 0.2), # box loss gain + "cls": tune.uniform(0.2, 4.0), # cls loss gain (scale with pixels) + "hsv_h": tune.uniform(0.0, 0.1), # image HSV-Hue augmentation (fraction) + "hsv_s": tune.uniform(0.0, 0.9), # image HSV-Saturation augmentation (fraction) + "hsv_v": tune.uniform(0.0, 0.9), # image HSV-Value augmentation (fraction) + "degrees": tune.uniform(0.0, 45.0), # image rotation (+/- deg) + "translate": tune.uniform(0.0, 0.9), # image translation (+/- fraction) + "scale": tune.uniform(0.0, 0.9), # image scale (+/- gain) + "shear": tune.uniform(0.0, 10.0), # image shear (+/- deg) + "perspective": tune.uniform(0.0, 0.001), # image perspective (+/- fraction), range 0-0.001 + "flipud": tune.uniform(0.0, 1.0), # image flip up-down (probability) + "fliplr": tune.uniform(0.0, 1.0), # image flip left-right (probability) + "bgr": tune.uniform(0.0, 1.0), # image channel BGR (probability) + "mosaic": tune.uniform(0.0, 1.0), # image mixup (probability) + "mixup": tune.uniform(0.0, 1.0), # image mixup (probability) + "copy_paste": tune.uniform(0.0, 1.0), # segment copy-paste (probability) + } + + # Put the model in ray store + task = model.task + model_in_store = ray.put(model) + + def _tune(config): + """ + Trains the YOLO model with the specified hyperparameters and additional arguments. + + Args: + config (dict): A dictionary of hyperparameters to use for training. + + Returns: + None + """ + model_to_train = ray.get(model_in_store) # get the model from ray store for tuning + model_to_train.reset_callbacks() + config.update(train_args) + results = model_to_train.train(**config) + return results.results_dict + + # Get search space + if not space: + space = default_space + LOGGER.warning("WARNING ⚠️ search space not provided, using default search space.") + + # Get dataset + data = train_args.get("data", TASK2DATA[task]) + space["data"] = data + if "data" not in train_args: + LOGGER.warning(f'WARNING ⚠️ data not provided, using default "data={data}".') + + # Define the trainable function with allocated resources + trainable_with_resources = tune.with_resources(_tune, {"cpu": NUM_THREADS, "gpu": gpu_per_trial or 0}) + + # Define the ASHA scheduler for hyperparameter search + asha_scheduler = ASHAScheduler( + time_attr="epoch", + metric=TASK2METRIC[task], + mode="max", + max_t=train_args.get("epochs") or DEFAULT_CFG_DICT["epochs"] or 100, + grace_period=grace_period, + reduction_factor=3, + ) + + # Define the callbacks for the hyperparameter search + tuner_callbacks = [WandbLoggerCallback(project="YOLOv8-tune")] if wandb else [] + + # Create the Ray Tune hyperparameter search tuner + tune_dir = get_save_dir(DEFAULT_CFG, name="tune").resolve() # must be absolute dir + tune_dir.mkdir(parents=True, exist_ok=True) + tuner = tune.Tuner( + trainable_with_resources, + param_space=space, + tune_config=tune.TuneConfig(scheduler=asha_scheduler, num_samples=max_samples), + run_config=RunConfig(callbacks=tuner_callbacks, storage_path=tune_dir), + ) + + # Run the hyperparameter search + tuner.fit() + + # Return the results of the hyperparameter search + return tuner.get_results() diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000..d33b924 --- /dev/null +++ b/utils/__init__.py @@ -0,0 +1,86 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +utils/initialization +""" + +import contextlib +import platform +import threading + + +def emojis(str=''): + # Return platform-dependent emoji-safe version of string + return str.encode().decode('ascii', 'ignore') if platform.system() == 'Windows' else str + + +class TryExcept(contextlib.ContextDecorator): + # YOLOv5 TryExcept class. Usage: @TryExcept() decorator or 'with TryExcept():' context manager + def __init__(self, msg=''): + self.msg = msg + + def __enter__(self): + pass + + def __exit__(self, exc_type, value, traceback): + if value: + print(emojis(f"{self.msg}{': ' if self.msg else ''}{value}")) + return True + + +def threaded(func): + # Multi-threads a target function and returns thread. Usage: @threaded decorator + def wrapper(*args, **kwargs): + thread = threading.Thread(target=func, args=args, kwargs=kwargs, daemon=True) + thread.start() + return thread + + return wrapper + + +def join_threads(verbose=False): + # Join all daemon threads, i.e. atexit.register(lambda: join_threads()) + main_thread = threading.current_thread() + for t in threading.enumerate(): + if t is not main_thread: + if verbose: + print(f'Joining thread {t.name}') + t.join() + + +def notebook_init(verbose=True): + # Check system software and hardware + print('Checking setup...') + + import os + import shutil + + from ultralytics.utils.checks import check_requirements + + from utils.general import check_font, is_colab + from utils.torch_utils import select_device # imports + + check_font() + + import psutil + + if check_requirements('wandb', install=False): + os.system('pip uninstall -y wandb') # eliminate unexpected account creation prompt with infinite hang + if is_colab(): + shutil.rmtree('/content/sample_data', ignore_errors=True) # remove colab /sample_data directory + + # System info + display = None + if verbose: + gb = 1 << 30 # bytes to GiB (1024 ** 3) + ram = psutil.virtual_memory().total + total, used, free = shutil.disk_usage('/') + with contextlib.suppress(Exception): # clear display if ipython is installed + from IPython import display + display.clear_output() + s = f'({os.cpu_count()} CPUs, {ram / gb:.1f} GB RAM, {(total - free) / gb:.1f}/{total / gb:.1f} GB disk)' + else: + s = '' + + select_device(newline=False) + print(emojis(f'Setup complete ✅ {s}')) + return display diff --git a/utils/__pycache__/__init__.cpython-312.pyc b/utils/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..13ea5e2 Binary files /dev/null and b/utils/__pycache__/__init__.cpython-312.pyc differ diff --git a/utils/__pycache__/__init__.cpython-39.pyc b/utils/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000..c4c6f4e Binary files /dev/null and b/utils/__pycache__/__init__.cpython-39.pyc differ diff --git a/utils/__pycache__/augmentations.cpython-312.pyc b/utils/__pycache__/augmentations.cpython-312.pyc new file mode 100644 index 0000000..3e1b3bf Binary files /dev/null and b/utils/__pycache__/augmentations.cpython-312.pyc differ diff --git a/utils/__pycache__/augmentations.cpython-39.pyc b/utils/__pycache__/augmentations.cpython-39.pyc new file mode 100644 index 0000000..54230c0 Binary files /dev/null and b/utils/__pycache__/augmentations.cpython-39.pyc differ diff --git a/utils/__pycache__/autoanchor.cpython-312.pyc b/utils/__pycache__/autoanchor.cpython-312.pyc new file mode 100644 index 0000000..77bd54c Binary files /dev/null and b/utils/__pycache__/autoanchor.cpython-312.pyc differ diff --git a/utils/__pycache__/autoanchor.cpython-39.pyc b/utils/__pycache__/autoanchor.cpython-39.pyc new file mode 100644 index 0000000..52d4743 Binary files /dev/null and b/utils/__pycache__/autoanchor.cpython-39.pyc differ diff --git a/utils/__pycache__/dataloaders.cpython-312.pyc b/utils/__pycache__/dataloaders.cpython-312.pyc new file mode 100644 index 0000000..f7f5f66 Binary files /dev/null and b/utils/__pycache__/dataloaders.cpython-312.pyc differ diff --git a/utils/__pycache__/dataloaders.cpython-39.pyc b/utils/__pycache__/dataloaders.cpython-39.pyc new file mode 100644 index 0000000..8da62af Binary files /dev/null and b/utils/__pycache__/dataloaders.cpython-39.pyc differ diff --git a/utils/__pycache__/downloads.cpython-312.pyc b/utils/__pycache__/downloads.cpython-312.pyc new file mode 100644 index 0000000..43ebc3e Binary files /dev/null and b/utils/__pycache__/downloads.cpython-312.pyc differ diff --git a/utils/__pycache__/downloads.cpython-39.pyc b/utils/__pycache__/downloads.cpython-39.pyc new file mode 100644 index 0000000..425628a Binary files /dev/null and b/utils/__pycache__/downloads.cpython-39.pyc differ diff --git a/utils/__pycache__/general.cpython-312.pyc b/utils/__pycache__/general.cpython-312.pyc new file mode 100644 index 0000000..9c0953e Binary files /dev/null and b/utils/__pycache__/general.cpython-312.pyc differ diff --git a/utils/__pycache__/general.cpython-39.pyc b/utils/__pycache__/general.cpython-39.pyc new file mode 100644 index 0000000..5a39f84 Binary files /dev/null and b/utils/__pycache__/general.cpython-39.pyc differ diff --git a/utils/__pycache__/getsource.cpython-312.pyc b/utils/__pycache__/getsource.cpython-312.pyc new file mode 100644 index 0000000..a0e5280 Binary files /dev/null and b/utils/__pycache__/getsource.cpython-312.pyc differ diff --git a/utils/__pycache__/getsource.cpython-39.pyc b/utils/__pycache__/getsource.cpython-39.pyc new file mode 100644 index 0000000..51e6471 Binary files /dev/null and b/utils/__pycache__/getsource.cpython-39.pyc differ diff --git a/utils/__pycache__/metrics.cpython-312.pyc b/utils/__pycache__/metrics.cpython-312.pyc new file mode 100644 index 0000000..98032a5 Binary files /dev/null and b/utils/__pycache__/metrics.cpython-312.pyc differ diff --git a/utils/__pycache__/metrics.cpython-39.pyc b/utils/__pycache__/metrics.cpython-39.pyc new file mode 100644 index 0000000..6254c35 Binary files /dev/null and b/utils/__pycache__/metrics.cpython-39.pyc differ diff --git a/utils/__pycache__/plots.cpython-312.pyc b/utils/__pycache__/plots.cpython-312.pyc new file mode 100644 index 0000000..6863a11 Binary files /dev/null and b/utils/__pycache__/plots.cpython-312.pyc differ diff --git a/utils/__pycache__/plots.cpython-39.pyc b/utils/__pycache__/plots.cpython-39.pyc new file mode 100644 index 0000000..9e0e6fe Binary files /dev/null and b/utils/__pycache__/plots.cpython-39.pyc differ diff --git a/utils/__pycache__/torch_utils.cpython-312.pyc b/utils/__pycache__/torch_utils.cpython-312.pyc new file mode 100644 index 0000000..deea80d Binary files /dev/null and b/utils/__pycache__/torch_utils.cpython-312.pyc differ diff --git a/utils/__pycache__/torch_utils.cpython-39.pyc b/utils/__pycache__/torch_utils.cpython-39.pyc new file mode 100644 index 0000000..be458cb Binary files /dev/null and b/utils/__pycache__/torch_utils.cpython-39.pyc differ diff --git a/utils/__pycache__/triton.cpython-312.pyc b/utils/__pycache__/triton.cpython-312.pyc new file mode 100644 index 0000000..a7b2aa5 Binary files /dev/null and b/utils/__pycache__/triton.cpython-312.pyc differ diff --git a/utils/activations.py b/utils/activations.py new file mode 100644 index 0000000..e4d4bbd --- /dev/null +++ b/utils/activations.py @@ -0,0 +1,103 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Activation functions +""" + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class SiLU(nn.Module): + # SiLU activation https://arxiv.org/pdf/1606.08415.pdf + @staticmethod + def forward(x): + return x * torch.sigmoid(x) + + +class Hardswish(nn.Module): + # Hard-SiLU activation + @staticmethod + def forward(x): + # return x * F.hardsigmoid(x) # for TorchScript and CoreML + return x * F.hardtanh(x + 3, 0.0, 6.0) / 6.0 # for TorchScript, CoreML and ONNX + + +class Mish(nn.Module): + # Mish activation https://github.com/digantamisra98/Mish + @staticmethod + def forward(x): + return x * F.softplus(x).tanh() + + +class MemoryEfficientMish(nn.Module): + # Mish activation memory-efficient + class F(torch.autograd.Function): + + @staticmethod + def forward(ctx, x): + ctx.save_for_backward(x) + return x.mul(torch.tanh(F.softplus(x))) # x * tanh(ln(1 + exp(x))) + + @staticmethod + def backward(ctx, grad_output): + x = ctx.saved_tensors[0] + sx = torch.sigmoid(x) + fx = F.softplus(x).tanh() + return grad_output * (fx + x * sx * (1 - fx * fx)) + + def forward(self, x): + return self.F.apply(x) + + +class FReLU(nn.Module): + # FReLU activation https://arxiv.org/abs/2007.11824 + def __init__(self, c1, k=3): # ch_in, kernel + super().__init__() + self.conv = nn.Conv2d(c1, c1, k, 1, 1, groups=c1, bias=False) + self.bn = nn.BatchNorm2d(c1) + + def forward(self, x): + return torch.max(x, self.bn(self.conv(x))) + + +class AconC(nn.Module): + r""" ACON activation (activate or not) + AconC: (p1*x-p2*x) * sigmoid(beta*(p1*x-p2*x)) + p2*x, beta is a learnable parameter + according to "Activate or Not: Learning Customized Activation" . + """ + + def __init__(self, c1): + super().__init__() + self.p1 = nn.Parameter(torch.randn(1, c1, 1, 1)) + self.p2 = nn.Parameter(torch.randn(1, c1, 1, 1)) + self.beta = nn.Parameter(torch.ones(1, c1, 1, 1)) + + def forward(self, x): + dpx = (self.p1 - self.p2) * x + return dpx * torch.sigmoid(self.beta * dpx) + self.p2 * x + + +class MetaAconC(nn.Module): + r""" ACON activation (activate or not) + MetaAconC: (p1*x-p2*x) * sigmoid(beta*(p1*x-p2*x)) + p2*x, beta is generated by a small network + according to "Activate or Not: Learning Customized Activation" . + """ + + def __init__(self, c1, k=1, s=1, r=16): # ch_in, kernel, stride, r + super().__init__() + c2 = max(r, c1 // r) + self.p1 = nn.Parameter(torch.randn(1, c1, 1, 1)) + self.p2 = nn.Parameter(torch.randn(1, c1, 1, 1)) + self.fc1 = nn.Conv2d(c1, c2, k, s, bias=True) + self.fc2 = nn.Conv2d(c2, c1, k, s, bias=True) + # self.bn1 = nn.BatchNorm2d(c2) + # self.bn2 = nn.BatchNorm2d(c1) + + def forward(self, x): + y = x.mean(dim=2, keepdims=True).mean(dim=3, keepdims=True) + # batch-size 1 bug/instabilities https://github.com/ultralytics/yolov5/issues/2891 + # beta = torch.sigmoid(self.bn2(self.fc2(self.bn1(self.fc1(y))))) # bug/unstable + beta = torch.sigmoid(self.fc2(self.fc1(y))) # bug patch BN layers removed + dpx = (self.p1 - self.p2) * x + return dpx * torch.sigmoid(beta * dpx) + self.p2 * x diff --git a/utils/augmentations.py b/utils/augmentations.py new file mode 100644 index 0000000..1e60930 --- /dev/null +++ b/utils/augmentations.py @@ -0,0 +1,397 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Image augmentation functions +""" + +import math +import random + +import cv2 +import numpy as np +import torch +import torchvision.transforms as T +import torchvision.transforms.functional as TF + +from utils.general import LOGGER, check_version, colorstr, resample_segments, segment2box, xywhn2xyxy +from utils.metrics import bbox_ioa + +IMAGENET_MEAN = 0.485, 0.456, 0.406 # RGB mean +IMAGENET_STD = 0.229, 0.224, 0.225 # RGB standard deviation + + +class Albumentations: + # YOLOv5 Albumentations class (optional, only used if package is installed) + def __init__(self, size=640): + self.transform = None + prefix = colorstr('albumentations: ') + try: + import albumentations as A + check_version(A.__version__, '1.0.3', hard=True) # version requirement + + T = [ + A.RandomResizedCrop(height=size, width=size, scale=(0.8, 1.0), ratio=(0.9, 1.11), p=0.0), + A.Blur(p=0.01), + A.MedianBlur(p=0.01), + A.ToGray(p=0.01), + A.CLAHE(p=0.01), + A.RandomBrightnessContrast(p=0.0), + A.RandomGamma(p=0.0), + A.ImageCompression(quality_lower=75, p=0.0)] # transforms + self.transform = A.Compose(T, bbox_params=A.BboxParams(format='yolo', label_fields=['class_labels'])) + + LOGGER.info(prefix + ', '.join(f'{x}'.replace('always_apply=False, ', '') for x in T if x.p)) + except ImportError: # package not installed, skip + pass + except Exception as e: + LOGGER.info(f'{prefix}{e}') + + def __call__(self, im, labels, p=1.0): + if self.transform and random.random() < p: + new = self.transform(image=im, bboxes=labels[:, 1:], class_labels=labels[:, 0]) # transformed + im, labels = new['image'], np.array([[c, *b] for c, b in zip(new['class_labels'], new['bboxes'])]) + return im, labels + + +def normalize(x, mean=IMAGENET_MEAN, std=IMAGENET_STD, inplace=False): + # Denormalize RGB images x per ImageNet stats in BCHW format, i.e. = (x - mean) / std + return TF.normalize(x, mean, std, inplace=inplace) + + +def denormalize(x, mean=IMAGENET_MEAN, std=IMAGENET_STD): + # Denormalize RGB images x per ImageNet stats in BCHW format, i.e. = x * std + mean + for i in range(3): + x[:, i] = x[:, i] * std[i] + mean[i] + return x + + +def augment_hsv(im, hgain=0.5, sgain=0.5, vgain=0.5): + # HSV color-space augmentation + if hgain or sgain or vgain: + r = np.random.uniform(-1, 1, 3) * [hgain, sgain, vgain] + 1 # random gains + hue, sat, val = cv2.split(cv2.cvtColor(im, cv2.COLOR_BGR2HSV)) + dtype = im.dtype # uint8 + + x = np.arange(0, 256, dtype=r.dtype) + lut_hue = ((x * r[0]) % 180).astype(dtype) + lut_sat = np.clip(x * r[1], 0, 255).astype(dtype) + lut_val = np.clip(x * r[2], 0, 255).astype(dtype) + + im_hsv = cv2.merge((cv2.LUT(hue, lut_hue), cv2.LUT(sat, lut_sat), cv2.LUT(val, lut_val))) + cv2.cvtColor(im_hsv, cv2.COLOR_HSV2BGR, dst=im) # no return needed + + +def hist_equalize(im, clahe=True, bgr=False): + # Equalize histogram on BGR image 'im' with im.shape(n,m,3) and range 0-255 + yuv = cv2.cvtColor(im, cv2.COLOR_BGR2YUV if bgr else cv2.COLOR_RGB2YUV) + if clahe: + c = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8)) + yuv[:, :, 0] = c.apply(yuv[:, :, 0]) + else: + yuv[:, :, 0] = cv2.equalizeHist(yuv[:, :, 0]) # equalize Y channel histogram + return cv2.cvtColor(yuv, cv2.COLOR_YUV2BGR if bgr else cv2.COLOR_YUV2RGB) # convert YUV image to RGB + + +def replicate(im, labels): + # Replicate labels + h, w = im.shape[:2] + boxes = labels[:, 1:].astype(int) + x1, y1, x2, y2 = boxes.T + s = ((x2 - x1) + (y2 - y1)) / 2 # side length (pixels) + for i in s.argsort()[:round(s.size * 0.5)]: # smallest indices + x1b, y1b, x2b, y2b = boxes[i] + bh, bw = y2b - y1b, x2b - x1b + yc, xc = int(random.uniform(0, h - bh)), int(random.uniform(0, w - bw)) # offset x, y + x1a, y1a, x2a, y2a = [xc, yc, xc + bw, yc + bh] + im[y1a:y2a, x1a:x2a] = im[y1b:y2b, x1b:x2b] # im4[ymin:ymax, xmin:xmax] + labels = np.append(labels, [[labels[i, 0], x1a, y1a, x2a, y2a]], axis=0) + + return im, labels + + +def letterbox(im, new_shape=(640, 640), color=(114, 114, 114), auto=True, scaleFill=False, scaleup=True, stride=32): + # Resize and pad image while meeting stride-multiple constraints + shape = im.shape[:2] # current shape [height, width] + if isinstance(new_shape, int): + new_shape = (new_shape, new_shape) + + # Scale ratio (new / old) + r = min(new_shape[0] / shape[0], new_shape[1] / shape[1]) + if not scaleup: # only scale down, do not scale up (for better val mAP) + r = min(r, 1.0) + + # Compute padding + ratio = r, r # width, height ratios + new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r)) + dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1] # wh padding + if auto: # minimum rectangle + dw, dh = np.mod(dw, stride), np.mod(dh, stride) # wh padding + elif scaleFill: # stretch + dw, dh = 0.0, 0.0 + new_unpad = (new_shape[1], new_shape[0]) + ratio = new_shape[1] / shape[1], new_shape[0] / shape[0] # width, height ratios + + dw /= 2 # divide padding into 2 sides + dh /= 2 + + if shape[::-1] != new_unpad: # resize + im = cv2.resize(im, new_unpad, interpolation=cv2.INTER_LINEAR) + top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1)) + left, right = int(round(dw - 0.1)), int(round(dw + 0.1)) + im = cv2.copyMakeBorder(im, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color) # add border + return im, ratio, (dw, dh) + + +def random_perspective(im, + targets=(), + segments=(), + degrees=10, + translate=.1, + scale=.1, + shear=10, + perspective=0.0, + border=(0, 0)): + # torchvision.transforms.RandomAffine(degrees=(-10, 10), translate=(0.1, 0.1), scale=(0.9, 1.1), shear=(-10, 10)) + # targets = [cls, xyxy] + + height = im.shape[0] + border[0] * 2 # shape(h,w,c) + width = im.shape[1] + border[1] * 2 + + # Center + C = np.eye(3) + C[0, 2] = -im.shape[1] / 2 # x translation (pixels) + C[1, 2] = -im.shape[0] / 2 # y translation (pixels) + + # Perspective + P = np.eye(3) + P[2, 0] = random.uniform(-perspective, perspective) # x perspective (about y) + P[2, 1] = random.uniform(-perspective, perspective) # y perspective (about x) + + # Rotation and Scale + R = np.eye(3) + a = random.uniform(-degrees, degrees) + # a += random.choice([-180, -90, 0, 90]) # add 90deg rotations to small rotations + s = random.uniform(1 - scale, 1 + scale) + # s = 2 ** random.uniform(-scale, scale) + R[:2] = cv2.getRotationMatrix2D(angle=a, center=(0, 0), scale=s) + + # Shear + S = np.eye(3) + S[0, 1] = math.tan(random.uniform(-shear, shear) * math.pi / 180) # x shear (deg) + S[1, 0] = math.tan(random.uniform(-shear, shear) * math.pi / 180) # y shear (deg) + + # Translation + T = np.eye(3) + T[0, 2] = random.uniform(0.5 - translate, 0.5 + translate) * width # x translation (pixels) + T[1, 2] = random.uniform(0.5 - translate, 0.5 + translate) * height # y translation (pixels) + + # Combined rotation matrix + M = T @ S @ R @ P @ C # order of operations (right to left) is IMPORTANT + if (border[0] != 0) or (border[1] != 0) or (M != np.eye(3)).any(): # image changed + if perspective: + im = cv2.warpPerspective(im, M, dsize=(width, height), borderValue=(114, 114, 114)) + else: # affine + im = cv2.warpAffine(im, M[:2], dsize=(width, height), borderValue=(114, 114, 114)) + + # Visualize + # import matplotlib.pyplot as plt + # ax = plt.subplots(1, 2, figsize=(12, 6))[1].ravel() + # ax[0].imshow(im[:, :, ::-1]) # base + # ax[1].imshow(im2[:, :, ::-1]) # warped + + # Transform label coordinates + n = len(targets) + if n: + use_segments = any(x.any() for x in segments) and len(segments) == n + new = np.zeros((n, 4)) + if use_segments: # warp segments + segments = resample_segments(segments) # upsample + for i, segment in enumerate(segments): + xy = np.ones((len(segment), 3)) + xy[:, :2] = segment + xy = xy @ M.T # transform + xy = xy[:, :2] / xy[:, 2:3] if perspective else xy[:, :2] # perspective rescale or affine + + # clip + new[i] = segment2box(xy, width, height) + + else: # warp boxes + xy = np.ones((n * 4, 3)) + xy[:, :2] = targets[:, [1, 2, 3, 4, 1, 4, 3, 2]].reshape(n * 4, 2) # x1y1, x2y2, x1y2, x2y1 + xy = xy @ M.T # transform + xy = (xy[:, :2] / xy[:, 2:3] if perspective else xy[:, :2]).reshape(n, 8) # perspective rescale or affine + + # create new boxes + x = xy[:, [0, 2, 4, 6]] + y = xy[:, [1, 3, 5, 7]] + new = np.concatenate((x.min(1), y.min(1), x.max(1), y.max(1))).reshape(4, n).T + + # clip + new[:, [0, 2]] = new[:, [0, 2]].clip(0, width) + new[:, [1, 3]] = new[:, [1, 3]].clip(0, height) + + # filter candidates + i = box_candidates(box1=targets[:, 1:5].T * s, box2=new.T, area_thr=0.01 if use_segments else 0.10) + targets = targets[i] + targets[:, 1:5] = new[i] + + return im, targets + + +def copy_paste(im, labels, segments, p=0.5): + # Implement Copy-Paste augmentation https://arxiv.org/abs/2012.07177, labels as nx5 np.array(cls, xyxy) + n = len(segments) + if p and n: + h, w, c = im.shape # height, width, channels + im_new = np.zeros(im.shape, np.uint8) + for j in random.sample(range(n), k=round(p * n)): + l, s = labels[j], segments[j] + box = w - l[3], l[2], w - l[1], l[4] + ioa = bbox_ioa(box, labels[:, 1:5]) # intersection over area + if (ioa < 0.30).all(): # allow 30% obscuration of existing labels + labels = np.concatenate((labels, [[l[0], *box]]), 0) + segments.append(np.concatenate((w - s[:, 0:1], s[:, 1:2]), 1)) + cv2.drawContours(im_new, [segments[j].astype(np.int32)], -1, (1, 1, 1), cv2.FILLED) + + result = cv2.flip(im, 1) # augment segments (flip left-right) + i = cv2.flip(im_new, 1).astype(bool) + im[i] = result[i] # cv2.imwrite('debug.jpg', im) # debug + + return im, labels, segments + + +def cutout(im, labels, p=0.5): + # Applies image cutout augmentation https://arxiv.org/abs/1708.04552 + if random.random() < p: + h, w = im.shape[:2] + scales = [0.5] * 1 + [0.25] * 2 + [0.125] * 4 + [0.0625] * 8 + [0.03125] * 16 # image size fraction + for s in scales: + mask_h = random.randint(1, int(h * s)) # create random masks + mask_w = random.randint(1, int(w * s)) + + # box + xmin = max(0, random.randint(0, w) - mask_w // 2) + ymin = max(0, random.randint(0, h) - mask_h // 2) + xmax = min(w, xmin + mask_w) + ymax = min(h, ymin + mask_h) + + # apply random color mask + im[ymin:ymax, xmin:xmax] = [random.randint(64, 191) for _ in range(3)] + + # return unobscured labels + if len(labels) and s > 0.03: + box = np.array([xmin, ymin, xmax, ymax], dtype=np.float32) + ioa = bbox_ioa(box, xywhn2xyxy(labels[:, 1:5], w, h)) # intersection over area + labels = labels[ioa < 0.60] # remove >60% obscured labels + + return labels + + +def mixup(im, labels, im2, labels2): + # Applies MixUp augmentation https://arxiv.org/pdf/1710.09412.pdf + r = np.random.beta(32.0, 32.0) # mixup ratio, alpha=beta=32.0 + im = (im * r + im2 * (1 - r)).astype(np.uint8) + labels = np.concatenate((labels, labels2), 0) + return im, labels + + +def box_candidates(box1, box2, wh_thr=2, ar_thr=100, area_thr=0.1, eps=1e-16): # box1(4,n), box2(4,n) + # Compute candidate boxes: box1 before augment, box2 after augment, wh_thr (pixels), aspect_ratio_thr, area_ratio + w1, h1 = box1[2] - box1[0], box1[3] - box1[1] + w2, h2 = box2[2] - box2[0], box2[3] - box2[1] + ar = np.maximum(w2 / (h2 + eps), h2 / (w2 + eps)) # aspect ratio + return (w2 > wh_thr) & (h2 > wh_thr) & (w2 * h2 / (w1 * h1 + eps) > area_thr) & (ar < ar_thr) # candidates + + +def classify_albumentations( + augment=True, + size=224, + scale=(0.08, 1.0), + ratio=(0.75, 1.0 / 0.75), # 0.75, 1.33 + hflip=0.5, + vflip=0.0, + jitter=0.4, + mean=IMAGENET_MEAN, + std=IMAGENET_STD, + auto_aug=False): + # YOLOv5 classification Albumentations (optional, only used if package is installed) + prefix = colorstr('albumentations: ') + try: + import albumentations as A + from albumentations.pytorch import ToTensorV2 + check_version(A.__version__, '1.0.3', hard=True) # version requirement + if augment: # Resize and crop + T = [A.RandomResizedCrop(height=size, width=size, scale=scale, ratio=ratio)] + if auto_aug: + # TODO: implement AugMix, AutoAug & RandAug in albumentation + LOGGER.info(f'{prefix}auto augmentations are currently not supported') + else: + if hflip > 0: + T += [A.HorizontalFlip(p=hflip)] + if vflip > 0: + T += [A.VerticalFlip(p=vflip)] + if jitter > 0: + color_jitter = (float(jitter), ) * 3 # repeat value for brightness, contrast, satuaration, 0 hue + T += [A.ColorJitter(*color_jitter, 0)] + else: # Use fixed crop for eval set (reproducibility) + T = [A.SmallestMaxSize(max_size=size), A.CenterCrop(height=size, width=size)] + T += [A.Normalize(mean=mean, std=std), ToTensorV2()] # Normalize and convert to Tensor + LOGGER.info(prefix + ', '.join(f'{x}'.replace('always_apply=False, ', '') for x in T if x.p)) + return A.Compose(T) + + except ImportError: # package not installed, skip + LOGGER.warning(f'{prefix}⚠️ not found, install with `pip install albumentations` (recommended)') + except Exception as e: + LOGGER.info(f'{prefix}{e}') + + +def classify_transforms(size=224): + # Transforms to apply if albumentations not installed + assert isinstance(size, int), f'ERROR: classify_transforms size {size} must be integer, not (list, tuple)' + # T.Compose([T.ToTensor(), T.Resize(size), T.CenterCrop(size), T.Normalize(IMAGENET_MEAN, IMAGENET_STD)]) + return T.Compose([CenterCrop(size), ToTensor(), T.Normalize(IMAGENET_MEAN, IMAGENET_STD)]) + + +class LetterBox: + # YOLOv5 LetterBox class for image preprocessing, i.e. T.Compose([LetterBox(size), ToTensor()]) + def __init__(self, size=(640, 640), auto=False, stride=32): + super().__init__() + self.h, self.w = (size, size) if isinstance(size, int) else size + self.auto = auto # pass max size integer, automatically solve for short side using stride + self.stride = stride # used with auto + + def __call__(self, im): # im = np.array HWC + imh, imw = im.shape[:2] + r = min(self.h / imh, self.w / imw) # ratio of new/old + h, w = round(imh * r), round(imw * r) # resized image + hs, ws = (math.ceil(x / self.stride) * self.stride for x in (h, w)) if self.auto else self.h, self.w + top, left = round((hs - h) / 2 - 0.1), round((ws - w) / 2 - 0.1) + im_out = np.full((self.h, self.w, 3), 114, dtype=im.dtype) + im_out[top:top + h, left:left + w] = cv2.resize(im, (w, h), interpolation=cv2.INTER_LINEAR) + return im_out + + +class CenterCrop: + # YOLOv5 CenterCrop class for image preprocessing, i.e. T.Compose([CenterCrop(size), ToTensor()]) + def __init__(self, size=640): + super().__init__() + self.h, self.w = (size, size) if isinstance(size, int) else size + + def __call__(self, im): # im = np.array HWC + imh, imw = im.shape[:2] + m = min(imh, imw) # min dimension + top, left = (imh - m) // 2, (imw - m) // 2 + return cv2.resize(im[top:top + m, left:left + m], (self.w, self.h), interpolation=cv2.INTER_LINEAR) + + +class ToTensor: + # YOLOv5 ToTensor class for image preprocessing, i.e. T.Compose([LetterBox(size), ToTensor()]) + def __init__(self, half=False): + super().__init__() + self.half = half + + def __call__(self, im): # im = np.array HWC in BGR order + im = np.ascontiguousarray(im.transpose((2, 0, 1))[::-1]) # HWC to CHW -> BGR to RGB -> contiguous + im = torch.from_numpy(im) # to torch + im = im.half() if self.half else im.float() # uint8 to fp16/32 + im /= 255.0 # 0-255 to 0.0-1.0 + return im diff --git a/utils/autoanchor.py b/utils/autoanchor.py new file mode 100644 index 0000000..4c11ab3 --- /dev/null +++ b/utils/autoanchor.py @@ -0,0 +1,169 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +AutoAnchor utils +""" + +import random + +import numpy as np +import torch +import yaml +from tqdm import tqdm + +from utils import TryExcept +from utils.general import LOGGER, TQDM_BAR_FORMAT, colorstr + +PREFIX = colorstr('AutoAnchor: ') + + +def check_anchor_order(m): + # Check anchor order against stride order for YOLOv5 Detect() module m, and correct if necessary + a = m.anchors.prod(-1).mean(-1).view(-1) # mean anchor area per output layer + da = a[-1] - a[0] # delta a + ds = m.stride[-1] - m.stride[0] # delta s + if da and (da.sign() != ds.sign()): # same order + LOGGER.info(f'{PREFIX}Reversing anchor order') + m.anchors[:] = m.anchors.flip(0) + + +@TryExcept(f'{PREFIX}ERROR') +def check_anchors(dataset, model, thr=4.0, imgsz=640): + # Check anchor fit to data, recompute if necessary + m = model.module.model[-1] if hasattr(model, 'module') else model.model[-1] # Detect() + shapes = imgsz * dataset.shapes / dataset.shapes.max(1, keepdims=True) + scale = np.random.uniform(0.9, 1.1, size=(shapes.shape[0], 1)) # augment scale + wh = torch.tensor(np.concatenate([l[:, 3:5] * s for s, l in zip(shapes * scale, dataset.labels)])).float() # wh + + def metric(k): # compute metric + r = wh[:, None] / k[None] + x = torch.min(r, 1 / r).min(2)[0] # ratio metric + best = x.max(1)[0] # best_x + aat = (x > 1 / thr).float().sum(1).mean() # anchors above threshold + bpr = (best > 1 / thr).float().mean() # best possible recall + return bpr, aat + + stride = m.stride.to(m.anchors.device).view(-1, 1, 1) # model strides + anchors = m.anchors.clone() * stride # current anchors + bpr, aat = metric(anchors.cpu().view(-1, 2)) + s = f'\n{PREFIX}{aat:.2f} anchors/target, {bpr:.3f} Best Possible Recall (BPR). ' + if bpr > 0.98: # threshold to recompute + LOGGER.info(f'{s}Current anchors are a good fit to dataset ✅') + else: + LOGGER.info(f'{s}Anchors are a poor fit to dataset ⚠️, attempting to improve...') + na = m.anchors.numel() // 2 # number of anchors + anchors = kmean_anchors(dataset, n=na, img_size=imgsz, thr=thr, gen=1000, verbose=False) + new_bpr = metric(anchors)[0] + if new_bpr > bpr: # replace anchors + anchors = torch.tensor(anchors, device=m.anchors.device).type_as(m.anchors) + m.anchors[:] = anchors.clone().view_as(m.anchors) + check_anchor_order(m) # must be in pixel-space (not grid-space) + m.anchors /= stride + s = f'{PREFIX}Done ✅ (optional: update model *.yaml to use these anchors in the future)' + else: + s = f'{PREFIX}Done ⚠️ (original anchors better than new anchors, proceeding with original anchors)' + LOGGER.info(s) + + +def kmean_anchors(dataset='./data/coco128.yaml', n=9, img_size=640, thr=4.0, gen=1000, verbose=True): + """ Creates kmeans-evolved anchors from training dataset + + Arguments: + dataset: path to data.yaml, or a loaded dataset + n: number of anchors + img_size: image size used for training + thr: anchor-label wh ratio threshold hyperparameter hyp['anchor_t'] used for training, default=4.0 + gen: generations to evolve anchors using genetic algorithm + verbose: print all results + + Return: + k: kmeans evolved anchors + + Usage: + from utils.autoanchor import *; _ = kmean_anchors() + """ + from scipy.cluster.vq import kmeans + + npr = np.random + thr = 1 / thr + + def metric(k, wh): # compute metrics + r = wh[:, None] / k[None] + x = torch.min(r, 1 / r).min(2)[0] # ratio metric + # x = wh_iou(wh, torch.tensor(k)) # iou metric + return x, x.max(1)[0] # x, best_x + + def anchor_fitness(k): # mutation fitness + _, best = metric(torch.tensor(k, dtype=torch.float32), wh) + return (best * (best > thr).float()).mean() # fitness + + def print_results(k, verbose=True): + k = k[np.argsort(k.prod(1))] # sort small to large + x, best = metric(k, wh0) + bpr, aat = (best > thr).float().mean(), (x > thr).float().mean() * n # best possible recall, anch > thr + s = f'{PREFIX}thr={thr:.2f}: {bpr:.4f} best possible recall, {aat:.2f} anchors past thr\n' \ + f'{PREFIX}n={n}, img_size={img_size}, metric_all={x.mean():.3f}/{best.mean():.3f}-mean/best, ' \ + f'past_thr={x[x > thr].mean():.3f}-mean: ' + for x in k: + s += '%i,%i, ' % (round(x[0]), round(x[1])) + if verbose: + LOGGER.info(s[:-2]) + return k + + if isinstance(dataset, str): # *.yaml file + with open(dataset, errors='ignore') as f: + data_dict = yaml.safe_load(f) # model dict + from utils.dataloaders import LoadImagesAndLabels + dataset = LoadImagesAndLabels(data_dict['train'], augment=True, rect=True) + + # Get label wh + shapes = img_size * dataset.shapes / dataset.shapes.max(1, keepdims=True) + wh0 = np.concatenate([l[:, 3:5] * s for s, l in zip(shapes, dataset.labels)]) # wh + + # Filter + i = (wh0 < 3.0).any(1).sum() + if i: + LOGGER.info(f'{PREFIX}WARNING ⚠️ Extremely small objects found: {i} of {len(wh0)} labels are <3 pixels in size') + wh = wh0[(wh0 >= 2.0).any(1)].astype(np.float32) # filter > 2 pixels + # wh = wh * (npr.rand(wh.shape[0], 1) * 0.9 + 0.1) # multiply by random scale 0-1 + + # Kmeans init + try: + LOGGER.info(f'{PREFIX}Running kmeans for {n} anchors on {len(wh)} points...') + assert n <= len(wh) # apply overdetermined constraint + s = wh.std(0) # sigmas for whitening + k = kmeans(wh / s, n, iter=30)[0] * s # points + assert n == len(k) # kmeans may return fewer points than requested if wh is insufficient or too similar + except Exception: + LOGGER.warning(f'{PREFIX}WARNING ⚠️ switching strategies from kmeans to random init') + k = np.sort(npr.rand(n * 2)).reshape(n, 2) * img_size # random init + wh, wh0 = (torch.tensor(x, dtype=torch.float32) for x in (wh, wh0)) + k = print_results(k, verbose=False) + + # Plot + # k, d = [None] * 20, [None] * 20 + # for i in tqdm(range(1, 21)): + # k[i-1], d[i-1] = kmeans(wh / s, i) # points, mean distance + # fig, ax = plt.subplots(1, 2, figsize=(14, 7), tight_layout=True) + # ax = ax.ravel() + # ax[0].plot(np.arange(1, 21), np.array(d) ** 2, marker='.') + # fig, ax = plt.subplots(1, 2, figsize=(14, 7)) # plot wh + # ax[0].hist(wh[wh[:, 0]<100, 0],400) + # ax[1].hist(wh[wh[:, 1]<100, 1],400) + # fig.savefig('wh.png', dpi=200) + + # Evolve + f, sh, mp, s = anchor_fitness(k), k.shape, 0.9, 0.1 # fitness, generations, mutation prob, sigma + pbar = tqdm(range(gen), bar_format=TQDM_BAR_FORMAT) # progress bar + for _ in pbar: + v = np.ones(sh) + while (v == 1).all(): # mutate until a change occurs (prevent duplicates) + v = ((npr.random(sh) < mp) * random.random() * npr.randn(*sh) * s + 1).clip(0.3, 3.0) + kg = (k.copy() * v).clip(min=2.0) + fg = anchor_fitness(kg) + if fg > f: + f, k = fg, kg.copy() + pbar.desc = f'{PREFIX}Evolving anchors with Genetic Algorithm: fitness = {f:.4f}' + if verbose: + print_results(k, verbose) + + return print_results(k).astype(np.float32) diff --git a/utils/autobatch.py b/utils/autobatch.py new file mode 100644 index 0000000..aa763b8 --- /dev/null +++ b/utils/autobatch.py @@ -0,0 +1,72 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Auto-batch utils +""" + +from copy import deepcopy + +import numpy as np +import torch + +from utils.general import LOGGER, colorstr +from utils.torch_utils import profile + + +def check_train_batch_size(model, imgsz=640, amp=True): + # Check YOLOv5 training batch size + with torch.cuda.amp.autocast(amp): + return autobatch(deepcopy(model).train(), imgsz) # compute optimal batch size + + +def autobatch(model, imgsz=640, fraction=0.8, batch_size=16): + # Automatically estimate best YOLOv5 batch size to use `fraction` of available CUDA memory + # Usage: + # import torch + # from utils.autobatch import autobatch + # model = torch.hub.load('ultralytics/yolov5', 'yolov5s', autoshape=False) + # print(autobatch(model)) + + # Check device + prefix = colorstr('AutoBatch: ') + LOGGER.info(f'{prefix}Computing optimal batch size for --imgsz {imgsz}') + device = next(model.parameters()).device # get model device + if device.type == 'cpu': + LOGGER.info(f'{prefix}CUDA not detected, using default CPU batch-size {batch_size}') + return batch_size + if torch.backends.cudnn.benchmark: + LOGGER.info(f'{prefix} ⚠️ Requires torch.backends.cudnn.benchmark=False, using default batch-size {batch_size}') + return batch_size + + # Inspect CUDA memory + gb = 1 << 30 # bytes to GiB (1024 ** 3) + d = str(device).upper() # 'CUDA:0' + properties = torch.cuda.get_device_properties(device) # device properties + t = properties.total_memory / gb # GiB total + r = torch.cuda.memory_reserved(device) / gb # GiB reserved + a = torch.cuda.memory_allocated(device) / gb # GiB allocated + f = t - (r + a) # GiB free + LOGGER.info(f'{prefix}{d} ({properties.name}) {t:.2f}G total, {r:.2f}G reserved, {a:.2f}G allocated, {f:.2f}G free') + + # Profile batch sizes + batch_sizes = [1, 2, 4, 8, 16] + try: + img = [torch.empty(b, 3, imgsz, imgsz) for b in batch_sizes] + results = profile(img, model, n=3, device=device) + except Exception as e: + LOGGER.warning(f'{prefix}{e}') + + # Fit a solution + y = [x[2] for x in results if x] # memory [2] + p = np.polyfit(batch_sizes[:len(y)], y, deg=1) # first degree polynomial fit + b = int((f * fraction - p[1]) / p[0]) # y intercept (optimal batch size) + if None in results: # some sizes failed + i = results.index(None) # first fail index + if b >= batch_sizes[i]: # y intercept above failure point + b = batch_sizes[max(i - 1, 0)] # select prior safe point + if b < 1 or b > 1024: # b outside of safe range + b = batch_size + LOGGER.warning(f'{prefix}WARNING ⚠️ CUDA anomaly detected, recommend restart environment and retry command.') + + fraction = (np.polyval(p, b) + r + a) / t # actual fraction predicted + LOGGER.info(f'{prefix}Using batch-size {b} for {d} {t * fraction:.2f}G/{t:.2f}G ({fraction * 100:.0f}%) ✅') + return b diff --git a/utils/aws/__init__.py b/utils/aws/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/utils/aws/mime.sh b/utils/aws/mime.sh new file mode 100644 index 0000000..c319a83 --- /dev/null +++ b/utils/aws/mime.sh @@ -0,0 +1,26 @@ +# AWS EC2 instance startup 'MIME' script https://aws.amazon.com/premiumsupport/knowledge-center/execute-user-data-ec2/ +# This script will run on every instance restart, not only on first start +# --- DO NOT COPY ABOVE COMMENTS WHEN PASTING INTO USERDATA --- + +Content-Type: multipart/mixed; boundary="//" +MIME-Version: 1.0 + +--// +Content-Type: text/cloud-config; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; filename="cloud-config.txt" + +#cloud-config +cloud_final_modules: +- [scripts-user, always] + +--// +Content-Type: text/x-shellscript; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; filename="userdata.txt" + +#!/bin/bash +# --- paste contents of userdata.sh here --- +--// diff --git a/utils/aws/resume.py b/utils/aws/resume.py new file mode 100644 index 0000000..b21731c --- /dev/null +++ b/utils/aws/resume.py @@ -0,0 +1,40 @@ +# Resume all interrupted trainings in yolov5/ dir including DDP trainings +# Usage: $ python utils/aws/resume.py + +import os +import sys +from pathlib import Path + +import torch +import yaml + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[2] # YOLOv5 root directory +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) # add ROOT to PATH + +port = 0 # --master_port +path = Path('').resolve() +for last in path.rglob('*/**/last.pt'): + ckpt = torch.load(last) + if ckpt['optimizer'] is None: + continue + + # Load opt.yaml + with open(last.parent.parent / 'opt.yaml', errors='ignore') as f: + opt = yaml.safe_load(f) + + # Get device count + d = opt['device'].split(',') # devices + nd = len(d) # number of devices + ddp = nd > 1 or (nd == 0 and torch.cuda.device_count() > 1) # distributed data parallel + + if ddp: # multi-GPU + port += 1 + cmd = f'python -m torch.distributed.run --nproc_per_node {nd} --master_port {port} train.py --resume {last}' + else: # single-GPU + cmd = f'python train.py --resume {last}' + + cmd += ' > /dev/null 2>&1 &' # redirect output to dev/null and run in daemon thread + print(cmd) + os.system(cmd) diff --git a/utils/aws/userdata.sh b/utils/aws/userdata.sh new file mode 100644 index 0000000..5fc1332 --- /dev/null +++ b/utils/aws/userdata.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# AWS EC2 instance startup script https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/user-data.html +# This script will run only once on first instance start (for a re-start script see mime.sh) +# /home/ubuntu (ubuntu) or /home/ec2-user (amazon-linux) is working dir +# Use >300 GB SSD + +cd home/ubuntu +if [ ! -d yolov5 ]; then + echo "Running first-time script." # install dependencies, download COCO, pull Docker + git clone https://github.com/ultralytics/yolov5 -b master && sudo chmod -R 777 yolov5 + cd yolov5 + bash data/scripts/get_coco.sh && echo "COCO done." & + sudo docker pull ultralytics/yolov5:latest && echo "Docker done." & + python -m pip install --upgrade pip && pip install -r requirements.txt && python detect.py && echo "Requirements done." & + wait && echo "All tasks done." # finish background tasks +else + echo "Running re-start script." # resume interrupted runs + i=0 + list=$(sudo docker ps -qa) # container list i.e. $'one\ntwo\nthree\nfour' + while IFS= read -r id; do + ((i++)) + echo "restarting container $i: $id" + sudo docker start $id + # sudo docker exec -it $id python train.py --resume # single-GPU + sudo docker exec -d $id python utils/aws/resume.py # multi-scenario + done <<<"$list" +fi diff --git a/utils/callbacks.py b/utils/callbacks.py new file mode 100644 index 0000000..c90fa82 --- /dev/null +++ b/utils/callbacks.py @@ -0,0 +1,76 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Callback utils +""" + +import threading + + +class Callbacks: + """" + Handles all registered callbacks for YOLOv5 Hooks + """ + + def __init__(self): + # Define the available callbacks + self._callbacks = { + 'on_pretrain_routine_start': [], + 'on_pretrain_routine_end': [], + 'on_train_start': [], + 'on_train_epoch_start': [], + 'on_train_batch_start': [], + 'optimizer_step': [], + 'on_before_zero_grad': [], + 'on_train_batch_end': [], + 'on_train_epoch_end': [], + 'on_val_start': [], + 'on_val_batch_start': [], + 'on_val_image_end': [], + 'on_val_batch_end': [], + 'on_val_end': [], + 'on_fit_epoch_end': [], # fit = train + val + 'on_model_save': [], + 'on_train_end': [], + 'on_params_update': [], + 'teardown': [], } + self.stop_training = False # set True to interrupt training + + def register_action(self, hook, name='', callback=None): + """ + Register a new action to a callback hook + + Args: + hook: The callback hook name to register the action to + name: The name of the action for later reference + callback: The callback to fire + """ + assert hook in self._callbacks, f"hook '{hook}' not found in callbacks {self._callbacks}" + assert callable(callback), f"callback '{callback}' is not callable" + self._callbacks[hook].append({'name': name, 'callback': callback}) + + def get_registered_actions(self, hook=None): + """" + Returns all the registered actions by callback hook + + Args: + hook: The name of the hook to check, defaults to all + """ + return self._callbacks[hook] if hook else self._callbacks + + def run(self, hook, *args, thread=False, **kwargs): + """ + Loop through the registered actions and fire all callbacks on main thread + + Args: + hook: The name of the hook to check, defaults to all + args: Arguments to receive from YOLOv5 + thread: (boolean) Run callbacks in daemon thread + kwargs: Keyword Arguments to receive from YOLOv5 + """ + + assert hook in self._callbacks, f"hook '{hook}' not found in callbacks {self._callbacks}" + for logger in self._callbacks[hook]: + if thread: + threading.Thread(target=logger['callback'], args=args, kwargs=kwargs, daemon=True).start() + else: + logger['callback'](*args, **kwargs) diff --git a/utils/dataloaders.py b/utils/dataloaders.py new file mode 100644 index 0000000..439b1c4 --- /dev/null +++ b/utils/dataloaders.py @@ -0,0 +1,1255 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Dataloaders and dataset utils +""" + +import contextlib +import glob +import hashlib +import json +import math +import os +import random +import shutil +import time +from itertools import repeat +from multiprocessing.pool import Pool, ThreadPool +from pathlib import Path +from threading import Thread +from urllib.parse import urlparse + +import numpy as np +import psutil +import torch +import torch.nn.functional as F +import torchvision +import yaml +from PIL import ExifTags, Image, ImageOps +from torch.utils.data import DataLoader, Dataset, dataloader, distributed +from tqdm import tqdm + +from utils.augmentations import (Albumentations, augment_hsv, classify_albumentations, classify_transforms, copy_paste, + letterbox, mixup, random_perspective) +from utils.general import (DATASETS_DIR, LOGGER, NUM_THREADS, TQDM_BAR_FORMAT, check_dataset, check_requirements, + check_yaml, clean_str, cv2, is_colab, is_kaggle, segments2boxes, unzip_file, xyn2xy, + xywh2xyxy, xywhn2xyxy, xyxy2xywhn) +from utils.torch_utils import torch_distributed_zero_first +import subprocess +import json + + +# Parameters +HELP_URL = 'See https://docs.ultralytics.com/yolov5/tutorials/train_custom_data' +IMG_FORMATS = 'bmp', 'dng', 'jpeg', 'jpg', 'mpo', 'png', 'tif', 'tiff', 'webp', 'pfm' # include image suffixes +VID_FORMATS = 'asf', 'avi', 'gif', 'm4v', 'mkv', 'mov', 'mp4', 'mpeg', 'mpg', 'ts', 'wmv' # include video suffixes +LOCAL_RANK = int(os.getenv('LOCAL_RANK', -1)) # https://pytorch.org/docs/stable/elastic/run.html +RANK = int(os.getenv('RANK', -1)) +PIN_MEMORY = str(os.getenv('PIN_MEMORY', True)).lower() == 'true' # global pin_memory for dataloaders + +# Get orientation exif tag +for orientation in ExifTags.TAGS.keys(): + if ExifTags.TAGS[orientation] == 'Orientation': + break + + +def get_hash(paths): + # Returns a single hash value of a list of paths (files or dirs) + size = sum(os.path.getsize(p) for p in paths if os.path.exists(p)) # sizes + h = hashlib.sha256(str(size).encode()) # hash sizes + h.update(''.join(paths).encode()) # hash paths + return h.hexdigest() # return hash + + +def exif_size(img): + # Returns exif-corrected PIL size + s = img.size # (width, height) + with contextlib.suppress(Exception): + rotation = dict(img._getexif().items())[orientation] + if rotation in [6, 8]: # rotation 270 or 90 + s = (s[1], s[0]) + return s + + +def exif_transpose(image): + """ + Transpose a PIL image accordingly if it has an EXIF Orientation tag. + Inplace version of https://github.com/python-pillow/Pillow/blob/master/src/PIL/ImageOps.py exif_transpose() + + :param image: The image to transpose. + :return: An image. + """ + exif = image.getexif() + orientation = exif.get(0x0112, 1) # default 1 + if orientation > 1: + method = { + 2: Image.FLIP_LEFT_RIGHT, + 3: Image.ROTATE_180, + 4: Image.FLIP_TOP_BOTTOM, + 5: Image.TRANSPOSE, + 6: Image.ROTATE_270, + 7: Image.TRANSVERSE, + 8: Image.ROTATE_90}.get(orientation) + if method is not None: + image = image.transpose(method) + del exif[0x0112] + image.info['exif'] = exif.tobytes() + return image + + +def seed_worker(worker_id): + # Set dataloader worker seed https://pytorch.org/docs/stable/notes/randomness.html#dataloader + worker_seed = torch.initial_seed() % 2 ** 32 + np.random.seed(worker_seed) + random.seed(worker_seed) + + +def create_dataloader(path, + imgsz, + batch_size, + stride, + single_cls=False, + hyp=None, + augment=False, + cache=False, + pad=0.0, + rect=False, + rank=-1, + workers=8, + image_weights=False, + quad=False, + prefix='', + shuffle=False, + seed=0): + if rect and shuffle: + LOGGER.warning('WARNING ⚠️ --rect is incompatible with DataLoader shuffle, setting shuffle=False') + shuffle = False + with torch_distributed_zero_first(rank): # init dataset *.cache only once if DDP + dataset = LoadImagesAndLabels( + path, + imgsz, + batch_size, + augment=augment, # augmentation + hyp=hyp, # hyperparameters + rect=rect, # rectangular batches + cache_images=cache, + single_cls=single_cls, + stride=int(stride), + pad=pad, + image_weights=image_weights, + prefix=prefix) + + batch_size = min(batch_size, len(dataset)) + nd = torch.cuda.device_count() # number of CUDA devices + nw = min([os.cpu_count() // max(nd, 1), batch_size if batch_size > 1 else 0, workers]) # number of workers + sampler = None if rank == -1 else distributed.DistributedSampler(dataset, shuffle=shuffle) + loader = DataLoader if image_weights else InfiniteDataLoader # only DataLoader allows for attribute updates + generator = torch.Generator() + generator.manual_seed(6148914691236517205 + seed + RANK) + return loader(dataset, + batch_size=batch_size, + shuffle=shuffle and sampler is None, + num_workers=nw, + sampler=sampler, + pin_memory=PIN_MEMORY, + collate_fn=LoadImagesAndLabels.collate_fn4 if quad else LoadImagesAndLabels.collate_fn, + worker_init_fn=seed_worker, + generator=generator), dataset + + +class InfiniteDataLoader(dataloader.DataLoader): + """ Dataloader that reuses workers + + Uses same syntax as vanilla DataLoader + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + object.__setattr__(self, 'batch_sampler', _RepeatSampler(self.batch_sampler)) + self.iterator = super().__iter__() + + def __len__(self): + return len(self.batch_sampler.sampler) + + def __iter__(self): + for _ in range(len(self)): + yield next(self.iterator) + + +class _RepeatSampler: + """ Sampler that repeats forever + + Args: + sampler (Sampler) + """ + + def __init__(self, sampler): + self.sampler = sampler + + def __iter__(self): + while True: + yield from iter(self.sampler) + + +class LoadScreenshots: + # YOLOv5 screenshot dataloader, i.e. `python detect.py --source "screen 0 100 100 512 256"` + def __init__(self, source, img_size=640, stride=32, auto=True, transforms=None): + # source = [screen_number left top width height] (pixels) + check_requirements('mss') + import mss + + source, *params = source.split() + self.screen, left, top, width, height = 0, None, None, None, None # default to full screen 0 + if len(params) == 1: + self.screen = int(params[0]) + elif len(params) == 4: + left, top, width, height = (int(x) for x in params) + elif len(params) == 5: + self.screen, left, top, width, height = (int(x) for x in params) + self.img_size = img_size + self.stride = stride + self.transforms = transforms + self.auto = auto + self.mode = 'stream' + self.frame = 0 + self.sct = mss.mss() + + # Parse monitor shape + monitor = self.sct.monitors[self.screen] + self.top = monitor['top'] if top is None else (monitor['top'] + top) + self.left = monitor['left'] if left is None else (monitor['left'] + left) + self.width = width or monitor['width'] + self.height = height or monitor['height'] + self.monitor = {'left': self.left, 'top': self.top, 'width': self.width, 'height': self.height} + + def __iter__(self): + return self + + def __next__(self): + # mss screen capture: get raw pixels from the screen as np array + im0 = np.array(self.sct.grab(self.monitor))[:, :, :3] # [:, :, :3] BGRA to BGR + s = f'screen {self.screen} (LTWH): {self.left},{self.top},{self.width},{self.height}: ' + + if self.transforms: + im = self.transforms(im0) # transforms + else: + im = letterbox(im0, self.img_size, stride=self.stride, auto=self.auto)[0] # padded resize + im = im.transpose((2, 0, 1))[::-1] # HWC to CHW, BGR to RGB + im = np.ascontiguousarray(im) # contiguous + self.frame += 1 + return str(self.screen), im, im0, None, s # screen, img, original img, im0s, s + + +class LoadImages: + # YOLOv5 image/video dataloader, i.e. `python detect.py --source image.jpg/vid.mp4` + def __init__(self, path, img_size=640, stride=32, auto=True, transforms=None, vid_stride=1): + if isinstance(path, str) and Path(path).suffix == '.txt': # *.txt file with img/vid/dir on each line + path = Path(path).read_text().rsplit() + files = [] + for p in sorted(path) if isinstance(path, (list, tuple)) else [path]: + p = str(Path(p).resolve()) + if '*' in p: + files.extend(sorted(glob.glob(p, recursive=True))) # glob + elif os.path.isdir(p): + files.extend(sorted(glob.glob(os.path.join(p, '*.*')))) # dir + elif os.path.isfile(p): + files.append(p) # files + else: + raise FileNotFoundError(f'{p} does not exist') + + images = [x for x in files if x.split('.')[-1].lower() in IMG_FORMATS] + videos = [x for x in files if x.split('.')[-1].lower() in VID_FORMATS] + ni, nv = len(images), len(videos) + + + + self.img_size = img_size + self.stride = stride + self.files = images + videos + self.nf = ni + nv # number of files + self.video_flag = [False] * ni + [True] * nv + self.mode = 'image' + self.auto = auto + self.transforms = transforms # optional + self.vid_stride = vid_stride # video frame-rate stride + if any(videos): + self._new_video(videos[0]) # new video + else: + self.cap = None + assert self.nf > 0, f'No images or videos found in {p}. ' \ + f'Supported formats are:\nimages: {IMG_FORMATS}\nvideos: {VID_FORMATS}' + + def __iter__(self): + self.count = 0 + return self + + def __next__(self): + if self.count == self.nf: + raise StopIteration + path = self.files[self.count] + + if self.video_flag[self.count]: + # Read video + self.mode = 'video' + for _ in range(self.vid_stride): + self.cap.grab() + ret_val, im0 = self.cap.retrieve() + while not ret_val: + self.count += 1 + self.cap.release() + if self.count == self.nf: # last video + raise StopIteration + path = self.files[self.count] + self._new_video(path) + ret_val, im0 = self.cap.read() + + self.frame += 1 + ##====================== + '''判断视频是否含旋转信息''' + rotation = self.get_rotation(path) + if rotation == 270: + im0 = cv2.rotate(im0, cv2.ROTATE_90_COUNTERCLOCKWISE) + ###====================== + + # if self.orientation == 270: + # im0 = cv2.rotate(im0, cv2.ROTATE_90_COUNTERCLOCKWISE) # for use if cv2 autorotation is False + s = f'video {self.count + 1}/{self.nf} ({self.frame}/{self.frames}) {path}: ' + + else: + # Read image + self.count += 1 + im0 = cv2.imread(path) # BGR + assert im0 is not None, f'Image Not Found {path}' + s = f'image {self.count}/{self.nf} {path}: ' + + if self.transforms: + im = self.transforms(im0) # transforms + else: + im = letterbox(im0, self.img_size, stride=self.stride, auto=self.auto)[0] # padded resize + im = im.transpose((2, 0, 1))[::-1] # HWC to CHW, BGR to RGB + im = np.ascontiguousarray(im) # contiguous + + return path, im, im0, self.cap, s + + def _new_video(self, path): + # Create a new video capture object + self.frame = 0 + self.cap = cv2.VideoCapture(path) + self.frames = int(self.cap.get(cv2.CAP_PROP_FRAME_COUNT) / self.vid_stride) + self.orientation = int(self.cap.get(cv2.CAP_PROP_ORIENTATION_META)) # rotation degrees + # self.cap.set(cv2.CAP_PROP_ORIENTATION_AUTO, 0) # disable https://github.com/ultralytics/yolov5/issues/8493 + + def get_rotation(self, filename): + cmd = [ + "ffprobe", # 注意是 ffprobe，不是 ffmpeg + "-v", "error", + "-select_streams", "v:0", + "-show_entries", "stream_tags=rotate", + "-of", "json", + filename + ] + result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if result.returncode == 0: + metadata = json.loads(result.stdout) + rotation = metadata.get("streams", [{}])[0].get("tags", {}).get("rotate", 0) + return int(rotation) + else: + return 0 + + def _cv2_rotate(self, im): + # Rotate a cv2 video manually + if self.orientation == 0: + return cv2.rotate(im, cv2.ROTATE_90_CLOCKWISE) + elif self.orientation == 180: + return cv2.rotate(im, cv2.ROTATE_90_COUNTERCLOCKWISE) + elif self.orientation == 90: + return cv2.rotate(im, cv2.ROTATE_180) + # if self.orientation == 270: + # return cv2.rotate(im, cv2.ROTATE_90_COUNTERCLOCKWISE) + + return im + + def __len__(self): + return self.nf # number of files + + +class LoadStreams: + # YOLOv5 streamloader, i.e. `python detect.py --source 'rtsp://example.com/media.mp4' # RTSP, RTMP, HTTP streams` + def __init__(self, sources='file.streams', img_size=640, stride=32, auto=True, transforms=None, vid_stride=1): + torch.backends.cudnn.benchmark = True # faster for fixed-size inference + self.mode = 'stream' + self.img_size = img_size + self.stride = stride + self.vid_stride = vid_stride # video frame-rate stride + sources = Path(sources).read_text().rsplit() if os.path.isfile(sources) else [sources] + n = len(sources) + self.sources = [clean_str(x) for x in sources] # clean source names for later + self.imgs, self.fps, self.frames, self.threads = [None] * n, [0] * n, [0] * n, [None] * n + for i, s in enumerate(sources): # index, source + # Start thread to read frames from video stream + st = f'{i + 1}/{n}: {s}... ' + if urlparse(s).hostname in ('www.youtube.com', 'youtube.com', 'youtu.be'): # if source is YouTube video + # YouTube format i.e. 'https://www.youtube.com/watch?v=Zgi9g1ksQHc' or 'https://youtu.be/Zgi9g1ksQHc' + check_requirements(('pafy', 'youtube_dl==2020.12.2')) + import pafy + s = pafy.new(s).getbest(preftype='mp4').url # YouTube URL + s = eval(s) if s.isnumeric() else s # i.e. s = '0' local webcam + if s == 0: + assert not is_colab(), '--source 0 webcam unsupported on Colab. Rerun command in a local environment.' + assert not is_kaggle(), '--source 0 webcam unsupported on Kaggle. Rerun command in a local environment.' + cap = cv2.VideoCapture(s) + assert cap.isOpened(), f'{st}Failed to open {s}' + w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + fps = cap.get(cv2.CAP_PROP_FPS) # warning: may return 0 or nan + self.frames[i] = max(int(cap.get(cv2.CAP_PROP_FRAME_COUNT)), 0) or float('inf') # infinite stream fallback + self.fps[i] = max((fps if math.isfinite(fps) else 0) % 100, 0) or 30 # 30 FPS fallback + + _, self.imgs[i] = cap.read() # guarantee first frame + self.threads[i] = Thread(target=self.update, args=([i, cap, s]), daemon=True) + LOGGER.info(f'{st} Success ({self.frames[i]} frames {w}x{h} at {self.fps[i]:.2f} FPS)') + self.threads[i].start() + LOGGER.info('') # newline + + # check for common shapes + s = np.stack([letterbox(x, img_size, stride=stride, auto=auto)[0].shape for x in self.imgs]) + self.rect = np.unique(s, axis=0).shape[0] == 1 # rect inference if all shapes equal + self.auto = auto and self.rect + self.transforms = transforms # optional + if not self.rect: + LOGGER.warning('WARNING ⚠️ Stream shapes differ. For optimal performance supply similarly-shaped streams.') + + def update(self, i, cap, stream): + # Read stream `i` frames in daemon thread + n, f = 0, self.frames[i] # frame number, frame array + while cap.isOpened() and n < f: + n += 1 + cap.grab() # .read() = .grab() followed by .retrieve() + if n % self.vid_stride == 0: + success, im = cap.retrieve() + if success: + self.imgs[i] = im + else: + LOGGER.warning('WARNING ⚠️ Video stream unresponsive, please check your IP camera connection.') + self.imgs[i] = np.zeros_like(self.imgs[i]) + cap.open(stream) # re-open stream if signal was lost + time.sleep(0.0) # wait time + + def __iter__(self): + self.count = -1 + return self + + def __next__(self): + self.count += 1 + if not all(x.is_alive() for x in self.threads) or cv2.waitKey(1) == ord('q'): # q to quit + cv2.destroyAllWindows() + raise StopIteration + + im0 = self.imgs.copy() + if self.transforms: + im = np.stack([self.transforms(x) for x in im0]) # transforms + else: + im = np.stack([letterbox(x, self.img_size, stride=self.stride, auto=self.auto)[0] for x in im0]) # resize + im = im[..., ::-1].transpose((0, 3, 1, 2)) # BGR to RGB, BHWC to BCHW + im = np.ascontiguousarray(im) # contiguous + + return self.sources, im, im0, None, '' + + def __len__(self): + return len(self.sources) # 1E12 frames = 32 streams at 30 FPS for 30 years + + +def img2label_paths(img_paths): + # Define label paths as a function of image paths + sa, sb = f'{os.sep}images{os.sep}', f'{os.sep}labels{os.sep}' # /images/, /labels/ substrings + return [sb.join(x.rsplit(sa, 1)).rsplit('.', 1)[0] + '.txt' for x in img_paths] + + +class LoadImagesAndLabels(Dataset): + # YOLOv5 train_loader/val_loader, loads images and labels for training and validation + cache_version = 0.6 # dataset labels *.cache version + rand_interp_methods = [cv2.INTER_NEAREST, cv2.INTER_LINEAR, cv2.INTER_CUBIC, cv2.INTER_AREA, cv2.INTER_LANCZOS4] + + def __init__(self, + path, + img_size=640, + batch_size=16, + augment=False, + hyp=None, + rect=False, + image_weights=False, + cache_images=False, + single_cls=False, + stride=32, + pad=0.0, + min_items=0, + prefix=''): + self.img_size = img_size + self.augment = augment + self.hyp = hyp + self.image_weights = image_weights + self.rect = False if image_weights else rect + self.mosaic = self.augment and not self.rect # load 4 images at a time into a mosaic (only during training) + self.mosaic_border = [-img_size // 2, -img_size // 2] + self.stride = stride + self.path = path + self.albumentations = Albumentations(size=img_size) if augment else None + + try: + f = [] # image files + for p in path if isinstance(path, list) else [path]: + p = Path(p) # os-agnostic + if p.is_dir(): # dir + f += glob.glob(str(p / '**' / '*.*'), recursive=True) + # f = list(p.rglob('*.*')) # pathlib + elif p.is_file(): # file + with open(p) as t: + t = t.read().strip().splitlines() + parent = str(p.parent) + os.sep + f += [x.replace('./', parent, 1) if x.startswith('./') else x for x in t] # to global path + # f += [p.parent / x.lstrip(os.sep) for x in t] # to global path (pathlib) + else: + raise FileNotFoundError(f'{prefix}{p} does not exist') + self.im_files = sorted(x.replace('/', os.sep) for x in f if x.split('.')[-1].lower() in IMG_FORMATS) + # self.img_files = sorted([x for x in f if x.suffix[1:].lower() in IMG_FORMATS]) # pathlib + assert self.im_files, f'{prefix}No images found' + except Exception as e: + raise Exception(f'{prefix}Error loading data from {path}: {e}\n{HELP_URL}') from e + + # Check cache + self.label_files = img2label_paths(self.im_files) # labels + cache_path = (p if p.is_file() else Path(self.label_files[0]).parent).with_suffix('.cache') + try: + cache, exists = np.load(cache_path, allow_pickle=True).item(), True # load dict + assert cache['version'] == self.cache_version # matches current version + assert cache['hash'] == get_hash(self.label_files + self.im_files) # identical hash + except Exception: + cache, exists = self.cache_labels(cache_path, prefix), False # run cache ops + + # Display cache + nf, nm, ne, nc, n = cache.pop('results') # found, missing, empty, corrupt, total + if exists and LOCAL_RANK in {-1, 0}: + d = f'Scanning {cache_path}... {nf} images, {nm + ne} backgrounds, {nc} corrupt' + tqdm(None, desc=prefix + d, total=n, initial=n, bar_format=TQDM_BAR_FORMAT) # display cache results + if cache['msgs']: + LOGGER.info('\n'.join(cache['msgs'])) # display warnings + assert nf > 0 or not augment, f'{prefix}No labels found in {cache_path}, can not start training. {HELP_URL}' + + # Read cache + [cache.pop(k) for k in ('hash', 'version', 'msgs')] # remove items + labels, shapes, self.segments = zip(*cache.values()) + nl = len(np.concatenate(labels, 0)) # number of labels + assert nl > 0 or not augment, f'{prefix}All labels empty in {cache_path}, can not start training. {HELP_URL}' + self.labels = list(labels) + self.shapes = np.array(shapes) + self.im_files = list(cache.keys()) # update + self.label_files = img2label_paths(cache.keys()) # update + + # Filter images + if min_items: + include = np.array([len(x) >= min_items for x in self.labels]).nonzero()[0].astype(int) + LOGGER.info(f'{prefix}{n - len(include)}/{n} images filtered from dataset') + self.im_files = [self.im_files[i] for i in include] + self.label_files = [self.label_files[i] for i in include] + self.labels = [self.labels[i] for i in include] + self.segments = [self.segments[i] for i in include] + self.shapes = self.shapes[include] # wh + + # Create indices + n = len(self.shapes) # number of images + bi = np.floor(np.arange(n) / batch_size).astype(int) # batch index + nb = bi[-1] + 1 # number of batches + self.batch = bi # batch index of image + self.n = n + self.indices = range(n) + + # Update labels + include_class = [] # filter labels to include only these classes (optional) + self.segments = list(self.segments) + include_class_array = np.array(include_class).reshape(1, -1) + for i, (label, segment) in enumerate(zip(self.labels, self.segments)): + if include_class: + j = (label[:, 0:1] == include_class_array).any(1) + self.labels[i] = label[j] + if segment: + self.segments[i] = [segment[idx] for idx, elem in enumerate(j) if elem] + if single_cls: # single-class training, merge all classes into 0 + self.labels[i][:, 0] = 0 + + # Rectangular Training + if self.rect: + # Sort by aspect ratio + s = self.shapes # wh + ar = s[:, 1] / s[:, 0] # aspect ratio + irect = ar.argsort() + self.im_files = [self.im_files[i] for i in irect] + self.label_files = [self.label_files[i] for i in irect] + self.labels = [self.labels[i] for i in irect] + self.segments = [self.segments[i] for i in irect] + self.shapes = s[irect] # wh + ar = ar[irect] + + # Set training image shapes + shapes = [[1, 1]] * nb + for i in range(nb): + ari = ar[bi == i] + mini, maxi = ari.min(), ari.max() + if maxi < 1: + shapes[i] = [maxi, 1] + elif mini > 1: + shapes[i] = [1, 1 / mini] + + self.batch_shapes = np.ceil(np.array(shapes) * img_size / stride + pad).astype(int) * stride + + # Cache images into RAM/disk for faster training + if cache_images == 'ram' and not self.check_cache_ram(prefix=prefix): + cache_images = False + self.ims = [None] * n + self.npy_files = [Path(f).with_suffix('.npy') for f in self.im_files] + if cache_images: + b, gb = 0, 1 << 30 # bytes of cached images, bytes per gigabytes + self.im_hw0, self.im_hw = [None] * n, [None] * n + fcn = self.cache_images_to_disk if cache_images == 'disk' else self.load_image + results = ThreadPool(NUM_THREADS).imap(fcn, range(n)) + pbar = tqdm(enumerate(results), total=n, bar_format=TQDM_BAR_FORMAT, disable=LOCAL_RANK > 0) + for i, x in pbar: + if cache_images == 'disk': + b += self.npy_files[i].stat().st_size + else: # 'ram' + self.ims[i], self.im_hw0[i], self.im_hw[i] = x # im, hw_orig, hw_resized = load_image(self, i) + b += self.ims[i].nbytes + pbar.desc = f'{prefix}Caching images ({b / gb:.1f}GB {cache_images})' + pbar.close() + + def check_cache_ram(self, safety_margin=0.1, prefix=''): + # Check image caching requirements vs available memory + b, gb = 0, 1 << 30 # bytes of cached images, bytes per gigabytes + n = min(self.n, 30) # extrapolate from 30 random images + for _ in range(n): + im = cv2.imread(random.choice(self.im_files)) # sample image + ratio = self.img_size / max(im.shape[0], im.shape[1]) # max(h, w) # ratio + b += im.nbytes * ratio ** 2 + mem_required = b * self.n / n # GB required to cache dataset into RAM + mem = psutil.virtual_memory() + cache = mem_required * (1 + safety_margin) < mem.available # to cache or not to cache, that is the question + if not cache: + LOGGER.info(f'{prefix}{mem_required / gb:.1f}GB RAM required, ' + f'{mem.available / gb:.1f}/{mem.total / gb:.1f}GB available, ' + f"{'caching images ✅' if cache else 'not caching images ⚠️'}") + return cache + + def cache_labels(self, path=Path('./labels.cache'), prefix=''): + # Cache dataset labels, check images and read shapes + x = {} # dict + nm, nf, ne, nc, msgs = 0, 0, 0, 0, [] # number missing, found, empty, corrupt, messages + desc = f'{prefix}Scanning {path.parent / path.stem}...' + with Pool(NUM_THREADS) as pool: + pbar = tqdm(pool.imap(verify_image_label, zip(self.im_files, self.label_files, repeat(prefix))), + desc=desc, + total=len(self.im_files), + bar_format=TQDM_BAR_FORMAT) + for im_file, lb, shape, segments, nm_f, nf_f, ne_f, nc_f, msg in pbar: + nm += nm_f + nf += nf_f + ne += ne_f + nc += nc_f + if im_file: + x[im_file] = [lb, shape, segments] + if msg: + msgs.append(msg) + pbar.desc = f'{desc} {nf} images, {nm + ne} backgrounds, {nc} corrupt' + + pbar.close() + if msgs: + LOGGER.info('\n'.join(msgs)) + if nf == 0: + LOGGER.warning(f'{prefix}WARNING ⚠️ No labels found in {path}. {HELP_URL}') + x['hash'] = get_hash(self.label_files + self.im_files) + x['results'] = nf, nm, ne, nc, len(self.im_files) + x['msgs'] = msgs # warnings + x['version'] = self.cache_version # cache version + try: + np.save(path, x) # save cache for next time + path.with_suffix('.cache.npy').rename(path) # remove .npy suffix + LOGGER.info(f'{prefix}New cache created: {path}') + except Exception as e: + LOGGER.warning(f'{prefix}WARNING ⚠️ Cache directory {path.parent} is not writeable: {e}') # not writeable + return x + + def __len__(self): + return len(self.im_files) + + # def __iter__(self): + # self.count = -1 + # print('ran dataset iter') + # #self.shuffled_vector = np.random.permutation(self.nF) if self.augment else np.arange(self.nF) + # return self + + def __getitem__(self, index): + index = self.indices[index] # linear, shuffled, or image_weights + + hyp = self.hyp + mosaic = self.mosaic and random.random() < hyp['mosaic'] + if mosaic: + # Load mosaic + img, labels = self.load_mosaic(index) + shapes = None + + # MixUp augmentation + if random.random() < hyp['mixup']: + img, labels = mixup(img, labels, *self.load_mosaic(random.randint(0, self.n - 1))) + + else: + # Load image + img, (h0, w0), (h, w) = self.load_image(index) + + # Letterbox + shape = self.batch_shapes[self.batch[index]] if self.rect else self.img_size # final letterboxed shape + img, ratio, pad = letterbox(img, shape, auto=False, scaleup=self.augment) + shapes = (h0, w0), ((h / h0, w / w0), pad) # for COCO mAP rescaling + + labels = self.labels[index].copy() + if labels.size: # normalized xywh to pixel xyxy format + labels[:, 1:] = xywhn2xyxy(labels[:, 1:], ratio[0] * w, ratio[1] * h, padw=pad[0], padh=pad[1]) + + if self.augment: + img, labels = random_perspective(img, + labels, + degrees=hyp['degrees'], + translate=hyp['translate'], + scale=hyp['scale'], + shear=hyp['shear'], + perspective=hyp['perspective']) + + nl = len(labels) # number of labels + if nl: + labels[:, 1:5] = xyxy2xywhn(labels[:, 1:5], w=img.shape[1], h=img.shape[0], clip=True, eps=1E-3) + + if self.augment: + # Albumentations + img, labels = self.albumentations(img, labels) + nl = len(labels) # update after albumentations + + # HSV color-space + augment_hsv(img, hgain=hyp['hsv_h'], sgain=hyp['hsv_s'], vgain=hyp['hsv_v']) + + # Flip up-down + if random.random() < hyp['flipud']: + img = np.flipud(img) + if nl: + labels[:, 2] = 1 - labels[:, 2] + + # Flip left-right + if random.random() < hyp['fliplr']: + img = np.fliplr(img) + if nl: + labels[:, 1] = 1 - labels[:, 1] + + # Cutouts + # labels = cutout(img, labels, p=0.5) + # nl = len(labels) # update after cutout + + labels_out = torch.zeros((nl, 6)) + if nl: + labels_out[:, 1:] = torch.from_numpy(labels) + + # Convert + img = img.transpose((2, 0, 1))[::-1] # HWC to CHW, BGR to RGB + img = np.ascontiguousarray(img) + + return torch.from_numpy(img), labels_out, self.im_files[index], shapes + + def load_image(self, i): + # Loads 1 image from dataset index 'i', returns (im, original hw, resized hw) + im, f, fn = self.ims[i], self.im_files[i], self.npy_files[i], + if im is None: # not cached in RAM + if fn.exists(): # load npy + im = np.load(fn) + else: # read image + im = cv2.imread(f) # BGR + assert im is not None, f'Image Not Found {f}' + h0, w0 = im.shape[:2] # orig hw + r = self.img_size / max(h0, w0) # ratio + if r != 1: # if sizes are not equal + interp = cv2.INTER_LINEAR if (self.augment or r > 1) else cv2.INTER_AREA + im = cv2.resize(im, (math.ceil(w0 * r), math.ceil(h0 * r)), interpolation=interp) + return im, (h0, w0), im.shape[:2] # im, hw_original, hw_resized + return self.ims[i], self.im_hw0[i], self.im_hw[i] # im, hw_original, hw_resized + + def cache_images_to_disk(self, i): + # Saves an image as an *.npy file for faster loading + f = self.npy_files[i] + if not f.exists(): + np.save(f.as_posix(), cv2.imread(self.im_files[i])) + + def load_mosaic(self, index): + # YOLOv5 4-mosaic loader. Loads 1 image + 3 random images into a 4-image mosaic + labels4, segments4 = [], [] + s = self.img_size + yc, xc = (int(random.uniform(-x, 2 * s + x)) for x in self.mosaic_border) # mosaic center x, y + indices = [index] + random.choices(self.indices, k=3) # 3 additional image indices + random.shuffle(indices) + for i, index in enumerate(indices): + # Load image + img, _, (h, w) = self.load_image(index) + + # place img in img4 + if i == 0: # top left + img4 = np.full((s * 2, s * 2, img.shape[2]), 114, dtype=np.uint8) # base image with 4 tiles + x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc # xmin, ymin, xmax, ymax (large image) + x1b, y1b, x2b, y2b = w - (x2a - x1a), h - (y2a - y1a), w, h # xmin, ymin, xmax, ymax (small image) + elif i == 1: # top right + x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc + x1b, y1b, x2b, y2b = 0, h - (y2a - y1a), min(w, x2a - x1a), h + elif i == 2: # bottom left + x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h) + x1b, y1b, x2b, y2b = w - (x2a - x1a), 0, w, min(y2a - y1a, h) + elif i == 3: # bottom right + x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h) + x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h) + + img4[y1a:y2a, x1a:x2a] = img[y1b:y2b, x1b:x2b] # img4[ymin:ymax, xmin:xmax] + padw = x1a - x1b + padh = y1a - y1b + + # Labels + labels, segments = self.labels[index].copy(), self.segments[index].copy() + if labels.size: + labels[:, 1:] = xywhn2xyxy(labels[:, 1:], w, h, padw, padh) # normalized xywh to pixel xyxy format + segments = [xyn2xy(x, w, h, padw, padh) for x in segments] + labels4.append(labels) + segments4.extend(segments) + + # Concat/clip labels + labels4 = np.concatenate(labels4, 0) + for x in (labels4[:, 1:], *segments4): + np.clip(x, 0, 2 * s, out=x) # clip when using random_perspective() + # img4, labels4 = replicate(img4, labels4) # replicate + + # Augment + img4, labels4, segments4 = copy_paste(img4, labels4, segments4, p=self.hyp['copy_paste']) + img4, labels4 = random_perspective(img4, + labels4, + segments4, + degrees=self.hyp['degrees'], + translate=self.hyp['translate'], + scale=self.hyp['scale'], + shear=self.hyp['shear'], + perspective=self.hyp['perspective'], + border=self.mosaic_border) # border to remove + + return img4, labels4 + + def load_mosaic9(self, index): + # YOLOv5 9-mosaic loader. Loads 1 image + 8 random images into a 9-image mosaic + labels9, segments9 = [], [] + s = self.img_size + indices = [index] + random.choices(self.indices, k=8) # 8 additional image indices + random.shuffle(indices) + hp, wp = -1, -1 # height, width previous + for i, index in enumerate(indices): + # Load image + img, _, (h, w) = self.load_image(index) + + # place img in img9 + if i == 0: # center + img9 = np.full((s * 3, s * 3, img.shape[2]), 114, dtype=np.uint8) # base image with 4 tiles + h0, w0 = h, w + c = s, s, s + w, s + h # xmin, ymin, xmax, ymax (base) coordinates + elif i == 1: # top + c = s, s - h, s + w, s + elif i == 2: # top right + c = s + wp, s - h, s + wp + w, s + elif i == 3: # right + c = s + w0, s, s + w0 + w, s + h + elif i == 4: # bottom right + c = s + w0, s + hp, s + w0 + w, s + hp + h + elif i == 5: # bottom + c = s + w0 - w, s + h0, s + w0, s + h0 + h + elif i == 6: # bottom left + c = s + w0 - wp - w, s + h0, s + w0 - wp, s + h0 + h + elif i == 7: # left + c = s - w, s + h0 - h, s, s + h0 + elif i == 8: # top left + c = s - w, s + h0 - hp - h, s, s + h0 - hp + + padx, pady = c[:2] + x1, y1, x2, y2 = (max(x, 0) for x in c) # allocate coords + + # Labels + labels, segments = self.labels[index].copy(), self.segments[index].copy() + if labels.size: + labels[:, 1:] = xywhn2xyxy(labels[:, 1:], w, h, padx, pady) # normalized xywh to pixel xyxy format + segments = [xyn2xy(x, w, h, padx, pady) for x in segments] + labels9.append(labels) + segments9.extend(segments) + + # Image + img9[y1:y2, x1:x2] = img[y1 - pady:, x1 - padx:] # img9[ymin:ymax, xmin:xmax] + hp, wp = h, w # height, width previous + + # Offset + yc, xc = (int(random.uniform(0, s)) for _ in self.mosaic_border) # mosaic center x, y + img9 = img9[yc:yc + 2 * s, xc:xc + 2 * s] + + # Concat/clip labels + labels9 = np.concatenate(labels9, 0) + labels9[:, [1, 3]] -= xc + labels9[:, [2, 4]] -= yc + c = np.array([xc, yc]) # centers + segments9 = [x - c for x in segments9] + + for x in (labels9[:, 1:], *segments9): + np.clip(x, 0, 2 * s, out=x) # clip when using random_perspective() + # img9, labels9 = replicate(img9, labels9) # replicate + + # Augment + img9, labels9, segments9 = copy_paste(img9, labels9, segments9, p=self.hyp['copy_paste']) + img9, labels9 = random_perspective(img9, + labels9, + segments9, + degrees=self.hyp['degrees'], + translate=self.hyp['translate'], + scale=self.hyp['scale'], + shear=self.hyp['shear'], + perspective=self.hyp['perspective'], + border=self.mosaic_border) # border to remove + + return img9, labels9 + + @staticmethod + def collate_fn(batch): + im, label, path, shapes = zip(*batch) # transposed + for i, lb in enumerate(label): + lb[:, 0] = i # add target image index for build_targets() + return torch.stack(im, 0), torch.cat(label, 0), path, shapes + + @staticmethod + def collate_fn4(batch): + im, label, path, shapes = zip(*batch) # transposed + n = len(shapes) // 4 + im4, label4, path4, shapes4 = [], [], path[:n], shapes[:n] + + ho = torch.tensor([[0.0, 0, 0, 1, 0, 0]]) + wo = torch.tensor([[0.0, 0, 1, 0, 0, 0]]) + s = torch.tensor([[1, 1, 0.5, 0.5, 0.5, 0.5]]) # scale + for i in range(n): # zidane torch.zeros(16,3,720,1280) # BCHW + i *= 4 + if random.random() < 0.5: + im1 = F.interpolate(im[i].unsqueeze(0).float(), scale_factor=2.0, mode='bilinear', + align_corners=False)[0].type(im[i].type()) + lb = label[i] + else: + im1 = torch.cat((torch.cat((im[i], im[i + 1]), 1), torch.cat((im[i + 2], im[i + 3]), 1)), 2) + lb = torch.cat((label[i], label[i + 1] + ho, label[i + 2] + wo, label[i + 3] + ho + wo), 0) * s + im4.append(im1) + label4.append(lb) + + for i, lb in enumerate(label4): + lb[:, 0] = i # add target image index for build_targets() + + return torch.stack(im4, 0), torch.cat(label4, 0), path4, shapes4 + + +# Ancillary functions -------------------------------------------------------------------------------------------------- +def flatten_recursive(path=DATASETS_DIR / 'coco128'): + # Flatten a recursive directory by bringing all files to top level + new_path = Path(f'{str(path)}_flat') + if os.path.exists(new_path): + shutil.rmtree(new_path) # delete output folder + os.makedirs(new_path) # make new output folder + for file in tqdm(glob.glob(f'{str(Path(path))}/**/*.*', recursive=True)): + shutil.copyfile(file, new_path / Path(file).name) + + +def extract_boxes(path=DATASETS_DIR / 'coco128'): # from utils.dataloaders import *; extract_boxes() + # Convert detection dataset into classification dataset, with one directory per class + path = Path(path) # images dir + shutil.rmtree(path / 'classification') if (path / 'classification').is_dir() else None # remove existing + files = list(path.rglob('*.*')) + n = len(files) # number of files + for im_file in tqdm(files, total=n): + if im_file.suffix[1:] in IMG_FORMATS: + # image + im = cv2.imread(str(im_file))[..., ::-1] # BGR to RGB + h, w = im.shape[:2] + + # labels + lb_file = Path(img2label_paths([str(im_file)])[0]) + if Path(lb_file).exists(): + with open(lb_file) as f: + lb = np.array([x.split() for x in f.read().strip().splitlines()], dtype=np.float32) # labels + + for j, x in enumerate(lb): + c = int(x[0]) # class + f = (path / 'classifier') / f'{c}' / f'{path.stem}_{im_file.stem}_{j}.jpg' # new filename + if not f.parent.is_dir(): + f.parent.mkdir(parents=True) + + b = x[1:] * [w, h, w, h] # box + # b[2:] = b[2:].max() # rectangle to square + b[2:] = b[2:] * 1.2 + 3 # pad + b = xywh2xyxy(b.reshape(-1, 4)).ravel().astype(int) + + b[[0, 2]] = np.clip(b[[0, 2]], 0, w) # clip boxes outside of image + b[[1, 3]] = np.clip(b[[1, 3]], 0, h) + assert cv2.imwrite(str(f), im[b[1]:b[3], b[0]:b[2]]), f'box failure in {f}' + + +def autosplit(path=DATASETS_DIR / 'coco128/images', weights=(0.9, 0.1, 0.0), annotated_only=False): + """ Autosplit a dataset into train/val/test splits and save path/autosplit_*.txt files + Usage: from utils.dataloaders import *; autosplit() + Arguments + path: Path to images directory + weights: Train, val, test weights (list, tuple) + annotated_only: Only use images with an annotated txt file + """ + path = Path(path) # images dir + files = sorted(x for x in path.rglob('*.*') if x.suffix[1:].lower() in IMG_FORMATS) # image files only + n = len(files) # number of files + random.seed(0) # for reproducibility + indices = random.choices([0, 1, 2], weights=weights, k=n) # assign each image to a split + + txt = ['autosplit_train.txt', 'autosplit_val.txt', 'autosplit_test.txt'] # 3 txt files + for x in txt: + if (path.parent / x).exists(): + (path.parent / x).unlink() # remove existing + + print(f'Autosplitting images from {path}' + ', using *.txt labeled images only' * annotated_only) + for i, img in tqdm(zip(indices, files), total=n): + if not annotated_only or Path(img2label_paths([str(img)])[0]).exists(): # check label + with open(path.parent / txt[i], 'a') as f: + f.write(f'./{img.relative_to(path.parent).as_posix()}' + '\n') # add image to txt file + + +def verify_image_label(args): + # Verify one image-label pair + im_file, lb_file, prefix = args + nm, nf, ne, nc, msg, segments = 0, 0, 0, 0, '', [] # number (missing, found, empty, corrupt), message, segments + try: + # verify images + im = Image.open(im_file) + im.verify() # PIL verify + shape = exif_size(im) # image size + assert (shape[0] > 9) & (shape[1] > 9), f'image size {shape} <10 pixels' + assert im.format.lower() in IMG_FORMATS, f'invalid image format {im.format}' + if im.format.lower() in ('jpg', 'jpeg'): + with open(im_file, 'rb') as f: + f.seek(-2, 2) + if f.read() != b'\xff\xd9': # corrupt JPEG + ImageOps.exif_transpose(Image.open(im_file)).save(im_file, 'JPEG', subsampling=0, quality=100) + msg = f'{prefix}WARNING ⚠️ {im_file}: corrupt JPEG restored and saved' + + # verify labels + if os.path.isfile(lb_file): + nf = 1 # label found + with open(lb_file) as f: + lb = [x.split() for x in f.read().strip().splitlines() if len(x)] + if any(len(x) > 6 for x in lb): # is segment + classes = np.array([x[0] for x in lb], dtype=np.float32) + segments = [np.array(x[1:], dtype=np.float32).reshape(-1, 2) for x in lb] # (cls, xy1...) + lb = np.concatenate((classes.reshape(-1, 1), segments2boxes(segments)), 1) # (cls, xywh) + lb = np.array(lb, dtype=np.float32) + nl = len(lb) + if nl: + assert lb.shape[1] == 5, f'labels require 5 columns, {lb.shape[1]} columns detected' + assert (lb >= 0).all(), f'negative label values {lb[lb < 0]}' + assert (lb[:, 1:] <= 1).all(), f'non-normalized or out of bounds coordinates {lb[:, 1:][lb[:, 1:] > 1]}' + _, i = np.unique(lb, axis=0, return_index=True) + if len(i) < nl: # duplicate row check + lb = lb[i] # remove duplicates + if segments: + segments = [segments[x] for x in i] + msg = f'{prefix}WARNING ⚠️ {im_file}: {nl - len(i)} duplicate labels removed' + else: + ne = 1 # label empty + lb = np.zeros((0, 5), dtype=np.float32) + else: + nm = 1 # label missing + lb = np.zeros((0, 5), dtype=np.float32) + return im_file, lb, shape, segments, nm, nf, ne, nc, msg + except Exception as e: + nc = 1 + msg = f'{prefix}WARNING ⚠️ {im_file}: ignoring corrupt image/label: {e}' + return [None, None, None, None, nm, nf, ne, nc, msg] + + +class HUBDatasetStats(): + """ Class for generating HUB dataset JSON and `-hub` dataset directory + + Arguments + path: Path to data.yaml or data.zip (with data.yaml inside data.zip) + autodownload: Attempt to download dataset if not found locally + + Usage + from utils.dataloaders import HUBDatasetStats + stats = HUBDatasetStats('coco128.yaml', autodownload=True) # usage 1 + stats = HUBDatasetStats('path/to/coco128.zip') # usage 2 + stats.get_json(save=False) + stats.process_images() + """ + + def __init__(self, path='coco128.yaml', autodownload=False): + # Initialize class + zipped, data_dir, yaml_path = self._unzip(Path(path)) + try: + with open(check_yaml(yaml_path), errors='ignore') as f: + data = yaml.safe_load(f) # data dict + if zipped: + data['path'] = data_dir + except Exception as e: + raise Exception('error/HUB/dataset_stats/yaml_load') from e + + check_dataset(data, autodownload) # download dataset if missing + self.hub_dir = Path(data['path'] + '-hub') + self.im_dir = self.hub_dir / 'images' + self.im_dir.mkdir(parents=True, exist_ok=True) # makes /images + self.stats = {'nc': data['nc'], 'names': list(data['names'].values())} # statistics dictionary + self.data = data + + @staticmethod + def _find_yaml(dir): + # Return data.yaml file + files = list(dir.glob('*.yaml')) or list(dir.rglob('*.yaml')) # try root level first and then recursive + assert files, f'No *.yaml file found in {dir}' + if len(files) > 1: + files = [f for f in files if f.stem == dir.stem] # prefer *.yaml files that match dir name + assert files, f'Multiple *.yaml files found in {dir}, only 1 *.yaml file allowed' + assert len(files) == 1, f'Multiple *.yaml files found: {files}, only 1 *.yaml file allowed in {dir}' + return files[0] + + def _unzip(self, path): + # Unzip data.zip + if not str(path).endswith('.zip'): # path is data.yaml + return False, None, path + assert Path(path).is_file(), f'Error unzipping {path}, file not found' + unzip_file(path, path=path.parent) + dir = path.with_suffix('') # dataset directory == zip name + assert dir.is_dir(), f'Error unzipping {path}, {dir} not found. path/to/abc.zip MUST unzip to path/to/abc/' + return True, str(dir), self._find_yaml(dir) # zipped, data_dir, yaml_path + + def _hub_ops(self, f, max_dim=1920): + # HUB ops for 1 image 'f': resize and save at reduced quality in /dataset-hub for web/app viewing + f_new = self.im_dir / Path(f).name # dataset-hub image filename + try: # use PIL + im = Image.open(f) + r = max_dim / max(im.height, im.width) # ratio + if r < 1.0: # image too large + im = im.resize((int(im.width * r), int(im.height * r))) + im.save(f_new, 'JPEG', quality=50, optimize=True) # save + except Exception as e: # use OpenCV + LOGGER.info(f'WARNING ⚠️ HUB ops PIL failure {f}: {e}') + im = cv2.imread(f) + im_height, im_width = im.shape[:2] + r = max_dim / max(im_height, im_width) # ratio + if r < 1.0: # image too large + im = cv2.resize(im, (int(im_width * r), int(im_height * r)), interpolation=cv2.INTER_AREA) + cv2.imwrite(str(f_new), im) + + def get_json(self, save=False, verbose=False): + # Return dataset JSON for Ultralytics HUB + def _round(labels): + # Update labels to integer class and 6 decimal place floats + return [[int(c), *(round(x, 4) for x in points)] for c, *points in labels] + + for split in 'train', 'val', 'test': + if self.data.get(split) is None: + self.stats[split] = None # i.e. no test set + continue + dataset = LoadImagesAndLabels(self.data[split]) # load dataset + x = np.array([ + np.bincount(label[:, 0].astype(int), minlength=self.data['nc']) + for label in tqdm(dataset.labels, total=dataset.n, desc='Statistics')]) # shape(128x80) + self.stats[split] = { + 'instance_stats': { + 'total': int(x.sum()), + 'per_class': x.sum(0).tolist()}, + 'image_stats': { + 'total': dataset.n, + 'unlabelled': int(np.all(x == 0, 1).sum()), + 'per_class': (x > 0).sum(0).tolist()}, + 'labels': [{ + str(Path(k).name): _round(v.tolist())} for k, v in zip(dataset.im_files, dataset.labels)]} + + # Save, print and return + if save: + stats_path = self.hub_dir / 'stats.json' + print(f'Saving {stats_path.resolve()}...') + with open(stats_path, 'w') as f: + json.dump(self.stats, f) # save stats.json + if verbose: + print(json.dumps(self.stats, indent=2, sort_keys=False)) + return self.stats + + def process_images(self): + # Compress images for Ultralytics HUB + for split in 'train', 'val', 'test': + if self.data.get(split) is None: + continue + dataset = LoadImagesAndLabels(self.data[split]) # load dataset + desc = f'{split} images' + for _ in tqdm(ThreadPool(NUM_THREADS).imap(self._hub_ops, dataset.im_files), total=dataset.n, desc=desc): + pass + print(f'Done. All images saved to {self.im_dir}') + return self.im_dir + + +# Classification dataloaders ------------------------------------------------------------------------------------------- +class ClassificationDataset(torchvision.datasets.ImageFolder): + """ + YOLOv5 Classification Dataset. + Arguments + root: Dataset path + transform: torchvision transforms, used by default + album_transform: Albumentations transforms, used if installed + """ + + def __init__(self, root, augment, imgsz, cache=False): + super().__init__(root=root) + self.torch_transforms = classify_transforms(imgsz) + self.album_transforms = classify_albumentations(augment, imgsz) if augment else None + self.cache_ram = cache is True or cache == 'ram' + self.cache_disk = cache == 'disk' + self.samples = [list(x) + [Path(x[0]).with_suffix('.npy'), None] for x in self.samples] # file, index, npy, im + + def __getitem__(self, i): + f, j, fn, im = self.samples[i] # filename, index, filename.with_suffix('.npy'), image + if self.cache_ram and im is None: + im = self.samples[i][3] = cv2.imread(f) + elif self.cache_disk: + if not fn.exists(): # load npy + np.save(fn.as_posix(), cv2.imread(f)) + im = np.load(fn) + else: # read image + im = cv2.imread(f) # BGR + if self.album_transforms: + sample = self.album_transforms(image=cv2.cvtColor(im, cv2.COLOR_BGR2RGB))['image'] + else: + sample = self.torch_transforms(im) + return sample, j + + +def create_classification_dataloader(path, + imgsz=224, + batch_size=16, + augment=True, + cache=False, + rank=-1, + workers=8, + shuffle=True): + # Returns Dataloader object to be used with YOLOv5 Classifier + with torch_distributed_zero_first(rank): # init dataset *.cache only once if DDP + dataset = ClassificationDataset(root=path, imgsz=imgsz, augment=augment, cache=cache) + batch_size = min(batch_size, len(dataset)) + nd = torch.cuda.device_count() + nw = min([os.cpu_count() // max(nd, 1), batch_size if batch_size > 1 else 0, workers]) + sampler = None if rank == -1 else distributed.DistributedSampler(dataset, shuffle=shuffle) + generator = torch.Generator() + generator.manual_seed(6148914691236517205 + RANK) + return InfiniteDataLoader(dataset, + batch_size=batch_size, + shuffle=shuffle and sampler is None, + num_workers=nw, + sampler=sampler, + pin_memory=PIN_MEMORY, + worker_init_fn=seed_worker, + generator=generator) # or DataLoader(persistent_workers=True) diff --git a/utils/docker/Dockerfile b/utils/docker/Dockerfile new file mode 100644 index 0000000..d1fc87f --- /dev/null +++ b/utils/docker/Dockerfile @@ -0,0 +1,73 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +# Builds ultralytics/yolov5:latest image on DockerHub https://hub.docker.com/r/ultralytics/yolov5 +# Image is CUDA-optimized for YOLOv5 single/multi-GPU training and inference + +# Start FROM PyTorch image https://hub.docker.com/r/pytorch/pytorch +FROM pytorch/pytorch:2.0.0-cuda11.7-cudnn8-runtime + +# Downloads to user config dir +ADD https://ultralytics.com/assets/Arial.ttf https://ultralytics.com/assets/Arial.Unicode.ttf /root/.config/Ultralytics/ + +# Install linux packages +ENV DEBIAN_FRONTEND noninteractive +RUN apt update +RUN TZ=Etc/UTC apt install -y tzdata +RUN apt install --no-install-recommends -y gcc git zip curl htop libgl1-mesa-glx libglib2.0-0 libpython3-dev gnupg +# RUN alias python=python3 + +# Security updates +# https://security.snyk.io/vuln/SNYK-UBUNTU1804-OPENSSL-3314796 +RUN apt upgrade --no-install-recommends -y openssl + +# Create working directory +RUN rm -rf /usr/src/app && mkdir -p /usr/src/app +WORKDIR /usr/src/app + +# Copy contents +COPY . /usr/src/app + +# Install pip packages +COPY requirements.txt . +RUN python3 -m pip install --upgrade pip wheel +RUN pip install --no-cache -r requirements.txt albumentations comet gsutil notebook \ + coremltools onnx onnx-simplifier onnxruntime 'openvino-dev>=2023.0' + # tensorflow tensorflowjs \ + +# Set environment variables +ENV OMP_NUM_THREADS=1 + +# Cleanup +ENV DEBIAN_FRONTEND teletype + + +# Usage Examples ------------------------------------------------------------------------------------------------------- + +# Build and Push +# t=ultralytics/yolov5:latest && sudo docker build -f utils/docker/Dockerfile -t $t . && sudo docker push $t + +# Pull and Run +# t=ultralytics/yolov5:latest && sudo docker pull $t && sudo docker run -it --ipc=host --gpus all $t + +# Pull and Run with local directory access +# t=ultralytics/yolov5:latest && sudo docker pull $t && sudo docker run -it --ipc=host --gpus all -v "$(pwd)"/datasets:/usr/src/datasets $t + +# Kill all +# sudo docker kill $(sudo docker ps -q) + +# Kill all image-based +# sudo docker kill $(sudo docker ps -qa --filter ancestor=ultralytics/yolov5:latest) + +# DockerHub tag update +# t=ultralytics/yolov5:latest tnew=ultralytics/yolov5:v6.2 && sudo docker pull $t && sudo docker tag $t $tnew && sudo docker push $tnew + +# Clean up +# sudo docker system prune -a --volumes + +# Update Ubuntu drivers +# https://www.maketecheasier.com/install-nvidia-drivers-ubuntu/ + +# DDP test +# python -m torch.distributed.run --nproc_per_node 2 --master_port 1 train.py --epochs 3 + +# GCP VM from Image +# docker.io/ultralytics/yolov5:latest diff --git a/utils/docker/Dockerfile-arm64 b/utils/docker/Dockerfile-arm64 new file mode 100644 index 0000000..802703e --- /dev/null +++ b/utils/docker/Dockerfile-arm64 @@ -0,0 +1,40 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +# Builds ultralytics/yolov5:latest-arm64 image on DockerHub https://hub.docker.com/r/ultralytics/yolov5 +# Image is aarch64-compatible for Apple M1 and other ARM architectures i.e. Jetson Nano and Raspberry Pi + +# Start FROM Ubuntu image https://hub.docker.com/_/ubuntu +FROM arm64v8/ubuntu:22.10 + +# Downloads to user config dir +ADD https://ultralytics.com/assets/Arial.ttf https://ultralytics.com/assets/Arial.Unicode.ttf /root/.config/Ultralytics/ + +# Install linux packages +ENV DEBIAN_FRONTEND noninteractive +RUN apt update +RUN TZ=Etc/UTC apt install -y tzdata +RUN apt install --no-install-recommends -y python3-pip git zip curl htop gcc libgl1-mesa-glx libglib2.0-0 libpython3-dev +# RUN alias python=python3 + +# Install pip packages +COPY requirements.txt . +RUN python3 -m pip install --upgrade pip wheel +RUN pip install --no-cache -r requirements.txt albumentations gsutil notebook \ + coremltools onnx onnxruntime + # tensorflow-aarch64 tensorflowjs \ + +# Create working directory +RUN mkdir -p /usr/src/app +WORKDIR /usr/src/app + +# Copy contents +COPY . /usr/src/app +ENV DEBIAN_FRONTEND teletype + + +# Usage Examples ------------------------------------------------------------------------------------------------------- + +# Build and Push +# t=ultralytics/yolov5:latest-arm64 && sudo docker build --platform linux/arm64 -f utils/docker/Dockerfile-arm64 -t $t . && sudo docker push $t + +# Pull and Run +# t=ultralytics/yolov5:latest-arm64 && sudo docker pull $t && sudo docker run -it --ipc=host -v "$(pwd)"/datasets:/usr/src/datasets $t diff --git a/utils/docker/Dockerfile-cpu b/utils/docker/Dockerfile-cpu new file mode 100644 index 0000000..2d1363a --- /dev/null +++ b/utils/docker/Dockerfile-cpu @@ -0,0 +1,42 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +# Builds ultralytics/yolov5:latest-cpu image on DockerHub https://hub.docker.com/r/ultralytics/yolov5 +# Image is CPU-optimized for ONNX, OpenVINO and PyTorch YOLOv5 deployments + +# Start FROM Ubuntu image https://hub.docker.com/_/ubuntu +FROM ubuntu:lunar-20230615 + +# Downloads to user config dir +ADD https://ultralytics.com/assets/Arial.ttf https://ultralytics.com/assets/Arial.Unicode.ttf /root/.config/Ultralytics/ + +# Install linux packages +# g++ required to build 'tflite_support' and 'lap' packages, libusb-1.0-0 required for 'tflite_support' package +RUN apt update \ + && apt install --no-install-recommends -y python3-pip git zip curl htop libgl1-mesa-glx libglib2.0-0 libpython3-dev gnupg g++ libusb-1.0-0 +# RUN alias python=python3 + +# Remove python3.11/EXTERNALLY-MANAGED or use 'pip install --break-system-packages' avoid 'externally-managed-environment' Ubuntu nightly error +RUN rm -rf /usr/lib/python3.11/EXTERNALLY-MANAGED + +# Install pip packages +COPY requirements.txt . +RUN python3 -m pip install --upgrade pip wheel +RUN pip install --no-cache -r requirements.txt albumentations gsutil notebook \ + coremltools onnx onnx-simplifier onnxruntime 'openvino-dev>=2023.0' \ + # tensorflow tensorflowjs \ + --extra-index-url https://download.pytorch.org/whl/cpu + +# Create working directory +RUN mkdir -p /usr/src/app +WORKDIR /usr/src/app + +# Copy contents +COPY . /usr/src/app + + +# Usage Examples ------------------------------------------------------------------------------------------------------- + +# Build and Push +# t=ultralytics/yolov5:latest-cpu && sudo docker build -f utils/docker/Dockerfile-cpu -t $t . && sudo docker push $t + +# Pull and Run +# t=ultralytics/yolov5:latest-cpu && sudo docker pull $t && sudo docker run -it --ipc=host -v "$(pwd)"/datasets:/usr/src/datasets $t diff --git a/utils/downloads.py b/utils/downloads.py new file mode 100644 index 0000000..9298259 --- /dev/null +++ b/utils/downloads.py @@ -0,0 +1,127 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Download utils +""" + +import logging +import subprocess +import urllib +from pathlib import Path + +import requests +import torch + + +def is_url(url, check=True): + # Check if string is URL and check if URL exists + try: + url = str(url) + result = urllib.parse.urlparse(url) + assert all([result.scheme, result.netloc]) # check if is url + return (urllib.request.urlopen(url).getcode() == 200) if check else True # check if exists online + except (AssertionError, urllib.request.HTTPError): + return False + + +def gsutil_getsize(url=''): + # gs://bucket/file size https://cloud.google.com/storage/docs/gsutil/commands/du + output = subprocess.check_output(['gsutil', 'du', url], shell=True, encoding='utf-8') + if output: + return int(output.split()[0]) + return 0 + + +def url_getsize(url='https://ultralytics.com/images/bus.jpg'): + # Return downloadable file size in bytes + response = requests.head(url, allow_redirects=True) + return int(response.headers.get('content-length', -1)) + + +def curl_download(url, filename, *, silent: bool = False) -> bool: + """ + Download a file from a url to a filename using curl. + """ + silent_option = 'sS' if silent else '' # silent + proc = subprocess.run([ + 'curl', + '-#', + f'-{silent_option}L', + url, + '--output', + filename, + '--retry', + '9', + '-C', + '-', ]) + return proc.returncode == 0 + + +def safe_download(file, url, url2=None, min_bytes=1E0, error_msg=''): + # Attempts to download file from url or url2, checks and removes incomplete downloads < min_bytes + from utils.general import LOGGER + + file = Path(file) + assert_msg = f"Downloaded file '{file}' does not exist or size is < min_bytes={min_bytes}" + try: # url1 + LOGGER.info(f'Downloading {url} to {file}...') + torch.hub.download_url_to_file(url, str(file), progress=LOGGER.level <= logging.INFO) + assert file.exists() and file.stat().st_size > min_bytes, assert_msg # check + except Exception as e: # url2 + if file.exists(): + file.unlink() # remove partial downloads + LOGGER.info(f'ERROR: {e}\nRe-attempting {url2 or url} to {file}...') + # curl download, retry and resume on fail + curl_download(url2 or url, file) + finally: + if not file.exists() or file.stat().st_size < min_bytes: # check + if file.exists(): + file.unlink() # remove partial downloads + LOGGER.info(f'ERROR: {assert_msg}\n{error_msg}') + LOGGER.info('') + + +def attempt_download(file, repo='ultralytics/yolov5', release='v7.0'): + # Attempt file download from GitHub release assets if not found locally. release = 'latest', 'v7.0', etc. + from utils.general import LOGGER + + def github_assets(repository, version='latest'): + # Return GitHub repo tag (i.e. 'v7.0') and assets (i.e. ['yolov5s.pt', 'yolov5m.pt', ...]) + if version != 'latest': + version = f'tags/{version}' # i.e. tags/v7.0 + response = requests.get(f'https://api.github.com/repos/{repository}/releases/{version}').json() # github api + return response['tag_name'], [x['name'] for x in response['assets']] # tag, assets + + file = Path(str(file).strip().replace("'", '')) + if not file.exists(): + # URL specified + name = Path(urllib.parse.unquote(str(file))).name # decode '%2F' to '/' etc. + if str(file).startswith(('http:/', 'https:/')): # download + url = str(file).replace(':/', '://') # Pathlib turns :// -> :/ + file = name.split('?')[0] # parse authentication https://url.com/file.txt?auth... + if Path(file).is_file(): + LOGGER.info(f'Found {url} locally at {file}') # file already exists + else: + safe_download(file=file, url=url, min_bytes=1E5) + return file + + # GitHub assets + assets = [f'yolov5{size}{suffix}.pt' for size in 'nsmlx' for suffix in ('', '6', '-cls', '-seg')] # default + try: + tag, assets = github_assets(repo, release) + except Exception: + try: + tag, assets = github_assets(repo) # latest release + except Exception: + try: + tag = subprocess.check_output('git tag', shell=True, stderr=subprocess.STDOUT).decode().split()[-1] + except Exception: + tag = release + + if name in assets: + file.parent.mkdir(parents=True, exist_ok=True) # make parent dir (if required) + safe_download(file, + url=f'https://github.com/{repo}/releases/download/{tag}/{name}', + min_bytes=1E5, + error_msg=f'{file} missing, try downloading from https://github.com/{repo}/releases/{tag}') + + return str(file) diff --git a/utils/flask_rest_api/README.md b/utils/flask_rest_api/README.md new file mode 100644 index 0000000..a726acb --- /dev/null +++ b/utils/flask_rest_api/README.md @@ -0,0 +1,73 @@ +# Flask REST API + +[REST](https://en.wikipedia.org/wiki/Representational_state_transfer) [API](https://en.wikipedia.org/wiki/API)s are +commonly used to expose Machine Learning (ML) models to other services. This folder contains an example REST API +created using Flask to expose the YOLOv5s model from [PyTorch Hub](https://pytorch.org/hub/ultralytics_yolov5/). + +## Requirements + +[Flask](https://palletsprojects.com/p/flask/) is required. Install with: + +```shell +$ pip install Flask +``` + +## Run + +After Flask installation run: + +```shell +$ python3 restapi.py --port 5000 +``` + +Then use [curl](https://curl.se/) to perform a request: + +```shell +$ curl -X POST -F image=@zidane.jpg 'http://localhost:5000/v1/object-detection/yolov5s' +``` + +The model inference results are returned as a JSON response: + +```json +[ + { + "class": 0, + "confidence": 0.8900438547, + "height": 0.9318675399, + "name": "person", + "width": 0.3264600933, + "xcenter": 0.7438579798, + "ycenter": 0.5207948685 + }, + { + "class": 0, + "confidence": 0.8440024257, + "height": 0.7155083418, + "name": "person", + "width": 0.6546785235, + "xcenter": 0.427829951, + "ycenter": 0.6334488392 + }, + { + "class": 27, + "confidence": 0.3771208823, + "height": 0.3902671337, + "name": "tie", + "width": 0.0696444362, + "xcenter": 0.3675483763, + "ycenter": 0.7991207838 + }, + { + "class": 27, + "confidence": 0.3527112305, + "height": 0.1540903747, + "name": "tie", + "width": 0.0336618312, + "xcenter": 0.7814827561, + "ycenter": 0.5065554976 + } +] +``` + +An example python script to perform inference using [requests](https://docs.python-requests.org/en/master/) is given +in `example_request.py` diff --git a/utils/flask_rest_api/example_request.py b/utils/flask_rest_api/example_request.py new file mode 100644 index 0000000..256ad13 --- /dev/null +++ b/utils/flask_rest_api/example_request.py @@ -0,0 +1,19 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Perform test request +""" + +import pprint + +import requests + +DETECTION_URL = 'http://localhost:5000/v1/object-detection/yolov5s' +IMAGE = 'zidane.jpg' + +# Read image +with open(IMAGE, 'rb') as f: + image_data = f.read() + +response = requests.post(DETECTION_URL, files={'image': image_data}).json() + +pprint.pprint(response) diff --git a/utils/flask_rest_api/restapi.py b/utils/flask_rest_api/restapi.py new file mode 100644 index 0000000..ae4756b --- /dev/null +++ b/utils/flask_rest_api/restapi.py @@ -0,0 +1,48 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Run a Flask REST API exposing one or more YOLOv5s models +""" + +import argparse +import io + +import torch +from flask import Flask, request +from PIL import Image + +app = Flask(__name__) +models = {} + +DETECTION_URL = '/v1/object-detection/' + + +@app.route(DETECTION_URL, methods=['POST']) +def predict(model): + if request.method != 'POST': + return + + if request.files.get('image'): + # Method 1 + # with request.files["image"] as f: + # im = Image.open(io.BytesIO(f.read())) + + # Method 2 + im_file = request.files['image'] + im_bytes = im_file.read() + im = Image.open(io.BytesIO(im_bytes)) + + if model in models: + results = models[model](im, size=640) # reduce size=320 for faster inference + return results.pandas().xyxy[0].to_json(orient='records') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Flask API exposing YOLOv5 model') + parser.add_argument('--port', default=5000, type=int, help='port number') + parser.add_argument('--model', nargs='+', default=['yolov5s'], help='model(s) to run, i.e. --model yolov5n yolov5s') + opt = parser.parse_args() + + for m in opt.model: + models[m] = torch.hub.load('ultralytics/yolov5', m, force_reload=True, skip_validation=True) + + app.run(host='0.0.0.0', port=opt.port) # debug=True causes Restarting with stat diff --git a/utils/general.py b/utils/general.py new file mode 100644 index 0000000..1016781 --- /dev/null +++ b/utils/general.py @@ -0,0 +1,1125 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +General utils +""" + +import contextlib +import glob +import inspect +import logging +import logging.config +import math +import os +import platform +import random +import re +import signal +import subprocess +import sys +import time +import urllib +from copy import deepcopy +from datetime import datetime +from itertools import repeat +from multiprocessing.pool import ThreadPool +from pathlib import Path +from subprocess import check_output +from tarfile import is_tarfile +from typing import Optional +from zipfile import ZipFile, is_zipfile + +import cv2 +import numpy as np +import pandas as pd +import pkg_resources as pkg +import torch +import torchvision +import yaml + +# Import 'ultralytics' package or install if if missing +try: + import ultralytics + + assert hasattr(ultralytics, '__version__') # verify package is not directory +except (ImportError, AssertionError): + os.system('pip install -U ultralytics') + import ultralytics + +from ultralytics.utils.checks import check_requirements + +from utils import TryExcept, emojis +from utils.downloads import curl_download, gsutil_getsize +from utils.metrics import box_iou, fitness + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[1] # YOLOv5 root directory +RANK = int(os.getenv('RANK', -1)) + +# Settings +NUM_THREADS = min(8, max(1, os.cpu_count() - 1)) # number of YOLOv5 multiprocessing threads +DATASETS_DIR = Path(os.getenv('YOLOv5_DATASETS_DIR', ROOT.parent / 'datasets')) # global datasets directory +AUTOINSTALL = str(os.getenv('YOLOv5_AUTOINSTALL', True)).lower() == 'true' # global auto-install mode +VERBOSE = str(os.getenv('YOLOv5_VERBOSE', True)).lower() == 'true' # global verbose mode +TQDM_BAR_FORMAT = '{l_bar}{bar:10}{r_bar}' # tqdm bar format +FONT = 'Arial.ttf' # https://ultralytics.com/assets/Arial.ttf + +torch.set_printoptions(linewidth=320, precision=5, profile='long') +np.set_printoptions(linewidth=320, formatter={'float_kind': '{:11.5g}'.format}) # format short g, %precision=5 +pd.options.display.max_columns = 10 +cv2.setNumThreads(0) # prevent OpenCV from multithreading (incompatible with PyTorch DataLoader) +os.environ['NUMEXPR_MAX_THREADS'] = str(NUM_THREADS) # NumExpr max threads +os.environ['OMP_NUM_THREADS'] = '1' if platform.system() == 'darwin' else str(NUM_THREADS) # OpenMP (PyTorch and SciPy) +os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' # suppress verbose TF compiler warnings in Colab + + +def is_ascii(s=''): + # Is string composed of all ASCII (no UTF) characters? (note str().isascii() introduced in python 3.7) + s = str(s) # convert list, tuple, None, etc. to str + return len(s.encode().decode('ascii', 'ignore')) == len(s) + + +def is_chinese(s='人工智能'): + # Is string composed of any Chinese characters? + return bool(re.search('[\u4e00-\u9fff]', str(s))) + + +def is_colab(): + # Is environment a Google Colab instance? + return 'google.colab' in sys.modules + + +def is_jupyter(): + """ + Check if the current script is running inside a Jupyter Notebook. + Verified on Colab, Jupyterlab, Kaggle, Paperspace. + + Returns: + bool: True if running inside a Jupyter Notebook, False otherwise. + """ + with contextlib.suppress(Exception): + from IPython import get_ipython + return get_ipython() is not None + return False + + +def is_kaggle(): + # Is environment a Kaggle Notebook? + return os.environ.get('PWD') == '/kaggle/working' and os.environ.get('KAGGLE_URL_BASE') == 'https://www.kaggle.com' + + +def is_docker() -> bool: + """Check if the process runs inside a docker container.""" + if Path('/.dockerenv').exists(): + return True + try: # check if docker is in control groups + with open('/proc/self/cgroup') as file: + return any('docker' in line for line in file) + except OSError: + return False + + +def is_writeable(dir, test=False): + # Return True if directory has write permissions, test opening a file with write permissions if test=True + if not test: + return os.access(dir, os.W_OK) # possible issues on Windows + file = Path(dir) / 'tmp.txt' + try: + with open(file, 'w'): # open file with write permissions + pass + file.unlink() # remove file + return True + except OSError: + return False + + +LOGGING_NAME = 'yolov5' + + +def set_logging(name=LOGGING_NAME, verbose=True): + # sets up logging for the given name + rank = int(os.getenv('RANK', -1)) # rank in world for Multi-GPU trainings + level = logging.INFO if verbose and rank in {-1, 0} else logging.ERROR + logging.config.dictConfig({ + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + name: { + 'format': '%(message)s'}}, + 'handlers': { + name: { + 'class': 'logging.StreamHandler', + 'formatter': name, + 'level': level, }}, + 'loggers': { + name: { + 'level': level, + 'handlers': [name], + 'propagate': False, }}}) + + +set_logging(LOGGING_NAME) # run before defining LOGGER +LOGGER = logging.getLogger(LOGGING_NAME) # define globally (used in train.py, val.py, detect.py, etc.) +if platform.system() == 'Windows': + for fn in LOGGER.info, LOGGER.warning: + setattr(LOGGER, fn.__name__, lambda x: fn(emojis(x))) # emoji safe logging + + +def user_config_dir(dir='Ultralytics', env_var='YOLOV5_CONFIG_DIR'): + # Return path of user configuration directory. Prefer environment variable if exists. Make dir if required. + env = os.getenv(env_var) + if env: + path = Path(env) # use environment variable + else: + cfg = {'Windows': 'AppData/Roaming', 'Linux': '.config', 'Darwin': 'Library/Application Support'} # 3 OS dirs + path = Path.home() / cfg.get(platform.system(), '') # OS-specific config dir + path = (path if is_writeable(path) else Path('/tmp')) / dir # GCP and AWS lambda fix, only /tmp is writeable + path.mkdir(exist_ok=True) # make if required + return path + + +CONFIG_DIR = user_config_dir() # Ultralytics settings dir + + +class Profile(contextlib.ContextDecorator): + # YOLOv5 Profile class. Usage: @Profile() decorator or 'with Profile():' context manager + def __init__(self, t=0.0): + self.t = t + self.cuda = torch.cuda.is_available() + + def __enter__(self): + self.start = self.time() + return self + + def __exit__(self, type, value, traceback): + self.dt = self.time() - self.start # delta-time + self.t += self.dt # accumulate dt + + def time(self): + if self.cuda: + torch.cuda.synchronize() + return time.time() + + +class Timeout(contextlib.ContextDecorator): + # YOLOv5 Timeout class. Usage: @Timeout(seconds) decorator or 'with Timeout(seconds):' context manager + def __init__(self, seconds, *, timeout_msg='', suppress_timeout_errors=True): + self.seconds = int(seconds) + self.timeout_message = timeout_msg + self.suppress = bool(suppress_timeout_errors) + + def _timeout_handler(self, signum, frame): + raise TimeoutError(self.timeout_message) + + def __enter__(self): + if platform.system() != 'Windows': # not supported on Windows + signal.signal(signal.SIGALRM, self._timeout_handler) # Set handler for SIGALRM + signal.alarm(self.seconds) # start countdown for SIGALRM to be raised + + def __exit__(self, exc_type, exc_val, exc_tb): + if platform.system() != 'Windows': + signal.alarm(0) # Cancel SIGALRM if it's scheduled + if self.suppress and exc_type is TimeoutError: # Suppress TimeoutError + return True + + +class WorkingDirectory(contextlib.ContextDecorator): + # Usage: @WorkingDirectory(dir) decorator or 'with WorkingDirectory(dir):' context manager + def __init__(self, new_dir): + self.dir = new_dir # new dir + self.cwd = Path.cwd().resolve() # current dir + + def __enter__(self): + os.chdir(self.dir) + + def __exit__(self, exc_type, exc_val, exc_tb): + os.chdir(self.cwd) + + +def methods(instance): + # Get class/instance methods + return [f for f in dir(instance) if callable(getattr(instance, f)) and not f.startswith('__')] + + +def print_args(args: Optional[dict] = None, show_file=True, show_func=False): + # Print function arguments (optional args dict) + x = inspect.currentframe().f_back # previous frame + file, _, func, _, _ = inspect.getframeinfo(x) + if args is None: # get args automatically + args, _, _, frm = inspect.getargvalues(x) + args = {k: v for k, v in frm.items() if k in args} + try: + file = Path(file).resolve().relative_to(ROOT).with_suffix('') + except ValueError: + file = Path(file).stem + s = (f'{file}: ' if show_file else '') + (f'{func}: ' if show_func else '') + LOGGER.info(colorstr(s) + ', '.join(f'{k}={v}' for k, v in args.items())) + + +def init_seeds(seed=0, deterministic=False): + # Initialize random number generator (RNG) seeds https://pytorch.org/docs/stable/notes/randomness.html + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) # for Multi-GPU, exception safe + # torch.backends.cudnn.benchmark = True # AutoBatch problem https://github.com/ultralytics/yolov5/issues/9287 + if deterministic and check_version(torch.__version__, '1.12.0'): # https://github.com/ultralytics/yolov5/pull/8213 + torch.use_deterministic_algorithms(True) + torch.backends.cudnn.deterministic = True + os.environ['CUBLAS_WORKSPACE_CONFIG'] = ':4096:8' + os.environ['PYTHONHASHSEED'] = str(seed) + + +def intersect_dicts(da, db, exclude=()): + # Dictionary intersection of matching keys and shapes, omitting 'exclude' keys, using da values + return {k: v for k, v in da.items() if k in db and all(x not in k for x in exclude) and v.shape == db[k].shape} + + +def get_default_args(func): + # Get func() default arguments + signature = inspect.signature(func) + return {k: v.default for k, v in signature.parameters.items() if v.default is not inspect.Parameter.empty} + + +def get_latest_run(search_dir='.'): + # Return path to most recent 'last.pt' in /runs (i.e. to --resume from) + last_list = glob.glob(f'{search_dir}/**/last*.pt', recursive=True) + return max(last_list, key=os.path.getctime) if last_list else '' + + +def file_age(path=__file__): + # Return days since last file update + dt = (datetime.now() - datetime.fromtimestamp(Path(path).stat().st_mtime)) # delta + return dt.days # + dt.seconds / 86400 # fractional days + + +def file_date(path=__file__): + # Return human-readable file modification date, i.e. '2021-3-26' + t = datetime.fromtimestamp(Path(path).stat().st_mtime) + return f'{t.year}-{t.month}-{t.day}' + + +def file_size(path): + # Return file/dir size (MB) + mb = 1 << 20 # bytes to MiB (1024 ** 2) + path = Path(path) + if path.is_file(): + return path.stat().st_size / mb + elif path.is_dir(): + return sum(f.stat().st_size for f in path.glob('**/*') if f.is_file()) / mb + else: + return 0.0 + + +def check_online(): + # Check internet connectivity + import socket + + def run_once(): + # Check once + try: + socket.create_connection(('1.1.1.1', 443), 5) # check host accessibility + return True + except OSError: + return False + + return run_once() or run_once() # check twice to increase robustness to intermittent connectivity issues + + +def git_describe(path=ROOT): # path must be a directory + # Return human-readable git description, i.e. v5.0-5-g3e25f1e https://git-scm.com/docs/git-describe + try: + assert (Path(path) / '.git').is_dir() + return check_output(f'git -C {path} describe --tags --long --always', shell=True).decode()[:-1] + except Exception: + return '' + + +@TryExcept() +@WorkingDirectory(ROOT) +def check_git_status(repo='ultralytics/yolov5', branch='master'): + # YOLOv5 status check, recommend 'git pull' if code is out of date + url = f'https://github.com/{repo}' + msg = f', for updates see {url}' + s = colorstr('github: ') # string + assert Path('.git').exists(), s + 'skipping check (not a git repository)' + msg + assert check_online(), s + 'skipping check (offline)' + msg + + splits = re.split(pattern=r'\s', string=check_output('git remote -v', shell=True).decode()) + matches = [repo in s for s in splits] + if any(matches): + remote = splits[matches.index(True) - 1] + else: + remote = 'ultralytics' + check_output(f'git remote add {remote} {url}', shell=True) + check_output(f'git fetch {remote}', shell=True, timeout=5) # git fetch + local_branch = check_output('git rev-parse --abbrev-ref HEAD', shell=True).decode().strip() # checked out + n = int(check_output(f'git rev-list {local_branch}..{remote}/{branch} --count', shell=True)) # commits behind + if n > 0: + pull = 'git pull' if remote == 'origin' else f'git pull {remote} {branch}' + s += f"⚠️ YOLOv5 is out of date by {n} commit{'s' * (n > 1)}. Use '{pull}' or 'git clone {url}' to update." + else: + s += f'up to date with {url} ✅' + LOGGER.info(s) + + +@WorkingDirectory(ROOT) +def check_git_info(path='.'): + # YOLOv5 git info check, return {remote, branch, commit} + check_requirements('gitpython') + import git + try: + repo = git.Repo(path) + remote = repo.remotes.origin.url.replace('.git', '') # i.e. 'https://github.com/ultralytics/yolov5' + commit = repo.head.commit.hexsha # i.e. '3134699c73af83aac2a481435550b968d5792c0d' + try: + branch = repo.active_branch.name # i.e. 'main' + except TypeError: # not on any branch + branch = None # i.e. 'detached HEAD' state + return {'remote': remote, 'branch': branch, 'commit': commit} + except git.exc.InvalidGitRepositoryError: # path is not a git dir + return {'remote': None, 'branch': None, 'commit': None} + + +def check_python(minimum='3.8.0'): + # Check current python version vs. required python version + check_version(platform.python_version(), minimum, name='Python ', hard=True) + + +def check_version(current='0.0.0', minimum='0.0.0', name='version ', pinned=False, hard=False, verbose=False): + # Check version vs. required version + current, minimum = (pkg.parse_version(x) for x in (current, minimum)) + result = (current == minimum) if pinned else (current >= minimum) # bool + s = f'WARNING ⚠️ {name}{minimum} is required by YOLOv5, but {name}{current} is currently installed' # string + if hard: + assert result, emojis(s) # assert min requirements met + if verbose and not result: + LOGGER.warning(s) + return result + + +def check_img_size(imgsz, s=32, floor=0): + # Verify image size is a multiple of stride s in each dimension + if isinstance(imgsz, int): # integer i.e. img_size=640 + new_size = max(make_divisible(imgsz, int(s)), floor) + else: # list i.e. img_size=[640, 480] + imgsz = list(imgsz) # convert to list if tuple + new_size = [max(make_divisible(x, int(s)), floor) for x in imgsz] + if new_size != imgsz: + LOGGER.warning(f'WARNING ⚠️ --img-size {imgsz} must be multiple of max stride {s}, updating to {new_size}') + return new_size + + +def check_imshow(warn=False): + # Check if environment supports image displays + try: + assert not is_jupyter() + assert not is_docker() + cv2.imshow('test', np.zeros((1, 1, 3))) + cv2.waitKey(1) + cv2.destroyAllWindows() + cv2.waitKey(1) + return True + except Exception as e: + if warn: + LOGGER.warning(f'WARNING ⚠️ Environment does not support cv2.imshow() or PIL Image.show()\n{e}') + return False + + +def check_suffix(file='yolov5s.pt', suffix=('.pt', ), msg=''): + # Check file(s) for acceptable suffix + if file and suffix: + if isinstance(suffix, str): + suffix = [suffix] + for f in file if isinstance(file, (list, tuple)) else [file]: + s = Path(f).suffix.lower() # file suffix + if len(s): + assert s in suffix, f'{msg}{f} acceptable suffix is {suffix}' + + +def check_yaml(file, suffix=('.yaml', '.yml')): + # Search/download YAML file (if necessary) and return path, checking suffix + return check_file(file, suffix) + + +def check_file(file, suffix=''): + # Search/download file (if necessary) and return path + check_suffix(file, suffix) # optional + file = str(file) # convert to str() + if os.path.isfile(file) or not file: # exists + return file + elif file.startswith(('http:/', 'https:/')): # download + url = file # warning: Pathlib turns :// -> :/ + file = Path(urllib.parse.unquote(file).split('?')[0]).name # '%2F' to '/', split https://url.com/file.txt?auth + if os.path.isfile(file): + LOGGER.info(f'Found {url} locally at {file}') # file already exists + else: + LOGGER.info(f'Downloading {url} to {file}...') + torch.hub.download_url_to_file(url, file) + assert Path(file).exists() and Path(file).stat().st_size > 0, f'File download failed: {url}' # check + return file + elif file.startswith('clearml://'): # ClearML Dataset ID + assert 'clearml' in sys.modules, "ClearML is not installed, so cannot use ClearML dataset. Try running 'pip install clearml'." + return file + else: # search + files = [] + for d in 'data', 'models', 'utils': # search directories + files.extend(glob.glob(str(ROOT / d / '**' / file), recursive=True)) # find file + assert len(files), f'File not found: {file}' # assert file was found + assert len(files) == 1, f"Multiple files match '{file}', specify exact path: {files}" # assert unique + return files[0] # return file + + +def check_font(font=FONT, progress=False): + # Download font to CONFIG_DIR if necessary + font = Path(font) + file = CONFIG_DIR / font.name + if not font.exists() and not file.exists(): + url = f'https://ultralytics.com/assets/{font.name}' + LOGGER.info(f'Downloading {url} to {file}...') + torch.hub.download_url_to_file(url, str(file), progress=progress) + + +def check_dataset(data, autodownload=True): + # Download, check and/or unzip dataset if not found locally + + # Download (optional) + extract_dir = '' + if isinstance(data, (str, Path)) and (is_zipfile(data) or is_tarfile(data)): + download(data, dir=f'{DATASETS_DIR}/{Path(data).stem}', unzip=True, delete=False, curl=False, threads=1) + data = next((DATASETS_DIR / Path(data).stem).rglob('*.yaml')) + extract_dir, autodownload = data.parent, False + + # Read yaml (optional) + if isinstance(data, (str, Path)): + data = yaml_load(data) # dictionary + + # Checks + for k in 'train', 'val', 'names': + assert k in data, emojis(f"data.yaml '{k}:' field missing ❌") + if isinstance(data['names'], (list, tuple)): # old array format + data['names'] = dict(enumerate(data['names'])) # convert to dict + assert all(isinstance(k, int) for k in data['names'].keys()), 'data.yaml names keys must be integers, i.e. 2: car' + data['nc'] = len(data['names']) + + # Resolve paths + path = Path(extract_dir or data.get('path') or '') # optional 'path' default to '.' + if not path.is_absolute(): + path = (ROOT / path).resolve() + data['path'] = path # download scripts + for k in 'train', 'val', 'test': + if data.get(k): # prepend path + if isinstance(data[k], str): + x = (path / data[k]).resolve() + if not x.exists() and data[k].startswith('../'): + x = (path / data[k][3:]).resolve() + data[k] = str(x) + else: + data[k] = [str((path / x).resolve()) for x in data[k]] + + # Parse yaml + train, val, test, s = (data.get(x) for x in ('train', 'val', 'test', 'download')) + if val: + val = [Path(x).resolve() for x in (val if isinstance(val, list) else [val])] # val path + if not all(x.exists() for x in val): + LOGGER.info('\nDataset not found ⚠️, missing paths %s' % [str(x) for x in val if not x.exists()]) + if not s or not autodownload: + raise Exception('Dataset not found ❌') + t = time.time() + if s.startswith('http') and s.endswith('.zip'): # URL + f = Path(s).name # filename + LOGGER.info(f'Downloading {s} to {f}...') + torch.hub.download_url_to_file(s, f) + Path(DATASETS_DIR).mkdir(parents=True, exist_ok=True) # create root + unzip_file(f, path=DATASETS_DIR) # unzip + Path(f).unlink() # remove zip + r = None # success + elif s.startswith('bash '): # bash script + LOGGER.info(f'Running {s} ...') + r = subprocess.run(s, shell=True) + else: # python script + r = exec(s, {'yaml': data}) # return None + dt = f'({round(time.time() - t, 1)}s)' + s = f"success ✅ {dt}, saved to {colorstr('bold', DATASETS_DIR)}" if r in (0, None) else f'failure {dt} ❌' + LOGGER.info(f'Dataset download {s}') + check_font('Arial.ttf' if is_ascii(data['names']) else 'Arial.Unicode.ttf', progress=True) # download fonts + return data # dictionary + + +def check_amp(model): + # Check PyTorch Automatic Mixed Precision (AMP) functionality. Return True on correct operation + from models.common import AutoShape, DetectMultiBackend + + def amp_allclose(model, im): + # All close FP32 vs AMP results + m = AutoShape(model, verbose=False) # model + a = m(im).xywhn[0] # FP32 inference + m.amp = True + b = m(im).xywhn[0] # AMP inference + return a.shape == b.shape and torch.allclose(a, b, atol=0.1) # close to 10% absolute tolerance + + prefix = colorstr('AMP: ') + device = next(model.parameters()).device # get model device + if device.type in ('cpu', 'mps'): + return False # AMP only used on CUDA devices + f = ROOT / 'data' / 'images' / 'bus.jpg' # image to check + im = f if f.exists() else 'https://ultralytics.com/images/bus.jpg' if check_online() else np.ones((640, 640, 3)) + try: + assert amp_allclose(deepcopy(model), im) or amp_allclose(DetectMultiBackend('yolov5n.pt', device), im) + LOGGER.info(f'{prefix}checks passed ✅') + return True + except Exception: + help_url = 'https://github.com/ultralytics/yolov5/issues/7908' + LOGGER.warning(f'{prefix}checks failed ❌, disabling Automatic Mixed Precision. See {help_url}') + return False + + +def yaml_load(file='data.yaml'): + # Single-line safe yaml loading + with open(file, errors='ignore') as f: + return yaml.safe_load(f) + + +def yaml_save(file='data.yaml', data={}): + # Single-line safe yaml saving + with open(file, 'w') as f: + yaml.safe_dump({k: str(v) if isinstance(v, Path) else v for k, v in data.items()}, f, sort_keys=False) + + +def unzip_file(file, path=None, exclude=('.DS_Store', '__MACOSX')): + # Unzip a *.zip file to path/, excluding files containing strings in exclude list + if path is None: + path = Path(file).parent # default path + with ZipFile(file) as zipObj: + for f in zipObj.namelist(): # list all archived filenames in the zip + if all(x not in f for x in exclude): + zipObj.extract(f, path=path) + + +def url2file(url): + # Convert URL to filename, i.e. https://url.com/file.txt?auth -> file.txt + url = str(Path(url)).replace(':/', '://') # Pathlib turns :// -> :/ + return Path(urllib.parse.unquote(url)).name.split('?')[0] # '%2F' to '/', split https://url.com/file.txt?auth + + +def download(url, dir='.', unzip=True, delete=True, curl=False, threads=1, retry=3): + # Multithreaded file download and unzip function, used in data.yaml for autodownload + def download_one(url, dir): + # Download 1 file + success = True + if os.path.isfile(url): + f = Path(url) # filename + else: # does not exist + f = dir / Path(url).name + LOGGER.info(f'Downloading {url} to {f}...') + for i in range(retry + 1): + if curl: + success = curl_download(url, f, silent=(threads > 1)) + else: + torch.hub.download_url_to_file(url, f, progress=threads == 1) # torch download + success = f.is_file() + if success: + break + elif i < retry: + LOGGER.warning(f'⚠️ Download failure, retrying {i + 1}/{retry} {url}...') + else: + LOGGER.warning(f'❌ Failed to download {url}...') + + if unzip and success and (f.suffix == '.gz' or is_zipfile(f) or is_tarfile(f)): + LOGGER.info(f'Unzipping {f}...') + if is_zipfile(f): + unzip_file(f, dir) # unzip + elif is_tarfile(f): + subprocess.run(['tar', 'xf', f, '--directory', f.parent], check=True) # unzip + elif f.suffix == '.gz': + subprocess.run(['tar', 'xfz', f, '--directory', f.parent], check=True) # unzip + if delete: + f.unlink() # remove zip + + dir = Path(dir) + dir.mkdir(parents=True, exist_ok=True) # make directory + if threads > 1: + pool = ThreadPool(threads) + pool.imap(lambda x: download_one(*x), zip(url, repeat(dir))) # multithreaded + pool.close() + pool.join() + else: + for u in [url] if isinstance(url, (str, Path)) else url: + download_one(u, dir) + + +def make_divisible(x, divisor): + # Returns nearest x divisible by divisor + if isinstance(divisor, torch.Tensor): + divisor = int(divisor.max()) # to int + return math.ceil(x / divisor) * divisor + + +def clean_str(s): + # Cleans a string by replacing special characters with underscore _ + return re.sub(pattern='[|@#!¡·$€%&()=?¿^*;:,¨´><+]', repl='_', string=s) + + +def one_cycle(y1=0.0, y2=1.0, steps=100): + # lambda function for sinusoidal ramp from y1 to y2 https://arxiv.org/pdf/1812.01187.pdf + return lambda x: ((1 - math.cos(x * math.pi / steps)) / 2) * (y2 - y1) + y1 + + +def colorstr(*input): + # Colors a string https://en.wikipedia.org/wiki/ANSI_escape_code, i.e. colorstr('blue', 'hello world') + *args, string = input if len(input) > 1 else ('blue', 'bold', input[0]) # color arguments, string + colors = { + 'black': '\033[30m', # basic colors + 'red': '\033[31m', + 'green': '\033[32m', + 'yellow': '\033[33m', + 'blue': '\033[34m', + 'magenta': '\033[35m', + 'cyan': '\033[36m', + 'white': '\033[37m', + 'bright_black': '\033[90m', # bright colors + 'bright_red': '\033[91m', + 'bright_green': '\033[92m', + 'bright_yellow': '\033[93m', + 'bright_blue': '\033[94m', + 'bright_magenta': '\033[95m', + 'bright_cyan': '\033[96m', + 'bright_white': '\033[97m', + 'end': '\033[0m', # misc + 'bold': '\033[1m', + 'underline': '\033[4m'} + return ''.join(colors[x] for x in args) + f'{string}' + colors['end'] + + +def labels_to_class_weights(labels, nc=80): + # Get class weights (inverse frequency) from training labels + if labels[0] is None: # no labels loaded + return torch.Tensor() + + labels = np.concatenate(labels, 0) # labels.shape = (866643, 5) for COCO + classes = labels[:, 0].astype(int) # labels = [class xywh] + weights = np.bincount(classes, minlength=nc) # occurrences per class + + # Prepend gridpoint count (for uCE training) + # gpi = ((320 / 32 * np.array([1, 2, 4])) ** 2 * 3).sum() # gridpoints per image + # weights = np.hstack([gpi * len(labels) - weights.sum() * 9, weights * 9]) ** 0.5 # prepend gridpoints to start + + weights[weights == 0] = 1 # replace empty bins with 1 + weights = 1 / weights # number of targets per class + weights /= weights.sum() # normalize + return torch.from_numpy(weights).float() + + +def labels_to_image_weights(labels, nc=80, class_weights=np.ones(80)): + # Produces image weights based on class_weights and image contents + # Usage: index = random.choices(range(n), weights=image_weights, k=1) # weighted image sample + class_counts = np.array([np.bincount(x[:, 0].astype(int), minlength=nc) for x in labels]) + return (class_weights.reshape(1, nc) * class_counts).sum(1) + + +def coco80_to_coco91_class(): # converts 80-index (val2014) to 91-index (paper) + # https://tech.amikelive.com/node-718/what-object-categories-labels-are-in-coco-dataset/ + # a = np.loadtxt('data/coco.names', dtype='str', delimiter='\n') + # b = np.loadtxt('data/coco_paper.names', dtype='str', delimiter='\n') + # x1 = [list(a[i] == b).index(True) + 1 for i in range(80)] # darknet to coco + # x2 = [list(b[i] == a).index(True) if any(b[i] == a) else None for i in range(91)] # coco to darknet + return [ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 27, 28, 31, 32, 33, 34, + 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, + 64, 65, 67, 70, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 84, 85, 86, 87, 88, 89, 90] + + +def xyxy2xywh(x): + # Convert nx4 boxes from [x1, y1, x2, y2] to [x, y, w, h] where xy1=top-left, xy2=bottom-right + y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x) + y[..., 0] = (x[..., 0] + x[..., 2]) / 2 # x center + y[..., 1] = (x[..., 1] + x[..., 3]) / 2 # y center + y[..., 2] = x[..., 2] - x[..., 0] # width + y[..., 3] = x[..., 3] - x[..., 1] # height + return y + + +def xywh2xyxy(x): + # Convert nx4 boxes from [x, y, w, h] to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right + y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x) + y[..., 0] = x[..., 0] - x[..., 2] / 2 # top left x + y[..., 1] = x[..., 1] - x[..., 3] / 2 # top left y + y[..., 2] = x[..., 0] + x[..., 2] / 2 # bottom right x + y[..., 3] = x[..., 1] + x[..., 3] / 2 # bottom right y + return y + + +def xywhn2xyxy(x, w=640, h=640, padw=0, padh=0): + # Convert nx4 boxes from [x, y, w, h] normalized to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right + y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x) + y[..., 0] = w * (x[..., 0] - x[..., 2] / 2) + padw # top left x + y[..., 1] = h * (x[..., 1] - x[..., 3] / 2) + padh # top left y + y[..., 2] = w * (x[..., 0] + x[..., 2] / 2) + padw # bottom right x + y[..., 3] = h * (x[..., 1] + x[..., 3] / 2) + padh # bottom right y + return y + + +def xyxy2xywhn(x, w=640, h=640, clip=False, eps=0.0): + # Convert nx4 boxes from [x1, y1, x2, y2] to [x, y, w, h] normalized where xy1=top-left, xy2=bottom-right + if clip: + clip_boxes(x, (h - eps, w - eps)) # warning: inplace clip + y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x) + y[..., 0] = ((x[..., 0] + x[..., 2]) / 2) / w # x center + y[..., 1] = ((x[..., 1] + x[..., 3]) / 2) / h # y center + y[..., 2] = (x[..., 2] - x[..., 0]) / w # width + y[..., 3] = (x[..., 3] - x[..., 1]) / h # height + return y + + +def xyn2xy(x, w=640, h=640, padw=0, padh=0): + # Convert normalized segments into pixel segments, shape (n,2) + y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x) + y[..., 0] = w * x[..., 0] + padw # top left x + y[..., 1] = h * x[..., 1] + padh # top left y + return y + + +def segment2box(segment, width=640, height=640): + # Convert 1 segment label to 1 box label, applying inside-image constraint, i.e. (xy1, xy2, ...) to (xyxy) + x, y = segment.T # segment xy + inside = (x >= 0) & (y >= 0) & (x <= width) & (y <= height) + x, y, = x[inside], y[inside] + return np.array([x.min(), y.min(), x.max(), y.max()]) if any(x) else np.zeros((1, 4)) # xyxy + + +def segments2boxes(segments): + # Convert segment labels to box labels, i.e. (cls, xy1, xy2, ...) to (cls, xywh) + boxes = [] + for s in segments: + x, y = s.T # segment xy + boxes.append([x.min(), y.min(), x.max(), y.max()]) # cls, xyxy + return xyxy2xywh(np.array(boxes)) # cls, xywh + + +def resample_segments(segments, n=1000): + # Up-sample an (n,2) segment + for i, s in enumerate(segments): + s = np.concatenate((s, s[0:1, :]), axis=0) + x = np.linspace(0, len(s) - 1, n) + xp = np.arange(len(s)) + segments[i] = np.concatenate([np.interp(x, xp, s[:, i]) for i in range(2)]).reshape(2, -1).T # segment xy + return segments + + +def scale_boxes(img1_shape, boxes, img0_shape, ratio_pad=None): + # Rescale boxes (xyxy) from img1_shape to img0_shape + if ratio_pad is None: # calculate from img0_shape + gain = min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1]) # gain = old / new + pad = (img1_shape[1] - img0_shape[1] * gain) / 2, (img1_shape[0] - img0_shape[0] * gain) / 2 # wh padding + else: + gain = ratio_pad[0][0] + pad = ratio_pad[1] + + boxes[..., [0, 2]] -= pad[0] # x padding + boxes[..., [1, 3]] -= pad[1] # y padding + boxes[..., :4] /= gain + clip_boxes(boxes, img0_shape) + return boxes + + +def scale_segments(img1_shape, segments, img0_shape, ratio_pad=None, normalize=False): + # Rescale coords (xyxy) from img1_shape to img0_shape + if ratio_pad is None: # calculate from img0_shape + gain = min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1]) # gain = old / new + pad = (img1_shape[1] - img0_shape[1] * gain) / 2, (img1_shape[0] - img0_shape[0] * gain) / 2 # wh padding + else: + gain = ratio_pad[0][0] + pad = ratio_pad[1] + + segments[:, 0] -= pad[0] # x padding + segments[:, 1] -= pad[1] # y padding + segments /= gain + clip_segments(segments, img0_shape) + if normalize: + segments[:, 0] /= img0_shape[1] # width + segments[:, 1] /= img0_shape[0] # height + return segments + + +def clip_boxes(boxes, shape): + # Clip boxes (xyxy) to image shape (height, width) + if isinstance(boxes, torch.Tensor): # faster individually + boxes[..., 0].clamp_(0, shape[1]) # x1 + boxes[..., 1].clamp_(0, shape[0]) # y1 + boxes[..., 2].clamp_(0, shape[1]) # x2 + boxes[..., 3].clamp_(0, shape[0]) # y2 + else: # np.array (faster grouped) + boxes[..., [0, 2]] = boxes[..., [0, 2]].clip(0, shape[1]) # x1, x2 + boxes[..., [1, 3]] = boxes[..., [1, 3]].clip(0, shape[0]) # y1, y2 + + +def clip_segments(segments, shape): + # Clip segments (xy1,xy2,...) to image shape (height, width) + if isinstance(segments, torch.Tensor): # faster individually + segments[:, 0].clamp_(0, shape[1]) # x + segments[:, 1].clamp_(0, shape[0]) # y + else: # np.array (faster grouped) + segments[:, 0] = segments[:, 0].clip(0, shape[1]) # x + segments[:, 1] = segments[:, 1].clip(0, shape[0]) # y + + +def non_max_suppression( + prediction, + conf_thres=0.25, + iou_thres=0.45, + classes=None, + agnostic=False, + multi_label=False, + labels=(), + max_det=300, + nm=0, # number of masks +): + """Non-Maximum Suppression (NMS) on inference results to reject overlapping detections + + Returns: + list of detections, on (n,6) tensor per image [xyxy, conf, cls] + """ + + # Checks + assert 0 <= conf_thres <= 1, f'Invalid Confidence threshold {conf_thres}, valid values are between 0.0 and 1.0' + assert 0 <= iou_thres <= 1, f'Invalid IoU {iou_thres}, valid values are between 0.0 and 1.0' + if isinstance(prediction, (list, tuple)): # YOLOv5 model in validation model, output = (inference_out, loss_out) + prediction = prediction[0] # select only inference output + + device = prediction.device + mps = 'mps' in device.type # Apple MPS + if mps: # MPS not fully supported yet, convert tensors to CPU before NMS + prediction = prediction.cpu() + bs = prediction.shape[0] # batch size + nc = prediction.shape[2] - nm - 5 # number of classes + xc = prediction[..., 4] > conf_thres # candidates + + # Settings + # min_wh = 2 # (pixels) minimum box width and height + max_wh = 7680 # (pixels) maximum box width and height + max_nms = 30000 # maximum number of boxes into torchvision.ops.nms() + time_limit = 0.5 + 0.05 * bs # seconds to quit after + redundant = True # require redundant detections + multi_label &= nc > 1 # multiple labels per box (adds 0.5ms/img) + merge = False # use merge-NMS + + t = time.time() + mi = 5 + nc # mask start index + output = [torch.zeros((0, 6 + nm), device=prediction.device)] * bs + for xi, x in enumerate(prediction): # image index, image inference + # Apply constraints + # x[((x[..., 2:4] < min_wh) | (x[..., 2:4] > max_wh)).any(1), 4] = 0 # width-height + x = x[xc[xi]] # confidence + + # Cat apriori labels if autolabelling + if labels and len(labels[xi]): + lb = labels[xi] + v = torch.zeros((len(lb), nc + nm + 5), device=x.device) + v[:, :4] = lb[:, 1:5] # box + v[:, 4] = 1.0 # conf + v[range(len(lb)), lb[:, 0].long() + 5] = 1.0 # cls + x = torch.cat((x, v), 0) + + # If none remain process next image + if not x.shape[0]: + continue + + # Compute conf + x[:, 5:] *= x[:, 4:5] # conf = obj_conf * cls_conf + + # Box/Mask + box = xywh2xyxy(x[:, :4]) # center_x, center_y, width, height) to (x1, y1, x2, y2) + mask = x[:, mi:] # zero columns if no masks + + # Detections matrix nx6 (xyxy, conf, cls) + if multi_label: + i, j = (x[:, 5:mi] > conf_thres).nonzero(as_tuple=False).T + x = torch.cat((box[i], x[i, 5 + j, None], j[:, None].float(), mask[i]), 1) + else: # best class only + conf, j = x[:, 5:mi].max(1, keepdim=True) + x = torch.cat((box, conf, j.float(), mask), 1)[conf.view(-1) > conf_thres] + + # Filter by class + if classes is not None: + x = x[(x[:, 5:6] == torch.tensor(classes, device=x.device)).any(1)] + + # Apply finite constraint + # if not torch.isfinite(x).all(): + # x = x[torch.isfinite(x).all(1)] + + # Check shape + n = x.shape[0] # number of boxes + if not n: # no boxes + continue + x = x[x[:, 4].argsort(descending=True)[:max_nms]] # sort by confidence and remove excess boxes + + # Batched NMS + c = x[:, 5:6] * (0 if agnostic else max_wh) # classes + boxes, scores = x[:, :4] + c, x[:, 4] # boxes (offset by class), scores + i = torchvision.ops.nms(boxes, scores, iou_thres) # NMS + i = i[:max_det] # limit detections + if merge and (1 < n < 3E3): # Merge NMS (boxes merged using weighted mean) + # update boxes as boxes(i,4) = weights(i,n) * boxes(n,4) + iou = box_iou(boxes[i], boxes) > iou_thres # iou matrix + weights = iou * scores[None] # box weights + x[i, :4] = torch.mm(weights, x[:, :4]).float() / weights.sum(1, keepdim=True) # merged boxes + if redundant: + i = i[iou.sum(1) > 1] # require redundancy + + ## ===================================================== written by WQG + idx = i.clone() + boxes_nocls, scores_nocls = x[idx, :4], x[idx, 4] + ix = torchvision.ops.nms(boxes_nocls, scores_nocls, 0.83) + idxx = idx[ix] + output[xi] = x[idxx] + + # output[xi] = x[i] + if mps: + output[xi] = output[xi].to(device) + if (time.time() - t) > time_limit: + LOGGER.warning(f'WARNING ⚠️ NMS time limit {time_limit:.3f}s exceeded') + break # time limit exceeded + + return output + + +def strip_optimizer(f='best.pt', s=''): # from utils.general import *; strip_optimizer() + # Strip optimizer from 'f' to finalize training, optionally save as 's' + x = torch.load(f, map_location=torch.device('cpu')) + if x.get('ema'): + x['model'] = x['ema'] # replace model with ema + for k in 'optimizer', 'best_fitness', 'ema', 'updates': # keys + x[k] = None + x['epoch'] = -1 + x['model'].half() # to FP16 + for p in x['model'].parameters(): + p.requires_grad = False + torch.save(x, s or f) + mb = os.path.getsize(s or f) / 1E6 # filesize + LOGGER.info(f"Optimizer stripped from {f},{f' saved as {s},' if s else ''} {mb:.1f}MB") + + +def print_mutation(keys, results, hyp, save_dir, bucket, prefix=colorstr('evolve: ')): + evolve_csv = save_dir / 'evolve.csv' + evolve_yaml = save_dir / 'hyp_evolve.yaml' + keys = tuple(keys) + tuple(hyp.keys()) # [results + hyps] + keys = tuple(x.strip() for x in keys) + vals = results + tuple(hyp.values()) + n = len(keys) + + # Download (optional) + if bucket: + url = f'gs://{bucket}/evolve.csv' + if gsutil_getsize(url) > (evolve_csv.stat().st_size if evolve_csv.exists() else 0): + subprocess.run(['gsutil', 'cp', f'{url}', f'{save_dir}']) # download evolve.csv if larger than local + + # Log to evolve.csv + s = '' if evolve_csv.exists() else (('%20s,' * n % keys).rstrip(',') + '\n') # add header + with open(evolve_csv, 'a') as f: + f.write(s + ('%20.5g,' * n % vals).rstrip(',') + '\n') + + # Save yaml + with open(evolve_yaml, 'w') as f: + data = pd.read_csv(evolve_csv, skipinitialspace=True) + data = data.rename(columns=lambda x: x.strip()) # strip keys + i = np.argmax(fitness(data.values[:, :4])) # + generations = len(data) + f.write('# YOLOv5 Hyperparameter Evolution Results\n' + f'# Best generation: {i}\n' + + f'# Last generation: {generations - 1}\n' + '# ' + ', '.join(f'{x.strip():>20s}' for x in keys[:7]) + + '\n' + '# ' + ', '.join(f'{x:>20.5g}' for x in data.values[i, :7]) + '\n\n') + yaml.safe_dump(data.loc[i][7:].to_dict(), f, sort_keys=False) + + # Print to screen + LOGGER.info(prefix + f'{generations} generations finished, current result:\n' + prefix + + ', '.join(f'{x.strip():>20s}' for x in keys) + '\n' + prefix + ', '.join(f'{x:20.5g}' + for x in vals) + '\n\n') + + if bucket: + subprocess.run(['gsutil', 'cp', f'{evolve_csv}', f'{evolve_yaml}', f'gs://{bucket}']) # upload + + +def apply_classifier(x, model, img, im0): + # Apply a second stage classifier to YOLO outputs + # Example model = torchvision.models.__dict__['efficientnet_b0'](pretrained=True).to(device).eval() + im0 = [im0] if isinstance(im0, np.ndarray) else im0 + for i, d in enumerate(x): # per image + if d is not None and len(d): + d = d.clone() + + # Reshape and pad cutouts + b = xyxy2xywh(d[:, :4]) # boxes + b[:, 2:] = b[:, 2:].max(1)[0].unsqueeze(1) # rectangle to square + b[:, 2:] = b[:, 2:] * 1.3 + 30 # pad + d[:, :4] = xywh2xyxy(b).long() + + # Rescale boxes from img_size to im0 size + scale_boxes(img.shape[2:], d[:, :4], im0[i].shape) + + # Classes + pred_cls1 = d[:, 5].long() + ims = [] + for a in d: + cutout = im0[i][int(a[1]):int(a[3]), int(a[0]):int(a[2])] + im = cv2.resize(cutout, (224, 224)) # BGR + + im = im[:, :, ::-1].transpose(2, 0, 1) # BGR to RGB, to 3x416x416 + im = np.ascontiguousarray(im, dtype=np.float32) # uint8 to float32 + im /= 255 # 0 - 255 to 0.0 - 1.0 + ims.append(im) + + pred_cls2 = model(torch.Tensor(ims).to(d.device)).argmax(1) # classifier prediction + x[i] = x[i][pred_cls1 == pred_cls2] # retain matching class detections + + return x + + +def increment_path(path, exist_ok=False, sep='', mkdir=False): + # Increment file or directory path, i.e. runs/exp --> runs/exp{sep}2, runs/exp{sep}3, ... etc. + path = Path(path) # os-agnostic + if path.exists() and not exist_ok: + path, suffix = (path.with_suffix(''), path.suffix) if path.is_file() else (path, '') + + # Method 1 + for n in range(2, 9999): + p = f'{path}{sep}{n}{suffix}' # increment path + if not os.path.exists(p): # + break + path = Path(p) + + # Method 2 (deprecated) + # dirs = glob.glob(f"{path}{sep}*") # similar paths + # matches = [re.search(rf"{path.stem}{sep}(\d+)", d) for d in dirs] + # i = [int(m.groups()[0]) for m in matches if m] # indices + # n = max(i) + 1 if i else 2 # increment number + # path = Path(f"{path}{sep}{n}{suffix}") # increment path + + if mkdir: + path.mkdir(parents=True, exist_ok=True) # make directory + + return path + + +# OpenCV Multilanguage-friendly functions ------------------------------------------------------------------------------------ +imshow_ = cv2.imshow # copy to avoid recursion errors + + +def imread(filename, flags=cv2.IMREAD_COLOR): + return cv2.imdecode(np.fromfile(filename, np.uint8), flags) + + +def imwrite(filename, img): + try: + cv2.imencode(Path(filename).suffix, img)[1].tofile(filename) + return True + except Exception: + return False + + +def imshow(path, im): + imshow_(path.encode('unicode_escape').decode(), im) + + +if Path(inspect.stack()[0].filename).parent.parent.as_posix() in inspect.stack()[-1].filename: + cv2.imread, cv2.imwrite, cv2.imshow = imread, imwrite, imshow # redefine + +# Variables ------------------------------------------------------------------------------------------------------------ diff --git a/utils/getsource.py b/utils/getsource.py new file mode 100644 index 0000000..6c918c1 --- /dev/null +++ b/utils/getsource.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +""" +Created on Wed Oct 30 13:18:59 2024 + +@author: ym +""" + +import os +import glob + +IMGFORMATS = '.bmp', '.jpeg', '.jpg', 'png', 'tif', 'tiff', 'webp', 'pfm' +VIDFORMATS = '.avi', '.gif', '.m4v', '.mkv', '.mov', '.mp4', '.ts', '.wmv' + +def get_image_pairs(p): + files = [] + files.extend(sorted(glob.glob(os.path.join(p, '*.jpg')))) + # images = [x for x in files if x.split('.')[-1].lower() in IMG_FORMATS] + + tamps_0, tamps_1 = [], [] + files_0, files_1 = [], [] + for file in files: + basename = os.path.basename(file) + if basename.find('frameId')<0: continue + + + f, ext = os.path.splitext(basename) + camer, tamp, _, frameId = f.split('_') + + if camer == '0': + tamps_0.append(int(tamp)) + files_0.append(file) + + if camer == '1': + tamps_1.append(int(tamp)) + files_1.append(file) + + idx0 = sorted(range(len(tamps_0)), key=lambda k: tamps_0[k]) + files0 = [files_0[i] for i in idx0] + + idx1 = sorted(range(len(tamps_1)), key=lambda k: tamps_1[k]) + files1 = [files_1[i] for i in idx1] + + files = (files0, files1) + + return files + + +def get_video_pairs(vpath): + vdieopath = [] + for filename in os.listdir(vpath): + file, ext = os.path.splitext(filename) + if ext in VIDFORMATS: + vdieopath.append(os.path.join(vpath, filename)) + return vdieopath + + + + + + diff --git a/utils/google_app_engine/Dockerfile b/utils/google_app_engine/Dockerfile new file mode 100644 index 0000000..0155618 --- /dev/null +++ b/utils/google_app_engine/Dockerfile @@ -0,0 +1,25 @@ +FROM gcr.io/google-appengine/python + +# Create a virtualenv for dependencies. This isolates these packages from +# system-level packages. +# Use -p python3 or -p python3.7 to select python version. Default is version 2. +RUN virtualenv /env -p python3 + +# Setting these environment variables are the same as running +# source /env/bin/activate. +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH + +RUN apt-get update && apt-get install -y python-opencv + +# Copy the application's requirements.txt and run pip to install all +# dependencies into the virtualenv. +ADD requirements.txt /app/requirements.txt +RUN pip install -r /app/requirements.txt + +# Add the application source code. +ADD . /app + +# Run a WSGI server to serve the application. gunicorn must be declared as +# a dependency in requirements.txt. +CMD gunicorn -b :$PORT main:app diff --git a/utils/google_app_engine/additional_requirements.txt b/utils/google_app_engine/additional_requirements.txt new file mode 100644 index 0000000..fce1511 --- /dev/null +++ b/utils/google_app_engine/additional_requirements.txt @@ -0,0 +1,5 @@ +# add these requirements in your app on top of the existing ones +pip==21.1 +Flask==2.3.2 +gunicorn==19.10.0 +werkzeug>=2.2.3 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/utils/google_app_engine/app.yaml b/utils/google_app_engine/app.yaml new file mode 100644 index 0000000..5056b7c --- /dev/null +++ b/utils/google_app_engine/app.yaml @@ -0,0 +1,14 @@ +runtime: custom +env: flex + +service: yolov5app + +liveness_check: + initial_delay_sec: 600 + +manual_scaling: + instances: 1 +resources: + cpu: 1 + memory_gb: 4 + disk_size_gb: 20 diff --git a/utils/loggers/__init__.py b/utils/loggers/__init__.py new file mode 100644 index 0000000..ba7d279 --- /dev/null +++ b/utils/loggers/__init__.py @@ -0,0 +1,401 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Logging utils +""" + +import os +import warnings +from pathlib import Path + +import pkg_resources as pkg +import torch + +from utils.general import LOGGER, colorstr, cv2 +from utils.loggers.clearml.clearml_utils import ClearmlLogger +from utils.loggers.wandb.wandb_utils import WandbLogger +from utils.plots import plot_images, plot_labels, plot_results +from utils.torch_utils import de_parallel + +LOGGERS = ('csv', 'tb', 'wandb', 'clearml', 'comet') # *.csv, TensorBoard, Weights & Biases, ClearML +RANK = int(os.getenv('RANK', -1)) + +try: + from torch.utils.tensorboard import SummaryWriter +except ImportError: + SummaryWriter = lambda *args: None # None = SummaryWriter(str) + +try: + import wandb + + assert hasattr(wandb, '__version__') # verify package import not local dir + if pkg.parse_version(wandb.__version__) >= pkg.parse_version('0.12.2') and RANK in {0, -1}: + try: + wandb_login_success = wandb.login(timeout=30) + except wandb.errors.UsageError: # known non-TTY terminal issue + wandb_login_success = False + if not wandb_login_success: + wandb = None +except (ImportError, AssertionError): + wandb = None + +try: + import clearml + + assert hasattr(clearml, '__version__') # verify package import not local dir +except (ImportError, AssertionError): + clearml = None + +try: + if RANK in {0, -1}: + import comet_ml + + assert hasattr(comet_ml, '__version__') # verify package import not local dir + from utils.loggers.comet import CometLogger + + else: + comet_ml = None +except (ImportError, AssertionError): + comet_ml = None + + +class Loggers(): + # YOLOv5 Loggers class + def __init__(self, save_dir=None, weights=None, opt=None, hyp=None, logger=None, include=LOGGERS): + self.save_dir = save_dir + self.weights = weights + self.opt = opt + self.hyp = hyp + self.plots = not opt.noplots # plot results + self.logger = logger # for printing results to console + self.include = include + self.keys = [ + 'train/box_loss', + 'train/obj_loss', + 'train/cls_loss', # train loss + 'metrics/precision', + 'metrics/recall', + 'metrics/mAP_0.5', + 'metrics/mAP_0.5:0.95', # metrics + 'val/box_loss', + 'val/obj_loss', + 'val/cls_loss', # val loss + 'x/lr0', + 'x/lr1', + 'x/lr2'] # params + self.best_keys = ['best/epoch', 'best/precision', 'best/recall', 'best/mAP_0.5', 'best/mAP_0.5:0.95'] + for k in LOGGERS: + setattr(self, k, None) # init empty logger dictionary + self.csv = True # always log to csv + + # Messages + if not comet_ml: + prefix = colorstr('Comet: ') + s = f"{prefix}run 'pip install comet_ml' to automatically track and visualize YOLOv5 🚀 runs in Comet" + self.logger.info(s) + # TensorBoard + s = self.save_dir + if 'tb' in self.include and not self.opt.evolve: + prefix = colorstr('TensorBoard: ') + self.logger.info(f"{prefix}Start with 'tensorboard --logdir {s.parent}', view at http://localhost:6006/") + self.tb = SummaryWriter(str(s)) + + # W&B + if wandb and 'wandb' in self.include: + self.opt.hyp = self.hyp # add hyperparameters + self.wandb = WandbLogger(self.opt) + else: + self.wandb = None + + # ClearML + if clearml and 'clearml' in self.include: + try: + self.clearml = ClearmlLogger(self.opt, self.hyp) + except Exception: + self.clearml = None + prefix = colorstr('ClearML: ') + LOGGER.warning(f'{prefix}WARNING ⚠️ ClearML is installed but not configured, skipping ClearML logging.' + f' See https://docs.ultralytics.com/yolov5/tutorials/clearml_logging_integration#readme') + + else: + self.clearml = None + + # Comet + if comet_ml and 'comet' in self.include: + if isinstance(self.opt.resume, str) and self.opt.resume.startswith('comet://'): + run_id = self.opt.resume.split('/')[-1] + self.comet_logger = CometLogger(self.opt, self.hyp, run_id=run_id) + + else: + self.comet_logger = CometLogger(self.opt, self.hyp) + + else: + self.comet_logger = None + + @property + def remote_dataset(self): + # Get data_dict if custom dataset artifact link is provided + data_dict = None + if self.clearml: + data_dict = self.clearml.data_dict + if self.wandb: + data_dict = self.wandb.data_dict + if self.comet_logger: + data_dict = self.comet_logger.data_dict + + return data_dict + + def on_train_start(self): + if self.comet_logger: + self.comet_logger.on_train_start() + + def on_pretrain_routine_start(self): + if self.comet_logger: + self.comet_logger.on_pretrain_routine_start() + + def on_pretrain_routine_end(self, labels, names): + # Callback runs on pre-train routine end + if self.plots: + plot_labels(labels, names, self.save_dir) + paths = self.save_dir.glob('*labels*.jpg') # training labels + if self.wandb: + self.wandb.log({'Labels': [wandb.Image(str(x), caption=x.name) for x in paths]}) + # if self.clearml: + # pass # ClearML saves these images automatically using hooks + if self.comet_logger: + self.comet_logger.on_pretrain_routine_end(paths) + + def on_train_batch_end(self, model, ni, imgs, targets, paths, vals): + log_dict = dict(zip(self.keys[:3], vals)) + # Callback runs on train batch end + # ni: number integrated batches (since train start) + if self.plots: + if ni < 3: + f = self.save_dir / f'train_batch{ni}.jpg' # filename + plot_images(imgs, targets, paths, f) + if ni == 0 and self.tb and not self.opt.sync_bn: + log_tensorboard_graph(self.tb, model, imgsz=(self.opt.imgsz, self.opt.imgsz)) + if ni == 10 and (self.wandb or self.clearml): + files = sorted(self.save_dir.glob('train*.jpg')) + if self.wandb: + self.wandb.log({'Mosaics': [wandb.Image(str(f), caption=f.name) for f in files if f.exists()]}) + if self.clearml: + self.clearml.log_debug_samples(files, title='Mosaics') + + if self.comet_logger: + self.comet_logger.on_train_batch_end(log_dict, step=ni) + + def on_train_epoch_end(self, epoch): + # Callback runs on train epoch end + if self.wandb: + self.wandb.current_epoch = epoch + 1 + + if self.comet_logger: + self.comet_logger.on_train_epoch_end(epoch) + + def on_val_start(self): + if self.comet_logger: + self.comet_logger.on_val_start() + + def on_val_image_end(self, pred, predn, path, names, im): + # Callback runs on val image end + if self.wandb: + self.wandb.val_one_image(pred, predn, path, names, im) + if self.clearml: + self.clearml.log_image_with_boxes(path, pred, names, im) + + def on_val_batch_end(self, batch_i, im, targets, paths, shapes, out): + if self.comet_logger: + self.comet_logger.on_val_batch_end(batch_i, im, targets, paths, shapes, out) + + def on_val_end(self, nt, tp, fp, p, r, f1, ap, ap50, ap_class, confusion_matrix): + # Callback runs on val end + if self.wandb or self.clearml: + files = sorted(self.save_dir.glob('val*.jpg')) + if self.wandb: + self.wandb.log({'Validation': [wandb.Image(str(f), caption=f.name) for f in files]}) + if self.clearml: + self.clearml.log_debug_samples(files, title='Validation') + + if self.comet_logger: + self.comet_logger.on_val_end(nt, tp, fp, p, r, f1, ap, ap50, ap_class, confusion_matrix) + + def on_fit_epoch_end(self, vals, epoch, best_fitness, fi): + # Callback runs at the end of each fit (train+val) epoch + x = dict(zip(self.keys, vals)) + if self.csv: + file = self.save_dir / 'results.csv' + n = len(x) + 1 # number of cols + s = '' if file.exists() else (('%20s,' * n % tuple(['epoch'] + self.keys)).rstrip(',') + '\n') # add header + with open(file, 'a') as f: + f.write(s + ('%20.5g,' * n % tuple([epoch] + vals)).rstrip(',') + '\n') + + if self.tb: + for k, v in x.items(): + self.tb.add_scalar(k, v, epoch) + elif self.clearml: # log to ClearML if TensorBoard not used + for k, v in x.items(): + title, series = k.split('/') + self.clearml.task.get_logger().report_scalar(title, series, v, epoch) + + if self.wandb: + if best_fitness == fi: + best_results = [epoch] + vals[3:7] + for i, name in enumerate(self.best_keys): + self.wandb.wandb_run.summary[name] = best_results[i] # log best results in the summary + self.wandb.log(x) + self.wandb.end_epoch() + + if self.clearml: + self.clearml.current_epoch_logged_images = set() # reset epoch image limit + self.clearml.current_epoch += 1 + + if self.comet_logger: + self.comet_logger.on_fit_epoch_end(x, epoch=epoch) + + def on_model_save(self, last, epoch, final_epoch, best_fitness, fi): + # Callback runs on model save event + if (epoch + 1) % self.opt.save_period == 0 and not final_epoch and self.opt.save_period != -1: + if self.wandb: + self.wandb.log_model(last.parent, self.opt, epoch, fi, best_model=best_fitness == fi) + if self.clearml: + self.clearml.task.update_output_model(model_path=str(last), + model_name='Latest Model', + auto_delete_file=False) + + if self.comet_logger: + self.comet_logger.on_model_save(last, epoch, final_epoch, best_fitness, fi) + + def on_train_end(self, last, best, epoch, results): + # Callback runs on training end, i.e. saving best model + if self.plots: + plot_results(file=self.save_dir / 'results.csv') # save results.png + files = ['results.png', 'confusion_matrix.png', *(f'{x}_curve.png' for x in ('F1', 'PR', 'P', 'R'))] + files = [(self.save_dir / f) for f in files if (self.save_dir / f).exists()] # filter + self.logger.info(f"Results saved to {colorstr('bold', self.save_dir)}") + + if self.tb and not self.clearml: # These images are already captured by ClearML by now, we don't want doubles + for f in files: + self.tb.add_image(f.stem, cv2.imread(str(f))[..., ::-1], epoch, dataformats='HWC') + + if self.wandb: + self.wandb.log(dict(zip(self.keys[3:10], results))) + self.wandb.log({'Results': [wandb.Image(str(f), caption=f.name) for f in files]}) + # Calling wandb.log. TODO: Refactor this into WandbLogger.log_model + if not self.opt.evolve: + wandb.log_artifact(str(best if best.exists() else last), + type='model', + name=f'run_{self.wandb.wandb_run.id}_model', + aliases=['latest', 'best', 'stripped']) + self.wandb.finish_run() + + if self.clearml and not self.opt.evolve: + self.clearml.task.update_output_model(model_path=str(best if best.exists() else last), + name='Best Model', + auto_delete_file=False) + + if self.comet_logger: + final_results = dict(zip(self.keys[3:10], results)) + self.comet_logger.on_train_end(files, self.save_dir, last, best, epoch, final_results) + + def on_params_update(self, params: dict): + # Update hyperparams or configs of the experiment + if self.wandb: + self.wandb.wandb_run.config.update(params, allow_val_change=True) + if self.comet_logger: + self.comet_logger.on_params_update(params) + + +class GenericLogger: + """ + YOLOv5 General purpose logger for non-task specific logging + Usage: from utils.loggers import GenericLogger; logger = GenericLogger(...) + Arguments + opt: Run arguments + console_logger: Console logger + include: loggers to include + """ + + def __init__(self, opt, console_logger, include=('tb', 'wandb')): + # init default loggers + self.save_dir = Path(opt.save_dir) + self.include = include + self.console_logger = console_logger + self.csv = self.save_dir / 'results.csv' # CSV logger + if 'tb' in self.include: + prefix = colorstr('TensorBoard: ') + self.console_logger.info( + f"{prefix}Start with 'tensorboard --logdir {self.save_dir.parent}', view at http://localhost:6006/") + self.tb = SummaryWriter(str(self.save_dir)) + + if wandb and 'wandb' in self.include: + self.wandb = wandb.init(project=web_project_name(str(opt.project)), + name=None if opt.name == 'exp' else opt.name, + config=opt) + else: + self.wandb = None + + def log_metrics(self, metrics, epoch): + # Log metrics dictionary to all loggers + if self.csv: + keys, vals = list(metrics.keys()), list(metrics.values()) + n = len(metrics) + 1 # number of cols + s = '' if self.csv.exists() else (('%23s,' * n % tuple(['epoch'] + keys)).rstrip(',') + '\n') # header + with open(self.csv, 'a') as f: + f.write(s + ('%23.5g,' * n % tuple([epoch] + vals)).rstrip(',') + '\n') + + if self.tb: + for k, v in metrics.items(): + self.tb.add_scalar(k, v, epoch) + + if self.wandb: + self.wandb.log(metrics, step=epoch) + + def log_images(self, files, name='Images', epoch=0): + # Log images to all loggers + files = [Path(f) for f in (files if isinstance(files, (tuple, list)) else [files])] # to Path + files = [f for f in files if f.exists()] # filter by exists + + if self.tb: + for f in files: + self.tb.add_image(f.stem, cv2.imread(str(f))[..., ::-1], epoch, dataformats='HWC') + + if self.wandb: + self.wandb.log({name: [wandb.Image(str(f), caption=f.name) for f in files]}, step=epoch) + + def log_graph(self, model, imgsz=(640, 640)): + # Log model graph to all loggers + if self.tb: + log_tensorboard_graph(self.tb, model, imgsz) + + def log_model(self, model_path, epoch=0, metadata={}): + # Log model to all loggers + if self.wandb: + art = wandb.Artifact(name=f'run_{wandb.run.id}_model', type='model', metadata=metadata) + art.add_file(str(model_path)) + wandb.log_artifact(art) + + def update_params(self, params): + # Update the parameters logged + if self.wandb: + wandb.run.config.update(params, allow_val_change=True) + + +def log_tensorboard_graph(tb, model, imgsz=(640, 640)): + # Log model graph to TensorBoard + try: + p = next(model.parameters()) # for device, type + imgsz = (imgsz, imgsz) if isinstance(imgsz, int) else imgsz # expand + im = torch.zeros((1, 3, *imgsz)).to(p.device).type_as(p) # input image (WARNING: must be zeros, not empty) + with warnings.catch_warnings(): + warnings.simplefilter('ignore') # suppress jit trace warning + tb.add_graph(torch.jit.trace(de_parallel(model), im, strict=False), []) + except Exception as e: + LOGGER.warning(f'WARNING ⚠️ TensorBoard graph visualization failure {e}') + + +def web_project_name(project): + # Convert local project name to web project name + if not project.startswith('runs/train'): + return project + suffix = '-Classify' if project.endswith('-cls') else '-Segment' if project.endswith('-seg') else '' + return f'YOLOv5{suffix}' diff --git a/utils/loggers/clearml/README.md b/utils/loggers/clearml/README.md new file mode 100644 index 0000000..ca41c04 --- /dev/null +++ b/utils/loggers/clearml/README.md @@ -0,0 +1,237 @@ +# ClearML Integration + +Clear|MLClear|ML + +## About ClearML + +[ClearML](https://cutt.ly/yolov5-tutorial-clearml) is an [open-source](https://github.com/allegroai/clearml) toolbox designed to save you time ⏱️. + +🔨 Track every YOLOv5 training run in the experiment manager + +🔧 Version and easily access your custom training data with the integrated ClearML Data Versioning Tool + +🔦 Remotely train and monitor your YOLOv5 training runs using ClearML Agent + +🔬 Get the very best mAP using ClearML Hyperparameter Optimization + +🔭 Turn your newly trained YOLOv5 model into an API with just a few commands using ClearML Serving + +
+And so much more. It's up to you how many of these tools you want to use, you can stick to the experiment manager, or chain them all together into an impressive pipeline! +
+
+ +![ClearML scalars dashboard](https://github.com/thepycoder/clearml_screenshots/raw/main/experiment_manager_with_compare.gif) + +
+
+ +## 🦾 Setting Things Up + +To keep track of your experiments and/or data, ClearML needs to communicate to a server. You have 2 options to get one: + +Either sign up for free to the [ClearML Hosted Service](https://cutt.ly/yolov5-tutorial-clearml) or you can set up your own server, see [here](https://clear.ml/docs/latest/docs/deploying_clearml/clearml_server). Even the server is open-source, so even if you're dealing with sensitive data, you should be good to go! + +1. Install the `clearml` python package: + + ```bash + pip install clearml + ``` + +1. Connect the ClearML SDK to the server by [creating credentials](https://app.clear.ml/settings/workspace-configuration) (go right top to Settings -> Workspace -> Create new credentials), then execute the command below and follow the instructions: + + ```bash + clearml-init + ``` + +That's it! You're done 😎 + +
+ +## 🚀 Training YOLOv5 With ClearML + +To enable ClearML experiment tracking, simply install the ClearML pip package. + +```bash +pip install clearml>=1.2.0 +``` + +This will enable integration with the YOLOv5 training script. Every training run from now on, will be captured and stored by the ClearML experiment manager. + +If you want to change the `project_name` or `task_name`, use the `--project` and `--name` arguments of the `train.py` script, by default the project will be called `YOLOv5` and the task `Training`. +PLEASE NOTE: ClearML uses `/` as a delimiter for subprojects, so be careful when using `/` in your project name! + +```bash +python train.py --img 640 --batch 16 --epochs 3 --data coco128.yaml --weights yolov5s.pt --cache +``` + +or with custom project and task name: + +```bash +python train.py --project my_project --name my_training --img 640 --batch 16 --epochs 3 --data coco128.yaml --weights yolov5s.pt --cache +``` + +This will capture: + +- Source code + uncommitted changes +- Installed packages +- (Hyper)parameters +- Model files (use `--save-period n` to save a checkpoint every n epochs) +- Console output +- Scalars (mAP_0.5, mAP_0.5:0.95, precision, recall, losses, learning rates, ...) +- General info such as machine details, runtime, creation date etc. +- All produced plots such as label correlogram and confusion matrix +- Images with bounding boxes per epoch +- Mosaic per epoch +- Validation images per epoch +- ... + +That's a lot right? 🤯 +Now, we can visualize all of this information in the ClearML UI to get an overview of our training progress. Add custom columns to the table view (such as e.g. mAP_0.5) so you can easily sort on the best performing model. Or select multiple experiments and directly compare them! + +There even more we can do with all of this information, like hyperparameter optimization and remote execution, so keep reading if you want to see how that works! + +
+ +## 🔗 Dataset Version Management + +Versioning your data separately from your code is generally a good idea and makes it easy to acquire the latest version too. This repository supports supplying a dataset version ID, and it will make sure to get the data if it's not there yet. Next to that, this workflow also saves the used dataset ID as part of the task parameters, so you will always know for sure which data was used in which experiment! + +![ClearML Dataset Interface](https://github.com/thepycoder/clearml_screenshots/raw/main/clearml_data.gif) + +### Prepare Your Dataset + +The YOLOv5 repository supports a number of different datasets by using yaml files containing their information. By default datasets are downloaded to the `../datasets` folder in relation to the repository root folder. So if you downloaded the `coco128` dataset using the link in the yaml or with the scripts provided by yolov5, you get this folder structure: + +``` +.. +|_ yolov5 +|_ datasets + |_ coco128 + |_ images + |_ labels + |_ LICENSE + |_ README.txt +``` + +But this can be any dataset you wish. Feel free to use your own, as long as you keep to this folder structure. + +Next, ⚠️**copy the corresponding yaml file to the root of the dataset folder**⚠️. This yaml files contains the information ClearML will need to properly use the dataset. You can make this yourself too, of course, just follow the structure of the example yamls. + +Basically we need the following keys: `path`, `train`, `test`, `val`, `nc`, `names`. + +``` +.. +|_ yolov5 +|_ datasets + |_ coco128 + |_ images + |_ labels + |_ coco128.yaml # <---- HERE! + |_ LICENSE + |_ README.txt +``` + +### Upload Your Dataset + +To get this dataset into ClearML as a versioned dataset, go to the dataset root folder and run the following command: + +```bash +cd coco128 +clearml-data sync --project YOLOv5 --name coco128 --folder . +``` + +The command `clearml-data sync` is actually a shorthand command. You could also run these commands one after the other: + +```bash +# Optionally add --parent if you want to base +# this version on another dataset version, so no duplicate files are uploaded! +clearml-data create --name coco128 --project YOLOv5 +clearml-data add --files . +clearml-data close +``` + +### Run Training Using A ClearML Dataset + +Now that you have a ClearML dataset, you can very simply use it to train custom YOLOv5 🚀 models! + +```bash +python train.py --img 640 --batch 16 --epochs 3 --data clearml:// --weights yolov5s.pt --cache +``` + +
+ +## 👀 Hyperparameter Optimization + +Now that we have our experiments and data versioned, it's time to take a look at what we can build on top! + +Using the code information, installed packages and environment details, the experiment itself is now **completely reproducible**. In fact, ClearML allows you to clone an experiment and even change its parameters. We can then just rerun it with these new parameters automatically, this is basically what HPO does! + +To **run hyperparameter optimization locally**, we've included a pre-made script for you. Just make sure a training task has been run at least once, so it is in the ClearML experiment manager, we will essentially clone it and change its hyperparameters. + +You'll need to fill in the ID of this `template task` in the script found at `utils/loggers/clearml/hpo.py` and then just run it :) You can change `task.execute_locally()` to `task.execute()` to put it in a ClearML queue and have a remote agent work on it instead. + +```bash +# To use optuna, install it first, otherwise you can change the optimizer to just be RandomSearch +pip install optuna +python utils/loggers/clearml/hpo.py +``` + +![HPO](https://github.com/thepycoder/clearml_screenshots/raw/main/hpo.png) + +## 🤯 Remote Execution (advanced) + +Running HPO locally is really handy, but what if we want to run our experiments on a remote machine instead? Maybe you have access to a very powerful GPU machine on-site, or you have some budget to use cloud GPUs. +This is where the ClearML Agent comes into play. Check out what the agent can do here: + +- [YouTube video](https://youtu.be/MX3BrXnaULs) +- [Documentation](https://clear.ml/docs/latest/docs/clearml_agent) + +In short: every experiment tracked by the experiment manager contains enough information to reproduce it on a different machine (installed packages, uncommitted changes etc.). So a ClearML agent does just that: it listens to a queue for incoming tasks and when it finds one, it recreates the environment and runs it while still reporting scalars, plots etc. to the experiment manager. + +You can turn any machine (a cloud VM, a local GPU machine, your own laptop ... ) into a ClearML agent by simply running: + +```bash +clearml-agent daemon --queue [--docker] +``` + +### Cloning, Editing And Enqueuing + +With our agent running, we can give it some work. Remember from the HPO section that we can clone a task and edit the hyperparameters? We can do that from the interface too! + +🪄 Clone the experiment by right-clicking it + +🎯 Edit the hyperparameters to what you wish them to be + +⏳ Enqueue the task to any of the queues by right-clicking it + +![Enqueue a task from the UI](https://github.com/thepycoder/clearml_screenshots/raw/main/enqueue.gif) + +### Executing A Task Remotely + +Now you can clone a task like we explained above, or simply mark your current script by adding `task.execute_remotely()` and on execution it will be put into a queue, for the agent to start working on! + +To run the YOLOv5 training script remotely, all you have to do is add this line to the training.py script after the clearml logger has been instantiated: + +```python +# ... +# Loggers +data_dict = None +if RANK in {-1, 0}: + loggers = Loggers(save_dir, weights, opt, hyp, LOGGER) # loggers instance + if loggers.clearml: + loggers.clearml.task.execute_remotely(queue="my_queue") # <------ ADD THIS LINE + # Data_dict is either None is user did not choose for ClearML dataset or is filled in by ClearML + data_dict = loggers.clearml.data_dict +# ... +``` + +When running the training script after this change, python will run the script up until that line, after which it will package the code and send it to the queue instead! + +### Autoscaling workers + +ClearML comes with autoscalers too! This tool will automatically spin up new remote machines in the cloud of your choice (AWS, GCP, Azure) and turn them into ClearML agents for you whenever there are experiments detected in the queue. Once the tasks are processed, the autoscaler will automatically shut down the remote machines, and you stop paying! + +Check out the autoscalers getting started video below. + +[![Watch the video](https://img.youtube.com/vi/j4XVMAaUt3E/0.jpg)](https://youtu.be/j4XVMAaUt3E) diff --git a/utils/loggers/clearml/__init__.py b/utils/loggers/clearml/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/utils/loggers/clearml/clearml_utils.py b/utils/loggers/clearml/clearml_utils.py new file mode 100644 index 0000000..4e999bf --- /dev/null +++ b/utils/loggers/clearml/clearml_utils.py @@ -0,0 +1,163 @@ +"""Main Logger class for ClearML experiment tracking.""" +import glob +import re +from pathlib import Path + +import numpy as np +import yaml +from ultralytics.utils.plotting import Annotator, colors + +try: + import clearml + from clearml import Dataset, Task + + assert hasattr(clearml, '__version__') # verify package import not local dir +except (ImportError, AssertionError): + clearml = None + + +def construct_dataset(clearml_info_string): + """Load in a clearml dataset and fill the internal data_dict with its contents. + """ + dataset_id = clearml_info_string.replace('clearml://', '') + dataset = Dataset.get(dataset_id=dataset_id) + dataset_root_path = Path(dataset.get_local_copy()) + + # We'll search for the yaml file definition in the dataset + yaml_filenames = list(glob.glob(str(dataset_root_path / '*.yaml')) + glob.glob(str(dataset_root_path / '*.yml'))) + if len(yaml_filenames) > 1: + raise ValueError('More than one yaml file was found in the dataset root, cannot determine which one contains ' + 'the dataset definition this way.') + elif len(yaml_filenames) == 0: + raise ValueError('No yaml definition found in dataset root path, check that there is a correct yaml file ' + 'inside the dataset root path.') + with open(yaml_filenames[0]) as f: + dataset_definition = yaml.safe_load(f) + + assert set(dataset_definition.keys()).issuperset( + {'train', 'test', 'val', 'nc', 'names'} + ), "The right keys were not found in the yaml file, make sure it at least has the following keys: ('train', 'test', 'val', 'nc', 'names')" + + data_dict = dict() + data_dict['train'] = str( + (dataset_root_path / dataset_definition['train']).resolve()) if dataset_definition['train'] else None + data_dict['test'] = str( + (dataset_root_path / dataset_definition['test']).resolve()) if dataset_definition['test'] else None + data_dict['val'] = str( + (dataset_root_path / dataset_definition['val']).resolve()) if dataset_definition['val'] else None + data_dict['nc'] = dataset_definition['nc'] + data_dict['names'] = dataset_definition['names'] + + return data_dict + + +class ClearmlLogger: + """Log training runs, datasets, models, and predictions to ClearML. + + This logger sends information to ClearML at app.clear.ml or to your own hosted server. By default, + this information includes hyperparameters, system configuration and metrics, model metrics, code information and + basic data metrics and analyses. + + By providing additional command line arguments to train.py, datasets, + models and predictions can also be logged. + """ + + def __init__(self, opt, hyp): + """ + - Initialize ClearML Task, this object will capture the experiment + - Upload dataset version to ClearML Data if opt.upload_dataset is True + + arguments: + opt (namespace) -- Commandline arguments for this run + hyp (dict) -- Hyperparameters for this run + + """ + self.current_epoch = 0 + # Keep tracked of amount of logged images to enforce a limit + self.current_epoch_logged_images = set() + # Maximum number of images to log to clearML per epoch + self.max_imgs_to_log_per_epoch = 16 + # Get the interval of epochs when bounding box images should be logged + self.bbox_interval = opt.bbox_interval + self.clearml = clearml + self.task = None + self.data_dict = None + if self.clearml: + self.task = Task.init( + project_name=opt.project if opt.project != 'runs/train' else 'YOLOv5', + task_name=opt.name if opt.name != 'exp' else 'Training', + tags=['YOLOv5'], + output_uri=True, + reuse_last_task_id=opt.exist_ok, + auto_connect_frameworks={'pytorch': False} + # We disconnect pytorch auto-detection, because we added manual model save points in the code + ) + # ClearML's hooks will already grab all general parameters + # Only the hyperparameters coming from the yaml config file + # will have to be added manually! + self.task.connect(hyp, name='Hyperparameters') + self.task.connect(opt, name='Args') + + # Make sure the code is easily remotely runnable by setting the docker image to use by the remote agent + self.task.set_base_docker('ultralytics/yolov5:latest', + docker_arguments='--ipc=host -e="CLEARML_AGENT_SKIP_PYTHON_ENV_INSTALL=1"', + docker_setup_bash_script='pip install clearml') + + # Get ClearML Dataset Version if requested + if opt.data.startswith('clearml://'): + # data_dict should have the following keys: + # names, nc (number of classes), test, train, val (all three relative paths to ../datasets) + self.data_dict = construct_dataset(opt.data) + # Set data to data_dict because wandb will crash without this information and opt is the best way + # to give it to them + opt.data = self.data_dict + + def log_debug_samples(self, files, title='Debug Samples'): + """ + Log files (images) as debug samples in the ClearML task. + + arguments: + files (List(PosixPath)) a list of file paths in PosixPath format + title (str) A title that groups together images with the same values + """ + for f in files: + if f.exists(): + it = re.search(r'_batch(\d+)', f.name) + iteration = int(it.groups()[0]) if it else 0 + self.task.get_logger().report_image(title=title, + series=f.name.replace(it.group(), ''), + local_path=str(f), + iteration=iteration) + + def log_image_with_boxes(self, image_path, boxes, class_names, image, conf_threshold=0.25): + """ + Draw the bounding boxes on a single image and report the result as a ClearML debug sample. + + arguments: + image_path (PosixPath) the path the original image file + boxes (list): list of scaled predictions in the format - [xmin, ymin, xmax, ymax, confidence, class] + class_names (dict): dict containing mapping of class int to class name + image (Tensor): A torch tensor containing the actual image data + """ + if len(self.current_epoch_logged_images) < self.max_imgs_to_log_per_epoch and self.current_epoch >= 0: + # Log every bbox_interval times and deduplicate for any intermittend extra eval runs + if self.current_epoch % self.bbox_interval == 0 and image_path not in self.current_epoch_logged_images: + im = np.ascontiguousarray(np.moveaxis(image.mul(255).clamp(0, 255).byte().cpu().numpy(), 0, 2)) + annotator = Annotator(im=im, pil=True) + for i, (conf, class_nr, box) in enumerate(zip(boxes[:, 4], boxes[:, 5], boxes[:, :4])): + color = colors(i) + + class_name = class_names[int(class_nr)] + confidence_percentage = round(float(conf) * 100, 2) + label = f'{class_name}: {confidence_percentage}%' + + if conf > conf_threshold: + annotator.rectangle(box.cpu().numpy(), outline=color) + annotator.box_label(box.cpu().numpy(), label=label, color=color) + + annotated_image = annotator.result() + self.task.get_logger().report_image(title='Bounding Boxes', + series=image_path.name, + iteration=self.current_epoch, + image=annotated_image) + self.current_epoch_logged_images.add(image_path) diff --git a/utils/loggers/clearml/hpo.py b/utils/loggers/clearml/hpo.py new file mode 100644 index 0000000..ee518b0 --- /dev/null +++ b/utils/loggers/clearml/hpo.py @@ -0,0 +1,84 @@ +from clearml import Task +# Connecting ClearML with the current process, +# from here on everything is logged automatically +from clearml.automation import HyperParameterOptimizer, UniformParameterRange +from clearml.automation.optuna import OptimizerOptuna + +task = Task.init(project_name='Hyper-Parameter Optimization', + task_name='YOLOv5', + task_type=Task.TaskTypes.optimizer, + reuse_last_task_id=False) + +# Example use case: +optimizer = HyperParameterOptimizer( + # This is the experiment we want to optimize + base_task_id='', + # here we define the hyper-parameters to optimize + # Notice: The parameter name should exactly match what you see in the UI: / + # For Example, here we see in the base experiment a section Named: "General" + # under it a parameter named "batch_size", this becomes "General/batch_size" + # If you have `argparse` for example, then arguments will appear under the "Args" section, + # and you should instead pass "Args/batch_size" + hyper_parameters=[ + UniformParameterRange('Hyperparameters/lr0', min_value=1e-5, max_value=1e-1), + UniformParameterRange('Hyperparameters/lrf', min_value=0.01, max_value=1.0), + UniformParameterRange('Hyperparameters/momentum', min_value=0.6, max_value=0.98), + UniformParameterRange('Hyperparameters/weight_decay', min_value=0.0, max_value=0.001), + UniformParameterRange('Hyperparameters/warmup_epochs', min_value=0.0, max_value=5.0), + UniformParameterRange('Hyperparameters/warmup_momentum', min_value=0.0, max_value=0.95), + UniformParameterRange('Hyperparameters/warmup_bias_lr', min_value=0.0, max_value=0.2), + UniformParameterRange('Hyperparameters/box', min_value=0.02, max_value=0.2), + UniformParameterRange('Hyperparameters/cls', min_value=0.2, max_value=4.0), + UniformParameterRange('Hyperparameters/cls_pw', min_value=0.5, max_value=2.0), + UniformParameterRange('Hyperparameters/obj', min_value=0.2, max_value=4.0), + UniformParameterRange('Hyperparameters/obj_pw', min_value=0.5, max_value=2.0), + UniformParameterRange('Hyperparameters/iou_t', min_value=0.1, max_value=0.7), + UniformParameterRange('Hyperparameters/anchor_t', min_value=2.0, max_value=8.0), + UniformParameterRange('Hyperparameters/fl_gamma', min_value=0.0, max_value=4.0), + UniformParameterRange('Hyperparameters/hsv_h', min_value=0.0, max_value=0.1), + UniformParameterRange('Hyperparameters/hsv_s', min_value=0.0, max_value=0.9), + UniformParameterRange('Hyperparameters/hsv_v', min_value=0.0, max_value=0.9), + UniformParameterRange('Hyperparameters/degrees', min_value=0.0, max_value=45.0), + UniformParameterRange('Hyperparameters/translate', min_value=0.0, max_value=0.9), + UniformParameterRange('Hyperparameters/scale', min_value=0.0, max_value=0.9), + UniformParameterRange('Hyperparameters/shear', min_value=0.0, max_value=10.0), + UniformParameterRange('Hyperparameters/perspective', min_value=0.0, max_value=0.001), + UniformParameterRange('Hyperparameters/flipud', min_value=0.0, max_value=1.0), + UniformParameterRange('Hyperparameters/fliplr', min_value=0.0, max_value=1.0), + UniformParameterRange('Hyperparameters/mosaic', min_value=0.0, max_value=1.0), + UniformParameterRange('Hyperparameters/mixup', min_value=0.0, max_value=1.0), + UniformParameterRange('Hyperparameters/copy_paste', min_value=0.0, max_value=1.0)], + # this is the objective metric we want to maximize/minimize + objective_metric_title='metrics', + objective_metric_series='mAP_0.5', + # now we decide if we want to maximize it or minimize it (accuracy we maximize) + objective_metric_sign='max', + # let us limit the number of concurrent experiments, + # this in turn will make sure we do dont bombard the scheduler with experiments. + # if we have an auto-scaler connected, this, by proxy, will limit the number of machine + max_number_of_concurrent_tasks=1, + # this is the optimizer class (actually doing the optimization) + # Currently, we can choose from GridSearch, RandomSearch or OptimizerBOHB (Bayesian optimization Hyper-Band) + optimizer_class=OptimizerOptuna, + # If specified only the top K performing Tasks will be kept, the others will be automatically archived + save_top_k_tasks_only=5, # 5, + compute_time_limit=None, + total_max_jobs=20, + min_iteration_per_job=None, + max_iteration_per_job=None, +) + +# report every 10 seconds, this is way too often, but we are testing here +optimizer.set_report_period(10 / 60) +# You can also use the line below instead to run all the optimizer tasks locally, without using queues or agent +# an_optimizer.start_locally(job_complete_callback=job_complete_callback) +# set the time limit for the optimization process (2 hours) +optimizer.set_time_limit(in_minutes=120.0) +# Start the optimization process in the local environment +optimizer.start_locally() +# wait until process is done (notice we are controlling the optimization process in the background) +optimizer.wait() +# make sure background optimization stopped +optimizer.stop() + +print('We are done, good bye') diff --git a/utils/loggers/comet/README.md b/utils/loggers/comet/README.md new file mode 100644 index 0000000..3ad52b0 --- /dev/null +++ b/utils/loggers/comet/README.md @@ -0,0 +1,258 @@ + + +# YOLOv5 with Comet + +This guide will cover how to use YOLOv5 with [Comet](https://bit.ly/yolov5-readme-comet2) + +# About Comet + +Comet builds tools that help data scientists, engineers, and team leaders accelerate and optimize machine learning and deep learning models. + +Track and visualize model metrics in real time, save your hyperparameters, datasets, and model checkpoints, and visualize your model predictions with [Comet Custom Panels](https://www.comet.com/docs/v2/guides/comet-dashboard/code-panels/about-panels/?utm_source=yolov5&utm_medium=partner&utm_campaign=partner_yolov5_2022&utm_content=github)! +Comet makes sure you never lose track of your work and makes it easy to share results and collaborate across teams of all sizes! + +# Getting Started + +## Install Comet + +```shell +pip install comet_ml +``` + +## Configure Comet Credentials + +There are two ways to configure Comet with YOLOv5. + +You can either set your credentials through environment variables + +**Environment Variables** + +```shell +export COMET_API_KEY= +export COMET_PROJECT_NAME= # This will default to 'yolov5' +``` + +Or create a `.comet.config` file in your working directory and set your credentials there. + +**Comet Configuration File** + +``` +[comet] +api_key= +project_name= # This will default to 'yolov5' +``` + +## Run the Training Script + +```shell +# Train YOLOv5s on COCO128 for 5 epochs +python train.py --img 640 --batch 16 --epochs 5 --data coco128.yaml --weights yolov5s.pt +``` + +That's it! Comet will automatically log your hyperparameters, command line arguments, training and validation metrics. You can visualize and analyze your runs in the Comet UI + +yolo-ui + +# Try out an Example! + +Check out an example of a [completed run here](https://www.comet.com/examples/comet-example-yolov5/a0e29e0e9b984e4a822db2a62d0cb357?experiment-tab=chart&showOutliers=true&smoothing=0&transformY=smoothing&xAxis=step&utm_source=yolov5&utm_medium=partner&utm_campaign=partner_yolov5_2022&utm_content=github) + +Or better yet, try it out yourself in this Colab Notebook + +[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/comet-ml/comet-examples/blob/master/integrations/model-training/yolov5/notebooks/Comet_and_YOLOv5.ipynb) + +# Log automatically + +By default, Comet will log the following items + +## Metrics + +- Box Loss, Object Loss, Classification Loss for the training and validation data +- mAP_0.5, mAP_0.5:0.95 metrics for the validation data. +- Precision and Recall for the validation data + +## Parameters + +- Model Hyperparameters +- All parameters passed through the command line options + +## Visualizations + +- Confusion Matrix of the model predictions on the validation data +- Plots for the PR and F1 curves across all classes +- Correlogram of the Class Labels + +# Configure Comet Logging + +Comet can be configured to log additional data either through command line flags passed to the training script +or through environment variables. + +```shell +export COMET_MODE=online # Set whether to run Comet in 'online' or 'offline' mode. Defaults to online +export COMET_MODEL_NAME= #Set the name for the saved model. Defaults to yolov5 +export COMET_LOG_CONFUSION_MATRIX=false # Set to disable logging a Comet Confusion Matrix. Defaults to true +export COMET_MAX_IMAGE_UPLOADS= # Controls how many total image predictions to log to Comet. Defaults to 100. +export COMET_LOG_PER_CLASS_METRICS=true # Set to log evaluation metrics for each detected class at the end of training. Defaults to false +export COMET_DEFAULT_CHECKPOINT_FILENAME= # Set this if you would like to resume training from a different checkpoint. Defaults to 'last.pt' +export COMET_LOG_BATCH_LEVEL_METRICS=true # Set this if you would like to log training metrics at the batch level. Defaults to false. +export COMET_LOG_PREDICTIONS=true # Set this to false to disable logging model predictions +``` + +## Logging Checkpoints with Comet + +Logging Models to Comet is disabled by default. To enable it, pass the `save-period` argument to the training script. This will save the +logged checkpoints to Comet based on the interval value provided by `save-period` + +```shell +python train.py \ +--img 640 \ +--batch 16 \ +--epochs 5 \ +--data coco128.yaml \ +--weights yolov5s.pt \ +--save-period 1 +``` + +## Logging Model Predictions + +By default, model predictions (images, ground truth labels and bounding boxes) will be logged to Comet. + +You can control the frequency of logged predictions and the associated images by passing the `bbox_interval` command line argument. Predictions can be visualized using Comet's Object Detection Custom Panel. This frequency corresponds to every Nth batch of data per epoch. In the example below, we are logging every 2nd batch of data for each epoch. + +**Note:** The YOLOv5 validation dataloader will default to a batch size of 32, so you will have to set the logging frequency accordingly. + +Here is an [example project using the Panel](https://www.comet.com/examples/comet-example-yolov5?shareable=YcwMiJaZSXfcEXpGOHDD12vA1&utm_source=yolov5&utm_medium=partner&utm_campaign=partner_yolov5_2022&utm_content=github) + +```shell +python train.py \ +--img 640 \ +--batch 16 \ +--epochs 5 \ +--data coco128.yaml \ +--weights yolov5s.pt \ +--bbox_interval 2 +``` + +### Controlling the number of Prediction Images logged to Comet + +When logging predictions from YOLOv5, Comet will log the images associated with each set of predictions. By default a maximum of 100 validation images are logged. You can increase or decrease this number using the `COMET_MAX_IMAGE_UPLOADS` environment variable. + +```shell +env COMET_MAX_IMAGE_UPLOADS=200 python train.py \ +--img 640 \ +--batch 16 \ +--epochs 5 \ +--data coco128.yaml \ +--weights yolov5s.pt \ +--bbox_interval 1 +``` + +### Logging Class Level Metrics + +Use the `COMET_LOG_PER_CLASS_METRICS` environment variable to log mAP, precision, recall, f1 for each class. + +```shell +env COMET_LOG_PER_CLASS_METRICS=true python train.py \ +--img 640 \ +--batch 16 \ +--epochs 5 \ +--data coco128.yaml \ +--weights yolov5s.pt +``` + +## Uploading a Dataset to Comet Artifacts + +If you would like to store your data using [Comet Artifacts](https://www.comet.com/docs/v2/guides/data-management/using-artifacts/#learn-more?utm_source=yolov5&utm_medium=partner&utm_campaign=partner_yolov5_2022&utm_content=github), you can do so using the `upload_dataset` flag. + +The dataset be organized in the way described in the [YOLOv5 documentation](https://docs.ultralytics.com/yolov5/tutorials/train_custom_data/). The dataset config `yaml` file must follow the same format as that of the `coco128.yaml` file. + +```shell +python train.py \ +--img 640 \ +--batch 16 \ +--epochs 5 \ +--data coco128.yaml \ +--weights yolov5s.pt \ +--upload_dataset +``` + +You can find the uploaded dataset in the Artifacts tab in your Comet Workspace +artifact-1 + +You can preview the data directly in the Comet UI. +artifact-2 + +Artifacts are versioned and also support adding metadata about the dataset. Comet will automatically log the metadata from your dataset `yaml` file +artifact-3 + +### Using a saved Artifact + +If you would like to use a dataset from Comet Artifacts, set the `path` variable in your dataset `yaml` file to point to the following Artifact resource URL. + +``` +# contents of artifact.yaml file +path: "comet:///:" +``` + +Then pass this file to your training script in the following way + +```shell +python train.py \ +--img 640 \ +--batch 16 \ +--epochs 5 \ +--data artifact.yaml \ +--weights yolov5s.pt +``` + +Artifacts also allow you to track the lineage of data as it flows through your Experimentation workflow. Here you can see a graph that shows you all the experiments that have used your uploaded dataset. +artifact-4 + +## Resuming a Training Run + +If your training run is interrupted for any reason, e.g. disrupted internet connection, you can resume the run using the `resume` flag and the Comet Run Path. + +The Run Path has the following format `comet:////`. + +This will restore the run to its state before the interruption, which includes restoring the model from a checkpoint, restoring all hyperparameters and training arguments and downloading Comet dataset Artifacts if they were used in the original run. The resumed run will continue logging to the existing Experiment in the Comet UI + +```shell +python train.py \ +--resume "comet://" +``` + +## Hyperparameter Search with the Comet Optimizer + +YOLOv5 is also integrated with Comet's Optimizer, making is simple to visualize hyperparameter sweeps in the Comet UI. + +### Configuring an Optimizer Sweep + +To configure the Comet Optimizer, you will have to create a JSON file with the information about the sweep. An example file has been provided in `utils/loggers/comet/optimizer_config.json` + +```shell +python utils/loggers/comet/hpo.py \ + --comet_optimizer_config "utils/loggers/comet/optimizer_config.json" +``` + +The `hpo.py` script accepts the same arguments as `train.py`. If you wish to pass additional arguments to your sweep simply add them after +the script. + +```shell +python utils/loggers/comet/hpo.py \ + --comet_optimizer_config "utils/loggers/comet/optimizer_config.json" \ + --save-period 1 \ + --bbox_interval 1 +``` + +### Running a Sweep in Parallel + +```shell +comet optimizer -j utils/loggers/comet/hpo.py \ + utils/loggers/comet/optimizer_config.json" +``` + +### Visualizing Results + +Comet provides a number of ways to visualize the results of your sweep. Take a look at a [project with a completed sweep here](https://www.comet.com/examples/comet-example-yolov5/view/PrlArHGuuhDTKC1UuBmTtOSXD/panels?utm_source=yolov5&utm_medium=partner&utm_campaign=partner_yolov5_2022&utm_content=github) + +hyperparameter-yolo diff --git a/utils/loggers/comet/__init__.py b/utils/loggers/comet/__init__.py new file mode 100644 index 0000000..c14a5f8 --- /dev/null +++ b/utils/loggers/comet/__init__.py @@ -0,0 +1,518 @@ +import glob +import json +import logging +import os +import sys +from pathlib import Path + +logger = logging.getLogger(__name__) + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[3] # YOLOv5 root directory +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) # add ROOT to PATH + +try: + import comet_ml + + # Project Configuration + config = comet_ml.config.get_config() + COMET_PROJECT_NAME = config.get_string(os.getenv('COMET_PROJECT_NAME'), 'comet.project_name', default='yolov5') +except ImportError: + comet_ml = None + COMET_PROJECT_NAME = None + +import PIL +import torch +import torchvision.transforms as T +import yaml + +from utils.dataloaders import img2label_paths +from utils.general import check_dataset, scale_boxes, xywh2xyxy +from utils.metrics import box_iou + +COMET_PREFIX = 'comet://' + +COMET_MODE = os.getenv('COMET_MODE', 'online') + +# Model Saving Settings +COMET_MODEL_NAME = os.getenv('COMET_MODEL_NAME', 'yolov5') + +# Dataset Artifact Settings +COMET_UPLOAD_DATASET = os.getenv('COMET_UPLOAD_DATASET', 'false').lower() == 'true' + +# Evaluation Settings +COMET_LOG_CONFUSION_MATRIX = (os.getenv('COMET_LOG_CONFUSION_MATRIX', 'true').lower() == 'true') +COMET_LOG_PREDICTIONS = os.getenv('COMET_LOG_PREDICTIONS', 'true').lower() == 'true' +COMET_MAX_IMAGE_UPLOADS = int(os.getenv('COMET_MAX_IMAGE_UPLOADS', 100)) + +# Confusion Matrix Settings +CONF_THRES = float(os.getenv('CONF_THRES', 0.001)) +IOU_THRES = float(os.getenv('IOU_THRES', 0.6)) + +# Batch Logging Settings +COMET_LOG_BATCH_METRICS = (os.getenv('COMET_LOG_BATCH_METRICS', 'false').lower() == 'true') +COMET_BATCH_LOGGING_INTERVAL = os.getenv('COMET_BATCH_LOGGING_INTERVAL', 1) +COMET_PREDICTION_LOGGING_INTERVAL = os.getenv('COMET_PREDICTION_LOGGING_INTERVAL', 1) +COMET_LOG_PER_CLASS_METRICS = (os.getenv('COMET_LOG_PER_CLASS_METRICS', 'false').lower() == 'true') + +RANK = int(os.getenv('RANK', -1)) + +to_pil = T.ToPILImage() + + +class CometLogger: + """Log metrics, parameters, source code, models and much more + with Comet + """ + + def __init__(self, opt, hyp, run_id=None, job_type='Training', **experiment_kwargs) -> None: + self.job_type = job_type + self.opt = opt + self.hyp = hyp + + # Comet Flags + self.comet_mode = COMET_MODE + + self.save_model = opt.save_period > -1 + self.model_name = COMET_MODEL_NAME + + # Batch Logging Settings + self.log_batch_metrics = COMET_LOG_BATCH_METRICS + self.comet_log_batch_interval = COMET_BATCH_LOGGING_INTERVAL + + # Dataset Artifact Settings + self.upload_dataset = self.opt.upload_dataset or COMET_UPLOAD_DATASET + self.resume = self.opt.resume + + # Default parameters to pass to Experiment objects + self.default_experiment_kwargs = { + 'log_code': False, + 'log_env_gpu': True, + 'log_env_cpu': True, + 'project_name': COMET_PROJECT_NAME, } + self.default_experiment_kwargs.update(experiment_kwargs) + self.experiment = self._get_experiment(self.comet_mode, run_id) + self.experiment.set_name(self.opt.name) + + self.data_dict = self.check_dataset(self.opt.data) + self.class_names = self.data_dict['names'] + self.num_classes = self.data_dict['nc'] + + self.logged_images_count = 0 + self.max_images = COMET_MAX_IMAGE_UPLOADS + + if run_id is None: + self.experiment.log_other('Created from', 'YOLOv5') + if not isinstance(self.experiment, comet_ml.OfflineExperiment): + workspace, project_name, experiment_id = self.experiment.url.split('/')[-3:] + self.experiment.log_other( + 'Run Path', + f'{workspace}/{project_name}/{experiment_id}', + ) + self.log_parameters(vars(opt)) + self.log_parameters(self.opt.hyp) + self.log_asset_data( + self.opt.hyp, + name='hyperparameters.json', + metadata={'type': 'hyp-config-file'}, + ) + self.log_asset( + f'{self.opt.save_dir}/opt.yaml', + metadata={'type': 'opt-config-file'}, + ) + + self.comet_log_confusion_matrix = COMET_LOG_CONFUSION_MATRIX + + if hasattr(self.opt, 'conf_thres'): + self.conf_thres = self.opt.conf_thres + else: + self.conf_thres = CONF_THRES + if hasattr(self.opt, 'iou_thres'): + self.iou_thres = self.opt.iou_thres + else: + self.iou_thres = IOU_THRES + + self.log_parameters({'val_iou_threshold': self.iou_thres, 'val_conf_threshold': self.conf_thres}) + + self.comet_log_predictions = COMET_LOG_PREDICTIONS + if self.opt.bbox_interval == -1: + self.comet_log_prediction_interval = (1 if self.opt.epochs < 10 else self.opt.epochs // 10) + else: + self.comet_log_prediction_interval = self.opt.bbox_interval + + if self.comet_log_predictions: + self.metadata_dict = {} + self.logged_image_names = [] + + self.comet_log_per_class_metrics = COMET_LOG_PER_CLASS_METRICS + + self.experiment.log_others({ + 'comet_mode': COMET_MODE, + 'comet_max_image_uploads': COMET_MAX_IMAGE_UPLOADS, + 'comet_log_per_class_metrics': COMET_LOG_PER_CLASS_METRICS, + 'comet_log_batch_metrics': COMET_LOG_BATCH_METRICS, + 'comet_log_confusion_matrix': COMET_LOG_CONFUSION_MATRIX, + 'comet_model_name': COMET_MODEL_NAME, }) + + # Check if running the Experiment with the Comet Optimizer + if hasattr(self.opt, 'comet_optimizer_id'): + self.experiment.log_other('optimizer_id', self.opt.comet_optimizer_id) + self.experiment.log_other('optimizer_objective', self.opt.comet_optimizer_objective) + self.experiment.log_other('optimizer_metric', self.opt.comet_optimizer_metric) + self.experiment.log_other('optimizer_parameters', json.dumps(self.hyp)) + + def _get_experiment(self, mode, experiment_id=None): + if mode == 'offline': + if experiment_id is not None: + return comet_ml.ExistingOfflineExperiment( + previous_experiment=experiment_id, + **self.default_experiment_kwargs, + ) + + return comet_ml.OfflineExperiment(**self.default_experiment_kwargs, ) + + else: + try: + if experiment_id is not None: + return comet_ml.ExistingExperiment( + previous_experiment=experiment_id, + **self.default_experiment_kwargs, + ) + + return comet_ml.Experiment(**self.default_experiment_kwargs) + + except ValueError: + logger.warning('COMET WARNING: ' + 'Comet credentials have not been set. ' + 'Comet will default to offline logging. ' + 'Please set your credentials to enable online logging.') + return self._get_experiment('offline', experiment_id) + + return + + def log_metrics(self, log_dict, **kwargs): + self.experiment.log_metrics(log_dict, **kwargs) + + def log_parameters(self, log_dict, **kwargs): + self.experiment.log_parameters(log_dict, **kwargs) + + def log_asset(self, asset_path, **kwargs): + self.experiment.log_asset(asset_path, **kwargs) + + def log_asset_data(self, asset, **kwargs): + self.experiment.log_asset_data(asset, **kwargs) + + def log_image(self, img, **kwargs): + self.experiment.log_image(img, **kwargs) + + def log_model(self, path, opt, epoch, fitness_score, best_model=False): + if not self.save_model: + return + + model_metadata = { + 'fitness_score': fitness_score[-1], + 'epochs_trained': epoch + 1, + 'save_period': opt.save_period, + 'total_epochs': opt.epochs, } + + model_files = glob.glob(f'{path}/*.pt') + for model_path in model_files: + name = Path(model_path).name + + self.experiment.log_model( + self.model_name, + file_or_folder=model_path, + file_name=name, + metadata=model_metadata, + overwrite=True, + ) + + def check_dataset(self, data_file): + with open(data_file) as f: + data_config = yaml.safe_load(f) + + path = data_config.get('path') + if path and path.startswith(COMET_PREFIX): + path = data_config['path'].replace(COMET_PREFIX, '') + data_dict = self.download_dataset_artifact(path) + + return data_dict + + self.log_asset(self.opt.data, metadata={'type': 'data-config-file'}) + + return check_dataset(data_file) + + def log_predictions(self, image, labelsn, path, shape, predn): + if self.logged_images_count >= self.max_images: + return + detections = predn[predn[:, 4] > self.conf_thres] + iou = box_iou(labelsn[:, 1:], detections[:, :4]) + mask, _ = torch.where(iou > self.iou_thres) + if len(mask) == 0: + return + + filtered_detections = detections[mask] + filtered_labels = labelsn[mask] + + image_id = path.split('/')[-1].split('.')[0] + image_name = f'{image_id}_curr_epoch_{self.experiment.curr_epoch}' + if image_name not in self.logged_image_names: + native_scale_image = PIL.Image.open(path) + self.log_image(native_scale_image, name=image_name) + self.logged_image_names.append(image_name) + + metadata = [] + for cls, *xyxy in filtered_labels.tolist(): + metadata.append({ + 'label': f'{self.class_names[int(cls)]}-gt', + 'score': 100, + 'box': { + 'x': xyxy[0], + 'y': xyxy[1], + 'x2': xyxy[2], + 'y2': xyxy[3]}, }) + for *xyxy, conf, cls in filtered_detections.tolist(): + metadata.append({ + 'label': f'{self.class_names[int(cls)]}', + 'score': conf * 100, + 'box': { + 'x': xyxy[0], + 'y': xyxy[1], + 'x2': xyxy[2], + 'y2': xyxy[3]}, }) + + self.metadata_dict[image_name] = metadata + self.logged_images_count += 1 + + return + + def preprocess_prediction(self, image, labels, shape, pred): + nl, _ = labels.shape[0], pred.shape[0] + + # Predictions + if self.opt.single_cls: + pred[:, 5] = 0 + + predn = pred.clone() + scale_boxes(image.shape[1:], predn[:, :4], shape[0], shape[1]) + + labelsn = None + if nl: + tbox = xywh2xyxy(labels[:, 1:5]) # target boxes + scale_boxes(image.shape[1:], tbox, shape[0], shape[1]) # native-space labels + labelsn = torch.cat((labels[:, 0:1], tbox), 1) # native-space labels + scale_boxes(image.shape[1:], predn[:, :4], shape[0], shape[1]) # native-space pred + + return predn, labelsn + + def add_assets_to_artifact(self, artifact, path, asset_path, split): + img_paths = sorted(glob.glob(f'{asset_path}/*')) + label_paths = img2label_paths(img_paths) + + for image_file, label_file in zip(img_paths, label_paths): + image_logical_path, label_logical_path = map(lambda x: os.path.relpath(x, path), [image_file, label_file]) + + try: + artifact.add( + image_file, + logical_path=image_logical_path, + metadata={'split': split}, + ) + artifact.add( + label_file, + logical_path=label_logical_path, + metadata={'split': split}, + ) + except ValueError as e: + logger.error('COMET ERROR: Error adding file to Artifact. Skipping file.') + logger.error(f'COMET ERROR: {e}') + continue + + return artifact + + def upload_dataset_artifact(self): + dataset_name = self.data_dict.get('dataset_name', 'yolov5-dataset') + path = str((ROOT / Path(self.data_dict['path'])).resolve()) + + metadata = self.data_dict.copy() + for key in ['train', 'val', 'test']: + split_path = metadata.get(key) + if split_path is not None: + metadata[key] = split_path.replace(path, '') + + artifact = comet_ml.Artifact(name=dataset_name, artifact_type='dataset', metadata=metadata) + for key in metadata.keys(): + if key in ['train', 'val', 'test']: + if isinstance(self.upload_dataset, str) and (key != self.upload_dataset): + continue + + asset_path = self.data_dict.get(key) + if asset_path is not None: + artifact = self.add_assets_to_artifact(artifact, path, asset_path, key) + + self.experiment.log_artifact(artifact) + + return + + def download_dataset_artifact(self, artifact_path): + logged_artifact = self.experiment.get_artifact(artifact_path) + artifact_save_dir = str(Path(self.opt.save_dir) / logged_artifact.name) + logged_artifact.download(artifact_save_dir) + + metadata = logged_artifact.metadata + data_dict = metadata.copy() + data_dict['path'] = artifact_save_dir + + metadata_names = metadata.get('names') + if isinstance(metadata_names, dict): + data_dict['names'] = {int(k): v for k, v in metadata.get('names').items()} + elif isinstance(metadata_names, list): + data_dict['names'] = {int(k): v for k, v in zip(range(len(metadata_names)), metadata_names)} + else: + raise "Invalid 'names' field in dataset yaml file. Please use a list or dictionary" + + return self.update_data_paths(data_dict) + + def update_data_paths(self, data_dict): + path = data_dict.get('path', '') + + for split in ['train', 'val', 'test']: + if data_dict.get(split): + split_path = data_dict.get(split) + data_dict[split] = (f'{path}/{split_path}' if isinstance(split, str) else [ + f'{path}/{x}' for x in split_path]) + + return data_dict + + def on_pretrain_routine_end(self, paths): + if self.opt.resume: + return + + for path in paths: + self.log_asset(str(path)) + + if self.upload_dataset: + if not self.resume: + self.upload_dataset_artifact() + + return + + def on_train_start(self): + self.log_parameters(self.hyp) + + def on_train_epoch_start(self): + return + + def on_train_epoch_end(self, epoch): + self.experiment.curr_epoch = epoch + + return + + def on_train_batch_start(self): + return + + def on_train_batch_end(self, log_dict, step): + self.experiment.curr_step = step + if self.log_batch_metrics and (step % self.comet_log_batch_interval == 0): + self.log_metrics(log_dict, step=step) + + return + + def on_train_end(self, files, save_dir, last, best, epoch, results): + if self.comet_log_predictions: + curr_epoch = self.experiment.curr_epoch + self.experiment.log_asset_data(self.metadata_dict, 'image-metadata.json', epoch=curr_epoch) + + for f in files: + self.log_asset(f, metadata={'epoch': epoch}) + self.log_asset(f'{save_dir}/results.csv', metadata={'epoch': epoch}) + + if not self.opt.evolve: + model_path = str(best if best.exists() else last) + name = Path(model_path).name + if self.save_model: + self.experiment.log_model( + self.model_name, + file_or_folder=model_path, + file_name=name, + overwrite=True, + ) + + # Check if running Experiment with Comet Optimizer + if hasattr(self.opt, 'comet_optimizer_id'): + metric = results.get(self.opt.comet_optimizer_metric) + self.experiment.log_other('optimizer_metric_value', metric) + + self.finish_run() + + def on_val_start(self): + return + + def on_val_batch_start(self): + return + + def on_val_batch_end(self, batch_i, images, targets, paths, shapes, outputs): + if not (self.comet_log_predictions and ((batch_i + 1) % self.comet_log_prediction_interval == 0)): + return + + for si, pred in enumerate(outputs): + if len(pred) == 0: + continue + + image = images[si] + labels = targets[targets[:, 0] == si, 1:] + shape = shapes[si] + path = paths[si] + predn, labelsn = self.preprocess_prediction(image, labels, shape, pred) + if labelsn is not None: + self.log_predictions(image, labelsn, path, shape, predn) + + return + + def on_val_end(self, nt, tp, fp, p, r, f1, ap, ap50, ap_class, confusion_matrix): + if self.comet_log_per_class_metrics: + if self.num_classes > 1: + for i, c in enumerate(ap_class): + class_name = self.class_names[c] + self.experiment.log_metrics( + { + 'mAP@.5': ap50[i], + 'mAP@.5:.95': ap[i], + 'precision': p[i], + 'recall': r[i], + 'f1': f1[i], + 'true_positives': tp[i], + 'false_positives': fp[i], + 'support': nt[c], }, + prefix=class_name, + ) + + if self.comet_log_confusion_matrix: + epoch = self.experiment.curr_epoch + class_names = list(self.class_names.values()) + class_names.append('background') + num_classes = len(class_names) + + self.experiment.log_confusion_matrix( + matrix=confusion_matrix.matrix, + max_categories=num_classes, + labels=class_names, + epoch=epoch, + column_label='Actual Category', + row_label='Predicted Category', + file_name=f'confusion-matrix-epoch-{epoch}.json', + ) + + def on_fit_epoch_end(self, result, epoch): + self.log_metrics(result, epoch=epoch) + + def on_model_save(self, last, epoch, final_epoch, best_fitness, fi): + if ((epoch + 1) % self.opt.save_period == 0 and not final_epoch) and self.opt.save_period != -1: + self.log_model(last.parent, self.opt, epoch, fi, best_model=best_fitness == fi) + + def on_params_update(self, params): + self.log_parameters(params) + + def finish_run(self): + self.experiment.end() diff --git a/utils/loggers/comet/comet_utils.py b/utils/loggers/comet/comet_utils.py new file mode 100644 index 0000000..2760076 --- /dev/null +++ b/utils/loggers/comet/comet_utils.py @@ -0,0 +1,150 @@ +import logging +import os +from urllib.parse import urlparse + +try: + import comet_ml +except (ModuleNotFoundError, ImportError): + comet_ml = None + +import yaml + +logger = logging.getLogger(__name__) + +COMET_PREFIX = 'comet://' +COMET_MODEL_NAME = os.getenv('COMET_MODEL_NAME', 'yolov5') +COMET_DEFAULT_CHECKPOINT_FILENAME = os.getenv('COMET_DEFAULT_CHECKPOINT_FILENAME', 'last.pt') + + +def download_model_checkpoint(opt, experiment): + model_dir = f'{opt.project}/{experiment.name}' + os.makedirs(model_dir, exist_ok=True) + + model_name = COMET_MODEL_NAME + model_asset_list = experiment.get_model_asset_list(model_name) + + if len(model_asset_list) == 0: + logger.error(f'COMET ERROR: No checkpoints found for model name : {model_name}') + return + + model_asset_list = sorted( + model_asset_list, + key=lambda x: x['step'], + reverse=True, + ) + logged_checkpoint_map = {asset['fileName']: asset['assetId'] for asset in model_asset_list} + + resource_url = urlparse(opt.weights) + checkpoint_filename = resource_url.query + + if checkpoint_filename: + asset_id = logged_checkpoint_map.get(checkpoint_filename) + else: + asset_id = logged_checkpoint_map.get(COMET_DEFAULT_CHECKPOINT_FILENAME) + checkpoint_filename = COMET_DEFAULT_CHECKPOINT_FILENAME + + if asset_id is None: + logger.error(f'COMET ERROR: Checkpoint {checkpoint_filename} not found in the given Experiment') + return + + try: + logger.info(f'COMET INFO: Downloading checkpoint {checkpoint_filename}') + asset_filename = checkpoint_filename + + model_binary = experiment.get_asset(asset_id, return_type='binary', stream=False) + model_download_path = f'{model_dir}/{asset_filename}' + with open(model_download_path, 'wb') as f: + f.write(model_binary) + + opt.weights = model_download_path + + except Exception as e: + logger.warning('COMET WARNING: Unable to download checkpoint from Comet') + logger.exception(e) + + +def set_opt_parameters(opt, experiment): + """Update the opts Namespace with parameters + from Comet's ExistingExperiment when resuming a run + + Args: + opt (argparse.Namespace): Namespace of command line options + experiment (comet_ml.APIExperiment): Comet API Experiment object + """ + asset_list = experiment.get_asset_list() + resume_string = opt.resume + + for asset in asset_list: + if asset['fileName'] == 'opt.yaml': + asset_id = asset['assetId'] + asset_binary = experiment.get_asset(asset_id, return_type='binary', stream=False) + opt_dict = yaml.safe_load(asset_binary) + for key, value in opt_dict.items(): + setattr(opt, key, value) + opt.resume = resume_string + + # Save hyperparameters to YAML file + # Necessary to pass checks in training script + save_dir = f'{opt.project}/{experiment.name}' + os.makedirs(save_dir, exist_ok=True) + + hyp_yaml_path = f'{save_dir}/hyp.yaml' + with open(hyp_yaml_path, 'w') as f: + yaml.dump(opt.hyp, f) + opt.hyp = hyp_yaml_path + + +def check_comet_weights(opt): + """Downloads model weights from Comet and updates the + weights path to point to saved weights location + + Args: + opt (argparse.Namespace): Command Line arguments passed + to YOLOv5 training script + + Returns: + None/bool: Return True if weights are successfully downloaded + else return None + """ + if comet_ml is None: + return + + if isinstance(opt.weights, str): + if opt.weights.startswith(COMET_PREFIX): + api = comet_ml.API() + resource = urlparse(opt.weights) + experiment_path = f'{resource.netloc}{resource.path}' + experiment = api.get(experiment_path) + download_model_checkpoint(opt, experiment) + return True + + return None + + +def check_comet_resume(opt): + """Restores run parameters to its original state based on the model checkpoint + and logged Experiment parameters. + + Args: + opt (argparse.Namespace): Command Line arguments passed + to YOLOv5 training script + + Returns: + None/bool: Return True if the run is restored successfully + else return None + """ + if comet_ml is None: + return + + if isinstance(opt.resume, str): + if opt.resume.startswith(COMET_PREFIX): + api = comet_ml.API() + resource = urlparse(opt.resume) + experiment_path = f'{resource.netloc}{resource.path}' + experiment = api.get(experiment_path) + set_opt_parameters(opt, experiment) + download_model_checkpoint(opt, experiment) + + return True + + return None diff --git a/utils/loggers/comet/hpo.py b/utils/loggers/comet/hpo.py new file mode 100644 index 0000000..fc49115 --- /dev/null +++ b/utils/loggers/comet/hpo.py @@ -0,0 +1,118 @@ +import argparse +import json +import logging +import os +import sys +from pathlib import Path + +import comet_ml + +logger = logging.getLogger(__name__) + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[3] # YOLOv5 root directory +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) # add ROOT to PATH + +from train import train +from utils.callbacks import Callbacks +from utils.general import increment_path +from utils.torch_utils import select_device + +# Project Configuration +config = comet_ml.config.get_config() +COMET_PROJECT_NAME = config.get_string(os.getenv('COMET_PROJECT_NAME'), 'comet.project_name', default='yolov5') + + +def get_args(known=False): + parser = argparse.ArgumentParser() + parser.add_argument('--weights', type=str, default=ROOT / 'yolov5s.pt', help='initial weights path') + parser.add_argument('--cfg', type=str, default='', help='model.yaml path') + parser.add_argument('--data', type=str, default=ROOT / 'data/coco128.yaml', help='dataset.yaml path') + parser.add_argument('--hyp', type=str, default=ROOT / 'data/hyps/hyp.scratch-low.yaml', help='hyperparameters path') + parser.add_argument('--epochs', type=int, default=300, help='total training epochs') + parser.add_argument('--batch-size', type=int, default=16, help='total batch size for all GPUs, -1 for autobatch') + parser.add_argument('--imgsz', '--img', '--img-size', type=int, default=640, help='train, val image size (pixels)') + parser.add_argument('--rect', action='store_true', help='rectangular training') + parser.add_argument('--resume', nargs='?', const=True, default=False, help='resume most recent training') + parser.add_argument('--nosave', action='store_true', help='only save final checkpoint') + parser.add_argument('--noval', action='store_true', help='only validate final epoch') + parser.add_argument('--noautoanchor', action='store_true', help='disable AutoAnchor') + parser.add_argument('--noplots', action='store_true', help='save no plot files') + parser.add_argument('--evolve', type=int, nargs='?', const=300, help='evolve hyperparameters for x generations') + parser.add_argument('--bucket', type=str, default='', help='gsutil bucket') + parser.add_argument('--cache', type=str, nargs='?', const='ram', help='--cache images in "ram" (default) or "disk"') + parser.add_argument('--image-weights', action='store_true', help='use weighted image selection for training') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--multi-scale', action='store_true', help='vary img-size +/- 50%%') + parser.add_argument('--single-cls', action='store_true', help='train multi-class data as single-class') + parser.add_argument('--optimizer', type=str, choices=['SGD', 'Adam', 'AdamW'], default='SGD', help='optimizer') + parser.add_argument('--sync-bn', action='store_true', help='use SyncBatchNorm, only available in DDP mode') + parser.add_argument('--workers', type=int, default=8, help='max dataloader workers (per RANK in DDP mode)') + parser.add_argument('--project', default=ROOT / 'runs/train', help='save to project/name') + parser.add_argument('--name', default='exp', help='save to project/name') + parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment') + parser.add_argument('--quad', action='store_true', help='quad dataloader') + parser.add_argument('--cos-lr', action='store_true', help='cosine LR scheduler') + parser.add_argument('--label-smoothing', type=float, default=0.0, help='Label smoothing epsilon') + parser.add_argument('--patience', type=int, default=100, help='EarlyStopping patience (epochs without improvement)') + parser.add_argument('--freeze', nargs='+', type=int, default=[0], help='Freeze layers: backbone=10, first3=0 1 2') + parser.add_argument('--save-period', type=int, default=-1, help='Save checkpoint every x epochs (disabled if < 1)') + parser.add_argument('--seed', type=int, default=0, help='Global training seed') + parser.add_argument('--local_rank', type=int, default=-1, help='Automatic DDP Multi-GPU argument, do not modify') + + # Weights & Biases arguments + parser.add_argument('--entity', default=None, help='W&B: Entity') + parser.add_argument('--upload_dataset', nargs='?', const=True, default=False, help='W&B: Upload data, "val" option') + parser.add_argument('--bbox_interval', type=int, default=-1, help='W&B: Set bounding-box image logging interval') + parser.add_argument('--artifact_alias', type=str, default='latest', help='W&B: Version of dataset artifact to use') + + # Comet Arguments + parser.add_argument('--comet_optimizer_config', type=str, help='Comet: Path to a Comet Optimizer Config File.') + parser.add_argument('--comet_optimizer_id', type=str, help='Comet: ID of the Comet Optimizer sweep.') + parser.add_argument('--comet_optimizer_objective', type=str, help="Comet: Set to 'minimize' or 'maximize'.") + parser.add_argument('--comet_optimizer_metric', type=str, help='Comet: Metric to Optimize.') + parser.add_argument('--comet_optimizer_workers', + type=int, + default=1, + help='Comet: Number of Parallel Workers to use with the Comet Optimizer.') + + return parser.parse_known_args()[0] if known else parser.parse_args() + + +def run(parameters, opt): + hyp_dict = {k: v for k, v in parameters.items() if k not in ['epochs', 'batch_size']} + + opt.save_dir = str(increment_path(Path(opt.project) / opt.name, exist_ok=opt.exist_ok or opt.evolve)) + opt.batch_size = parameters.get('batch_size') + opt.epochs = parameters.get('epochs') + + device = select_device(opt.device, batch_size=opt.batch_size) + train(hyp_dict, opt, device, callbacks=Callbacks()) + + +if __name__ == '__main__': + opt = get_args(known=True) + + opt.weights = str(opt.weights) + opt.cfg = str(opt.cfg) + opt.data = str(opt.data) + opt.project = str(opt.project) + + optimizer_id = os.getenv('COMET_OPTIMIZER_ID') + if optimizer_id is None: + with open(opt.comet_optimizer_config) as f: + optimizer_config = json.load(f) + optimizer = comet_ml.Optimizer(optimizer_config) + else: + optimizer = comet_ml.Optimizer(optimizer_id) + + opt.comet_optimizer_id = optimizer.id + status = optimizer.status() + + opt.comet_optimizer_objective = status['spec']['objective'] + opt.comet_optimizer_metric = status['spec']['metric'] + + logger.info('COMET INFO: Starting Hyperparameter Sweep') + for parameter in optimizer.get_parameters(): + run(parameter['parameters'], opt) diff --git a/utils/loggers/wandb/__init__.py b/utils/loggers/wandb/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/utils/loggers/wandb/wandb_utils.py b/utils/loggers/wandb/wandb_utils.py new file mode 100644 index 0000000..4ea32b1 --- /dev/null +++ b/utils/loggers/wandb/wandb_utils.py @@ -0,0 +1,193 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license + +# WARNING ⚠️ wandb is deprecated and will be removed in future release. +# See supported integrations at https://github.com/ultralytics/yolov5#integrations + +import logging +import os +import sys +from contextlib import contextmanager +from pathlib import Path + +from utils.general import LOGGER, colorstr + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[3] # YOLOv5 root directory +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) # add ROOT to PATH +RANK = int(os.getenv('RANK', -1)) +DEPRECATION_WARNING = f"{colorstr('wandb')}: WARNING ⚠️ wandb is deprecated and will be removed in a future release. " \ + f'See supported integrations at https://github.com/ultralytics/yolov5#integrations.' + +try: + import wandb + + assert hasattr(wandb, '__version__') # verify package import not local dir + LOGGER.warning(DEPRECATION_WARNING) +except (ImportError, AssertionError): + wandb = None + + +class WandbLogger(): + """Log training runs, datasets, models, and predictions to Weights & Biases. + + This logger sends information to W&B at wandb.ai. By default, this information + includes hyperparameters, system configuration and metrics, model metrics, + and basic data metrics and analyses. + + By providing additional command line arguments to train.py, datasets, + models and predictions can also be logged. + + For more on how this logger is used, see the Weights & Biases documentation: + https://docs.wandb.com/guides/integrations/yolov5 + """ + + def __init__(self, opt, run_id=None, job_type='Training'): + """ + - Initialize WandbLogger instance + - Upload dataset if opt.upload_dataset is True + - Setup training processes if job_type is 'Training' + + arguments: + opt (namespace) -- Commandline arguments for this run + run_id (str) -- Run ID of W&B run to be resumed + job_type (str) -- To set the job_type for this run + + """ + # Pre-training routine -- + self.job_type = job_type + self.wandb, self.wandb_run = wandb, wandb.run if wandb else None + self.val_artifact, self.train_artifact = None, None + self.train_artifact_path, self.val_artifact_path = None, None + self.result_artifact = None + self.val_table, self.result_table = None, None + self.max_imgs_to_log = 16 + self.data_dict = None + if self.wandb: + self.wandb_run = wandb.init(config=opt, + resume='allow', + project='YOLOv5' if opt.project == 'runs/train' else Path(opt.project).stem, + entity=opt.entity, + name=opt.name if opt.name != 'exp' else None, + job_type=job_type, + id=run_id, + allow_val_change=True) if not wandb.run else wandb.run + + if self.wandb_run: + if self.job_type == 'Training': + if isinstance(opt.data, dict): + # This means another dataset manager has already processed the dataset info (e.g. ClearML) + # and they will have stored the already processed dict in opt.data + self.data_dict = opt.data + self.setup_training(opt) + + def setup_training(self, opt): + """ + Setup the necessary processes for training YOLO models: + - Attempt to download model checkpoint and dataset artifacts if opt.resume stats with WANDB_ARTIFACT_PREFIX + - Update data_dict, to contain info of previous run if resumed and the paths of dataset artifact if downloaded + - Setup log_dict, initialize bbox_interval + + arguments: + opt (namespace) -- commandline arguments for this run + + """ + self.log_dict, self.current_epoch = {}, 0 + self.bbox_interval = opt.bbox_interval + if isinstance(opt.resume, str): + model_dir, _ = self.download_model_artifact(opt) + if model_dir: + self.weights = Path(model_dir) / 'last.pt' + config = self.wandb_run.config + opt.weights, opt.save_period, opt.batch_size, opt.bbox_interval, opt.epochs, opt.hyp, opt.imgsz = str( + self.weights), config.save_period, config.batch_size, config.bbox_interval, config.epochs, \ + config.hyp, config.imgsz + + if opt.bbox_interval == -1: + self.bbox_interval = opt.bbox_interval = (opt.epochs // 10) if opt.epochs > 10 else 1 + if opt.evolve or opt.noplots: + self.bbox_interval = opt.bbox_interval = opt.epochs + 1 # disable bbox_interval + + def log_model(self, path, opt, epoch, fitness_score, best_model=False): + """ + Log the model checkpoint as W&B artifact + + arguments: + path (Path) -- Path of directory containing the checkpoints + opt (namespace) -- Command line arguments for this run + epoch (int) -- Current epoch number + fitness_score (float) -- fitness score for current epoch + best_model (boolean) -- Boolean representing if the current checkpoint is the best yet. + """ + model_artifact = wandb.Artifact('run_' + wandb.run.id + '_model', + type='model', + metadata={ + 'original_url': str(path), + 'epochs_trained': epoch + 1, + 'save period': opt.save_period, + 'project': opt.project, + 'total_epochs': opt.epochs, + 'fitness_score': fitness_score}) + model_artifact.add_file(str(path / 'last.pt'), name='last.pt') + wandb.log_artifact(model_artifact, + aliases=['latest', 'last', 'epoch ' + str(self.current_epoch), 'best' if best_model else '']) + LOGGER.info(f'Saving model artifact on epoch {epoch + 1}') + + def val_one_image(self, pred, predn, path, names, im): + pass + + def log(self, log_dict): + """ + save the metrics to the logging dictionary + + arguments: + log_dict (Dict) -- metrics/media to be logged in current step + """ + if self.wandb_run: + for key, value in log_dict.items(): + self.log_dict[key] = value + + def end_epoch(self): + """ + commit the log_dict, model artifacts and Tables to W&B and flush the log_dict. + + arguments: + best_result (boolean): Boolean representing if the result of this evaluation is best or not + """ + if self.wandb_run: + with all_logging_disabled(): + try: + wandb.log(self.log_dict) + except BaseException as e: + LOGGER.info( + f'An error occurred in wandb logger. The training will proceed without interruption. More info\n{e}' + ) + self.wandb_run.finish() + self.wandb_run = None + self.log_dict = {} + + def finish_run(self): + """ + Log metrics if any and finish the current W&B run + """ + if self.wandb_run: + if self.log_dict: + with all_logging_disabled(): + wandb.log(self.log_dict) + wandb.run.finish() + LOGGER.warning(DEPRECATION_WARNING) + + +@contextmanager +def all_logging_disabled(highest_level=logging.CRITICAL): + """ source - https://gist.github.com/simon-weber/7853144 + A context manager that will prevent any logging messages triggered during the body from being processed. + :param highest_level: the maximum logging level in use. + This would only need to be changed if a custom level greater than CRITICAL is defined. + """ + previous_level = logging.root.manager.disable + logging.disable(highest_level) + try: + yield + finally: + logging.disable(previous_level) diff --git a/utils/loss.py b/utils/loss.py new file mode 100644 index 0000000..26cca87 --- /dev/null +++ b/utils/loss.py @@ -0,0 +1,234 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Loss functions +""" + +import torch +import torch.nn as nn + +from utils.metrics import bbox_iou +from utils.torch_utils import de_parallel + + +def smooth_BCE(eps=0.1): # https://github.com/ultralytics/yolov3/issues/238#issuecomment-598028441 + # return positive, negative label smoothing BCE targets + return 1.0 - 0.5 * eps, 0.5 * eps + + +class BCEBlurWithLogitsLoss(nn.Module): + # BCEwithLogitLoss() with reduced missing label effects. + def __init__(self, alpha=0.05): + super().__init__() + self.loss_fcn = nn.BCEWithLogitsLoss(reduction='none') # must be nn.BCEWithLogitsLoss() + self.alpha = alpha + + def forward(self, pred, true): + loss = self.loss_fcn(pred, true) + pred = torch.sigmoid(pred) # prob from logits + dx = pred - true # reduce only missing label effects + # dx = (pred - true).abs() # reduce missing label and false label effects + alpha_factor = 1 - torch.exp((dx - 1) / (self.alpha + 1e-4)) + loss *= alpha_factor + return loss.mean() + + +class FocalLoss(nn.Module): + # Wraps focal loss around existing loss_fcn(), i.e. criteria = FocalLoss(nn.BCEWithLogitsLoss(), gamma=1.5) + def __init__(self, loss_fcn, gamma=1.5, alpha=0.25): + super().__init__() + self.loss_fcn = loss_fcn # must be nn.BCEWithLogitsLoss() + self.gamma = gamma + self.alpha = alpha + self.reduction = loss_fcn.reduction + self.loss_fcn.reduction = 'none' # required to apply FL to each element + + def forward(self, pred, true): + loss = self.loss_fcn(pred, true) + # p_t = torch.exp(-loss) + # loss *= self.alpha * (1.000001 - p_t) ** self.gamma # non-zero power for gradient stability + + # TF implementation https://github.com/tensorflow/addons/blob/v0.7.1/tensorflow_addons/losses/focal_loss.py + pred_prob = torch.sigmoid(pred) # prob from logits + p_t = true * pred_prob + (1 - true) * (1 - pred_prob) + alpha_factor = true * self.alpha + (1 - true) * (1 - self.alpha) + modulating_factor = (1.0 - p_t) ** self.gamma + loss *= alpha_factor * modulating_factor + + if self.reduction == 'mean': + return loss.mean() + elif self.reduction == 'sum': + return loss.sum() + else: # 'none' + return loss + + +class QFocalLoss(nn.Module): + # Wraps Quality focal loss around existing loss_fcn(), i.e. criteria = FocalLoss(nn.BCEWithLogitsLoss(), gamma=1.5) + def __init__(self, loss_fcn, gamma=1.5, alpha=0.25): + super().__init__() + self.loss_fcn = loss_fcn # must be nn.BCEWithLogitsLoss() + self.gamma = gamma + self.alpha = alpha + self.reduction = loss_fcn.reduction + self.loss_fcn.reduction = 'none' # required to apply FL to each element + + def forward(self, pred, true): + loss = self.loss_fcn(pred, true) + + pred_prob = torch.sigmoid(pred) # prob from logits + alpha_factor = true * self.alpha + (1 - true) * (1 - self.alpha) + modulating_factor = torch.abs(true - pred_prob) ** self.gamma + loss *= alpha_factor * modulating_factor + + if self.reduction == 'mean': + return loss.mean() + elif self.reduction == 'sum': + return loss.sum() + else: # 'none' + return loss + + +class ComputeLoss: + sort_obj_iou = False + + # Compute losses + def __init__(self, model, autobalance=False): + device = next(model.parameters()).device # get model device + h = model.hyp # hyperparameters + + # Define criteria + BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h['cls_pw']], device=device)) + BCEobj = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h['obj_pw']], device=device)) + + # Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3 + self.cp, self.cn = smooth_BCE(eps=h.get('label_smoothing', 0.0)) # positive, negative BCE targets + + # Focal loss + g = h['fl_gamma'] # focal loss gamma + if g > 0: + BCEcls, BCEobj = FocalLoss(BCEcls, g), FocalLoss(BCEobj, g) + + m = de_parallel(model).model[-1] # Detect() module + self.balance = {3: [4.0, 1.0, 0.4]}.get(m.nl, [4.0, 1.0, 0.25, 0.06, 0.02]) # P3-P7 + self.ssi = list(m.stride).index(16) if autobalance else 0 # stride 16 index + self.BCEcls, self.BCEobj, self.gr, self.hyp, self.autobalance = BCEcls, BCEobj, 1.0, h, autobalance + self.na = m.na # number of anchors + self.nc = m.nc # number of classes + self.nl = m.nl # number of layers + self.anchors = m.anchors + self.device = device + + def __call__(self, p, targets): # predictions, targets + lcls = torch.zeros(1, device=self.device) # class loss + lbox = torch.zeros(1, device=self.device) # box loss + lobj = torch.zeros(1, device=self.device) # object loss + tcls, tbox, indices, anchors = self.build_targets(p, targets) # targets + + # Losses + for i, pi in enumerate(p): # layer index, layer predictions + b, a, gj, gi = indices[i] # image, anchor, gridy, gridx + tobj = torch.zeros(pi.shape[:4], dtype=pi.dtype, device=self.device) # target obj + + n = b.shape[0] # number of targets + if n: + # pxy, pwh, _, pcls = pi[b, a, gj, gi].tensor_split((2, 4, 5), dim=1) # faster, requires torch 1.8.0 + pxy, pwh, _, pcls = pi[b, a, gj, gi].split((2, 2, 1, self.nc), 1) # target-subset of predictions + + # Regression + pxy = pxy.sigmoid() * 2 - 0.5 + pwh = (pwh.sigmoid() * 2) ** 2 * anchors[i] + pbox = torch.cat((pxy, pwh), 1) # predicted box + iou = bbox_iou(pbox, tbox[i], CIoU=True).squeeze() # iou(prediction, target) + lbox += (1.0 - iou).mean() # iou loss + + # Objectness + iou = iou.detach().clamp(0).type(tobj.dtype) + if self.sort_obj_iou: + j = iou.argsort() + b, a, gj, gi, iou = b[j], a[j], gj[j], gi[j], iou[j] + if self.gr < 1: + iou = (1.0 - self.gr) + self.gr * iou + tobj[b, a, gj, gi] = iou # iou ratio + + # Classification + if self.nc > 1: # cls loss (only if multiple classes) + t = torch.full_like(pcls, self.cn, device=self.device) # targets + t[range(n), tcls[i]] = self.cp + lcls += self.BCEcls(pcls, t) # BCE + + # Append targets to text file + # with open('targets.txt', 'a') as file: + # [file.write('%11.5g ' * 4 % tuple(x) + '\n') for x in torch.cat((txy[i], twh[i]), 1)] + + obji = self.BCEobj(pi[..., 4], tobj) + lobj += obji * self.balance[i] # obj loss + if self.autobalance: + self.balance[i] = self.balance[i] * 0.9999 + 0.0001 / obji.detach().item() + + if self.autobalance: + self.balance = [x / self.balance[self.ssi] for x in self.balance] + lbox *= self.hyp['box'] + lobj *= self.hyp['obj'] + lcls *= self.hyp['cls'] + bs = tobj.shape[0] # batch size + + return (lbox + lobj + lcls) * bs, torch.cat((lbox, lobj, lcls)).detach() + + def build_targets(self, p, targets): + # Build targets for compute_loss(), input targets(image,class,x,y,w,h) + na, nt = self.na, targets.shape[0] # number of anchors, targets + tcls, tbox, indices, anch = [], [], [], [] + gain = torch.ones(7, device=self.device) # normalized to gridspace gain + ai = torch.arange(na, device=self.device).float().view(na, 1).repeat(1, nt) # same as .repeat_interleave(nt) + targets = torch.cat((targets.repeat(na, 1, 1), ai[..., None]), 2) # append anchor indices + + g = 0.5 # bias + off = torch.tensor( + [ + [0, 0], + [1, 0], + [0, 1], + [-1, 0], + [0, -1], # j,k,l,m + # [1, 1], [1, -1], [-1, 1], [-1, -1], # jk,jm,lk,lm + ], + device=self.device).float() * g # offsets + + for i in range(self.nl): + anchors, shape = self.anchors[i], p[i].shape + gain[2:6] = torch.tensor(shape)[[3, 2, 3, 2]] # xyxy gain + + # Match targets to anchors + t = targets * gain # shape(3,n,7) + if nt: + # Matches + r = t[..., 4:6] / anchors[:, None] # wh ratio + j = torch.max(r, 1 / r).max(2)[0] < self.hyp['anchor_t'] # compare + # j = wh_iou(anchors, t[:, 4:6]) > model.hyp['iou_t'] # iou(3,n)=wh_iou(anchors(3,2), gwh(n,2)) + t = t[j] # filter + + # Offsets + gxy = t[:, 2:4] # grid xy + gxi = gain[[2, 3]] - gxy # inverse + j, k = ((gxy % 1 < g) & (gxy > 1)).T + l, m = ((gxi % 1 < g) & (gxi > 1)).T + j = torch.stack((torch.ones_like(j), j, k, l, m)) + t = t.repeat((5, 1, 1))[j] + offsets = (torch.zeros_like(gxy)[None] + off[:, None])[j] + else: + t = targets[0] + offsets = 0 + + # Define + bc, gxy, gwh, a = t.chunk(4, 1) # (image, class), grid xy, grid wh, anchors + a, (b, c) = a.long().view(-1), bc.long().T # anchors, image, class + gij = (gxy - offsets).long() + gi, gj = gij.T # grid indices + + # Append + indices.append((b, a, gj.clamp_(0, shape[2] - 1), gi.clamp_(0, shape[3] - 1))) # image, anchor, grid + tbox.append(torch.cat((gxy - gij, gwh), 1)) # box + anch.append(anchors[a]) # anchors + tcls.append(c) # class + + return tcls, tbox, indices, anch diff --git a/utils/metrics.py b/utils/metrics.py new file mode 100644 index 0000000..5646f40 --- /dev/null +++ b/utils/metrics.py @@ -0,0 +1,360 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Model validation metrics +""" + +import math +import warnings +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np +import torch + +from utils import TryExcept, threaded + + +def fitness(x): + # Model fitness as a weighted combination of metrics + w = [0.0, 0.0, 0.1, 0.9] # weights for [P, R, mAP@0.5, mAP@0.5:0.95] + return (x[:, :4] * w).sum(1) + + +def smooth(y, f=0.05): + # Box filter of fraction f + nf = round(len(y) * f * 2) // 2 + 1 # number of filter elements (must be odd) + p = np.ones(nf // 2) # ones padding + yp = np.concatenate((p * y[0], y, p * y[-1]), 0) # y padded + return np.convolve(yp, np.ones(nf) / nf, mode='valid') # y-smoothed + + +def ap_per_class(tp, conf, pred_cls, target_cls, plot=False, save_dir='.', names=(), eps=1e-16, prefix=''): + """ Compute the average precision, given the recall and precision curves. + Source: https://github.com/rafaelpadilla/Object-Detection-Metrics. + # Arguments + tp: True positives (nparray, nx1 or nx10). + conf: Objectness value from 0-1 (nparray). + pred_cls: Predicted object classes (nparray). + target_cls: True object classes (nparray). + plot: Plot precision-recall curve at mAP@0.5 + save_dir: Plot save directory + # Returns + The average precision as computed in py-faster-rcnn. + """ + + # Sort by objectness + i = np.argsort(-conf) + tp, conf, pred_cls = tp[i], conf[i], pred_cls[i] + + # Find unique classes + unique_classes, nt = np.unique(target_cls, return_counts=True) + nc = unique_classes.shape[0] # number of classes, number of detections + + # Create Precision-Recall curve and compute AP for each class + px, py = np.linspace(0, 1, 1000), [] # for plotting + ap, p, r = np.zeros((nc, tp.shape[1])), np.zeros((nc, 1000)), np.zeros((nc, 1000)) + for ci, c in enumerate(unique_classes): + i = pred_cls == c + n_l = nt[ci] # number of labels + n_p = i.sum() # number of predictions + if n_p == 0 or n_l == 0: + continue + + # Accumulate FPs and TPs + fpc = (1 - tp[i]).cumsum(0) + tpc = tp[i].cumsum(0) + + # Recall + recall = tpc / (n_l + eps) # recall curve + r[ci] = np.interp(-px, -conf[i], recall[:, 0], left=0) # negative x, xp because xp decreases + + # Precision + precision = tpc / (tpc + fpc) # precision curve + p[ci] = np.interp(-px, -conf[i], precision[:, 0], left=1) # p at pr_score + + # AP from recall-precision curve + for j in range(tp.shape[1]): + ap[ci, j], mpre, mrec = compute_ap(recall[:, j], precision[:, j]) + if plot and j == 0: + py.append(np.interp(px, mrec, mpre)) # precision at mAP@0.5 + + # Compute F1 (harmonic mean of precision and recall) + f1 = 2 * p * r / (p + r + eps) + names = [v for k, v in names.items() if k in unique_classes] # list: only classes that have data + names = dict(enumerate(names)) # to dict + if plot: + plot_pr_curve(px, py, ap, Path(save_dir) / f'{prefix}PR_curve.png', names) + plot_mc_curve(px, f1, Path(save_dir) / f'{prefix}F1_curve.png', names, ylabel='F1') + plot_mc_curve(px, p, Path(save_dir) / f'{prefix}P_curve.png', names, ylabel='Precision') + plot_mc_curve(px, r, Path(save_dir) / f'{prefix}R_curve.png', names, ylabel='Recall') + + i = smooth(f1.mean(0), 0.1).argmax() # max F1 index + p, r, f1 = p[:, i], r[:, i], f1[:, i] + tp = (r * nt).round() # true positives + fp = (tp / (p + eps) - tp).round() # false positives + return tp, fp, p, r, f1, ap, unique_classes.astype(int) + + +def compute_ap(recall, precision): + """ Compute the average precision, given the recall and precision curves + # Arguments + recall: The recall curve (list) + precision: The precision curve (list) + # Returns + Average precision, precision curve, recall curve + """ + + # Append sentinel values to beginning and end + mrec = np.concatenate(([0.0], recall, [1.0])) + mpre = np.concatenate(([1.0], precision, [0.0])) + + # Compute the precision envelope + mpre = np.flip(np.maximum.accumulate(np.flip(mpre))) + + # Integrate area under curve + method = 'interp' # methods: 'continuous', 'interp' + if method == 'interp': + x = np.linspace(0, 1, 101) # 101-point interp (COCO) + ap = np.trapz(np.interp(x, mrec, mpre), x) # integrate + else: # 'continuous' + i = np.where(mrec[1:] != mrec[:-1])[0] # points where x axis (recall) changes + ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) # area under curve + + return ap, mpre, mrec + + +class ConfusionMatrix: + # Updated version of https://github.com/kaanakan/object_detection_confusion_matrix + def __init__(self, nc, conf=0.25, iou_thres=0.45): + self.matrix = np.zeros((nc + 1, nc + 1)) + self.nc = nc # number of classes + self.conf = conf + self.iou_thres = iou_thres + + def process_batch(self, detections, labels): + """ + Return intersection-over-union (Jaccard index) of boxes. + Both sets of boxes are expected to be in (x1, y1, x2, y2) format. + Arguments: + detections (Array[N, 6]), x1, y1, x2, y2, conf, class + labels (Array[M, 5]), class, x1, y1, x2, y2 + Returns: + None, updates confusion matrix accordingly + """ + if detections is None: + gt_classes = labels.int() + for gc in gt_classes: + self.matrix[self.nc, gc] += 1 # background FN + return + + detections = detections[detections[:, 4] > self.conf] + gt_classes = labels[:, 0].int() + detection_classes = detections[:, 5].int() + iou = box_iou(labels[:, 1:], detections[:, :4]) + + x = torch.where(iou > self.iou_thres) + if x[0].shape[0]: + matches = torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1).cpu().numpy() + if x[0].shape[0] > 1: + matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 1], return_index=True)[1]] + matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 0], return_index=True)[1]] + else: + matches = np.zeros((0, 3)) + + n = matches.shape[0] > 0 + m0, m1, _ = matches.transpose().astype(int) + for i, gc in enumerate(gt_classes): + j = m0 == i + if n and sum(j) == 1: + self.matrix[detection_classes[m1[j]], gc] += 1 # correct + else: + self.matrix[self.nc, gc] += 1 # true background + + if n: + for i, dc in enumerate(detection_classes): + if not any(m1 == i): + self.matrix[dc, self.nc] += 1 # predicted background + + def tp_fp(self): + tp = self.matrix.diagonal() # true positives + fp = self.matrix.sum(1) - tp # false positives + # fn = self.matrix.sum(0) - tp # false negatives (missed detections) + return tp[:-1], fp[:-1] # remove background class + + @TryExcept('WARNING ⚠️ ConfusionMatrix plot failure') + def plot(self, normalize=True, save_dir='', names=()): + import seaborn as sn + + array = self.matrix / ((self.matrix.sum(0).reshape(1, -1) + 1E-9) if normalize else 1) # normalize columns + array[array < 0.005] = np.nan # don't annotate (would appear as 0.00) + + fig, ax = plt.subplots(1, 1, figsize=(12, 9), tight_layout=True) + nc, nn = self.nc, len(names) # number of classes, names + sn.set(font_scale=1.0 if nc < 50 else 0.8) # for label size + labels = (0 < nn < 99) and (nn == nc) # apply names to ticklabels + ticklabels = (names + ['background']) if labels else 'auto' + with warnings.catch_warnings(): + warnings.simplefilter('ignore') # suppress empty matrix RuntimeWarning: All-NaN slice encountered + sn.heatmap(array, + ax=ax, + annot=nc < 30, + annot_kws={ + 'size': 8}, + cmap='Blues', + fmt='.2f', + square=True, + vmin=0.0, + xticklabels=ticklabels, + yticklabels=ticklabels).set_facecolor((1, 1, 1)) + ax.set_xlabel('True') + ax.set_ylabel('Predicted') + ax.set_title('Confusion Matrix') + fig.savefig(Path(save_dir) / 'confusion_matrix.png', dpi=250) + plt.close(fig) + + def print(self): + for i in range(self.nc + 1): + print(' '.join(map(str, self.matrix[i]))) + + +def bbox_iou(box1, box2, xywh=True, GIoU=False, DIoU=False, CIoU=False, eps=1e-7): + # Returns Intersection over Union (IoU) of box1(1,4) to box2(n,4) + + # Get the coordinates of bounding boxes + if xywh: # transform from xywh to xyxy + (x1, y1, w1, h1), (x2, y2, w2, h2) = box1.chunk(4, -1), box2.chunk(4, -1) + w1_, h1_, w2_, h2_ = w1 / 2, h1 / 2, w2 / 2, h2 / 2 + b1_x1, b1_x2, b1_y1, b1_y2 = x1 - w1_, x1 + w1_, y1 - h1_, y1 + h1_ + b2_x1, b2_x2, b2_y1, b2_y2 = x2 - w2_, x2 + w2_, y2 - h2_, y2 + h2_ + else: # x1, y1, x2, y2 = box1 + b1_x1, b1_y1, b1_x2, b1_y2 = box1.chunk(4, -1) + b2_x1, b2_y1, b2_x2, b2_y2 = box2.chunk(4, -1) + w1, h1 = b1_x2 - b1_x1, (b1_y2 - b1_y1).clamp(eps) + w2, h2 = b2_x2 - b2_x1, (b2_y2 - b2_y1).clamp(eps) + + # Intersection area + inter = (b1_x2.minimum(b2_x2) - b1_x1.maximum(b2_x1)).clamp(0) * \ + (b1_y2.minimum(b2_y2) - b1_y1.maximum(b2_y1)).clamp(0) + + # Union Area + union = w1 * h1 + w2 * h2 - inter + eps + + # IoU + iou = inter / union + if CIoU or DIoU or GIoU: + cw = b1_x2.maximum(b2_x2) - b1_x1.minimum(b2_x1) # convex (smallest enclosing box) width + ch = b1_y2.maximum(b2_y2) - b1_y1.minimum(b2_y1) # convex height + if CIoU or DIoU: # Distance or Complete IoU https://arxiv.org/abs/1911.08287v1 + c2 = cw ** 2 + ch ** 2 + eps # convex diagonal squared + rho2 = ((b2_x1 + b2_x2 - b1_x1 - b1_x2) ** 2 + (b2_y1 + b2_y2 - b1_y1 - b1_y2) ** 2) / 4 # center dist ** 2 + if CIoU: # https://github.com/Zzh-tju/DIoU-SSD-pytorch/blob/master/utils/box/box_utils.py#L47 + v = (4 / math.pi ** 2) * (torch.atan(w2 / h2) - torch.atan(w1 / h1)).pow(2) + with torch.no_grad(): + alpha = v / (v - iou + (1 + eps)) + return iou - (rho2 / c2 + v * alpha) # CIoU + return iou - rho2 / c2 # DIoU + c_area = cw * ch + eps # convex area + return iou - (c_area - union) / c_area # GIoU https://arxiv.org/pdf/1902.09630.pdf + return iou # IoU + + +def box_iou(box1, box2, eps=1e-7): + # https://github.com/pytorch/vision/blob/master/torchvision/ops/boxes.py + """ + Return intersection-over-union (Jaccard index) of boxes. + Both sets of boxes are expected to be in (x1, y1, x2, y2) format. + Arguments: + box1 (Tensor[N, 4]) + box2 (Tensor[M, 4]) + Returns: + iou (Tensor[N, M]): the NxM matrix containing the pairwise + IoU values for every element in boxes1 and boxes2 + """ + + # inter(N,M) = (rb(N,M,2) - lt(N,M,2)).clamp(0).prod(2) + (a1, a2), (b1, b2) = box1.unsqueeze(1).chunk(2, 2), box2.unsqueeze(0).chunk(2, 2) + inter = (torch.min(a2, b2) - torch.max(a1, b1)).clamp(0).prod(2) + + # IoU = inter / (area1 + area2 - inter) + return inter / ((a2 - a1).prod(2) + (b2 - b1).prod(2) - inter + eps) + + +def bbox_ioa(box1, box2, eps=1e-7): + """ Returns the intersection over box2 area given box1, box2. Boxes are x1y1x2y2 + box1: np.array of shape(4) + box2: np.array of shape(nx4) + returns: np.array of shape(n) + """ + + # Get the coordinates of bounding boxes + b1_x1, b1_y1, b1_x2, b1_y2 = box1 + b2_x1, b2_y1, b2_x2, b2_y2 = box2.T + + # Intersection area + inter_area = (np.minimum(b1_x2, b2_x2) - np.maximum(b1_x1, b2_x1)).clip(0) * \ + (np.minimum(b1_y2, b2_y2) - np.maximum(b1_y1, b2_y1)).clip(0) + + # box2 area + box2_area = (b2_x2 - b2_x1) * (b2_y2 - b2_y1) + eps + + # Intersection over box2 area + return inter_area / box2_area + + +def wh_iou(wh1, wh2, eps=1e-7): + # Returns the nxm IoU matrix. wh1 is nx2, wh2 is mx2 + wh1 = wh1[:, None] # [N,1,2] + wh2 = wh2[None] # [1,M,2] + inter = torch.min(wh1, wh2).prod(2) # [N,M] + return inter / (wh1.prod(2) + wh2.prod(2) - inter + eps) # iou = inter / (area1 + area2 - inter) + + +# Plots ---------------------------------------------------------------------------------------------------------------- + + +@threaded +def plot_pr_curve(px, py, ap, save_dir=Path('pr_curve.png'), names=()): + # Precision-recall curve + fig, ax = plt.subplots(1, 1, figsize=(9, 6), tight_layout=True) + py = np.stack(py, axis=1) + + if 0 < len(names) < 21: # display per-class legend if < 21 classes + for i, y in enumerate(py.T): + ax.plot(px, y, linewidth=1, label=f'{names[i]} {ap[i, 0]:.3f}') # plot(recall, precision) + else: + ax.plot(px, py, linewidth=1, color='grey') # plot(recall, precision) + + ax.plot(px, py.mean(1), linewidth=3, color='blue', label='all classes %.3f mAP@0.5' % ap[:, 0].mean()) + ax.set_xlabel('Recall') + ax.set_ylabel('Precision') + ax.set_xlim(0, 1) + ax.set_ylim(0, 1) + ax.legend(bbox_to_anchor=(1.04, 1), loc='upper left') + ax.set_title('Precision-Recall Curve') + fig.savefig(save_dir, dpi=250) + plt.close(fig) + + +@threaded +def plot_mc_curve(px, py, save_dir=Path('mc_curve.png'), names=(), xlabel='Confidence', ylabel='Metric'): + # Metric-confidence curve + fig, ax = plt.subplots(1, 1, figsize=(9, 6), tight_layout=True) + + if 0 < len(names) < 21: # display per-class legend if < 21 classes + for i, y in enumerate(py): + ax.plot(px, y, linewidth=1, label=f'{names[i]}') # plot(confidence, metric) + else: + ax.plot(px, py.T, linewidth=1, color='grey') # plot(confidence, metric) + + y = smooth(py.mean(0), 0.05) + ax.plot(px, y, linewidth=3, color='blue', label=f'all classes {y.max():.2f} at {px[y.argmax()]:.3f}') + ax.set_xlabel(xlabel) + ax.set_ylabel(ylabel) + ax.set_xlim(0, 1) + ax.set_ylim(0, 1) + ax.legend(bbox_to_anchor=(1.04, 1), loc='upper left') + ax.set_title(f'{ylabel}-Confidence Curve') + fig.savefig(save_dir, dpi=250) + plt.close(fig) diff --git a/utils/plots.py b/utils/plots.py new file mode 100644 index 0000000..db6f94a --- /dev/null +++ b/utils/plots.py @@ -0,0 +1,446 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Plotting utils +""" + +import contextlib +import math +import os +from copy import copy +from pathlib import Path + +import cv2 +import matplotlib +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import seaborn as sn +import torch +from PIL import Image, ImageDraw +from scipy.ndimage.filters import gaussian_filter1d +from ultralytics.utils.plotting import Annotator + +from utils import TryExcept, threaded +from utils.general import LOGGER, clip_boxes, increment_path, xywh2xyxy, xyxy2xywh +from utils.metrics import fitness + +# Settings +RANK = int(os.getenv('RANK', -1)) +matplotlib.rc('font', **{'size': 11}) +matplotlib.use('Agg') # for writing to files only + + +class Colors: + # Ultralytics color palette https://ultralytics.com/ + def __init__(self): + # hex = matplotlib.colors.TABLEAU_COLORS.values() + hexs = ('FF3838', 'FF9D97', 'FF701F', 'FFB21D', 'CFD231', '48F90A', '92CC17', '3DDB86', '1A9334', '00D4BB', + '2C99A8', '00C2FF', '344593', '6473FF', '0018EC', '8438FF', '520085', 'CB38FF', 'FF95C8', 'FF37C7') + self.palette = [self.hex2rgb(f'#{c}') for c in hexs] + self.n = len(self.palette) + + def __call__(self, i, bgr=False): + c = self.palette[int(i) % self.n] + return (c[2], c[1], c[0]) if bgr else c + + @staticmethod + def hex2rgb(h): # rgb order (PIL) + return tuple(int(h[1 + i:1 + i + 2], 16) for i in (0, 2, 4)) + + +colors = Colors() # create instance for 'from utils.plots import colors' + + +def feature_visualization(x, module_type, stage, n=32, save_dir=Path('runs/detect/exp')): + """ + x: Features to be visualized + module_type: Module type + stage: Module stage within model + n: Maximum number of feature maps to plot + save_dir: Directory to save results + """ + if 'Detect' not in module_type: + batch, channels, height, width = x.shape # batch, channels, height, width + if height > 1 and width > 1: + f = save_dir / f"stage{stage}_{module_type.split('.')[-1]}_features.png" # filename + + blocks = torch.chunk(x[0].cpu(), channels, dim=0) # select batch index 0, block by channels + n = min(n, channels) # number of plots + fig, ax = plt.subplots(math.ceil(n / 8), 8, tight_layout=True) # 8 rows x n/8 cols + ax = ax.ravel() + plt.subplots_adjust(wspace=0.05, hspace=0.05) + for i in range(n): + ax[i].imshow(blocks[i].squeeze()) # cmap='gray' + ax[i].axis('off') + + LOGGER.info(f'Saving {f}... ({n}/{channels})') + plt.savefig(f, dpi=300, bbox_inches='tight') + plt.close() + np.save(str(f.with_suffix('.npy')), x[0].cpu().numpy()) # npy save + + +def hist2d(x, y, n=100): + # 2d histogram used in labels.png and evolve.png + xedges, yedges = np.linspace(x.min(), x.max(), n), np.linspace(y.min(), y.max(), n) + hist, xedges, yedges = np.histogram2d(x, y, (xedges, yedges)) + xidx = np.clip(np.digitize(x, xedges) - 1, 0, hist.shape[0] - 1) + yidx = np.clip(np.digitize(y, yedges) - 1, 0, hist.shape[1] - 1) + return np.log(hist[xidx, yidx]) + + +def butter_lowpass_filtfilt(data, cutoff=1500, fs=50000, order=5): + from scipy.signal import butter, filtfilt + + # https://stackoverflow.com/questions/28536191/how-to-filter-smooth-with-scipy-numpy + def butter_lowpass(cutoff, fs, order): + nyq = 0.5 * fs + normal_cutoff = cutoff / nyq + return butter(order, normal_cutoff, btype='low', analog=False) + + b, a = butter_lowpass(cutoff, fs, order=order) + return filtfilt(b, a, data) # forward-backward filter + + +def output_to_target(output, max_det=300): + # Convert model output to target format [batch_id, class_id, x, y, w, h, conf] for plotting + targets = [] + for i, o in enumerate(output): + box, conf, cls = o[:max_det, :6].cpu().split((4, 1, 1), 1) + j = torch.full((conf.shape[0], 1), i) + targets.append(torch.cat((j, cls, xyxy2xywh(box), conf), 1)) + return torch.cat(targets, 0).numpy() + + +@threaded +def plot_images(images, targets, paths=None, fname='images.jpg', names=None): + # Plot image grid with labels + if isinstance(images, torch.Tensor): + images = images.cpu().float().numpy() + if isinstance(targets, torch.Tensor): + targets = targets.cpu().numpy() + + max_size = 1920 # max image size + max_subplots = 16 # max image subplots, i.e. 4x4 + bs, _, h, w = images.shape # batch size, _, height, width + bs = min(bs, max_subplots) # limit plot images + ns = np.ceil(bs ** 0.5) # number of subplots (square) + if np.max(images[0]) <= 1: + images *= 255 # de-normalise (optional) + + # Build Image + mosaic = np.full((int(ns * h), int(ns * w), 3), 255, dtype=np.uint8) # init + for i, im in enumerate(images): + if i == max_subplots: # if last batch has fewer images than we expect + break + x, y = int(w * (i // ns)), int(h * (i % ns)) # block origin + im = im.transpose(1, 2, 0) + mosaic[y:y + h, x:x + w, :] = im + + # Resize (optional) + scale = max_size / ns / max(h, w) + if scale < 1: + h = math.ceil(scale * h) + w = math.ceil(scale * w) + mosaic = cv2.resize(mosaic, tuple(int(x * ns) for x in (w, h))) + + # Annotate + fs = int((h + w) * ns * 0.01) # font size + annotator = Annotator(mosaic, line_width=round(fs / 10), font_size=fs, pil=True, example=names) + for i in range(i + 1): + x, y = int(w * (i // ns)), int(h * (i % ns)) # block origin + annotator.rectangle([x, y, x + w, y + h], None, (255, 255, 255), width=2) # borders + if paths: + annotator.text([x + 5, y + 5], text=Path(paths[i]).name[:40], txt_color=(220, 220, 220)) # filenames + if len(targets) > 0: + ti = targets[targets[:, 0] == i] # image targets + boxes = xywh2xyxy(ti[:, 2:6]).T + classes = ti[:, 1].astype('int') + labels = ti.shape[1] == 6 # labels if no conf column + conf = None if labels else ti[:, 6] # check for confidence presence (label vs pred) + + if boxes.shape[1]: + if boxes.max() <= 1.01: # if normalized with tolerance 0.01 + boxes[[0, 2]] *= w # scale to pixels + boxes[[1, 3]] *= h + elif scale < 1: # absolute coords need scale if image scales + boxes *= scale + boxes[[0, 2]] += x + boxes[[1, 3]] += y + for j, box in enumerate(boxes.T.tolist()): + cls = classes[j] + color = colors(cls) + cls = names[cls] if names else cls + if labels or conf[j] > 0.25: # 0.25 conf thresh + label = f'{cls}' if labels else f'{cls} {conf[j]:.1f}' + annotator.box_label(box, label, color=color) + annotator.im.save(fname) # save + + +def plot_lr_scheduler(optimizer, scheduler, epochs=300, save_dir=''): + # Plot LR simulating training for full epochs + optimizer, scheduler = copy(optimizer), copy(scheduler) # do not modify originals + y = [] + for _ in range(epochs): + scheduler.step() + y.append(optimizer.param_groups[0]['lr']) + plt.plot(y, '.-', label='LR') + plt.xlabel('epoch') + plt.ylabel('LR') + plt.grid() + plt.xlim(0, epochs) + plt.ylim(0) + plt.savefig(Path(save_dir) / 'LR.png', dpi=200) + plt.close() + + +def plot_val_txt(): # from utils.plots import *; plot_val() + # Plot val.txt histograms + x = np.loadtxt('val.txt', dtype=np.float32) + box = xyxy2xywh(x[:, :4]) + cx, cy = box[:, 0], box[:, 1] + + fig, ax = plt.subplots(1, 1, figsize=(6, 6), tight_layout=True) + ax.hist2d(cx, cy, bins=600, cmax=10, cmin=0) + ax.set_aspect('equal') + plt.savefig('hist2d.png', dpi=300) + + fig, ax = plt.subplots(1, 2, figsize=(12, 6), tight_layout=True) + ax[0].hist(cx, bins=600) + ax[1].hist(cy, bins=600) + plt.savefig('hist1d.png', dpi=200) + + +def plot_targets_txt(): # from utils.plots import *; plot_targets_txt() + # Plot targets.txt histograms + x = np.loadtxt('targets.txt', dtype=np.float32).T + s = ['x targets', 'y targets', 'width targets', 'height targets'] + fig, ax = plt.subplots(2, 2, figsize=(8, 8), tight_layout=True) + ax = ax.ravel() + for i in range(4): + ax[i].hist(x[i], bins=100, label=f'{x[i].mean():.3g} +/- {x[i].std():.3g}') + ax[i].legend() + ax[i].set_title(s[i]) + plt.savefig('targets.jpg', dpi=200) + + +def plot_val_study(file='', dir='', x=None): # from utils.plots import *; plot_val_study() + # Plot file=study.txt generated by val.py (or plot all study*.txt in dir) + save_dir = Path(file).parent if file else Path(dir) + plot2 = False # plot additional results + if plot2: + ax = plt.subplots(2, 4, figsize=(10, 6), tight_layout=True)[1].ravel() + + fig2, ax2 = plt.subplots(1, 1, figsize=(8, 4), tight_layout=True) + # for f in [save_dir / f'study_coco_{x}.txt' for x in ['yolov5n6', 'yolov5s6', 'yolov5m6', 'yolov5l6', 'yolov5x6']]: + for f in sorted(save_dir.glob('study*.txt')): + y = np.loadtxt(f, dtype=np.float32, usecols=[0, 1, 2, 3, 7, 8, 9], ndmin=2).T + x = np.arange(y.shape[1]) if x is None else np.array(x) + if plot2: + s = ['P', 'R', 'mAP@.5', 'mAP@.5:.95', 't_preprocess (ms/img)', 't_inference (ms/img)', 't_NMS (ms/img)'] + for i in range(7): + ax[i].plot(x, y[i], '.-', linewidth=2, markersize=8) + ax[i].set_title(s[i]) + + j = y[3].argmax() + 1 + ax2.plot(y[5, 1:j], + y[3, 1:j] * 1E2, + '.-', + linewidth=2, + markersize=8, + label=f.stem.replace('study_coco_', '').replace('yolo', 'YOLO')) + + ax2.plot(1E3 / np.array([209, 140, 97, 58, 35, 18]), [34.6, 40.5, 43.0, 47.5, 49.7, 51.5], + 'k.-', + linewidth=2, + markersize=8, + alpha=.25, + label='EfficientDet') + + ax2.grid(alpha=0.2) + ax2.set_yticks(np.arange(20, 60, 5)) + ax2.set_xlim(0, 57) + ax2.set_ylim(25, 55) + ax2.set_xlabel('GPU Speed (ms/img)') + ax2.set_ylabel('COCO AP val') + ax2.legend(loc='lower right') + f = save_dir / 'study.png' + print(f'Saving {f}...') + plt.savefig(f, dpi=300) + + +@TryExcept() # known issue https://github.com/ultralytics/yolov5/issues/5395 +def plot_labels(labels, names=(), save_dir=Path('')): + # plot dataset labels + LOGGER.info(f"Plotting labels to {save_dir / 'labels.jpg'}... ") + c, b = labels[:, 0], labels[:, 1:].transpose() # classes, boxes + nc = int(c.max() + 1) # number of classes + x = pd.DataFrame(b.transpose(), columns=['x', 'y', 'width', 'height']) + + # seaborn correlogram + sn.pairplot(x, corner=True, diag_kind='auto', kind='hist', diag_kws=dict(bins=50), plot_kws=dict(pmax=0.9)) + plt.savefig(save_dir / 'labels_correlogram.jpg', dpi=200) + plt.close() + + # matplotlib labels + matplotlib.use('svg') # faster + ax = plt.subplots(2, 2, figsize=(8, 8), tight_layout=True)[1].ravel() + y = ax[0].hist(c, bins=np.linspace(0, nc, nc + 1) - 0.5, rwidth=0.8) + with contextlib.suppress(Exception): # color histogram bars by class + [y[2].patches[i].set_color([x / 255 for x in colors(i)]) for i in range(nc)] # known issue #3195 + ax[0].set_ylabel('instances') + if 0 < len(names) < 30: + ax[0].set_xticks(range(len(names))) + ax[0].set_xticklabels(list(names.values()), rotation=90, fontsize=10) + else: + ax[0].set_xlabel('classes') + sn.histplot(x, x='x', y='y', ax=ax[2], bins=50, pmax=0.9) + sn.histplot(x, x='width', y='height', ax=ax[3], bins=50, pmax=0.9) + + # rectangles + labels[:, 1:3] = 0.5 # center + labels[:, 1:] = xywh2xyxy(labels[:, 1:]) * 2000 + img = Image.fromarray(np.ones((2000, 2000, 3), dtype=np.uint8) * 255) + for cls, *box in labels[:1000]: + ImageDraw.Draw(img).rectangle(box, width=1, outline=colors(cls)) # plot + ax[1].imshow(img) + ax[1].axis('off') + + for a in [0, 1, 2, 3]: + for s in ['top', 'right', 'left', 'bottom']: + ax[a].spines[s].set_visible(False) + + plt.savefig(save_dir / 'labels.jpg', dpi=200) + matplotlib.use('Agg') + plt.close() + + +def imshow_cls(im, labels=None, pred=None, names=None, nmax=25, verbose=False, f=Path('images.jpg')): + # Show classification image grid with labels (optional) and predictions (optional) + from utils.augmentations import denormalize + + names = names or [f'class{i}' for i in range(1000)] + blocks = torch.chunk(denormalize(im.clone()).cpu().float(), len(im), + dim=0) # select batch index 0, block by channels + n = min(len(blocks), nmax) # number of plots + m = min(8, round(n ** 0.5)) # 8 x 8 default + fig, ax = plt.subplots(math.ceil(n / m), m) # 8 rows x n/8 cols + ax = ax.ravel() if m > 1 else [ax] + # plt.subplots_adjust(wspace=0.05, hspace=0.05) + for i in range(n): + ax[i].imshow(blocks[i].squeeze().permute((1, 2, 0)).numpy().clip(0.0, 1.0)) + ax[i].axis('off') + if labels is not None: + s = names[labels[i]] + (f'—{names[pred[i]]}' if pred is not None else '') + ax[i].set_title(s, fontsize=8, verticalalignment='top') + plt.savefig(f, dpi=300, bbox_inches='tight') + plt.close() + if verbose: + LOGGER.info(f'Saving {f}') + if labels is not None: + LOGGER.info('True: ' + ' '.join(f'{names[i]:3s}' for i in labels[:nmax])) + if pred is not None: + LOGGER.info('Predicted:' + ' '.join(f'{names[i]:3s}' for i in pred[:nmax])) + return f + + +def plot_evolve(evolve_csv='path/to/evolve.csv'): # from utils.plots import *; plot_evolve() + # Plot evolve.csv hyp evolution results + evolve_csv = Path(evolve_csv) + data = pd.read_csv(evolve_csv) + keys = [x.strip() for x in data.columns] + x = data.values + f = fitness(x) + j = np.argmax(f) # max fitness index + plt.figure(figsize=(10, 12), tight_layout=True) + matplotlib.rc('font', **{'size': 8}) + print(f'Best results from row {j} of {evolve_csv}:') + for i, k in enumerate(keys[7:]): + v = x[:, 7 + i] + mu = v[j] # best single result + plt.subplot(6, 5, i + 1) + plt.scatter(v, f, c=hist2d(v, f, 20), cmap='viridis', alpha=.8, edgecolors='none') + plt.plot(mu, f.max(), 'k+', markersize=15) + plt.title(f'{k} = {mu:.3g}', fontdict={'size': 9}) # limit to 40 characters + if i % 5 != 0: + plt.yticks([]) + print(f'{k:>15}: {mu:.3g}') + f = evolve_csv.with_suffix('.png') # filename + plt.savefig(f, dpi=200) + plt.close() + print(f'Saved {f}') + + +def plot_results(file='path/to/results.csv', dir=''): + # Plot training results.csv. Usage: from utils.plots import *; plot_results('path/to/results.csv') + save_dir = Path(file).parent if file else Path(dir) + fig, ax = plt.subplots(2, 5, figsize=(12, 6), tight_layout=True) + ax = ax.ravel() + files = list(save_dir.glob('results*.csv')) + assert len(files), f'No results.csv files found in {save_dir.resolve()}, nothing to plot.' + for f in files: + try: + data = pd.read_csv(f) + s = [x.strip() for x in data.columns] + x = data.values[:, 0] + for i, j in enumerate([1, 2, 3, 4, 5, 8, 9, 10, 6, 7]): + y = data.values[:, j].astype('float') + # y[y == 0] = np.nan # don't show zero values + ax[i].plot(x, y, marker='.', label=f.stem, linewidth=2, markersize=8) # actual results + ax[i].plot(x, gaussian_filter1d(y, sigma=3), ':', label='smooth', linewidth=2) # smoothing line + ax[i].set_title(s[j], fontsize=12) + # if j in [8, 9, 10]: # share train and val loss y axes + # ax[i].get_shared_y_axes().join(ax[i], ax[i - 5]) + except Exception as e: + LOGGER.info(f'Warning: Plotting error for {f}: {e}') + ax[1].legend() + fig.savefig(save_dir / 'results.png', dpi=200) + plt.close() + + +def profile_idetection(start=0, stop=0, labels=(), save_dir=''): + # Plot iDetection '*.txt' per-image logs. from utils.plots import *; profile_idetection() + ax = plt.subplots(2, 4, figsize=(12, 6), tight_layout=True)[1].ravel() + s = ['Images', 'Free Storage (GB)', 'RAM Usage (GB)', 'Battery', 'dt_raw (ms)', 'dt_smooth (ms)', 'real-world FPS'] + files = list(Path(save_dir).glob('frames*.txt')) + for fi, f in enumerate(files): + try: + results = np.loadtxt(f, ndmin=2).T[:, 90:-30] # clip first and last rows + n = results.shape[1] # number of rows + x = np.arange(start, min(stop, n) if stop else n) + results = results[:, x] + t = (results[0] - results[0].min()) # set t0=0s + results[0] = x + for i, a in enumerate(ax): + if i < len(results): + label = labels[fi] if len(labels) else f.stem.replace('frames_', '') + a.plot(t, results[i], marker='.', label=label, linewidth=1, markersize=5) + a.set_title(s[i]) + a.set_xlabel('time (s)') + # if fi == len(files) - 1: + # a.set_ylim(bottom=0) + for side in ['top', 'right']: + a.spines[side].set_visible(False) + else: + a.remove() + except Exception as e: + print(f'Warning: Plotting error for {f}; {e}') + ax[1].legend() + plt.savefig(Path(save_dir) / 'idetection_profile.png', dpi=200) + + +def save_one_box(xyxy, im, file=Path('im.jpg'), gain=1.02, pad=10, square=False, BGR=False, save=True): + # Save image crop as {file} with crop size multiple {gain} and {pad} pixels. Save and/or return crop + xyxy = torch.tensor(xyxy).view(-1, 4) + b = xyxy2xywh(xyxy) # boxes + if square: + b[:, 2:] = b[:, 2:].max(1)[0].unsqueeze(1) # attempt rectangle to square + b[:, 2:] = b[:, 2:] * gain + pad # box wh * gain + pad + xyxy = xywh2xyxy(b).long() + clip_boxes(xyxy, im.shape) + crop = im[int(xyxy[0, 1]):int(xyxy[0, 3]), int(xyxy[0, 0]):int(xyxy[0, 2]), ::(1 if BGR else -1)] + if save: + file.parent.mkdir(parents=True, exist_ok=True) # make directory + f = str(increment_path(file).with_suffix('.jpg')) + # cv2.imwrite(f, crop) # save BGR, https://github.com/ultralytics/yolov5/issues/7007 chroma subsampling issue + Image.fromarray(crop[..., ::-1]).save(f, quality=95, subsampling=0) # save RGB + return crop diff --git a/utils/segment/__init__.py b/utils/segment/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/utils/segment/augmentations.py b/utils/segment/augmentations.py new file mode 100644 index 0000000..f8154b8 --- /dev/null +++ b/utils/segment/augmentations.py @@ -0,0 +1,104 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Image augmentation functions +""" + +import math +import random + +import cv2 +import numpy as np + +from ..augmentations import box_candidates +from ..general import resample_segments, segment2box + + +def mixup(im, labels, segments, im2, labels2, segments2): + # Applies MixUp augmentation https://arxiv.org/pdf/1710.09412.pdf + r = np.random.beta(32.0, 32.0) # mixup ratio, alpha=beta=32.0 + im = (im * r + im2 * (1 - r)).astype(np.uint8) + labels = np.concatenate((labels, labels2), 0) + segments = np.concatenate((segments, segments2), 0) + return im, labels, segments + + +def random_perspective(im, + targets=(), + segments=(), + degrees=10, + translate=.1, + scale=.1, + shear=10, + perspective=0.0, + border=(0, 0)): + # torchvision.transforms.RandomAffine(degrees=(-10, 10), translate=(.1, .1), scale=(.9, 1.1), shear=(-10, 10)) + # targets = [cls, xyxy] + + height = im.shape[0] + border[0] * 2 # shape(h,w,c) + width = im.shape[1] + border[1] * 2 + + # Center + C = np.eye(3) + C[0, 2] = -im.shape[1] / 2 # x translation (pixels) + C[1, 2] = -im.shape[0] / 2 # y translation (pixels) + + # Perspective + P = np.eye(3) + P[2, 0] = random.uniform(-perspective, perspective) # x perspective (about y) + P[2, 1] = random.uniform(-perspective, perspective) # y perspective (about x) + + # Rotation and Scale + R = np.eye(3) + a = random.uniform(-degrees, degrees) + # a += random.choice([-180, -90, 0, 90]) # add 90deg rotations to small rotations + s = random.uniform(1 - scale, 1 + scale) + # s = 2 ** random.uniform(-scale, scale) + R[:2] = cv2.getRotationMatrix2D(angle=a, center=(0, 0), scale=s) + + # Shear + S = np.eye(3) + S[0, 1] = math.tan(random.uniform(-shear, shear) * math.pi / 180) # x shear (deg) + S[1, 0] = math.tan(random.uniform(-shear, shear) * math.pi / 180) # y shear (deg) + + # Translation + T = np.eye(3) + T[0, 2] = (random.uniform(0.5 - translate, 0.5 + translate) * width) # x translation (pixels) + T[1, 2] = (random.uniform(0.5 - translate, 0.5 + translate) * height) # y translation (pixels) + + # Combined rotation matrix + M = T @ S @ R @ P @ C # order of operations (right to left) is IMPORTANT + if (border[0] != 0) or (border[1] != 0) or (M != np.eye(3)).any(): # image changed + if perspective: + im = cv2.warpPerspective(im, M, dsize=(width, height), borderValue=(114, 114, 114)) + else: # affine + im = cv2.warpAffine(im, M[:2], dsize=(width, height), borderValue=(114, 114, 114)) + + # Visualize + # import matplotlib.pyplot as plt + # ax = plt.subplots(1, 2, figsize=(12, 6))[1].ravel() + # ax[0].imshow(im[:, :, ::-1]) # base + # ax[1].imshow(im2[:, :, ::-1]) # warped + + # Transform label coordinates + n = len(targets) + new_segments = [] + if n: + new = np.zeros((n, 4)) + segments = resample_segments(segments) # upsample + for i, segment in enumerate(segments): + xy = np.ones((len(segment), 3)) + xy[:, :2] = segment + xy = xy @ M.T # transform + xy = (xy[:, :2] / xy[:, 2:3] if perspective else xy[:, :2]) # perspective rescale or affine + + # clip + new[i] = segment2box(xy, width, height) + new_segments.append(xy) + + # filter candidates + i = box_candidates(box1=targets[:, 1:5].T * s, box2=new.T, area_thr=0.01) + targets = targets[i] + targets[:, 1:5] = new[i] + new_segments = np.array(new_segments)[i] + + return im, targets, new_segments diff --git a/utils/segment/dataloaders.py b/utils/segment/dataloaders.py new file mode 100644 index 0000000..3ee826d --- /dev/null +++ b/utils/segment/dataloaders.py @@ -0,0 +1,332 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Dataloaders +""" + +import os +import random + +import cv2 +import numpy as np +import torch +from torch.utils.data import DataLoader, distributed + +from ..augmentations import augment_hsv, copy_paste, letterbox +from ..dataloaders import InfiniteDataLoader, LoadImagesAndLabels, seed_worker +from ..general import LOGGER, xyn2xy, xywhn2xyxy, xyxy2xywhn +from ..torch_utils import torch_distributed_zero_first +from .augmentations import mixup, random_perspective + +RANK = int(os.getenv('RANK', -1)) + + +def create_dataloader(path, + imgsz, + batch_size, + stride, + single_cls=False, + hyp=None, + augment=False, + cache=False, + pad=0.0, + rect=False, + rank=-1, + workers=8, + image_weights=False, + quad=False, + prefix='', + shuffle=False, + mask_downsample_ratio=1, + overlap_mask=False, + seed=0): + if rect and shuffle: + LOGGER.warning('WARNING ⚠️ --rect is incompatible with DataLoader shuffle, setting shuffle=False') + shuffle = False + with torch_distributed_zero_first(rank): # init dataset *.cache only once if DDP + dataset = LoadImagesAndLabelsAndMasks( + path, + imgsz, + batch_size, + augment=augment, # augmentation + hyp=hyp, # hyperparameters + rect=rect, # rectangular batches + cache_images=cache, + single_cls=single_cls, + stride=int(stride), + pad=pad, + image_weights=image_weights, + prefix=prefix, + downsample_ratio=mask_downsample_ratio, + overlap=overlap_mask) + + batch_size = min(batch_size, len(dataset)) + nd = torch.cuda.device_count() # number of CUDA devices + nw = min([os.cpu_count() // max(nd, 1), batch_size if batch_size > 1 else 0, workers]) # number of workers + sampler = None if rank == -1 else distributed.DistributedSampler(dataset, shuffle=shuffle) + loader = DataLoader if image_weights else InfiniteDataLoader # only DataLoader allows for attribute updates + generator = torch.Generator() + generator.manual_seed(6148914691236517205 + seed + RANK) + return loader( + dataset, + batch_size=batch_size, + shuffle=shuffle and sampler is None, + num_workers=nw, + sampler=sampler, + pin_memory=True, + collate_fn=LoadImagesAndLabelsAndMasks.collate_fn4 if quad else LoadImagesAndLabelsAndMasks.collate_fn, + worker_init_fn=seed_worker, + generator=generator, + ), dataset + + +class LoadImagesAndLabelsAndMasks(LoadImagesAndLabels): # for training/testing + + def __init__( + self, + path, + img_size=640, + batch_size=16, + augment=False, + hyp=None, + rect=False, + image_weights=False, + cache_images=False, + single_cls=False, + stride=32, + pad=0, + min_items=0, + prefix='', + downsample_ratio=1, + overlap=False, + ): + super().__init__(path, img_size, batch_size, augment, hyp, rect, image_weights, cache_images, single_cls, + stride, pad, min_items, prefix) + self.downsample_ratio = downsample_ratio + self.overlap = overlap + + def __getitem__(self, index): + index = self.indices[index] # linear, shuffled, or image_weights + + hyp = self.hyp + mosaic = self.mosaic and random.random() < hyp['mosaic'] + masks = [] + if mosaic: + # Load mosaic + img, labels, segments = self.load_mosaic(index) + shapes = None + + # MixUp augmentation + if random.random() < hyp['mixup']: + img, labels, segments = mixup(img, labels, segments, *self.load_mosaic(random.randint(0, self.n - 1))) + + else: + # Load image + img, (h0, w0), (h, w) = self.load_image(index) + + # Letterbox + shape = self.batch_shapes[self.batch[index]] if self.rect else self.img_size # final letterboxed shape + img, ratio, pad = letterbox(img, shape, auto=False, scaleup=self.augment) + shapes = (h0, w0), ((h / h0, w / w0), pad) # for COCO mAP rescaling + + labels = self.labels[index].copy() + # [array, array, ....], array.shape=(num_points, 2), xyxyxyxy + segments = self.segments[index].copy() + if len(segments): + for i_s in range(len(segments)): + segments[i_s] = xyn2xy( + segments[i_s], + ratio[0] * w, + ratio[1] * h, + padw=pad[0], + padh=pad[1], + ) + if labels.size: # normalized xywh to pixel xyxy format + labels[:, 1:] = xywhn2xyxy(labels[:, 1:], ratio[0] * w, ratio[1] * h, padw=pad[0], padh=pad[1]) + + if self.augment: + img, labels, segments = random_perspective(img, + labels, + segments=segments, + degrees=hyp['degrees'], + translate=hyp['translate'], + scale=hyp['scale'], + shear=hyp['shear'], + perspective=hyp['perspective']) + + nl = len(labels) # number of labels + if nl: + labels[:, 1:5] = xyxy2xywhn(labels[:, 1:5], w=img.shape[1], h=img.shape[0], clip=True, eps=1e-3) + if self.overlap: + masks, sorted_idx = polygons2masks_overlap(img.shape[:2], + segments, + downsample_ratio=self.downsample_ratio) + masks = masks[None] # (640, 640) -> (1, 640, 640) + labels = labels[sorted_idx] + else: + masks = polygons2masks(img.shape[:2], segments, color=1, downsample_ratio=self.downsample_ratio) + + masks = (torch.from_numpy(masks) if len(masks) else torch.zeros(1 if self.overlap else nl, img.shape[0] // + self.downsample_ratio, img.shape[1] // + self.downsample_ratio)) + # TODO: albumentations support + if self.augment: + # Albumentations + # there are some augmentation that won't change boxes and masks, + # so just be it for now. + img, labels = self.albumentations(img, labels) + nl = len(labels) # update after albumentations + + # HSV color-space + augment_hsv(img, hgain=hyp['hsv_h'], sgain=hyp['hsv_s'], vgain=hyp['hsv_v']) + + # Flip up-down + if random.random() < hyp['flipud']: + img = np.flipud(img) + if nl: + labels[:, 2] = 1 - labels[:, 2] + masks = torch.flip(masks, dims=[1]) + + # Flip left-right + if random.random() < hyp['fliplr']: + img = np.fliplr(img) + if nl: + labels[:, 1] = 1 - labels[:, 1] + masks = torch.flip(masks, dims=[2]) + + # Cutouts # labels = cutout(img, labels, p=0.5) + + labels_out = torch.zeros((nl, 6)) + if nl: + labels_out[:, 1:] = torch.from_numpy(labels) + + # Convert + img = img.transpose((2, 0, 1))[::-1] # HWC to CHW, BGR to RGB + img = np.ascontiguousarray(img) + + return (torch.from_numpy(img), labels_out, self.im_files[index], shapes, masks) + + def load_mosaic(self, index): + # YOLOv5 4-mosaic loader. Loads 1 image + 3 random images into a 4-image mosaic + labels4, segments4 = [], [] + s = self.img_size + yc, xc = (int(random.uniform(-x, 2 * s + x)) for x in self.mosaic_border) # mosaic center x, y + + # 3 additional image indices + indices = [index] + random.choices(self.indices, k=3) # 3 additional image indices + for i, index in enumerate(indices): + # Load image + img, _, (h, w) = self.load_image(index) + + # place img in img4 + if i == 0: # top left + img4 = np.full((s * 2, s * 2, img.shape[2]), 114, dtype=np.uint8) # base image with 4 tiles + x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc # xmin, ymin, xmax, ymax (large image) + x1b, y1b, x2b, y2b = w - (x2a - x1a), h - (y2a - y1a), w, h # xmin, ymin, xmax, ymax (small image) + elif i == 1: # top right + x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc + x1b, y1b, x2b, y2b = 0, h - (y2a - y1a), min(w, x2a - x1a), h + elif i == 2: # bottom left + x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h) + x1b, y1b, x2b, y2b = w - (x2a - x1a), 0, w, min(y2a - y1a, h) + elif i == 3: # bottom right + x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h) + x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h) + + img4[y1a:y2a, x1a:x2a] = img[y1b:y2b, x1b:x2b] # img4[ymin:ymax, xmin:xmax] + padw = x1a - x1b + padh = y1a - y1b + + labels, segments = self.labels[index].copy(), self.segments[index].copy() + + if labels.size: + labels[:, 1:] = xywhn2xyxy(labels[:, 1:], w, h, padw, padh) # normalized xywh to pixel xyxy format + segments = [xyn2xy(x, w, h, padw, padh) for x in segments] + labels4.append(labels) + segments4.extend(segments) + + # Concat/clip labels + labels4 = np.concatenate(labels4, 0) + for x in (labels4[:, 1:], *segments4): + np.clip(x, 0, 2 * s, out=x) # clip when using random_perspective() + # img4, labels4 = replicate(img4, labels4) # replicate + + # Augment + img4, labels4, segments4 = copy_paste(img4, labels4, segments4, p=self.hyp['copy_paste']) + img4, labels4, segments4 = random_perspective(img4, + labels4, + segments4, + degrees=self.hyp['degrees'], + translate=self.hyp['translate'], + scale=self.hyp['scale'], + shear=self.hyp['shear'], + perspective=self.hyp['perspective'], + border=self.mosaic_border) # border to remove + return img4, labels4, segments4 + + @staticmethod + def collate_fn(batch): + img, label, path, shapes, masks = zip(*batch) # transposed + batched_masks = torch.cat(masks, 0) + for i, l in enumerate(label): + l[:, 0] = i # add target image index for build_targets() + return torch.stack(img, 0), torch.cat(label, 0), path, shapes, batched_masks + + +def polygon2mask(img_size, polygons, color=1, downsample_ratio=1): + """ + Args: + img_size (tuple): The image size. + polygons (np.ndarray): [N, M], N is the number of polygons, + M is the number of points(Be divided by 2). + """ + mask = np.zeros(img_size, dtype=np.uint8) + polygons = np.asarray(polygons) + polygons = polygons.astype(np.int32) + shape = polygons.shape + polygons = polygons.reshape(shape[0], -1, 2) + cv2.fillPoly(mask, polygons, color=color) + nh, nw = (img_size[0] // downsample_ratio, img_size[1] // downsample_ratio) + # NOTE: fillPoly firstly then resize is trying the keep the same way + # of loss calculation when mask-ratio=1. + mask = cv2.resize(mask, (nw, nh)) + return mask + + +def polygons2masks(img_size, polygons, color, downsample_ratio=1): + """ + Args: + img_size (tuple): The image size. + polygons (list[np.ndarray]): each polygon is [N, M], + N is the number of polygons, + M is the number of points(Be divided by 2). + """ + masks = [] + for si in range(len(polygons)): + mask = polygon2mask(img_size, [polygons[si].reshape(-1)], color, downsample_ratio) + masks.append(mask) + return np.array(masks) + + +def polygons2masks_overlap(img_size, segments, downsample_ratio=1): + """Return a (640, 640) overlap mask.""" + masks = np.zeros((img_size[0] // downsample_ratio, img_size[1] // downsample_ratio), + dtype=np.int32 if len(segments) > 255 else np.uint8) + areas = [] + ms = [] + for si in range(len(segments)): + mask = polygon2mask( + img_size, + [segments[si].reshape(-1)], + downsample_ratio=downsample_ratio, + color=1, + ) + ms.append(mask) + areas.append(mask.sum()) + areas = np.asarray(areas) + index = np.argsort(-areas) + ms = np.array(ms)[index] + for i in range(len(segments)): + mask = ms[i] * (i + 1) + masks = masks + mask + masks = np.clip(masks, a_min=0, a_max=i + 1) + return masks, index diff --git a/utils/segment/general.py b/utils/segment/general.py new file mode 100644 index 0000000..f1b2f1d --- /dev/null +++ b/utils/segment/general.py @@ -0,0 +1,160 @@ +import cv2 +import numpy as np +import torch +import torch.nn.functional as F + + +def crop_mask(masks, boxes): + """ + "Crop" predicted masks by zeroing out everything not in the predicted bbox. + Vectorized by Chong (thanks Chong). + + Args: + - masks should be a size [n, h, w] tensor of masks + - boxes should be a size [n, 4] tensor of bbox coords in relative point form + """ + + n, h, w = masks.shape + x1, y1, x2, y2 = torch.chunk(boxes[:, :, None], 4, 1) # x1 shape(1,1,n) + r = torch.arange(w, device=masks.device, dtype=x1.dtype)[None, None, :] # rows shape(1,w,1) + c = torch.arange(h, device=masks.device, dtype=x1.dtype)[None, :, None] # cols shape(h,1,1) + + return masks * ((r >= x1) * (r < x2) * (c >= y1) * (c < y2)) + + +def process_mask_upsample(protos, masks_in, bboxes, shape): + """ + Crop after upsample. + protos: [mask_dim, mask_h, mask_w] + masks_in: [n, mask_dim], n is number of masks after nms + bboxes: [n, 4], n is number of masks after nms + shape: input_image_size, (h, w) + + return: h, w, n + """ + + c, mh, mw = protos.shape # CHW + masks = (masks_in @ protos.float().view(c, -1)).sigmoid().view(-1, mh, mw) + masks = F.interpolate(masks[None], shape, mode='bilinear', align_corners=False)[0] # CHW + masks = crop_mask(masks, bboxes) # CHW + return masks.gt_(0.5) + + +def process_mask(protos, masks_in, bboxes, shape, upsample=False): + """ + Crop before upsample. + proto_out: [mask_dim, mask_h, mask_w] + out_masks: [n, mask_dim], n is number of masks after nms + bboxes: [n, 4], n is number of masks after nms + shape:input_image_size, (h, w) + + return: h, w, n + """ + + c, mh, mw = protos.shape # CHW + ih, iw = shape + masks = (masks_in @ protos.float().view(c, -1)).sigmoid().view(-1, mh, mw) # CHW + + downsampled_bboxes = bboxes.clone() + downsampled_bboxes[:, 0] *= mw / iw + downsampled_bboxes[:, 2] *= mw / iw + downsampled_bboxes[:, 3] *= mh / ih + downsampled_bboxes[:, 1] *= mh / ih + + masks = crop_mask(masks, downsampled_bboxes) # CHW + if upsample: + masks = F.interpolate(masks[None], shape, mode='bilinear', align_corners=False)[0] # CHW + return masks.gt_(0.5) + + +def process_mask_native(protos, masks_in, bboxes, shape): + """ + Crop after upsample. + protos: [mask_dim, mask_h, mask_w] + masks_in: [n, mask_dim], n is number of masks after nms + bboxes: [n, 4], n is number of masks after nms + shape: input_image_size, (h, w) + + return: h, w, n + """ + c, mh, mw = protos.shape # CHW + masks = (masks_in @ protos.float().view(c, -1)).sigmoid().view(-1, mh, mw) + gain = min(mh / shape[0], mw / shape[1]) # gain = old / new + pad = (mw - shape[1] * gain) / 2, (mh - shape[0] * gain) / 2 # wh padding + top, left = int(pad[1]), int(pad[0]) # y, x + bottom, right = int(mh - pad[1]), int(mw - pad[0]) + masks = masks[:, top:bottom, left:right] + + masks = F.interpolate(masks[None], shape, mode='bilinear', align_corners=False)[0] # CHW + masks = crop_mask(masks, bboxes) # CHW + return masks.gt_(0.5) + + +def scale_image(im1_shape, masks, im0_shape, ratio_pad=None): + """ + img1_shape: model input shape, [h, w] + img0_shape: origin pic shape, [h, w, 3] + masks: [h, w, num] + """ + # Rescale coordinates (xyxy) from im1_shape to im0_shape + if ratio_pad is None: # calculate from im0_shape + gain = min(im1_shape[0] / im0_shape[0], im1_shape[1] / im0_shape[1]) # gain = old / new + pad = (im1_shape[1] - im0_shape[1] * gain) / 2, (im1_shape[0] - im0_shape[0] * gain) / 2 # wh padding + else: + pad = ratio_pad[1] + top, left = int(pad[1]), int(pad[0]) # y, x + bottom, right = int(im1_shape[0] - pad[1]), int(im1_shape[1] - pad[0]) + + if len(masks.shape) < 2: + raise ValueError(f'"len of masks shape" should be 2 or 3, but got {len(masks.shape)}') + masks = masks[top:bottom, left:right] + # masks = masks.permute(2, 0, 1).contiguous() + # masks = F.interpolate(masks[None], im0_shape[:2], mode='bilinear', align_corners=False)[0] + # masks = masks.permute(1, 2, 0).contiguous() + masks = cv2.resize(masks, (im0_shape[1], im0_shape[0])) + + if len(masks.shape) == 2: + masks = masks[:, :, None] + return masks + + +def mask_iou(mask1, mask2, eps=1e-7): + """ + mask1: [N, n] m1 means number of predicted objects + mask2: [M, n] m2 means number of gt objects + Note: n means image_w x image_h + + return: masks iou, [N, M] + """ + intersection = torch.matmul(mask1, mask2.t()).clamp(0) + union = (mask1.sum(1)[:, None] + mask2.sum(1)[None]) - intersection # (area1 + area2) - intersection + return intersection / (union + eps) + + +def masks_iou(mask1, mask2, eps=1e-7): + """ + mask1: [N, n] m1 means number of predicted objects + mask2: [N, n] m2 means number of gt objects + Note: n means image_w x image_h + + return: masks iou, (N, ) + """ + intersection = (mask1 * mask2).sum(1).clamp(0) # (N, ) + union = (mask1.sum(1) + mask2.sum(1))[None] - intersection # (area1 + area2) - intersection + return intersection / (union + eps) + + +def masks2segments(masks, strategy='largest'): + # Convert masks(n,160,160) into segments(n,xy) + segments = [] + for x in masks.int().cpu().numpy().astype('uint8'): + c = cv2.findContours(x, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[0] + if c: + if strategy == 'concat': # concatenate all segments + c = np.concatenate([x.reshape(-1, 2) for x in c]) + elif strategy == 'largest': # select largest segment + c = np.array(c[np.array([len(x) for x in c]).argmax()]).reshape(-1, 2) + else: + c = np.zeros((0, 2)) # no segments found + segments.append(c.astype('float32')) + return segments diff --git a/utils/segment/loss.py b/utils/segment/loss.py new file mode 100644 index 0000000..caeff3c --- /dev/null +++ b/utils/segment/loss.py @@ -0,0 +1,185 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..general import xywh2xyxy +from ..loss import FocalLoss, smooth_BCE +from ..metrics import bbox_iou +from ..torch_utils import de_parallel +from .general import crop_mask + + +class ComputeLoss: + # Compute losses + def __init__(self, model, autobalance=False, overlap=False): + self.sort_obj_iou = False + self.overlap = overlap + device = next(model.parameters()).device # get model device + h = model.hyp # hyperparameters + + # Define criteria + BCEcls = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h['cls_pw']], device=device)) + BCEobj = nn.BCEWithLogitsLoss(pos_weight=torch.tensor([h['obj_pw']], device=device)) + + # Class label smoothing https://arxiv.org/pdf/1902.04103.pdf eqn 3 + self.cp, self.cn = smooth_BCE(eps=h.get('label_smoothing', 0.0)) # positive, negative BCE targets + + # Focal loss + g = h['fl_gamma'] # focal loss gamma + if g > 0: + BCEcls, BCEobj = FocalLoss(BCEcls, g), FocalLoss(BCEobj, g) + + m = de_parallel(model).model[-1] # Detect() module + self.balance = {3: [4.0, 1.0, 0.4]}.get(m.nl, [4.0, 1.0, 0.25, 0.06, 0.02]) # P3-P7 + self.ssi = list(m.stride).index(16) if autobalance else 0 # stride 16 index + self.BCEcls, self.BCEobj, self.gr, self.hyp, self.autobalance = BCEcls, BCEobj, 1.0, h, autobalance + self.na = m.na # number of anchors + self.nc = m.nc # number of classes + self.nl = m.nl # number of layers + self.nm = m.nm # number of masks + self.anchors = m.anchors + self.device = device + + def __call__(self, preds, targets, masks): # predictions, targets, model + p, proto = preds + bs, nm, mask_h, mask_w = proto.shape # batch size, number of masks, mask height, mask width + lcls = torch.zeros(1, device=self.device) + lbox = torch.zeros(1, device=self.device) + lobj = torch.zeros(1, device=self.device) + lseg = torch.zeros(1, device=self.device) + tcls, tbox, indices, anchors, tidxs, xywhn = self.build_targets(p, targets) # targets + + # Losses + for i, pi in enumerate(p): # layer index, layer predictions + b, a, gj, gi = indices[i] # image, anchor, gridy, gridx + tobj = torch.zeros(pi.shape[:4], dtype=pi.dtype, device=self.device) # target obj + + n = b.shape[0] # number of targets + if n: + pxy, pwh, _, pcls, pmask = pi[b, a, gj, gi].split((2, 2, 1, self.nc, nm), 1) # subset of predictions + + # Box regression + pxy = pxy.sigmoid() * 2 - 0.5 + pwh = (pwh.sigmoid() * 2) ** 2 * anchors[i] + pbox = torch.cat((pxy, pwh), 1) # predicted box + iou = bbox_iou(pbox, tbox[i], CIoU=True).squeeze() # iou(prediction, target) + lbox += (1.0 - iou).mean() # iou loss + + # Objectness + iou = iou.detach().clamp(0).type(tobj.dtype) + if self.sort_obj_iou: + j = iou.argsort() + b, a, gj, gi, iou = b[j], a[j], gj[j], gi[j], iou[j] + if self.gr < 1: + iou = (1.0 - self.gr) + self.gr * iou + tobj[b, a, gj, gi] = iou # iou ratio + + # Classification + if self.nc > 1: # cls loss (only if multiple classes) + t = torch.full_like(pcls, self.cn, device=self.device) # targets + t[range(n), tcls[i]] = self.cp + lcls += self.BCEcls(pcls, t) # BCE + + # Mask regression + if tuple(masks.shape[-2:]) != (mask_h, mask_w): # downsample + masks = F.interpolate(masks[None], (mask_h, mask_w), mode='nearest')[0] + marea = xywhn[i][:, 2:].prod(1) # mask width, height normalized + mxyxy = xywh2xyxy(xywhn[i] * torch.tensor([mask_w, mask_h, mask_w, mask_h], device=self.device)) + for bi in b.unique(): + j = b == bi # matching index + if self.overlap: + mask_gti = torch.where(masks[bi][None] == tidxs[i][j].view(-1, 1, 1), 1.0, 0.0) + else: + mask_gti = masks[tidxs[i]][j] + lseg += self.single_mask_loss(mask_gti, pmask[j], proto[bi], mxyxy[j], marea[j]) + + obji = self.BCEobj(pi[..., 4], tobj) + lobj += obji * self.balance[i] # obj loss + if self.autobalance: + self.balance[i] = self.balance[i] * 0.9999 + 0.0001 / obji.detach().item() + + if self.autobalance: + self.balance = [x / self.balance[self.ssi] for x in self.balance] + lbox *= self.hyp['box'] + lobj *= self.hyp['obj'] + lcls *= self.hyp['cls'] + lseg *= self.hyp['box'] / bs + + loss = lbox + lobj + lcls + lseg + return loss * bs, torch.cat((lbox, lseg, lobj, lcls)).detach() + + def single_mask_loss(self, gt_mask, pred, proto, xyxy, area): + # Mask loss for one image + pred_mask = (pred @ proto.view(self.nm, -1)).view(-1, *proto.shape[1:]) # (n,32) @ (32,80,80) -> (n,80,80) + loss = F.binary_cross_entropy_with_logits(pred_mask, gt_mask, reduction='none') + return (crop_mask(loss, xyxy).mean(dim=(1, 2)) / area).mean() + + def build_targets(self, p, targets): + # Build targets for compute_loss(), input targets(image,class,x,y,w,h) + na, nt = self.na, targets.shape[0] # number of anchors, targets + tcls, tbox, indices, anch, tidxs, xywhn = [], [], [], [], [], [] + gain = torch.ones(8, device=self.device) # normalized to gridspace gain + ai = torch.arange(na, device=self.device).float().view(na, 1).repeat(1, nt) # same as .repeat_interleave(nt) + if self.overlap: + batch = p[0].shape[0] + ti = [] + for i in range(batch): + num = (targets[:, 0] == i).sum() # find number of targets of each image + ti.append(torch.arange(num, device=self.device).float().view(1, num).repeat(na, 1) + 1) # (na, num) + ti = torch.cat(ti, 1) # (na, nt) + else: + ti = torch.arange(nt, device=self.device).float().view(1, nt).repeat(na, 1) + targets = torch.cat((targets.repeat(na, 1, 1), ai[..., None], ti[..., None]), 2) # append anchor indices + + g = 0.5 # bias + off = torch.tensor( + [ + [0, 0], + [1, 0], + [0, 1], + [-1, 0], + [0, -1], # j,k,l,m + # [1, 1], [1, -1], [-1, 1], [-1, -1], # jk,jm,lk,lm + ], + device=self.device).float() * g # offsets + + for i in range(self.nl): + anchors, shape = self.anchors[i], p[i].shape + gain[2:6] = torch.tensor(shape)[[3, 2, 3, 2]] # xyxy gain + + # Match targets to anchors + t = targets * gain # shape(3,n,7) + if nt: + # Matches + r = t[..., 4:6] / anchors[:, None] # wh ratio + j = torch.max(r, 1 / r).max(2)[0] < self.hyp['anchor_t'] # compare + # j = wh_iou(anchors, t[:, 4:6]) > model.hyp['iou_t'] # iou(3,n)=wh_iou(anchors(3,2), gwh(n,2)) + t = t[j] # filter + + # Offsets + gxy = t[:, 2:4] # grid xy + gxi = gain[[2, 3]] - gxy # inverse + j, k = ((gxy % 1 < g) & (gxy > 1)).T + l, m = ((gxi % 1 < g) & (gxi > 1)).T + j = torch.stack((torch.ones_like(j), j, k, l, m)) + t = t.repeat((5, 1, 1))[j] + offsets = (torch.zeros_like(gxy)[None] + off[:, None])[j] + else: + t = targets[0] + offsets = 0 + + # Define + bc, gxy, gwh, at = t.chunk(4, 1) # (image, class), grid xy, grid wh, anchors + (a, tidx), (b, c) = at.long().T, bc.long().T # anchors, image, class + gij = (gxy - offsets).long() + gi, gj = gij.T # grid indices + + # Append + indices.append((b, a, gj.clamp_(0, shape[2] - 1), gi.clamp_(0, shape[3] - 1))) # image, anchor, grid + tbox.append(torch.cat((gxy - gij, gwh), 1)) # box + anch.append(anchors[a]) # anchors + tcls.append(c) # class + tidxs.append(tidx) + xywhn.append(torch.cat((gxy, gwh), 1) / gain[2:6]) # xywh normalized + + return tcls, tbox, indices, anch, tidxs, xywhn diff --git a/utils/segment/metrics.py b/utils/segment/metrics.py new file mode 100644 index 0000000..787961b --- /dev/null +++ b/utils/segment/metrics.py @@ -0,0 +1,210 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Model validation metrics +""" + +import numpy as np + +from ..metrics import ap_per_class + + +def fitness(x): + # Model fitness as a weighted combination of metrics + w = [0.0, 0.0, 0.1, 0.9, 0.0, 0.0, 0.1, 0.9] + return (x[:, :8] * w).sum(1) + + +def ap_per_class_box_and_mask( + tp_m, + tp_b, + conf, + pred_cls, + target_cls, + plot=False, + save_dir='.', + names=(), +): + """ + Args: + tp_b: tp of boxes. + tp_m: tp of masks. + other arguments see `func: ap_per_class`. + """ + results_boxes = ap_per_class(tp_b, + conf, + pred_cls, + target_cls, + plot=plot, + save_dir=save_dir, + names=names, + prefix='Box')[2:] + results_masks = ap_per_class(tp_m, + conf, + pred_cls, + target_cls, + plot=plot, + save_dir=save_dir, + names=names, + prefix='Mask')[2:] + + results = { + 'boxes': { + 'p': results_boxes[0], + 'r': results_boxes[1], + 'ap': results_boxes[3], + 'f1': results_boxes[2], + 'ap_class': results_boxes[4]}, + 'masks': { + 'p': results_masks[0], + 'r': results_masks[1], + 'ap': results_masks[3], + 'f1': results_masks[2], + 'ap_class': results_masks[4]}} + return results + + +class Metric: + + def __init__(self) -> None: + self.p = [] # (nc, ) + self.r = [] # (nc, ) + self.f1 = [] # (nc, ) + self.all_ap = [] # (nc, 10) + self.ap_class_index = [] # (nc, ) + + @property + def ap50(self): + """AP@0.5 of all classes. + Return: + (nc, ) or []. + """ + return self.all_ap[:, 0] if len(self.all_ap) else [] + + @property + def ap(self): + """AP@0.5:0.95 + Return: + (nc, ) or []. + """ + return self.all_ap.mean(1) if len(self.all_ap) else [] + + @property + def mp(self): + """mean precision of all classes. + Return: + float. + """ + return self.p.mean() if len(self.p) else 0.0 + + @property + def mr(self): + """mean recall of all classes. + Return: + float. + """ + return self.r.mean() if len(self.r) else 0.0 + + @property + def map50(self): + """Mean AP@0.5 of all classes. + Return: + float. + """ + return self.all_ap[:, 0].mean() if len(self.all_ap) else 0.0 + + @property + def map(self): + """Mean AP@0.5:0.95 of all classes. + Return: + float. + """ + return self.all_ap.mean() if len(self.all_ap) else 0.0 + + def mean_results(self): + """Mean of results, return mp, mr, map50, map""" + return (self.mp, self.mr, self.map50, self.map) + + def class_result(self, i): + """class-aware result, return p[i], r[i], ap50[i], ap[i]""" + return (self.p[i], self.r[i], self.ap50[i], self.ap[i]) + + def get_maps(self, nc): + maps = np.zeros(nc) + self.map + for i, c in enumerate(self.ap_class_index): + maps[c] = self.ap[i] + return maps + + def update(self, results): + """ + Args: + results: tuple(p, r, ap, f1, ap_class) + """ + p, r, all_ap, f1, ap_class_index = results + self.p = p + self.r = r + self.all_ap = all_ap + self.f1 = f1 + self.ap_class_index = ap_class_index + + +class Metrics: + """Metric for boxes and masks.""" + + def __init__(self) -> None: + self.metric_box = Metric() + self.metric_mask = Metric() + + def update(self, results): + """ + Args: + results: Dict{'boxes': Dict{}, 'masks': Dict{}} + """ + self.metric_box.update(list(results['boxes'].values())) + self.metric_mask.update(list(results['masks'].values())) + + def mean_results(self): + return self.metric_box.mean_results() + self.metric_mask.mean_results() + + def class_result(self, i): + return self.metric_box.class_result(i) + self.metric_mask.class_result(i) + + def get_maps(self, nc): + return self.metric_box.get_maps(nc) + self.metric_mask.get_maps(nc) + + @property + def ap_class_index(self): + # boxes and masks have the same ap_class_index + return self.metric_box.ap_class_index + + +KEYS = [ + 'train/box_loss', + 'train/seg_loss', # train loss + 'train/obj_loss', + 'train/cls_loss', + 'metrics/precision(B)', + 'metrics/recall(B)', + 'metrics/mAP_0.5(B)', + 'metrics/mAP_0.5:0.95(B)', # metrics + 'metrics/precision(M)', + 'metrics/recall(M)', + 'metrics/mAP_0.5(M)', + 'metrics/mAP_0.5:0.95(M)', # metrics + 'val/box_loss', + 'val/seg_loss', # val loss + 'val/obj_loss', + 'val/cls_loss', + 'x/lr0', + 'x/lr1', + 'x/lr2', ] + +BEST_KEYS = [ + 'best/epoch', + 'best/precision(B)', + 'best/recall(B)', + 'best/mAP_0.5(B)', + 'best/mAP_0.5:0.95(B)', + 'best/precision(M)', + 'best/recall(M)', + 'best/mAP_0.5(M)', + 'best/mAP_0.5:0.95(M)', ] diff --git a/utils/segment/plots.py b/utils/segment/plots.py new file mode 100644 index 0000000..f9938cd --- /dev/null +++ b/utils/segment/plots.py @@ -0,0 +1,143 @@ +import contextlib +import math +from pathlib import Path + +import cv2 +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import torch + +from .. import threaded +from ..general import xywh2xyxy +from ..plots import Annotator, colors + + +@threaded +def plot_images_and_masks(images, targets, masks, paths=None, fname='images.jpg', names=None): + # Plot image grid with labels + if isinstance(images, torch.Tensor): + images = images.cpu().float().numpy() + if isinstance(targets, torch.Tensor): + targets = targets.cpu().numpy() + if isinstance(masks, torch.Tensor): + masks = masks.cpu().numpy().astype(int) + + max_size = 1920 # max image size + max_subplots = 16 # max image subplots, i.e. 4x4 + bs, _, h, w = images.shape # batch size, _, height, width + bs = min(bs, max_subplots) # limit plot images + ns = np.ceil(bs ** 0.5) # number of subplots (square) + if np.max(images[0]) <= 1: + images *= 255 # de-normalise (optional) + + # Build Image + mosaic = np.full((int(ns * h), int(ns * w), 3), 255, dtype=np.uint8) # init + for i, im in enumerate(images): + if i == max_subplots: # if last batch has fewer images than we expect + break + x, y = int(w * (i // ns)), int(h * (i % ns)) # block origin + im = im.transpose(1, 2, 0) + mosaic[y:y + h, x:x + w, :] = im + + # Resize (optional) + scale = max_size / ns / max(h, w) + if scale < 1: + h = math.ceil(scale * h) + w = math.ceil(scale * w) + mosaic = cv2.resize(mosaic, tuple(int(x * ns) for x in (w, h))) + + # Annotate + fs = int((h + w) * ns * 0.01) # font size + annotator = Annotator(mosaic, line_width=round(fs / 10), font_size=fs, pil=True, example=names) + for i in range(i + 1): + x, y = int(w * (i // ns)), int(h * (i % ns)) # block origin + annotator.rectangle([x, y, x + w, y + h], None, (255, 255, 255), width=2) # borders + if paths: + annotator.text([x + 5, y + 5], text=Path(paths[i]).name[:40], txt_color=(220, 220, 220)) # filenames + if len(targets) > 0: + idx = targets[:, 0] == i + ti = targets[idx] # image targets + + boxes = xywh2xyxy(ti[:, 2:6]).T + classes = ti[:, 1].astype('int') + labels = ti.shape[1] == 6 # labels if no conf column + conf = None if labels else ti[:, 6] # check for confidence presence (label vs pred) + + if boxes.shape[1]: + if boxes.max() <= 1.01: # if normalized with tolerance 0.01 + boxes[[0, 2]] *= w # scale to pixels + boxes[[1, 3]] *= h + elif scale < 1: # absolute coords need scale if image scales + boxes *= scale + boxes[[0, 2]] += x + boxes[[1, 3]] += y + for j, box in enumerate(boxes.T.tolist()): + cls = classes[j] + color = colors(cls) + cls = names[cls] if names else cls + if labels or conf[j] > 0.25: # 0.25 conf thresh + label = f'{cls}' if labels else f'{cls} {conf[j]:.1f}' + annotator.box_label(box, label, color=color) + + # Plot masks + if len(masks): + if masks.max() > 1.0: # mean that masks are overlap + image_masks = masks[[i]] # (1, 640, 640) + nl = len(ti) + index = np.arange(nl).reshape(nl, 1, 1) + 1 + image_masks = np.repeat(image_masks, nl, axis=0) + image_masks = np.where(image_masks == index, 1.0, 0.0) + else: + image_masks = masks[idx] + + im = np.asarray(annotator.im).copy() + for j, box in enumerate(boxes.T.tolist()): + if labels or conf[j] > 0.25: # 0.25 conf thresh + color = colors(classes[j]) + mh, mw = image_masks[j].shape + if mh != h or mw != w: + mask = image_masks[j].astype(np.uint8) + mask = cv2.resize(mask, (w, h)) + mask = mask.astype(bool) + else: + mask = image_masks[j].astype(bool) + with contextlib.suppress(Exception): + im[y:y + h, x:x + w, :][mask] = im[y:y + h, x:x + w, :][mask] * 0.4 + np.array(color) * 0.6 + annotator.fromarray(im) + annotator.im.save(fname) # save + + +def plot_results_with_masks(file='path/to/results.csv', dir='', best=True): + # Plot training results.csv. Usage: from utils.plots import *; plot_results('path/to/results.csv') + save_dir = Path(file).parent if file else Path(dir) + fig, ax = plt.subplots(2, 8, figsize=(18, 6), tight_layout=True) + ax = ax.ravel() + files = list(save_dir.glob('results*.csv')) + assert len(files), f'No results.csv files found in {save_dir.resolve()}, nothing to plot.' + for f in files: + try: + data = pd.read_csv(f) + index = np.argmax(0.9 * data.values[:, 8] + 0.1 * data.values[:, 7] + 0.9 * data.values[:, 12] + + 0.1 * data.values[:, 11]) + s = [x.strip() for x in data.columns] + x = data.values[:, 0] + for i, j in enumerate([1, 2, 3, 4, 5, 6, 9, 10, 13, 14, 15, 16, 7, 8, 11, 12]): + y = data.values[:, j] + # y[y == 0] = np.nan # don't show zero values + ax[i].plot(x, y, marker='.', label=f.stem, linewidth=2, markersize=2) + if best: + # best + ax[i].scatter(index, y[index], color='r', label=f'best:{index}', marker='*', linewidth=3) + ax[i].set_title(s[j] + f'\n{round(y[index], 5)}') + else: + # last + ax[i].scatter(x[-1], y[-1], color='r', label='last', marker='*', linewidth=3) + ax[i].set_title(s[j] + f'\n{round(y[-1], 5)}') + # if j in [8, 9, 10]: # share train and val loss y axes + # ax[i].get_shared_y_axes().join(ax[i], ax[i - 5]) + except Exception as e: + print(f'Warning: Plotting error for {f}: {e}') + ax[1].legend() + fig.savefig(save_dir / 'results.png', dpi=200) + plt.close() diff --git a/utils/torch_utils.py b/utils/torch_utils.py new file mode 100644 index 0000000..13a356f --- /dev/null +++ b/utils/torch_utils.py @@ -0,0 +1,432 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +PyTorch utils +""" + +import math +import os +import platform +import subprocess +import time +import warnings +from contextlib import contextmanager +from copy import deepcopy +from pathlib import Path + +import torch +import torch.distributed as dist +import torch.nn as nn +import torch.nn.functional as F +from torch.nn.parallel import DistributedDataParallel as DDP + +from utils.general import LOGGER, check_version, colorstr, file_date, git_describe + +LOCAL_RANK = int(os.getenv('LOCAL_RANK', -1)) # https://pytorch.org/docs/stable/elastic/run.html +RANK = int(os.getenv('RANK', -1)) +WORLD_SIZE = int(os.getenv('WORLD_SIZE', 1)) + +try: + import thop # for FLOPs computation +except ImportError: + thop = None + +# Suppress PyTorch warnings +warnings.filterwarnings('ignore', message='User provided device_type of \'cuda\', but CUDA is not available. Disabling') +warnings.filterwarnings('ignore', category=UserWarning) + + +def smart_inference_mode(torch_1_9=check_version(torch.__version__, '1.9.0')): + # Applies torch.inference_mode() decorator if torch>=1.9.0 else torch.no_grad() decorator + def decorate(fn): + return (torch.inference_mode if torch_1_9 else torch.no_grad)()(fn) + + return decorate + + +def smartCrossEntropyLoss(label_smoothing=0.0): + # Returns nn.CrossEntropyLoss with label smoothing enabled for torch>=1.10.0 + if check_version(torch.__version__, '1.10.0'): + return nn.CrossEntropyLoss(label_smoothing=label_smoothing) + if label_smoothing > 0: + LOGGER.warning(f'WARNING ⚠️ label smoothing {label_smoothing} requires torch>=1.10.0') + return nn.CrossEntropyLoss() + + +def smart_DDP(model): + # Model DDP creation with checks + assert not check_version(torch.__version__, '1.12.0', pinned=True), \ + 'torch==1.12.0 torchvision==0.13.0 DDP training is not supported due to a known issue. ' \ + 'Please upgrade or downgrade torch to use DDP. See https://github.com/ultralytics/yolov5/issues/8395' + if check_version(torch.__version__, '1.11.0'): + return DDP(model, device_ids=[LOCAL_RANK], output_device=LOCAL_RANK, static_graph=True) + else: + return DDP(model, device_ids=[LOCAL_RANK], output_device=LOCAL_RANK) + + +def reshape_classifier_output(model, n=1000): + # Update a TorchVision classification model to class count 'n' if required + from models.common import Classify + name, m = list((model.model if hasattr(model, 'model') else model).named_children())[-1] # last module + if isinstance(m, Classify): # YOLOv5 Classify() head + if m.linear.out_features != n: + m.linear = nn.Linear(m.linear.in_features, n) + elif isinstance(m, nn.Linear): # ResNet, EfficientNet + if m.out_features != n: + setattr(model, name, nn.Linear(m.in_features, n)) + elif isinstance(m, nn.Sequential): + types = [type(x) for x in m] + if nn.Linear in types: + i = types.index(nn.Linear) # nn.Linear index + if m[i].out_features != n: + m[i] = nn.Linear(m[i].in_features, n) + elif nn.Conv2d in types: + i = types.index(nn.Conv2d) # nn.Conv2d index + if m[i].out_channels != n: + m[i] = nn.Conv2d(m[i].in_channels, n, m[i].kernel_size, m[i].stride, bias=m[i].bias is not None) + + +@contextmanager +def torch_distributed_zero_first(local_rank: int): + # Decorator to make all processes in distributed training wait for each local_master to do something + if local_rank not in [-1, 0]: + dist.barrier(device_ids=[local_rank]) + yield + if local_rank == 0: + dist.barrier(device_ids=[0]) + + +def device_count(): + # Returns number of CUDA devices available. Safe version of torch.cuda.device_count(). Supports Linux and Windows + assert platform.system() in ('Linux', 'Windows'), 'device_count() only supported on Linux or Windows' + try: + cmd = 'nvidia-smi -L | wc -l' if platform.system() == 'Linux' else 'nvidia-smi -L | find /c /v ""' # Windows + return int(subprocess.run(cmd, shell=True, capture_output=True, check=True).stdout.decode().split()[-1]) + except Exception: + return 0 + + +def select_device(device='', batch_size=0, newline=True): + # device = None or 'cpu' or 0 or '0' or '0,1,2,3' + s = f'YOLOv5 🚀 {git_describe() or file_date()} Python-{platform.python_version()} torch-{torch.__version__} ' + device = str(device).strip().lower().replace('cuda:', '').replace('none', '') # to string, 'cuda:0' to '0' + cpu = device == 'cpu' + mps = device == 'mps' # Apple Metal Performance Shaders (MPS) + if cpu or mps: + os.environ['CUDA_VISIBLE_DEVICES'] = '-1' # force torch.cuda.is_available() = False + elif device: # non-cpu device requested + os.environ['CUDA_VISIBLE_DEVICES'] = device # set environment variable - must be before assert is_available() + assert torch.cuda.is_available() and torch.cuda.device_count() >= len(device.replace(',', '')), \ + f"Invalid CUDA '--device {device}' requested, use '--device cpu' or pass valid CUDA device(s)" + + if not cpu and not mps and torch.cuda.is_available(): # prefer GPU if available + devices = device.split(',') if device else '0' # range(torch.cuda.device_count()) # i.e. 0,1,6,7 + n = len(devices) # device count + if n > 1 and batch_size > 0: # check batch_size is divisible by device_count + assert batch_size % n == 0, f'batch-size {batch_size} not multiple of GPU count {n}' + space = ' ' * (len(s) + 1) + for i, d in enumerate(devices): + p = torch.cuda.get_device_properties(i) + s += f"{'' if i == 0 else space}CUDA:{d} ({p.name}, {p.total_memory / (1 << 20):.0f}MiB)\n" # bytes to MB + arg = 'cuda:0' + elif mps and getattr(torch, 'has_mps', False) and torch.backends.mps.is_available(): # prefer MPS if available + s += 'MPS\n' + arg = 'mps' + else: # revert to CPU + s += 'CPU\n' + arg = 'cpu' + + if not newline: + s = s.rstrip() + LOGGER.info(s) + return torch.device(arg) + + +def time_sync(): + # PyTorch-accurate time + if torch.cuda.is_available(): + torch.cuda.synchronize() + return time.time() + + +def profile(input, ops, n=10, device=None): + """ YOLOv5 speed/memory/FLOPs profiler + Usage: + input = torch.randn(16, 3, 640, 640) + m1 = lambda x: x * torch.sigmoid(x) + m2 = nn.SiLU() + profile(input, [m1, m2], n=100) # profile over 100 iterations + """ + results = [] + if not isinstance(device, torch.device): + device = select_device(device) + print(f"{'Params':>12s}{'GFLOPs':>12s}{'GPU_mem (GB)':>14s}{'forward (ms)':>14s}{'backward (ms)':>14s}" + f"{'input':>24s}{'output':>24s}") + + for x in input if isinstance(input, list) else [input]: + x = x.to(device) + x.requires_grad = True + for m in ops if isinstance(ops, list) else [ops]: + m = m.to(device) if hasattr(m, 'to') else m # device + m = m.half() if hasattr(m, 'half') and isinstance(x, torch.Tensor) and x.dtype is torch.float16 else m + tf, tb, t = 0, 0, [0, 0, 0] # dt forward, backward + try: + flops = thop.profile(m, inputs=(x, ), verbose=False)[0] / 1E9 * 2 # GFLOPs + except Exception: + flops = 0 + + try: + for _ in range(n): + t[0] = time_sync() + y = m(x) + t[1] = time_sync() + try: + _ = (sum(yi.sum() for yi in y) if isinstance(y, list) else y).sum().backward() + t[2] = time_sync() + except Exception: # no backward method + # print(e) # for debug + t[2] = float('nan') + tf += (t[1] - t[0]) * 1000 / n # ms per op forward + tb += (t[2] - t[1]) * 1000 / n # ms per op backward + mem = torch.cuda.memory_reserved() / 1E9 if torch.cuda.is_available() else 0 # (GB) + s_in, s_out = (tuple(x.shape) if isinstance(x, torch.Tensor) else 'list' for x in (x, y)) # shapes + p = sum(x.numel() for x in m.parameters()) if isinstance(m, nn.Module) else 0 # parameters + print(f'{p:12}{flops:12.4g}{mem:>14.3f}{tf:14.4g}{tb:14.4g}{str(s_in):>24s}{str(s_out):>24s}') + results.append([p, flops, mem, tf, tb, s_in, s_out]) + except Exception as e: + print(e) + results.append(None) + torch.cuda.empty_cache() + return results + + +def is_parallel(model): + # Returns True if model is of type DP or DDP + return type(model) in (nn.parallel.DataParallel, nn.parallel.DistributedDataParallel) + + +def de_parallel(model): + # De-parallelize a model: returns single-GPU model if model is of type DP or DDP + return model.module if is_parallel(model) else model + + +def initialize_weights(model): + for m in model.modules(): + t = type(m) + if t is nn.Conv2d: + pass # nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') + elif t is nn.BatchNorm2d: + m.eps = 1e-3 + m.momentum = 0.03 + elif t in [nn.Hardswish, nn.LeakyReLU, nn.ReLU, nn.ReLU6, nn.SiLU]: + m.inplace = True + + +def find_modules(model, mclass=nn.Conv2d): + # Finds layer indices matching module class 'mclass' + return [i for i, m in enumerate(model.module_list) if isinstance(m, mclass)] + + +def sparsity(model): + # Return global model sparsity + a, b = 0, 0 + for p in model.parameters(): + a += p.numel() + b += (p == 0).sum() + return b / a + + +def prune(model, amount=0.3): + # Prune model to requested global sparsity + import torch.nn.utils.prune as prune + for name, m in model.named_modules(): + if isinstance(m, nn.Conv2d): + prune.l1_unstructured(m, name='weight', amount=amount) # prune + prune.remove(m, 'weight') # make permanent + LOGGER.info(f'Model pruned to {sparsity(model):.3g} global sparsity') + + +def fuse_conv_and_bn(conv, bn): + # Fuse Conv2d() and BatchNorm2d() layers https://tehnokv.com/posts/fusing-batchnorm-and-conv/ + fusedconv = nn.Conv2d(conv.in_channels, + conv.out_channels, + kernel_size=conv.kernel_size, + stride=conv.stride, + padding=conv.padding, + dilation=conv.dilation, + groups=conv.groups, + bias=True).requires_grad_(False).to(conv.weight.device) + + # Prepare filters + w_conv = conv.weight.clone().view(conv.out_channels, -1) + w_bn = torch.diag(bn.weight.div(torch.sqrt(bn.eps + bn.running_var))) + fusedconv.weight.copy_(torch.mm(w_bn, w_conv).view(fusedconv.weight.shape)) + + # Prepare spatial bias + b_conv = torch.zeros(conv.weight.size(0), device=conv.weight.device) if conv.bias is None else conv.bias + b_bn = bn.bias - bn.weight.mul(bn.running_mean).div(torch.sqrt(bn.running_var + bn.eps)) + fusedconv.bias.copy_(torch.mm(w_bn, b_conv.reshape(-1, 1)).reshape(-1) + b_bn) + + return fusedconv + + +def model_info(model, verbose=False, imgsz=640): + # Model information. img_size may be int or list, i.e. img_size=640 or img_size=[640, 320] + n_p = sum(x.numel() for x in model.parameters()) # number parameters + n_g = sum(x.numel() for x in model.parameters() if x.requires_grad) # number gradients + if verbose: + print(f"{'layer':>5} {'name':>40} {'gradient':>9} {'parameters':>12} {'shape':>20} {'mu':>10} {'sigma':>10}") + for i, (name, p) in enumerate(model.named_parameters()): + name = name.replace('module_list.', '') + print('%5g %40s %9s %12g %20s %10.3g %10.3g' % + (i, name, p.requires_grad, p.numel(), list(p.shape), p.mean(), p.std())) + + try: # FLOPs + p = next(model.parameters()) + stride = max(int(model.stride.max()), 32) if hasattr(model, 'stride') else 32 # max stride + im = torch.empty((1, p.shape[1], stride, stride), device=p.device) # input image in BCHW format + flops = thop.profile(deepcopy(model), inputs=(im, ), verbose=False)[0] / 1E9 * 2 # stride GFLOPs + imgsz = imgsz if isinstance(imgsz, list) else [imgsz, imgsz] # expand if int/float + fs = f', {flops * imgsz[0] / stride * imgsz[1] / stride:.1f} GFLOPs' # 640x640 GFLOPs + except Exception: + fs = '' + + name = Path(model.yaml_file).stem.replace('yolov5', 'YOLOv5') if hasattr(model, 'yaml_file') else 'Model' + LOGGER.info(f'{name} summary: {len(list(model.modules()))} layers, {n_p} parameters, {n_g} gradients{fs}') + + +def scale_img(img, ratio=1.0, same_shape=False, gs=32): # img(16,3,256,416) + # Scales img(bs,3,y,x) by ratio constrained to gs-multiple + if ratio == 1.0: + return img + h, w = img.shape[2:] + s = (int(h * ratio), int(w * ratio)) # new size + img = F.interpolate(img, size=s, mode='bilinear', align_corners=False) # resize + if not same_shape: # pad/crop img + h, w = (math.ceil(x * ratio / gs) * gs for x in (h, w)) + return F.pad(img, [0, w - s[1], 0, h - s[0]], value=0.447) # value = imagenet mean + + +def copy_attr(a, b, include=(), exclude=()): + # Copy attributes from b to a, options to only include [...] and to exclude [...] + for k, v in b.__dict__.items(): + if (len(include) and k not in include) or k.startswith('_') or k in exclude: + continue + else: + setattr(a, k, v) + + +def smart_optimizer(model, name='Adam', lr=0.001, momentum=0.9, decay=1e-5): + # YOLOv5 3-param group optimizer: 0) weights with decay, 1) weights no decay, 2) biases no decay + g = [], [], [] # optimizer parameter groups + bn = tuple(v for k, v in nn.__dict__.items() if 'Norm' in k) # normalization layers, i.e. BatchNorm2d() + for v in model.modules(): + for p_name, p in v.named_parameters(recurse=0): + if p_name == 'bias': # bias (no decay) + g[2].append(p) + elif p_name == 'weight' and isinstance(v, bn): # weight (no decay) + g[1].append(p) + else: + g[0].append(p) # weight (with decay) + + if name == 'Adam': + optimizer = torch.optim.Adam(g[2], lr=lr, betas=(momentum, 0.999)) # adjust beta1 to momentum + elif name == 'AdamW': + optimizer = torch.optim.AdamW(g[2], lr=lr, betas=(momentum, 0.999), weight_decay=0.0) + elif name == 'RMSProp': + optimizer = torch.optim.RMSprop(g[2], lr=lr, momentum=momentum) + elif name == 'SGD': + optimizer = torch.optim.SGD(g[2], lr=lr, momentum=momentum, nesterov=True) + else: + raise NotImplementedError(f'Optimizer {name} not implemented.') + + optimizer.add_param_group({'params': g[0], 'weight_decay': decay}) # add g0 with weight_decay + optimizer.add_param_group({'params': g[1], 'weight_decay': 0.0}) # add g1 (BatchNorm2d weights) + LOGGER.info(f"{colorstr('optimizer:')} {type(optimizer).__name__}(lr={lr}) with parameter groups " + f'{len(g[1])} weight(decay=0.0), {len(g[0])} weight(decay={decay}), {len(g[2])} bias') + return optimizer + + +def smart_hub_load(repo='ultralytics/yolov5', model='yolov5s', **kwargs): + # YOLOv5 torch.hub.load() wrapper with smart error/issue handling + if check_version(torch.__version__, '1.9.1'): + kwargs['skip_validation'] = True # validation causes GitHub API rate limit errors + if check_version(torch.__version__, '1.12.0'): + kwargs['trust_repo'] = True # argument required starting in torch 0.12 + try: + return torch.hub.load(repo, model, **kwargs) + except Exception: + return torch.hub.load(repo, model, force_reload=True, **kwargs) + + +def smart_resume(ckpt, optimizer, ema=None, weights='yolov5s.pt', epochs=300, resume=True): + # Resume training from a partially trained checkpoint + best_fitness = 0.0 + start_epoch = ckpt['epoch'] + 1 + if ckpt['optimizer'] is not None: + optimizer.load_state_dict(ckpt['optimizer']) # optimizer + best_fitness = ckpt['best_fitness'] + if ema and ckpt.get('ema'): + ema.ema.load_state_dict(ckpt['ema'].float().state_dict()) # EMA + ema.updates = ckpt['updates'] + if resume: + assert start_epoch > 0, f'{weights} training to {epochs} epochs is finished, nothing to resume.\n' \ + f"Start a new training without --resume, i.e. 'python train.py --weights {weights}'" + LOGGER.info(f'Resuming training from {weights} from epoch {start_epoch} to {epochs} total epochs') + if epochs < start_epoch: + LOGGER.info(f"{weights} has been trained for {ckpt['epoch']} epochs. Fine-tuning for {epochs} more epochs.") + epochs += ckpt['epoch'] # finetune additional epochs + return best_fitness, start_epoch, epochs + + +class EarlyStopping: + # YOLOv5 simple early stopper + def __init__(self, patience=30): + self.best_fitness = 0.0 # i.e. mAP + self.best_epoch = 0 + self.patience = patience or float('inf') # epochs to wait after fitness stops improving to stop + self.possible_stop = False # possible stop may occur next epoch + + def __call__(self, epoch, fitness): + if fitness >= self.best_fitness: # >= 0 to allow for early zero-fitness stage of training + self.best_epoch = epoch + self.best_fitness = fitness + delta = epoch - self.best_epoch # epochs without improvement + self.possible_stop = delta >= (self.patience - 1) # possible stop may occur next epoch + stop = delta >= self.patience # stop training if patience exceeded + if stop: + LOGGER.info(f'Stopping training early as no improvement observed in last {self.patience} epochs. ' + f'Best results observed at epoch {self.best_epoch}, best model saved as best.pt.\n' + f'To update EarlyStopping(patience={self.patience}) pass a new patience value, ' + f'i.e. `python train.py --patience 300` or use `--patience 0` to disable EarlyStopping.') + return stop + + +class ModelEMA: + """ Updated Exponential Moving Average (EMA) from https://github.com/rwightman/pytorch-image-models + Keeps a moving average of everything in the model state_dict (parameters and buffers) + For EMA details see https://www.tensorflow.org/api_docs/python/tf/train/ExponentialMovingAverage + """ + + def __init__(self, model, decay=0.9999, tau=2000, updates=0): + # Create EMA + self.ema = deepcopy(de_parallel(model)).eval() # FP32 EMA + self.updates = updates # number of EMA updates + self.decay = lambda x: decay * (1 - math.exp(-x / tau)) # decay exponential ramp (to help early epochs) + for p in self.ema.parameters(): + p.requires_grad_(False) + + def update(self, model): + # Update EMA parameters + self.updates += 1 + d = self.decay(self.updates) + + msd = de_parallel(model).state_dict() # model state_dict + for k, v in self.ema.state_dict().items(): + if v.dtype.is_floating_point: # true for FP16 and FP32 + v *= d + v += (1 - d) * msd[k].detach() + # assert v.dtype == msd[k].dtype == torch.float32, f'{k}: EMA {v.dtype} and model {msd[k].dtype} must be FP32' + + def update_attr(self, model, include=(), exclude=('process_group', 'reducer')): + # Update EMA attributes + copy_attr(self.ema, model, include, exclude) diff --git a/utils/triton.py b/utils/triton.py new file mode 100644 index 0000000..b5153da --- /dev/null +++ b/utils/triton.py @@ -0,0 +1,85 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" Utils to interact with the Triton Inference Server +""" + +import typing +from urllib.parse import urlparse + +import torch + + +class TritonRemoteModel: + """ A wrapper over a model served by the Triton Inference Server. It can + be configured to communicate over GRPC or HTTP. It accepts Torch Tensors + as input and returns them as outputs. + """ + + def __init__(self, url: str): + """ + Keyword arguments: + url: Fully qualified address of the Triton server - for e.g. grpc://localhost:8000 + """ + + parsed_url = urlparse(url) + if parsed_url.scheme == 'grpc': + from tritonclient.grpc import InferenceServerClient, InferInput + + self.client = InferenceServerClient(parsed_url.netloc) # Triton GRPC client + model_repository = self.client.get_model_repository_index() + self.model_name = model_repository.models[0].name + self.metadata = self.client.get_model_metadata(self.model_name, as_json=True) + + def create_input_placeholders() -> typing.List[InferInput]: + return [ + InferInput(i['name'], [int(s) for s in i['shape']], i['datatype']) for i in self.metadata['inputs']] + + else: + from tritonclient.http import InferenceServerClient, InferInput + + self.client = InferenceServerClient(parsed_url.netloc) # Triton HTTP client + model_repository = self.client.get_model_repository_index() + self.model_name = model_repository[0]['name'] + self.metadata = self.client.get_model_metadata(self.model_name) + + def create_input_placeholders() -> typing.List[InferInput]: + return [ + InferInput(i['name'], [int(s) for s in i['shape']], i['datatype']) for i in self.metadata['inputs']] + + self._create_input_placeholders_fn = create_input_placeholders + + @property + def runtime(self): + """Returns the model runtime""" + return self.metadata.get('backend', self.metadata.get('platform')) + + def __call__(self, *args, **kwargs) -> typing.Union[torch.Tensor, typing.Tuple[torch.Tensor, ...]]: + """ Invokes the model. Parameters can be provided via args or kwargs. + args, if provided, are assumed to match the order of inputs of the model. + kwargs are matched with the model input names. + """ + inputs = self._create_inputs(*args, **kwargs) + response = self.client.infer(model_name=self.model_name, inputs=inputs) + result = [] + for output in self.metadata['outputs']: + tensor = torch.as_tensor(response.as_numpy(output['name'])) + result.append(tensor) + return result[0] if len(result) == 1 else result + + def _create_inputs(self, *args, **kwargs): + args_len, kwargs_len = len(args), len(kwargs) + if not args_len and not kwargs_len: + raise RuntimeError('No inputs provided.') + if args_len and kwargs_len: + raise RuntimeError('Cannot specify args and kwargs at the same time') + + placeholders = self._create_input_placeholders_fn() + if args_len: + if args_len != len(placeholders): + raise RuntimeError(f'Expected {len(placeholders)} inputs, got {args_len}.') + for input, value in zip(placeholders, args): + input.set_data_from_numpy(value.cpu().numpy()) + else: + for input in placeholders: + value = kwargs[input.name] + input.set_data_from_numpy(value.cpu().numpy()) + return placeholders diff --git a/val.py b/val.py new file mode 100644 index 0000000..8da3ef7 --- /dev/null +++ b/val.py @@ -0,0 +1,411 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Validate a trained YOLOv5 detection model on a detection dataset + +Usage: + $ python val.py --weights yolov5s.pt --data coco128.yaml --img 640 + +Usage - formats: + $ python val.py --weights yolov5s.pt # PyTorch + yolov5s.torchscript # TorchScript + yolov5s.onnx # ONNX Runtime or OpenCV DNN with --dnn + yolov5s_openvino_model # OpenVINO + yolov5s.engine # TensorRT + yolov5s.mlmodel # CoreML (macOS-only) + yolov5s_saved_model # TensorFlow SavedModel + yolov5s.pb # TensorFlow GraphDef + yolov5s.tflite # TensorFlow Lite + yolov5s_edgetpu.tflite # TensorFlow Edge TPU + yolov5s_paddle_model # PaddlePaddle +""" + +import argparse +import json +import os +import subprocess +import sys +from pathlib import Path + +import numpy as np +import torch +from tqdm import tqdm + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[0] # YOLOv5 root directory +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) # add ROOT to PATH +ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative + +from models.common import DetectMultiBackend +from utils.callbacks import Callbacks +from utils.dataloaders import create_dataloader +from utils.general import (LOGGER, TQDM_BAR_FORMAT, Profile, check_dataset, check_img_size, check_requirements, + check_yaml, coco80_to_coco91_class, colorstr, increment_path, non_max_suppression, + print_args, scale_boxes, xywh2xyxy, xyxy2xywh) +from utils.metrics import ConfusionMatrix, ap_per_class, box_iou +from utils.plots import output_to_target, plot_images, plot_val_study +from utils.torch_utils import select_device, smart_inference_mode + + +def save_one_txt(predn, save_conf, shape, file): + # Save one txt result + gn = torch.tensor(shape)[[1, 0, 1, 0]] # normalization gain whwh + for *xyxy, conf, cls in predn.tolist(): + xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh + line = (cls, *xywh, conf) if save_conf else (cls, *xywh) # label format + with open(file, 'a') as f: + f.write(('%g ' * len(line)).rstrip() % line + '\n') + + +def save_one_json(predn, jdict, path, class_map): + # Save one JSON result {"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236} + image_id = int(path.stem) if path.stem.isnumeric() else path.stem + box = xyxy2xywh(predn[:, :4]) # xywh + box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner + for p, b in zip(predn.tolist(), box.tolist()): + jdict.append({ + 'image_id': image_id, + 'category_id': class_map[int(p[5])], + 'bbox': [round(x, 3) for x in b], + 'score': round(p[4], 5)}) + + +def process_batch(detections, labels, iouv): + """ + Return correct prediction matrix + Arguments: + detections (array[N, 6]), x1, y1, x2, y2, conf, class + labels (array[M, 5]), class, x1, y1, x2, y2 + Returns: + correct (array[N, 10]), for 10 IoU levels + """ + correct = np.zeros((detections.shape[0], iouv.shape[0])).astype(bool) + iou = box_iou(labels[:, 1:], detections[:, :4]) + correct_class = labels[:, 0:1] == detections[:, 5] + for i in range(len(iouv)): + x = torch.where((iou >= iouv[i]) & correct_class) # IoU > threshold and classes match + if x[0].shape[0]: + matches = torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1).cpu().numpy() # [label, detect, iou] + if x[0].shape[0] > 1: + matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 1], return_index=True)[1]] + # matches = matches[matches[:, 2].argsort()[::-1]] + matches = matches[np.unique(matches[:, 0], return_index=True)[1]] + correct[matches[:, 1].astype(int), i] = True + return torch.tensor(correct, dtype=torch.bool, device=iouv.device) + + +@smart_inference_mode() +def run( + data, + weights=None, # model.pt path(s) + batch_size=32, # batch size + imgsz=640, # inference size (pixels) + conf_thres=0.001, # confidence threshold + iou_thres=0.6, # NMS IoU threshold + max_det=300, # maximum detections per image + task='val', # train, val, test, speed or study + device='', # cuda device, i.e. 0 or 0,1,2,3 or cpu + workers=8, # max dataloader workers (per RANK in DDP mode) + single_cls=False, # treat as single-class dataset + augment=False, # augmented inference + verbose=False, # verbose output + save_txt=False, # save results to *.txt + save_hybrid=False, # save label+prediction hybrid results to *.txt + save_conf=False, # save confidences in --save-txt labels + save_json=False, # save a COCO-JSON results file + project=ROOT / 'runs/val', # save to project/name + name='exp', # save to project/name + exist_ok=False, # existing project/name ok, do not increment + half=True, # use FP16 half-precision inference + dnn=False, # use OpenCV DNN for ONNX inference + model=None, + dataloader=None, + save_dir=Path(''), + plots=True, + callbacks=Callbacks(), + compute_loss=None, +): + # Initialize/load model and set device + training = model is not None + if training: # called by train.py + device, pt, jit, engine = next(model.parameters()).device, True, False, False # get model device, PyTorch model + half &= device.type != 'cpu' # half precision only supported on CUDA + model.half() if half else model.float() + else: # called directly + device = select_device(device, batch_size=batch_size) + + # Directories + save_dir = increment_path(Path(project) / name, exist_ok=exist_ok) # increment run + (save_dir / 'labels' if save_txt else save_dir).mkdir(parents=True, exist_ok=True) # make dir + + # Load model + model = DetectMultiBackend(weights, device=device, dnn=dnn, data=data, fp16=half) + stride, pt, jit, engine = model.stride, model.pt, model.jit, model.engine + imgsz = check_img_size(imgsz, s=stride) # check image size + half = model.fp16 # FP16 supported on limited backends with CUDA + if engine: + batch_size = model.batch_size + else: + device = model.device + if not (pt or jit): + batch_size = 1 # export.py models default to batch-size 1 + LOGGER.info(f'Forcing --batch-size 1 square inference (1,3,{imgsz},{imgsz}) for non-PyTorch models') + + # Data + data = check_dataset(data) # check + + # Configure + model.eval() + cuda = device.type != 'cpu' + is_coco = isinstance(data.get('val'), str) and data['val'].endswith(f'coco{os.sep}val2017.txt') # COCO dataset + nc = 1 if single_cls else int(data['nc']) # number of classes + iouv = torch.linspace(0.5, 0.95, 10, device=device) # iou vector for mAP@0.5:0.95 + niou = iouv.numel() + + # Dataloader + if not training: + if pt and not single_cls: # check --weights are trained on --data + ncm = model.model.nc + assert ncm == nc, f'{weights} ({ncm} classes) trained on different --data than what you passed ({nc} ' \ + f'classes). Pass correct combination of --weights and --data that are trained together.' + model.warmup(imgsz=(1 if pt else batch_size, 3, imgsz, imgsz)) # warmup + pad, rect = (0.0, False) if task == 'speed' else (0.5, pt) # square inference for benchmarks + task = task if task in ('train', 'val', 'test') else 'val' # path to train/val/test images + dataloader = create_dataloader(data[task], + imgsz, + batch_size, + stride, + single_cls, + pad=pad, + rect=rect, + workers=workers, + prefix=colorstr(f'{task}: '))[0] + + seen = 0 + confusion_matrix = ConfusionMatrix(nc=nc) + names = model.names if hasattr(model, 'names') else model.module.names # get class names + if isinstance(names, (list, tuple)): # old format + names = dict(enumerate(names)) + class_map = coco80_to_coco91_class() if is_coco else list(range(1000)) + s = ('%22s' + '%11s' * 6) % ('Class', 'Images', 'Instances', 'P', 'R', 'mAP50', 'mAP50-95') + tp, fp, p, r, f1, mp, mr, map50, ap50, map = 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0 + dt = Profile(), Profile(), Profile() # profiling times + loss = torch.zeros(3, device=device) + jdict, stats, ap, ap_class = [], [], [], [] + callbacks.run('on_val_start') + pbar = tqdm(dataloader, desc=s, bar_format=TQDM_BAR_FORMAT) # progress bar + for batch_i, (im, targets, paths, shapes) in enumerate(pbar): + callbacks.run('on_val_batch_start') + with dt[0]: + if cuda: + im = im.to(device, non_blocking=True) + targets = targets.to(device) + im = im.half() if half else im.float() # uint8 to fp16/32 + im /= 255 # 0 - 255 to 0.0 - 1.0 + nb, _, height, width = im.shape # batch size, channels, height, width + + # Inference + with dt[1]: + preds, train_out = model(im) if compute_loss else (model(im, augment=augment), None) + + # Loss + if compute_loss: + loss += compute_loss(train_out, targets)[1] # box, obj, cls + + # NMS + targets[:, 2:] *= torch.tensor((width, height, width, height), device=device) # to pixels + lb = [targets[targets[:, 0] == i, 1:] for i in range(nb)] if save_hybrid else [] # for autolabelling + with dt[2]: + preds = non_max_suppression(preds, + conf_thres, + iou_thres, + labels=lb, + multi_label=True, + agnostic=single_cls, + max_det=max_det) + + # Metrics + for si, pred in enumerate(preds): + labels = targets[targets[:, 0] == si, 1:] + nl, npr = labels.shape[0], pred.shape[0] # number of labels, predictions + path, shape = Path(paths[si]), shapes[si][0] + correct = torch.zeros(npr, niou, dtype=torch.bool, device=device) # init + seen += 1 + + if npr == 0: + if nl: + stats.append((correct, *torch.zeros((2, 0), device=device), labels[:, 0])) + if plots: + confusion_matrix.process_batch(detections=None, labels=labels[:, 0]) + continue + + # Predictions + if single_cls: + pred[:, 5] = 0 + predn = pred.clone() + scale_boxes(im[si].shape[1:], predn[:, :4], shape, shapes[si][1]) # native-space pred + + # Evaluate + if nl: + tbox = xywh2xyxy(labels[:, 1:5]) # target boxes + scale_boxes(im[si].shape[1:], tbox, shape, shapes[si][1]) # native-space labels + labelsn = torch.cat((labels[:, 0:1], tbox), 1) # native-space labels + correct = process_batch(predn, labelsn, iouv) + if plots: + confusion_matrix.process_batch(predn, labelsn) + stats.append((correct, pred[:, 4], pred[:, 5], labels[:, 0])) # (correct, conf, pcls, tcls) + + # Save/log + if save_txt: + save_one_txt(predn, save_conf, shape, file=save_dir / 'labels' / f'{path.stem}.txt') + if save_json: + save_one_json(predn, jdict, path, class_map) # append to COCO-JSON dictionary + callbacks.run('on_val_image_end', pred, predn, path, names, im[si]) + + # Plot images + if plots and batch_i < 3: + plot_images(im, targets, paths, save_dir / f'val_batch{batch_i}_labels.jpg', names) # labels + plot_images(im, output_to_target(preds), paths, save_dir / f'val_batch{batch_i}_pred.jpg', names) # pred + + callbacks.run('on_val_batch_end', batch_i, im, targets, paths, shapes, preds) + + # Compute metrics + stats = [torch.cat(x, 0).cpu().numpy() for x in zip(*stats)] # to numpy + if len(stats) and stats[0].any(): + tp, fp, p, r, f1, ap, ap_class = ap_per_class(*stats, plot=plots, save_dir=save_dir, names=names) + ap50, ap = ap[:, 0], ap.mean(1) # AP@0.5, AP@0.5:0.95 + mp, mr, map50, map = p.mean(), r.mean(), ap50.mean(), ap.mean() + nt = np.bincount(stats[3].astype(int), minlength=nc) # number of targets per class + + # Print results + pf = '%22s' + '%11i' * 2 + '%11.3g' * 4 # print format + LOGGER.info(pf % ('all', seen, nt.sum(), mp, mr, map50, map)) + if nt.sum() == 0: + LOGGER.warning(f'WARNING ⚠️ no labels found in {task} set, can not compute metrics without labels') + + # Print results per class + if (verbose or (nc < 50 and not training)) and nc > 1 and len(stats): + for i, c in enumerate(ap_class): + LOGGER.info(pf % (names[c], seen, nt[c], p[i], r[i], ap50[i], ap[i])) + + # Print speeds + t = tuple(x.t / seen * 1E3 for x in dt) # speeds per image + if not training: + shape = (batch_size, 3, imgsz, imgsz) + LOGGER.info(f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {shape}' % t) + + # Plots + if plots: + confusion_matrix.plot(save_dir=save_dir, names=list(names.values())) + callbacks.run('on_val_end', nt, tp, fp, p, r, f1, ap, ap50, ap_class, confusion_matrix) + + # Save JSON + if save_json and len(jdict): + w = Path(weights[0] if isinstance(weights, list) else weights).stem if weights is not None else '' # weights + anno_json = str(Path('../datasets/coco/annotations/instances_val2017.json')) # annotations + if not os.path.exists(anno_json): + anno_json = os.path.join(data['path'], 'annotations', 'instances_val2017.json') + pred_json = str(save_dir / f'{w}_predictions.json') # predictions + LOGGER.info(f'\nEvaluating pycocotools mAP... saving {pred_json}...') + with open(pred_json, 'w') as f: + json.dump(jdict, f) + + try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb + check_requirements('pycocotools>=2.0.6') + from pycocotools.coco import COCO + from pycocotools.cocoeval import COCOeval + + anno = COCO(anno_json) # init annotations api + pred = anno.loadRes(pred_json) # init predictions api + eval = COCOeval(anno, pred, 'bbox') + if is_coco: + eval.params.imgIds = [int(Path(x).stem) for x in dataloader.dataset.im_files] # image IDs to evaluate + eval.evaluate() + eval.accumulate() + eval.summarize() + map, map50 = eval.stats[:2] # update results (mAP@0.5:0.95, mAP@0.5) + except Exception as e: + LOGGER.info(f'pycocotools unable to run: {e}') + + # Return results + model.float() # for training + if not training: + s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else '' + LOGGER.info(f"Results saved to {colorstr('bold', save_dir)}{s}") + maps = np.zeros(nc) + map + for i, c in enumerate(ap_class): + maps[c] = ap[i] + return (mp, mr, map50, map, *(loss.cpu() / len(dataloader)).tolist()), maps, t + + +def parse_opt(): + parser = argparse.ArgumentParser() + parser.add_argument('--data', type=str, default=ROOT / 'data/coco128.yaml', help='dataset.yaml path') + parser.add_argument('--weights', nargs='+', type=str, default=ROOT / 'yolov5s.pt', help='model path(s)') + parser.add_argument('--batch-size', type=int, default=32, help='batch size') + parser.add_argument('--imgsz', '--img', '--img-size', type=int, default=640, help='inference size (pixels)') + parser.add_argument('--conf-thres', type=float, default=0.001, help='confidence threshold') + parser.add_argument('--iou-thres', type=float, default=0.6, help='NMS IoU threshold') + parser.add_argument('--max-det', type=int, default=300, help='maximum detections per image') + parser.add_argument('--task', default='val', help='train, val, test, speed or study') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--workers', type=int, default=8, help='max dataloader workers (per RANK in DDP mode)') + parser.add_argument('--single-cls', action='store_true', help='treat as single-class dataset') + parser.add_argument('--augment', action='store_true', help='augmented inference') + parser.add_argument('--verbose', action='store_true', help='report mAP by class') + parser.add_argument('--save-txt', action='store_true', help='save results to *.txt') + parser.add_argument('--save-hybrid', action='store_true', help='save label+prediction hybrid results to *.txt') + parser.add_argument('--save-conf', action='store_true', help='save confidences in --save-txt labels') + parser.add_argument('--save-json', action='store_true', help='save a COCO-JSON results file') + parser.add_argument('--project', default=ROOT / 'runs/val', help='save to project/name') + parser.add_argument('--name', default='exp', help='save to project/name') + parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment') + parser.add_argument('--half', action='store_true', help='use FP16 half-precision inference') + parser.add_argument('--dnn', action='store_true', help='use OpenCV DNN for ONNX inference') + opt = parser.parse_args() + opt.data = check_yaml(opt.data) # check YAML + opt.save_json |= opt.data.endswith('coco.yaml') + opt.save_txt |= opt.save_hybrid + print_args(vars(opt)) + return opt + + +def main(opt): + check_requirements(ROOT / 'requirements.txt', exclude=('tensorboard', 'thop')) + + if opt.task in ('train', 'val', 'test'): # run normally + if opt.conf_thres > 0.001: # https://github.com/ultralytics/yolov5/issues/1466 + LOGGER.info(f'WARNING ⚠️ confidence threshold {opt.conf_thres} > 0.001 produces invalid results') + if opt.save_hybrid: + LOGGER.info('WARNING ⚠️ --save-hybrid will return high mAP from hybrid labels, not from predictions alone') + run(**vars(opt)) + + else: + weights = opt.weights if isinstance(opt.weights, list) else [opt.weights] + opt.half = torch.cuda.is_available() and opt.device != 'cpu' # FP16 for fastest results + if opt.task == 'speed': # speed benchmarks + # python val.py --task speed --data coco.yaml --batch 1 --weights yolov5n.pt yolov5s.pt... + opt.conf_thres, opt.iou_thres, opt.save_json = 0.25, 0.45, False + for opt.weights in weights: + run(**vars(opt), plots=False) + + elif opt.task == 'study': # speed vs mAP benchmarks + # python val.py --task study --data coco.yaml --iou 0.7 --weights yolov5n.pt yolov5s.pt... + for opt.weights in weights: + f = f'study_{Path(opt.data).stem}_{Path(opt.weights).stem}.txt' # filename to save to + x, y = list(range(256, 1536 + 128, 128)), [] # x axis (image sizes), y axis + for opt.imgsz in x: # img-size + LOGGER.info(f'\nRunning {f} --imgsz {opt.imgsz}...') + r, _, t = run(**vars(opt), plots=False) + y.append(r + t) # results and times + np.savetxt(f, y, fmt='%10.4g') # save + subprocess.run(['zip', '-r', 'study.zip', 'study_*.txt']) + plot_val_study(x=x) # plot + else: + raise NotImplementedError(f'--task {opt.task} not in ("train", "val", "test", "speed", "study")') + + +if __name__ == '__main__': + opt = parse_opt() + main(opt) diff --git a/videos_select.py b/videos_select.py new file mode 100644 index 0000000..d1f22e9 --- /dev/null +++ b/videos_select.py @@ -0,0 +1,641 @@ +# YOLOv5 🚀 by Ultralytics, AGPL-3.0 license +""" +Run YOLOv5 detection inference on images, videos, directories, globs, YouTube, webcam, streams, etc. + +Usage - sources: + $ python detect.py --weights yolov5s.pt --source 0 # webcam + img.jpg # image + vid.mp4 # video + screen # screenshot + path/ # directory + list.txt # list of images + list.streams # list of streams + 'path/*.jpg' # glob + 'https://youtu.be/Zgi9g1ksQHc' # YouTube + 'rtsp://example.com/media.mp4' # RTSP, RTMP, HTTP stream + +Usage - formats: + $ python detect.py --weights yolov5s.pt # PyTorch + yolov5s.torchscript # TorchScript + yolov5s.onnx # ONNX Runtime or OpenCV DNN with --dnn + yolov5s_openvino_model # OpenVINO + yolov5s.engine # TensorRT + yolov5s.mlmodel # CoreML (macOS-only) + yolov5s_saved_model # TensorFlow SavedModel + yolov5s.pb # TensorFlow GraphDef + yolov5s.tflite # TensorFlow Lite + yolov5s_edgetpu.tflite # TensorFlow Edge TPU + yolov5s_paddle_model # PaddlePaddle +""" + +import argparse +import csv +import os +import platform +import sys +from pathlib import Path +import glob +import numpy as np +import pickle +import torch + +FILE = Path(__file__).resolve() +ROOT = FILE.parents[0] # YOLOv5 root directory +if str(ROOT) not in sys.path: + sys.path.append(str(ROOT)) # add ROOT to PATH +sys.path.append((str(ROOT)+'\\tracking\\utils')) + +ROOT = Path(os.path.relpath(ROOT, Path.cwd())) # relative + + +from models.common import DetectMultiBackend +from utils.dataloaders import IMG_FORMATS, VID_FORMATS, LoadImages, LoadScreenshots, LoadStreams +from utils.general import (LOGGER, Profile, check_file, check_img_size, check_imshow, check_requirements, colorstr, cv2, + increment_path, non_max_suppression, print_args, scale_boxes, strip_optimizer, xyxy2xywh) +from utils.torch_utils import select_device, smart_inference_mode + +'''集成跟踪模块，输出跟踪结果文件 .npy''' +# from ultralytics.engine.results import Boxes # Results +# from ultralytics.utils import IterableSimpleNamespace, yaml_load +from tracking.utils.plotting import Annotator, colors +from tracking.utils import Boxes, IterableSimpleNamespace, yaml_load, boxes_add_fid +from tracking.trackers import BOTSORT, BYTETracker +from tracking.utils.showtrack import drawtracks +from hands.hand_inference import hand_pose +from tracking.dotrack.dotracks_back import doBackTracks +from tracking.dotrack.dotracks_front import doFrontTracks + +from tracking.trackers.reid.reid_interface import ReIDInterface +from tracking.trackers.reid.config import config as ReIDConfig +ReIDEncoder = ReIDInterface(ReIDConfig) + +# tracker_yaml = r"./tracking/trackers/cfg/botsort.yaml" + +def inference_image(image, detections): + H, W, _ = np.shape(image) + imgs = [] + batch_patches = [] + patches = [] + for d in range(np.size(detections, 0)): + tlbr = detections[d, :4].astype(np.int_) + tlbr[0] = max(0, tlbr[0]) + tlbr[1] = max(0, tlbr[1]) + tlbr[2] = min(W - 1, tlbr[2]) + tlbr[3] = min(H - 1, tlbr[3]) + img1 = image[tlbr[1]:tlbr[3], tlbr[0]:tlbr[2], :] + + img = img1[:, :, ::-1].copy() # the model expects RGB inputs + patch = ReIDEncoder.transform(img) + + imgs.append(img1) + # patch = patch.to(device=self.device).half() + if str(ReIDEncoder.device) != "cpu": + patch = patch.to(device=ReIDEncoder.device).half() + else: + patch = patch.to(device=ReIDEncoder.device) + + patches.append(patch) + if (d + 1) % ReIDEncoder.batch_size == 0: + patches = torch.stack(patches, dim=0) + batch_patches.append(patches) + patches = [] + + if len(patches): + patches = torch.stack(patches, dim=0) + batch_patches.append(patches) + + features = np.zeros((0, ReIDEncoder.embedding_size)) + for patches in batch_patches: + pred = ReIDEncoder.model(patches) + pred[torch.isinf(pred)] = 1.0 + feat = pred.cpu().data.numpy() + features = np.vstack((features, feat)) + + return imgs, features + + + +def init_trackers(tracker_yaml = None, bs=1): + """ + Initialize trackers for object tracking during prediction. + """ + # tracker_yaml = r"./tracking/trackers/cfg/botsort.yaml" + + TRACKER_MAP = {'bytetrack': BYTETracker, 'botsort': BOTSORT} + + cfg = IterableSimpleNamespace(**yaml_load(tracker_yaml)) + trackers = [] + for _ in range(bs): + tracker = TRACKER_MAP[cfg.tracker_type](args=cfg, frame_rate=30) + trackers.append(tracker) + + return trackers + +def detect_start_end(bboxes, features_dict, filename): + + boxes = np.empty(shape=(0, 9), dtype = np.float64) + if filename.find("back") >= 0: + vts = doBackTracks(bboxes, features_dict) + vts.classify() + # vtx = [t for t in vts.tracks if t.cls != 0] + # for track in vtx: + for track in vts.Residual: + if track.moving_index.size: + boxes = np.concatenate((boxes, track.moving_index), axis=0) + + elif filename.find("front") >= 0: + vts = doFrontTracks(bboxes, features_dict) + vts.classify() + # vtx = [t for t in vts.tracks if t.cls != 0] + # for track in vtx: + for track in vts.Residual: + for start, end in track.dynamic_y2: + boxes = np.concatenate((boxes, track.boxes[start:end+1, :]), axis=0) + for start, end in track.dynamic_y1: + boxes = np.concatenate((boxes, track.boxes[start:end+1, :]), axis=0) + + if boxes.size > 0: + start = np.min(boxes[:, 7]) + end = np.max(boxes[:, 7]) + else: + start, end = 0, 0 + + return start, end + +@smart_inference_mode() +def run( + weights=ROOT / 'yolov5s.pt', # model path or triton URL + source=ROOT / 'data/images', # file/dir/URL/glob/screen/0(webcam) + + project=ROOT / 'runs/detect', # save results to project/name + name='exp', # save results to project/name + + tracker_yaml = "./tracking/trackers/cfg/botsort.yaml", + imgsz=(640, 640), # inference size (height, width) + conf_thres=0.25, # confidence threshold + iou_thres=0.45, # NMS IOU threshold + max_det=1000, # maximum detections per image + device='', # cuda device, i.e. 0 or 0,1,2,3 or cpu + view_img=False, # show results + save_txt=False, # save results to *.txt + save_csv=False, # save results in CSV format + save_conf=False, # save confidences in --save-txt labels + save_crop=False, # save cropped prediction boxes + nosave=False, # do not save images/videos + classes=None, # filter by class: --class 0, or --class 0 2 3 + agnostic_nms=False, # class-agnostic NMS + augment=False, # augmented inference + visualize=False, # visualize features + update=False, # update all models + exist_ok=False, # existing project/name ok, do not increment + line_thickness=3, # bounding box thickness (pixels) + hide_labels=False, # hide labels + hide_conf=False, # hide confidencesL + half=False, # use FP16 half-precision inference + dnn=False, # use OpenCV DNN for ONNX inference + vid_stride=1, # video frame-rate stride + data=ROOT / 'data/coco128.yaml', # dataset.yaml path +): + source = str(source) + # filename = os.path.split(source)[-1] + + save_img = not nosave and not source.endswith('.txt') # save inference images + is_file = Path(source).suffix[1:] in (IMG_FORMATS + VID_FORMATS) + is_url = source.lower().startswith(('rtsp://', 'rtmp://', 'http://', 'https://')) + webcam = source.isnumeric() or source.endswith('.streams') or (is_url and not is_file) + screenshot = source.lower().startswith('screen') + if is_url and is_file: + source = check_file(source) # download + + save_dir = Path(project) / Path(source).stem + if save_dir.exists(): + print(Path(source).stem) + # return + + save_dir = increment_path(Path(project) / name, exist_ok=exist_ok) # increment run + (save_dir / 'labels' if save_txt else save_dir).mkdir(parents=True, exist_ok=True) # make dir + else: + save_dir.mkdir(parents=True, exist_ok=True) + + # Load model + device = select_device(device) + model = DetectMultiBackend(weights, device=device, dnn=dnn, data=data, fp16=half) + stride, names, pt = model.stride, model.names, model.pt + imgsz = check_img_size(imgsz, s=stride) # check image size + + # Dataloader + bs = 1 # batch_size + + dataset = LoadImages(source, img_size=imgsz, stride=stride, auto=pt, vid_stride=vid_stride) + vid_path, vid_writer = [None] * bs, [None] * bs + + + # Run inference + model.warmup(imgsz=(1 if pt or model.triton else bs, 3, *imgsz)) # warmup + seen, dt = 0, (Profile(), Profile(), Profile()) + + tracker = init_trackers(tracker_yaml, bs)[0] + + handpose = hand_pose() + handlocals_dict = {} + + boxes_and_imgs = [] + + BoxesFeats = [] + + + + track_boxes = np.empty((0, 9), dtype = np.float32) + det_boxes = np.empty((0, 9), dtype = np.float32) + + DetBoxes = np.empty((0, 6), dtype = np.float32) + TrackerBoxes = np.empty((0, 9), dtype = np.float32) + TrackerFeats = np.empty((0, 256), dtype = np.float32) + + images = [] + + features_dict = {} + TracksDict = {} + for path, im, im0s, vid_cap, s in dataset: + if save_img and 'imgshow' not in locals().keys(): + imgshow = im0s.copy() + + ## ============================= tracking 功能只处理视频，writed by WQG + if dataset.mode == 'image': + continue + + with dt[0]: + im = torch.from_numpy(im).to(model.device) + im = im.half() if model.fp16 else im.float() # uint8 to fp16/32 + im /= 255 # 0 - 255 to 0.0 - 1.0 + if len(im.shape) == 3: + im = im[None] # expand for batch dim + + # Inference + with dt[1]: + visualize = increment_path(save_dir / Path(path).stem, mkdir=True) if visualize else False + pred = model(im, augment=augment, visualize=visualize) + + # NMS + with dt[2]: + pred = non_max_suppression(pred, conf_thres, iou_thres, classes, agnostic_nms, max_det=max_det) + + # Process predictions + for i, det in enumerate(pred): # per image + seen += 1 + if webcam: # batch_size >= 1 + p, im0, frame = path[i], im0s[i].copy(), dataset.count + s += f'{i}: ' + else: + p, im0, frame = path, im0s.copy(), getattr(dataset, 'frame', 0) + + images.append((dataset.frame, im0)) + + p = Path(p) # to Path + save_path = str(save_dir / p.name) # im.jpg + s += '%gx%g ' % im.shape[2:] # print string + + # im0_ant = im0.copy() + annotator = Annotator(im0.copy(), line_width=line_thickness, example=str(names)) + + nd = len(det) + if nd: + # Rescale boxes from img_size to im0 size + det[:, :4] = scale_boxes(im.shape[2:], det[:, :4], im0.shape).round() + + det = det.cpu().numpy() + det = np.concatenate([det[:, :4], np.arange(nd).reshape(-1, 1), det[:, 4:]], axis=-1) + + DetBoxes = np.concatenate([DetBoxes, det[:, :6]], axis=0) + + ## ============================================================ 前后帧相同 boxes 的特征赋值 + # def static_estimate(box1, box2, TH1=8, TH2=12): + # dij_abs = max(np.abs(box1 - box2)) + # dij_euc = max([np.linalg.norm((box1[:2] - box2[:2])), + # np.linalg.norm((box1[2:4] - box2[2:4])) + # ]) + # if dij_abs < TH1 and dij_euc < TH2: + # return True + # else: + # return False + + # nw = 3 # 向前递推检查的窗口大小 + # nf = len(BoxesFeats) # 已经检测+特征提取的帧数 + # feat_curr = [None] * nd # nd: 当前帧检测出的boxes数 + # for ii in range(nd): + # box = det[ii, :4] + + # kk=1 + # feat = None + # while kk <= nw and nf>=kk: + # ki = -1 * kk + # boxes_ = BoxesFeats[ki][0] + # feats_ = BoxesFeats[ki][1] + + # flag = [jj for jj in range(len(boxes_)) if static_estimate(box, boxes_[jj, :4])] + # if len(flag) == 1: + # feat = feats_[flag[0]] + # break + # kk += 1 + # if feat is not None: + # feat_curr[ii] = feat + + + + + ## ================================================================ writed by WQG + + '''tracks: [x1, y1, x2, y2, track_id, score, cls, frame_index, box_index] + 0 1 2 3 4 5 6 7 8 + 这里，frame_index 也可以用视频的 帧ID 代替, box_index 保持不变 + ''' + + det_tracking = Boxes(det, im0.shape).cpu().numpy() + tracks = tracker.update(det_tracking, im0) + if len(tracks) == 0: + continue + tracks[:, 7] = dataset.frame + + '''================== 1. 存储 dets/subimgs/features Dict =============''' + imgs, features = inference_image(im0, tracks) + + + TrackerFeats = np.concatenate([TrackerFeats, features], axis=0) + + + + + + imgdict = {} + boxdict = {} + featdict = {} + for ii, bid in enumerate(tracks[:, 8]): + imgdict.update({int(bid): imgs[ii]}) # [f"img_{int(bid)}"] = imgs[i] + boxdict.update({int(bid): tracks[ii, :]}) # [f"box_{int(bid)}"] = tracks[i, :] + featdict.update({int(bid): features[ii, :]}) # [f"feat_{int(bid)}"] = features[i, :] + TracksDict[f"frame_{int(dataset.frame)}"] = {"imgs":imgdict, "boxes":boxdict, "feats":featdict} + + track_boxes = np.concatenate([track_boxes, tracks], axis=0) + + + '''================== 2. 提取手势位置 ===================''' + # idx_0 = tracks[:, 6].astype(np.int_) == 0 + # hn = 0 + # for j, index in enumerate(idx_0): + # if index: + # track = tracks[j, :] + # hand_local, imgshow = handpose.get_hand_local(track, im0) + # handlocals_dict.update({int(track[7]): {int(track[8]): hand_local}}) + + # # '''yoloV5和手势检测的召回率并不一直，用hand_local代替tracks中手部的(x1, y1, x2, y2)，会使得两种坐标方式混淆''' + # # if hand_local: tracks[j, :4] = hand_local + + # hn += 1 + # cv2.imwrite(f"D:\DeepLearning\yolov5\hands\images\{Path(source).stem}_{int(track[7])}_{hn}.png", imgshow) + + for *xyxy, id, conf, cls, fid, bid in reversed(tracks): + name = ('' if id==-1 else f'id:{int(id)} ') + names[int(cls)] + label = None if hide_labels else (name if hide_conf else f'{name} {conf:.2f}') + + if id >=0 and cls==0: + color = colors(int(cls), True) + elif id >=0 and cls!=0: + color = colors(int(id), True) + else: + color = colors(19, True) # 19为调色板的最后一个元素 + + annotator.box_label(xyxy, label, color=color) + + # Save results (image and video with tracking) + im0 = annotator.result() + save_path_img, ext = os.path.splitext(save_path) + if save_img: + if dataset.mode == 'image': + imgpath = save_path_img + f"_{dataset}.png" + else: + imgpath = save_path_img + f"_{dataset.frame}.png" + + cv2.imwrite(Path(imgpath), im0) + + if vid_path[i] != save_path: # new video + vid_path[i] = save_path + if isinstance(vid_writer[i], cv2.VideoWriter): + vid_writer[i].release() # release previous video writer + if vid_cap: # video + fps = vid_cap.get(cv2.CAP_PROP_FPS) + w = int(vid_cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + h = int(vid_cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + else: # stream + fps, w, h = 30, im0.shape[1], im0.shape[0] + save_path = str(Path(save_path).with_suffix('.mp4')) # force *.mp4 suffix on results videos + vid_writer[i] = cv2.VideoWriter(save_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (w, h)) + vid_writer[i].write(im0) + + # Print time (inference-only) + LOGGER.info(f"{s}{'' if len(det) else '(no detections), '}{dt[1].dt * 1E3:.1f}ms") + + + ## ======================================================================== written by WQG + ## track_boxes: Array, [x1, y1, x2, y2, track_id, score, cls, frame_index, box_id] + + '''上面保存了检测结果是视频和图像，以下还保存五种类型的数据''' + filename = os.path.split(save_path_img)[-1] + file, ext = os.path.splitext(filename) + + mfid = np.max(track_boxes[:, 7]) + start, end = detect_start_end(track_boxes, features_dict, filename) + + + if start == end: + return + + if start > 5: + start = start - 5 + else: + start = 0 + + if mfid - end > 5: + end = end + 5 + + + # img_path = Path(f'./runs/images/{file}/') + + img_path = savepath / f'{file}' + + if not img_path.exists(): + img_path.mkdir(parents=True, exist_ok=True) + + '''抽帧间隔数''' + Interval = 3 + for i, img in images: + if i >= start and i % Interval == 0: + imgpath = img_path / f'{file}_{int(i)}.png' + cv2.imwrite(Path(imgpath), img) + if i == end: + break + + + '''======================== 3. save hand_local data ==================''' + # handlocal_dir = Path('./tracking/data/handlocal/') + # if not handlocal_dir.exists(): + # handlocal_dir.mkdir(parents=True, exist_ok=True) + # handlocal_path = handlocal_dir.joinpath(f'{filename}.pkl') + # with open(handlocal_path, 'wb') as file: + # pickle.dump(handlocals_dict, file) + + + # Print results + t = tuple(x.t / seen * 1E3 for x in dt) # speeds per image + LOGGER.info(f'Speed: %.1fms pre-process, %.1fms inference, %.1fms NMS per image at shape {(1, 3, *imgsz)}' % t) + if save_txt or save_img: + s = f"\n{len(list(save_dir.glob('labels/*.txt')))} labels saved to {save_dir / 'labels'}" if save_txt else '' + LOGGER.info(f"Results saved to {colorstr('bold', save_dir)}{s}") + if update: + strip_optimizer(weights[0]) # update model (to fix SourceChangeWarning) + + +def parse_opt(): + modelpath = ROOT / 'ckpts/best_yolov5m_250000.pt' # 'ckpts/best_15000_0908.pt', 'ckpts/yolov5s.pt', 'ckpts/best_20000_cls30.pt' + + '''datapath为视频文件目录或视频文件''' + datapath = r"D:/datasets/ym/videos/标记视频/" # ROOT/'data/videos', ROOT/'data/images' images + # datapath = r"D:\datasets\ym\highvalue\videos" + # datapath = r"D:/dcheng/videos/" + # modelpath = ROOT / 'ckpts/yolov5s.pt' + + parser = argparse.ArgumentParser() + parser.add_argument('--weights', nargs='+', type=str, default=modelpath, help='model path or triton URL') # 'yolov5s.pt', best_15000_0908.pt + parser.add_argument('--source', type=str, default=datapath, help='file/dir/URL/glob/screen/0(webcam)') # images, videos + parser.add_argument('--data', type=str, default=ROOT / 'data/coco128.yaml', help='(optional) dataset.yaml path') + parser.add_argument('--imgsz', '--img', '--img-size', nargs='+', type=int, default=[640], help='inference size h,w') + parser.add_argument('--conf-thres', type=float, default=0.25, help='confidence threshold') + parser.add_argument('--iou-thres', type=float, default=0.45, help='NMS IoU threshold') + parser.add_argument('--max-det', type=int, default=1000, help='maximum detections per image') + parser.add_argument('--device', default='', help='cuda device, i.e. 0 or 0,1,2,3 or cpu') + parser.add_argument('--view-img', action='store_true', help='show results') + parser.add_argument('--save-txt', action='store_true', help='save results to *.txt') + parser.add_argument('--save-csv', action='store_true', help='save results in CSV format') + parser.add_argument('--save-conf', action='store_true', help='save confidences in --save-txt labels') + parser.add_argument('--save-crop', action='store_true', help='save cropped prediction boxes') + parser.add_argument('--nosave', action='store_true', help='do not save images/videos') + parser.add_argument('--classes', nargs='+', type=int, help='filter by class: --classes 0, or --classes 0 2 3') + parser.add_argument('--agnostic-nms', action='store_true', help='class-agnostic NMS') + parser.add_argument('--augment', action='store_true', help='augmented inference') + parser.add_argument('--visualize', action='store_true', help='visualize features') + parser.add_argument('--update', action='store_true', help='update all models') + parser.add_argument('--project', default=ROOT / 'runs/detect', help='save results to project/name') + parser.add_argument('--name', default='exp', help='save results to project/name') + parser.add_argument('--exist-ok', action='store_true', help='existing project/name ok, do not increment') + parser.add_argument('--line-thickness', default=3, type=int, help='bounding box thickness (pixels)') + parser.add_argument('--hide-labels', default=False, action='store_true', help='hide labels') + parser.add_argument('--hide-conf', default=False, action='store_true', help='hide confidences') + parser.add_argument('--half', action='store_true', help='use FP16 half-precision inference') + parser.add_argument('--dnn', action='store_true', help='use OpenCV DNN for ONNX inference') + parser.add_argument('--vid-stride', type=int, default=1, help='video frame-rate stride') + opt = parser.parse_args() + opt.imgsz *= 2 if len(opt.imgsz) == 1 else 1 # expand + print_args(vars(opt)) + return opt + +def find_files_in_nested_dirs(root_dir): + all_files = [] + extensions = ['.mp4'] + for dirpath, dirnames, filenames in os.walk(root_dir): + for filename in filenames: + file, ext = os.path.splitext(filename) + if ext in extensions: + all_files.append(os.path.join(dirpath, filename)) + return all_files + +print('=======') + +def main(opt): + check_requirements(ROOT / 'requirements.txt', exclude=('tensorboard', 'thop')) + + p = r"D:\datasets\ym\永辉测试数据_202404\20240402" + + optdict = vars(opt) + files = [] + k = 0 + if os.path.isdir(p): + files.extend(sorted(glob.glob(os.path.join(p, '*.*')))) + for file in files: + optdict["source"] = file + run(**optdict) + + k += 1 + if k == 2: + break + elif os.path.isfile(p): + run(**vars(opt)) + +def main_loop(opt): + check_requirements(ROOT / 'requirements.txt', exclude=('tensorboard', 'thop')) + + optdict = vars(opt) + + # p = r"D:\datasets\ym\永辉测试数据_比对" + p = r"D:\datasets\ym\广告板遮挡测试\8" + # p = r"D:\datasets\ym\videos\标记视频" + # p = r"D:\datasets\ym\实验室测试" + # p = r"D:\datasets\ym\永辉双摄视频\新建文件夹" + + k = 0 + if os.path.isdir(p): + files = find_files_in_nested_dirs(p) + + # files = [r"D:\datasets\ym\广告板遮挡测试\8\6926636301004_20240508-175300_back_addGood_70f754088050_215_17327712807.mp4", + # r"D:\datasets\ym\videos\标记视频\test_20240402-173935_6920152400975_back_174037372.mp4", + # r"D:\datasets\ym\videos\标记视频\test_20240402-173935_6920152400975_front_174037379.mp4", + # r"D:\datasets\ym\广告板遮挡测试\8\2500441577966_20240508-175946_front_addGood_70f75407b7ae_155_17788571404.mp4" + # ] + + files = [r"D:\datasets\ym\广告板遮挡测试\8\6907149227609_20240508-174733_back_returnGood_70f754088050_425_17327712807.mp4"] + + + for file in files: + optdict["source"] = file + run(**optdict) + + k += 1 + if k == 1: + break + elif os.path.isfile(p): + optdict["source"] = p + run(**vars(opt)) + + + + +if __name__ == '__main__': + opt = parse_opt() + # main(opt) + savepath = ROOT / 'runs/images' + main_loop(opt) + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/说明文档.txt b/说明文档.txt new file mode 100644 index 0000000..689676f --- /dev/null +++ b/说明文档.txt @@ -0,0 +1,390 @@ +五个功能模块 +1. Yolo + Tracker + Resnet, 其中 Resnet 的实现在./contrast中 + track_reid.py + +2. 轨迹分析模块，目录为：./tracking + (1) 基于模块（Yolo + Tracker + Resnet）的输出 + tracking_test.py + + (2) 基于测试过程数据（track.data, tracking_output.data）的输出 + module_analysis.py + +3. 比对分析模块，目录为：./contrast + 2个场景：1:1，1:n + 1:1场景： + (1) 利用现场保存数据进行比对 + OneToOneCompare.txt（现场保存数据） + one2one_onsite.py + (2) 利用本地算法进行特征提取 + one2one_contrast.py + 1:n场景： + (1) 直接利用 deletedBarcode.txt 中数据 + one2n_contrast.py + (2) 构造取出、放入事件，设计不同的特征， + feat_select.py + + +4. 整体流程仿真 + pipeline.py + 需要指定是个变量： + (1) evtdir: 保存各个事件的文件夹 + (2) SourceType: "image", "video", yolo+resent+tracker模块输入数据类型 + (3) savepath: 结果数据保存位置 + (4) weights: YoloV5 模型权重 + + savepath + (1) Yolos_Tracking + eventname文件夹，保存yolo+resent+tracker模块输出，包括： + (1) images 或 {videoname} + (2) subimgs + (3) trajectory + (4) front_y2.png + (5) trajectory.png + (2) ShoppingDict_pkfile + 保存 ShoppingDict 字典的 pickle 文件，可用于构造 ShoppingEvent 对象。 + + (3) contrast + 保存1:1、1:SN、1:n数据 + + (4) error_events.txt + + 事件字典结构： + ShoppingDict = {"eventPath": eventpath, + "eventName": evtname, + "barcode": barcode, + "eventType": '', # "input", "output", "other" + "frontCamera": {}, + "backCamera": {}, + "one2n": [] + } + + 相机字典： + CameraEvent = {"cameraType": '', # "front", "back" + "videoPath": '', + "imagePaths": [], + "yoloResnetTracker": [], + "tracking": [], + } + + yoloResnetTracker，列表，元素 "frameDict" 为有关帧数据的字典结构 + frameDict = {"path": path, + "fid": int(frameId), + "bboxes": det, + "tboxes": tracks, + "imgs": imgdict, + "feats": featdict} + + + + + + +5. 全实时时间切分仿真分析 + time_devide.py + 需分 2 步运行模块: + (1) runyolo() + + 该模块调用 imgs_inference.py 中模块 run_yolo, 该模块重新定义了类 LoadImages, 对图像进行了旋转。 + 后续工作： + 1). 将run_yolo模块与track_redi.yolo_resnet_tracker模块合并 + 2). 图像文件名标准化 + + (2) show_seri() + 该模块调用 event_time_specify.py 中模块 devide_motion_state + + +具体实现： +./tracking + tracking_test.py + have_tracked(): + 轨迹分析测试。遍历track_reid.py输出的文件夹trackdict下的所有.pkl文件。 + + time_test.py + 统计Pipeline整体流程中各模块耗时 + + module_analysis.py + 该模块中需要借助 try...except... 捕获data文件中的异常 + main(): + 遍历文件夹下的每一个子文件夹，对子文件夹执行tracking_simulate() 函数； + + main_loop()： + (1) 根据 deletedBarcode.txt 生成事件对，并利用事件对生成存储地址 + (2) 调用 tracking_simulate() 函数 + + tracking_simulate(eventpath, savepath)： + (1) 根据event_names获取事件名enent_name + (2) 遍历并执行 eventpath 文件夹下的 0_track.data、1_track.data 文件，并调用do_tracking() 执行 + (3) 将前后摄、本地与现场，工8幅子图合并为1幅大图。 + 上下子图分别显示的是前后摄，每一行4个子图，分别为： + (a) tracker输出原始轨迹; + (b) 本地tracking输出; + (c) 现场算法轨迹选择前轨迹; + (d) 现场算法轨迹选择后的轨迹 + + + + do_tracking(fpath, savedir, event_name) + inputs： + fpath: 0/1_track.data文件，并核验是否存在 0/1_tracking_output.data，若不存在该文件，直接返回 None, None + + savedir: 在该文件夹下会建立3个子文件夹及一个png轨迹图 + ./savedir/event_name_images, 画boxes后的图像序列 + ./savedir/event_name_subimgs, 轨迹分析后的目标子图，无 _x 为PC机结果，有 _x 为平板单摄像头轨迹选择后的输出 + ./savedir/trajectory, 存储每一个轨迹图 + ./savedir/event_name_tracking_select.png 或 enent_name_tracking.png 或 enent_name_select.png + + outputs: + img_tracking：本机tracker、tracking 输出的结果比较图 + abimg: 部署算法的tracking、轨迹选择分析比较图 + + + ./utils/read_data.py + 0/1_track.data 文件保存：yolo、Resnet、tracker、tracking模块的输出 + 函数: extract_data(datapath) + 异常排除： + (1) assert len(boxes)==len(feats)，确保每一帧内boxes数和feats数相等 + (2) assert(len(bboxes)==len(ffeats)), 确保关于bboxes的帧数和关于ffeats的帧数相等 + (3) assert(len(trackerboxes)==len(trackerfeats))，确保tracker输出的boxes可以对应到相应的feats上 + 这里未对 len(box)!=9、len(feat)!=256, 的情况做出约束 + 输出： + bboxes + ffeats + trackerboxes + tracker_feat_dict[f"frame_{fid}"]["feats"]{{bid}: (256,) + } + trackingboxes + tracking_feat_dict[f"track_{tid}"]["feats"]{f"{fid}_{bid}": tracker_feat_dict[f"frame_{fid}"]["feats"][bid]}) + + + + 0/1_tracking_output.data 文件保存用于比对的boxes、features + 函数: read_tracking_output(filepath) + 异常排除： + (1) assert len(feats)==len(boxes) + (2) box.size==9、feat.size=256 + + + + ./deprecated/contrast_one2one.py + 1:1 比对评估。have Deprecated! + ./enentmatch.py + 1:n 模拟测试，have Deprecated! + + +./contrast + seqfeat_compare.py + similarity_compare_sequence(root_dir) + inputs: + root_dir：文件夹，包含"subimgs"字段，对该文件夹中的相邻图像进行相似度比较 + silimarity_compare() + 功能：对imgpaths文件夹中的图像进行相似度比较 + select_subimgs.py + 为吴华琦实现的样本选择程序 + + + event_test.py + calc_simil(event, stdfeat) + 事件与标准库的对比策略，该函数在one2one_contrast.py中调用 + calsiml(feat1, feat2, topkp=75, cluth=0.15): + 功能：计算2个样本集的相似度 + （1）(feat1, feat2)，对于单样本，选择 topkp 的相似度，并求均值，得到单样本对另一个特征集的相似度； + （2）simi_max：样本集 feat1 对样本集 feat2 的相似度，是一个向量 + （3）对 simi_max 进行聚类，阈值为 cluth，选取值最大的聚类中心，作为2个样本集相似度的度量 + 前后摄轨迹选择： + 情况1：前后摄轨迹（与标准集的）相似度的差 > 阈值，表明前后摄中有一个轨迹与标准集相差较远，丢弃相应轨迹，保留具有较大相似度的轨迹 + 情况2：前后摄轨迹（与标准集的）相似度的差 >=阈值，表明前后摄轨迹与标准集均相似，求轨迹相似度均值 + + + + + + + + + input_getout_compare.py + creatd_deletedBarcode_front(filepath) + (1) 基于 deletedBarcode.txt, 构造取出事件和相应的放入事件，构成列表并更新这些列表。 + MatchList = [(getout_event, InputList), ...] + + (2) 设计不同的特征选择方式，计算 getout 事件和各 input 事件的相似度，并保存于文件： + deletedBarcodeTest.txt + + + precision_compare(filepath, savepath) + 读取 deletedBarcode.txt 和 deletedBarcodeTest.txt 中的数据，进行相似度比较 + + stdfeat_analys()： + feat_analysis(featpath): 标准特征集中样本类内、类间相似度分布 + + feat_infer.py + 简单的subimg特征提取 + + + + genfeats.py + get_std_barcodeDict(bcdpath, savepath) + 功能: 生成并保存只有一个key值的字典 {barcode: [imgpath1, imgpath1, ...]} + + stdfeat_infer(imgPath, featPath, bcdSet=None) + 功能: 对 imgPath 中图像进行特征提取，生成只有一个key值的字典。 + {barcode: features}，features.shape=(nsample, 256)，并保存至 featPath 中 + + + onsite_contrast_pr.py + one2one_pr() + 现场试验输出数据的性能评估，输出1:1, 1:SN, 1:n的PR曲线和相似度直方图分布。 + + + test_compare() + 适用于202410前数据保存版本的，需调用 OneToOneCompare.txt + + + one2one_contrast.py + 已修改，未更新。 + 共6个地址： + (1) stdSamplePath: 用于生成比对标准特征集的原始图像地址 + (2) stdBarcodePath: 比对标准特征集原始图像地址的pickle文件存储，{barcode: [imgpath1, imgpath1, ...]} + (3) stdFeaturePath: 比对标准特征集特征存储地址 + (4) eventFeatPath: 用于1:1比对的购物事件特征存储地址、对应子图存储地址 + (5) subimgPath: 1:1比对购物事件轨迹、标准barcode所对应的 subimgs 存储地址 + (6) resultPath: 1:1比对结果存储地址 + + (1), (2), (3): 保存标准特征集向量，只需运行一次 + (4): 保存测试的事件字典，只需运行一次 + + + test_one2one() + 1:1比对 + (1) 初始化事件和标准特征集, 只需运行一次 + init_std_evt_dict() + (2) 事件barcode集和标准库barcode求交集 + build_std_evt_dict() + (3) 1:1性能评估 + one2one_simi() + (4) 计算PR曲线 + one2one_pr() + + test_one2SN() + 1:SN比对 + (1) 初始化事件和标准特征集, 只需运行一次 + init_std_evt_dict() + (2) 事件barcode集和标准库barcode求交集 + build_std_evt__dict() + (3) 1:SN性能评估 + one2SN_pr() + + + init_std_evt_dict() + 生成标准特征集与事件对象并存储，并存储相关数据，只需运行一次 + + (1) Barcode图像进行特征推理并保存 + gen_bcd_features(stdSamplePath, stdBarcodePath, stdFeaturePath, bcdSet) + + (2) 基于data文件或pipeline输出的pickle文件生成 ShoppingEvent 对象 + gen_eventdict(eventDatePath, saveimg) + + build_std_evt_dict() + 基于事件barcode集和标准库barcode交集构造用于评估性能的事件集合。 + + simi_calc() + 计算事件和标准特征集的相似度 + + data_precision_compare() + 不同数据精度下的性能比较 + + + + int8_to_ft16() + + + ft16_to_uint8() + + one2n_contrast.py + 执行1:n共需要3步，分别对应3个函数： + (1) 生成事件字典并保存至 eventDataPath, 只需运行一次 + (2) 读取事件字典 + (3) 1:n 比对事件评估 + + (1) gen_eventdict(sourcePath) + + + (2) read_eventdict() + inputs： + eventDataPath: 类 ShoppingEvent 对象存储地址，该对象由函数 gen_eventdict() 生成。 + + output： + evtDicts + + + (3) one2n_pr() + inputs： + evtDicts + + output： + fpevents：FP事件地址 + + (4) simi_calc() + 计算两个事件的相似度 + + + + one2n_contrast_old.py (disused) + test_one2n() + 1:n 现场测试性能评估，输出 PR 曲线 + 兼容 2 种 txt 文件格式：returnGoods.txt, deletedBarcode.txt, + 分别对应不同的文件读取函数： + - read_deletedBarcode_file() + - read_returnGoods_file() + + one2n_return(all_list) + 输入：从returnGoods.txt读取的数据 + 输出： + corrpairs：(取出事件, 正确匹配的放入事件) + errpairs：(取出事件, 放入事件, 错误匹配的放入事件) + corr_similarity: (正确匹配时的相似度) + err_similarity: (错误匹配时的相似度) + + + one2n_deleted(all_list) + 输入: 从deletedBarcode.txt读取的数据 + 输出： + corrpairs：(取出事件, 取出的barcode) + errpairs：(取出事件, 取出的barcode, 错误匹配的barcode) + corr_similarity: (正确匹配时的相似度) + err_similarity: (错误匹配时的相似度) + + save_tracking_imgpairs(pairs, savepath) + 输入： + pairs：匹配时间对，len(2)=2 or 3, 对应正确匹配与错误匹配 + savepath：结果保存地址，其中图像文件的命名为：取出事件 + 放入事件 + 错误匹配时间 + 子函数 get_event_path(), 扫码放入的对齐名 + 对于 returnGoods.txt, 放入事件的事件名和对应的文件夹名不一致，需要对齐 + + test_rpath_deleted() + 功能： + 针对 eletedBarcode.txt 格式的 1:n 数据结果文件 + 获得 1:n 情况下正确或匹配事件对（取出事件、放入事件、错误匹配事件） + 匹配事件分析, 实现函数：save_tracking_imgpairs() + 重要参数： + del_barcode_file: + basepath: 对应事件路径 + savepath: 存储路径, 是函数 save_tracking_imgpairs() 的输入 + saveimgs: Ture, False, 是否保存错误匹配的事件对 + + get_contrast_paths() + 针对 eletedBarcode.txt 格式的 1:n 数据结果文件,返回三元时间元组（getoutpath, inputpath, errorpath） + + test_rpath_return() + 针对 returnGoods.txt 格式 1:n 数据文件，不需要调用函数get_contrast_paths() + 获得 1:n 情况下正确或匹配事件对（取出事件、放入事件、错误匹配事件） + 匹配事件分析, 实现函数：save_tracking_imgpairs() + + +time_devide.py + runyolo() + 执行 imgs_inference.py 中的 run_yolo()模块，该模块重新定义了类 LoadImages, 对图像进行了旋转。 + + + show_seri() + + \ No newline at end of file