s3

ieemoo-ai-searchv2.py

@@ -31,7 +31,7 @@ app = Flask(__name__)
 
 parser = argparse.ArgumentParser()
 #parser.add_argument('--model', default='../module/ieemoo-ai-search/model/now/raft-things.pth',help="restore checkpoint")
-parser.add_argument('--model', default='../module/ieemoo-ai-search/model/now/raft-small.pth',help="restore checkpoint")
+parser.add_argument('--model', default='../module/ieemoo-ai-searchv2/model/now/raft-small.pth',help="restore checkpoint")
 #parser.add_argument('--small', action='store_true', help='use small model')
 parser.add_argument('--small', type=bool, default=True, help='use small model')
 parser.add_argument('--mixed_precision', action='store_true', help='use mixed precision')
@@ -58,11 +58,13 @@ def setup_logging(path):
     return logger
 
 logger = setup_logging('utils/logging.json')
-@app.route('/searchv2', methods=['POST'])
+@app.route('/searchv2', methods=['POST', 'GET'])
 def search():
     pre_status = False
     try:
         video_name = request.form.get('video_name')
+        if video_name == None:
+            return 'Need video_name'
         logger.info('get video '+video_name)
         ocr_file_path = os.sep.join([cfg.Ocrtxt, video_name.split('.')[0]+'.txt'])
         video_extra_info = request.form.get('video_extra_info')
@@ -80,16 +82,23 @@ def search():
         else:
             state = analysis_video(raft_model, videoPath, '',uuid_barcode,None,net=net, transform=transform,ms=ms, match=True)
         result = uuid_barcode+'_'+state #参数修改返回结果
+        try:
+            thread = Thread(target=AddObs, kwargs={'file_path':videoPath, 'status':state})
+            thread.start()
+            logger.info(result)
+            print('result >>>>> {}'.format(result))
+            return result
+        except Exception as e:
+            print('Exception >>>>> {}'.format(result))
+            return result
     except Exception as e:
         logger.warning(e)   #异常返回00
-        thread = Thread(target=AddObs, kwargs={'file_path':videoPath, 'status':status[3]})
-        thread.start()
-        return uuid_barcode+'_'+status[3] #参数修改返回00   
-    thread = Thread(target=AddObs, kwargs={'file_path':videoPath, 'status':state})
-    thread.start()
-    logger.info(result)
-    print('result >>>>> {}'.format(result))
-    return result
+        try:
+            thread = Thread(target=AddObs, kwargs={'file_path':videoPath, 'status':status[3]})
+            thread.start()
+            return uuid_barcode+'_'+status[3] #参数修改返回00
+        except Exception as e:
+            return uuid_barcode+'_'+status[3] #参数修改返回00
 
 if __name__ == '__main__':
     app.run(host='0.0.0.0', port=8085)

requirements.txt

@@ -1,22 +1,21 @@
-esdk_obs_python==3.21.8
-Flask==2.0.0
-gevent==21.1.2
-matplotlib==3.4.1
-numpy==1.20.2
-esdk-obs-python --trusted-host pypi.org
-opencv_python==4.5.5.64
-opencv-contrib-python==4.5.5.64
-Pillow==9.1.0
-scipy==1.6.2
-setuptools==49.6.0
-coremltools==5.2.0
-onnx==1.7.0
-pandas==1.2.4
-pycocotools==2.0.2
+apscheduler==3.10.4
+cos_python_sdk_v5==1.9.26
+esdk_obs_python==3.23.9.1
+Flask==3.0.0
+gevent==23.9.1
+h5py==3.7.0
+matplotlib==3.6.2
+numpy==1.19.2
+numpy==1.24.4
+opencv_python==4.8.1.78
+opencv_python_headless==4.7.0.68
+pandas==1.4.4
+Pillow==9.4.0
 PyYAML==6.0
-requests==2.25.1
-seaborn==0.11.1
-thop==0.0.31.post2005241907
-tqdm==4.60.0
-ml-collections==0.1.1
-apache-skywalking
+Requests==2.31.0
+scipy==1.3.1
+setuptools==68.0.0
+torch==1.8.1+cu111
+torchvision==0.9.1+cu111
+Werkzeug==3.0.1
+yacs==0.1.8

utils/config.py

@@ -51,10 +51,37 @@ _C.fgbgmask_old = '../module/ieemoo-ai-searchv2/model/now/ori_old.jpg'
 _C.URL = 'https://api.ieemoo.com/emoo-api/intelligence'  #online
 #_C.URL = 'http://api.test.ieemoo.com/emoo-api/intelligence'
 
-_C.VreTest = 'http://api.test.ieemoo.cn/emoo-api/intelligence/queryVideoCompareResult.do'
-_C.VreTest2 = 'http://api.test2.ieemoo.cn/emoo-api/intelligence/queryVideoCompareResult.do'
-#_C.Vre = 'https://api.ieemoo.com/emoo-api/intelligence/queryVideoCompareResult.do' #online
+_C.VreTest = 'http://api.test.ieemoo.cn/emoo-api/intelligence/queryVideoCompareResult.do' #test
+_C.VreTest2 = 'http://api.test2.ieemoo.cn/emoo-api/intelligence/queryVideoCompareResult.do' #test2
+_C.Vre = 'https://api.ieemoo.com/emoo-api/intelligence/queryVideoCompareResult.do' #online
+
+_C.backups = '../module/output/videos'
+_C.riskControl = 'http://api.test2.ieemoo.com/emoo-api/riskControl/shoporder/checkUploadAiImages?storeId=32001001'
 
 _C.Ocrimg = '../module/ieemoo-ai-assist/imgs'#post ocr img
 _C.Ocrtxt = '../module/ieemoo-ai-assist/document'#post ocr txts
 _C.Ocrvideo = '../module/ieemoo-ai-assist/videos'#post ocr video
+
+#obs
+_C.obs_access_key_id='LHXJC7GIC2NNUUHHTNVI'
+_C.obs_secret_access_key='sVWvEItrFKWPp5DxeMvX8jLFU69iXPpzkjuMX3iM'
+_C.obs_server='https://obs.cn-east-3.myhuaweicloud.com'
+_C.obs_bucketName = 'ieemoo-ai'
+
+#cos
+_C.cos_secret_id = 'AKIDIVcVFnBMMLCqbFF6lPr6930tc8VlrlS0'
+_C.cos_secret_key = 'oYGrJG2d3LTUPBoFTZVEmC1kQ1ObdTkr'
+_C.cos_region = 'ap-shanghai'
+_C.cos_Bucket = 'ieemoo-ai-1321281601'
+
+#s3
+_C.s3_secret_id = 'v1wAfxqZqJYbA7kuB9dc'
+_C.s3_secret_key = 'ZICGRJetjNIXhnqH1yfraoigydWoF2WTggtO9AhR'
+_C.s3_region = '172.16.0.186:9000'
+_C.s3_Bucket = 'ieemoo-ai'
+
+#cos or obs
+_C.obs = True
+_C.cos = False
+_C.s3 = True
+

utils/up_load_file.py

@@ -0,0 +1,68 @@
+from obs import ObsClient
+import obs
+from utils.config import cfg
+from qcloud_cos import CosConfig,CosS3Client
+from qcloud_cos.cos_exception import CosClientError, CosServiceError
+from minio import Minio
+from minio.error import S3Error
+
+def obsInit():
+    print('start init obs <<')
+    obsClient = ObsClient(
+        access_key_id=cfg.obs_access_key_id,
+        secret_access_key=cfg.obs_secret_access_key,
+        server=cfg.obs_server)
+    bucketName = cfg.obs_bucketName
+    headers = obs.SetObjectMetadataHeader()
+    headers.cacheControl = "no-cache"
+    return obsClient, headers
+
+def cosInit():
+    print('start init cos <<')
+    secret_id = cfg.cos_secret_id
+    secret_key = cfg.cos_secret_key
+    region = cfg.cos_region
+    config = CosConfig(Region=region, Secret_id=secret_id, Secret_key=secret_key)
+    client = CosS3Client(config)
+    return client
+
+def s3Init():
+    print('start init s3 <<')
+    secret_id = cfg.s3_secret_id
+    secret_key = cfg.s3_secret_key
+    region = cfg.s3_region
+    client = Minio(region, secure=False, access_key=secret_id, secret_key=secret_key)
+    return client
+
+
+class up_load:
+    def __init__(self):
+        if cfg.cos:
+            self.cosclient = cosInit()
+        if cfg.obs:
+            self.obsclient, self.headers = obsInit()
+        if cfg.s3:
+            self.s3client = s3Init()
+
+    def upLoad(self, key=None, datapath=None):
+        # 使用高级接口断点续传，失败重试时不会上传已成功的分块(这里重试10次)
+        response = 'get re'
+        if cfg.cos:
+            for i in range(0, 10):
+                try:
+                    response = self.cosclient.upload_file(
+                        Bucket=cfg.cos_Bucket,
+                        Key=key,
+                        LocalFilePath=datapath)
+                    break
+                except CosClientError or CosServiceError as e:
+                    print(e)
+        if cfg.obs:
+            response = self.obsclient.putFile(cfg.obs_bucketName, key, datapath)
+        if cfg.s3:
+            found = self.s3client.bucket_exists(cfg.s3_Bucket)
+            if not found:
+                self.s3client.make_bucket(cfg.s3_Bucket)
+            self.s3client.fput_object(cfg.obs_bucketName, key, datapath)
+        return response
+

utils/updateObs.py

@@ -2,52 +2,46 @@ from obs import ObsClient
 from datetime import datetime
 #from config import cfg
 from utils.config import cfg
-import os,threading
+import os, sys
 import time as ti
 import base64,requests,cv2,shutil
-lock = threading.Lock()
-obsClient = ObsClient(
-    access_key_id='LHXJC7GIC2NNUUHHTNVI',
-    secret_access_key='sVWvEItrFKWPp5DxeMvX8jLFU69iXPpzkjuMX3iM',
-    server='https://obs.cn-east-3.myhuaweicloud.com'
-)
-bucketName = 'ieemoo-ai'
+from utils.up_load_file import up_load
 
+up_load_data = up_load()
 def AddObs(file_path, status):
-    with lock:
-        if not cfg.flag:
+    if not cfg.flag:
+        addobs(file_path, status)
+    else:
+        if status == '02':
             addobs(file_path, status)
         else:
-            if status == '02':
-                addobs(file_path, status)
-            else:
-                objectkey = os.path.basename(file_path)
-                f_dist = os.sep.join([cfg.tempvideos, status+'_'+objectkey])
-                shutil.move(file_path, f_dst)
+            objectkey = os.path.basename(file_path)
+            f_dist = os.sep.join([cfg.tempvideos, status+'_'+objectkey])
+            shutil.move(file_path, f_dst)
 
 def addobs(file_path, status): #save videos
-    ti.sleep(3)
+    ti.sleep(0.5)
     videoUuid = os.path.basename(file_path).split('_')[1]
     json_data = {'videoUuid': videoUuid}
-    for vre in [cfg.VreTest, cfg.VreTest2]:
+    for vre in [cfg.VreTest, cfg.VreTest2, cfg.Vre]:
         resp = requests.post(url=vre, data=json_data)
         status = resp.json()
         if not 'data' in status:#延迟设置
-           ti.sleep(5)
-           resp = requests.post(url=cfg.Vre, data=json_data)
+           ti.sleep(0.5)
+           resp = requests.post(url=vre, data=json_data)
            status = resp.json()
            if 'data' in status:
                break
         else:
             break
+    if not 'data' in status:
+        return 'can not get the status'
     status = status['data']
-
     objectkey = os.path.basename(file_path)
     time = os.path.basename(file_path).split('-')[0]
-    if objectkey.split('.')[-1] in ['avi','mp4']:
-        objectkey = 'videos/'+time+'/'+status+'/'+status+'_'+objectkey
-        resp = obsClient.putFile(bucketName, objectkey, file_path)
-    os.remove(file_path)
+    objectkey = 'videos/'+time+'/'+status+'/'+status+'_'+objectkey
+    if status == '02' or get_weightStatus(objectkey):
+        up_load_data.upLoad(objectkey, file_path)
 
 def Addimg(uuid_barcode):
     time = uuid_barcode.split('-')[0].split('_')[-1]
@@ -57,14 +51,27 @@ def Addimg(uuid_barcode):
         file_path = os.sep.join([cfg.Tempimg, '3_'+uuid_barcode+'.jpg'])
     if not os.path.exists(file_path):
         file_path = os.sep.join([cfg.Tempimg, 'ex_'+uuid_barcode+'.jpg'])
-    resp = obsClient.putFile(bucketName, objectkey, file_path)
+    uuid = uuid_barcode.split('_')[-1]
+    if get_weightStatus(uuid):
+        up_load_data.upLoad(objectkey, file_path)
 
 def Addimg_content(uuid_barcode, context):
+    file_path = "context.jpg"
+    cv2.imwrite(file_path, context)
     success, encoded_image = cv2.imencode(".jpg",context)
     context = encoded_image.tobytes()
     time = uuid_barcode.split('-')[0]
     objectkey = 'imgs/'+time+'/'+uuid_barcode+'.jpg'
-    resp = obsClient.putContent(bucketName, objectkey, context)
+    if get_weightStatus(uuid_barcode):
+        up_load_data.upLoad(objectkey, file_path)
+
+def get_weightStatus(uuid):
+    uuid = uuid.split('_')[1]
+    resp = requests.post(url = cfg.riskControl)
+    up_uuid = resp.json()['data']
+    if uuid in up_uuid:
+        return True
+    return False
 
 if __name__ == '__main__':
     import cv2