S3

floder/config.py

@@ -15,3 +15,26 @@ _C.maskAreaImg1 = '../module/ieemoo-ai-conpurchase/model/now/maskAreaImg1.jpg'
 _C.streamModel = '../module/ieemoo-ai-conpurchase/model/now/raft-things.pth'
 _C.hFile = '../module/ieemoo-ai-conpurchase/document'
 _C.videoPath = '../module/ieemoo-ai-conpurchase/videos'
+
+#obs
+_C.obs_access_key_id='LHXJC7GIC2NNUUHHTNVI'
+_C.obs_secret_access_key='sVWvEItrFKWPp5DxeMvX8jLFU69iXPpzkjuMX3iM'
+_C.obs_server='https://obs.cn-east-3.myhuaweicloud.com'
+_C.obs_bucketName = 'ieemoo-ai'
+
+#cos
+_C.cos_secret_id = 'AKIDIVcVFnBMMLCqbFF6lPr6930tc8VlrlS0'
+_C.cos_secret_key = 'oYGrJG2d3LTUPBoFTZVEmC1kQ1ObdTkr'
+_C.cos_region = 'ap-shanghai'
+_C.cos_Bucket = 'ieemoo-ai-1321281601'
+
+#s3
+_C.s3_secret_id = 'v1wAfxqZqJYbA7kuB9dc'
+_C.s3_secret_key = 'ZICGRJetjNIXhnqH1yfraoigydWoF2WTggtO9AhR'
+_C.s3_region = '172.16.0.186:9000'
+_C.s3_Bucket = 'ieemoo-ai'
+
+#obs or cos
+_C.obs = True
+_C.cos = False
+_C.s3 = True

utils/opfile.py

@@ -1,15 +1,8 @@
-import h5py, os
+import h5py, os, sys, logging, obs
 from  floder.config import cfg
-from obs import ObsClient
+from up_load_file import up_load
-import obs
-obsClient = ObsClient(
-access_key_id='LHXJC7GIC2NNUUHHTNVI',
-secret_access_key='sVWvEItrFKWPp5DxeMvX8jLFU69iXPpzkjuMX3iM',
-server='https://obs.cn-east-3.myhuaweicloud.com')
-bucketName = 'ieemoo-ai'
-headers = obs.SetObjectMetadataHeader()
-headers.cacheControl = "no-cache"
 
+up_load_data = up_load()
 def writef(uuid, num_id, feature):
     fname = os.sep.join([cfg.hFile, uuid+'.h5'])
     if not os.path.exists(fname):
@@ -36,4 +29,10 @@ def uploadf(video_name, num, file_path):
     time = video_name.split('-')[0]
     objectkey = video_name.split('.')[0]+'_'+str(num)+'.mp4'
     objectkey = 'conpurchase/'+time+'/'+objectkey
-    resp = obsClient.putFile(bucketName, objectkey, file_path, headers=headers)
+    up_load_data.upLoad(objectkey, file_path)
+    #if obs:
+    #    obsClient, headers = obsInit()
+    #    resp = obsClient.putFile(cfg.obs_bucketName, objectkey, file_path, headers=headers)
+    #if cos:
+    #    upLoadCos(cfg.cos_Bucket, objectkey, file_path)
+

utils/up_load_file.py

@@ -0,0 +1,68 @@
+from obs import ObsClient
+import obs
+from floder.config import cfg
+from qcloud_cos import CosConfig,CosS3Client
+from qcloud_cos.cos_exception import CosClientError, CosServiceError
+from minio import Minio
+from minio.error import S3Error
+
+def obsInit():
+    print('start init obs <<')
+    obsClient = ObsClient(
+        access_key_id=cfg.obs_access_key_id,
+        secret_access_key=cfg.obs_secret_access_key,
+        server=cfg.obs_server)
+    bucketName = cfg.obs_bucketName
+    headers = obs.SetObjectMetadataHeader()
+    headers.cacheControl = "no-cache"
+    return obsClient, headers
+
+def cosInit():
+    print('start init cos <<')
+    secret_id = cfg.cos_secret_id
+    secret_key = cfg.cos_secret_key
+    region = cfg.cos_region
+    config = CosConfig(Region=region, Secret_id=secret_id, Secret_key=secret_key)
+    client = CosS3Client(config)
+    return client
+
+def s3Init():
+    print('start init s3 <<')
+    secret_id = cfg.s3_secret_id
+    secret_key = cfg.s3_secret_key
+    region = cfg.s3_region
+    client = Minio(region, secure=False, access_key=secret_id, secret_key=secret_key)
+    return client
+
+
+class up_load:
+    def __init__(self):
+        if cfg.cos:
+            self.cosclient = cosInit()
+        if cfg.obs:
+            self.obsclient, self.headers = obsInit()
+        if cfg.s3:
+            self.s3client = s3Init()
+
+    def upLoad(self, key=None, datapath=None):
+        # 使用高级接口断点续传，失败重试时不会上传已成功的分块(这里重试10次)
+        response = 'get re'
+        if cfg.cos:
+            for i in range(0, 10):
+                try:
+                    response = self.cosclient.upload_file(
+                        Bucket=cfg.cos_Bucket,
+                        Key=key,
+                        LocalFilePath=datapath)
+                    break
+                except CosClientError or CosServiceError as e:
+                    print(e)
+        if cfg.obs:
+            response = self.obsclient.putFile(cfg.obs_bucketName, key, datapath)
+        if cfg.s3:
+            found = self.s3client.bucket_exists(cfg.s3_Bucket)
+            if not found:
+                self.s3client.make_bucket(cfg.s3_Bucket)
+            self.s3client.fput_object(cfg.obs_bucketName, key, datapath)
+        return response
+