From bdd6f14399b982e2643f8282c4cf2ac6e1eae845 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E6=9D=8E=E6=99=A8?= <lichen@ieemoo.com>
Date: Wed, 29 Nov 2023 15:23:08 +0800
Subject: [PATCH] s3

---
 ieemoo-ai-searchv2.py |  4 ++--
 requirements.txt      | 41 ++++++++++++++++++++---------------------
 utils/config.py       | 10 +++++++++-
 utils/up_load_file.py | 26 ++++++++++++++++++++++++--
 4 files changed, 55 insertions(+), 26 deletions(-)

diff --git a/ieemoo-ai-searchv2.py b/ieemoo-ai-searchv2.py
index 5e2068a..f756c96 100755
--- a/ieemoo-ai-searchv2.py
+++ b/ieemoo-ai-searchv2.py
@@ -25,13 +25,13 @@ if SW_SERVER and SW_SERVICE_NAME:
     config.init() #采集服务的地址，给自己的服务起个名称
     #config.init(collector="123.60.56.51:11800", service='ieemoo-ai-search') #采集服务的地址，给自己的服务起个名称
     agent.start()
-os.environ["CUDA_VISIBLE_DEVICES"] = '1'
+os.environ["CUDA_VISIBLE_DEVICES"] = '0'
 
 app = Flask(__name__)
 
 parser = argparse.ArgumentParser()
 #parser.add_argument('--model', default='../module/ieemoo-ai-search/model/now/raft-things.pth',help="restore checkpoint")
-parser.add_argument('--model', default='../module/ieemoo-ai-search/model/now/raft-small.pth',help="restore checkpoint")
+parser.add_argument('--model', default='../module/ieemoo-ai-searchv2/model/now/raft-small.pth',help="restore checkpoint")
 #parser.add_argument('--small', action='store_true', help='use small model')
 parser.add_argument('--small', type=bool, default=True, help='use small model')
 parser.add_argument('--mixed_precision', action='store_true', help='use mixed precision')
diff --git a/requirements.txt b/requirements.txt
index 717bfea..324fb4c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,22 +1,21 @@
-esdk_obs_python==3.21.8
-Flask==2.0.0
-gevent==21.1.2
-matplotlib==3.4.1
-numpy==1.20.2
-esdk-obs-python --trusted-host pypi.org
-opencv_python==4.5.5.64
-opencv-contrib-python==4.5.5.64
-Pillow==9.1.0
-scipy==1.6.2
-setuptools==49.6.0
-coremltools==5.2.0
-onnx==1.7.0
-pandas==1.2.4
-pycocotools==2.0.2
+apscheduler==3.10.4
+cos_python_sdk_v5==1.9.26
+esdk_obs_python==3.23.9.1
+Flask==3.0.0
+gevent==23.9.1
+h5py==3.7.0
+matplotlib==3.6.2
+numpy==1.19.2
+numpy==1.24.4
+opencv_python==4.8.1.78
+opencv_python_headless==4.7.0.68
+pandas==1.4.4
+Pillow==9.4.0
 PyYAML==6.0
-requests==2.25.1
-seaborn==0.11.1
-thop==0.0.31.post2005241907
-tqdm==4.60.0
-ml-collections==0.1.1
-apache-skywalking
+Requests==2.31.0
+scipy==1.3.1
+setuptools==68.0.0
+torch==1.8.1+cu111
+torchvision==0.9.1+cu111
+Werkzeug==3.0.1
+yacs==0.1.8
diff --git a/utils/config.py b/utils/config.py
index 41ab5d4..34f7022 100644
--- a/utils/config.py
+++ b/utils/config.py
@@ -74,6 +74,14 @@ _C.cos_secret_key = 'oYGrJG2d3LTUPBoFTZVEmC1kQ1ObdTkr'
 _C.cos_region = 'ap-shanghai'
 _C.cos_Bucket = 'ieemoo-ai-1321281601'
 
+#s3
+_C.s3_secret_id = 'v1wAfxqZqJYbA7kuB9dc'
+_C.s3_secret_key = 'ZICGRJetjNIXhnqH1yfraoigydWoF2WTggtO9AhR'
+_C.s3_region = '172.16.0.186:9000'
+_C.s3_Bucket = 'ieemoo-ai'
+
 #cos or obs
-_C.cos = True
 _C.obs = True
+_C.cos = False
+_C.s3 = True
+
diff --git a/utils/up_load_file.py b/utils/up_load_file.py
index 1945968..efa38a4 100644
--- a/utils/up_load_file.py
+++ b/utils/up_load_file.py
@@ -3,6 +3,8 @@ import obs
 from utils.config import cfg
 from qcloud_cos import CosConfig,CosS3Client
 from qcloud_cos.cos_exception import CosClientError, CosServiceError
+from minio import Minio
+from minio.error import S3Error
 
 def obsInit():
     print('start init obs <<')
@@ -24,13 +26,27 @@ def cosInit():
     client = CosS3Client(config)
     return client
 
+def s3Init():
+    print('start init s3 <<')
+    secret_id = cfg.s3_secret_id
+    secret_key = cfg.s3_secret_key
+    region = cfg.s3_region
+    client = Minio(region, secure=False, access_key=secret_id, secret_key=secret_key)
+    return client
+
+
 class up_load:
     def __init__(self):
-        self.cosclient = cosInit()
-        self.obsclient, self.headers = obsInit()
+        if cfg.cos:
+            self.cosclient = cosInit()
+        if cfg.obs:
+            self.obsclient, self.headers = obsInit()
+        if cfg.s3:
+            self.s3client = s3Init()
 
     def upLoad(self, key=None, datapath=None):
         # 使用高级接口断点续传，失败重试时不会上传已成功的分块(这里重试10次)
+        response = 'get re'
         if cfg.cos:
             for i in range(0, 10):
                 try:
@@ -43,4 +59,10 @@ class up_load:
                     print(e)
         if cfg.obs:
             response = self.obsclient.putFile(cfg.obs_bucketName, key, datapath)
+        if cfg.s3:
+            found = self.s3client.bucket_exists(cfg.s3_Bucket)
+            if not found:
+                self.s3client.make_bucket(cfg.s3_Bucket)
+            self.s3client.fput_object(cfg.obs_bucketName, key, datapath)
         return response
+