Merge branch 'feature/REQ-2119' into 'master'

REQ-2119 图框数据标注1.0

See merge request universal/infrastructure/backend/oss!126

oss-client/src/main/java/cn/axzo/oss/client/controller/ServerFileController.java

@@ -8,16 +8,22 @@ import cn.axzo.oss.http.model.*;
 import cn.axzo.oss.manager.api.dto.request.FindFileKeyDto;
 import cn.axzo.oss.manager.api.dto.request.FindFileUrlDto;
 import cn.axzo.oss.manager.api.dto.request.ServerFileDeleteDto;
+import cn.axzo.oss.manager.api.dto.request.ServerFileDownloadDto;
 import cn.axzo.oss.manager.api.dto.request.ServerFileUploadDto;
+import cn.axzo.oss.manager.api.dto.response.ServerFileDownloadResponse;
 import cn.axzo.oss.service.api.FileService;
 import cn.azxo.framework.common.model.CommonResponse;
 import cn.hutool.json.JSONUtil;
+import feign.Response;
+import org.apache.commons.io.IOUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RestController;
 
 import javax.servlet.http.HttpServletRequest;
 import javax.validation.Valid;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.util.List;
 
 /**

oss-client/src/main/java/cn/axzo/oss/client/controller/WebFileController.java

@@ -6,15 +6,20 @@ import cn.axzo.framework.auth.domain.ContextInfo;
 import cn.axzo.framework.auth.domain.ContextInfoHolder;
 import cn.axzo.oss.client.vo.*;
 import cn.axzo.oss.common.enums.ChannelTypeEnum;
+import cn.axzo.oss.common.enums.FileDownloadTypeEnum;
 import cn.axzo.oss.common.enums.FileUploadTypeEnum;
 import cn.axzo.oss.common.enums.StorageUnitEnum;
 import cn.axzo.oss.common.exception.BizException;
 import cn.axzo.oss.common.utils.BeanConvertUtil;
+import cn.axzo.oss.http.model.TemporaryUrlAccessRes;
+import cn.axzo.oss.http.model.WebFileUploadVo;
 import cn.axzo.oss.manager.api.dto.request.*;
 import cn.axzo.oss.manager.api.dto.response.*;
 import cn.axzo.oss.service.api.FileService;
 import cn.azxo.framework.common.model.CommonResponse;
+import cn.hutool.json.JSONUtil;
 import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.IOUtils;
 import org.springframework.web.bind.annotation.*;
 import org.springframework.web.multipart.MultipartFile;
@@ -36,6 +41,7 @@ import static org.springframework.http.MediaType.MULTIPART_FORM_DATA_VALUE;
  **/
 @RestController
 @RequestMapping("/webApi")
+@Slf4j
 public class WebFileController {
 
     private static int FILE_NAME_MAX_LENGTH = 128;
@@ -262,4 +268,88 @@ public class WebFileController {
         WebFileUploadVo result = BeanConvertUtil.copyBean(response, WebFileUploadVo.class);
         return CommonResponse.success(result);
     }
+
+    /**
+     * obs：流式下载
+     *
+     * @param dto ServerFileDownloadDto
+     * @param response HttpServletResponse
+     */
+    @SneakyThrows
+    @GetMapping("/v1/obs/getObject")
+    @CrossOrigin
+    public void getObject(@Valid ServerFileDownloadDto dto, HttpServletResponse response) {
+        ServerFileDownloadResponse result = fileService.getObject(dto, FileDownloadTypeEnum.STREAM_DOWNLOAD.getCode());
+        try (OutputStream outputStream = response.getOutputStream(); InputStream inputStream = result.getFileStream()) {
+            response.setContentType("image/jpg");
+            response.setCharacterEncoding("UTF-8");
+            response.addHeader("Content-Disposition", "attachment;filename="
+                    + result.getFileName() + "." + result.getFileFormat());
+            IOUtils.copy(inputStream, outputStream);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * obs：断点续传下载
+     *
+     * @param dto ServerFileDownloadDto
+     * @param response HttpServletResponse
+     */
+    @SneakyThrows
+    @GetMapping("/v1/obs/downloadFile")
+    @CrossOrigin
+    public void downloadFile(@Valid ServerFileDownloadDto dto, HttpServletResponse response) {
+        ServerFileDownloadResponse result = fileService.getObject(dto, FileDownloadTypeEnum.CHECK_POINT_DOWNLOAD.getCode());
+        try (OutputStream outputStream = response.getOutputStream(); InputStream inputStream = result.getFileStream()) {
+            response.setContentType("image/jpg");
+            response.setCharacterEncoding("UTF-8");
+            response.addHeader("Content-Disposition", "attachment;filename="
+                    + result.getFileName() + "." + result.getFileFormat());
+            IOUtils.copy(inputStream, outputStream);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * 远程下载接口
+     *
+     * @param fileUuId 文件uuid
+     * @param response HttpServletResponse
+     */
+    @GetMapping("/v1/obs/downloadFileFromObs")
+    public void downloadFileFromObs(@RequestParam("fileUuid") String fileUuId, HttpServletResponse response) {
+        ServerFileDownloadDto dto = new ServerFileDownloadDto();
+        dto.setFileKey(fileUuId);
+        ServerFileDownloadResponse result = fileService.getObject(dto, FileDownloadTypeEnum.STREAM_DOWNLOAD.getCode());
+
+        try (OutputStream outputStream = response.getOutputStream(); InputStream inputStream = result.getFileStream()) {
+            response.setHeader("content-type","application/octet-stream");
+            response.setContentType("application/octet-stream");
+            response.setCharacterEncoding("UTF-8");
+            response.addHeader("Content-Disposition", "attachment;filename=" + result.getFileName() + "." + result.getFileFormat());
+            IOUtils.copy(inputStream, outputStream);
+            log.info("response设置文件流成功");
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * 通过临时url访问OBS
+     *
+     * @param fileUuId 文件uuid
+     * @return TemporaryUrlAccessRes
+     */
+    @GetMapping("/v1/obs/temporaryUrlAccess")
+    public CommonResponse<TemporaryUrlAccessRes> temporaryUrlAccess(@RequestParam("fileUuid") String fileUuId) {
+        ServerFileDownloadDto dto = new ServerFileDownloadDto();
+        dto.setFileKey(fileUuId);
+        ServerFileDownloadResponse response = fileService.getObject(dto, FileDownloadTypeEnum.TEMPORARY_URL_ACCESS.getCode());
+        TemporaryUrlAccessRes result = BeanConvertUtil.copyBean(response, TemporaryUrlAccessRes.class);
+        log.info("下载结果, result = {}", JSONUtil.toJsonStr(result));
+        return CommonResponse.success(result);
+    }
 }

oss-common/src/main/java/cn/axzo/oss/common/enums/FileDownloadTypeEnum.java

@@ -0,0 +1,24 @@
+package cn.axzo.oss.common.enums;
+
+import lombok.Getter;
+
+/**
+ * 文件下载类型枚举类
+ *
+ * @author hucf
+ * @since 2024/1/23 17:57
+ **/
+@Getter
+public enum FileDownloadTypeEnum {
+    STREAM_DOWNLOAD(1, "流式下载"),
+    CHECK_POINT_DOWNLOAD(2, "断点续传下载"),
+    TEMPORARY_URL_ACCESS(3, "通过临时url访问OBs")
+    ;
+    private final Integer code;
+    private final String message;
+
+    FileDownloadTypeEnum(Integer code, String message) {
+        this.code = code;
+        this.message = message;
+    }
+}

oss-http-api/pom.xml

@@ -35,6 +35,10 @@
             <groupId>org.springframework.cloud</groupId>
             <artifactId>spring-cloud-openfeign-core</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.springframework.cloud</groupId>
+            <artifactId>spring-cloud-starter-openfeign</artifactId>
+        </dependency>
     </dependencies>
 
 </project>

oss-http-api/src/main/java/cn/axzo/oss/http/api/DownloadFileApi.java

@@ -0,0 +1,24 @@
+package cn.axzo.oss.http.api;
+
+import cn.axzo.oss.http.model.TemporaryUrlAccessRes;
+import cn.azxo.framework.common.model.CommonResponse;
+import feign.Response;
+import org.springframework.cloud.openfeign.FeignClient;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+
+/**
+ * @author axzo
+ * @since 2024/3/7 16:56
+ **/
+@FeignClient(
+        name = "oss",
+        url = "http://oss:9123"
+)
+public interface DownloadFileApi {
+    @GetMapping(value = "/webApi/v1/obs/downloadFileFromObs")
+    CommonResponse<Response> downloadFileFromObs(@RequestParam("fileUuid") String fileUuId);
+
+    @GetMapping(value = "/webApi/v1/obs/temporaryUrlAccess")
+    CommonResponse<TemporaryUrlAccessRes> temporaryUrlAccess(@RequestParam("fileUuid") String fileUuId);
+}

oss-http-api/src/main/java/cn/axzo/oss/http/api/ServerFileServiceApi.java

@@ -3,9 +3,13 @@ package cn.axzo.oss.http.api;
 import cn.axzo.oss.http.model.*;
 import cn.azxo.framework.common.model.CommonResponse;
 import org.springframework.cloud.openfeign.FeignClient;
+import org.springframework.http.MediaType;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestMethod;
 
+import feign.Response;
 import java.util.List;
 
 /**

oss-http-api/src/main/java/cn/axzo/oss/http/api/WebFileServiceApi.java

@@ -0,0 +1,38 @@
+package cn.axzo.oss.http.api;
+
+import cn.axzo.oss.http.model.WebFileUploadVo;
+import cn.azxo.framework.common.model.CommonResponse;
+import feign.Response;
+import org.springframework.cloud.openfeign.FeignClient;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RequestPart;
+import org.springframework.web.multipart.MultipartFile;
+
+import javax.validation.Valid;
+
+import static org.springframework.http.MediaType.MULTIPART_FORM_DATA_VALUE;
+
+/**
+ * @author wangsiqian
+ * @since 2024/02/27
+ */
+@FeignClient(
+        name = "oss",
+        url = "http://oss:9123"
+)
+public interface WebFileServiceApi {
+    /**
+     * OBS：文件上传（断点续传）
+     *
+     * @param appCode 应用编码
+     * @param bizScene 业务场景
+     * @param file MultipartFile
+     * @return WebFileUploadVo
+     */
+    @PostMapping(value = "/webApi/v1/file2", consumes = MULTIPART_FORM_DATA_VALUE)
+    CommonResponse<WebFileUploadVo> uploadObs(@Valid @RequestParam("appCode") String appCode,
+                                              @Valid @RequestParam("bizScene") String bizScene,
+                                              @Valid @RequestPart MultipartFile file);
+}

oss-http-api/src/main/java/cn/axzo/oss/http/model/DownloadFileFromObsRequest.java

@@ -0,0 +1,30 @@
+package cn.axzo.oss.http.model;
+
+import lombok.Data;
+
+import javax.validation.constraints.NotBlank;
+import javax.validation.constraints.NotNull;
+
+/**
+ * @author hucf
+ * @since 2024/3/6 15:11
+ **/
+@Data
+public class DownloadFileFromObsRequest {
+    /**
+     * 文件uuid
+     */
+    @NotBlank(message = "文件uuid不能为空")
+    private String fileKey;
+
+    /**
+     * 图片样式
+     */
+    private String style;
+
+    /**
+     * 下载方式：1:流式下载；2:断点续传方式下载
+     */
+    @NotNull(message = "下载方式不能为空")
+    private Integer fileDownloadType;
+}

oss-http-api/src/main/java/cn/axzo/oss/http/model/DownloadFileFromObsResponse.java

@@ -0,0 +1,46 @@
+package cn.axzo.oss.http.model;
+
+import lombok.Data;
+
+import java.io.InputStream;
+
+/**
+ * @author hucf
+ * @since 2024/3/6 15:15
+ **/
+@Data
+public class DownloadFileFromObsResponse {
+    /**
+     * 文件 URL
+     */
+    private String url;
+    /**
+     * 文件 Key
+     */
+    private String fileKey;
+
+    /**
+     * URL MD5
+     */
+    private String urlMd5;
+
+    /**
+     * 文件名称
+     */
+    private String fileName;
+
+    /**
+     * 文件大小
+     */
+    private Long fileSize;
+
+    /**
+     * 文件格式
+     */
+    private String fileFormat;
+
+    /**
+     * 文件流
+     */
+    private InputStream fileStream;
+}

oss-http-api/src/main/java/cn/axzo/oss/http/model/TemporaryUrlAccessRes.java

@@ -0,0 +1,44 @@
+package cn.axzo.oss.http.model;
+
+import lombok.Data;
+
+/**
+ * @author hucf
+ * @since 2024/3/11 18:21
+ **/
+@Data
+public class TemporaryUrlAccessRes {
+    /**
+     * 文件 URL
+     */
+    private String url;
+    /**
+     * 文件 Key
+     */
+    private String fileKey;
+
+    /**
+     * URL MD5
+     */
+    private String urlMd5;
+
+    /**
+     * 文件名称
+     */
+    private String fileName;
+
+    /**
+     * 文件大小
+     */
+    private Long fileSize;
+
+    /**
+     * 文件格式
+     */
+    private String fileFormat;
+
+    /**
+     * obs:临时url
+     */
+    private String signedUrl;
+}

oss-http-api/src/main/java/cn/axzo/oss/http/model/WebFileUploadVo.java

@@ -1,4 +1,4 @@
-package cn.axzo.oss.client.vo;
+package cn.axzo.oss.http.model;
 
 import lombok.Data;
 

oss-integration/src/main/java/cn/axzo/oss/integration/s3/HuaWeiCloudService.java

@@ -1,5 +1,7 @@
 package cn.axzo.oss.integration.s3;
 
+import com.obs.services.model.TemporarySignatureResponse;
+
 import java.io.InputStream;
 
 /**
@@ -14,4 +16,8 @@ public interface HuaWeiCloudService {
     String checkPointUploadFile(String bucketName, String fileName, String appCode, InputStream srcStream, String filePath);
 
     String multipartUpload(String bucketName, String fileName, String appCode, InputStream srcStream);
+
+    InputStream getObject(String bucketName, String objectKey, Integer fileDownloadType);
+
+    TemporarySignatureResponse createTemporarySignature(String bucketName, String objectKey);
 }

oss-integration/src/main/java/cn/axzo/oss/integration/s3/impl/HuaWeiCloudServiceImpl.java

@@ -1,10 +1,12 @@
 package cn.axzo.oss.integration.s3.impl;
 
+import cn.axzo.oss.common.enums.FileDownloadTypeEnum;
 import cn.axzo.oss.integration.s3.HuaWeiCloudService;
 import cn.axzo.oss.integration.s3.client.HuaWeiCloudObsClient;
 import cn.axzo.oss.integration.s3.config.HuaWeiCloudObsConfig;
 import cn.azxo.framework.common.utils.LogUtil;
 import cn.hutool.core.io.FileUtil;
+import cn.hutool.json.JSONUtil;
 import com.obs.services.ObsClient;
 import com.obs.services.exception.ObsException;
 import com.obs.services.model.*;
@@ -15,6 +17,7 @@ import org.springframework.stereotype.Service;
 import java.io.InputStream;
 import java.net.URLEncoder;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.List;
 
 /**
@@ -225,4 +228,96 @@ public class HuaWeiCloudServiceImpl implements HuaWeiCloudService {
         request.setObjectMetadata(metadata);
         return obsClient.uploadFile(request);
     }
+
+    /**
+     * obs下载文件：流式下载
+     *
+     * @param bucketName 桶名称
+     * @param objectKey 对象名称
+     * @return InputStream
+     */
+    @Override
+    public InputStream getObject(String bucketName, String objectKey, Integer fileDownloadType) {
+        ObsClient obsClient = huaWeiCloudObsClient.getClient();
+        if (FileDownloadTypeEnum.STREAM_DOWNLOAD.getCode().equals(fileDownloadType)) {
+            try {
+                GetObjectRequest request = new GetObjectRequest();
+                request.setBucketName(bucketName);
+                request.setObjectKey(objectKey);
+
+                ObsObject object = obsClient.getObject(request);
+                log.info("下载华为云OBS文件成功");
+                return object.getObjectContent();
+            } catch (ObsException obsException) {
+                log.warn("下载华为云OBS文件失败，HTTP Code:{}, Error Code:{}, Request ID:{}, Host ID:{}, Error Message:{}",
+                        obsException.getResponseCode(),
+                        obsException.getErrorCode(),
+                        obsException.getErrorRequestId(),
+                        obsException.getErrorHostId(),
+                        obsException.getErrorMessage());
+            }
+        } else if (FileDownloadTypeEnum.CHECK_POINT_DOWNLOAD.getCode().equals(fileDownloadType)) {
+            try {
+                DownloadFileRequest request = new DownloadFileRequest(bucketName, objectKey);
+                // 设置下载对象的本地文件全路径，当该值为空时，默认为当前程序的运行目录。
+                request.setDownloadFile("");
+                // 设置分段下载时的最大并发数
+                request.setTaskNum(Integer.parseInt(huaWeiCloudObsConfig.getTaskNum()));
+                // 设置分段大小为10MB
+                request.setPartSize(Long.parseLong(huaWeiCloudObsConfig.getPartSize()));
+                // 是否开启断点续传模式
+                request.setEnableCheckpoint(Boolean.parseBoolean(huaWeiCloudObsConfig.getEnableCheckPoint()));
+                // 是否自动解码响应头
+                request.setIsEncodeHeaders(Boolean.parseBoolean(huaWeiCloudObsConfig.getEncodeHeaders()));
+
+                DownloadFileResult downloadFileResult = obsClient.downloadFile(request);
+                String etag = downloadFileResult.getObjectMetadata().getEtag();
+                log.info("下载华为云OBS文件成功");
+            } catch (NumberFormatException e) {
+                throw new RuntimeException(e);
+            } catch (ObsException obsException) {
+                log.warn("下载华为云OBS文件失败，HTTP Code:{}, Error Code:{}, Request ID:{}, Host ID:{}, Error Message:{}",
+                        obsException.getResponseCode(),
+                        obsException.getErrorCode(),
+                        obsException.getErrorRequestId(),
+                        obsException.getErrorHostId(),
+                        obsException.getErrorMessage());
+            }
+        }
+        return null;
+    }
+
+    /**
+     * 获取OBS临时url
+     *
+     * @param bucketName 桶名称
+     * @param objectKey 对象名称
+     * @return TemporarySignatureResponse
+     */
+    @Override
+    public TemporarySignatureResponse createTemporarySignature(String bucketName, String objectKey) {
+        TemporarySignatureRequest request = new TemporarySignatureRequest();
+        // 桶名称
+        request.setBucketName(bucketName);
+        // 对象名称
+        request.setObjectKey(objectKey);
+        // HTTP方法类型
+        request.setMethod(HttpMethodEnum.GET);
+        // 带授权信息的URL的过期时间300秒
+        request.setExpires(300L);
+        // 发起请求的时间
+        request.setRequestDate(new Date());
+
+        try {
+            ObsClient obsClient = huaWeiCloudObsClient.getClient();
+            TemporarySignatureResponse response = obsClient.createTemporarySignature(request);
+            log.info("response = {}", JSONUtil.toJsonStr(response));
+            return response;
+        } catch (Exception exception) {
+            log.warn("获取临时url发生异常, exception = {}", exception.getMessage());
+            log.warn("异常信息：");
+            exception.printStackTrace();
+            return null;
+        }
+    }
 }

oss-manager-api/src/main/java/cn/axzo/oss/manager/api/FileManager.java

@@ -51,4 +51,8 @@ public interface FileManager {
     String multipartUploadComplete(String bucketName, String tgtFileKey, String uploadId, List<PartETag> partETags);
 
     String multipartUploadFile(String bucketName, String tgtFileKey, MultipartFile file);
+
+    InputStream downloadObsFile(String bucketName, String objectKey, Integer fileDownloadType);
+
+    String getTempUrlFromOns(String bucketName, String objectKey);
 }

oss-manager-api/src/main/java/cn/axzo/oss/manager/api/dto/response/ServerFileDownloadResponse.java

@@ -45,4 +45,9 @@ public class ServerFileDownloadResponse {
      * 文件流
      */
     private InputStream fileStream;
+
+    /**
+     * obs:临时url
+     */
+    private String signedUrl;
 }

oss-manager/src/main/java/cn/axzo/oss/manager/impl/FileManagerImpl.java

@@ -9,6 +9,7 @@ import cn.axzo.oss.integration.s3.HuaWeiCloudService;
 import cn.axzo.oss.manager.api.FileManager;
 import cn.axzo.oss.manager.api.dto.PartETag;
 import cn.axzo.oss.manager.api.dto.request.MultipartUploadDto;
+import com.obs.services.model.TemporarySignatureResponse;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -124,4 +125,13 @@ public class FileManagerImpl implements FileManager {
         }
         return partETagList;
     }
+
+    public InputStream downloadObsFile(String bucketName, String objectKey, Integer fileDownloadType) {
+        return huaWeiCloudService.getObject(bucketName, objectKey, fileDownloadType);
+    }
+
+    public String getTempUrlFromOns(String bucketName, String objectKey) {
+        TemporarySignatureResponse response = huaWeiCloudService.createTemporarySignature(bucketName, objectKey);
+        return response.getSignedUrl();
+    }
 }

oss-service-api/src/main/java/cn/axzo/oss/service/api/FileService.java

@@ -51,4 +51,6 @@ public interface FileService {
                                        Integer fileUploadType);
 
     String getFilePath(MultipartFile file);
+
+    ServerFileDownloadResponse getObject(ServerFileDownloadDto dto, Integer fileDownloadType);
 }

oss-service/src/main/java/cn/axzo/oss/service/impl/FileServiceImpl.java

@@ -697,4 +697,38 @@ public class FileServiceImpl implements FileService {
             throw new RuntimeException(e);
         }
     }
+
+    public ServerFileDownloadResponse getObject(ServerFileDownloadDto dto, Integer fileDownloadType) {
+        log.info("obs download file = {}", JsonUtil.obj2Str(dto));
+        File file = null;
+        String fileKey = dto.getFileKey();
+        if (fileKey.contains(IS_URL)) {
+            String urlMd5 = Utility.getMd5(fileKey);
+            file = fileDao.getByUrlMd5(urlMd5);
+        } else {
+            file = fileDao.getByFileUuid(fileKey);
+        }
+        if (Utility.objIsNull(file)) {
+            log.warn("obs download file is null, fileKey = {}", fileKey);
+            BizException.error(false, CodeEnum.FILE_NOT_FOUND);
+        }
+
+        // 对象名称：目录+文件名
+        String objectKey = file.getDirectory() + SEPARATOR + file.getFileName();
+        // 桶名称
+        String bucketName = file.getBucketName();
+        InputStream fileStream = null;
+        String tempUrlFromOns = null;
+        if (FileDownloadTypeEnum.TEMPORARY_URL_ACCESS.getCode().equals(fileDownloadType)) {
+            tempUrlFromOns = fileManager.getTempUrlFromOns(bucketName, objectKey);
+        } else {
+            fileStream = fileManager.downloadObsFile(bucketName, objectKey, fileDownloadType);
+            if (Objects.isNull(fileStream)) {
+                log.warn("下载的文件流为空");
+            }
+        }
+        ServerFileDownloadResponse response = setFileDownloadResponse(file, fileStream);
+        response.setSignedUrl(tempUrlFromOns);
+        return response;
+    }
 }