wangli/oss
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'feature/REQ-2119' into 'master'

Browse Source 
REQ-2119 图框数据标注1.0

See merge request universal/infrastructure/backend/oss!126
This commit is contained in:
金海洋 2024-03-13 10:43:08 +00:00
commit a2dffea9b5
18 changed files with 467 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
oss-client/src/main/java/cn/axzo/oss/client/controller/ServerFileController.java
View File

@ -8,16 +8,22 @@ import cn.axzo.oss.http.model.*;
import cn.axzo.oss.manager.api.dto.request.FindFileKeyDto;
import cn.axzo.oss.manager.api.dto.request.FindFileUrlDto;
import cn.axzo.oss.manager.api.dto.request.ServerFileDeleteDto;
import cn.axzo.oss.manager.api.dto.request.ServerFileDownloadDto;
import cn.axzo.oss.manager.api.dto.request.ServerFileUploadDto;
import cn.axzo.oss.manager.api.dto.response.ServerFileDownloadResponse;
import cn.axzo.oss.service.api.FileService;
import cn.azxo.framework.common.model.CommonResponse;
import cn.hutool.json.JSONUtil;
import feign.Response;
import org.apache.commons.io.IOUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RestController;
import javax.servlet.http.HttpServletRequest;
import javax.validation.Valid;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.List;
/**

90
oss-client/src/main/java/cn/axzo/oss/client/controller/WebFileController.java
View File

@ -6,15 +6,20 @@ import cn.axzo.framework.auth.domain.ContextInfo;
import cn.axzo.framework.auth.domain.ContextInfoHolder;
import cn.axzo.oss.client.vo.*;
import cn.axzo.oss.common.enums.ChannelTypeEnum;
import cn.axzo.oss.common.enums.FileDownloadTypeEnum;
import cn.axzo.oss.common.enums.FileUploadTypeEnum;
import cn.axzo.oss.common.enums.StorageUnitEnum;
import cn.axzo.oss.common.exception.BizException;
import cn.axzo.oss.common.utils.BeanConvertUtil;
import cn.axzo.oss.http.model.TemporaryUrlAccessRes;
import cn.axzo.oss.http.model.WebFileUploadVo;
import cn.axzo.oss.manager.api.dto.request.*;
import cn.axzo.oss.manager.api.dto.response.*;
import cn.axzo.oss.service.api.FileService;
import cn.azxo.framework.common.model.CommonResponse;
import cn.hutool.json.JSONUtil;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
@ -36,6 +41,7 @@ import static org.springframework.http.MediaType.MULTIPART_FORM_DATA_VALUE;
**/
@RestController
@RequestMapping("/webApi")
@Slf4j
public class WebFileController {
private static int FILE_NAME_MAX_LENGTH = 128;
@ -262,4 +268,88 @@ public class WebFileController {
WebFileUploadVo result = BeanConvertUtil.copyBean(response, WebFileUploadVo.class);
return CommonResponse.success(result);
}
/**
* obs：流式下载
*
* @param dto ServerFileDownloadDto
* @param response HttpServletResponse
*/
@SneakyThrows
@GetMapping("/v1/obs/getObject")
@CrossOrigin
public void getObject(@Valid ServerFileDownloadDto dto, HttpServletResponse response) {
ServerFileDownloadResponse result = fileService.getObject(dto, FileDownloadTypeEnum.STREAM_DOWNLOAD.getCode());
try (OutputStream outputStream = response.getOutputStream(); InputStream inputStream = result.getFileStream()) {
response.setContentType("image/jpg");
response.setCharacterEncoding("UTF-8");
response.addHeader("Content-Disposition", "attachment;filename="
+ result.getFileName() + "." + result.getFileFormat());
IOUtils.copy(inputStream, outputStream);
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* obs：断点续传下载
*
* @param dto ServerFileDownloadDto
* @param response HttpServletResponse
*/
@SneakyThrows
@GetMapping("/v1/obs/downloadFile")
@CrossOrigin
public void downloadFile(@Valid ServerFileDownloadDto dto, HttpServletResponse response) {
ServerFileDownloadResponse result = fileService.getObject(dto, FileDownloadTypeEnum.CHECK_POINT_DOWNLOAD.getCode());
try (OutputStream outputStream = response.getOutputStream(); InputStream inputStream = result.getFileStream()) {
response.setContentType("image/jpg");
response.setCharacterEncoding("UTF-8");
response.addHeader("Content-Disposition", "attachment;filename="
+ result.getFileName() + "." + result.getFileFormat());
IOUtils.copy(inputStream, outputStream);
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 远程下载接口
*
* @param fileUuId 文件uuid
* @param response HttpServletResponse
*/
@GetMapping("/v1/obs/downloadFileFromObs")
public void downloadFileFromObs(@RequestParam("fileUuid") String fileUuId, HttpServletResponse response) {
ServerFileDownloadDto dto = new ServerFileDownloadDto();
dto.setFileKey(fileUuId);
ServerFileDownloadResponse result = fileService.getObject(dto, FileDownloadTypeEnum.STREAM_DOWNLOAD.getCode());
try (OutputStream outputStream = response.getOutputStream(); InputStream inputStream = result.getFileStream()) {
response.setHeader("content-type","application/octet-stream");
response.setContentType("application/octet-stream");
response.setCharacterEncoding("UTF-8");
response.addHeader("Content-Disposition", "attachment;filename=" + result.getFileName() + "." + result.getFileFormat());
IOUtils.copy(inputStream, outputStream);
log.info("response设置文件流成功");
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 通过临时url访问OBS
*
* @param fileUuId 文件uuid
* @return TemporaryUrlAccessRes
*/
@GetMapping("/v1/obs/temporaryUrlAccess")
public CommonResponse<TemporaryUrlAccessRes> temporaryUrlAccess(@RequestParam("fileUuid") String fileUuId) {
ServerFileDownloadDto dto = new ServerFileDownloadDto();
dto.setFileKey(fileUuId);
ServerFileDownloadResponse response = fileService.getObject(dto, FileDownloadTypeEnum.TEMPORARY_URL_ACCESS.getCode());
TemporaryUrlAccessRes result = BeanConvertUtil.copyBean(response, TemporaryUrlAccessRes.class);
log.info("下载结果, result = {}", JSONUtil.toJsonStr(result));
return CommonResponse.success(result);
}
}

24
oss-common/src/main/java/cn/axzo/oss/common/enums/FileDownloadTypeEnum.java Normal file
View File

@ -0,0 +1,24 @@
package cn.axzo.oss.common.enums;
import lombok.Getter;
/**
* 文件下载类型枚举类
*
* @author hucf
* @since 2024/1/23 17:57
**/
@Getter
public enum FileDownloadTypeEnum {
STREAM_DOWNLOAD(1, "流式下载"),
CHECK_POINT_DOWNLOAD(2, "断点续传下载"),
TEMPORARY_URL_ACCESS(3, "通过临时url访问OBs")
;
private final Integer code;
private final String message;
FileDownloadTypeEnum(Integer code, String message) {
this.code = code;
this.message = message;
}
}

4
oss-http-api/pom.xml
View File

@ -35,6 +35,10 @@
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-openfeign-core</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-openfeign</artifactId>
</dependency>
</dependencies>
</project>

24
oss-http-api/src/main/java/cn/axzo/oss/http/api/DownloadFileApi.java Normal file
View File

@ -0,0 +1,24 @@
package cn.axzo.oss.http.api;
import cn.axzo.oss.http.model.TemporaryUrlAccessRes;
import cn.azxo.framework.common.model.CommonResponse;
import feign.Response;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestParam;
/**
* @author axzo
* @since 2024/3/7 16:56
**/
@FeignClient(
name = "oss",
url = "http://oss:9123"
)
public interface DownloadFileApi {
@GetMapping(value = "/webApi/v1/obs/downloadFileFromObs")
CommonResponse<Response> downloadFileFromObs(@RequestParam("fileUuid") String fileUuId);
@GetMapping(value = "/webApi/v1/obs/temporaryUrlAccess")
CommonResponse<TemporaryUrlAccessRes> temporaryUrlAccess(@RequestParam("fileUuid") String fileUuId);
}

4
oss-http-api/src/main/java/cn/axzo/oss/http/api/ServerFileServiceApi.java
View File

@ -3,9 +3,13 @@ package cn.axzo.oss.http.api;
import cn.axzo.oss.http.model.*;
import cn.azxo.framework.common.model.CommonResponse;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import feign.Response;
import java.util.List;
/**

38
oss-http-api/src/main/java/cn/axzo/oss/http/api/WebFileServiceApi.java Normal file
View File

@ -0,0 +1,38 @@
package cn.axzo.oss.http.api;
import cn.axzo.oss.http.model.WebFileUploadVo;
import cn.azxo.framework.common.model.CommonResponse;
import feign.Response;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RequestPart;
import org.springframework.web.multipart.MultipartFile;
import javax.validation.Valid;
import static org.springframework.http.MediaType.MULTIPART_FORM_DATA_VALUE;
/**
* @author wangsiqian
* @since 2024/02/27
*/
@FeignClient(
name = "oss",
url = "http://oss:9123"
)
public interface WebFileServiceApi {
/**
* OBS：文件上传（断点续传）
*
* @param appCode 应用编码
* @param bizScene 业务场景
* @param file MultipartFile
* @return WebFileUploadVo
*/
@PostMapping(value = "/webApi/v1/file2", consumes = MULTIPART_FORM_DATA_VALUE)
CommonResponse<WebFileUploadVo> uploadObs(@Valid @RequestParam("appCode") String appCode,
@Valid @RequestParam("bizScene") String bizScene,
@Valid @RequestPart MultipartFile file);
}

30
oss-http-api/src/main/java/cn/axzo/oss/http/model/DownloadFileFromObsRequest.java Normal file
View File

@ -0,0 +1,30 @@
package cn.axzo.oss.http.model;
import lombok.Data;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
/**
* @author hucf
* @since 2024/3/6 15:11
**/
@Data
public class DownloadFileFromObsRequest {
/**
* 文件uuid
*/
@NotBlank(message = "文件uuid不能为空")
private String fileKey;
/**
* 图片样式
*/
private String style;
/**
* 下载方式：1:流式下载；2:断点续传方式下载
*/
@NotNull(message = "下载方式不能为空")
private Integer fileDownloadType;
}

46
oss-http-api/src/main/java/cn/axzo/oss/http/model/DownloadFileFromObsResponse.java Normal file
View File

@ -0,0 +1,46 @@
package cn.axzo.oss.http.model;
import lombok.Data;
import java.io.InputStream;
/**
* @author hucf
* @since 2024/3/6 15:15
**/
@Data
public class DownloadFileFromObsResponse {
/**
* 文件 URL
*/
private String url;
/**
* 文件 Key
*/
private String fileKey;
/**
* URL MD5
*/
private String urlMd5;
/**
* 文件名称
*/
private String fileName;
/**
* 文件大小
*/
private Long fileSize;
/**
* 文件格式
*/
private String fileFormat;
/**
* 文件流
*/
private InputStream fileStream;
}

44
oss-http-api/src/main/java/cn/axzo/oss/http/model/TemporaryUrlAccessRes.java Normal file
View File

@ -0,0 +1,44 @@
package cn.axzo.oss.http.model;
import lombok.Data;
/**
* @author hucf
* @since 2024/3/11 18:21
**/
@Data
public class TemporaryUrlAccessRes {
/**
* 文件 URL
*/
private String url;
/**
* 文件 Key
*/
private String fileKey;
/**
* URL MD5
*/
private String urlMd5;
/**
* 文件名称
*/
private String fileName;
/**
* 文件大小
*/
private Long fileSize;
/**
* 文件格式
*/
private String fileFormat;
/**
* obs:临时url
*/
private String signedUrl;
}

2
oss-client/src/main/java/cn/axzo/oss/client/vo/WebFileUploadVo.java → oss-http-api/src/main/java/cn/axzo/oss/http/model/WebFileUploadVo.java
View File

@ -1,4 +1,4 @@
package cn.axzo.oss.client.vo;
package cn.axzo.oss.http.model;
import lombok.Data;

6
oss-integration/src/main/java/cn/axzo/oss/integration/s3/HuaWeiCloudService.java
View File

@ -1,5 +1,7 @@
package cn.axzo.oss.integration.s3;
import com.obs.services.model.TemporarySignatureResponse;
import java.io.InputStream;
/**
@ -14,4 +16,8 @@ public interface HuaWeiCloudService {
String checkPointUploadFile(String bucketName, String fileName, String appCode, InputStream srcStream, String filePath);
String multipartUpload(String bucketName, String fileName, String appCode, InputStream srcStream);
InputStream getObject(String bucketName, String objectKey, Integer fileDownloadType);
TemporarySignatureResponse createTemporarySignature(String bucketName, String objectKey);
}

95
oss-integration/src/main/java/cn/axzo/oss/integration/s3/impl/HuaWeiCloudServiceImpl.java
View File

@ -1,10 +1,12 @@
package cn.axzo.oss.integration.s3.impl;
import cn.axzo.oss.common.enums.FileDownloadTypeEnum;
import cn.axzo.oss.integration.s3.HuaWeiCloudService;
import cn.axzo.oss.integration.s3.client.HuaWeiCloudObsClient;
import cn.axzo.oss.integration.s3.config.HuaWeiCloudObsConfig;
import cn.azxo.framework.common.utils.LogUtil;
import cn.hutool.core.io.FileUtil;
import cn.hutool.json.JSONUtil;
import com.obs.services.ObsClient;
import com.obs.services.exception.ObsException;
import com.obs.services.model.*;
@ -15,6 +17,7 @@ import org.springframework.stereotype.Service;
import java.io.InputStream;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
/**
@ -225,4 +228,96 @@ public class HuaWeiCloudServiceImpl implements HuaWeiCloudService {
request.setObjectMetadata(metadata);
return obsClient.uploadFile(request);
}
/**
* obs下载文件：流式下载
*
* @param bucketName 桶名称
* @param objectKey 对象名称
* @return InputStream
*/
@Override
public InputStream getObject(String bucketName, String objectKey, Integer fileDownloadType) {
ObsClient obsClient = huaWeiCloudObsClient.getClient();
if (FileDownloadTypeEnum.STREAM_DOWNLOAD.getCode().equals(fileDownloadType)) {
try {
GetObjectRequest request = new GetObjectRequest();
request.setBucketName(bucketName);
request.setObjectKey(objectKey);
ObsObject object = obsClient.getObject(request);
log.info("下载华为云OBS文件成功");
return object.getObjectContent();
} catch (ObsException obsException) {
log.warn("下载华为云OBS文件失败，HTTP Code:{}, Error Code:{}, Request ID:{}, Host ID:{}, Error Message:{}",
obsException.getResponseCode(),
obsException.getErrorCode(),
obsException.getErrorRequestId(),
obsException.getErrorHostId(),
obsException.getErrorMessage());
}
} else if (FileDownloadTypeEnum.CHECK_POINT_DOWNLOAD.getCode().equals(fileDownloadType)) {
try {
DownloadFileRequest request = new DownloadFileRequest(bucketName, objectKey);
// 设置下载对象的本地文件全路径，当该值为空时，默认为当前程序的运行目录。
request.setDownloadFile("");
// 设置分段下载时的最大并发数
request.setTaskNum(Integer.parseInt(huaWeiCloudObsConfig.getTaskNum()));
// 设置分段大小为10MB
request.setPartSize(Long.parseLong(huaWeiCloudObsConfig.getPartSize()));
// 是否开启断点续传模式
request.setEnableCheckpoint(Boolean.parseBoolean(huaWeiCloudObsConfig.getEnableCheckPoint()));
// 是否自动解码响应头
request.setIsEncodeHeaders(Boolean.parseBoolean(huaWeiCloudObsConfig.getEncodeHeaders()));
DownloadFileResult downloadFileResult = obsClient.downloadFile(request);
String etag = downloadFileResult.getObjectMetadata().getEtag();
log.info("下载华为云OBS文件成功");
} catch (NumberFormatException e) {
throw new RuntimeException(e);
} catch (ObsException obsException) {
log.warn("下载华为云OBS文件失败，HTTP Code:{}, Error Code:{}, Request ID:{}, Host ID:{}, Error Message:{}",
obsException.getResponseCode(),
obsException.getErrorCode(),
obsException.getErrorRequestId(),
obsException.getErrorHostId(),
obsException.getErrorMessage());
}
}
return null;
}
/**
* 获取OBS临时url
*
* @param bucketName 桶名称
* @param objectKey 对象名称
* @return TemporarySignatureResponse
*/
@Override
public TemporarySignatureResponse createTemporarySignature(String bucketName, String objectKey) {
TemporarySignatureRequest request = new TemporarySignatureRequest();
// 桶名称
request.setBucketName(bucketName);
// 对象名称
request.setObjectKey(objectKey);
// HTTP方法类型
request.setMethod(HttpMethodEnum.GET);
// 带授权信息的URL的过期时间300秒
request.setExpires(300L);
// 发起请求的时间
request.setRequestDate(new Date());
try {
ObsClient obsClient = huaWeiCloudObsClient.getClient();
TemporarySignatureResponse response = obsClient.createTemporarySignature(request);
log.info("response = {}", JSONUtil.toJsonStr(response));
return response;
} catch (Exception exception) {
log.warn("获取临时url发生异常, exception = {}", exception.getMessage());
log.warn("异常信息：");
exception.printStackTrace();
return null;
}
}
}

4
oss-manager-api/src/main/java/cn/axzo/oss/manager/api/FileManager.java
View File

@ -51,4 +51,8 @@ public interface FileManager {
String multipartUploadComplete(String bucketName, String tgtFileKey, String uploadId, List<PartETag> partETags);
String multipartUploadFile(String bucketName, String tgtFileKey, MultipartFile file);
InputStream downloadObsFile(String bucketName, String objectKey, Integer fileDownloadType);
String getTempUrlFromOns(String bucketName, String objectKey);
}

5
oss-manager-api/src/main/java/cn/axzo/oss/manager/api/dto/response/ServerFileDownloadResponse.java
View File

@ -45,4 +45,9 @@ public class ServerFileDownloadResponse {
* 文件流
*/
private InputStream fileStream;
/**
* obs:临时url
*/
private String signedUrl;
}

10
oss-manager/src/main/java/cn/axzo/oss/manager/impl/FileManagerImpl.java
View File

@ -9,6 +9,7 @@ import cn.axzo.oss.integration.s3.HuaWeiCloudService;
import cn.axzo.oss.manager.api.FileManager;
import cn.axzo.oss.manager.api.dto.PartETag;
import cn.axzo.oss.manager.api.dto.request.MultipartUploadDto;
import com.obs.services.model.TemporarySignatureResponse;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
@ -124,4 +125,13 @@ public class FileManagerImpl implements FileManager {
}
return partETagList;
}
public InputStream downloadObsFile(String bucketName, String objectKey, Integer fileDownloadType) {
return huaWeiCloudService.getObject(bucketName, objectKey, fileDownloadType);
}
public String getTempUrlFromOns(String bucketName, String objectKey) {
TemporarySignatureResponse response = huaWeiCloudService.createTemporarySignature(bucketName, objectKey);
return response.getSignedUrl();
}
}

2
oss-service-api/src/main/java/cn/axzo/oss/service/api/FileService.java
View File

@ -51,4 +51,6 @@ public interface FileService {
Integer fileUploadType);
String getFilePath(MultipartFile file);
ServerFileDownloadResponse getObject(ServerFileDownloadDto dto, Integer fileDownloadType);
}

34
oss-service/src/main/java/cn/axzo/oss/service/impl/FileServiceImpl.java
View File

@ -697,4 +697,38 @@ public class FileServiceImpl implements FileService {
throw new RuntimeException(e);
}
}
public ServerFileDownloadResponse getObject(ServerFileDownloadDto dto, Integer fileDownloadType) {
log.info("obs download file = {}", JsonUtil.obj2Str(dto));
File file = null;
String fileKey = dto.getFileKey();
if (fileKey.contains(IS_URL)) {
String urlMd5 = Utility.getMd5(fileKey);
file = fileDao.getByUrlMd5(urlMd5);
} else {
file = fileDao.getByFileUuid(fileKey);
}
if (Utility.objIsNull(file)) {
log.warn("obs download file is null, fileKey = {}", fileKey);
BizException.error(false, CodeEnum.FILE_NOT_FOUND);
}
// 对象名称：目录+文件名
String objectKey = file.getDirectory() + SEPARATOR + file.getFileName();
// 桶名称
String bucketName = file.getBucketName();
InputStream fileStream = null;
String tempUrlFromOns = null;
if (FileDownloadTypeEnum.TEMPORARY_URL_ACCESS.getCode().equals(fileDownloadType)) {
tempUrlFromOns = fileManager.getTempUrlFromOns(bucketName, objectKey);
} else {
fileStream = fileManager.downloadObsFile(bucketName, objectKey, fileDownloadType);
if (Objects.isNull(fileStream)) {
log.warn("下载的文件流为空");
}
}
ServerFileDownloadResponse response = setFileDownloadResponse(file, fileStream);
response.setSignedUrl(tempUrlFromOns);
return response;
}
}