diff --git a/README.md b/README.md index cdf2eda..2ed0473 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,44 @@ -# oss +# oss 对象存储服务 -存储系统 \ No newline at end of file +## 介绍 +- 项目介绍说明地址:https://showdoc.axzo.cn/web/#/102?page_id=987 +- 项目接入说明地址:https://showdoc.axzo.cn/web/#/102?page_id=908 +- 项目API说明地址:https://yapi.axzo.cn/project/950/interface/api + +## 业务支持场景 + +### 业务服务新接入进行sql语句配置 +以前端app有文件上传需求,进行接入oss服务进行sql语句配置为例:appCode=app +```mysql +INSERT INTO `app_channel_bucket` (`app_channel_bucket_no`, `app_code`, `channel_code`, `bucket_name`, `access_control`, `remark`, `extension`, `create_at`, `update_at`, `create_by`, `update_by`, `is_delete`) VALUES ('app:aliyun:app', 'app', 'aliyun', 'axzo-public', '', NULL, NULL, now(), now(), '', '', 0); +INSERT INTO `file_app` (`app_code`, `app_name`, `status`, `remark`, `extension`, `create_at`, `update_at`, `create_by`, `update_by`, `is_delete`) VALUES ('app', 'app', 1, 'app', NULL, now(), now(), '', '', 0); +INSERT INTO `file_business_scene` (`app_channel_bucket_no`, `app_code`, `channel_code`, `bucket_name`, `business_scene`, `directory`, `create_at`, `update_at`, `create_by`, `update_by`, `is_delete`) VALUES ('app:aliyun:app', 'app', 'aliyun', 'axzo-public', 'app', 'app/app', now(), now(), NULL, NULL, 0); +INSERT INTO `file_upload_config` (`app_channel_bucket_no`, `app_code`, `channel_code`, `bucket_name`, `directory`, `storage_unit`, `storage_size`, `file_format`, `create_at`, `update_at`, `create_by`, `update_by`, `is_delete`) VALUES ('app:aliyun:app', 'app', 'aliyun', 'axzo-public', 'app/app', 'MB', 100, 'png,jpg,xls,xlsx,docx,pdf,zip,dwg,doc,jpeg', now(), now(), NULL, NULL, 0); +``` + +### 已接入业务服务支持文件格式配置 +例如appCode=cms,需要支持rar格式文件上传, 支持处理的方式为: +修改file_upload_config表对字段file_format进行调整,执行以下语句即可: +```mysql +update `file_upload_config` SET `file_format` = 'png,jpg,zip,jpeg,pdf,gif,xls,doc,docx,xlsx,bmp,rar' WHERE `app_code` = 'cms' +``` + +## 当前提供的能力 +- 文件上传功能 +- 上传功能对接操作日志记录服务。 +- 对接用户授权获取用户信息。 +- 根据fileKey获取文件流返回。 +- 获取文件url兼容处理,传输的为url直接返回。 +- 支持app端历史数据不存在,进行新增处理。 +- 获取文件Url支持返回原文件名。 +- 支持图片处理尺寸样式。 +- 文件流下载。 +- 分片上传。 + +## 未来可扩展 + +- oss服务业务支持场景手动配置sql语句可切换为管理界面处理,效率提高,可追溯。方案: + - oss封装提供给表的curd api接口。 + - 基础技术管理后台对接调用 oss提供的api接口进行业务接入或者已接入业务服务调整文件格式配置。 +- oss服务重构,简化流程 + - 多子模块架构,调用链路太长,新功能支持开发较繁琐。 diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000..a033ae0 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,2 @@ +# 发布记录 + diff --git a/oss-client/src/main/java/cn/axzo/oss/client/controller/WebFileController.java b/oss-client/src/main/java/cn/axzo/oss/client/controller/WebFileController.java index 399ee1c..145ad69 100644 --- a/oss-client/src/main/java/cn/axzo/oss/client/controller/WebFileController.java +++ b/oss-client/src/main/java/cn/axzo/oss/client/controller/WebFileController.java @@ -4,18 +4,11 @@ import cn.axzo.core.utils.converter.BeanConverter; import cn.axzo.framework.auth.annotation.PreBuildContext; import cn.axzo.framework.auth.domain.ContextInfo; import cn.axzo.framework.auth.domain.ContextInfoHolder; -import cn.axzo.oss.client.vo.FileInformationVo; -import cn.axzo.oss.client.vo.FindFileUrlVo; -import cn.axzo.oss.client.vo.WebFileUploadVo; +import cn.axzo.oss.client.vo.*; import cn.axzo.oss.common.exception.BizException; import cn.axzo.oss.common.utils.BeanConvertUtil; -import cn.axzo.oss.manager.api.dto.request.FindFileUrlDto; -import cn.axzo.oss.manager.api.dto.request.ServerFileDownloadDto; -import cn.axzo.oss.manager.api.dto.request.ServerFileUploadDto; -import cn.axzo.oss.manager.api.dto.response.FileInformationResponse; -import cn.axzo.oss.manager.api.dto.response.FindFileUrlResponse; -import cn.axzo.oss.manager.api.dto.response.ServerFileDownloadResponse; -import cn.axzo.oss.manager.api.dto.response.ServerFileUploadResponse; +import cn.axzo.oss.manager.api.dto.request.*; +import cn.axzo.oss.manager.api.dto.response.*; import cn.axzo.oss.service.api.FileService; import cn.azxo.framework.common.model.CommonResponse; import lombok.SneakyThrows; @@ -99,6 +92,48 @@ public class WebFileController { return CommonResponse.success(result); } + @SneakyThrows + @PostMapping("/v1/multipart-upload/init") + @CrossOrigin + public CommonResponse multipartUploadInit(@Valid @RequestBody MultipartUploadInitDto dto) { + MultipartUploadInitResponse response = fileService.multipartUploadInit(dto); + return CommonResponse.success(BeanConvertUtil.copyBean(response, MultipartUploadInitVo.class)); + } + + @SneakyThrows + @PostMapping(value = "/v1/multipart-upload", consumes = MULTIPART_FORM_DATA_VALUE) + @CrossOrigin + public CommonResponse multipartUpload(@Valid @RequestParam("appCode") String appCode, + @Valid @RequestParam("bizScene") String bizScene, + @Valid @RequestParam("fileName") String fileName, + @Valid @RequestParam("tgtFileKey") String tgtFileKey, + @Valid @RequestParam("partNumber") Integer partNumber, + @Valid @RequestParam("partSize") Long partSize, + @Valid @RequestParam("uploadId") String uploadId, + @Valid @RequestPart MultipartFile file) { + MultipartUploadDto dto = MultipartUploadDto.builder() + .appCode(appCode) + .bizScene(bizScene) + .fileName(fileName) + .tgtFileKey(tgtFileKey) + .fileContent(file.getBytes()) + .partNumber(partNumber) + .partSize(partSize) + .uploadId(uploadId) + .build(); + MultipartUploadResponse response = fileService.multipartUpload(dto); + return CommonResponse.success(BeanConvertUtil.copyBean(response, MultipartUploadVo.class)); + } + + @PostMapping("/v1/multipart-upload/complete") + @CrossOrigin + @SneakyThrows + //@PreBuildContext + public CommonResponse multipartUploadComplete(@Valid @RequestBody MultipartUploadCompleteDto dto) { + FileInformationResponse response = fileService.multipartUploadComplete(dto); + return CommonResponse.success(BeanConvertUtil.copyBean(response, FileInformationVo.class)); + } + @SneakyThrows @PostMapping("/v1/file/getUrl") @CrossOrigin diff --git a/oss-client/src/main/java/cn/axzo/oss/client/vo/MultipartUploadInitVo.java b/oss-client/src/main/java/cn/axzo/oss/client/vo/MultipartUploadInitVo.java new file mode 100644 index 0000000..a5d96ed --- /dev/null +++ b/oss-client/src/main/java/cn/axzo/oss/client/vo/MultipartUploadInitVo.java @@ -0,0 +1,17 @@ +package cn.axzo.oss.client.vo; + +import lombok.Builder; +import lombok.Data; + +/** + * @Author: liyong.tian + * @Date: 2023/8/15 17:29 + * @Description: + */ +@Data +public class MultipartUploadInitVo { + + private String uploadId; + + private String tgtFileKey; +} diff --git a/oss-client/src/main/java/cn/axzo/oss/client/vo/MultipartUploadVo.java b/oss-client/src/main/java/cn/axzo/oss/client/vo/MultipartUploadVo.java new file mode 100644 index 0000000..3d8bc24 --- /dev/null +++ b/oss-client/src/main/java/cn/axzo/oss/client/vo/MultipartUploadVo.java @@ -0,0 +1,17 @@ +package cn.axzo.oss.client.vo; + +import cn.axzo.oss.manager.api.dto.PartETag; +import lombok.Data; + +/** + * @Author: liyong.tian + * @Date: 2023/8/15 11:56 + * @Description: + */ +@Data +public class MultipartUploadVo { + + private String uploadId; + + private PartETag partETag; +} diff --git a/oss-client/src/main/resources/bootstrap.yml b/oss-client/src/main/resources/bootstrap.yml index 5d6f794..579e4c7 100644 --- a/oss-client/src/main/resources/bootstrap.yml +++ b/oss-client/src/main/resources/bootstrap.yml @@ -9,7 +9,7 @@ spring: namespace: ${NACOS_NAMESPACE_ID:35eada10-9574-4db8-9fea-bc6a4960b6c7} prefix: ${spring.application.name} profiles: - active: ${NACOS_PROFILES_ACTIVE:dev} + active: ${NACOS_PROFILES_ACTIVE:local} main: allow-bean-definition-overriding: true diff --git a/oss-common/src/main/java/cn/axzo/oss/common/enums/CodeEnum.java b/oss-common/src/main/java/cn/axzo/oss/common/enums/CodeEnum.java index a1b6830..f6bbdbe 100644 --- a/oss-common/src/main/java/cn/axzo/oss/common/enums/CodeEnum.java +++ b/oss-common/src/main/java/cn/axzo/oss/common/enums/CodeEnum.java @@ -44,6 +44,9 @@ public enum CodeEnum implements EnumBase { FILE_NAME_TOO_LONG(111, "file name too long max 128"), FILE_NOT_FOUND(112, "file not found"), + MULTIPART_UPLOAD_INIT_ERROR(113, "file multipart upload init error"), + MULTIPART_UPLOAD_ERROR(114, "file multipart upload error"), + MULTIPART_UPLOAD_COMPLETE_ERROR(115, "file multipart upload complete error") ; diff --git a/oss-common/src/main/java/cn/axzo/oss/common/utils/BeanConvertUtil.java b/oss-common/src/main/java/cn/axzo/oss/common/utils/BeanConvertUtil.java index 32a54cd..19905bf 100644 --- a/oss-common/src/main/java/cn/axzo/oss/common/utils/BeanConvertUtil.java +++ b/oss-common/src/main/java/cn/axzo/oss/common/utils/BeanConvertUtil.java @@ -1,9 +1,15 @@ package cn.axzo.oss.common.utils; +import cn.hutool.core.collection.CollUtil; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.BeanUtils; import org.springframework.beans.BeansException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.function.BiConsumer; + /** * @Author admin * @Description @@ -25,4 +31,34 @@ public class BeanConvertUtil { } return target; } + + public static List copyList(Collection list, Class targetCls) { + List result = new ArrayList(); + if (list == null || list.size() == 0) { + return result; + } + + for (Object obj : list) { + result.add(copyBean(obj, targetCls)); + } + + return result; + } + + public static List copyList(Collection list, Class targetCls, BiConsumer action) { + List result = new ArrayList(); + if (CollUtil.isEmpty(list)) { + return result; + } + + for (S obj : list) { + T res = copyBean(obj, targetCls); + if (action != null) { + action.accept(obj, res); + } + result.add(res); + } + + return result; + } } diff --git a/oss-integration/src/main/java/cn/axzo/oss/integration/s3/base/BaseS3Service.java b/oss-integration/src/main/java/cn/axzo/oss/integration/s3/base/BaseS3Service.java index 1b563dd..04287a6 100644 --- a/oss-integration/src/main/java/cn/axzo/oss/integration/s3/base/BaseS3Service.java +++ b/oss-integration/src/main/java/cn/axzo/oss/integration/s3/base/BaseS3Service.java @@ -1,10 +1,12 @@ package cn.axzo.oss.integration.s3.base; +import com.aliyun.oss.model.PartETag; import org.springframework.http.ResponseEntity; import org.springframework.web.multipart.MultipartFile; import java.io.FilterInputStream; import java.io.InputStream; +import java.util.List; /** * @Author admin @@ -54,5 +56,10 @@ public interface BaseS3Service { FilterInputStream getS3ObjectStream(String tgtFileKey); + String multipartUploadInit(String bucketName, String tgtFileKey); + PartETag multipartUpload(String bucketName, String tgtFileKey, String uploadId, InputStream instream, + Integer partNumber, long partSize); + + String multipartUploadComplete(String bucketName, String tgtFileKey, String uploadId, List partETags); } diff --git a/oss-integration/src/main/java/cn/axzo/oss/integration/s3/impl/AliOssServiceImpl.java b/oss-integration/src/main/java/cn/axzo/oss/integration/s3/impl/AliOssServiceImpl.java index 8c9e50c..6104bdf 100644 --- a/oss-integration/src/main/java/cn/axzo/oss/integration/s3/impl/AliOssServiceImpl.java +++ b/oss-integration/src/main/java/cn/axzo/oss/integration/s3/impl/AliOssServiceImpl.java @@ -1,14 +1,13 @@ package cn.axzo.oss.integration.s3.impl; +import cn.axzo.oss.common.exception.BizException; import cn.axzo.oss.integration.s3.AliOssService; import cn.axzo.oss.integration.s3.client.AliOssAppProClient; import cn.axzo.oss.integration.s3.client.AliOssClient; import cn.azxo.framework.common.utils.LogUtil; import com.aliyun.oss.OSS; import com.aliyun.oss.OSSException; -import com.aliyun.oss.model.GetObjectRequest; -import com.aliyun.oss.model.OSSObject; -import com.aliyun.oss.model.ObjectMetadata; +import com.aliyun.oss.model.*; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -19,8 +18,10 @@ import org.springframework.web.multipart.MultipartFile; import java.io.FilterInputStream; import java.io.InputStream; import java.net.URLEncoder; +import java.util.List; import static cn.axzo.oss.common.constans.CommonConstants.APP_PRO_BUCKET_NAME; +import static cn.axzo.oss.common.enums.CodeEnum.*; /** * @program: oss @@ -65,6 +66,10 @@ public class AliOssServiceImpl implements AliOssService { return ""; } + return getUrl(bucketName, tgtFileKey); + } + + private String getUrl(String bucketName, String tgtFileKey) { StringBuilder allBuilder = new StringBuilder(); allBuilder.append("https://"); allBuilder.append(bucketName); @@ -141,4 +146,64 @@ public class AliOssServiceImpl implements AliOssService { public FilterInputStream getS3ObjectStream(String tgtFileKey) { return null; } + + @Override + public String multipartUploadInit(String bucketName, String tgtFileKey) { + OSS client = aliOssClient.getClient(); + try { + // 创建分片上传对象 + InitiateMultipartUploadRequest uploadRequest = new InitiateMultipartUploadRequest(bucketName, tgtFileKey); + // 初始化分片 + InitiateMultipartUploadResult result = client.initiateMultipartUpload(uploadRequest); + // 返回uploadId,它是分片上传事件的唯一标识,您可以根据这个uploadId发起相关的操作,如取消分片上传、查询分片上传等。 + return result.getUploadId(); + } catch (Exception e) { + e.printStackTrace(); + LogUtil.error("ali oss multipart upload init error = {}", e); + throw new BizException(MULTIPART_UPLOAD_INIT_ERROR); + } + } + + @Override + public PartETag multipartUpload(String bucketName, String tgtFileKey, String uploadId, InputStream instream, + Integer partNumber, long partSize) { + OSS client = aliOssClient.getClient(); + try { + UploadPartRequest partRequest = new UploadPartRequest(); + // 阿里云 oss 文件根目录 + partRequest.setBucketName(bucketName); + // 文件key + partRequest.setKey(tgtFileKey); + // 分片上传uploadId + partRequest.setUploadId(uploadId); + // 分片文件 + instream.skip((partNumber - 1) * partSize); + partRequest.setInputStream(instream); + // 分片大小。除了最后一个分片没有大小限制,其他的分片最小为100 KB。 + partRequest.setPartSize(partSize); + // 分片号。每一个上传的分片都有一个分片号,取值范围是1~10000,如果超出这个范围,OSS将返回InvalidArgument的错误码。 + partRequest.setPartNumber(partNumber); + // 每个分片不需要按顺序上传,甚至可以在不同客户端上传,OSS会按照分片号排序组成完整的文件。 + UploadPartResult uploadPartResult = client.uploadPart(partRequest); + // 每次上传分片之后,OSS的返回结果包含PartETag。 + return uploadPartResult.getPartETag(); + } catch (Exception e) { + e.printStackTrace(); + throw new BizException(MULTIPART_UPLOAD_ERROR); + } + } + + @Override + public String multipartUploadComplete(String bucketName, String tgtFileKey, String uploadId, List partETags) { + OSS client = aliOssClient.getClient(); + try { + CompleteMultipartUploadRequest completeMultipartUploadRequest = + new CompleteMultipartUploadRequest(bucketName, tgtFileKey, uploadId, partETags); + client.completeMultipartUpload(completeMultipartUploadRequest); + return getUrl(bucketName, tgtFileKey); + } catch (Exception e) { + e.printStackTrace(); + throw new BizException(MULTIPART_UPLOAD_COMPLETE_ERROR); + } + } } diff --git a/oss-manager-api/src/main/java/cn/axzo/oss/manager/api/FileManager.java b/oss-manager-api/src/main/java/cn/axzo/oss/manager/api/FileManager.java index 20b6ad6..87372c4 100644 --- a/oss-manager-api/src/main/java/cn/axzo/oss/manager/api/FileManager.java +++ b/oss-manager-api/src/main/java/cn/axzo/oss/manager/api/FileManager.java @@ -1,6 +1,11 @@ package cn.axzo.oss.manager.api; +import cn.axzo.oss.manager.api.dto.request.MultipartUploadDto; +import cn.axzo.oss.manager.api.dto.response.MultipartUploadResponse; +import com.aliyun.oss.model.PartETag; + import java.io.InputStream; +import java.util.List; /** * @Author admin @@ -37,4 +42,9 @@ public interface FileManager { */ InputStream downloadFile(String bucketName, String tgtFileKey, String style); + String multipartUploadInit(String bucketName, String tgtFileKey); + + PartETag multipartUpload(String bucketName, String tgtFileKey, byte[] fileContent, MultipartUploadDto dto); + + String multipartUploadComplete(String bucketName, String tgtFileKey, String uploadId, List partETags); } diff --git a/oss-manager-api/src/main/java/cn/axzo/oss/manager/api/dto/PartETag.java b/oss-manager-api/src/main/java/cn/axzo/oss/manager/api/dto/PartETag.java new file mode 100644 index 0000000..58d0ee6 --- /dev/null +++ b/oss-manager-api/src/main/java/cn/axzo/oss/manager/api/dto/PartETag.java @@ -0,0 +1,17 @@ +package cn.axzo.oss.manager.api.dto; + +import lombok.Data; + +/** + * @Author: liyong.tian + * @Date: 2023/8/18 16:13 + * @Description: + */ +@Data +public class PartETag { + + private int partNumber; + private String eTag; + private long partSize; + private Long partCRC; +} diff --git a/oss-manager-api/src/main/java/cn/axzo/oss/manager/api/dto/request/MultipartUploadCompleteDto.java b/oss-manager-api/src/main/java/cn/axzo/oss/manager/api/dto/request/MultipartUploadCompleteDto.java new file mode 100644 index 0000000..c6cfdfc --- /dev/null +++ b/oss-manager-api/src/main/java/cn/axzo/oss/manager/api/dto/request/MultipartUploadCompleteDto.java @@ -0,0 +1,29 @@ +package cn.axzo.oss.manager.api.dto.request; + +import cn.axzo.oss.manager.api.dto.PartETag; +import lombok.Builder; +import lombok.Data; + +import java.util.List; + +/** + * @Author: liyong.tian + * @Date: 2023/8/15 15:03 + * @Description: + */ +@Data +@Builder +public class MultipartUploadCompleteDto { + + private String appCode; + + private String bizScene; + + private String fileName; + + private String tgtFileKey; + + private String uploadId; + + private List partETags; +} diff --git a/oss-manager-api/src/main/java/cn/axzo/oss/manager/api/dto/request/MultipartUploadDto.java b/oss-manager-api/src/main/java/cn/axzo/oss/manager/api/dto/request/MultipartUploadDto.java new file mode 100644 index 0000000..e87efca --- /dev/null +++ b/oss-manager-api/src/main/java/cn/axzo/oss/manager/api/dto/request/MultipartUploadDto.java @@ -0,0 +1,49 @@ +package cn.axzo.oss.manager.api.dto.request; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import javax.validation.constraints.NotBlank; + +/** + * @Author: liyong.tian + * @Date: 2023/8/15 11:39 + * @Description: + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class MultipartUploadDto { + + @NotBlank + private String appCode; + + @NotBlank + private String bizScene; + + @NotBlank + private String fileName; + + @NotBlank + private String tgtFileKey; + + private byte[] fileContent; + + /** + * 当前为第几分片 + */ + private int partNumber; + + /** + * 当前分片大小 + */ + private long partSize; + + /** + * oss上传时的上传id + */ + private String uploadId; +} diff --git a/oss-manager-api/src/main/java/cn/axzo/oss/manager/api/dto/request/MultipartUploadInitDto.java b/oss-manager-api/src/main/java/cn/axzo/oss/manager/api/dto/request/MultipartUploadInitDto.java new file mode 100644 index 0000000..e1cd593 --- /dev/null +++ b/oss-manager-api/src/main/java/cn/axzo/oss/manager/api/dto/request/MultipartUploadInitDto.java @@ -0,0 +1,29 @@ +package cn.axzo.oss.manager.api.dto.request; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import javax.validation.constraints.NotBlank; + +/** + * @Author: liyong.tian + * @Date: 2023/8/15 10:28 + * @Description: + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class MultipartUploadInitDto { + + @NotBlank + private String appCode; + + @NotBlank + private String bizScene; + + @NotBlank + private String fileName; +} diff --git a/oss-manager-api/src/main/java/cn/axzo/oss/manager/api/dto/response/MultipartUploadInitResponse.java b/oss-manager-api/src/main/java/cn/axzo/oss/manager/api/dto/response/MultipartUploadInitResponse.java new file mode 100644 index 0000000..f66c957 --- /dev/null +++ b/oss-manager-api/src/main/java/cn/axzo/oss/manager/api/dto/response/MultipartUploadInitResponse.java @@ -0,0 +1,18 @@ +package cn.axzo.oss.manager.api.dto.response; + +import lombok.Builder; +import lombok.Data; + +/** + * @Author: liyong.tian + * @Date: 2023/8/15 17:28 + * @Description: + */ +@Data +@Builder +public class MultipartUploadInitResponse { + + private String uploadId; + + private String tgtFileKey; +} diff --git a/oss-manager-api/src/main/java/cn/axzo/oss/manager/api/dto/response/MultipartUploadResponse.java b/oss-manager-api/src/main/java/cn/axzo/oss/manager/api/dto/response/MultipartUploadResponse.java new file mode 100644 index 0000000..a63f10e --- /dev/null +++ b/oss-manager-api/src/main/java/cn/axzo/oss/manager/api/dto/response/MultipartUploadResponse.java @@ -0,0 +1,20 @@ +package cn.axzo.oss.manager.api.dto.response; + +import cn.axzo.oss.manager.api.dto.PartETag; +import lombok.Builder; +import lombok.Data; + + +/** + * @Author: liyong.tian + * @Date: 2023/8/15 13:39 + * @Description: + */ +@Data +@Builder +public class MultipartUploadResponse { + + private String uploadId; + + private PartETag partETag; +} diff --git a/oss-manager/src/main/java/cn/axzo/oss/manager/impl/FileManagerImpl.java b/oss-manager/src/main/java/cn/axzo/oss/manager/impl/FileManagerImpl.java index 19d5754..2be8d4d 100644 --- a/oss-manager/src/main/java/cn/axzo/oss/manager/impl/FileManagerImpl.java +++ b/oss-manager/src/main/java/cn/axzo/oss/manager/impl/FileManagerImpl.java @@ -2,12 +2,16 @@ package cn.axzo.oss.manager.impl; import cn.axzo.oss.integration.s3.AliOssService; import cn.axzo.oss.manager.api.FileManager; +import cn.axzo.oss.manager.api.dto.request.MultipartUploadDto; +import cn.axzo.oss.manager.api.dto.response.MultipartUploadResponse; +import com.aliyun.oss.model.PartETag; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.io.ByteArrayInputStream; import java.io.InputStream; +import java.util.List; /** * @Author admin @@ -50,4 +54,21 @@ public class FileManagerImpl implements FileManager { return aliOssService.downloadFile(bucketName, tgtFileKey, style); } + @Override + public String multipartUploadInit(String bucketName, String tgtFileKey) { + return aliOssService.multipartUploadInit(bucketName, tgtFileKey); + } + + @Override + public PartETag multipartUpload(String bucketName, String tgtFileKey, byte[] fileContent, MultipartUploadDto dto) { + InputStream inputStream = new ByteArrayInputStream(fileContent); + + return aliOssService.multipartUpload(bucketName, tgtFileKey, dto.getUploadId(), inputStream, + dto.getPartNumber(), dto.getPartSize()); + } + + @Override + public String multipartUploadComplete(String bucketName, String tgtFileKey, String uploadId, List partETags) { + return aliOssService.multipartUploadComplete(bucketName, tgtFileKey, uploadId, partETags); + } } diff --git a/oss-service-api/src/main/java/cn/axzo/oss/service/api/FileService.java b/oss-service-api/src/main/java/cn/axzo/oss/service/api/FileService.java index 36b0bea..6c3aa45 100644 --- a/oss-service-api/src/main/java/cn/axzo/oss/service/api/FileService.java +++ b/oss-service-api/src/main/java/cn/axzo/oss/service/api/FileService.java @@ -4,6 +4,7 @@ import cn.axzo.framework.auth.domain.ContextInfo; import cn.axzo.oss.manager.api.dto.request.*; import cn.axzo.oss.manager.api.dto.response.*; +import java.io.InputStream; import java.util.List; /** @@ -35,4 +36,10 @@ public interface FileService { FileInformationResponse uploadV2(String serviceName, ServerFileUploadDto request, ContextInfo.LiteSaasContext liteSaasContext); ServerFileDownloadResponse download(ServerFileDownloadDto dto); + + MultipartUploadInitResponse multipartUploadInit(MultipartUploadInitDto multipartUploadInitDto); + + MultipartUploadResponse multipartUpload(MultipartUploadDto dto); + + FileInformationResponse multipartUploadComplete(MultipartUploadCompleteDto dto); } diff --git a/oss-service/src/main/java/cn/axzo/oss/service/impl/FileServiceImpl.java b/oss-service/src/main/java/cn/axzo/oss/service/impl/FileServiceImpl.java index 3ed9dbc..31e12ba 100644 --- a/oss-service/src/main/java/cn/axzo/oss/service/impl/FileServiceImpl.java +++ b/oss-service/src/main/java/cn/axzo/oss/service/impl/FileServiceImpl.java @@ -10,6 +10,7 @@ import cn.axzo.oss.common.enums.CodeEnum; import cn.axzo.oss.common.enums.FileClassEnum; import cn.axzo.oss.common.enums.FileStatusEnum; import cn.axzo.oss.common.exception.BizException; +import cn.axzo.oss.common.utils.BeanConvertUtil; import cn.axzo.oss.common.utils.JsonUtil; import cn.axzo.oss.common.utils.Utility; import cn.axzo.oss.dal.entity.*; @@ -23,8 +24,10 @@ import cn.axzo.oss.manager.api.dto.request.*; import cn.axzo.oss.manager.api.dto.response.*; import cn.axzo.oss.service.api.FileService; import cn.hutool.core.collection.CollectionUtil; +import com.aliyun.oss.model.PartETag; import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; @@ -148,6 +151,47 @@ public class FileServiceImpl implements FileService { return setFileInfoResp(ossFile); } + @Override + public MultipartUploadInitResponse multipartUploadInit(MultipartUploadInitDto dto) { + FileUploadConfig fileUploadConfig = getFileUploadConfig(dto.getAppCode(), dto.getBizScene()); + // 判断容量 + String fileConform = isFileConform(fileUploadConfig, 10, dto.getFileName()); + String uuid = Utility.getUUID(); + // 生成上传文件的唯一key + String tgtFileKey = Utility.generateFileKey(fileUploadConfig.getDirectory(), uuid, fileConform); + + String uploadId = fileManager.multipartUploadInit(fileUploadConfig.getBucketName(), tgtFileKey); + return MultipartUploadInitResponse.builder() + .uploadId(uploadId) + .tgtFileKey(tgtFileKey) + .build(); + } + + @Override + public MultipartUploadResponse multipartUpload(MultipartUploadDto dto) { + FileUploadConfig fileUploadConfig = getFileUploadConfig(dto.getAppCode(), dto.getBizScene()); + // 判断容量 + isFileConform(fileUploadConfig, dto.getFileContent().length, dto.getFileName()); + + PartETag partETag = fileManager.multipartUpload(fileUploadConfig.getBucketName(), + dto.getTgtFileKey(), dto.getFileContent(), dto); + MultipartUploadResponse response = MultipartUploadResponse.builder() + .uploadId(dto.getUploadId()) + .partETag(BeanConvertUtil.copyBean(partETag, cn.axzo.oss.manager.api.dto.PartETag.class)) + .build(); + return response; + } + + @Override + public FileInformationResponse multipartUploadComplete(MultipartUploadCompleteDto dto) { + FileUploadConfig fileUploadConfig = getFileUploadConfig(dto.getAppCode(), dto.getBizScene()); + List partETags = convertPartETags(dto.getPartETags()); + // 文件合并并生成 file对象 + File ossFile = completeFile(fileUploadConfig, dto.getFileName(), dto.getTgtFileKey(), + dto.getUploadId(), partETags); + return setFileInfoResp(ossFile); + } + @Override public ServerFileDownloadResponse download(ServerFileDownloadDto dto) { log.info("file download dto = {}", JsonUtil.obj2Str(dto)); @@ -163,7 +207,6 @@ public class FileServiceImpl implements FileService { log.warn("file download is null, fileKey = {}", fileKey); BizException.error(Utility.objIsNotNull(file), CodeEnum.FILE_NOT_FOUND); } - //String tgtFileKey = Utility.generateFileKey(file.getDirectory(), file.getFileUuid(), file.getFileFormat()); //兼容app端oss历史上传文件的url处理 String tgtFileKey = file.getFileUrl().substring(file.getFileUrl().indexOf(HTTP_COM) + 5); log.info("file download tgtFileKey = {}", tgtFileKey); @@ -176,6 +219,74 @@ public class FileServiceImpl implements FileService { return setFileDownloadResponse(file, fileStream); } + private List convertPartETags(List partETags) { + List partETagList = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(partETags)) { + partETags.stream().forEach(partETag -> { + PartETag ossPartETag = new PartETag(partETag.getPartNumber(), partETag.getETag(), + partETag.getPartSize(), partETag.getPartCRC()); + partETagList.add(ossPartETag); + }); + } + return partETagList; + } + + private FileUploadConfig getFileUploadConfig(String appCode, String bizScene) { + // 检查appCode + checkAppCode(appCode); + + // 通过appcode获取文件渠道桶信息 + AppChannelBucket appChannelBucket = appChannelBucketManager.getByAppCode(appCode); + + // 通过渠道桶编码获取到具体文件业务场景 + FileBusinessScene scene = fileBusinessSceneManager + .getByBucketNoAndScene(appChannelBucket.getAppChannelBucketNo(), bizScene); + + // 通过渠道码和桶名称获取获取指定上传配置 + FileUploadConfig fileUploadConfig = fileUploadConfigManager + .getByUploadConfig(scene.getAppChannelBucketNo(), scene.getDirectory()); + return fileUploadConfig; + } + + private File completeFile(FileUploadConfig fileUploadConfig, String fileName, String tgtFileKey, String uploadId, List partETags) { + // 文件合并 + String fileUrl = fileManager.multipartUploadComplete(fileUploadConfig.getBucketName(), tgtFileKey, uploadId, partETags); + // 保存失败 + if (Utility.isBlank(fileUrl)) { + log.error("fileUrl is empty"); + throw new BizException(CodeEnum.MULTIPART_UPLOAD_COMPLETE_ERROR); + } + + int lastIndexOf = tgtFileKey.lastIndexOf(FileClassEnum.SEPARATOR_CHAR.type); + String newFileName = tgtFileKey.substring(lastIndexOf + 1); + int indexOf = newFileName.indexOf(FileClassEnum.DOT.type); + String fileConform = newFileName.substring(indexOf + 1).toLowerCase(); + String uuid = newFileName.substring(0, indexOf); + return getOssFile(fileUploadConfig, fileName, fileConform, uuid, fileUrl, Utility.getMd5(uploadId)); + } + + private File getOssFile(FileUploadConfig fileUploadConfig, String fileName, String fileConform, String uuid, + String fileUrl, String fileMd5) { + File ossFile = new File(); + ossFile.setAppChannelBucketNo(fileUploadConfig.getAppChannelBucketNo()); + ossFile.setAppCode(fileUploadConfig.getAppCode()); + ossFile.setChannelCode(fileUploadConfig.getChannelCode()); + ossFile.setBucketName(fileUploadConfig.getBucketName()); + ossFile.setDirectory(fileUploadConfig.getDirectory()); + ossFile.setStatus(FileStatusEnum.STATUS_UPLOAD_FAIL.getCode()); + ossFile.setStorageUnit(fileUploadConfig.getStorageUnit()); + ossFile.setStorageSize(fileUploadConfig.getStorageSize()); + ossFile.setFileFormat(fileConform); + ossFile.setFileUuid(uuid); + ossFile.setFileUrl(fileUrl); + ossFile.setUrlMd5(Utility.getMd5(fileUrl)); + ossFile.setStatus(FileStatusEnum.STATUS_UPLOAD_SUCCESS.getCode()); + ossFile.setFileName(fileName); + ossFile.setFileMd5(fileMd5); + fileDao.save(ossFile); + return ossFile; + } + private ServerFileDownloadResponse setFileDownloadResponse(File file, InputStream fileStream) { ServerFileDownloadResponse resp = new ServerFileDownloadResponse(); resp.setUrl(file.getFileUrl()); @@ -217,19 +328,7 @@ public class FileServiceImpl implements FileService { private File uploadFileAndGetFile(ServerFileUploadDto dto) { log.info("update fileName:{},appCode:{},bizScene:{}", dto.getFileName(), dto.getAppCode(), dto.getBizScene()); - // 检查appCode - checkAppCode(dto.getAppCode()); - - // 通过appcode获取文件渠道桶信息 - AppChannelBucket appChannelBucket = appChannelBucketManager.getByAppCode(dto.getAppCode()); - - // 通过渠道桶编码获取到具体文件业务场景 - FileBusinessScene scene = fileBusinessSceneManager - .getByBucketNoAndScene(appChannelBucket.getAppChannelBucketNo(), dto.getBizScene()); - - // 通过渠道码和桶名称获取获取指定上传配置 - FileUploadConfig fileUploadConfig = fileUploadConfigManager - .getByUploadConfig(scene.getAppChannelBucketNo(), scene.getDirectory()); + FileUploadConfig fileUploadConfig = getFileUploadConfig(dto.getAppCode(), dto.getBizScene()); // 上传文件并生成file对象 return generateFile(fileUploadConfig, dto); } @@ -474,24 +573,7 @@ public class FileServiceImpl implements FileService { throw new BizException(CodeEnum.FILE_UPLOAD_FAILED); } - File ossFile = new File(); - ossFile.setAppChannelBucketNo(fileUploadConfig.getAppChannelBucketNo()); - ossFile.setAppCode(fileUploadConfig.getAppCode()); - ossFile.setChannelCode(fileUploadConfig.getChannelCode()); - ossFile.setBucketName(fileUploadConfig.getBucketName()); - ossFile.setDirectory(fileUploadConfig.getDirectory()); - ossFile.setStatus(FileStatusEnum.STATUS_UPLOAD_FAIL.getCode()); - ossFile.setStorageUnit(fileUploadConfig.getStorageUnit()); - ossFile.setStorageSize(fileUploadConfig.getStorageSize()); - ossFile.setFileFormat(fileConform); - ossFile.setFileUuid(uuid); - ossFile.setFileUrl(fileUrl); - ossFile.setUrlMd5(Utility.getMd5(fileUrl)); - ossFile.setStatus(FileStatusEnum.STATUS_UPLOAD_SUCCESS.getCode()); - ossFile.setFileName(dto.getFileName()); - ossFile.setFileMd5(Utility.getMd5(dto.getFileContent())); - fileDao.save(ossFile); - return ossFile; + return getOssFile(fileUploadConfig, dto.getFileName(), fileConform, uuid, fileUrl, Utility.getMd5(dto.getFileContent())); } private ServerFileUploadResponse setResponse(File ossFile) {