wangli/oss
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'feature/joint_log' into 'pre'

Browse Source 
Feature/joint log

See merge request infra/oss!35
This commit is contained in:
田立勇 2022-12-13 10:11:13 +00:00
commit 26cf455ebc
9 changed files with 136 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
oss-client/src/main/java/cn/axzo/oss/client/controller/WebFileController.java
View File

@ -25,12 +25,14 @@ import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.Part;
import javax.validation.Valid;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.List;
import static cn.axzo.oss.common.enums.CodeEnum.FILE_NAME_TOO_LONG;
import static org.springframework.http.MediaType.MULTIPART_FORM_DATA_VALUE;
/**
* 前端文件controller
@ -47,11 +49,11 @@ public class WebFileController {
private FileService fileService;
@SneakyThrows
@PostMapping("/v1/file")
@PostMapping(value = "/v1/file", consumes = MULTIPART_FORM_DATA_VALUE)
@CrossOrigin
public CommonResponse<WebFileUploadVo> upload(@Valid @RequestParam String appCode,
@Valid @RequestParam String bizScene,
@Valid @RequestParam MultipartFile file) {
public CommonResponse<WebFileUploadVo> upload(@Valid @RequestParam("appCode") String appCode,
@Valid @RequestParam("bizScene") String bizScene,
@Valid @RequestPart MultipartFile file) {
//获取用户信息
ContextInfo.LiteSaasContext liteSaasContext = null;
ContextInfo contextInfo = ContextInfoHolder.get();
@ -71,14 +73,14 @@ public class WebFileController {
return CommonResponse.success(result);
}
@PostMapping("/v2/file")
@PostMapping(value = "/v2/file", consumes = MULTIPART_FORM_DATA_VALUE)
@CrossOrigin
@SneakyThrows
@PreBuildContext
public CommonResponse<FileInformationVo> uploadV2(@Valid @RequestParam String appCode,
@Valid @RequestParam String bizScene,
@Valid @RequestParam String serviceName,
@Valid @RequestParam MultipartFile file) {
public CommonResponse<FileInformationVo> uploadV2(@Valid @RequestParam("appCode") String appCode,
@Valid @RequestParam("bizScene") String bizScene,
@Valid @RequestParam("serviceName") String serviceName,
@Valid @RequestPart MultipartFile file) {
//获取用户信息
ContextInfo.LiteSaasContext liteSaasContext = null;
ContextInfo contextInfo = ContextInfoHolder.get();
@ -111,10 +113,7 @@ public class WebFileController {
@GetMapping("/v1/file/download")
@CrossOrigin
@PreBuildContext
public void download(@Valid @RequestParam String fileKey, HttpServletResponse response) {
ServerFileDownloadDto dto = ServerFileDownloadDto.builder()
.fileKey(fileKey)
.build();
public void download(@Valid ServerFileDownloadDto dto, HttpServletResponse response) {
ServerFileDownloadResponse result = fileService.download(dto);
InputStream inputStream = null;
OutputStream outputStream = response.getOutputStream();

9
oss-dal/src/main/java/cn/axzo/oss/dal/repository/impl/FileDaoImpl.java
View File

@ -5,6 +5,7 @@ import cn.axzo.oss.common.constans.CommonConstants.TableDelete;
import cn.axzo.oss.dal.entity.File;
import cn.axzo.oss.dal.mapper.FileMapper;
import cn.axzo.oss.dal.repository.FileDao;
import cn.hutool.core.collection.CollectionUtil;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import org.springframework.stereotype.Repository;
@ -38,16 +39,16 @@ public class FileDaoImpl extends ServiceImpl<FileMapper, File> implements FileDa
}
@Override
public List<File> getByFileUuids(List<String> fileKey) {
return lambdaQuery().in(File::getFileUuid, fileKey)
public List<File> getByFileUuids(List<String> fileKeys) {
return lambdaQuery().in(CollectionUtil.isNotEmpty(fileKeys), File::getFileUuid, fileKeys)
.eq(File::getStatus, FileStatus.SUCCESS)
.eq(File::getIsDelete, TableDelete.UN_DELETED)
.list();
}
@Override
public List<File> getByUrlMd5s(List<String> urlMd5List) {
return lambdaQuery().in(File::getUrlMd5, urlMd5List)
public List<File> getByUrlMd5s(List<String> urlMd5s) {
return lambdaQuery().in(CollectionUtil.isNotEmpty(urlMd5s), File::getUrlMd5, urlMd5s)
.eq(File::getStatus, FileStatus.SUCCESS)
.eq(File::getIsDelete, TableDelete.UN_DELETED)
.list();

2
oss-integration/src/main/java/cn/axzo/oss/integration/s3/base/BaseS3Service.java
View File

@ -39,7 +39,7 @@ public interface BaseS3Service {
*/
ResponseEntity<byte[]> download(String tgtFileKey);
InputStream downloadFile(String bucket, String tgtFileKey);
InputStream downloadFile(String bucket, String tgtFileKey, String style);
/**
* get file address

10
oss-integration/src/main/java/cn/axzo/oss/integration/s3/impl/AliOssServiceImpl.java
View File

@ -6,8 +6,10 @@ import cn.azxo.framework.common.utils.LogUtil;
import com.aliyun.oss.ClientException;
import com.aliyun.oss.OSS;
import com.aliyun.oss.OSSException;
import com.aliyun.oss.model.GetObjectRequest;
import com.aliyun.oss.model.OSSObject;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
@ -87,9 +89,13 @@ public class AliOssServiceImpl implements AliOssService {
}
@Override
public InputStream downloadFile(String bucket, String tgtFileKey) {
public InputStream downloadFile(String bucket, String tgtFileKey, String style) {
OSS client = aliOssClient.getClient();
OSSObject ossObject = client.getObject(bucket, tgtFileKey);
GetObjectRequest request = new GetObjectRequest(bucket, tgtFileKey);
if (StringUtils.isNotEmpty(style)) {
request.setProcess(style);
}
OSSObject ossObject = client.getObject(request);
return ossObject.getObjectContent();
}

2
oss-manager-api/src/main/java/cn/axzo/oss/manager/api/FileManager.java
View File

@ -35,6 +35,6 @@ public interface FileManager {
* @param bucketName 桶名称
* @param tgtFileKey 目标文件key
*/
InputStream downloadFile(String bucketName, String tgtFileKey);
InputStream downloadFile(String bucketName, String tgtFileKey, String style);
}

2
oss-manager-api/src/main/java/cn/axzo/oss/manager/api/dto/request/FindFileUrlDto.java
View File

@ -5,6 +5,7 @@ import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import javax.validation.constraints.NotNull;
import java.util.List;
/**
@ -21,5 +22,6 @@ public class FindFileUrlDto {
/**
* 文件uuid
*/
@NotNull
private List<String> fileKey;
}

8
oss-manager-api/src/main/java/cn/axzo/oss/manager/api/dto/request/ServerFileDownloadDto.java
View File

@ -5,6 +5,8 @@ import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import javax.validation.constraints.NotBlank;
/**
* @Author: liyong.tian
* @Date: 2022/11/29 15:42
@ -19,5 +21,11 @@ public class ServerFileDownloadDto {
/**
* 文件uuid
*/
@NotBlank
private String fileKey;
/**
* 图片样式
*/
private String style;
}

4
oss-manager/src/main/java/cn/axzo/oss/manager/impl/FileManagerImpl.java
View File

@ -46,8 +46,8 @@ public class FileManagerImpl implements FileManager {
}
@Override
public InputStream downloadFile(String bucketName, String tgtFileKey) {
return aliOssService.downloadFile(bucketName, tgtFileKey);
public InputStream downloadFile(String bucketName, String tgtFileKey, String style) {
return aliOssService.downloadFile(bucketName, tgtFileKey, style);
}
}

144
oss-service/src/main/java/cn/axzo/oss/service/impl/FileServiceImpl.java
View File

@ -31,10 +31,7 @@ import org.springframework.core.env.Environment;
import org.springframework.stereotype.Service;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
@ -57,6 +54,8 @@ public class FileServiceImpl implements FileService {
private static String APPCODE_BANKCARD = "bankcard";
private static String SCENE_BANKCARD = "bankcard";
private static String IS_URL = "https://";
@Autowired
private FileManager fileManager;
@ -148,16 +147,16 @@ public class FileServiceImpl implements FileService {
@Override
public ServerFileDownloadResponse download(ServerFileDownloadDto dto) {
log.info("file download dto = {}", JsonUtil.obj2Str(dto));
File file = fileDao.getByFileUuid(dto.getFileKey());
String fileKey = dto.getFileKey();
File file = fileDao.getByFileUuid(fileKey);
if (Utility.objIsNull(file)) {
log.warn("file download is null, fileKey = {}", dto.getFileKey());
log.warn("file download is null, fileKey = {}", fileKey);
return null;
}
String tgtFileKey = Utility
.generateFileKey(file.getDirectory(), file.getFileUuid(), file.getFileFormat());
log.info("file download tgtFileKey = {}", tgtFileKey);
InputStream fileStream = fileManager.downloadFile(file.getBucketName(), tgtFileKey);
InputStream fileStream = fileManager.downloadFile(file.getBucketName(), tgtFileKey, dto.getStyle());
return setFileDownloadResponse(file, fileStream);
}
@ -226,20 +225,58 @@ public class FileServiceImpl implements FileService {
// 做兼容处理，如果List<String> fileKey如果存在url取出url，不做查询处理
List<String> urlList = new ArrayList<>();
List<String> fileKeyList = new ArrayList<>();
Map<String, String> fileKeyStyleMap = new HashMap<>();
for (String fileKey : dto.getFileKey()) {
if (fileKey.contains("http")) {
if (fileKey.contains(IS_URL)) {
urlList.add(fileKey);
} else {
//对fileKey带图片样式进行处理
if (fileKey.contains("?")) {
String[] fileKeyArr = fileKey.split("\\?");
fileKeyList.add(fileKeyArr[0]);
fileKeyStyleMap.put(fileKeyArr[0], fileKey);
} else {
fileKeyList.add(fileKey);
}
}
List<File> fileList = fileDao.getByFileUuids(fileKeyList);
if (CollectionUtil.isEmpty(fileList)) {
log.warn("find file url is null,key = {}", Arrays.toString(dto.getFileKey().toArray()));
return new ArrayList<>();
}
return setFileUrlRes(urlList, fileList);
List<File> fileList = new ArrayList<>();
if (CollectionUtil.isNotEmpty(fileKeyList)) {
fileList = fileDao.getByFileUuids(fileKeyList);
}
return setFileUrlRes(urlList, fileKeyStyleMap, fileList);
}
private List<FindFileUrlResponse> setFileUrlRes(List<String> urlList) {
List<FindFileUrlResponse> resList = new ArrayList<>();
if (CollectionUtil.isNotEmpty(urlList)) {
urlList.stream().forEach(url -> {
FindFileUrlResponse response = new FindFileUrlResponse();
response.setUrl(url);
response.setFileKey(url);
resList.add(response);
});
}
return resList;
}
private List<FindFileUrlResponse> setFileUrlRes(List<String> urlList, Map<String, String> fileKeyStyleMap, List<File> fileList) {
List<FindFileUrlResponse> resList = setFileUrlRes(urlList);
if (CollectionUtil.isNotEmpty(fileList)) {
fileList.stream().forEach(file -> {
FindFileUrlResponse response = new FindFileUrlResponse();
response.setUrl(file.getFileUrl());
String fileKey = file.getFileUuid();
if (fileKeyStyleMap.containsKey(fileKey)) {
response.setFileKey(fileKeyStyleMap.get(fileKey));
} else {
response.setFileKey(fileKey);
}
resList.add(response);
});
}
return resList;
}
@Override
@ -247,23 +284,52 @@ public class FileServiceImpl implements FileService {
log.info("find file key dto = {}", JsonUtil.obj2Str(dto));
List<String> urlList = dto.getUrl();
List<String> urlMd5List = urlList.stream()
.map(url -> Utility.getMd5(url))
.collect(Collectors.toList());
List<File> fileList = fileDao.getByUrlMd5s(urlMd5List);
//用于对请求参数中为fileKey的兼容处理
List<String> fileKeyList = new ArrayList<>();
List<String> urlMd5List = new ArrayList<>();
urlList.stream().forEach(url -> {
if (url.contains(IS_URL)) {
urlMd5List.add(Utility.getMd5(url));
} else {
fileKeyList.add(url);
}
});
List<File> fileList = new ArrayList<>();
List<FindFileKeyResponse> resList = new ArrayList<>();
if (CollectionUtil.isNotEmpty(urlMd5List)) {
fileList = fileDao.getByUrlMd5s(urlMd5List);
//处理app端历史数据不在file表中的情况
if (urlMd5List.size() > fileList.size()) {
List<String> existUrlList = fileList.stream().map(File::getFileUrl).collect(Collectors.toList());
urlList.removeAll(existUrlList);
List<FindFileKeyResponse> resList = setFileKeyResByUrl(urlList);
resList = setFileKeyResByUrl(urlList);
// 异步新增file数据
asyncSaveFile(resList);
}
}
resList.addAll(setFileKeyRes(fileKeyList, fileList));
return resList;
}
return setFileKeyResByFile(fileList);
private List<FindFileKeyResponse> setFileKeyRes(List<String> fileKeyList, List<File> fileList) {
List<FindFileKeyResponse> resList = new ArrayList<>();
if (CollectionUtil.isNotEmpty(fileKeyList)) {
fileKeyList.forEach(fileKey -> {
FindFileKeyResponse response = new FindFileKeyResponse();
response.setUrl(fileKey);
response.setFileKey(fileKey);
resList.add(response);
});
}
if (CollectionUtil.isNotEmpty(fileList)) {
fileList.forEach(file -> {
FindFileKeyResponse response = new FindFileKeyResponse();
response.setUrl(file.getFileUrl());
response.setFileKey(file.getFileUuid());
resList.add(response);
});
}
return resList;
}
private void asyncSaveFile(List<FindFileKeyResponse> resList) {
@ -324,27 +390,6 @@ public class FileServiceImpl implements FileService {
return ossFile;
}
private List<FindFileUrlResponse> setFileUrlRes(List<String> urlList, List<File> fileList) {
List<FindFileUrlResponse> resList = new ArrayList<>();
if (CollectionUtil.isNotEmpty(urlList)) {
urlList.stream().forEach(url -> {
FindFileUrlResponse response = new FindFileUrlResponse();
response.setUrl(url);
response.setFileKey(url);
resList.add(response);
});
}
fileList.stream().forEach(file -> {
FindFileUrlResponse response = new FindFileUrlResponse();
response.setUrl(file.getFileUrl());
response.setFileKey(file.getFileUuid());
resList.add(response);
});
return resList;
}
private List<FindFileKeyResponse> setFileKeyResByUrl(List<String> urlList) {
List<FindFileKeyResponse> resList = new ArrayList<>();
urlList.stream().forEach(url -> {
@ -356,17 +401,6 @@ public class FileServiceImpl implements FileService {
return resList;
}
private List<FindFileKeyResponse> setFileKeyResByFile(List<File> fileList) {
List<FindFileKeyResponse> resList = new ArrayList<>();
fileList.forEach(file -> {
FindFileKeyResponse response = new FindFileKeyResponse();
response.setUrl(file.getFileUrl());
response.setFileKey(file.getFileUuid());
resList.add(response);
});
return resList;
}
/**
* 判断文件是否符合要求
*/