package com.zilber.boot.file.service; import com.aliyun.oss.*; import com.aliyun.oss.model.*; import com.baomidou.mybatisplus.core.toolkit.CollectionUtils; import com.zilber.boot.file.dao.FileDao; import com.zilber.boot.file.entity.*; import com.zilber.boot.utils.file.FileUtils; import com.zilber.boot.utils.sign.Md5Utils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang3.StringUtils; import org.apache.tomcat.util.security.MD5Encoder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import reactor.util.annotation.Nullable; import javax.annotation.Resource; import java.io.*; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; /** * @Author LJX * @TIME 2023-01-09 9:11 * @PROJECT cppcc * created by Intellij IDEA * Description */ @Service @Slf4j public class UploadUtils { @Value("${zilber.localtion}") private String localtion; @Resource private OssProperties ossProperties; @Autowired private FileDao fileDao; public Map upload(MultipartFile file, String md5) { try { Map objectMap = fileDao.listFileUrl(md5, 1); if ( objectMap != null ){ objectMap.put("name", file.getOriginalFilename()); return objectMap; } String type = "." + FilenameUtils.getExtension(file.getOriginalFilename()); String name = FilenameUtils.getName(file.getOriginalFilename()); String filePath = localtion + "/upload"; String fileName = UUID.randomUUID() + type; File desc = getAbsoluteFile(filePath, fileName); file.transferTo(desc); Map pathFileName = getPathFileName(filePath, fileName, name, type); ` fileDao.addFile(md5, pathFileName.get("url"), Integer.parseInt(pathFileName.get("type")), 1); return pathFileName; } catch (Exception e) { log.error(e.getLocalizedMessage()); return new ConcurrentHashMap<>(); } } private Map getPathFileName(String uploadDir, String fileName, String name, String type) throws IOException { Map map = new ConcurrentHashMap(3); int dirLastIndex = localtion.length() + 1; String currentDir = StringUtils.substring(uploadDir, dirLastIndex); String pathFileName = "/profile" + "/" + currentDir + "/" + fileName; map.put("url", pathFileName); map.put("name", name); map.put("type", String.valueOf(getFileType(type))); return map; } private File getAbsoluteFile(String uploadDir, String fileName) { File desc = new File(uploadDir + File.separator + fileName); if (!desc.exists()) { if (!desc.getParentFile().exists()) { desc.getParentFile().mkdirs(); } } return desc; } //TODO 上传至oss /** * 上传文件 * * @param file 上传的文件 * @param module oss目录 * @return */ public Map upload(MultipartFile file, @Nullable String module, String md5) { Map objectMap = fileDao.listFileUrl(md5, 2); if ( objectMap != null ){ objectMap.put("name", file.getOriginalFilename()); return objectMap; } /** * 获取oss的属性 */ String endpoint = ossProperties.getEndpoint(); String accessKeyId = ossProperties.getKeyId(); String accessKeySecret = ossProperties.getKeySecret(); String bucketName = ossProperties.getBucketName(); if ( StringUtils.isBlank(module) ){ module = ossProperties.getModule(); } String extension = "." + FilenameUtils.getExtension(file.getOriginalFilename()); String objectName = module + "/" + UUID.randomUUID().toString() + extension; Map resultMap = new HashMap<>(); // 创建OSSClient实例。 OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret); try { // 创建PutObjectRequest对象。 PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, objectName, file.getInputStream()); // 设置该属性可以返回response。如果不设置,则返回的response为空。 putObjectRequest.setProcess("true"); // 上传字符串。 PutObjectResult result = ossClient.putObject(putObjectRequest); if ( result.getResponse().getStatusCode() == 200 ){ Integer fileType = getFileType(extension); String uri = result.getResponse().getUri(); resultMap.put("url", uri); resultMap.put("name", file.getOriginalFilename()); resultMap.put("type", String.valueOf(fileType)); fileDao.addFile(md5, uri, fileType, 2); return resultMap; } } catch (OSSException oe) { throw new ServiceException(oe.getMessage()); } catch (ClientException ce) { throw new ServiceException(ce.getMessage()); } catch (IOException e) { e.printStackTrace(); } finally { if (ossClient != null) { // 关闭OSSClient。 ossClient.shutdown(); } } return null; } //TODO 分片上传至服务器 /** * 清除临时目录下的临时文件 * * @param root */ private void clearTempFiles(File root) { File[] files = root.listFiles(); if (files != null) { List fileList = Arrays.asList(files); for (File f : fileList) { f.delete(); } } root.delete(); } /********************************************************************************************/ public ETag sliceUploads(FileUploadsRequestDTO param) { boolean isOk = true; File root = new File(localtion + "/upload", param.getUUID()); if (!root.exists() || !root.isDirectory()) { isOk = root.mkdirs(); } File targetFile = new File(localtion + "/upload", param.getUUID() + param.getType()); if (!targetFile.exists() || !targetFile.isFile()) { try { targetFile.createNewFile(); } catch (IOException e) { e.printStackTrace(); } } Map map = new HashMap<>(); map.put(param.getChunk(), param.getUUID()); if (isOk) { String sliceName = param.getChunk().toString(); return upload(root, sliceName, param); } return null; } /** * 文件检测 * { * "requestId": "63BD04BA1253C53638A9F751", * "bucketName": "zilber-public", * "key": "upload/3a3cd46a-d966-4258-a6e7-a3b318cbe133.jpg", * "uploadId": "70F44C7139E14DE0AB6BD7442F0F4AFE", * "maxParts": 100, * "partNumberMarker": 1, * "storageClass": "Standard", * "nextPartNumberMarker": 2, * "parts": [ * { * "partNumber": 2, * "lastModified": "2023-01-10T14:08:37.000+0800", * "size": 566470, * "etag": "21DE9DD8AE549FF2285DDC5E23B67867" * } * ], * "truncated": false * } * @param md5 md5 * @param UUID UUID * @return * @throws IOException */ public PartList fileCheck(String md5, String UUID) throws IOException { PartList list = new PartList(); List eTags = new ArrayList<>(); File root = new File(localtion + "/upload", UUID); if (!root.exists()) { return null; } //获取总块数 // File aConf = new File(root.getAbsolutePath(), UUID + ".conf"); // FileReader fr = new FileReader(aConf); // char[] chunkCs = new char[(int) aConf.length()]; // fr.read(chunkCs); Long chunks = chunks(root, UUID); list.setTotal(chunks); FilenameFilter filter = new FilenameFilter() { @Override public boolean accept(File file, String s) { return s.endsWith(".conf") && !s.contains("-"); } }; String[] fileNames = root.list(filter); if (fileNames == null) { return null; } List fileNameList = Arrays.asList(fileNames) .stream().distinct().collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(fileNameList)) { for (String fileName : fileNameList) { String prefix = fileName.split("\\.")[0]; ETag tag = new ETag(); tag.setPartSize(new File(root.getAbsolutePath(), fileName).getTotalSpace()); tag.setPartNumber(Integer.parseInt(prefix)); eTags.add(tag); } list.setTags(eTags); return list; } return list; } /** * { * "partNumber": 2, * "partSize": 566470, * "partCRC": 997575365254752264, * "etag": "21DE9DD8AE549FF2285DDC5E23B67867" * } * @return * @author liuao */ public ETag upload(File root, String name, FileUploadsRequestDTO param) { //保存最大块数 ETag tag = new ETag(); try { tag.setPartNumber(param.getChunk().intValue()); tag.setPartSize(param.getChunkSize()); tag.setETag(MD5Encoder.encode(param.getFile().getBytes())); } catch (IOException e) { e.printStackTrace(); } File aConfFile = new File(root.getAbsolutePath(), param.getUUID() + ".conf"); File sizeFile = new File(root.getAbsolutePath(), "size.conf"); if (!aConfFile.exists()) { FileWriter w = null; try { aConfFile.createNewFile(); w = new FileWriter(aConfFile); w.write(param.getChunks().toString()); } catch (IOException e) { e.printStackTrace(); } finally { try { assert w != null; w.close(); } catch (IOException e) { e.printStackTrace(); } } } RandomAccessFile accessTmpFile = null; int length = 0; File[] files = root.listFiles(); if (files != null) { length = files.length; } // String uploadDirPath = localtion + "/upload"; File tmpFile = new File(root.getAbsolutePath(), name); File tmpConfFile = new File(root.getAbsolutePath(), name + ".conf"); try { accessTmpFile = new RandomAccessFile(tmpFile, "rw"); accessTmpFile.write(param.getFile().getBytes()); log.info(param.getUUID() + "文件块" + param.getChunk() + "上传完成"); return tag; } catch (IOException e) { log.error(e.getMessage(), e); if (tmpConfFile.exists()) { tmpConfFile.delete(); } } finally { try { if (tmpFile.length() == param.getFile().getSize()) { if (!tmpConfFile.exists()) { tmpConfFile.createNewFile(); } } assert accessTmpFile != null; accessTmpFile.close(); } catch (IOException e) { log.error(e.getLocalizedMessage()); if (tmpConfFile.exists()) { tmpConfFile.delete(); } } } return null; } /** * { * "requestId": "63BD05267FFDC23733FE3B95", * "clientCRC": -1453100417491105764, * "serverCRC": -1453100417491105764, * "bucketName": "zilber-public", * "key": "upload/3a3cd46a-d966-4258-a6e7-a3b318cbe133.jpg", * "location": "http://zilber-public.oss-cn-beijing.aliyuncs.com/upload/3a3cd46a-d966-4258-a6e7-a3b318cbe133.jpg", * "etag": "8DDB27878F7D65485F8E83A2B5E7F11E-2" * } * @param UUID * @param type * @param md5 * @return */ public Complete compose(String UUID, String type, String md5) { long startTime = 0, endTime = 0; log.info(String.valueOf(System.currentTimeMillis())); Complete complete = new Complete(); File root = new File(localtion + "/upload", UUID); File targetFile = new File(localtion + "/upload", UUID + type); Long chunks = chunks(root, UUID); File[] fileNames = root.listFiles((file1, s1) -> !s1.endsWith(".conf")); if (fileNames == null || fileNames.length < chunks) { return null; } List collect = Arrays.asList(fileNames) .stream().sorted((file, t1) -> { int f = Integer.parseInt(file.getName()); int t = Integer.parseInt(t1.getName()); return f < t ? -1 : 1; }).collect(Collectors.toList()); FileOutputStream fos; FileInputStream fis; // 一次读取10M数据,将读取到的数据保存到byte字节数组中 byte[] buffer = new byte[1024 * 1024 * 10]; int len; try { fos = new FileOutputStream(targetFile); log.info(String.valueOf(startTime = System.currentTimeMillis())); for (File file : collect) { fis = new FileInputStream(file); len = 0; while ((len = fis.read(buffer)) != -1) { // buffer从指定字节数组写入。buffer:数据中的起始偏移量,len:写入的字数。 fos.write(buffer, 0, len); } fos.flush(); fis.close(); } fos.close(); log.info("传输完成:" + (endTime = System.currentTimeMillis())); } catch (IOException ex) { targetFile.delete(); ex.printStackTrace(); } log.info(String.valueOf(endTime - startTime)); // String fileName = java.util.UUID.randomUUID().toString(); int dirLastIndex = localtion.length() + 1; String currentDir = StringUtils.substring(localtion + "/upload", dirLastIndex); String pathFileName = "/profile" + "/" + currentDir + "/" + UUID + type; Integer t = getFileType(type); fileDao.addFile(md5, pathFileName, t, 1); complete.setLocation(pathFileName); complete.setType(t); clearTempFiles(root); return complete; } private Long chunks(File root, String UUID) { File aConf = new File(root.getAbsolutePath(), UUID + ".conf"); FileReader fr = null; char[] chunkCs = null; try { fr = new FileReader(aConf); chunkCs = new char[(int) aConf.length()]; fr.read(chunkCs); } catch (FileNotFoundException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } finally { try { fr.close(); } catch (IOException e) { e.printStackTrace(); } } return Long.valueOf(String.valueOf(chunkCs)); } public static Integer getFileType(String type) { String pic = ".jpeg,.bmp,.jpg,.png,.tif,.gif,.pcx,.tga,.exif,.fpx,.svg,.psd,.cdr,.pcd,.dxf,.ufo,.eps,.ai,.raw,.WMF,.webp,.avif,.apng"; String video = ".wmv,.asf,.asx,.mp4,.m4v,.rm,.rmvb,.mpg,.mpeg,.mpe,.3gp,.mov,.avi,.dat,.mkv,.flv,.vob"; if (pic.contains(type)) { return 1; } if (video.contains(type)) { return 3; } return 2; } //TODO 分片上传至oss /** * 初始化切片上传 * @param fileName 文件名 * @param module 目标文件夹 * @return 文件唯一标识 */ public OssPartUpload InitiateMultipartUpload(String fileName, @Nullable String module,@Nullable OssProperties provided) { /** * 获取oss的属性 */ String endpoint = ""; String accessKeyId = ""; String accessKeySecret = ""; String bucketName = ""; if ( provided != null ){ if ( StringUtils.isNotBlank(provided.getEndpoint())){ endpoint = provided.getEndpoint(); }else { endpoint = ossProperties.getEndpoint(); } if ( StringUtils.isNotBlank(provided.getKeyId())){ accessKeyId = provided.getKeyId(); }else { accessKeyId = ossProperties.getKeyId(); } if ( StringUtils.isNotBlank(provided.getKeySecret())){ accessKeySecret = provided.getKeySecret(); }else { accessKeySecret = ossProperties.getKeySecret(); } if ( StringUtils.isNotBlank(provided.getBucketName())){ bucketName = provided.getBucketName(); }else { bucketName = ossProperties.getBucketName(); } }else { endpoint = ossProperties.getEndpoint(); accessKeyId = ossProperties.getKeyId(); accessKeySecret = ossProperties.getKeySecret(); bucketName = ossProperties.getBucketName(); } if ( StringUtils.isBlank(module) ){ module = ossProperties.getModule(); } String extension = "." + FilenameUtils.getExtension(fileName); String objectName = module + "/" + UUID.randomUUID().toString() + extension; Map resultMap = new HashMap<>(); // 创建OSSClient实例。 OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret); // 创建InitiateMultipartUploadRequest对象。 InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, objectName); // 初始化分片。 InitiateMultipartUploadResult upresult = ossClient.initiateMultipartUpload(request); // 返回uploadId,它是分片上传事件的唯一标识。您可以根据该uploadId发起相关的操作,例如取消分片上传、查询分片上传等。 OssPartUpload upload = new OssPartUpload(); upload.setKey(objectName); upload.setUploadId(upresult.getUploadId()); return upload; } /** * 分片上传至oss * @param upload 分片信息 * @return */ public PartETag uploadPart(OssPartUpload upload) { //ValidatorUtils.validateEntity(upload); /** * 获取oss的属性 */ String endpoint = ""; String accessKeyId = ""; String accessKeySecret = ""; String bucketName = ""; if ( StringUtils.isNotBlank(upload.getEndpoint())){ endpoint = upload.getEndpoint(); }else { endpoint = ossProperties.getEndpoint(); } if ( StringUtils.isNotBlank(upload.getKeyId())){ accessKeyId = upload.getKeyId(); }else { accessKeyId = ossProperties.getKeyId(); } if ( StringUtils.isNotBlank(upload.getKeySecret())){ accessKeySecret = upload.getKeySecret(); }else { accessKeySecret = ossProperties.getKeySecret(); } if ( StringUtils.isNotBlank(upload.getBucketName())){ bucketName = upload.getBucketName(); }else { bucketName = ossProperties.getBucketName(); } // 创建OSSClient实例。 OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret); try { UploadPartRequest uploadPartRequest = new UploadPartRequest(); uploadPartRequest.setBucketName(bucketName); uploadPartRequest.setKey(upload.getKey()); uploadPartRequest.setUploadId(upload.getUploadId()); uploadPartRequest.setInputStream(upload.getFile().getInputStream()); // 设置分片大小。除了最后一个分片没有大小限制,其他的分片最小为100 KB。 uploadPartRequest.setPartSize(upload.getPartSize()); // 设置分片号。每一个上传的分片都有一个分片号,取值范围是1~10000,如果超出此范围,OSS将返回InvalidArgument错误码。 uploadPartRequest.setPartNumber(upload.getPartNumber()); // 每个分片不需要按顺序上传,甚至可以在不同客户端上传,OSS会按照分片号排序组成完整的文件。 return ossClient.uploadPart(uploadPartRequest).getPartETag(); } catch (IOException e) { e.printStackTrace(); } return null; } /** * 获取已上传分页列表 * @param parts * @return */ public PartListing list(OssListParts parts) { //ValidatorUtils.validateEntity(parts); /** * 获取oss的属性 */ String endpoint = ""; String accessKeyId = ""; String accessKeySecret = ""; String bucketName = ""; if ( StringUtils.isNotBlank(parts.getEndpoint())){ endpoint = parts.getEndpoint(); }else { endpoint = ossProperties.getEndpoint(); } if ( StringUtils.isNotBlank(parts.getKeyId())){ accessKeyId = parts.getKeyId(); }else { accessKeyId = ossProperties.getKeyId(); } if ( StringUtils.isNotBlank(parts.getKeySecret())){ accessKeySecret = parts.getKeySecret(); }else { accessKeySecret = ossProperties.getKeySecret(); } if ( StringUtils.isNotBlank(parts.getBucketName())){ bucketName = parts.getBucketName(); }else { bucketName = ossProperties.getBucketName(); } // 创建OSSClient实例。 OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret); try { // 列举已上传的分片。 ListPartsRequest listPartsRequest = new ListPartsRequest(bucketName, parts.getKey(), parts.getUploadId()); // 设置uploadId。 //listPartsRequest.setUploadId(uploadId); // 设置分页时每一页中分片数量为100个。默认列举1000个分片。 listPartsRequest.setMaxParts(parts.getMaxParts()); // 指定List的起始位置。只有分片号大于此参数值的分片会被列举。 listPartsRequest.setPartNumberMarker(parts.getPartNumberMarker()); return ossClient.listParts(listPartsRequest); }catch (Exception e){ e.printStackTrace(); } return null; } /** * 完成分片上传 * @return */ public CompleteMultipartUploadResult completeUpload(OssComplete complete) { //ValidatorUtils.validateEntity(complete, DefaultGroup.class); /** * 获取oss的属性 */ String endpoint = ""; String accessKeyId = ""; String accessKeySecret = ""; String bucketName = ""; if ( StringUtils.isNotBlank(complete.getEndpoint())){ endpoint = complete.getEndpoint(); }else { endpoint = ossProperties.getEndpoint(); } if ( StringUtils.isNotBlank(complete.getKeyId())){ accessKeyId = complete.getKeyId(); }else { accessKeyId = ossProperties.getKeyId(); } if ( StringUtils.isNotBlank(complete.getKeySecret())){ accessKeySecret = complete.getKeySecret(); }else { accessKeySecret = ossProperties.getKeySecret(); } if ( StringUtils.isNotBlank(complete.getBucketName())){ bucketName = complete.getBucketName(); }else { bucketName = ossProperties.getBucketName(); } List partETags = new ArrayList<>(); if ( CollectionUtils.isNotEmpty(complete.getTags())){ for ( ETag e : complete.getTags()){ PartETag partETag = new PartETag(e.getPartNumber(), e.getETag(), e.getPartSize(), e.getPartCRC()); partETags.add(partETag); } } // 创建OSSClient实例。 OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret); // 在执行完成分片上传操作时,需要提供所有有效的partETags。OSS收到提交的partETags后,会逐一验证每个分片的有效性。当所有的数据分片验证通过后,OSS将把这些分片组合成一个完整的文件。 CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, complete.getKey(), complete.getUploadId(), partETags); // 完成分片上传。 return ossClient.completeMultipartUpload(completeMultipartUploadRequest); } /** * 取消oss分片上传 * @param cancel */ public void abortUpload(OssCancel cancel) { //ValidatorUtils.validateEntity(cancel, DefaultGroup.class); /** * 获取oss的属性 */ String endpoint = ""; String accessKeyId = ""; String accessKeySecret = ""; String bucketName = ""; if ( StringUtils.isNotBlank(cancel.getEndpoint())){ endpoint = cancel.getEndpoint(); }else { endpoint = ossProperties.getEndpoint(); } if ( StringUtils.isNotBlank(cancel.getKeyId())){ accessKeyId = cancel.getKeyId(); }else { accessKeyId = ossProperties.getKeyId(); } if ( StringUtils.isNotBlank(cancel.getKeySecret())){ accessKeySecret = cancel.getKeySecret(); }else { accessKeySecret = ossProperties.getKeySecret(); } if ( StringUtils.isNotBlank(cancel.getBucketName())){ bucketName = cancel.getBucketName(); }else { bucketName = ossProperties.getBucketName(); } // 创建OSSClient实例。 OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret); // 取消分片上传。 AbortMultipartUploadRequest abortMultipartUploadRequest = new AbortMultipartUploadRequest(bucketName, cancel.getKey(), cancel.getUploadId()); ossClient.abortMultipartUpload(abortMultipartUploadRequest); } }