package com.zilber.boot.file.service; import com.aliyun.oss.*; import com.aliyun.oss.model.*; import com.baomidou.mybatisplus.core.toolkit.CollectionUtils; import com.cppcc.common.validator.ValidatorUtils; import com.cppcc.files.*; import com.cppcc.files.dao.FileDao; import com.zilber.boot.file.entity.*; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import reactor.util.annotation.Nullable; import javax.annotation.Resource; import java.io.*; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; /** * @Author LJX * @TIME 2023-01-09 9:11 * @PROJECT cppcc * created by Intellij IDEA * Description */ @Service @Slf4j public class UploadUtils { @Value("${cppcc.localtion}") private String localtion; @Resource private OssProperties ossProperties; @Autowired private FileDao fileDao; public Map upload(MultipartFile file) { try { String type = "." + FilenameUtils.getExtension(file.getOriginalFilename()); String name = FilenameUtils.getName(file.getOriginalFilename()); String filePath = localtion + "/upload"; String fileName = UUID.randomUUID() + type; File desc = getAbsoluteFile(filePath, fileName); file.transferTo(desc); return getPathFileName(filePath, fileName, name, type); } catch (Exception e) { log.error(e.getLocalizedMessage()); return new ConcurrentHashMap<>(); } } private Map getPathFileName(String uploadDir, String fileName, String name, String type) throws IOException { Map map = new ConcurrentHashMap(3); int dirLastIndex = localtion.length() + 1; String currentDir = StringUtils.substring(uploadDir, dirLastIndex); String pathFileName = "/profile" + "/" + currentDir + "/" + fileName; map.put("url", pathFileName); map.put("name", name); map.put("type", String.valueOf(SliceUploadsTemplate.getFileType(type))); return map; } private File getAbsoluteFile(String uploadDir, String fileName) { File desc = new File(uploadDir + File.separator + fileName); if (!desc.exists()) { if (!desc.getParentFile().exists()) { desc.getParentFile().mkdirs(); } } return desc; } //TODO 上传至oss /** * 上传文件 * * @param file 上传的文件 * @param module oss目录 * @return */ public PutObjectResult upload(MultipartFile file, @Nullable String module) { /** * 获取oss的属性 */ String endpoint = ossProperties.getEndpoint(); String accessKeyId = ossProperties.getKeyId(); String accessKeySecret = ossProperties.getKeySecret(); String bucketName = ossProperties.getBucketName(); if ( StringUtils.isBlank(module) ){ module = ossProperties.getModule(); } String extension = file.getName().substring(file.getName().lastIndexOf(".")); String objectName = module + "/" + UUID.randomUUID().toString() + extension; // 创建OSSClient实例。 OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret); try { // 创建PutObjectRequest对象。 PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, objectName, file.getInputStream()); // 设置该属性可以返回response。如果不设置,则返回的response为空。 putObjectRequest.setProcess("true"); // 上传字符串。 PutObjectResult result = ossClient.putObject(putObjectRequest); return result; } catch (OSSException oe) { throw new ServiceException(oe.getMessage()); } catch (ClientException ce) { throw new ServiceException(ce.getMessage()); } catch (IOException e) { e.printStackTrace(); } finally { if (ossClient != null) { // 关闭OSSClient。 ossClient.shutdown(); } } return null; } //TODO 分片上传至服务器 /** * 清除临时目录下的临时文件 * * @param root */ private void clearTempFiles(File root) { File[] files = root.listFiles(); if (files != null) { List fileList = Arrays.asList(files); for (File f : fileList) { f.delete(); } } root.delete(); } /********************************************************************************************/ public FileUploadDTO sliceUploads(FileUploadsRequestDTO param) { boolean isOk = true; File root = new File(localtion + "/upload", param.getUUID()); if (!root.exists() || !root.isDirectory()) { isOk = root.mkdirs(); } File targetFile = new File(localtion + "/upload", param.getUUID() + param.getType()); if (!targetFile.exists() || !targetFile.isFile()) { try { targetFile.createNewFile(); } catch (IOException e) { e.printStackTrace(); } } Map map = new HashMap<>(); map.put(param.getChunk(), param.getUUID()); if (isOk) { String sliceName = param.getChunk().toString(); upload(root, sliceName, param); } return FileUploadDTO.builder().chunkMd5Info(map).build(); } /** * 文件检测 * * @param md5 md5 * @param UUID UUID * @return * @throws IOException */ public FileCheckDTO fileCheck(String md5, String UUID) throws IOException { FileCheckDTO fileCheckDTO = new FileCheckDTO(); //根据文件md5查询文件是否存在,如果存在,则直接返回文件路径,否则返回块信息 //TODO Map objectMap = fileDao.listFileUrl(md5); if (objectMap != null) { fileCheckDTO.setUrl(objectMap.get("url").toString()); fileCheckDTO.setType(Integer.parseInt(objectMap.get("type").toString())); fileCheckDTO.setExist(true); return fileCheckDTO; } //不存在 return checkUnexist(UUID, fileCheckDTO); } /** * @return * @author liuao */ public boolean upload(File root, String name, FileUploadsRequestDTO param) { //保存最大块数 File aConfFile = new File(root.getAbsolutePath(), param.getUUID() + ".conf"); File sizeFile = new File(root.getAbsolutePath(), "size.conf"); if (!aConfFile.exists()) { FileWriter w = null; try { aConfFile.createNewFile(); w = new FileWriter(aConfFile); w.write(param.getChunks().toString()); } catch (IOException e) { e.printStackTrace(); } finally { try { assert w != null; w.close(); } catch (IOException e) { e.printStackTrace(); } } } RandomAccessFile accessTmpFile = null; int length = 0; File[] files = root.listFiles(); if (files != null) { length = files.length; } // String uploadDirPath = localtion + "/upload"; File tmpFile = new File(root.getAbsolutePath(), name); File tmpConfFile = new File(root.getAbsolutePath(), name + ".conf"); try { accessTmpFile = new RandomAccessFile(tmpFile, "rw"); accessTmpFile.write(param.getFile().getBytes()); log.info(param.getUUID() + "文件块" + param.getChunk() + "上传完成"); return true; } catch (IOException e) { log.error(e.getMessage(), e); if (tmpConfFile.exists()) { tmpConfFile.delete(); } } finally { try { if (tmpFile.length() == param.getFile().getSize()) { if (!tmpConfFile.exists()) { tmpConfFile.createNewFile(); } } assert accessTmpFile != null; accessTmpFile.close(); } catch (IOException e) { log.error(e.getLocalizedMessage()); if (tmpConfFile.exists()) { tmpConfFile.delete(); } } } return false; } public FileCheckDTO compose(String UUID, String type, String md5) { long startTime = 0, endTime = 0; log.info(String.valueOf(System.currentTimeMillis())); FileCheckDTO fileCheckDTO = new FileCheckDTO(); File root = new File(localtion + "/upload", UUID); File targetFile = new File(localtion + "/upload", UUID + type); Long chunks = chunks(root, UUID); File[] fileNames = root.listFiles((file1, s1) -> !s1.endsWith(".conf")); if (fileNames == null || fileNames.length < chunks) { return checkUnexist(UUID, new FileCheckDTO()); } List collect = Arrays.asList(fileNames) .stream().sorted(new Comparator() { @Override public int compare(File file, File t1) { int f = Integer.parseInt(file.getName()); int t = Integer.parseInt(t1.getName()); return f < t ? -1 : 1; } }).collect(Collectors.toList()); FileOutputStream fos; FileInputStream fis; // 一次读取10M数据,将读取到的数据保存到byte字节数组中 byte[] buffer = new byte[1024 * 1024 * 10]; int len; try { fos = new FileOutputStream(targetFile); log.info(String.valueOf(startTime = System.currentTimeMillis())); for (File file : collect) { fis = new FileInputStream(file); len = 0; while ((len = fis.read(buffer)) != -1) { // buffer从指定字节数组写入。buffer:数据中的起始偏移量,len:写入的字数。 fos.write(buffer, 0, len); } fos.flush(); fis.close(); } fos.close(); log.info("传输完成:" + (endTime = System.currentTimeMillis())); } catch (IOException ex) { targetFile.delete(); ex.printStackTrace(); } log.info(String.valueOf(endTime - startTime)); // String fileName = java.util.UUID.randomUUID().toString(); int dirLastIndex = localtion.length() + 1; String currentDir = StringUtils.substring(localtion + "/upload", dirLastIndex); String pathFileName = "/profile" + "/" + currentDir + "/" + UUID + type; Integer t = getFileType(type); fileDao.addFile(md5, pathFileName, t); fileCheckDTO.setUrl(pathFileName); fileCheckDTO.setType(t); fileCheckDTO.setExist(true); clearTempFiles(root); return fileCheckDTO; } private FileCheckDTO checkUnexist(String UUID, FileCheckDTO fileCheckDTO) { //不存在 fileCheckDTO.setExist(false); File root = new File(localtion + "/upload", UUID); if (!root.exists()) { fileCheckDTO.setUploaded(false); return fileCheckDTO; } //获取总块数 // File aConf = new File(root.getAbsolutePath(), UUID + ".conf"); // FileReader fr = new FileReader(aConf); // char[] chunkCs = new char[(int) aConf.length()]; // fr.read(chunkCs); Long chunks = chunks(root, UUID); FilenameFilter filter = new FilenameFilter() { @Override public boolean accept(File file, String s) { return s.endsWith(".conf") && !s.contains("-"); } }; String[] fileNames = root.list(filter); if (fileNames == null) { fileCheckDTO.setUploaded(false); return fileCheckDTO; } List fileNameList = Arrays.asList(fileNames) .stream().distinct().collect(Collectors.toList()); Map chunkInfo = new HashMap<>(); if (CollectionUtils.isNotEmpty(fileNameList)) { for (String fileName : fileNameList) { String prefix = fileName.split("\\.")[0]; chunkInfo.put(prefix, true); } fileCheckDTO.setChunkInfo(chunkInfo); fileCheckDTO.setUploaded(true); fileCheckDTO.setChunks(chunks); return fileCheckDTO; } fileCheckDTO.setUploaded(false); return fileCheckDTO; } private Long chunks(File root, String UUID) { File aConf = new File(root.getAbsolutePath(), UUID + ".conf"); FileReader fr = null; char[] chunkCs = null; try { fr = new FileReader(aConf); chunkCs = new char[(int) aConf.length()]; fr.read(chunkCs); } catch (FileNotFoundException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } finally { try { fr.close(); } catch (IOException e) { e.printStackTrace(); } } return Long.valueOf(String.valueOf(chunkCs)); } public static Integer getFileType(String type) { String pic = ".jpeg,.bmp,.jpg,.png,.tif,.gif,.pcx,.tga,.exif,.fpx,.svg,.psd,.cdr,.pcd,.dxf,.ufo,.eps,.ai,.raw,.WMF,.webp,.avif,.apng"; String video = ".wmv,.asf,.asx,.mp4,.m4v,.rm,.rmvb,.mpg,.mpeg,.mpe,.3gp,.mov,.avi,.dat,.mkv,.flv,.vob"; if (pic.contains(type)) { return 1; } if (video.contains(type)) { return 3; } return 2; } //TODO 分片上传至oss /** * 初始化切片上传 * @param fileName 文件名 * @param module 目标文件夹 * @return 文件唯一标识 */ public OssPartUpload InitiateMultipartUpload(String fileName, @Nullable String module, OssProperties provided) { /** * 获取oss的属性 */ String endpoint = ""; String accessKeyId = ""; String accessKeySecret = ""; String bucketName = ""; if ( StringUtils.isNotBlank(provided.getEndpoint())){ endpoint = provided.getEndpoint(); }else { endpoint = ossProperties.getEndpoint(); } if ( StringUtils.isNotBlank(provided.getKeyId())){ accessKeyId = provided.getKeyId(); }else { accessKeyId = ossProperties.getKeyId(); } if ( StringUtils.isNotBlank(provided.getKeySecret())){ accessKeySecret = provided.getKeySecret(); }else { accessKeySecret = ossProperties.getKeySecret(); } if ( StringUtils.isNotBlank(provided.getBucketName())){ bucketName = provided.getBucketName(); }else { bucketName = ossProperties.getBucketName(); } if ( StringUtils.isBlank(module) ){ module = ossProperties.getModule(); } String extension = fileName.substring(fileName.lastIndexOf(".")); String objectName = module + "/" + UUID.randomUUID().toString() + extension; Map resultMap = new HashMap<>(); // 创建OSSClient实例。 OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret); // 创建InitiateMultipartUploadRequest对象。 InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, objectName); // 初始化分片。 InitiateMultipartUploadResult upresult = ossClient.initiateMultipartUpload(request); // 返回uploadId,它是分片上传事件的唯一标识。您可以根据该uploadId发起相关的操作,例如取消分片上传、查询分片上传等。 OssPartUpload upload = new OssPartUpload(); upload.setKey(objectName); upload.setUploadId(upresult.getUploadId()); return upload; } /** * 分片上传至oss * @param upload 分片信息 * @return */ public UploadPartResult uploadPart(OssPartUpload upload) { ValidatorUtils.validateEntity(upload, DefaultGroup.class); /** * 获取oss的属性 */ String endpoint = ""; String accessKeyId = ""; String accessKeySecret = ""; String bucketName = ""; if ( StringUtils.isNotBlank(upload.getEndpoint())){ endpoint = upload.getEndpoint(); }else { endpoint = ossProperties.getEndpoint(); } if ( StringUtils.isNotBlank(upload.getKeyId())){ accessKeyId = upload.getKeyId(); }else { accessKeyId = ossProperties.getKeyId(); } if ( StringUtils.isNotBlank(upload.getKeySecret())){ accessKeySecret = upload.getKeySecret(); }else { accessKeySecret = ossProperties.getKeySecret(); } if ( StringUtils.isNotBlank(upload.getBucketName())){ bucketName = upload.getBucketName(); }else { bucketName = ossProperties.getBucketName(); } // 创建OSSClient实例。 OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret); try { UploadPartRequest uploadPartRequest = new UploadPartRequest(); uploadPartRequest.setBucketName(bucketName); uploadPartRequest.setKey(upload.getKey()); uploadPartRequest.setUploadId(upload.getUploadId()); uploadPartRequest.setInputStream(upload.getFile().getInputStream()); // 设置分片大小。除了最后一个分片没有大小限制,其他的分片最小为100 KB。 uploadPartRequest.setPartSize(upload.getPartSize()); // 设置分片号。每一个上传的分片都有一个分片号,取值范围是1~10000,如果超出此范围,OSS将返回InvalidArgument错误码。 uploadPartRequest.setPartNumber(upload.getPartNumber()); // 每个分片不需要按顺序上传,甚至可以在不同客户端上传,OSS会按照分片号排序组成完整的文件。 return ossClient.uploadPart(uploadPartRequest); } catch (IOException e) { e.printStackTrace(); } return null; } /** * 获取已上传分页列表 * @param parts * @return */ public PartListing list(OssListParts parts) { ValidatorUtils.validateEntity(parts, DefaultGroup.class); /** * 获取oss的属性 */ String endpoint = ""; String accessKeyId = ""; String accessKeySecret = ""; String bucketName = ""; if ( StringUtils.isNotBlank(parts.getEndpoint())){ endpoint = parts.getEndpoint(); }else { endpoint = ossProperties.getEndpoint(); } if ( StringUtils.isNotBlank(parts.getKeyId())){ accessKeyId = parts.getKeyId(); }else { accessKeyId = ossProperties.getKeyId(); } if ( StringUtils.isNotBlank(parts.getKeySecret())){ accessKeySecret = parts.getKeySecret(); }else { accessKeySecret = ossProperties.getKeySecret(); } if ( StringUtils.isNotBlank(parts.getBucketName())){ bucketName = parts.getBucketName(); }else { bucketName = ossProperties.getBucketName(); } // 创建OSSClient实例。 OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret); try { // 列举已上传的分片。 ListPartsRequest listPartsRequest = new ListPartsRequest(bucketName, parts.getKey(), parts.getUploadId()); // 设置uploadId。 //listPartsRequest.setUploadId(uploadId); // 设置分页时每一页中分片数量为100个。默认列举1000个分片。 listPartsRequest.setMaxParts(parts.getMaxParts()); // 指定List的起始位置。只有分片号大于此参数值的分片会被列举。 listPartsRequest.setPartNumberMarker(parts.getPartNumberMarker()); return ossClient.listParts(listPartsRequest); }catch (Exception e){ e.printStackTrace(); } return null; } /** * 完成分片上传 * @return */ public CompleteMultipartUploadResult completeUpload(OssComplete complete) { ValidatorUtils.validateEntity(complete, DefaultGroup.class); /** * 获取oss的属性 */ String endpoint = ""; String accessKeyId = ""; String accessKeySecret = ""; String bucketName = ""; if ( StringUtils.isNotBlank(complete.getEndpoint())){ endpoint = complete.getEndpoint(); }else { endpoint = ossProperties.getEndpoint(); } if ( StringUtils.isNotBlank(complete.getKeyId())){ accessKeyId = complete.getKeyId(); }else { accessKeyId = ossProperties.getKeyId(); } if ( StringUtils.isNotBlank(complete.getKeySecret())){ accessKeySecret = complete.getKeySecret(); }else { accessKeySecret = ossProperties.getKeySecret(); } if ( StringUtils.isNotBlank(complete.getBucketName())){ bucketName = complete.getBucketName(); }else { bucketName = ossProperties.getBucketName(); } // 创建OSSClient实例。 OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret); // 在执行完成分片上传操作时,需要提供所有有效的partETags。OSS收到提交的partETags后,会逐一验证每个分片的有效性。当所有的数据分片验证通过后,OSS将把这些分片组合成一个完整的文件。 CompleteMultipartUploadRequest completeMultipartUploadRequest = new CompleteMultipartUploadRequest(bucketName, complete.getKey(), complete.getUploadId(), complete.getTags()); // 完成分片上传。 return ossClient.completeMultipartUpload(completeMultipartUploadRequest); } /** * 取消oss分片上传 * @param cancel */ public void abortUpload(OssCancel cancel) { ValidatorUtils.validateEntity(cancel, DefaultGroup.class); /** * 获取oss的属性 */ String endpoint = ""; String accessKeyId = ""; String accessKeySecret = ""; String bucketName = ""; if ( StringUtils.isNotBlank(cancel.getEndpoint())){ endpoint = cancel.getEndpoint(); }else { endpoint = ossProperties.getEndpoint(); } if ( StringUtils.isNotBlank(cancel.getKeyId())){ accessKeyId = cancel.getKeyId(); }else { accessKeyId = ossProperties.getKeyId(); } if ( StringUtils.isNotBlank(cancel.getKeySecret())){ accessKeySecret = cancel.getKeySecret(); }else { accessKeySecret = ossProperties.getKeySecret(); } if ( StringUtils.isNotBlank(cancel.getBucketName())){ bucketName = cancel.getBucketName(); }else { bucketName = ossProperties.getBucketName(); } // 创建OSSClient实例。 OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret); // 取消分片上传。 AbortMultipartUploadRequest abortMultipartUploadRequest = new AbortMultipartUploadRequest(bucketName, cancel.getKey(), cancel.getUploadId()); ossClient.abortMultipartUpload(abortMultipartUploadRequest); } }