文章目录
- 什么是断点续传
- 后端实现
- JAVA实现大文件分片上传断点续传
什么是断点续传
用户上传大文件,网络差点的需要历时数小时，万一线路中断，不具备断点续传的服务器就只能从头重传，而断点续传就是，允许用户从上传断线的地方继续传送，这样大大减少了用户的烦恼。
解决上传大文件服务器内存不够的问题
解决如果因为其他因素导致上传终止的问题,并且刷新浏览器后仍然能够续传，重启浏览器（关闭浏览器后再打开）仍然能够继续上传，重启电脑后仍然能够上传
检测上传过程中因网络波动导致文件出现了内容丢失那么需要自动检测并且从新上传
解决方案
前端
需要进行分割上传的文件
需要对上传的分片文件进行指定文件序号
需要监控上传进度,控制进度条
上传完毕后需要发送合并请求
Blob 对象,操作文件
后端
上传分片的接口
合并分片的接口
获取分片的接口
其他工具方法,用于辅助
前端端需要注意的就是: 文件的切割,和进度条
后端需要注意的就是: 分片存储的地方和如何进行合并分片
后端实现
package com.controller.commontools.fIleupload;import com.alibaba.fastjson.JSON;
import com.application.Result;
import com.container.ArrayByteUtil;
import com.encryption.hash.HashUtil;
import com.file.FileUtils;
import com.file.FileWebUpload;
import com.file.ReadWriteFileUtils;
import com.function.impl.ExecutorUtils;
import com.path.ResourceFileUtil;
import com.string.PatternCommon;
import org.springframework.web.bind.annotation.*;import javax.servlet.http.HttpServletRequest;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;@RestController
@RequestMapping("/fileslice")
public class FIleSliceUploadController {private final String identification="-slice-";private final String uploadslicedir="uploads"+File.separator+"slice"+File.separator;//分片目录private final String uploaddir="uploads"+File.separator+"real"+File.separator;//实际文件目录//获取分片@GetMapping("/testing/{fileName}/{fileSlicSize}/{fileSize}")public Result testing(@PathVariable String fileName,@PathVariable long fileSlicSize,@PathVariable long fileSize ) throws Exception {String dir = fileNameMd5Dir(fileName,fileSize);String absoluteFilePathAndCreate = ResourceFileUtil.getAbsoluteFilePathAndCreate(uploadslicedir)+File.separator+dir;File file = new File(absoluteFilePathAndCreate);if (file.exists()) {List<String> filesAll = FileUtils.getFilesAll(file.getAbsolutePath());if (filesAll.size()<2){//分片缺少 删除全部分片文件 ,从新上传FileUtils.delFilesAllReview(absoluteFilePathAndCreate,true);return Result.Error();}//从小到大文件进行按照序号排序,和判断分片是否损坏List<String> collect = fileSliceIsbadAndSort(file, fileSlicSize);//获取最后一个分片String fileSliceName = collect.get(collect.size() - 1);fileSliceName = new File(fileSliceName).getName();int code = fileId(fileSliceName);//服务器的分片总大小必须小于或者等于文件的总大小if ((code*fileSlicSize)<=fileSize) {Result result = new Result();String finalFileSliceName = fileSliceName;String str = PatternCommon.renderString("{\"code\":\"$[code]\",\"fileSliceName\":\"${fileSliceName}\"}", new HashMap<String, String>() {{put("code", String.valueOf(code));put("fileSliceName", finalFileSliceName);}});result.setData(JSON.parse(str));return result;}else {//分片异常 ,删除全部分片文件,从新上传FileUtils.delFilesAllReview(absoluteFilePathAndCreate,true);return Result.Error();}}//不存在return Result.Error();}@PostMapping(value = "/uploads")public Result uploads(HttpServletRequest request) {String fileSliceName = request.getParameter("fileSliceName");long fileSize = Long.parseLong(request.getParameter("fileSize")); //文件大小String dir = fileSliceMd5Dir(fileSliceName,fileSize);String absoluteFilePathAndCreate = ResourceFileUtil.getAbsoluteFilePathAndCreate(uploadslicedir+dir);FileWebUpload.fileUpload(absoluteFilePathAndCreate,fileSliceName,request);int i = fileId(fileSliceName); //返回上传成功的文件id,用于前端计算进度Result result=new Result();result.setData(i);return result;}// 合并分片@GetMapping(value = "/merge-file-slice/{fileSlicNamee}/{fileSlicSize}/{fileSize}")public Result mergeFileSlice(@PathVariable String fileSlicNamee,@PathVariable long fileSlicSize,@PathVariable long fileSize ) throws Exception {int l =(int) Math.ceil((double) fileSize / fileSlicSize); //有多少个分片String dir = fileSliceMd5Dir(fileSlicNamee,fileSize); //分片所在的目录String absoluteFilePathAndCreate = ResourceFileUtil.getAbsoluteFilePathAndCreate(uploadslicedir+dir);File file=new File(absoluteFilePathAndCreate);if (file.exists()){List<String> filesAll = FileUtils.getFilesAll(file.getAbsolutePath());//阻塞循环判断是否还在上传 ,解决前端进行ajax异步上传的问题int beforeSize=filesAll.size();while (true){Thread.sleep(1000);//之前分片数量和现在分片数据只差,如果大于1那么就在上传,那么继续filesAll = FileUtils.getFilesAll(file.getAbsolutePath());if (filesAll.size()-beforeSize>=1){beforeSize=filesAll.size();//继续检测continue;}//如果是之前分片和现在的分片相等的,那么在阻塞2秒后检测是否发生变化,如果还没变化那么上传全部完成,可以进行合并了//当然这不是绝对的,只能解决网络短暂的波动,因为有可能发生断网很长时间,网络恢复后文件恢复上传, 这个问题是避免不了的,所以我们在下面的代码进行数量的效验// 因为我们不可能一直等着他网好,所以如果1~3秒内没有上传新的内容,那么我们默认判定上传完毕if (beforeSize==filesAll.size()){Thread.sleep(2000);filesAll = FileUtils.getFilesAll(file.getAbsolutePath());if (beforeSize==filesAll.size()){break;}}}//分片数量效验if (filesAll.size()!=l){//分片缺少 ,删除全部分片文件,从新上传FileUtils.delFilesAllReview(absoluteFilePathAndCreate,true);return Result.Error();}//获取实际的文件名称,组装路径String realFileName = realFileName(fileSlicNamee);String realFileNamePath = ResourceFileUtil.getAbsoluteFilePathAndCreate(uploaddir+ realFileName);//从小到大文件进行按照序号排序 ,和检查分片文件是否有问题List<String> collect = fileSliceIsbadAndSort(file, fileSlicSize);int fileSliceSize = collect.size();List<Future<?>> futures = new ArrayList<>();// 将文件按照序号进行合并 ,算出Runtime.getRuntime().availableProcessors()个线程 ,每个线程需要读取多少分片, 和每个线程需要读取多少字节大小//有人会说一个分片一个线程不行吗,你想想如果上千或者上万分片的话,你创建这么多的线程需要多少时间,以及线程切换上下文切换和销毁需要多少时间? // 就算使用线程池,也顶不住啊,你内存又有多大,能存下多少队列?,并发高的话直接怼爆int availableProcessors = Runtime.getRuntime().availableProcessors();//每个线程读取多少文件int readFileSize = (int)Math.ceil((double)fileSliceSize / availableProcessors);//每个线程需要读取的文件大小long readSliceSize = readFileSize * fileSlicSize;for (int i = 0; i < availableProcessors; i++) {int finalI = i;Future<?> future = ExecutorUtils.createFuture("FIleSliceUploadController",()->{//每个线程需要读取多少字节byte[] bytes=new byte[(int) readSliceSize];int index=0;for (int i1 = finalI *readFileSize,i2 = readFileSize*(finalI+1)>fileSliceSize?fileSliceSize:readFileSize*(finalI+1); i1 < i2; i1++) {try ( RandomAccessFile r = new RandomAccessFile(collect.get(i1), "r");){r.read(bytes, (int)(index*fileSlicSize),(int)fileSlicSize);} catch (IOException e) {e.printStackTrace();}index++;}if(finalI==availableProcessors-1){//需要调整数组bytes = ArrayByteUtil.getActualBytes(bytes);}try ( RandomAccessFile w = new RandomAccessFile(realFileNamePath, "rw");){//当前文件写入的位置w.seek(finalI*readSliceSize);w.write(bytes);} catch (IOException e) {e.printStackTrace();}});futures.add(future);}//阻塞到全部线程执行完毕后ExecutorUtils.waitComplete(futures);//删除全部分片文件FileUtils.delFilesAllReview(absoluteFilePathAndCreate,true);}else {//没有这个分片相关的的目录return Result.Error();}return Result.Ok();}//获取分片文件的目录private String fileSliceMd5Dir(String fileSliceName,long fileSize){int i = fileSliceName.indexOf(identification) ;String substring = fileSliceName.substring(0, i);String dir = HashUtil.md5(substring+fileSize);return dir;}//通过文件名称获取文件目录private String fileNameMd5Dir(String fileName,long fileSize){return HashUtil.md5(fileName+fileSize);}//获取分片的实际文件名private String realFileName(String fileSliceName){int i = fileSliceName.indexOf(identification) ;String substring = fileSliceName.substring(0, i);return substring;}//获取文件序号private int fileId(String fileSliceName){int i = fileSliceName.indexOf(identification)+identification.length() ;String fileId = fileSliceName.substring(i);return Integer.parseInt(fileId);}//判断是否损坏private List<String> fileSliceIsbadAndSort(File file,long fileSlicSize) throws Exception {String absolutePath = file.getAbsolutePath();List<String> filesAll = FileUtils.getFilesAll(absolutePath);if (filesAll.size()<1){//分片缺少,删除全部分片文件 ,从新上传FileUtils.delFilesAllReview(absolutePath,true);throw new Exception("分片损坏");}//从小到大文件进行按照序号排序List<String> collect = filesAll.stream().sorted((a, b) -> fileId(a) - fileId(b)).collect(Collectors.toList());//判断文件是否损坏,将文件排序后,进行前后序号相差大于1那么就代表少分片了for (int i = 0; i < collect.size()-1; i++) {//检测分片的连续度if (fileId(collect.get(i)) - fileId(collect.get(i+1))!=-1) {//分片损坏 删除全部分片文件 ,从新上传FileUtils.delFilesAllReview(absolutePath,true);throw new Exception("分片损坏");}//检测分片的完整度if (new File(collect.get(i)).length()!=fileSlicSize) {//分片损坏 删除全部分片文件 ,从新上传FileUtils.delFilesAllReview(absolutePath,true);throw new Exception("分片损坏");}}return collect;}
}
JAVA实现大文件分片上传断点续传
import org.springframework.web.multipart.MultipartFile;
import java.util.concurrent.CompletableFuture;
import org.apache.commons.lang3.StringUtils;
import lombok.extern.slf4j.Slf4j;
import java.text.DecimalFormat;
import java.io.*;@Slf4j
public class FileSliceUpload {// 文件上传地址 private final String uploadPath = "/data/upload/";/** * @description: 文件分片上传 * @date: 2024/3/12 15:25 * @param fileSliceDTO * @return boolean */public boolean fileUpload(FileSliceDTO fileSliceDTO) {// 当前分片序号Integer chunkNumber = fileSliceDTO.getChunkNumber();// 当前分片大小 Long currentChunkSize = fileSliceDTO.getCurrentChunkSize();// 总分片数 Integer totalChunks = fileSliceDTO.getTotalChunks();// 原文件md5 String fileMd5 = fileSliceDTO.getFileMd5();// 文件名称 String fileName = fileSliceDTO.getFileName();// 用户账号 String userAccount = fileSliceDTO.getUserAccount();// 文件总大小 Long totalSize = fileSliceDTO.getTotalSize();// 当前分片文件流 MultipartFile mFile = fileSliceDTO.getFile();log.info("接收到文件：{}，总大小：{} 总分片：{} 当前分片：{}", fileName, this.readableFileSize(totalSize), totalChunks, chunkNumber);String path = uploadPath + fileMd5 + "_" + userAccount + File.separator;File dirfile = new File(path);if (!dirfile.exists()) {dirfile.mkdirs();}String currentChunkFileName = path + fileMd5 + "_" + chunkNumber + ".tmp";File file = new File(currentChunkFileName);boolean uploadFlag = this.fileUpload(mFile, file);if (uploadFlag) {if (chunkNumber < totalChunks) {return true;} else {// 如果是最后一个分片，上传成功后就进行文件合并String mergeFileName = null;try {mergeFileName = this.fileMerge(fileMd5, totalChunks, fileName, totalSize, userAccount);} catch (IOException e) {log.info("文件合并发生异常：{}, {}", fileName, e.toString());}if (StringUtils.isBlank(mergeFileName)) {// 合并失败后异步删除文件夹CompletableFuture.runAsync(() - > { delFile(new File(path)); });return false;}log.info("文件合并成功：{}, 总大小: {}", fileName, this.readableFileSize(totalSize)); // 合并成功后异步删除文件夹CompletableFuture.runAsync(() - > {delFile(new File(path)); });return true;}}return false;}/** * @description: 文件上传私有方法* @date: 2024/3/12 15:25 * @param multipartFile 分片文件* @param file 目标文件* @return boolean */private boolean fileUpload(MultipartFile multipartFile, File file) {boolean flag = false;try {if (!file.exists()) {file.createNewFile();multipartFile.transferTo(file);} else {log.info("当前分片文件已存在:{}", file.getName());}flag = true;} catch (Exception e) {log.info("文件上传失败：{}, {}", multipartFile.getOriginalFilename(), e.toString());}return flag;} /** * @description: 合并文件 * @date: 2024/3/12 15:21 * @param fileMd5 * @param chunks * @param fileName * @param totalSize * @param userAccount * @return java.lang.String */private String fileMerge(String fileMd5, Integer chunks, String fileName, long totalSize, String userAccount) throws IOException {String fileType = this.getFileSuffix(fileName);String mergePath = uploadPath + UUIDUtil.uuid32() + "." + fileType;FileOutputStream fileOutputStream = new FileOutputStream(mergePath);String mergeFileName = null;long fileSize = 0 L;try {byte[] buf = new byte[1024 * 4];for (int i = 1; i <= chunks; i++) {String chunkFile = i + ".tmp";File file = new File(uploadPath + fileMd5 + "_" + userAccount + File.separator + fileMd5 + "_" + chunkFile);fileSize = fileSize + file.length();InputStream inputStream = new FileInputStream(file);int len;while ((len = inputStream.read(buf)) != -1) {fileOutputStream.write(buf, 0, len);}inputStream.close();}mergeFileName = mergePath;} catch (Exception e) {log.info("合并文件失败：{},{}", fileName, e.toString());} finally {fileOutputStream.close();}if (fileSize != totalSize) {log.info("文件总大小不一致:{}", fileName);return null;}return mergeFileName;} /*** @description: 删除文件/文件夹 * @date: 2024/3/12 15:19* @param file * @return boolean*/private boolean delFile(File file) {if (file == null || !file.exists()) {return true;}if (!file.isDirectory()) {file.delete();return true;} else {File[] files = file.listFiles();for (File f: files) {if (f.isDirectory()) {delFile(f);} else {f.delete();}}file.delete();return true;}} /** * @description: 文件大小可读化转换* @date: 2024/3/12 15:18 * @param size* @return java.lang.String */private String readableFileSize(long size) {if (size <= 0) {return "0";}final String[] units = new String[] {"B", "KB", "MB", "GB", "TB"};int digitGroups = (int)(Math.log10(size) / Math.log10(1024));return new DecimalFormat("#,###.##").format(size / Math.pow(1024, digitGroups)) + " " + units[digitGroups];} /** * @description: 获取文件的拓展名 支持形式 abc.jpg or D:/data/abc.txt; * @date: 2024/3/12 15:16 * @param fileName * @return java.lang.String */private String getFileSuffix(String fileName) {String newFileName = fileName; if(fileName.contains("/") || fileName.contains("\\")){ String replaceFileName = fileName.replaceAll("\\\\", "/"); int lastIndexOf = replaceFileName.lastIndexOf("/"); if(lastIndexOf > 0) { newFileName = replaceFileName.substring(lastIndexOf); } } int lastIndexOf = newFileName.lastIndexOf(".");String fileSuffix = lastIndexOf >= 0 ? newFileName.substring(lastIndexOf + 1).toLowerCase() : "";fileSuffix = newFileName.toLowerCase().endsWith(".tar.gz") ? "tar.gz" : fileSuffix;return fileSuffix;}
}
