java对大文件分片上传-编程知识

这里记录一下，Java对大文件的切分，和后端接口分片上传的实现逻辑
正常，前后端分离的项目其实是前端去切分文件，后端接口接收到切分后的分片文件去合并，这里都用java来记录一下。
特别说明：我这里用的是zip包的上传，里面是音频文件，如果你的文件是单个文件，切分和合并文件逻辑都是一样的，只是不用后续的解压。
因为是测试代码，所以部分代码规范不严谨

1.文件的切分

public static void chunkFile(){//每片的大小，这里是10Mint TENMB = 10485760;//要切分的大文件和切分后的文件放到目录String PATH = "D:\\test\\fenpian\\";try {File file = new File(PATH, "55.zip");RandomAccessFile accessFile = new RandomAccessFile(file, "r");// 文件的大小long size = FileUtil.size(file);int chunkSize = (int) Math.ceil((double) size / TENMB);for (int i = 0; i < chunkSize; i++) {// 文件操作的指针位置long filePointer = accessFile.getFilePointer();byte[] bytes;if (i == chunkSize - 1) {int len = (int) (size - filePointer);bytes = new byte[len];accessFile.read(bytes, 0, bytes.length);} else {bytes = new byte[TENMB];accessFile.read(bytes, 0, bytes.length);}FileUtil.writeBytes(bytes, new File(PATH, String.valueOf(i) + ".zip"));}} catch (FileNotFoundException e) {e.printStackTrace();} catch (IOException e) {e.printStackTrace();}}

55.zip切分成了10个小的zip包

2.Spring boot分片上传接口

controller层

	@Resourceprivate FileUploadService fileUploadService;@RequestMapping(value = "/upload")public String upload(MultipartFileParam fileParam) throws IOException {try{return fileUploadService.fileUpload(fileParam);}catch (Exception e){e.printStackTrace();return "error";}}

service层

public interface FileUploadService {String fileUpload(MultipartFileParam fileParam) throws IOException;}

service实现层

@Slf4j
@Service
public class FileUploadServiceImpl implements FileUploadService {//合并后的文件的父目录private String FILE_UPLOAD_DIR = "D:\\test\\fenpian";//分片文件大小private Integer CHUNK_SIZE = 10485760;/*** 分片上传* @param fileParam* @return* @throws IOException*/private String chunkUpload(MultipartFileParam fileParam) throws IOException {// 是否为最后一片boolean lastFlag = false;int currentChunk = fileParam.getChunk();int totalChunk = fileParam.getTotalChunk();long totalSize = fileParam.getTotalSize();String fileName = fileParam.getName();String fileMd5 = fileParam.getMd5();MultipartFile multipartFile = fileParam.getFile();String parentDir = FILE_UPLOAD_DIR + File.separator + fileMd5 + File.separator;String tempFileName = fileName + "_tmp";// 写入到临时文件File tmpFile = tmpFile(parentDir, tempFileName, multipartFile, currentChunk, totalSize, fileMd5);// 检测是否为最后一个分片（这里吧每个分片数据放到了一张表里，后续可以改为用redis记录）FileChunkRecordExample example = new FileChunkRecordExample();example.createCriteria().andMd5EqualTo(fileMd5);long count = fileChunkRecordMapper.countByExample(example);if (count == totalChunk) {lastFlag = true;}if (lastFlag) {// 检查md5是否一致log.info("是否最后一个分片:{}","是");if (!checkFileMd5(tmpFile, fileMd5)) {cleanUp(tmpFile, fileMd5);throw new RuntimeException("文件md5检测不符合要求, 请检查!");}System.out.println("开始重命名....");File newFile = renameFile(tmpFile, fileName);//解析文件数据 -解压缩unzipSystem.out.println("开始解压缩....");File zipFile = ZipUtil.unzip(newFile);//得到压缩包内所有文件System.out.println("遍历zipFile.....");File[] files = zipFile.listFiles();System.out.println("打印fileName.....");//解析文件，处理业务数据for (File file : files) {System.out.println(file.getName());}log.info("所有文件上传完成, 时间是:{}, 文件名称是:{}", DateUtil.now(), fileName);//所有数据都处理完成后，删除文件和数据库记录cleanUp(new File(parentDir + fileName),fileMd5);}else{log.info("是否最后一个分片:{}","否");}return "success";}private File tmpFile(String parentDir, String tempFileName, MultipartFile file,int currentChunk, long totalSize, String fileMd5) throws IOException {log.info("开始上传文件, 时间是:{}, 文件名称是:{}", DateUtil.now(), tempFileName);long position = (currentChunk - 1) * CHUNK_SIZE;File tmpDir = new File(parentDir);File tmpFile = new File(parentDir, tempFileName);if (!tmpDir.exists()) {tmpDir.mkdirs();}RandomAccessFile tempRaf = new RandomAccessFile(tmpFile, "rw");if (tempRaf.length() == 0) {tempRaf.setLength(totalSize);}// 写入该分片数据FileChannel fc = tempRaf.getChannel();MappedByteBuffer map = fc.map(FileChannel.MapMode.READ_WRITE, position, file.getSize());map.put(file.getBytes());clean(map);fc.close();tempRaf.close();// 记录已经完成的分片FileChunkRecord fileChunkRecord = new FileChunkRecord();fileChunkRecord.setMd5(fileMd5);fileChunkRecord.setUploadStatus(1);fileChunkRecord.setChunk(currentChunk);fileChunkRecordMapper.insert(fileChunkRecord);log.info("分片文件上传完成, 时间是:{}, 文件名称是:{}", DateUtil.now(), tempFileName);return tmpFile;}private void cleanUp(File file, String md5) {if (file.exists()) {file.delete();}// 删除上传记录FileChunkRecordExample example = new FileChunkRecordExample();example.createCriteria().andMd5EqualTo(md5);fileChunkRecordMapper.deleteByExample(example);}/*** 最后一片接受完后执行* @param toBeRenamed* @param toFileNewName* @return*/private File renameFile(File toBeRenamed, String toFileNewName) {// 检查要重命名的文件是否存在，是否是文件if (!toBeRenamed.exists() || toBeRenamed.isDirectory()) {log.info("File does not exist: " + toBeRenamed.getName());throw new RuntimeException("File does not exist");}String parentPath = toBeRenamed.getParent();File newFile = new File(parentPath + File.separatorChar + toFileNewName);// 如果存在, 先删除if (newFile.exists()) {newFile.delete();}toBeRenamed.renameTo(newFile);return newFile;}private static void clean(MappedByteBuffer map) {try {Method getCleanerMethod = map.getClass().getMethod("cleaner");Cleaner.create(map, null);getCleanerMethod.setAccessible(true);Cleaner cleaner = (Cleaner) getCleanerMethod.invoke(map);cleaner.clean();} catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {e.printStackTrace();}}/*** 文件md5值检查，最后一片文件合并后执行，* @param file    所有分片文件合并后的文件（正常情况下md5应该和前端传过来的大文件的md5一致）* @param fileMd5 大文件的md5值* @return* @throws IOException*/private boolean checkFileMd5(File file, String fileMd5) throws IOException {FileInputStream fis = new FileInputStream(file);String checkMd5 = DigestUtils.md5DigestAsHex(fis).toUpperCase();fis.close();if (checkMd5.equals(fileMd5.toUpperCase())) {return true;}return false;}/*** 不分片* @param fileParam* @return*/private String singleUpload(MultipartFileParam fileParam) {MultipartFile file = fileParam.getFile();File baseFile = new File(FILE_UPLOAD_DIR);if (!baseFile.exists()) {baseFile.mkdirs();}try {file.transferTo(new File(baseFile, fileParam.getName()));Date now = new Date();FileRecord fileRecord = new FileRecord();String filePath = FILE_UPLOAD_DIR + File.separator + fileParam.getName();long size = FileUtil.size(new File(filePath));String sizeStr = size / (1024 * 1024) + "Mb";fileRecord.setFileName(fileParam.getName()).setFilePath(filePath).setUploadStatus(1).setFileMd5(fileParam.getMd5()).setCreateTime(now).setUpdateTime(now).setFileSize(sizeStr);//fileRecordMapper.insert(fileRecord);} catch (IOException e) {log.error("单独上传文件错误, 问题是:{}, 时间是:{}", e.getMessage(), DateUtil.now());}return "success";}}

后端入参实体类

package com.server.controller.bigFileUpload;import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import lombok.experimental.Accessors;
import org.springframework.web.multipart.MultipartFile;/*** @description:* @date: created in 2021/10/6* @modified:*/
@Getter
@Setter
@ToString
@Accessors(chain = true)
public class MultipartFileParam {/*** 是否分片*/private boolean chunkFlag;/*** 当前为第几块分片*/private int chunk;/*** 总分片数量*/private int totalChunk;/*** 文件总大小, 单位是byte*/private long totalSize;/*** 文件名*/private String name;/*** 文件*/private MultipartFile file;/*** md5值*/private String md5;}

合并后的文件放到了文件mdf的文件夹内
在这里插入图片描述