Springboot实现分片上传,分片下载(断点下载)

本文详细介绍了如何使用 Springboot 框架实现文件的分片上传和断点续传下载功能,涉及到的核心组件包括 pom.xml 配置、application.yml 配置、logback 日志配置、前端 HTML 以及 DTO、Controller 和 Service 层的实现。

本文实现断点续传,分片下载,废话不多说,接下来带大家分步实现

pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>
	<parent>
		<groupId>org.springframework.boot</groupId>
		<artifactId>spring-boot-starter-parent</artifactId>
		<version>2.7.6</version>
		<relativePath/> <!-- lookup parent from repository -->
	</parent>
	<groupId>springboot-file-senior-operate</groupId>
	<artifactId>springboot-file-senior-operate-demo</artifactId>
	<version>0.0.1-SNAPSHOT</version>
	<name>demo</name>
	<description>Demo project for Spring Boot</description>
	<properties>
		<java.version>8</java.version>
	</properties>
	<dependencies>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter</artifactId>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-web</artifactId>
		</dependency>
		<dependency>
			<groupId>org.projectlombok</groupId>
			<artifactId>lombok</artifactId>
		</dependency>
		<dependency>
			<groupId>com.alibaba</groupId>
			<artifactId>fastjson</artifactId>
			<version>1.2.71</version>
		</dependency>
		<dependency>
			<groupId>org.apache.commons</groupId>
			<artifactId>commons-collections4</artifactId>
			<version>4.2</version>
		</dependency>
		<dependency>
			<groupId>org.apache.commons</groupId>
			<artifactId>commons-lang3</artifactId>
		</dependency>
		<dependency>
			<groupId>commons-io</groupId>
			<artifactId>commons-io</artifactId>
			<version>2.11.0</version>
		</dependency>
		<dependency>
			<groupId>org.apache.commons</groupId>
			<artifactId>commons-pool2</artifactId>
			<version>2.10.0</version>
		</dependency>
		<dependency>
			<groupId>commons-codec</groupId>
			<artifactId>commons-codec</artifactId>
			<version>1.15</version>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-logging</artifactId>
		</dependency>
		<dependency>
			<groupId>commons-fileupload</groupId>
			<artifactId>commons-fileupload</artifactId>
			<version>1.3.1</version>
		</dependency>
		<dependency>
			<groupId>commons-io</groupId>
			<artifactId>commons-io</artifactId>
			<version>2.4</version>
		</dependency>
		<!-- 做断点下载使用-->
		<dependency>
			<groupId>org.apache.httpcomponents</groupId>
			<artifactId>httpcore</artifactId>
		</dependency>
		<dependency>
			<groupId>org.apache.httpcomponents</groupId>
			<artifactId>httpclient</artifactId>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-test</artifactId>
			<scope>test</scope>
		</dependency>
	</dependencies>

	<build>
		<plugins>
			<plugin>
				<groupId>org.springframework.boot</groupId>
				<artifactId>spring-boot-maven-plugin</artifactId>
			</plugin>
			<plugin>
				<groupId>org.apache.maven.plugins</groupId>
				<artifactId>maven-compiler-plugin</artifactId>
				<version>3.8.1</version>
				<configuration>
					<source>1.8</source>
					<target>1.8</target>
				</configuration>
			</plugin>
		</plugins>
	</build>

</project>

resources/application.yml

server:
  port: 8080
  
spring:
  servlet:
   multipart:
    enabled: false

file:
 upload-path: D:\upload
 download-path: D:\download
 download-file:D:\文件路径+文件名

resources/logback-spring.xml

<?xml version="1.0" encoding="UTF-8"?>
<configuration>
    <!--控制台输出-->
    <appender name="stdout" class="ch.qos.logback.core.ConsoleAppender">
        <encoder>
            <pattern>%d{yyyy-MM-dd HH:mm:ss} [%-5level] [%-5thread] %logger{20} - %msg%n</pattern>
        </encoder>
    </appender>


    <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
        <!--过滤-->
        <filter class="ch.qos.logback.classic.filter.LevelFilter">
            <level>INFO</level>
        </filter>
        <file>logs/file-svc.log</file>
        <encoder>
            <pattern>%d{yyyy-MM-dd HH:mm:ss} [%-5level] [%-5thread] %logger{20} - %msg%n</pattern>
        </encoder>
        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
            <FileNamePattern>logs/file-svc.%d{yyyy-MM-dd}.%i.log</FileNamePattern>
            <MaxHistory>5</MaxHistory>
            <maxFileSize>100MB</maxFileSize>
        </rollingPolicy>
    </appender>


    <root level="INFO">
        <appender-ref ref="stdout"/>
        <appender-ref ref="FILE"/>
    </root>

</configuration>

resources/static/index.html

需要集成【jquery.js】、百度【webuploader.js、webuploader.css、Uploader.swf】

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <title>upload</title>
    <link rel="stylesheet" type="text/css" href="webuploader.css">
    <script src="jquery.js"></script>
    <script src="webuploader.js"></script>

    <style>

        #upload-container {
            width: 500px;
            height: 500px;
            background: lightskyblue;
            padding-bottom: 10px;
        }
    </style>


</head>
<body>
<div id="upload-container">
    <span>文件拖拽上传</span>
</div>
<div id="upload-list"></div>


<button id="picker" style="margin-top: 50px">点击上传</button>
</body>
<script>
    $('#upload-container').click(function (event) {
        $("#picker").find('input').click();
    });
    //初始化上传组件
    const uploader = WebUploader.create({
        auto: true,
        swf: 'Uploader.swf', //swf文件路径
        server: 'http://localhost:8080/upload', //上传接口
        dnd: '#upload-container',
        pick: '#picker',  //内部根据当前运行创建
        multiple: true,     //选择多个
        chunked: true,      //开启分片
        threads: 20,        //并发数
        method: 'POST',
        fileSizeLimit: 1024 * 1024 * 1024 * 100, // 文件总大小为100G
        fileSingleSizeLimit: 1024 * 1024 * 1024 * 5,  //单个文件大小最大为1G
        fileVal: 'upload'
    });
    //入队之前触发事件
    uploader.on("beforeFileQueued", function (file) {
        console.log(file); //获取文件后缀
    });
    //入队触发事件
    uploader.on('fileQueued', function (file) {
        //选中文件要做的事
        console.log(file.ext);
        console.log(file.size);
        console.log(file.name);
        const html = '<div class="upload-item"><span>文件名:' + file.name + '</span><span data-file_id="' + file.id + '"class="btn-delete">删除</span><span data-file_id="' + file.id + '"class="btn-retry">重试</span><div class="percentage ' + file.id + '" style="width: 0%;"></div></div>';
        $('#upload-list').append(html);
        uploader.md5File(file)  //给文件定义唯一的md5值,当再次上传相同文件时,就不用传了  大文件秒传实际上是没传,直接拷贝之前文件地址
            //显示进度
            .progress(function (percentage) {
                console.log('Percentage:', percentage);
            })
            //完成
            .then(function (val) {
                console.log('md5 result', val);
            });
    });
</script>
</html>

dto/FileInfoDto

package com.file.dto;

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;

/**
 * @author: pingyao.ming
 * @Date: 2022/12/08 18:00
 **/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class FileInfoDto {

    private long fileSize;

    private String fileName;

}

dto/UploadFileInfoDto

package com.file.dto;

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;

/**
 * @author: pingyao.ming
 * @Date: 2022/12/08 18:30
 **/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class UploadFileInfoDto {

    // 文件名称
    private String fileName;

    // 当前分片
    private Integer currentChunk;

    // 总分片
    private Integer chunks;

}

dto/DownloadFileInfoDto

package com.file.dto;

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;

/**
 * @author: pingyao.ming
 * @Date: 2022/12/08 18:33
 **/
@Data
@AllArgsConstructor
@NoArgsConstructor
public class DownloadFileInfoDto{

    // 文件总大小
    private long fSize;

    // 断点起始位置
    private long pos;

    // 断点结束位置
    private long last;

    /**
     * rang响应
     */

    private long rangeLength;

    private String contentRange;

}

controller/FileController

package com.file.controller;

import com.file.service.FileService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RestController;

import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;

/**
 * @author: pingyao.ming
 * @Date: 2022/12/08 18:38
 **/
@RestController
public class FileController {

    @Autowired
    private FileService fileService;

    /**
     * 单文件上传(支持断点续传)
     *
     * @param request
     * @param response
     */
    @PostMapping("/upload")
    public void upload(HttpServletRequest request, HttpServletResponse response) throws Exception {
        fileService.upload(request, response);
    }

    /**
     * 文件下载
     * @param request
     * @param response
     */
    @GetMapping("/download")
    public void download(HttpServletRequest request, HttpServletResponse response) throws IOException {
        fileService.download(request, response);
    }

    /**
     * 分片下载
     */
    @GetMapping("/shardingDownload")
    public void shardingDownload() throws Exception {
        fileService.shardingDownload();
    }

}

service/FileService

package com.file.service;

import com.file.dto.DownloadFileInfoDto;
import com.file.dto.FileInfoDto;
import com.file.dto.UploadFileInfoDto;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.fileupload.FileItem;
import org.apache.commons.fileupload.disk.DiskFileItemFactory;
import org.apache.commons.fileupload.servlet.ServletFileUpload;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClients;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.*;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 * @author: pingyao.ming
 * @Date: 2022/12/08 18:38
 **/
@Slf4j
@Service
public class FileService {

    // 分片下载每一片大小为5M
    private static final Long PER_SHARDING = 1024 * 1024 * 5L;

    private static final String RANGE = "Range";

    // 编码
    private static final String UTF_8 = "utf-8";

    // 上传文件路径
    @Value("${file.upload-path}")
    private String uploadPath;

    // 下载指定文件
    @Value("${file.download-file}")
    private String downloadFile;

    // 下载地址
    @Value("${file.download-path}")
    private String downloadPath;

    // 定义分片下载线程池
    private ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());


    public void shardingDownload() throws Exception {
        // 探测下载 获取文件相关信息
        FileInfoDto fileInfoDto = shardingDownload(1, 10, -1, null);
        // 如果不为空,执行分片下载
        if (fileInfoDto != null) {
            // 计算有多少分片
            long pages = fileInfoDto.getFileSize() / PER_SHARDING;
            // 适配最后一个分片
            for (long i = 0; i <= pages; i++) {
                long start = i * PER_SHARDING;
                long end = (i + 1) * PER_SHARDING - 1;
                executorService.execute(new ShardingDownloadRunnable(start, end, i, fileInfoDto.getFileName()));
            }
        }
    }

    private FileInfoDto shardingDownload(long start, long end, long page, String fName) throws Exception {
        // 断点下载
        File file = new File(downloadPath, page + "-" + fName);
        // 如果当前文件已经存在,并且不是探测任务,并且文件的长度等于分片的大小,不用下载当前文件
        if (file.exists() && page != -1 && file.length() == PER_SHARDING) {
            return null;
        }
        // 创建HttpClient
        HttpClient httpClient = HttpClients.createDefault();
        HttpGet httpGet = new HttpGet("http://localhost:8080/download");
        httpGet.setHeader(RANGE, "bytes=" + start + "-" + end);
        HttpResponse httpResponse = httpClient.execute(httpGet);
        String fSize = httpResponse.getFirstHeader("fSize").getValue();
        fName = URLDecoder.decode(httpResponse.getFirstHeader("fName").getValue(), UTF_8);
        HttpEntity httpEntity = httpResponse.getEntity();
        // 下载
        try (InputStream is = httpEntity.getContent(); FileOutputStream fos = new FileOutputStream(file)) {
            byte[] buffer = new byte[1024];
            int ch;
            while ((ch = is.read(buffer)) != -1) {
                fos.write(buffer, 0, ch);
            }
            fos.close();
        }
        // 判断是否是最后一个分片,如果是那么合并
        if (end - Long.parseLong(fSize) > 0) {
            mergeFile(fName, page);
        }
        return new FileInfoDto(Long.parseLong(fSize), fName);
    }

    private void mergeFile(String fName, long page) throws Exception {
        File file = new File(downloadPath, fName);
        try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(file))) {
            for (int i = 0; i <= page; i++) {
                File tempFile = new File(downloadPath, i + "-" + fName);
                // 文件不存在或文件还没写完
                while (!tempFile.exists() || (i != page && tempFile.length() < PER_SHARDING)) {
                    Thread.sleep(100);
                }
                byte [] bytes = FileUtils.readFileToByteArray(tempFile);
                bos.write(bytes);
                bos.flush();
                tempFile.delete();
            }
            // 删除探测文件
            File f = new File(downloadPath,"-1"+"-null");
            if (f.exists()) {
                f.delete();
            }
        }
    }

    public void download(HttpServletRequest request, HttpServletResponse response) throws IOException {
        // 获取文件
        File file = new File(downloadFile);
        // 获取下载文件信息
        DownloadFileInfoDto downloadFileInfoDto = getDownloadFileInfoDto(file.length(), request, response);
        // 设置响应头
        setResponse(response, file.getName(), downloadFileInfoDto);
        // 下载文件
        try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file))) {
            OutputStream os = new BufferedOutputStream(response.getOutputStream());
            // 跳过已经读取的文件
            bis.skip(downloadFileInfoDto.getPos());
            byte[] buffer = new byte[1024];
            long sum = 0;
            // 读取
            while (sum < downloadFileInfoDto.getRangeLength()) {
                int length = bis.read(buffer, 0, (downloadFileInfoDto.getRangeLength() - sum) <= buffer.length
                        ? (int) (downloadFileInfoDto.getRangeLength() - sum) : buffer.length);
                sum += length;
                os.write(buffer, 0, length);
            }
        }
    }

    private void setResponse(HttpServletResponse response, String fileName, DownloadFileInfoDto downloadFileInfoDto) throws UnsupportedEncodingException {
        response.setCharacterEncoding(UTF_8);
        response.setContentType("application/x-download");
        response.addHeader("Content-Disposition", "attachment;filename=" + URLEncoder.encode(fileName, UTF_8));
        // 支持分片下载
        response.setHeader("Accept-Range", "bytes");
        response.setHeader("fSize", String.valueOf(downloadFileInfoDto.getFSize()));
        response.setHeader("fName", URLEncoder.encode(fileName, UTF_8));
        // range响应头
        response.setHeader("Content-Range", downloadFileInfoDto.getContentRange());
        response.setHeader("Content-Length", String.valueOf(downloadFileInfoDto.getRangeLength()));
    }

    private DownloadFileInfoDto getDownloadFileInfoDto(long fSize, HttpServletRequest request, HttpServletResponse response) {
        long pos = 0;
        long last = fSize - 1;
        //前端需要分片下载
        if (request.getHeader(RANGE) != null) {
            response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
            String numRange = request.getHeader(RANGE).replace("bytes=", "");
            String[] strRange = numRange.split("-");
            if (strRange.length == 2) {
                pos = Long.parseLong(strRange[0].trim());
                last = Long.parseLong(strRange[1].trim());
                //若结束字节超出文件大小 取文件大小
                if (last > fSize - 1) {
                    last = fSize - 1;
                }
            } else {
                //若只给一个长度  开始位置一直到结束
                pos = Long.parseLong(numRange.replace("-", "").trim());
            }
        }
        long rangeLength = last - pos + 1;
        String contentRange = "bytes " + pos + "-" + last + "/" + fSize;
        return new DownloadFileInfoDto(fSize, pos, last, rangeLength, contentRange);
    }

    /**
     * 上传文件
     *
     * @param request
     * @param response
     */
    public void upload(HttpServletRequest request, HttpServletResponse response) throws Exception {
        // 获取ServletFileUpload
        ServletFileUpload servletFileUpload = getServletFileUpload();
        List<FileItem> items = servletFileUpload.parseRequest(request);
        // 获取文件信息
        UploadFileInfoDto uploadFileInfoDto = getUploadFileInfoDto(items);
        // 写临时文件
        writeTempFile(items, uploadFileInfoDto);
        // 判断是否合并
        mergeFile(uploadFileInfoDto);
        // 返回结果
        response.setCharacterEncoding(UTF_8);
        response.getWriter().write("Upload succeeded");
    }

    private ServletFileUpload getServletFileUpload() {
        // 设置缓冲区大小 先读取到内存 后从内存写
        DiskFileItemFactory factory = new DiskFileItemFactory();
        factory.setSizeThreshold(1024);
        factory.setRepository(new File(uploadPath));
        // 解析
        ServletFileUpload upload = new ServletFileUpload(factory);
        // 设置单个大小
        upload.setFileSizeMax(5 * 1024 * 1024 * 1024L);
        // 设置最大大小
        upload.setSizeMax(100 * 1024 * 1024 * 1024L);
        return upload;
    }

    private UploadFileInfoDto getUploadFileInfoDto(List<FileItem> items) throws UnsupportedEncodingException {
        UploadFileInfoDto uploadFileInfoDto = new UploadFileInfoDto();
        for (FileItem item : items) {
            if (item.isFormField()) {
                // 获取分片数赋值给变量
                if ("chunk".equals(item.getFieldName())) {
                    uploadFileInfoDto.setCurrentChunk(Integer.parseInt(item.getString(UTF_8)));
                }
                if ("chunks".equals(item.getFieldName())) {
                    uploadFileInfoDto.setChunks(Integer.parseInt(item.getString(UTF_8)));
                }
                if ("name".equals(item.getFieldName())) {
                    uploadFileInfoDto.setFileName(item.getString(UTF_8));
                }
            }
        }
        return uploadFileInfoDto;
    }

    private void writeTempFile(List<FileItem> items, UploadFileInfoDto uploadFileInfoDto) throws Exception {
        // 获取文件基本信息后
        for (FileItem item : items) {
            // 判断某项是否是普通的表单类型
            if (!item.isFormField()) {
                String tempFileName = uploadFileInfoDto.getFileName();
                if (StringUtils.isNotBlank(tempFileName)) {
                    if (uploadFileInfoDto.getCurrentChunk() != null) {
                        tempFileName = uploadFileInfoDto.getCurrentChunk() + "_" + uploadFileInfoDto.getFileName();
                    }
                    // 判断文件是否存在
                    File tempFile = new File(uploadPath, tempFileName);
                    // 断点续传 判断文件是否存在 若存在则不传
                    if (!tempFile.exists()) {
                        item.write(tempFile);
                    }
                }
            }
        }
    }

    private void mergeFile(UploadFileInfoDto uploadFileInfoDto) throws IOException, InterruptedException {

        Integer currentChunk = uploadFileInfoDto.getCurrentChunk();
        Integer chunks = uploadFileInfoDto.getChunks();
        // 如果当前分片等于总分片,那么合并文件
        if (currentChunk != null && chunks != null && currentChunk.equals(chunks - 1)) {
            File tempFile = new File(uploadPath, uploadFileInfoDto.getFileName());
            try (BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(tempFile))) {
                for (int i = 0; i < chunks; i++) {
                    File file = new File(uploadPath, i + "_" + uploadFileInfoDto.getFileName());
                    // 并发情况,需要判断所有:因为可能最后一个分片传完,之前有的还没有传完
                    while (!file.exists()) {
                        // 不存在 休眠100毫秒,然后从新判断
                        Thread.sleep(100);
                    }
                    // 分片存在 读入数组中
                    byte[] bytes = FileUtils.readFileToByteArray(file);
                    os.write(bytes);
                    os.flush();
                    file.delete();
                }
                os.flush();
            }
        }
    }

    private class ShardingDownloadRunnable implements Runnable {

        private final long start;
        private final long end;
        private final long page;
        private final String fName;

        private ShardingDownloadRunnable(long start, long end, long page, String fName) {
            this.start = start;
            this.end = end;
            this.page = page;
            this.fName = fName;
        }

        @Override
        public void run() {
            try {
                shardingDownload(start, end, page, fName);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }
}


Application

package com.file;

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;

@SpringBootApplication
public class Application {

	public static void main(String[] args) {
		SpringApplication.run(Application.class, args);
	}

}

评论 1
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值