大文件切片上传

  1. 拿到文件对象
  2. 使用线程切割文件
  3. 拿到切片数组循环上传
  4. 上传完毕后通知后端合并文件

后端服务

import cors from "cors";
import multer from "multer";
import express from "express";
import fs from "node:fs";
import path from "node:path";
/**
 * 大文件上传
 * 1. 客户端切片上传文件(给每个文件打上hash)
 * 2. 服务端接收切片, 切片合并
 * 3. 切片上传失败, 切片重传
 * 4. 切片上传成功, 通知服务端合并切片
 * 5. 合并切片成功, 通知客户端上传完成
 */

//初始化multer
const storage = multer.diskStorage({
  //文件保存路径
  destination: function (req, file, cb) {
    cb(null, path.resolve(__dirname, "./test/uploads/"));
  },
  //文件重命名
  filename: function (req, file, cb) {
    cb(null, Date.now() + "-" + req.body.fileName);
  },
});
//初始化multer
const upload = multer({ storage: storage });
const app = express();
app.use(cors());
app.use(express.json());
//接收客户端上传的切片 chunk要与前端文件对象key一致
app.post("/upload", upload.single("chunk"), (req, res) => {
  res.send({ ok: true });
});
//合并切片
app.post("/merge", async (req, res) => {
  //读取切片内容
  const uploadDir = path.join(__dirname, "./test/uploads/");
  //读取目录下的文件
  let dirs = fs.readdirSync(uploadDir);
  //根据时间戳排序
  dirs.sort((a, b) => a.localeCompare(b, "zh-CN"));
  //合并目标路径
  const video = path.join(
    __dirname,
    "./test/video",
    `${req.query.fileName}.mp4`
  );
  //合并切片
  dirs.forEach((item) => {
    //读取文件内容
    const data = fs.readFileSync(path.join(uploadDir, item));
    //写入文件(合并)
    fs.appendFileSync(video, data);
    //删除切片文件
    fs.unlinkSync(path.join(uploadDir, item));
  });
  res.send({ ok: true });
});
app.listen(8080, () => {
  console.log("server is running at http://localhost:8080");
});

前端实现

<!DOCTYPE html>
<html lang="en">
  <head>
    <meta charset="UTF-8" />
    <title>大文件分片</title>
  </head>
  <body>
    <input type="file" class="input" />
    <script src="./main.js" type="module"></script>
  </body>
</html>

main.js

import { cutFile } from "./cutFile.js";
//获取文件
const fileDom = document.querySelector(".input");

fileDom.onchange = async (e) => {
  const file = e.target.files;
  const chunks = await cutFile(file[0]);
  const list = [];
  //上传分片
  chunks.forEach((chunk) => {
    const formData = new FormData();
    formData.append("hash", chunk.spark);
    formData.append("index", chunk.index);
    formData.append("chunk", chunk.blob);
    //上传
    list.push(
      fetch("http://localhost:8080/upload", {
        method: "POST",
        body: formData,
      })
    );
  });
  //通知后端合并
  Promise.all(list).then(() => {
    fetch("http://127.0.0.1:8080/merge?fileName=李世民", {
      method: "POST",
    });
  });
};

cutFile.js

const FILE_SIZE = 1024 * 1024 * 5; //文件大小限制
const CUP_NUM = navigator.hardwareConcurrency || 4; //线程数量
export function cutFile(file) {
  return new Promise((resolve) => {
    const fileCount = Math.ceil(file.size / FILE_SIZE); // 文件分片数量
    const taskNum = Math.ceil(fileCount / CUP_NUM); //每个线程分配任务数量
    let doneNum = 0; //完成线程数
    const result = []; //存放结果
    //创建一个线程并分配任务
    for (let i = 0, len = CUP_NUM; i < len; i++) {
      const start = i * taskNum; //每个线程处理任务数
      let end = (i + 1) * taskNum; //下一个线程处理任务数
      if (end > fileCount) end = fileCount;
      //创建线程
      const worker = new Worker("./worker.js", {
        type: "module",
      });
      //给线程传递数据
      worker.postMessage({
        file,
        fileCount,
        start,
        end,
      });
      //监听线程返回结果
      worker.onmessage = (e) => {
        for (let i = start, len = end; i < len; i++) {
          result[i] = e.data[i - start];
        }
        worker.terminate(); //销毁线程
        doneNum++;
        //判断是否所有线程执行完毕
        doneNum === CUP_NUM ? resolve(result) : "";
      };
    }
  });
}

worker.js

import { createChunk } from "./createChunk.js";
onmessage = async (e) => {
  const { file, fileCount, start, end } = e.data;
  const proms = [];
  for (let i = start, len = end; i < len; i++) {
    const chunk = await createChunk(file, i, fileCount);
    proms.push(chunk);
  }
  const chunks = await Promise.all(proms);
  postMessage(chunks);
};

createChunk.js
由于我使用spark-md5来进行唯一标识的时候，浏览器网络控制台无法返回期望信息error，也不给我报错，单纯引入这个库都没使用都会返回无法加载。可是我在vue又能用?希望有大佬解答下。所以我使用随机数来模拟唯一标识。

/**
 * 创建分片
 * @param file {File} 文件对象
 * @param index {number} 当前分片索引
 * @param chunkSize  {number} 分片大小
 */
export async function createChunk(file, index, chunkSize) {
  return new Promise((resolve) => {
    const start = index * chunkSize; //当前分片起始位置
    const end = start + chunkSize; //当前分片结束位置
    const spark = Math.random() * 1000; //随机数 标识是否为同一分段
    const blob = file.slice(start, end); //总分片 服务器需要的字段
    const reader = new FileReader(); //文件对象
    //文件完成加载
    reader.onload = (e) => {
      resolve({
        start,
        end,
        index,
        blob,
        spark,
      });
    };
    reader.readAsArrayBuffer(blob);
  });
}