切片上传思路

前端

使用Blob.prototype.slice方法对文件切片
使用spark-md5库计算文件hash（该库不能直接计算整个文件，需要分片进行计算，具体可以看它官网的例子spark-md5-example。
创建web worker进行计算hash，减少页面卡顿。
将各个分片文件上传到服务器，并携带hash
当所有文件上传完后，需要发送合并请求，携带文件名和分片大小

服务端

根据接收到的hash创建文件夹，将分片文件存储到文件夹中
收到合并请求后，读取各个分片文件，进行合并
删除存储分片文件的文件夹及其内容

秒传思路

客户端

上传文件是的时候，先计算hash，然后将hash和文件名发送到服务器
服务器返回文件是否存在的状态
如果存在，客户端提示文件上传成功，否则执行上传动作

服务端

服务器设置一个hash=文件的映射表
根据客户端发送过来的hash去表中查找
返回存在的状态

web worker踩坑

self.importScript函数只能导入绝对路径的文件，但是看文档说是可以使用相对路径，但测试多次都不行。所以将spark.md5.js文件放在public文件夹中。

微信截图_20220310115642.png

效果图

动画6.gif

客户端

upload.js

import React, { useState, useEffect, useMemo } from "react";
import request from "../utils/request";
import styled from "styled-components";
import hashWorker from "../utils/hash-worker";
import WorkerBuilder from "../utils/worker-build";

const chunkSize = 1 * 1024 * 1024;
const UpLoadFile = function () {
  const [sourceFile, setSourceFile] = useState(null);
  const [chunksData, setChunksData] = useState([])
  const [myWorker, setMyWorker] = useState(null)
  const [fileHash, setFileHash] = useState("")
  const [hashPercentage, setHashPercentage] = useState(0)

  const handleFileChange = (e) => {
    const { files } = e.target;
    if (files.length === 0) return;
    // 保存源文件
    setSourceFile(files[0]);
    // 文件分片
    splitFile(files[0])
  }
  // 发送合并请求
  const mergeRequest = () => {
    request({
      url: "http://localhost:3001/merge",
      method: "post",
      headers: {
        "content-type": "application/json"
      },
      data: JSON.stringify({
        filename: sourceFile.name,
        // 用于服务器合并文件
        size: chunkSize
      })
    })
  }
  // 上传分片
  const uploadChunks = async (chunksData) => {
    const formDataList = chunksData.map(({ chunk, hash }) => {
      const formData = new FormData()
      formData.append("chunk", chunk);
      formData.append("hash", hash);
      formData.append("filename", sourceFile.name);
      return { formData };
    })

    const requestList = formDataList.map(({ formData }, index) => {
      return request({
        url: "http://localhost:3001/upload",
        data: formData,
        onprogress: e => {
          let list = [...chunksData];
          list[index].progress = parseInt(String((e.loaded / e.total) * 100));
          setChunksData(list)
        }
      })
    })
    // 上传文件
    await Promise.all(requestList);
    // 延迟发送合并请求
    setTimeout(() => {
      mergeRequest();
    }, 500);
  }
  // 计算文件hash
  const calculateHash = (chunkList) => {
    return new Promise(resolve => {
      const w = new WorkerBuilder(hashWorker)
      w.postMessage({ chunkList: chunkList })
      w.onmessage = e => {
        const { percentage, hash } = e.data;
        setHashPercentage(percentage);
        if (hash) {
          // 当hash计算完成时，执行resolve
          resolve(hash)
        }
      }
      setMyWorker(w)
    })
  }
  // 上传文件
  const handleUpload = async (e) => {
    if (!sourceFile) {
      alert("请先选择文件")
      return;
    }
    // 拆分文件
    const chunklist = splitFile(sourceFile);
    // 计算hash
    const hash = await calculateHash(chunklist)
    console.log("hash=======", hash)
    setFileHash(hash)
    const { shouldUpload } = await verfileIsExist(sourceFile.name, hash);
    if (!shouldUpload) {
      alert("文件已存在，无需重复上传");
      return;
    }
    const chunksData = chunklist.map(({ chunk }, index) => ({
      chunk: chunk,
      hash: hash + "-" + index,
      progress: 0
    }))
    // 保存分片数据
    setChunksData(chunksData)
    // 开始上传分片
    uploadChunks(chunksData)
  }

  // 拆分文件
  const splitFile = (file, size = chunkSize) => {
    const fileChunkList = [];
    let curChunkIndex = 0;
    while (curChunkIndex <= file.size) {
      const chunk = file.slice(curChunkIndex, curChunkIndex + size);
      fileChunkList.push({ chunk: chunk, })
      curChunkIndex += size;
    }
    return fileChunkList;
  }
  // 秒传：验证文件是否存在服务器
  const verfileIsExist = async (fileName, fileHash) => {
    const { data } = await request({
      url: "http://localhost:3001/verFileIsExist",
      headers: {
        "content-type": "application/json"
      },
      data: JSON.stringify({
        fileName: fileName,
        fileHash: fileHash
      })
    })
    return JSON.parse(data);
  }

  return (
    <div>
      <input type="file" onChange={handleFileChange} /><br />
      <button onClick={handleUpload}>上传</button>
      <ProgressBox chunkList={chunksData} />
    </div>
  )
}
const BlockWraper = styled.div`
  width: ${({ size }) => size + "px"};
  height: ${({ size }) => size + "px"};
  text-align: center;
  font-size: 12px;
  line-height: ${({ size }) => size + "px"}; 
  border: 1px solid #ccc;
  position: relative;
  float: left;
  &:before {
    content: "${({ chunkIndex }) => chunkIndex}";
    position: absolute;
    width: 100%;
    height: 10px;
    left: 0;
    top: 0;
    font-size: 12px;
    text-align: left;
    line-height: initial;
    color: #000
  }
  &:after {
    content: "";
    position: absolute;
    width: 100%;
    height: ${({ progress }) => progress + "%"};
    background-color: pink;
    left: 0;
    top: 0;
    z-index: -1;
  }
`
const ChunksProgress = styled.div`
  *zoom: 1;
  &:after {
    content: "";
    display: block;
    clear: both;
  }
`
const Label = styled.h3``
const ProgressWraper = styled.div``
const Block = ({ progress, size, chunkIndex }) => {
  return (<BlockWraper size={size} chunkIndex={chunkIndex} progress={progress}>
    {progress}%
  </BlockWraper>)
}

const ProgressBox = ({ chunkList = [], size = 40 }) => {
  const sumProgress = useMemo(() => {
    if (chunkList.length === 0) return 0
    return chunkList.reduce((pre, cur, sum) => pre + cur.progress / 100, 0) * 100 / (chunkList.length)
  }, [chunkList])

  return (
    <ProgressWraper>
      <Label>文件切分为{chunkList.length}段，每段上传进度如下：</Label>
      <ChunksProgress>
        {chunkList.map(({ progress }, index) => (
          <Block key={index} size={size} chunkIndex={index} progress={progress} />
        ))}
      </ChunksProgress>
      <Label>总进度:{sumProgress.toFixed(2)}%</Label>
    </ProgressWraper >
  )
}

export default UpLoadFile;
复制代码

hash-worker.js

const hashWorker = () => {
  self.importScripts("http://localhost:3000/spark-md5.min.js")
  self.onmessage = (e) => {
    const { chunkList } = e.data;
    const spark = new self.SparkMD5.ArrayBuffer();
    let percentage = 0;
    let count = 0;
    const loadNext = index => {
      const reader = new FileReader();
      reader.readAsArrayBuffer(chunkList[index].chunk);
      reader.onload = event => {
        count++;
        spark.append(event.target.result);
        if (count === chunkList.length) {
          self.postMessage({
            percentage: 100,
            hash: spark.end()
          })
          self.close();
        } else {
          percentage += (100 / chunkList.length)
          self.postMessage({
            percentage
          })
          loadNext(count)
        }
      }
    }
    loadNext(count)
  }
}

export default hashWorker
复制代码

worker-build.js

export default class WorkerBuilder extends Worker {
  constructor(worker) {
    const code = worker.toString();
    const blob = new Blob([`(${code})()`]);
    return new Worker(URL.createObjectURL(blob));
  }
}
复制代码

request.js

const request = ({
  url,
  method = "post",
  data,
  headers = {},
  onprogress
}) => {
  return new Promise(resolve => {
    const xhr = new XMLHttpRequest();
    xhr.open(method, url);
    Object.keys(headers).forEach(key =>
      xhr.setRequestHeader(key, headers[key])
    );
    xhr.upload.onprogress = onprogress
    xhr.send(data);
    xhr.onload = e => {
      resolve({
        data: e.target.response
      });
    };
  });
}

export default request;
复制代码

服务端

import express from 'express'
import path from "path";
import fse from "fs-extra";
import multiparty from "multiparty";
import bodyParser from "body-parser"
let app = express()
const __dirname = path.resolve(path.dirname(''));
const upload_files_dir = path.resolve(__dirname, "./filelist")

const jsonParser = bodyParser.json({ extended: false });

app.use(function (req, res, next) {
  res.setHeader("Access-Control-Allow-Origin", "*");
  res.setHeader("Access-Control-Allow-Headers", "*");
  next()
})

app.post('/verFileIsExist', jsonParser, async (req, res) => {
  const { fileName, fileHash } = req.body;
  const filePath = path.resolve(upload_files_dir, fileName);
  if (fse.existsSync(filePath)) {
    res.send({
      code: 200,
      shouldUpload: false
    })
  } else {
    res.send({
      code: 200,
      shouldUpload: true
    })
  }
})

app.post('/upload', async (req, res) => {
  const multipart = new multiparty.Form();
  multipart.parse(req, async (err, fields, files) => {
    if (err) return;
    const [chunk] = files.chunk;
    const [hash] = fields.hash;
    const [filename] = fields.filename;
    // 加上“_dir”，是为了防止合并的时候，文件名与文件夹名发生冲突
    const chunkDir = path.resolve(upload_files_dir, filename + "_dir");
    if (!fse.existsSync(chunkDir)) {
      await fse.mkdirs(chunkDir);
    }
    await fse.move(chunk.path, `${chunkDir}/${hash}`);
  })
  res.status(200).send("received file chunk")
})

const pipeStream = (path, writeStream) =>
  new Promise(resolve => {
    const readStream = fse.createReadStream(path);
    readStream.on("end", () => {
      fse.unlinkSync(path);
      resolve();
    });
    readStream.pipe(writeStream);
  });

// 合并切片
const mergeFileChunk = async (filePath, filename, size) => {
  const chunkDir = path.resolve(upload_files_dir, filename + "_dir");
  const chunkPaths = await fse.readdir(chunkDir);
  // 根据切片下标进行排序
  // 否则直接读取目录的获得的顺序可能会错乱
  chunkPaths.sort((a, b) => a.split("-")[1] - b.split("-")[1]);
  console.log("指定位置创建可写流", filePath);
  await Promise.all(
    chunkPaths.map((chunkPath, index) =>
      pipeStream(
        path.resolve(chunkDir, chunkPath),
        // 指定位置创建可写流
        fse.createWriteStream(filePath, {
          start: index * size,
          end: (index + 1) * size
        })
      )
    )
  );
  fse.rmdirSync(chunkDir); // 合并后删除保存切片的目录
};

app.post('/merge', jsonParser, async (req, res) => {
  const { filename, size } = req.body;
  const filePath = path.resolve(upload_files_dir, filename);
  await mergeFileChunk(filePath, filename, size);
  res.send({
    code: 200,
    message: "success"
  });
})

app.listen(3001, () => {
  console.log('listen:3001')
})
复制代码