#4960 fix-4953 数据集上传分片交互优化

Merged
chenshihai merged 9 commits from fix-4953 into V20231211 5 months ago
  1. +98
    -74
      web_src/js/components/MinioUploader.vue
  2. +1
    -1
      web_src/vuepages/pages/modelmanage/fileupload/index.vue

+ 98
- 74
web_src/js/components/MinioUploader.vue View File

@@ -217,13 +217,6 @@ export default {
this.uploadProgressList = [];
},
updateProgress(file, status, progress, infoCode, failedInfo = "") {
console.log("this.uploadProgressList:", this.uploadProgressList);
// file.previewTemplate.querySelector(
// ".dz-upload"
// ).style.width = `${progress}%`;
// file.previewTemplate.querySelector(".dz-upload").style.background =
// "#409eff";
// file.previewTemplate.querySelector(".dz-upload").style.opacity = 0;
this.uploadProgressList.forEach((item, index) => {
if (item.name === file.name) {
this.uploadProgressList[index].status = status;
@@ -265,7 +258,6 @@ export default {
// this.dropzoneUploader.emit('complete', file);
},
finishUpload(file) {
console.log("finish", file);
file.previewTemplate.querySelector(".dz-success-mark").style.opacity = 1;
file.previewTemplate.querySelector(".dz-progress").style.opacity = 0;
if (this.uploadLength === this.uploadFiles.length) {
@@ -314,12 +306,6 @@ export default {
(currentChunk / chunks) *
100
).toFixed(2)}% (${currentChunk}/${chunks})`;
console.log("-=-=-=-=--=", this.status);
// this.updateProgress(
// file,
// this.dropzoneParams.data("md5-computing"),
// Number(((currentChunk / chunks) * 100).toFixed(1))
// );
loadMd5Next();
return;
}
@@ -330,11 +316,6 @@ export default {
file.size
} 用时:${(new Date().getTime() - time) / 1000} s`
);
// this.updateProgress(
// file,
// this.dropzoneParams.data("md5-computing"),
// 100
// );
spark.destroy(); // 释放缓存
file.uniqueIdentifier = md5; // 将文件md5赋值给文件唯一标识
file.cmd5 = false; // 取消计算md5状态
@@ -358,7 +339,6 @@ export default {

async computeMD5Success(md5edFile) {
const file = await this.getSuccessChunks(md5edFile);
console.log(file);
try {
if (file.uploadID == "" || file.uuid == "") {
// 未上传过
@@ -485,7 +465,7 @@ export default {
let currentChunk = 0;
let _this = this;

function loadNext() {
function loadNext() {
const start = currentChunk * chunkSize;
const end =
start + chunkSize >= file.size ? file.size : start + chunkSize;
@@ -534,7 +514,6 @@ export default {
xhr.setRequestHeader("Content-Type", "");
xhr.send(e.target.result);
var etagValue = xhr.getResponseHeader("ETag");
//console.log(etagValue);
etags[currentChunk] = etagValue;
}
}
@@ -550,7 +529,7 @@ export default {
})
);
}
async function uploadChunk(e) {
async function uploadChunk(e,_this) {
try {
if (!checkSuccessChunks()) {
const start = currentChunk * chunkSize;
@@ -562,10 +541,16 @@ export default {
// 上传到minio
//await uploadMinio(urls[currentChunk], e);
await uploadMinioNewMethod(urls[currentChunk], e);

if (etags[currentChunk] != "") {
// 更新数据库:分片上传结果
//await updateChunk(currentChunk);
succesLength++
_this.updateProgress(
file,
_this.dropzoneParams.data("uploading"),
Number(((succesLength / chunks) * 100).toFixed(2)),
3
);
} else {
console.log(
"上传到minio uploadChunk etags[currentChunk] == ''"
@@ -577,6 +562,8 @@ export default {
}
} catch (error) {
// this.emitDropzoneFailed(file);
//stopFlag = true
errorChunkLength++
console.log(error);
}
}
@@ -596,56 +583,22 @@ export default {
})
);
}

const successChunks = [];
let successParts = [];
successParts = file.chunks.split(",");
for (let i = 0; i < successParts.length; i++) {
successChunks[i] = successParts[i].split("-")[0];
}
const urls = []; // TODO const ?
const etags = [];
console.log("上传分片...");
this.status = this.dropzoneParams.data("uploading");
loadNext();
fileReader.onload = async (e) => {
try {
await uploadChunk(e);
} catch (err) {
console.log(err);
}

fileReader.abort();
currentChunk++;
if (currentChunk < chunks) {
console.log(
`第${currentChunk}个分片上传完成, 开始第${
currentChunk + 1
}/${chunks}个分片上传`
);
this.progress = Math.ceil((currentChunk / chunks) * 100);
this.updateProgress(
file,
this.dropzoneParams.data("uploading"),
Number(((currentChunk / chunks) * 100).toFixed(2)),
3
);
this.status = `${this.dropzoneParams.data("uploading")} ${(
(currentChunk / chunks) *
100
).toFixed(2)}%`;
await loadNext();
} else {
async function checkCompleteUpload() {
if(succesLength===chunks) {
try {
await completeUpload();
} catch (err) {
let info = "上传失败";
this.allUploadLength++;
this.uploadError(file, info);
this.updateProgress(
_this.allUploadLength++;
_this.uploadError(file, info);
let completeProgress=Number(((succesLength / chunks) * 100).toFixed(2))
if(completeProgress==100){
completeProgress = completeProgress -1
}
_this.updateProgress(
file,
info,
Number(((currentChunk / chunks) * 100).toFixed(2)) - 1,
completeProgress,
2
);
// this.allUploadFiles.push({
@@ -663,18 +616,89 @@ export default {
file.size
} 用时:${(new Date().getTime() - time) / 1000} s`
);
this.uploadLength++;
this.allUploadLength++;
_this.uploadLength++;
_this.allUploadLength++;
// this.allUploadFiles.push({
// name: file.name,
// status: 0,
// info: "上传成功",
// });

this.updateProgress(file, "上传成功", 100, 0);
this.progress = 100;
this.status = this.dropzoneParams.data("upload-complete");
this.finishUpload(file);
_this.updateProgress(file, "上传成功", 100, 0);
// this.progress = 100;
// this.status = this.dropzoneParams.data("upload-complete");
_this.finishUpload(file);
}
}
const successChunks = [];
let successParts = [];
successParts = file.chunks.split(",");
for (let i = 0; i < successParts.length; i++) {
successChunks[i] = successParts[i].split("-")[0];
}
if(successChunks.length>1){
this.updateProgress(
file,
this.dropzoneParams.data("uploading"),
Number((((successChunks.length-1) / chunks) * 100).toFixed(2)),
3
);
}
let succesLength = successChunks.length-1
let errorChunkLength = 0
const urls = []; // TODO const ?
const etags = [];
console.log("上传分片...");
this.status = this.dropzoneParams.data("uploading");
for (currentChunk; currentChunk < chunks; currentChunk++) {
if(!checkSuccessChunks()){
loadNext();
break
}
}
checkCompleteUpload()
fileReader.onload = async (e) => {
try {
await uploadChunk(e,_this);
} catch (err) {
console.log(err);
}
fileReader.abort();
currentChunk++;
if(succesLength<chunks && errorChunkLength>20){
this.updateProgress(
file,
"上传失败",
Number(((succesLength / chunks) * 100).toFixed(2)),
2
);
return
}
if (currentChunk < chunks) {
console.log(
`第${currentChunk}个分片上传完成, 开始第${
currentChunk + 1
}/${chunks}个分片上传`
);
for (currentChunk; currentChunk < chunks; currentChunk++) {
if(!checkSuccessChunks()){
loadNext();
break
}
}
}
if(succesLength<chunks && currentChunk === chunks){
this.updateProgress(
file,
"上传失败",
Number(((succesLength / chunks) * 100).toFixed(2)),
2
);
}
if(succesLength===chunks){
checkCompleteUpload()
}
};
},


+ 1
- 1
web_src/vuepages/pages/modelmanage/fileupload/index.vue View File

@@ -99,7 +99,7 @@ Dropzone.autoDiscover = false;
const uploadChunkSize = 1024 * 1024 * 64;
const md5ChunkSize = 1024 * 1024 * 64;
const maxFileSize = 10;
const maxModelFilesSize = window.MAX_MODEL_SIZE || 200 * 1024 * 1024 * 1024; // 200 GB
const maxModelFilesSize = window.MAX_MODEL_SIZE || 250 * 1024 * 1024 * 1024; // 200 GB

export default {
data() {


Loading…
Cancel
Save