chunked fileupload

Author: sparkoo

pkg/list/list.go

@@ -10,7 +10,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"strings"
+	"regexp"
 
 	"cloud.google.com/go/storage"
 	"go.uber.org/zap"
@@ -68,16 +68,17 @@ func (s *ListService) UploadMatch(w http.ResponseWriter, r *http.Request) {
 	// r.ParseMultipartForm(100 << 20)
 	file, handler, err := r.FormFile("demoFile")
 	if err != nil {
-		// fmt.Println("Error Retrieving the File")
-		fmt.Println(err)
+		http.Error(w, fmt.Sprintf("Error Retrieving the File [%s]", err.Error()), http.StatusBadRequest)
+		log.L().Error("failed upload file request", zap.Error(err))
 		return
 	}
 	defer file.Close()
-	if !strings.HasSuffix(handler.Filename, ".dem.gz") {
+	log.L().Info("uploading file", zap.String("filename", handler.Filename), zap.Int64("filesize", handler.Size), zap.Any("mime", handler.Header))
+	sampleRegex := regexp.MustCompile(`.dem.gz_[0-9]$`)
+	if !sampleRegex.Match([]byte(handler.Filename)) {
 		http.Error(w, fmt.Sprintf("unexpected file type [%s]", handler.Filename), http.StatusBadRequest)
 		return
 	}
-	log.L().Info("uploading file", zap.String("filename", handler.Filename), zap.Int64("filesize", handler.Size), zap.Any("mime", handler.Header))
 	objHandle := s.gcpBucket.Object(handler.Filename)
 	log.L().Info("1")
 	objWriter := objHandle.NewWriter(r.Context())

web/index/src/Uploader/Uploader.js

@@ -8,41 +8,37 @@ const Uploader = (props) => {
     console.log("file upload", event)
   }
 
+  const onProgress = (event) => {
+    console.log("progress", event)
+  }
+
   const uploadHandler = function ({ files }) {
-    const [file] = files;
-    let formData = new FormData();
-    formData.append('demoFile', file);
 
-    fetch(serverHost + "/match/upload",
-      {
-        method: 'POST',
-        body: formData
-      },
-    ).catch(err => {
-      console.log("failed to upload")
-    })
+    const uploadChunk = function (chunk, filename) {
+      console.log("uploading chunk", filename)
 
-    // const fileReader = new FileReader();
-    // fileReader.onload = (e) => {
-    //   uploadDemo(e.target.result);
-    // };
-    // fileReader.readAsDataURL(file);
-  }
+      let formData = new FormData();
+      formData.append("demoFile", chunk, filename);
 
-  const uploadDemo = async (demoFile) => {
-    let formData = new FormData();
-    formData.append('demoFile', demoFile);
+      let xhr = new XMLHttpRequest();
+      xhr.upload.addEventListener('progress', onProgress);
+      xhr.onreadystatechange = (xhr, event) => {
+        console.log("onreadystatechange", xhr, event)
+      };
+      xhr.open('POST', serverHost + "/match/upload", true);
+      xhr.withCredentials = props.withCredentials;
+      xhr.send(formData);
+    }
 
-    const response = await fetch(serverHost + "/match/upload",
-      {
-        method: 'POST',
-        body: formData
-      },
-    );
-    console.log(response)
-  };
-  const onProgress = (event) => {
-    console.log("progress", event)
+    const [file] = files;
+    const chunkSize = 1024 * 1024 * 30; // 30MB
+
+    let start = 0;
+    let chunkNo = 0;
+    while (start < file.size) {
+      uploadChunk(file.slice(start, start + chunkSize), file.name + "_" + chunkNo++);
+      start += chunkSize;
+    }
   }
 
   return (
@@ -55,8 +51,8 @@ const Uploader = (props) => {
         maxFileSize={200_000_000}
         onUpload={onUpload}
         onProgress={onProgress}
-        // customUpload={true}
-        // uploadHandler={uploadHandler}
+        customUpload={true}
+        uploadHandler={uploadHandler}
         auto />
     </div>
   )