Merge pull request #787 from 4nshuman/issue/zip-operation

Issue/zip operation

eslint.config.js

@@ -72,7 +72,7 @@ export default [
                 "iro": true,            // iro.js color picker
                 "$": true,              // jQuery
                 "jQuery": true,         // jQuery
-                "JSZip": true,          // JSZip
+                "fflate": true,         // fflate
                 "_": true,              // lodash
                 "QRCode": true,         // qrcode
                 "io": true,             // socket.io

src/gui/puter-gui.json

@@ -14,7 +14,7 @@
             "/lib/timeago.min.js",
             "/lib/iro.min.js",
             "/lib/isMobile.min.js",
-            "/lib/jszip-3.10.1.min.js"
+            "/lib/fflate-0.8.2.min.js"
         ],
         "css_paths": [
             "/css/normalize.css",

src/gui/src/UI/UIItem.js

@@ -1200,24 +1200,8 @@ function UIItem(options){
                 menu_items.push({
                     html: i18n('unzip'),
                     onClick: async function(){
-                        const zip = new JSZip();
+                        let filePath = $(el_item).attr('data-path');
-                        let filPath = $(el_item).attr('data-path');
+                        window.unzipItem(filePath)
-                        let file = puter.fs.read($(el_item).attr('data-path'));
-
-                        zip.loadAsync(file).then(async function (zip) {
-                            const rootdir = await puter.fs.mkdir(path.dirname(filPath) + '/' + path.basename(filPath, '.zip'), {dedupeName: true});
-                            Object.keys(zip.files).forEach(async function (filename) {
-                                if(filename.endsWith('/'))
-                                    await puter.fs.mkdir(rootdir.path +'/' + filename, {createMissingParents: true});
-                                zip.files[filename].async('blob').then(async function (fileData) {
-                                    await puter.fs.write(rootdir.path +'/' + filename, fileData);
-                                }).catch(function (e) {
-                                    // UIAlert(e.message);
-                                })
-                            })
-                        }).catch(function (e) {
-                            // UIAlert(e.message);
-                        })
                     }
                 })
             }

src/gui/src/globals.js

@@ -109,6 +109,8 @@ window.taskbar_height = window.default_taskbar_height;
 window.upload_progress_hide_delay = 500;
 window.active_uploads = {};
 window.copy_progress_hide_delay = 1000;
+window.zip_progress_hide_delay = 2000;
+window.unzip_progress_hide_delay = 2000;
 window.busy_indicator_hide_delay = 600;
 window.global_element_id = 0;
 window.operation_id = 0;
@@ -126,6 +128,17 @@ window.watchItems = [];
 window.appdata_signatures = {};
 window.appCallbackFunctions = [];
 
+// Defines how much weight each operation has in the zipping progress
+window.zippingProgressConfig = {
+    TOTAL: 100
+}
+//Assuming uInt8Array conversion a file takes betwneen 45% to 60% of the total progress
+window.zippingProgressConfig.SEQUENCING = Math.floor(Math.random() * (60 - 45 + 1)) + 45,
+//Assuming zipping up uInt8Arrays takes betwneen 20% to 23% of the total progress
+window.zippingProgressConfig.ZIPPING = Math.floor(Math.random() * (23 - 20 + 1)) + 20,
+//Assuming writing a zip file takes betwneen 10% to 14% of the total progress
+window.zippingProgressConfig.WRITING = Math.floor(Math.random() * (14 - 10 + 1)) + 14,
+
 // 'Launch' apps
 window.launch_apps = [];
 window.launch_apps.recent = []

src/gui/src/helpers.js

@@ -1959,9 +1959,34 @@ window.checkUserSiteRelationship = async function(origin) {
     }
 }
 
+// Converts a Blob to a Uint8Array [local helper module]
+async function blobToUint8Array(blob) {
+    const totalLength = blob.size;
+    const reader = blob.stream().getReader();
+    let chunks = [];
+    let receivedLength = 0;
+
+    while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+
+        chunks.push(value);
+        receivedLength += value.length;
+    }
+    let uint8Array = new Uint8Array(receivedLength);
+    let position = 0;
+
+    for (let chunk of chunks) {
+        uint8Array.set(chunk, position);
+        position += chunk.length;
+    }
+    return uint8Array;
+}
 
 window.zipItems = async function(el_items, targetDirPath, download = true) {
-    const zip = new JSZip();
+    const zip_operation_id = window.operation_id++;
+    window.operation_cancelled[zip_operation_id] = false;
+    let terminateOp = () => {}
 
     // if single item, convert to array
     el_items = Array.isArray(el_items) ? el_items : [el_items];
@@ -1969,44 +1994,80 @@ window.zipItems = async function(el_items, targetDirPath, download = true) {
     // create progress window
     let start_ts = Date.now();
     let progwin, progwin_timeout;
-    // only show progress window if it takes longer than 500ms to download
+    // only show progress window if it takes longer than 500ms
     progwin_timeout = setTimeout(async () => {
-        progwin = await UIWindowProgress();
+        progwin = await UIWindowProgress({
+            title: i18n('zip'),
+            icon: window.icons[`app-icon-uploader.svg`],
+            operation_id: zip_operation_id,
+            show_progress: true,
+            on_cancel: () => {
+                window.operation_cancelled[zip_operation_id] = true;
+                terminateOp();
+            },
+        });
+        progwin?.set_status(i18n('zip', 'Selection(s)'));
     }, 500);
 
-    for (const el_item of el_items) {
+    let toBeZipped = {};
+    
+    let perItemAdditionProgress = window.zippingProgressConfig.SEQUENCING / el_items.length;
+    let currentProgress = 0;
+    for (let idx = 0; idx < el_items.length; idx++) {
+        const el_item = el_items[idx];
+        if(window.operation_cancelled[zip_operation_id]) return;
         let targetPath = $(el_item).attr('data-path');
+
         // if directory, zip the directory
         if($(el_item).attr('data-is_dir') === '1'){
-            progwin?.set_status(i18n('reading_file', targetPath));
+            progwin?.set_status(i18n('reading', path.basename(targetPath)));
             // Recursively read the directory
             let children = await readDirectoryRecursive(targetPath);
 
             // Add files to the zip
-            for (const child of children) {
+            for (let cIdx = 0; cIdx < children.length; cIdx++) {
-                let relativePath;
+                const child = children[cIdx];
-                if(el_items.length === 1)
-                    relativePath = child.relativePath;
-                else
-                    relativePath = path.basename(targetPath) + '/' + child.relativePath;
-
-                // update progress window
-                progwin?.set_status(i18n('zipping_file', relativePath));
                 
-                // read file content
+                if (!child.relativePath) {
-                let content = await puter.fs.read(child.path);
+                    // Add empty directiories to the zip
-                try{
+                    toBeZipped = {
-                    zip.file(relativePath, content, {binary: true});
+                        ...toBeZipped,
-                }catch(e){
+                        [path.basename(child.path)+"/"]: [await blobToUint8Array(new Blob()), { level: 9 }]
-                    console.error(e);
+                    }
-                }
+                } else {
-            }
+                    // Add files from directory to the zip
+                    let relativePath;
+                    if (el_items.length === 1)
+                        relativePath = child.relativePath;
+                    else
+                        relativePath = path.basename(targetPath) + '/' + child.relativePath;
 
+                    // read file content
+                    progwin?.set_status(i18n('sequencing', child.relativePath));
+                    let content = await puter.fs.read(child.path);
+                    try {
+                        toBeZipped = {
+                            ...toBeZipped,
+                            [relativePath]: [await blobToUint8Array(content), { level: 9 }]
+                        }
+                    } catch (e) {
+                        console.error(e);
+                    }
+                }
+                currentProgress += perItemAdditionProgress / children.length;
+                progwin?.set_progress(currentProgress.toPrecision(2));
+            }
         }
-        // if item is a file, zip the file
+        // if item is a file, add the file to be zipped
         else{
-            let content = await puter.fs.read(targetPath);
+            progwin?.set_status(i18n('reading', path.basename($(el_items[0]).attr('data-path'))));
-            zip.file(path.basename(targetPath), content, {binary: true});
+            let content = await puter.fs.read(targetPath)
+            toBeZipped = {
+                ...toBeZipped,
+                [path.basename(targetPath)]: [await blobToUint8Array(content), {level: 9}]
+            }
+            currentProgress += perItemAdditionProgress;
+            progwin?.set_progress(currentProgress.toPrecision(2));
         }
     }
 
@@ -2017,15 +2078,28 @@ window.zipItems = async function(el_items, targetDirPath, download = true) {
     else
         zipName = 'Archive';
 
-    // Generate the zip file
+    progwin?.set_status(i18n('zipping', zipName + ".zip"));
-    zip.generateAsync({ type: "blob" })
+    progwin?.set_progress(currentProgress.toPrecision(2));
-        .then(async function (content) {
+    terminateOp = fflate.zip(toBeZipped, { level: 9 }, async (err, zippedContents)=>{
+        currentProgress += window.zippingProgressConfig.ZIPPING;
+        if(err) {
+            // close progress window
+            clearTimeout(progwin_timeout);
+            setTimeout(() => {
+                progwin?.close();
+            }, Math.max(0, window.copy_progress_hide_delay - (Date.now() - start_ts)));
+            // handle errors
+            // TODO: Display in progress dialog
+            console.error("Error in zipping files: ", err);
+        } else {
+            let zippedBlob = new Blob([new Uint8Array(zippedContents, zippedContents.byteOffset, zippedContents.length)]);
+            
             // Trigger the download
             if(download){
-                const url = URL.createObjectURL(content);
+                const url = URL.createObjectURL(zippedBlob);
                 const a = document.createElement("a");
                 a.href = url;
-                a.download = zipName;
+                a.download = zipName+".zip";
                 document.body.appendChild(a);
                 a.click();
 
@@ -2034,26 +2108,21 @@ window.zipItems = async function(el_items, targetDirPath, download = true) {
                 URL.revokeObjectURL(url);
             }
             // save
-            else
+            else {
-                await puter.fs.write(targetDirPath + '/' + zipName + ".zip", content, {overwrite: false, dedupeName: true})
+                progwin?.set_status(i18n('writing', zipName + ".zip"));
+                currentProgress += window.zippingProgressConfig.WRITING;
+                progwin?.set_progress(currentProgress.toPrecision(2));
+                await puter.fs.write(targetDirPath + '/' + zipName + ".zip", zippedBlob, { overwrite: false, dedupeName: true })
+                progwin?.set_progress(window.zippingProgressConfig.TOTAL);
+            }
 
             // close progress window
             clearTimeout(progwin_timeout);
             setTimeout(() => {
                 progwin?.close();
-            }, Math.max(0, window.copy_progress_hide_delay - (Date.now() - start_ts)));
+            }, Math.max(0, window.zip_progress_hide_delay - (Date.now() - start_ts)));
-        })
+        }
-        .catch(function (err) {
+    });
-            // close progress window
-            clearTimeout(progwin_timeout);
-            setTimeout(() => {
-                progwin?.close();
-            }, Math.max(0, window.copy_progress_hide_delay - (Date.now() - start_ts)));
-
-            // handle errors
-            // TODO: Display in progress dialog
-            console.error("Error in zipping files: ", err);
-        });
 }
 
 async function readDirectoryRecursive(path, baseDir = '') {
@@ -2062,16 +2131,20 @@ async function readDirectoryRecursive(path, baseDir = '') {
     // Read the directory
     const entries = await puter.fs.readdir(path);
 
-    // Process each entry
+    if (entries.length === 0) {
-    for (const entry of entries) {
+        allFiles.push({ path });
-        const fullPath = `${path}/${entry.name}`;
+    } else {
-        if (entry.is_dir) {
+        // Process each entry
-            // If entry is a directory, recursively read it
+        for (const entry of entries) {
-            const subDirFiles = await readDirectoryRecursive(fullPath, `${baseDir}${entry.name}/`);
+            const fullPath = `${path}/${entry.name}`;
-            allFiles = allFiles.concat(subDirFiles);
+            if (entry.is_dir) {
-        } else {
+                // If entry is a directory, recursively read it
-            // If entry is a file, add it to the list
+                const subDirFiles = await readDirectoryRecursive(fullPath, `${baseDir}${entry.name}/`);
-            allFiles.push({ path: fullPath, relativePath: `${baseDir}${entry.name}` });
+                allFiles = allFiles.concat(subDirFiles);
+            } else {
+                // If entry is a file, add it to the list
+                allFiles.push({ path: fullPath, relativePath: `${baseDir}${entry.name}` });
+            }
         }
     }
 
@@ -2089,43 +2162,87 @@ window.sleep = function(ms){
 }
 
 window.unzipItem = async function(itemPath) {
+    const unzip_operation_id = window.operation_id++;
+    window.operation_cancelled[unzip_operation_id] = false;
+    let terminateOp = () => {};
     // create progress window
     let start_ts = Date.now();
     let progwin, progwin_timeout;
     // only show progress window if it takes longer than 500ms to download
     progwin_timeout = setTimeout(async () => {
-        progwin = await UIWindowProgress();
+        progwin = await UIWindowProgress({
+            title: i18n('unzip'),
+            icon: window.icons[`app-icon-uploader.svg`],
+            operation_id: unzip_operation_id,
+            show_progress: true,
+            on_cancel: () => {
+                window.operation_cancelled[unzip_operation_id] = true;
+                terminateOp();
+            },
+        });
+        progwin?.set_status(i18n('unzip', 'Selection'));
     }, 500);
 
-    const zip = new JSZip();
+    let filePath = itemPath;
-    let filPath = itemPath;
+    let currentProgress = window.zippingProgressConfig.SEQUENCING;
-    let file = puter.fs.read(filPath);
 
-    zip.loadAsync(file).then(async function (zip) {
+    progwin?.set_status(i18n('sequencing', path.basename(filePath)));
-        const rootdir = await puter.fs.mkdir(path.dirname(filPath) + '/' + path.basename(filPath, '.zip'), {dedupeName: true});
+    let file = await blobToUint8Array(await puter.fs.read(filePath));
-        Object.keys(zip.files).forEach(async function (filename) {
+    progwin?.set_progress(currentProgress.toPrecision(2));
-            if(filename.endsWith('/'))
-                await puter.fs.mkdir(rootdir.path +'/' + filename, {createMissingParents: true});
-            zip.files[filename].async('blob').then(async function (fileData) {
-                await puter.fs.write(rootdir.path +'/' + filename, fileData);
-            }).catch(function (e) {
-                // UIAlert(e.message);
-            })
-        })
-        // close progress window
-        clearTimeout(progwin_timeout);
-        setTimeout(() => {
-            progwin?.close();
-        }, Math.max(0, window.copy_progress_hide_delay - (Date.now() - start_ts)));
 
-    }).catch(function (e) {
+    progwin?.set_status(i18n('unzipping', path.basename(filePath)));
-        // UIAlert(e.message);
+    terminateOp = fflate.unzip(file, async (err, unzipped) => {
-        // close progress window
+        currentProgress += window.zippingProgressConfig.ZIPPING;
-        clearTimeout(progwin_timeout);
+        progwin?.set_progress(currentProgress.toPrecision(2));
-        setTimeout(() => {
+        if(err) {
-            progwin?.close();
+            UIAlert(e.message);
-        }, Math.max(0, window.copy_progress_hide_delay - (Date.now() - start_ts)));
+            // close progress window
-    })
+            clearTimeout(progwin_timeout);
+            setTimeout(() => {
+                progwin?.close();
+            }, Math.max(0, window.copy_progress_hide_delay - (Date.now() - start_ts)));
+        } else {
+            const rootdir = await puter.fs.mkdir(path.dirname(filePath) + '/' + path.basename(filePath, '.zip'), { dedupeName: true });
+            let perItemProgress = window.zippingProgressConfig.WRITING / Object.keys(unzipped).length;
+            let queuedFileWrites = []
+            Object.keys(unzipped).forEach(fileItem => {
+                try {
+                    let fileData = new Blob([new Uint8Array(unzipped[fileItem], unzipped[fileItem].byteOffset, unzipped[fileItem].length)]);
+                    progwin?.set_status(i18n('writing', fileItem));
+                    queuedFileWrites.push(new File([fileData], fileItem))
+                    currentProgress += perItemProgress;
+                    progwin?.set_progress(currentProgress.toPrecision(2));
+                } catch (e) {
+                    UIAlert(e.message);
+                }
+            });
+            queuedFileWrites.length && puter.fs.upload(
+                // what to upload
+                queuedFileWrites, 
+                // where to upload
+                rootdir.path + '/',
+                // options
+                {
+                    createFileParent: true,
+                    progress: async function(operation_id, op_progress){
+                        progwin.set_progress(op_progress);
+                        // update title if window is not visible
+                        if(document.visibilityState !== "visible"){
+                            update_title_based_on_uploads();
+                        }
+                    },
+                    success: async function(items){
+                        progwin?.set_progress(window.zippingProgressConfig.TOTAL.toPrecision(2));
+                        // close progress window
+                        clearTimeout(progwin_timeout);
+                        setTimeout(() => {
+                            progwin?.close();
+                        }, Math.max(0, window.unzip_progress_hide_delay - (Date.now() - start_ts)));
+                    }
+                }
+            );
+        }
+    });
 }
 
 window.rename_file = async(options, new_name, old_name, old_path, el_item, el_item_name, el_item_icon, el_item_name_editor, website_url, is_undo = false)=>{

src/gui/src/i18n/translations/en.js

@@ -226,7 +226,8 @@ const en = {
         publish: "Publish",
         publish_as_website: 'Publish as website',
         puter_description: `Puter is a privacy-first personal cloud to keep all your files, apps, and games in one secure place, accessible from anywhere at any time.`,
-        reading_file: "Reading %strong%",
+        reading: "Reading %strong%",
+        writing: "Writing %strong%",
         recent: "Recent",
         recommended: "Recommended",
         recover_password: "Recover Password",
@@ -303,6 +304,7 @@ const en = {
         undo: 'Undo',
         unlimited: 'Unlimited',
         unzip: "Unzip",
+        unzipping: "Unzipping %strong%",
         upload: 'Upload',
         upload_here: 'Upload here',
         usage: 'Usage',
@@ -316,7 +318,8 @@ const en = {
         yes_release_it: 'Yes, Release It',
         you_have_been_referred_to_puter_by_a_friend: "You have been referred to Puter by a friend!",
         zip: "Zip",
-        zipping_file: "Zipping %strong%",
+        sequencing: "Sequencing %strong%",
+        zipping: "Zipping %strong%",
 
         // === 2FA Setup ===
         setup2fa_1_step_heading: 'Open your authenticator app',      

src/gui/src/static-assets.js

@@ -31,7 +31,7 @@ const lib_paths =[
     `/lib/timeago.min.js`,
     `/lib/iro.min.js`,
     `/lib/isMobile.min.js`,
-    `/lib/jszip-3.10.1.min.js`,
+    `/lib/fflate-0.8.2.min.js`,
 ]
 
 // Ordered list of CSS stylesheets

src/gui/webpack/libPaths.cjs

@@ -11,5 +11,5 @@ module.exports = [
     "timeago.min.js",
     "iro.min.js",
     "isMobile.min.js",
-    "jszip-3.10.1.min.js"
+    "fflate-0.8.2.min.js"
 ];

src/puter-js/src/modules/FileSystem/operations/upload.js

@@ -130,6 +130,7 @@ const upload = async function(items, dirPath, options = {}){
 
         // Will hold directories and files to be uploaded
         let dirs = [];
+        let uniqueDirs = {}
         let files = [];
 
         // Separate files from directories
@@ -141,8 +142,28 @@ const upload = async function(items, dirPath, options = {}){
             if(entries[i].isDirectory)
                 dirs.push({path: path.join(dirPath, entries[i].finalPath ? entries[i].finalPath : entries[i].fullPath)});
             // also files
-            else
+            else{
-                files.push(entries[i])
+                // Dragged and dropped files do not have a finalPath property and hence the fileItem will go undefined.
+                // In such cases, we need default to creating the files as uploaded by the user.
+                let fileItem = entries[i].finalPath ? entries[i].finalPath : entries[i].fullPath;
+                let [dirLevel, fileName] = [fileItem?.slice(0, fileItem?.lastIndexOf("/")), fileItem?.slice(fileItem?.lastIndexOf("/") + 1)]
+                
+                // If file name is blank then we need to create only an empty directory.
+                // On the other hand if the file name is not blank(could be undefined), we need to create the file.
+                fileName != "" && files.push(entries[i])
+                if (options.createFileParent && fileItem.includes('/')) {
+                    let incrementalDir;
+                    dirLevel.split('/').forEach((directory) => {
+                        incrementalDir = incrementalDir ? incrementalDir + '/' + directory : directory;
+                        let filePath = path.join(dirPath, incrementalDir)
+                        // Prevent duplicate parent directory creation
+                        if(!uniqueDirs[filePath]){
+                            uniqueDirs[filePath] = true;
+                            dirs.push({path: filePath});
+                        }
+                    })
+                }
+            }
             // stats about the upload to come
             if(entries[i].size !== undefined){
                 total_size += (entries[i].size);
@@ -185,7 +206,7 @@ const upload = async function(items, dirPath, options = {}){
         // Generate the requests to create all the 
         // folders in this upload
         //-------------------------------------------------
-        dirs.sort();
+        dirs.sort((a, b) => b.path.length - a.path.length);
         let mkdir_requests = [];
     
         for(let i=0; i < dirs.length; i++){