mirrors/puter
1
0
Fork 0
mirror of https://github.com/HeyPuter/puter synced 2024-11-14 22:06:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #787 from 4nshuman/issue/zip-operation

Browse Source 
Issue/zip operation
This commit is contained in:
Nariman Jelveh 2024-10-19 15:45:39 -07:00 committed by GitHub
commit 90e7098cc7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 257 additions and 123 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
eslint.config.js
View File

@ -72,7 +72,7 @@ export default [
"iro": true, // iro.js color picker
"$": true, // jQuery
"jQuery": true, // jQuery
"JSZip": true, // JSZip
"fflate": true, // fflate
"_": true, // lodash
"QRCode": true, // qrcode
"io": true, // socket.io

2
src/gui/puter-gui.json
View File

@ -14,7 +14,7 @@
"/lib/timeago.min.js",
"/lib/iro.min.js",
"/lib/isMobile.min.js",
"/lib/jszip-3.10.1.min.js"
"/lib/fflate-0.8.2.min.js"
],
"css_paths": [
"/css/normalize.css",

20
src/gui/src/UI/UIItem.js
View File

@ -1200,24 +1200,8 @@ function UIItem(options){
menu_items.push({
html: i18n('unzip'),
onClick: async function(){
const zip = new JSZip();
let filPath = $(el_item).attr('data-path');
let file = puter.fs.read($(el_item).attr('data-path'));
zip.loadAsync(file).then(async function (zip) {
const rootdir = await puter.fs.mkdir(path.dirname(filPath) + '/' + path.basename(filPath, '.zip'), {dedupeName: true});
Object.keys(zip.files).forEach(async function (filename) {
if(filename.endsWith('/'))
await puter.fs.mkdir(rootdir.path +'/' + filename, {createMissingParents: true});
zip.files[filename].async('blob').then(async function (fileData) {
await puter.fs.write(rootdir.path +'/' + filename, fileData);
}).catch(function (e) {
// UIAlert(e.message);
})
})
}).catch(function (e) {
// UIAlert(e.message);
})
let filePath = $(el_item).attr('data-path');
window.unzipItem(filePath)
}
})
}

13
src/gui/src/globals.js
View File

@ -109,6 +109,8 @@ window.taskbar_height = window.default_taskbar_height;
window.upload_progress_hide_delay = 500;
window.active_uploads = {};
window.copy_progress_hide_delay = 1000;
window.zip_progress_hide_delay = 2000;
window.unzip_progress_hide_delay = 2000;
window.busy_indicator_hide_delay = 600;
window.global_element_id = 0;
window.operation_id = 0;
@ -126,6 +128,17 @@ window.watchItems = [];
window.appdata_signatures = {};
window.appCallbackFunctions = [];
// Defines how much weight each operation has in the zipping progress
window.zippingProgressConfig = {
TOTAL: 100
}
//Assuming uInt8Array conversion a file takes betwneen 45% to 60% of the total progress
window.zippingProgressConfig.SEQUENCING = Math.floor(Math.random() * (60 - 45 + 1)) + 45,
//Assuming zipping up uInt8Arrays takes betwneen 20% to 23% of the total progress
window.zippingProgressConfig.ZIPPING = Math.floor(Math.random() * (23 - 20 + 1)) + 20,
//Assuming writing a zip file takes betwneen 10% to 14% of the total progress
window.zippingProgressConfig.WRITING = Math.floor(Math.random() * (14 - 10 + 1)) + 14,
// 'Launch' apps
window.launch_apps = [];
window.launch_apps.recent = []

283
src/gui/src/helpers.js
View File

@ -1959,9 +1959,34 @@ window.checkUserSiteRelationship = async function(origin) {
}
}
// Converts a Blob to a Uint8Array [local helper module]
async function blobToUint8Array(blob) {
const totalLength = blob.size;
const reader = blob.stream().getReader();
let chunks = [];
let receivedLength = 0;
while (true) {
const { done, value } = await reader.read();
if (done) break;
chunks.push(value);
receivedLength += value.length;
}
let uint8Array = new Uint8Array(receivedLength);
let position = 0;
for (let chunk of chunks) {
uint8Array.set(chunk, position);
position += chunk.length;
}
return uint8Array;
}
window.zipItems = async function(el_items, targetDirPath, download = true) {
const zip = new JSZip();
const zip_operation_id = window.operation_id++;
window.operation_cancelled[zip_operation_id] = false;
let terminateOp = () => {}
// if single item, convert to array
el_items = Array.isArray(el_items) ? el_items : [el_items];
@ -1969,44 +1994,80 @@ window.zipItems = async function(el_items, targetDirPath, download = true) {
// create progress window
let start_ts = Date.now();
let progwin, progwin_timeout;
// only show progress window if it takes longer than 500ms to download
// only show progress window if it takes longer than 500ms
progwin_timeout = setTimeout(async () => {
progwin = await UIWindowProgress();
progwin = await UIWindowProgress({
title: i18n('zip'),
icon: window.icons[`app-icon-uploader.svg`],
operation_id: zip_operation_id,
show_progress: true,
on_cancel: () => {
window.operation_cancelled[zip_operation_id] = true;
terminateOp();
},
});
progwin?.set_status(i18n('zip', 'Selection(s)'));
}, 500);
for (const el_item of el_items) {
let toBeZipped = {};
let perItemAdditionProgress = window.zippingProgressConfig.SEQUENCING / el_items.length;
let currentProgress = 0;
for (let idx = 0; idx < el_items.length; idx++) {
const el_item = el_items[idx];
if(window.operation_cancelled[zip_operation_id]) return;
let targetPath = $(el_item).attr('data-path');
// if directory, zip the directory
if($(el_item).attr('data-is_dir') === '1'){
progwin?.set_status(i18n('reading_file', targetPath));
progwin?.set_status(i18n('reading', path.basename(targetPath)));
// Recursively read the directory
let children = await readDirectoryRecursive(targetPath);
// Add files to the zip
for (const child of children) {
let relativePath;
if(el_items.length === 1)
relativePath = child.relativePath;
else
relativePath = path.basename(targetPath) + '/' + child.relativePath;
// update progress window
progwin?.set_status(i18n('zipping_file', relativePath));
for (let cIdx = 0; cIdx < children.length; cIdx++) {
const child = children[cIdx];
// read file content
let content = await puter.fs.read(child.path);
try{
zip.file(relativePath, content, {binary: true});
}catch(e){
console.error(e);
}
}
if (!child.relativePath) {
// Add empty directiories to the zip
toBeZipped = {
...toBeZipped,
[path.basename(child.path)+"/"]: [await blobToUint8Array(new Blob()), { level: 9 }]
}
} else {
// Add files from directory to the zip
let relativePath;
if (el_items.length === 1)
relativePath = child.relativePath;
else
relativePath = path.basename(targetPath) + '/' + child.relativePath;
// read file content
progwin?.set_status(i18n('sequencing', child.relativePath));
let content = await puter.fs.read(child.path);
try {
toBeZipped = {
...toBeZipped,
[relativePath]: [await blobToUint8Array(content), { level: 9 }]
}
} catch (e) {
console.error(e);
}
}
currentProgress += perItemAdditionProgress / children.length;
progwin?.set_progress(currentProgress.toPrecision(2));
}
}
// if item is a file, zip the file
// if item is a file, add the file to be zipped
else{
let content = await puter.fs.read(targetPath);
zip.file(path.basename(targetPath), content, {binary: true});
progwin?.set_status(i18n('reading', path.basename($(el_items[0]).attr('data-path'))));
let content = await puter.fs.read(targetPath)
toBeZipped = {
...toBeZipped,
[path.basename(targetPath)]: [await blobToUint8Array(content), {level: 9}]
}
currentProgress += perItemAdditionProgress;
progwin?.set_progress(currentProgress.toPrecision(2));
}
}
@ -2017,15 +2078,28 @@ window.zipItems = async function(el_items, targetDirPath, download = true) {
else
zipName = 'Archive';
// Generate the zip file
zip.generateAsync({ type: "blob" })
.then(async function (content) {
progwin?.set_status(i18n('zipping', zipName + ".zip"));
progwin?.set_progress(currentProgress.toPrecision(2));
terminateOp = fflate.zip(toBeZipped, { level: 9 }, async (err, zippedContents)=>{
currentProgress += window.zippingProgressConfig.ZIPPING;
if(err) {
// close progress window
clearTimeout(progwin_timeout);
setTimeout(() => {
progwin?.close();
}, Math.max(0, window.copy_progress_hide_delay - (Date.now() - start_ts)));
// handle errors
// TODO: Display in progress dialog
console.error("Error in zipping files: ", err);
} else {
let zippedBlob = new Blob([new Uint8Array(zippedContents, zippedContents.byteOffset, zippedContents.length)]);
// Trigger the download
if(download){
const url = URL.createObjectURL(content);
const url = URL.createObjectURL(zippedBlob);
const a = document.createElement("a");
a.href = url;
a.download = zipName;
a.download = zipName+".zip";
document.body.appendChild(a);
a.click();
@ -2034,26 +2108,21 @@ window.zipItems = async function(el_items, targetDirPath, download = true) {
URL.revokeObjectURL(url);
}
// save
else
await puter.fs.write(targetDirPath + '/' + zipName + ".zip", content, {overwrite: false, dedupeName: true})
else {
progwin?.set_status(i18n('writing', zipName + ".zip"));
currentProgress += window.zippingProgressConfig.WRITING;
progwin?.set_progress(currentProgress.toPrecision(2));
await puter.fs.write(targetDirPath + '/' + zipName + ".zip", zippedBlob, { overwrite: false, dedupeName: true })
progwin?.set_progress(window.zippingProgressConfig.TOTAL);
}
// close progress window
clearTimeout(progwin_timeout);
setTimeout(() => {
progwin?.close();
}, Math.max(0, window.copy_progress_hide_delay - (Date.now() - start_ts)));
})
.catch(function (err) {
// close progress window
clearTimeout(progwin_timeout);
setTimeout(() => {
progwin?.close();
}, Math.max(0, window.copy_progress_hide_delay - (Date.now() - start_ts)));
// handle errors
// TODO: Display in progress dialog
console.error("Error in zipping files: ", err);
});
}, Math.max(0, window.zip_progress_hide_delay - (Date.now() - start_ts)));
}
});
}
async function readDirectoryRecursive(path, baseDir = '') {
@ -2062,16 +2131,20 @@ async function readDirectoryRecursive(path, baseDir = '') {
// Read the directory
const entries = await puter.fs.readdir(path);
// Process each entry
for (const entry of entries) {
const fullPath = `${path}/${entry.name}`;
if (entry.is_dir) {
// If entry is a directory, recursively read it
const subDirFiles = await readDirectoryRecursive(fullPath, `${baseDir}${entry.name}/`);
allFiles = allFiles.concat(subDirFiles);
} else {
// If entry is a file, add it to the list
allFiles.push({ path: fullPath, relativePath: `${baseDir}${entry.name}` });
if (entries.length === 0) {
allFiles.push({ path });
} else {
// Process each entry
for (const entry of entries) {
const fullPath = `${path}/${entry.name}`;
if (entry.is_dir) {
// If entry is a directory, recursively read it
const subDirFiles = await readDirectoryRecursive(fullPath, `${baseDir}${entry.name}/`);
allFiles = allFiles.concat(subDirFiles);
} else {
// If entry is a file, add it to the list
allFiles.push({ path: fullPath, relativePath: `${baseDir}${entry.name}` });
}
}
}
@ -2089,43 +2162,87 @@ window.sleep = function(ms){
}
window.unzipItem = async function(itemPath) {
const unzip_operation_id = window.operation_id++;
window.operation_cancelled[unzip_operation_id] = false;
let terminateOp = () => {};
// create progress window
let start_ts = Date.now();
let progwin, progwin_timeout;
// only show progress window if it takes longer than 500ms to download
progwin_timeout = setTimeout(async () => {
progwin = await UIWindowProgress();
progwin = await UIWindowProgress({
title: i18n('unzip'),
icon: window.icons[`app-icon-uploader.svg`],
operation_id: unzip_operation_id,
show_progress: true,
on_cancel: () => {
window.operation_cancelled[unzip_operation_id] = true;
terminateOp();
},
});
progwin?.set_status(i18n('unzip', 'Selection'));
}, 500);
const zip = new JSZip();
let filPath = itemPath;
let file = puter.fs.read(filPath);
let filePath = itemPath;
let currentProgress = window.zippingProgressConfig.SEQUENCING;
zip.loadAsync(file).then(async function (zip) {
const rootdir = await puter.fs.mkdir(path.dirname(filPath) + '/' + path.basename(filPath, '.zip'), {dedupeName: true});
Object.keys(zip.files).forEach(async function (filename) {
if(filename.endsWith('/'))
await puter.fs.mkdir(rootdir.path +'/' + filename, {createMissingParents: true});
zip.files[filename].async('blob').then(async function (fileData) {
await puter.fs.write(rootdir.path +'/' + filename, fileData);
}).catch(function (e) {
// UIAlert(e.message);
})
})
// close progress window
clearTimeout(progwin_timeout);
setTimeout(() => {
progwin?.close();
}, Math.max(0, window.copy_progress_hide_delay - (Date.now() - start_ts)));
progwin?.set_status(i18n('sequencing', path.basename(filePath)));
let file = await blobToUint8Array(await puter.fs.read(filePath));
progwin?.set_progress(currentProgress.toPrecision(2));
}).catch(function (e) {
// UIAlert(e.message);
// close progress window
clearTimeout(progwin_timeout);
setTimeout(() => {
progwin?.close();
}, Math.max(0, window.copy_progress_hide_delay - (Date.now() - start_ts)));
})
progwin?.set_status(i18n('unzipping', path.basename(filePath)));
terminateOp = fflate.unzip(file, async (err, unzipped) => {
currentProgress += window.zippingProgressConfig.ZIPPING;
progwin?.set_progress(currentProgress.toPrecision(2));
if(err) {
UIAlert(e.message);
// close progress window
clearTimeout(progwin_timeout);
setTimeout(() => {
progwin?.close();
}, Math.max(0, window.copy_progress_hide_delay - (Date.now() - start_ts)));
} else {
const rootdir = await puter.fs.mkdir(path.dirname(filePath) + '/' + path.basename(filePath, '.zip'), { dedupeName: true });
let perItemProgress = window.zippingProgressConfig.WRITING / Object.keys(unzipped).length;
let queuedFileWrites = []
Object.keys(unzipped).forEach(fileItem => {
try {
let fileData = new Blob([new Uint8Array(unzipped[fileItem], unzipped[fileItem].byteOffset, unzipped[fileItem].length)]);
progwin?.set_status(i18n('writing', fileItem));
queuedFileWrites.push(new File([fileData], fileItem))
currentProgress += perItemProgress;
progwin?.set_progress(currentProgress.toPrecision(2));
} catch (e) {
UIAlert(e.message);
}
});
queuedFileWrites.length && puter.fs.upload(
// what to upload
queuedFileWrites,
// where to upload
rootdir.path + '/',
// options
{
createFileParent: true,
progress: async function(operation_id, op_progress){
progwin.set_progress(op_progress);
// update title if window is not visible
if(document.visibilityState !== "visible"){
update_title_based_on_uploads();
}
},
success: async function(items){
progwin?.set_progress(window.zippingProgressConfig.TOTAL.toPrecision(2));
// close progress window
clearTimeout(progwin_timeout);
setTimeout(() => {
progwin?.close();
}, Math.max(0, window.unzip_progress_hide_delay - (Date.now() - start_ts)));
}
}
);
}
});
}
window.rename_file = async(options, new_name, old_name, old_path, el_item, el_item_name, el_item_icon, el_item_name_editor, website_url, is_undo = false)=>{

7
src/gui/src/i18n/translations/en.js
View File

@ -226,7 +226,8 @@ const en = {
publish: "Publish",
publish_as_website: 'Publish as website',
puter_description: `Puter is a privacy-first personal cloud to keep all your files, apps, and games in one secure place, accessible from anywhere at any time.`,
reading_file: "Reading %strong%",
reading: "Reading %strong%",
writing: "Writing %strong%",
recent: "Recent",
recommended: "Recommended",
recover_password: "Recover Password",
@ -303,6 +304,7 @@ const en = {
undo: 'Undo',
unlimited: 'Unlimited',
unzip: "Unzip",
unzipping: "Unzipping %strong%",
upload: 'Upload',
upload_here: 'Upload here',
usage: 'Usage',
@ -316,7 +318,8 @@ const en = {
yes_release_it: 'Yes, Release It',
you_have_been_referred_to_puter_by_a_friend: "You have been referred to Puter by a friend!",
zip: "Zip",
zipping_file: "Zipping %strong%",
sequencing: "Sequencing %strong%",
zipping: "Zipping %strong%",
// === 2FA Setup ===
setup2fa_1_step_heading: 'Open your authenticator app',

9
src/gui/src/lib/fflate-0.8.2.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long

13
src/gui/src/lib/jszip-3.10.1.min.js vendored
View File

File diff suppressed because one or more lines are too long

2
src/gui/src/static-assets.js
View File

@ -31,7 +31,7 @@ const lib_paths =[
`/lib/timeago.min.js`,
`/lib/iro.min.js`,
`/lib/isMobile.min.js`,
`/lib/jszip-3.10.1.min.js`,
`/lib/fflate-0.8.2.min.js`,
]
// Ordered list of CSS stylesheets

2
src/gui/webpack/libPaths.cjs
View File

@ -11,5 +11,5 @@ module.exports = [
"timeago.min.js",
"iro.min.js",
"isMobile.min.js",
"jszip-3.10.1.min.js"
"fflate-0.8.2.min.js"
];

27
src/puter-js/src/modules/FileSystem/operations/upload.js
View File

@ -130,6 +130,7 @@ const upload = async function(items, dirPath, options = {}){
// Will hold directories and files to be uploaded
let dirs = [];
let uniqueDirs = {}
let files = [];
// Separate files from directories
@ -141,8 +142,28 @@ const upload = async function(items, dirPath, options = {}){
if(entries[i].isDirectory)
dirs.push({path: path.join(dirPath, entries[i].finalPath ? entries[i].finalPath : entries[i].fullPath)});
// also files
else
files.push(entries[i])
else{
// Dragged and dropped files do not have a finalPath property and hence the fileItem will go undefined.
// In such cases, we need default to creating the files as uploaded by the user.
let fileItem = entries[i].finalPath ? entries[i].finalPath : entries[i].fullPath;
let [dirLevel, fileName] = [fileItem?.slice(0, fileItem?.lastIndexOf("/")), fileItem?.slice(fileItem?.lastIndexOf("/") + 1)]
// If file name is blank then we need to create only an empty directory.
// On the other hand if the file name is not blank(could be undefined), we need to create the file.
fileName != "" && files.push(entries[i])
if (options.createFileParent && fileItem.includes('/')) {
let incrementalDir;
dirLevel.split('/').forEach((directory) => {
incrementalDir = incrementalDir ? incrementalDir + '/' + directory : directory;
let filePath = path.join(dirPath, incrementalDir)
// Prevent duplicate parent directory creation
if(!uniqueDirs[filePath]){
uniqueDirs[filePath] = true;
dirs.push({path: filePath});
}
})
}
}
// stats about the upload to come
if(entries[i].size !== undefined){
total_size += (entries[i].size);
@ -185,7 +206,7 @@ const upload = async function(items, dirPath, options = {}){
// Generate the requests to create all the
// folders in this upload
//-------------------------------------------------
dirs.sort();
dirs.sort((a, b) => b.path.length - a.path.length);
let mkdir_requests = [];
for(let i=0; i < dirs.length; i++){