From 11fbcb27b0ae282c82975820b988e710b9ab5168 Mon Sep 17 00:00:00 2001 From: KernelDeimos Date: Fri, 11 Oct 2024 23:51:38 -0400 Subject: [PATCH] dev: cache readdir --- .../src/modules/FileSystem/CacheFS.js | 155 ++++++++++++++++-- src/puter-js/src/modules/FileSystem/index.js | 1 + .../modules/FileSystem/operations/readdir.js | 6 +- 3 files changed, 143 insertions(+), 19 deletions(-) diff --git a/src/puter-js/src/modules/FileSystem/CacheFS.js b/src/puter-js/src/modules/FileSystem/CacheFS.js index d5bed510..f3395188 100644 --- a/src/puter-js/src/modules/FileSystem/CacheFS.js +++ b/src/puter-js/src/modules/FileSystem/CacheFS.js @@ -1,44 +1,65 @@ import putility from "@heyputer/putility"; import { RWLock } from "@heyputer/putility/src/libs/promise"; import { ProxyFilesystem, TFilesystem } from "./definitions"; +import { uuidv4 } from "../../lib/utils"; export const ROOT_UUID = '00000000-0000-0000-0000-000000000000'; +const TTL = 10 * 1000; export class CacheFS extends putility.AdvancedBase { static PROPERTIES = { - // 'internal_uuid' maps a path or external UUID to - // the respective cache entry UUID - // (which for now is the same as the public UUID) - internal_uuid: () => ({}), + assocs_path_: () => ({}), + assocs_uuid_: () => ({}), entries: () => ({}), }; get_entry_ei (external_identifier) { - const internal_identifier = this.internal_uuid[external_identifier]; + if ( Array.isArray(external_identifier) ) { + for ( const ei of external_identifier ) { + const entry = this.get_entry_ei(ei); + if ( entry ) return entry; + } + return; + } + + console.log('GET ENTRY EI', external_identifier); + + const internal_identifier = + this.assocs_path_[external_identifier] || + this.assocs_uuid_[external_identifier] || + external_identifier; + if ( ! internal_identifier ) { return; } return this.entries[internal_identifier]; } - add_entry ({ - external_identifiers, - internal_identifier, - }) { + add_entry ({ id } = {}) { + const internal_identifier = id ?? uuidv4(); const entry = { + id: internal_identifier, stat_has: {}, stat_exp: 0, locks: { stat: new RWLock(), + members: new RWLock(), }, }; - for ( const ident of external_identifiers ) { - this.internal_uuid[ident] = internal_identifier; - } this.entries[internal_identifier] = entry; - console.log('cREATED ENTRY', this.internal_uuid, this.entries); return entry; } + + assoc_path (path, internal_identifier) { + console.log('ASSOC PATH', path, internal_identifier); + this.assocs_path_[path] = internal_identifier; + } + + assoc_uuid (uuid, internal_identifier) { + if ( uuid === internal_identifier ) return; + this.assocs_uuid_[uuid] = internal_identifier; + } + } export class CachedFilesystem extends ProxyFilesystem { @@ -101,23 +122,121 @@ export class CachedFilesystem extends ProxyFilesystem { console.log('DOING THE STAT', o); const entry = await this.delegate.stat(o); + // We might have new information to identify a relevant cache entry + let cent_replaced = !! cent; + cent = this.cacheFS.get_entry_ei([entry.uid, entry.path]); + if ( cent ) { + if ( cent_replaced ) l.unlock(); + l = await cent.locks.stat.wlock(); + } + if ( ! cent ) { - cent = this.cacheFS.add_entry({ - external_identifiers: [entry.path, entry.uid], - internal_identifier: entry.uid, - }); + cent = this.cacheFS.add_entry({ id: entry.uid }); + this.cacheFS.assoc_path(entry.path, cent.id); + this.cacheFS.assoc_uuid(entry.uid, cent.id); + l = await cent.locks.stat.wlock(); } cent.stat = entry; cent.stat_has = { ...values_requested }; // TODO: increase cache TTL once invalidation works - cent.stat_exp = Date.now() + 1000*3; + cent.stat_exp = Date.now() + TTL; l.unlock(); console.log('RETRUNING THE ENTRY', entry); return entry; + }, + readdir: async function (o) { + let cent = this.cacheFS.get_entry_ei([o.path, o.uid]); + + console.log('CENT', cent, o); + let stats = null; + if ( cent && cent.members && cent.members_exp > Date.now() ) { + console.log('MEMBERS', cent.members); + stats = []; + const l = await cent.locks.stat.rlock(); + + for ( const id of cent.members ) { + const member = this.cacheFS.get_entry_ei(id); + if ( ! member || ! member.stat || member.stat_exp <= Date.now() ) { + console.log('NO MEMBER OR STAT', member); + stats = null; + break; + } + console.log('member', member); + if ( ! o.no_assocs && ! member.stat_has.subdomains ) { + stats = null; + break; + } + if ( ! o.no_assocs && ! member.stat_has.apps ) { + stats = null; + break; + } + if ( ! o.no_thumbs && ! member.stat_has.thumbnail ) { + stats = null; + break; + } + console.log('PUSHING', member.stat); + + stats.push(member.stat); + } + + l.unlock(); + } + + console.log('STATS????', stats); + if ( stats ) { + return stats; + } + + let l; + if ( cent ) { + l = await cent.locks.members.wlock(); + } + + const entries = await this.delegate.readdir(o); + if ( ! cent ) { + cent = this.cacheFS.add_entry(o.uid ? { id: o.uid } : {}); + if ( o.path ) this.cacheFS.assoc_path(o.path, cent.id); + l = await cent.locks.members.wlock(); + } + + let cent_ids = []; + for ( const entry of entries ) { + let entry_cent = this.cacheFS.get_entry_ei([entry.path, entry.uid]); + if ( ! entry_cent ) { + entry_cent = this.cacheFS.add_entry({ id: entry.uid }); + this.cacheFS.assoc_path(entry.path, entry.uid); + } + cent_ids.push(entry_cent.id); + // TODO: update_stat_ is not implemented + // this.cacheFS.update_stat_(entry_cent, entry, { + // subdomains: ! o.no_assocs, + // apps: ! o.no_assocs, + // thumbnail: ! o.no_thumbs, + // }); + entry_cent.stat = entry; + entry_cent.stat_has = { + subdomains: ! o.no_assocs, + apps: ! o.no_assocs, + thumbnail: ! o.no_thumbs, + } + entry_cent.stat_exp = Date.now() + 1000*3; + } + + cent.members = [] + for ( const id of cent_ids ) { + cent.members.push(id); + } + cent.members_exp = Date.now() + TTL; + + l.unlock(); + + console.log('CACHE ENTRY?', cent); + + return entries; } } } diff --git a/src/puter-js/src/modules/FileSystem/index.js b/src/puter-js/src/modules/FileSystem/index.js index df4dcc4d..5832dfca 100644 --- a/src/puter-js/src/modules/FileSystem/index.js +++ b/src/puter-js/src/modules/FileSystem/index.js @@ -82,6 +82,7 @@ export class PuterJSFileSystemModule extends AdvancedBase { // this.filesystem = this.fs_nocache; this.fs_proxy_ = new ProxyFilesystem({ delegate: this.fs_nocache_ }); this.filesystem = this.fs_proxy_.as(TFilesystem); + this.fs_proxy_.delegate = this.fs_cache_; } cache_on () { diff --git a/src/puter-js/src/modules/FileSystem/operations/readdir.js b/src/puter-js/src/modules/FileSystem/operations/readdir.js index fe2f2e14..6f4f3b09 100644 --- a/src/puter-js/src/modules/FileSystem/operations/readdir.js +++ b/src/puter-js/src/modules/FileSystem/operations/readdir.js @@ -39,7 +39,11 @@ const readdir = async function (...args) { // set up event handlers for load and error events utils.setupXhrEventHandlers(xhr, options.success, options.error, resolve, reject); - xhr.send(JSON.stringify({path: getAbsolutePathForApp(options.path)})); + xhr.send(JSON.stringify({ + path: getAbsolutePathForApp(options.path), + no_thumbs: options.no_thumbs, + no_assocs: options.no_assocs, + })); }) }