From 8f77f720e68b342a8f6c787928554a6237f3dfa4 Mon Sep 17 00:00:00 2001 From: theit8514 Date: Sun, 12 Dec 2021 19:39:53 -0500 Subject: [PATCH 1/2] Cache the blobs generated by scripts --- package-lock.json | 11 ++++++ package.json | 1 + src/NetscriptEvaluator.ts | 5 ++- src/NetscriptJSEvaluator.ts | 59 +++++++++++++++++++++++--------- src/Script/RunningScript.ts | 5 +++ src/Script/Script.ts | 31 ++++++++++++++++- src/utils/BlobCache.ts | 18 ++++++++++ src/utils/ImportCache.ts | 14 ++++++++ src/utils/helpers/computeHash.ts | 12 +++++++ 9 files changed, 136 insertions(+), 20 deletions(-) create mode 100644 src/utils/BlobCache.ts create mode 100644 src/utils/ImportCache.ts create mode 100644 src/utils/helpers/computeHash.ts diff --git a/package-lock.json b/package-lock.json index ceb318ee0..5e67128dc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -34,6 +34,7 @@ "file-saver": "^1.3.8", "fs": "^0.0.1-security", "jquery": "^3.5.0", + "js-sha256": "^0.9.0", "jszip": "^3.7.0", "material-ui-color": "^1.2.0", "monaco-editor": "^0.27.0", @@ -13699,6 +13700,11 @@ "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.6.0.tgz", "integrity": "sha512-JVzAR/AjBvVt2BmYhxRCSYysDsPcssdmTFnzyLEts9qNwmjmu4JTAMYubEfwVOSwpQ1I1sKKFcxhZCI2buerfw==" }, + "node_modules/js-sha256": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/js-sha256/-/js-sha256-0.9.0.tgz", + "integrity": "sha512-sga3MHh9sgQN2+pJ9VYZ+1LPwXOxuBJBA5nrR5/ofPfuiJBE2hnjsaN8se8JznOmGLN2p49Pe5U/ttafcs/apA==" + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -32343,6 +32349,11 @@ "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.6.0.tgz", "integrity": "sha512-JVzAR/AjBvVt2BmYhxRCSYysDsPcssdmTFnzyLEts9qNwmjmu4JTAMYubEfwVOSwpQ1I1sKKFcxhZCI2buerfw==" }, + "js-sha256": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/js-sha256/-/js-sha256-0.9.0.tgz", + "integrity": "sha512-sga3MHh9sgQN2+pJ9VYZ+1LPwXOxuBJBA5nrR5/ofPfuiJBE2hnjsaN8se8JznOmGLN2p49Pe5U/ttafcs/apA==" + }, "js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", diff --git a/package.json b/package.json index 1c0ec4dfe..712c8f2b4 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "file-saver": "^1.3.8", "fs": "^0.0.1-security", "jquery": "^3.5.0", + "js-sha256": "^0.9.0", "jszip": "^3.7.0", "material-ui-color": "^1.2.0", "monaco-editor": "^0.27.0", diff --git a/src/NetscriptEvaluator.ts b/src/NetscriptEvaluator.ts index e12eb1406..d5a35b214 100644 --- a/src/NetscriptEvaluator.ts +++ b/src/NetscriptEvaluator.ts @@ -1,7 +1,6 @@ import { isString } from "./utils/helpers/isString"; import { GetServer } from "./Server/AllServers"; import { WorkerScript } from "./Netscript/WorkerScript"; -import { BlobsMap } from "./NetscriptJSEvaluator"; export function netscriptDelay(time: number, workerScript: WorkerScript): Promise { return new Promise(function (resolve) { @@ -22,8 +21,8 @@ export function makeRuntimeRejectMsg(workerScript: WorkerScript, msg: string): s throw new Error(`WorkerScript constructed with invalid server ip: ${workerScript.hostname}`); } - for (const url in BlobsMap) { - msg = msg.replace(new RegExp(url, "g"), BlobsMap[url]); + for (const scriptUrl of workerScript.scriptRef.dependencies) { + msg = msg.replace(new RegExp(scriptUrl.url, "g"), scriptUrl.filename); } return "|DELIMITER|" + server.hostname + "|DELIMITER|" + workerScript.name + "|DELIMITER|" + msg; diff --git a/src/NetscriptJSEvaluator.ts b/src/NetscriptJSEvaluator.ts index 28205a85b..d4c7b6dbc 100644 --- a/src/NetscriptJSEvaluator.ts +++ b/src/NetscriptJSEvaluator.ts @@ -2,10 +2,11 @@ import { makeRuntimeRejectMsg } from "./NetscriptEvaluator"; import { ScriptUrl } from "./Script/ScriptUrl"; import { WorkerScript } from "./Netscript/WorkerScript"; import { Script } from "./Script/Script"; +import { computeHash } from "./utils/helpers/computeHash"; +import { BlobCache } from "./utils/BlobCache"; +import { ImportCache } from "./utils/ImportCache"; import { areImportsEquals } from "./Terminal/DirectoryHelpers"; -export const BlobsMap: { [key: string]: string } = {}; - // Makes a blob that contains the code of a given script. function makeScriptBlob(code: string): Blob { return new Blob([code], { type: "text/javascript" }); @@ -22,13 +23,22 @@ export async function compile(script: Script, scripts: Script[]): Promise // by placing it inside an eval call. await script.updateRamUsage(scripts); const uurls = _getScriptUrls(script, scripts, []); - if (script.url) { - URL.revokeObjectURL(script.url); // remove the old reference. - delete BlobsMap[script.url]; + const url = uurls[uurls.length - 1].url; + if (script.url && script.url !== url) { + // Thoughts: Should we be revoking any URLs here? + // If a script is modified repeatedly between two states, + // we could reuse the blob at a later time. + // BlobCache.removeByValue(script.url); + // URL.revokeObjectURL(script.url); + // if (script.dependencies.length > 0) { + // script.dependencies.forEach((dep) => { + // removeBlobFromCache(dep.url); + // URL.revokeObjectURL(dep.url); + // }); + // } } - if (script.dependencies.length > 0) script.dependencies.forEach((dep) => URL.revokeObjectURL(dep.url)); - script.url = uurls[uurls.length - 1].url; - script.module = new Promise((resolve) => resolve(eval("import(uurls[uurls.length - 1].url)"))); + script.url = url; + script.module = new Promise((resolve) => resolve(eval("import(url)"))); script.dependencies = uurls; } @@ -124,12 +134,21 @@ function _getScriptUrls(script: Script, scripts: Script[], seen: Script[]): Scri // Find the corresponding script. const [importedScript] = scripts.filter((s) => areImportsEquals(s.filename, filename)); - // Try to get a URL for the requested script and its dependencies. - const urls = _getScriptUrls(importedScript, scripts, seen); + // Check to see if the urls for this script are stored in the cache by the hash value. + let urls = ImportCache.get(importedScript.hash()); + // If we don't have it in the cache, then we need to generate the urls for it. + if (!urls) { + // Try to get a URL for the requested script and its dependencies. + urls = _getScriptUrls(importedScript, scripts, seen); + } // The top url in the stack is the replacement import file for this script. urlStack.push(...urls); - return [prefix, urls[urls.length - 1].url, suffix].join(""); + const blob = urls[urls.length - 1].url; + ImportCache.store(importedScript.hash(), urls); + + // Replace the blob inside the import statement. + return [prefix, blob, suffix].join(""); }, ); @@ -137,11 +156,19 @@ function _getScriptUrls(script: Script, scripts: Script[], seen: Script[]): Scri // accidental calls to window.print() do not bring up the "print screen" dialog transformedCode += `\n\nfunction print() {throw new Error("Invalid call to window.print(). Did you mean to use Netscript's print()?");}`; - // If we successfully transformed the code, create a blob url for it and - // push that URL onto the top of the stack. - const su = new ScriptUrl(script.filename, URL.createObjectURL(makeScriptBlob(transformedCode))); - urlStack.push(su); - BlobsMap[su.url] = su.filename; + // If we successfully transformed the code, create a blob url for it + // Compute the hash for the transformed code + const transformedHash = computeHash(transformedCode); + // Check to see if this transformed hash is in our cache + let blob = BlobCache.get(transformedHash); + if (!blob) { + blob = URL.createObjectURL(makeScriptBlob(transformedCode)); + } + // Store this blob in the cache. Any script that transforms the same + // (e.g. same scripts on server, same hash value, etc) can use this blob url. + BlobCache.store(transformedHash, blob); + // Push the blob URL onto the top of the stack. + urlStack.push(new ScriptUrl(script.filename, blob)); return urlStack; } catch (err) { // If there is an error, we need to clean up the URLs. diff --git a/src/Script/RunningScript.ts b/src/Script/RunningScript.ts index 469851261..0f9b2b3bb 100644 --- a/src/Script/RunningScript.ts +++ b/src/Script/RunningScript.ts @@ -3,6 +3,7 @@ * A Script can have multiple active instances */ import { Script } from "./Script"; +import { ScriptUrl } from "./ScriptUrl"; import { Settings } from "../Settings/Settings"; import { IMap } from "../types"; import { Terminal } from "../Terminal"; @@ -58,6 +59,9 @@ export class RunningScript { // Number of threads that this script is running with threads = 1; + // Script urls for the current running script for translating urls back to file names in errors + dependencies: ScriptUrl[] = []; + constructor(script: Script | null = null, args: any[] = []) { if (script == null) { return; @@ -66,6 +70,7 @@ export class RunningScript { this.args = args; this.server = script.server; this.ramUsage = script.ramUsage; + this.dependencies = script.dependencies; } log(txt: string): void { diff --git a/src/Script/Script.ts b/src/Script/Script.ts index 62d71ede2..ccdfa578e 100644 --- a/src/Script/Script.ts +++ b/src/Script/Script.ts @@ -9,6 +9,7 @@ import { ScriptUrl } from "./ScriptUrl"; import { Generic_fromJSON, Generic_toJSON, Reviver } from "../utils/JSONReviver"; import { roundToTwo } from "../utils/helpers/roundToTwo"; +import { computeHash } from "../utils/helpers/computeHash"; let globalModuleSequenceNumber = 0; @@ -40,6 +41,9 @@ export class Script { // hostname of server that this script is on. server = ""; + // sha256 hash of the code in the Script. Do not access directly. + _hash = ""; + constructor(fn = "", code = "", server = "", otherScripts: Script[] = []) { this.filename = fn; this.code = code; @@ -47,8 +51,10 @@ export class Script { this.server = server; // hostname of server this script is on this.module = ""; this.moduleSequenceNumber = ++globalModuleSequenceNumber; + this._hash = ""; if (this.code !== "") { this.updateRamUsage(otherScripts); + this.rehash(); } } @@ -84,6 +90,24 @@ export class Script { markUpdated(): void { this.module = ""; this.moduleSequenceNumber = ++globalModuleSequenceNumber; + this.rehash(); + } + + /** + * Force update of the computed hash based on the source code. + */ + rehash(): void { + this._hash = computeHash(this.code); + } + + /** + * If the hash is not computed, computes the hash. Otherwise return the computed hash. + * @returns the computed hash of the script + */ + hash(): string { + if (!this._hash) + this.rehash(); + return this._hash; } /** @@ -125,7 +149,12 @@ export class Script { // Initializes a Script Object from a JSON save state // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types static fromJSON(value: any): Script { - return Generic_fromJSON(Script, value.data); + const s = Generic_fromJSON(Script, value.data); + // Force the url to blank from the save data. Urls are not valid outside the current browser page load. + s.url = ""; + // Rehash the code to ensure that hash is set properly. + s.rehash(); + return s; } } diff --git a/src/utils/BlobCache.ts b/src/utils/BlobCache.ts new file mode 100644 index 000000000..fed85e91f --- /dev/null +++ b/src/utils/BlobCache.ts @@ -0,0 +1,18 @@ + +const blobCache: { [hash: string]: string } = {}; + +export class BlobCache { + static get(hash: string) { + return blobCache[hash]; + } + + static store(hash: string, value: string): void { + if (blobCache[hash]) return; + blobCache[hash] = value; + } + + static removeByValue(value: string) { + const keys = Object.keys(blobCache).filter((key) => blobCache[key] === value); + keys.forEach((key) => delete blobCache[key]); + } +} diff --git a/src/utils/ImportCache.ts b/src/utils/ImportCache.ts new file mode 100644 index 000000000..ab8672b68 --- /dev/null +++ b/src/utils/ImportCache.ts @@ -0,0 +1,14 @@ +import { ScriptUrl } from "../Script/ScriptUrl"; + +const importCache: { [hash: string]: ScriptUrl[] } = {}; + +export class ImportCache { + static get(hash: string) { + return importCache[hash]; + } + + static store(hash: string, value: ScriptUrl[]): void { + if (importCache[hash]) return; + importCache[hash] = value; + } +} diff --git a/src/utils/helpers/computeHash.ts b/src/utils/helpers/computeHash.ts new file mode 100644 index 000000000..a6c9f45af --- /dev/null +++ b/src/utils/helpers/computeHash.ts @@ -0,0 +1,12 @@ +import { sha256 } from "js-sha256"; + +/** + * Computes a SHA-256 hash of a string synchronously + * @param message The input string + * @returns The SHA-256 hash in hex + */ +export function computeHash(message: string): string { + var hash = sha256.create(); + hash.update(message); + return hash.hex(); +} From fd93c6d3387b5df0e508ebae152f6ffc3ee0cbec Mon Sep 17 00:00:00 2001 From: theit8514 Date: Thu, 23 Dec 2021 23:26:05 -0500 Subject: [PATCH 2/2] Fix linting errors --- src/utils/BlobCache.ts | 4 ++-- src/utils/ImportCache.ts | 2 +- src/utils/helpers/computeHash.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/utils/BlobCache.ts b/src/utils/BlobCache.ts index fed85e91f..e58a9abb6 100644 --- a/src/utils/BlobCache.ts +++ b/src/utils/BlobCache.ts @@ -2,7 +2,7 @@ const blobCache: { [hash: string]: string } = {}; export class BlobCache { - static get(hash: string) { + static get(hash: string): string { return blobCache[hash]; } @@ -11,7 +11,7 @@ export class BlobCache { blobCache[hash] = value; } - static removeByValue(value: string) { + static removeByValue(value: string): void { const keys = Object.keys(blobCache).filter((key) => blobCache[key] === value); keys.forEach((key) => delete blobCache[key]); } diff --git a/src/utils/ImportCache.ts b/src/utils/ImportCache.ts index ab8672b68..9a8383e21 100644 --- a/src/utils/ImportCache.ts +++ b/src/utils/ImportCache.ts @@ -3,7 +3,7 @@ import { ScriptUrl } from "../Script/ScriptUrl"; const importCache: { [hash: string]: ScriptUrl[] } = {}; export class ImportCache { - static get(hash: string) { + static get(hash: string): ScriptUrl[] { return importCache[hash]; } diff --git a/src/utils/helpers/computeHash.ts b/src/utils/helpers/computeHash.ts index a6c9f45af..f7c1435bb 100644 --- a/src/utils/helpers/computeHash.ts +++ b/src/utils/helpers/computeHash.ts @@ -6,7 +6,7 @@ import { sha256 } from "js-sha256"; * @returns The SHA-256 hash in hex */ export function computeHash(message: string): string { - var hash = sha256.create(); + const hash = sha256.create(); hash.update(message); return hash.hex(); }