Merge pull request #2172 from theit8514/script-caching

Cache the blobs generated by scripts

package-lock.json

@@ -34,6 +34,7 @@
         "file-saver": "^1.3.8",
         "fs": "^0.0.1-security",
         "jquery": "^3.5.0",
+        "js-sha256": "^0.9.0",
         "jszip": "^3.7.0",
         "material-ui-color": "^1.2.0",
         "monaco-editor": "^0.27.0",
@@ -13699,6 +13700,11 @@
       "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.6.0.tgz",
       "integrity": "sha512-JVzAR/AjBvVt2BmYhxRCSYysDsPcssdmTFnzyLEts9qNwmjmu4JTAMYubEfwVOSwpQ1I1sKKFcxhZCI2buerfw=="
     },
+    "node_modules/js-sha256": {
+      "version": "0.9.0",
+      "resolved": "https://registry.npmjs.org/js-sha256/-/js-sha256-0.9.0.tgz",
+      "integrity": "sha512-sga3MHh9sgQN2+pJ9VYZ+1LPwXOxuBJBA5nrR5/ofPfuiJBE2hnjsaN8se8JznOmGLN2p49Pe5U/ttafcs/apA=="
+    },
     "node_modules/js-tokens": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
@@ -32343,6 +32349,11 @@
       "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.6.0.tgz",
       "integrity": "sha512-JVzAR/AjBvVt2BmYhxRCSYysDsPcssdmTFnzyLEts9qNwmjmu4JTAMYubEfwVOSwpQ1I1sKKFcxhZCI2buerfw=="
     },
+    "js-sha256": {
+      "version": "0.9.0",
+      "resolved": "https://registry.npmjs.org/js-sha256/-/js-sha256-0.9.0.tgz",
+      "integrity": "sha512-sga3MHh9sgQN2+pJ9VYZ+1LPwXOxuBJBA5nrR5/ofPfuiJBE2hnjsaN8se8JznOmGLN2p49Pe5U/ttafcs/apA=="
+    },
     "js-tokens": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",

package.json

@@ -36,6 +36,7 @@
     "file-saver": "^1.3.8",
     "fs": "^0.0.1-security",
     "jquery": "^3.5.0",
+    "js-sha256": "^0.9.0",
     "jszip": "^3.7.0",
     "material-ui-color": "^1.2.0",
     "material-ui-popup-state": "^1.5.3",

src/NetscriptEvaluator.ts

@@ -1,7 +1,6 @@
 import { isString } from "./utils/helpers/isString";
 import { GetServer } from "./Server/AllServers";
 import { WorkerScript } from "./Netscript/WorkerScript";
-import { BlobsMap } from "./NetscriptJSEvaluator";
 
 export function netscriptDelay(time: number, workerScript: WorkerScript): Promise<void> {
   return new Promise(function (resolve) {
@@ -22,8 +21,8 @@ export function makeRuntimeRejectMsg(workerScript: WorkerScript, msg: string): s
     throw new Error(`WorkerScript constructed with invalid server ip: ${workerScript.hostname}`);
   }
 
-  for (const url in BlobsMap) {
-    msg = msg.replace(new RegExp(url, "g"), BlobsMap[url]);
+  for (const scriptUrl of workerScript.scriptRef.dependencies) {
+    msg = msg.replace(new RegExp(scriptUrl.url, "g"), scriptUrl.filename);
   }
 
   return "|DELIMITER|" + server.hostname + "|DELIMITER|" + workerScript.name + "|DELIMITER|" + msg;

src/NetscriptJSEvaluator.ts

@@ -2,10 +2,11 @@ import { makeRuntimeRejectMsg } from "./NetscriptEvaluator";
 import { ScriptUrl } from "./Script/ScriptUrl";
 import { WorkerScript } from "./Netscript/WorkerScript";
 import { Script } from "./Script/Script";
+import { computeHash } from "./utils/helpers/computeHash";
+import { BlobCache } from "./utils/BlobCache";
+import { ImportCache } from "./utils/ImportCache";
 import { areImportsEquals } from "./Terminal/DirectoryHelpers";
 
-export const BlobsMap: { [key: string]: string } = {};
-
 // Makes a blob that contains the code of a given script.
 function makeScriptBlob(code: string): Blob {
   return new Blob([code], { type: "text/javascript" });
@@ -22,13 +23,22 @@ export async function compile(script: Script, scripts: Script[]): Promise<void>
   // by placing it inside an eval call.
   await script.updateRamUsage(scripts);
   const uurls = _getScriptUrls(script, scripts, []);
-  if (script.url) {
-    URL.revokeObjectURL(script.url); // remove the old reference.
-    delete BlobsMap[script.url];
+  const url = uurls[uurls.length - 1].url;
+  if (script.url && script.url !== url) {
+    // Thoughts: Should we be revoking any URLs here?
+    // If a script is modified repeatedly between two states,
+    // we could reuse the blob at a later time.
+    // BlobCache.removeByValue(script.url);
+    // URL.revokeObjectURL(script.url);
+    // if (script.dependencies.length > 0) {
+    //   script.dependencies.forEach((dep) => {
+    //     removeBlobFromCache(dep.url);
+    //     URL.revokeObjectURL(dep.url);
+    //   });
+    // }
   }
-  if (script.dependencies.length > 0) script.dependencies.forEach((dep) => URL.revokeObjectURL(dep.url));
-  script.url = uurls[uurls.length - 1].url;
-  script.module = new Promise((resolve) => resolve(eval("import(uurls[uurls.length - 1].url)")));
+  script.url = url;
+  script.module = new Promise((resolve) => resolve(eval("import(url)")));
   script.dependencies = uurls;
 }
 
@@ -124,12 +134,21 @@ function _getScriptUrls(script: Script, scripts: Script[], seen: Script[]): Scri
         // Find the corresponding script.
         const [importedScript] = scripts.filter((s) => areImportsEquals(s.filename, filename));
 
+        // Check to see if the urls for this script are stored in the cache by the hash value.
+        let urls = ImportCache.get(importedScript.hash());
+        // If we don't have it in the cache, then we need to generate the urls for it.
+        if (!urls) {
           // Try to get a URL for the requested script and its dependencies.
-        const urls = _getScriptUrls(importedScript, scripts, seen);
+          urls = _getScriptUrls(importedScript, scripts, seen);
+        }
 
         // The top url in the stack is the replacement import file for this script.
         urlStack.push(...urls);
-        return [prefix, urls[urls.length - 1].url, suffix].join("");
+        const blob = urls[urls.length - 1].url;
+        ImportCache.store(importedScript.hash(), urls);
+
+        // Replace the blob inside the import statement.
+        return [prefix, blob, suffix].join("");
       },
     );
 
@@ -137,11 +156,19 @@ function _getScriptUrls(script: Script, scripts: Script[], seen: Script[]): Scri
     // accidental calls to window.print() do not bring up the "print screen" dialog
     transformedCode += `\n\nfunction print() {throw new Error("Invalid call to window.print(). Did you mean to use Netscript's print()?");}`;
 
-    // If we successfully transformed the code, create a blob url for it and
-    // push that URL onto the top of the stack.
-    const su = new ScriptUrl(script.filename, URL.createObjectURL(makeScriptBlob(transformedCode)));
-    urlStack.push(su);
-    BlobsMap[su.url] = su.filename;
+    // If we successfully transformed the code, create a blob url for it
+    // Compute the hash for the transformed code
+    const transformedHash = computeHash(transformedCode);
+    // Check to see if this transformed hash is in our cache
+    let blob = BlobCache.get(transformedHash);
+    if (!blob) {
+      blob = URL.createObjectURL(makeScriptBlob(transformedCode));
+    }
+    // Store this blob in the cache. Any script that transforms the same
+    // (e.g. same scripts on server, same hash value, etc) can use this blob url.
+    BlobCache.store(transformedHash, blob);
+    // Push the blob URL onto the top of the stack.
+    urlStack.push(new ScriptUrl(script.filename, blob));
     return urlStack;
   } catch (err) {
     // If there is an error, we need to clean up the URLs.

src/Script/RunningScript.ts

@@ -3,6 +3,7 @@
  * A Script can have multiple active instances
  */
 import { Script } from "./Script";
+import { ScriptUrl } from "./ScriptUrl";
 import { Settings } from "../Settings/Settings";
 import { IMap } from "../types";
 import { Terminal } from "../Terminal";
@@ -58,6 +59,9 @@ export class RunningScript {
   // Number of threads that this script is running with
   threads = 1;
 
+  // Script urls for the current running script for translating urls back to file names in errors
+  dependencies: ScriptUrl[] = [];
+
   constructor(script: Script | null = null, args: any[] = []) {
     if (script == null) {
       return;
@@ -66,6 +70,7 @@ export class RunningScript {
     this.args = args;
     this.server = script.server;
     this.ramUsage = script.ramUsage;
+    this.dependencies = script.dependencies;
   }
 
   log(txt: string): void {

src/Script/Script.ts

@@ -9,6 +9,7 @@ import { ScriptUrl } from "./ScriptUrl";
 
 import { Generic_fromJSON, Generic_toJSON, Reviver } from "../utils/JSONReviver";
 import { roundToTwo } from "../utils/helpers/roundToTwo";
+import { computeHash } from "../utils/helpers/computeHash";
 
 let globalModuleSequenceNumber = 0;
 
@@ -40,6 +41,9 @@ export class Script {
   // hostname of server that this script is on.
   server = "";
 
+  // sha256 hash of the code in the Script. Do not access directly.
+  _hash = "";
+
   constructor(fn = "", code = "", server = "", otherScripts: Script[] = []) {
     this.filename = fn;
     this.code = code;
@@ -47,8 +51,10 @@ export class Script {
     this.server = server; // hostname of server this script is on
     this.module = "";
     this.moduleSequenceNumber = ++globalModuleSequenceNumber;
+    this._hash = "";
     if (this.code !== "") {
       this.updateRamUsage(otherScripts);
+      this.rehash();
     }
   }
 
@@ -84,6 +90,24 @@ export class Script {
   markUpdated(): void {
     this.module = "";
     this.moduleSequenceNumber = ++globalModuleSequenceNumber;
+    this.rehash();
+  }
+
+  /**
+   * Force update of the computed hash based on the source code.
+   */
+  rehash(): void {
+    this._hash = computeHash(this.code);
+  }
+
+  /**
+   * If the hash is not computed, computes the hash. Otherwise return the computed hash.
+   * @returns the computed hash of the script
+   */
+  hash(): string {
+    if (!this._hash)
+      this.rehash();
+    return this._hash;
   }
 
   /**
@@ -125,7 +149,12 @@ export class Script {
   // Initializes a Script Object from a JSON save state
   // eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
   static fromJSON(value: any): Script {
-    return Generic_fromJSON(Script, value.data);
+    const s = Generic_fromJSON(Script, value.data);
+    // Force the url to blank from the save data. Urls are not valid outside the current browser page load.
+    s.url = "";
+    // Rehash the code to ensure that hash is set properly.
+    s.rehash();
+    return s;
   }
 }
 

src/utils/BlobCache.ts

@@ -0,0 +1,18 @@
+
+const blobCache: { [hash: string]: string } = {};
+
+export class BlobCache {
+  static get(hash: string): string {
+    return blobCache[hash];
+  }
+
+  static store(hash: string, value: string): void {
+    if (blobCache[hash]) return;
+    blobCache[hash] = value;
+  }
+
+  static removeByValue(value: string): void {
+    const keys = Object.keys(blobCache).filter((key) => blobCache[key] === value);
+    keys.forEach((key) => delete blobCache[key]);
+  }
+}

src/utils/ImportCache.ts

@@ -0,0 +1,14 @@
+import { ScriptUrl } from "../Script/ScriptUrl";
+
+const importCache: { [hash: string]: ScriptUrl[] } = {};
+
+export class ImportCache {
+  static get(hash: string): ScriptUrl[] {
+    return importCache[hash];
+  }
+
+  static store(hash: string, value: ScriptUrl[]): void {
+    if (importCache[hash]) return;
+    importCache[hash] = value;
+  }
+}

src/utils/helpers/computeHash.ts

@@ -0,0 +1,12 @@
+import { sha256 } from "js-sha256";
+
+/**
+ * Computes a SHA-256 hash of a string synchronously
+ * @param message The input string
+ * @returns The SHA-256 hash in hex
+ */
+export function computeHash(message: string): string {
+  const hash = sha256.create();
+  hash.update(message);
+  return hash.hex();
+}