Merge pull request #2172 from theit8514/script-caching

Cache the blobs generated by scripts
This commit is contained in:
hydroflame 2021-12-29 11:53:30 -05:00 committed by GitHub
commit 5c5cc9708d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 136 additions and 20 deletions

11
package-lock.json generated

@ -34,6 +34,7 @@
"file-saver": "^1.3.8",
"fs": "^0.0.1-security",
"jquery": "^3.5.0",
"js-sha256": "^0.9.0",
"jszip": "^3.7.0",
"material-ui-color": "^1.2.0",
"monaco-editor": "^0.27.0",
@ -13699,6 +13700,11 @@
"resolved": "https://registry.npmjs.org/jquery/-/jquery-3.6.0.tgz",
"integrity": "sha512-JVzAR/AjBvVt2BmYhxRCSYysDsPcssdmTFnzyLEts9qNwmjmu4JTAMYubEfwVOSwpQ1I1sKKFcxhZCI2buerfw=="
},
"node_modules/js-sha256": {
"version": "0.9.0",
"resolved": "https://registry.npmjs.org/js-sha256/-/js-sha256-0.9.0.tgz",
"integrity": "sha512-sga3MHh9sgQN2+pJ9VYZ+1LPwXOxuBJBA5nrR5/ofPfuiJBE2hnjsaN8se8JznOmGLN2p49Pe5U/ttafcs/apA=="
},
"node_modules/js-tokens": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
@ -32343,6 +32349,11 @@
"resolved": "https://registry.npmjs.org/jquery/-/jquery-3.6.0.tgz",
"integrity": "sha512-JVzAR/AjBvVt2BmYhxRCSYysDsPcssdmTFnzyLEts9qNwmjmu4JTAMYubEfwVOSwpQ1I1sKKFcxhZCI2buerfw=="
},
"js-sha256": {
"version": "0.9.0",
"resolved": "https://registry.npmjs.org/js-sha256/-/js-sha256-0.9.0.tgz",
"integrity": "sha512-sga3MHh9sgQN2+pJ9VYZ+1LPwXOxuBJBA5nrR5/ofPfuiJBE2hnjsaN8se8JznOmGLN2p49Pe5U/ttafcs/apA=="
},
"js-tokens": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",

@ -36,6 +36,7 @@
"file-saver": "^1.3.8",
"fs": "^0.0.1-security",
"jquery": "^3.5.0",
"js-sha256": "^0.9.0",
"jszip": "^3.7.0",
"material-ui-color": "^1.2.0",
"material-ui-popup-state": "^1.5.3",

@ -1,7 +1,6 @@
import { isString } from "./utils/helpers/isString";
import { GetServer } from "./Server/AllServers";
import { WorkerScript } from "./Netscript/WorkerScript";
import { BlobsMap } from "./NetscriptJSEvaluator";
export function netscriptDelay(time: number, workerScript: WorkerScript): Promise<void> {
return new Promise(function (resolve) {
@ -22,8 +21,8 @@ export function makeRuntimeRejectMsg(workerScript: WorkerScript, msg: string): s
throw new Error(`WorkerScript constructed with invalid server ip: ${workerScript.hostname}`);
}
for (const url in BlobsMap) {
msg = msg.replace(new RegExp(url, "g"), BlobsMap[url]);
for (const scriptUrl of workerScript.scriptRef.dependencies) {
msg = msg.replace(new RegExp(scriptUrl.url, "g"), scriptUrl.filename);
}
return "|DELIMITER|" + server.hostname + "|DELIMITER|" + workerScript.name + "|DELIMITER|" + msg;

@ -2,10 +2,11 @@ import { makeRuntimeRejectMsg } from "./NetscriptEvaluator";
import { ScriptUrl } from "./Script/ScriptUrl";
import { WorkerScript } from "./Netscript/WorkerScript";
import { Script } from "./Script/Script";
import { computeHash } from "./utils/helpers/computeHash";
import { BlobCache } from "./utils/BlobCache";
import { ImportCache } from "./utils/ImportCache";
import { areImportsEquals } from "./Terminal/DirectoryHelpers";
export const BlobsMap: { [key: string]: string } = {};
// Makes a blob that contains the code of a given script.
function makeScriptBlob(code: string): Blob {
return new Blob([code], { type: "text/javascript" });
@ -22,13 +23,22 @@ export async function compile(script: Script, scripts: Script[]): Promise<void>
// by placing it inside an eval call.
await script.updateRamUsage(scripts);
const uurls = _getScriptUrls(script, scripts, []);
if (script.url) {
URL.revokeObjectURL(script.url); // remove the old reference.
delete BlobsMap[script.url];
const url = uurls[uurls.length - 1].url;
if (script.url && script.url !== url) {
// Thoughts: Should we be revoking any URLs here?
// If a script is modified repeatedly between two states,
// we could reuse the blob at a later time.
// BlobCache.removeByValue(script.url);
// URL.revokeObjectURL(script.url);
// if (script.dependencies.length > 0) {
// script.dependencies.forEach((dep) => {
// removeBlobFromCache(dep.url);
// URL.revokeObjectURL(dep.url);
// });
// }
}
if (script.dependencies.length > 0) script.dependencies.forEach((dep) => URL.revokeObjectURL(dep.url));
script.url = uurls[uurls.length - 1].url;
script.module = new Promise((resolve) => resolve(eval("import(uurls[uurls.length - 1].url)")));
script.url = url;
script.module = new Promise((resolve) => resolve(eval("import(url)")));
script.dependencies = uurls;
}
@ -124,12 +134,21 @@ function _getScriptUrls(script: Script, scripts: Script[], seen: Script[]): Scri
// Find the corresponding script.
const [importedScript] = scripts.filter((s) => areImportsEquals(s.filename, filename));
// Try to get a URL for the requested script and its dependencies.
const urls = _getScriptUrls(importedScript, scripts, seen);
// Check to see if the urls for this script are stored in the cache by the hash value.
let urls = ImportCache.get(importedScript.hash());
// If we don't have it in the cache, then we need to generate the urls for it.
if (!urls) {
// Try to get a URL for the requested script and its dependencies.
urls = _getScriptUrls(importedScript, scripts, seen);
}
// The top url in the stack is the replacement import file for this script.
urlStack.push(...urls);
return [prefix, urls[urls.length - 1].url, suffix].join("");
const blob = urls[urls.length - 1].url;
ImportCache.store(importedScript.hash(), urls);
// Replace the blob inside the import statement.
return [prefix, blob, suffix].join("");
},
);
@ -137,11 +156,19 @@ function _getScriptUrls(script: Script, scripts: Script[], seen: Script[]): Scri
// accidental calls to window.print() do not bring up the "print screen" dialog
transformedCode += `\n\nfunction print() {throw new Error("Invalid call to window.print(). Did you mean to use Netscript's print()?");}`;
// If we successfully transformed the code, create a blob url for it and
// push that URL onto the top of the stack.
const su = new ScriptUrl(script.filename, URL.createObjectURL(makeScriptBlob(transformedCode)));
urlStack.push(su);
BlobsMap[su.url] = su.filename;
// If we successfully transformed the code, create a blob url for it
// Compute the hash for the transformed code
const transformedHash = computeHash(transformedCode);
// Check to see if this transformed hash is in our cache
let blob = BlobCache.get(transformedHash);
if (!blob) {
blob = URL.createObjectURL(makeScriptBlob(transformedCode));
}
// Store this blob in the cache. Any script that transforms the same
// (e.g. same scripts on server, same hash value, etc) can use this blob url.
BlobCache.store(transformedHash, blob);
// Push the blob URL onto the top of the stack.
urlStack.push(new ScriptUrl(script.filename, blob));
return urlStack;
} catch (err) {
// If there is an error, we need to clean up the URLs.

@ -3,6 +3,7 @@
* A Script can have multiple active instances
*/
import { Script } from "./Script";
import { ScriptUrl } from "./ScriptUrl";
import { Settings } from "../Settings/Settings";
import { IMap } from "../types";
import { Terminal } from "../Terminal";
@ -58,6 +59,9 @@ export class RunningScript {
// Number of threads that this script is running with
threads = 1;
// Script urls for the current running script for translating urls back to file names in errors
dependencies: ScriptUrl[] = [];
constructor(script: Script | null = null, args: any[] = []) {
if (script == null) {
return;
@ -66,6 +70,7 @@ export class RunningScript {
this.args = args;
this.server = script.server;
this.ramUsage = script.ramUsage;
this.dependencies = script.dependencies;
}
log(txt: string): void {

@ -9,6 +9,7 @@ import { ScriptUrl } from "./ScriptUrl";
import { Generic_fromJSON, Generic_toJSON, Reviver } from "../utils/JSONReviver";
import { roundToTwo } from "../utils/helpers/roundToTwo";
import { computeHash } from "../utils/helpers/computeHash";
let globalModuleSequenceNumber = 0;
@ -40,6 +41,9 @@ export class Script {
// hostname of server that this script is on.
server = "";
// sha256 hash of the code in the Script. Do not access directly.
_hash = "";
constructor(fn = "", code = "", server = "", otherScripts: Script[] = []) {
this.filename = fn;
this.code = code;
@ -47,8 +51,10 @@ export class Script {
this.server = server; // hostname of server this script is on
this.module = "";
this.moduleSequenceNumber = ++globalModuleSequenceNumber;
this._hash = "";
if (this.code !== "") {
this.updateRamUsage(otherScripts);
this.rehash();
}
}
@ -84,6 +90,24 @@ export class Script {
markUpdated(): void {
this.module = "";
this.moduleSequenceNumber = ++globalModuleSequenceNumber;
this.rehash();
}
/**
* Force update of the computed hash based on the source code.
*/
rehash(): void {
this._hash = computeHash(this.code);
}
/**
* If the hash is not computed, computes the hash. Otherwise return the computed hash.
* @returns the computed hash of the script
*/
hash(): string {
if (!this._hash)
this.rehash();
return this._hash;
}
/**
@ -125,7 +149,12 @@ export class Script {
// Initializes a Script Object from a JSON save state
// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
static fromJSON(value: any): Script {
return Generic_fromJSON(Script, value.data);
const s = Generic_fromJSON(Script, value.data);
// Force the url to blank from the save data. Urls are not valid outside the current browser page load.
s.url = "";
// Rehash the code to ensure that hash is set properly.
s.rehash();
return s;
}
}

18
src/utils/BlobCache.ts Normal file

@ -0,0 +1,18 @@
const blobCache: { [hash: string]: string } = {};
export class BlobCache {
static get(hash: string): string {
return blobCache[hash];
}
static store(hash: string, value: string): void {
if (blobCache[hash]) return;
blobCache[hash] = value;
}
static removeByValue(value: string): void {
const keys = Object.keys(blobCache).filter((key) => blobCache[key] === value);
keys.forEach((key) => delete blobCache[key]);
}
}

14
src/utils/ImportCache.ts Normal file

@ -0,0 +1,14 @@
import { ScriptUrl } from "../Script/ScriptUrl";
const importCache: { [hash: string]: ScriptUrl[] } = {};
export class ImportCache {
static get(hash: string): ScriptUrl[] {
return importCache[hash];
}
static store(hash: string, value: ScriptUrl[]): void {
if (importCache[hash]) return;
importCache[hash] = value;
}
}

@ -0,0 +1,12 @@
import { sha256 } from "js-sha256";
/**
* Computes a SHA-256 hash of a string synchronously
* @param message The input string
* @returns The SHA-256 hash in hex
*/
export function computeHash(message: string): string {
const hash = sha256.create();
hash.update(message);
return hash.hex();
}