Merge pull request #3985 from Snarling/synchronize

NETSCRIPT: ns.scp and ns.write are now synchronous + fix exec race condition
This commit is contained in:
hydroflame 2022-08-23 12:25:24 -03:00 committed by GitHub
commit 849046df3a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 142 additions and 222 deletions

@ -829,142 +829,106 @@ const base: InternalAPI<NS> = {
},
scp:
(ctx: NetscriptContext) =>
async (
_scriptname: unknown,
_destination: unknown,
_source: unknown = ctx.workerScript.hostname,
): Promise<boolean> => {
(_files: unknown, _destination: unknown, _source: unknown = ctx.workerScript.hostname): boolean => {
const destination = helpers.string(ctx, "destination", _destination);
const source = helpers.string(ctx, "source", _source);
if (Array.isArray(_scriptname)) {
// Recursively call scp on all elements of array
const scripts: string[] = _scriptname;
if (scripts.length === 0) {
throw helpers.makeRuntimeErrorMsg(ctx, "No scripts to copy");
}
let res = true;
await Promise.all(
scripts.map(async function (script) {
if (!(await NetscriptFunctions(ctx.workerScript).scp(script, destination, source))) {
res = false;
}
}),
);
return Promise.resolve(res);
}
const scriptName = helpers.string(ctx, "scriptName", _scriptname);
// Invalid file type
if (!isValidFilePath(scriptName)) {
throw helpers.makeRuntimeErrorMsg(ctx, `Invalid filename: '${scriptName}'`);
}
// Invalid file name
if (!scriptName.endsWith(".lit") && !isScriptFilename(scriptName) && !scriptName.endsWith("txt")) {
throw helpers.makeRuntimeErrorMsg(ctx, "Only works for scripts, .lit and .txt files");
}
const destServer = helpers.getServer(ctx, destination);
const currServ = helpers.getServer(ctx, source);
const sourceServ = helpers.getServer(ctx, source);
const files = Array.isArray(_files) ? _files : [_files];
// Scp for lit files
if (scriptName.endsWith(".lit")) {
let found = false;
for (let i = 0; i < currServ.messages.length; ++i) {
if (currServ.messages[i] == scriptName) {
found = true;
break;
//First loop through filenames to find all errors before moving anything.
for (const file of files) {
// Not a string
if (typeof file !== "string")
throw helpers.makeRuntimeErrorMsg(ctx, "files should be a string or an array of strings.");
// Invalid file name
if (!isValidFilePath(file)) throw helpers.makeRuntimeErrorMsg(ctx, `Invalid filename: '${file}'`);
// Invalid file type
if (!file.endsWith(".lit") && !isScriptFilename(file) && !file.endsWith("txt")) {
throw helpers.makeRuntimeErrorMsg(ctx, "Only works for scripts, .lit and .txt files.");
}
}
let noFailures = true;
//ts detects files as any[] here even though we would have thrown in the above loop if it wasn't string[]
for (const file of files as string[]) {
// Scp for lit files
if (file.endsWith(".lit")) {
const sourceMessage = sourceServ.messages.find((message) => message === file);
if (!sourceMessage) {
helpers.log(ctx, () => `File '${file}' does not exist.`);
noFailures = false;
continue;
}
}
if (!found) {
helpers.log(ctx, () => `File '${scriptName}' does not exist.`);
return Promise.resolve(false);
}
for (let i = 0; i < destServer.messages.length; ++i) {
if (destServer.messages[i] === scriptName) {
helpers.log(ctx, () => `File '${scriptName}' copied over to '${destServer?.hostname}'.`);
return Promise.resolve(true); // Already exists
const destMessage = destServer.messages.find((message) => message === file);
if (destMessage) {
helpers.log(ctx, () => `File '${file}' was already on '${destServer?.hostname}'.`);
continue;
}
}
destServer.messages.push(scriptName);
helpers.log(ctx, () => `File '${scriptName}' copied over to '${destServer?.hostname}'.`);
return Promise.resolve(true);
}
// Scp for text files
if (scriptName.endsWith(".txt")) {
let txtFile;
for (let i = 0; i < currServ.textFiles.length; ++i) {
if (currServ.textFiles[i].fn === scriptName) {
txtFile = currServ.textFiles[i];
break;
destServer.messages.push(file);
helpers.log(ctx, () => `File '${file}' copied over to '${destServer?.hostname}'.`);
continue;
}
// Scp for text files
if (file.endsWith(".txt")) {
const sourceTextFile = sourceServ.textFiles.find((textFile) => textFile.fn === file);
if (!sourceTextFile) {
helpers.log(ctx, () => `File '${file}' does not exist.`);
noFailures = false;
continue;
}
}
if (txtFile === undefined) {
helpers.log(ctx, () => `File '${scriptName}' does not exist.`);
return Promise.resolve(false);
}
for (let i = 0; i < destServer.textFiles.length; ++i) {
if (destServer.textFiles[i].fn === scriptName) {
// Overwrite
destServer.textFiles[i].text = txtFile.text;
helpers.log(ctx, () => `File '${scriptName}' copied over to '${destServer?.hostname}'.`);
return Promise.resolve(true);
const destTextFile = destServer.textFiles.find((textFile) => textFile.fn === file);
if (destTextFile) {
destTextFile.text = sourceTextFile.text;
helpers.log(ctx, () => `File '${file}' overwritten on '${destServer?.hostname}'.`);
continue;
}
const newFile = new TextFile(sourceTextFile.fn, sourceTextFile.text);
destServer.textFiles.push(newFile);
helpers.log(ctx, () => `File '${file}' copied over to '${destServer?.hostname}'.`);
continue;
}
const newFile = new TextFile(txtFile.fn, txtFile.text);
destServer.textFiles.push(newFile);
helpers.log(ctx, () => `File '${scriptName}' copied over to '${destServer?.hostname}'.`);
return Promise.resolve(true);
// Scp for script files
const sourceScript = sourceServ.scripts.find((script) => script.filename === file);
if (!sourceScript) {
helpers.log(ctx, () => `File '${file}' does not exist.`);
noFailures = false;
continue;
}
// Overwrite script if it already exists
const destScript = destServer.scripts.find((script) => script.filename === file);
if (destScript) {
if (destScript.code === sourceScript.code) {
helpers.log(ctx, () => `Identical file '${file}' was already on '${destServer?.hostname}'`);
continue;
}
destScript.code = sourceScript.code;
destScript.ramUsage = destScript.ramUsage;
destScript.markUpdated();
helpers.log(ctx, () => `WARNING: File '${file}' overwritten on '${destServer?.hostname}'`);
continue;
}
// Create new script if it does not already exist
const newScript = new Script(Player, file);
newScript.code = sourceScript.code;
newScript.ramUsage = sourceScript.ramUsage;
newScript.server = destServer.hostname;
destServer.scripts.push(newScript);
helpers.log(ctx, () => `File '${file}' copied over to '${destServer?.hostname}'.`);
newScript.updateRamUsage(Player, destServer.scripts);
}
// Scp for script files
let sourceScript = null;
for (let i = 0; i < currServ.scripts.length; ++i) {
if (scriptName == currServ.scripts[i].filename) {
sourceScript = currServ.scripts[i];
break;
}
}
if (sourceScript == null) {
helpers.log(ctx, () => `File '${scriptName}' does not exist.`);
return Promise.resolve(false);
}
// Overwrite script if it already exists
for (let i = 0; i < destServer.scripts.length; ++i) {
if (scriptName == destServer.scripts[i].filename) {
helpers.log(ctx, () => `WARNING: File '${scriptName}' overwritten on '${destServer?.hostname}'`);
const oldScript = destServer.scripts[i];
// If it's the exact same file don't actually perform the
// copy to avoid recompiling uselessly. Players tend to scp
// liberally.
if (oldScript.code === sourceScript.code) return Promise.resolve(true);
oldScript.code = sourceScript.code;
oldScript.ramUsage = sourceScript.ramUsage;
oldScript.markUpdated();
return Promise.resolve(true);
}
}
// Create new script if it does not already exist
const newScript = new Script(Player, scriptName);
newScript.code = sourceScript.code;
newScript.ramUsage = sourceScript.ramUsage;
newScript.server = destServer.hostname;
destServer.scripts.push(newScript);
helpers.log(ctx, () => `File '${scriptName}' copied over to '${destServer?.hostname}'.`);
return new Promise((resolve) => {
if (destServer === null) {
resolve(false);
return;
}
newScript.updateRamUsage(Player, destServer.scripts).then(() => resolve(true));
});
return noFailures;
},
ls:
(ctx: NetscriptContext) =>
@ -1501,34 +1465,19 @@ const base: InternalAPI<NS> = {
},
write:
(ctx: NetscriptContext) =>
(_port: unknown, data: unknown = "", _mode: unknown = "a"): Promise<void> => {
(_port: unknown, _data: unknown = "", _mode: unknown = "a"): void => {
const port = helpers.string(ctx, "port", _port);
const data = helpers.string(ctx, "data", _data);
const mode = helpers.string(ctx, "mode", _mode);
if (isString(port)) {
// Write to script or text file
let fn = port;
if (!isValidFilePath(fn)) {
throw helpers.makeRuntimeErrorMsg(ctx, `Invalid filepath: ${fn}`);
}
if (!isValidFilePath(fn)) throw helpers.makeRuntimeErrorMsg(ctx, `Invalid filepath: ${fn}`);
if (fn.lastIndexOf("/") === 0) {
fn = removeLeadingSlash(fn);
}
if (fn.lastIndexOf("/") === 0) fn = removeLeadingSlash(fn);
// Coerce 'data' to be a string
try {
data = String(data);
} catch (e: unknown) {
throw helpers.makeRuntimeErrorMsg(
ctx,
`Invalid data (${String(e)}). Data being written must be convertible to a string`,
);
}
const server = helpers.getServer(ctx, ctx.workerScript.hostname);
const server = ctx.workerScript.getServer();
if (server == null) {
throw helpers.makeRuntimeErrorMsg(ctx, "Error getting Server. This is a bug. Report to dev.");
}
if (isScriptFilename(fn)) {
// Write to script
let script = ctx.workerScript.getScriptOnServer(fn, server);
@ -1545,7 +1494,7 @@ const base: InternalAPI<NS> = {
const txtFile = getTextFile(fn, server);
if (txtFile == null) {
createTextFile(fn, String(data), server);
return Promise.resolve();
return;
}
if (mode === "w") {
txtFile.write(String(data));
@ -1553,7 +1502,7 @@ const base: InternalAPI<NS> = {
txtFile.append(String(data));
}
}
return Promise.resolve();
return;
} else {
throw helpers.makeRuntimeErrorMsg(ctx, `Invalid argument: ${port}`);
}

@ -11,6 +11,7 @@ import { WorkerScript } from "./Netscript/WorkerScript";
import { Script } from "./Script/Script";
import { areImportsEquals } from "./Terminal/DirectoryHelpers";
import { IPlayer } from "./PersonObjects/IPlayer";
import { ScriptModule } from "./Script/ScriptModule";
// Acorn type def is straight up incomplete so we have to fill with our own.
export type Node = any;
@ -20,8 +21,23 @@ function makeScriptBlob(code: string): Blob {
return new Blob([code], { type: "text/javascript" });
}
export async function compile(player: IPlayer, script: Script, scripts: Script[]): Promise<void> {
if (!shouldCompile(script, scripts)) return;
export async function compile(player: IPlayer, script: Script, scripts: Script[]): Promise<ScriptModule> {
//!shouldCompile ensures that script.module is non-null, hence the "as".
if (!shouldCompile(script, scripts)) return script.module as Promise<ScriptModule>;
script.queueCompile = true;
//If we're already in the middle of compiling (script.module has not resolved yet), wait for the previous compilation to finish
//If script.module is null, this does nothing.
await script.module;
//If multiple compiles were called on the same script before a compilation could be completed this ensures only one complilation is actually performed.
if (!script.queueCompile) return script.module as Promise<ScriptModule>;
script.queueCompile = false;
script.updateRamUsage(player, scripts);
const uurls = _getScriptUrls(script, scripts, []);
const url = uurls[uurls.length - 1].url;
if (script.url && script.url !== url) URL.revokeObjectURL(script.url);
if (script.dependencies.length > 0) script.dependencies.forEach((dep) => URL.revokeObjectURL(dep.url));
script.url = uurls[uurls.length - 1].url;
// The URL at the top is the one we want to import. It will
// recursively import all the other modules in the urlStack.
//
@ -29,27 +45,9 @@ export async function compile(player: IPlayer, script: Script, scripts: Script[]
// but not really behaves like import. Particularly, it cannot
// load fully dynamic content. So we hide the import from webpack
// by placing it inside an eval call.
await script.updateRamUsage(player, scripts);
const uurls = _getScriptUrls(script, scripts, []);
const url = uurls[uurls.length - 1].url;
if (script.url && script.url !== url) {
URL.revokeObjectURL(script.url);
// Thoughts: Should we be revoking any URLs here?
// If a script is modified repeatedly between two states,
// we could reuse the blob at a later time.
// BlobCache.removeByValue(script.url);
// URL.revokeObjectURL(script.url);
// if (script.dependencies.length > 0) {
// script.dependencies.forEach((dep) => {
// removeBlobFromCache(dep.url);
// URL.revokeObjectURL(dep.url);
// });
// }
}
if (script.dependencies.length > 0) script.dependencies.forEach((dep) => URL.revokeObjectURL(dep.url));
script.url = uurls[uurls.length - 1].url;
script.module = new Promise((resolve) => resolve(eval("import(uurls[uurls.length - 1].url)")));
script.dependencies = uurls;
return script.module;
}
// Begin executing a user JS script, and return a promise that resolves
@ -67,9 +65,8 @@ export async function executeJSScript(
): Promise<void> {
const script = workerScript.getScript();
if (script === null) throw new Error("script is null");
await compile(player, script, scripts);
const loadedModule = await compile(player, script, scripts);
workerScript.ramUsage = script.ramUsage;
const loadedModule = await script.module;
const ns = workerScript.env.vars;
@ -113,7 +110,11 @@ function isDependencyOutOfDate(filename: string, scripts: Script[], scriptModule
*/
function shouldCompile(script: Script, scripts: Script[]): boolean {
if (!script.module) return true;
return script.dependencies.some((dep) => isDependencyOutOfDate(dep.filename, scripts, script.moduleSequenceNumber));
if (script.dependencies.some((dep) => isDependencyOutOfDate(dep.filename, scripts, script.moduleSequenceNumber))) {
script.module = null;
return true;
}
return false;
}
// Gets a stack of blob urls, the top/right-most element being

@ -44,7 +44,7 @@ const memCheckGlobalKey = ".__GLOBAL__";
* @param {WorkerScript} workerScript - Object containing RAM costs of Netscript functions. Also used to
* keep track of what functions have/havent been accounted for
*/
async function parseOnlyRamCalculate(player: IPlayer, otherScripts: Script[], code: string): Promise<RamCalculation> {
function parseOnlyRamCalculate(player: IPlayer, otherScripts: Script[], code: string): RamCalculation {
try {
/**
* Maps dependent identifiers to their dependencies.
@ -88,47 +88,22 @@ async function parseOnlyRamCalculate(player: IPlayer, otherScripts: Script[], co
while (parseQueue.length > 0) {
const nextModule = parseQueue.shift();
if (nextModule === undefined) throw new Error("nextModule should not be undefined");
if (nextModule.startsWith("https://") || nextModule.startsWith("http://")) continue;
// Additional modules can either be imported from the web (in which case we use
// a dynamic import), or from other in-game scripts
let code;
if (nextModule.startsWith("https://") || nextModule.startsWith("http://")) {
try {
// eslint-disable-next-line no-await-in-loop
const module = await eval("import(nextModule)");
code = "";
for (const prop in module) {
if (typeof module[prop] === "function") {
code += module[prop].toString() + ";\n";
}
}
} catch (e) {
console.error(`Error dynamically importing module from ${nextModule} for RAM calculations: ${e}`);
return { cost: RamCalculationErrorCode.URLImportError };
let script = null;
const fn = nextModule.startsWith("./") ? nextModule.slice(2) : nextModule;
for (const s of otherScripts) {
if (areImportsEquals(s.filename, fn)) {
script = s;
break;
}
} else {
if (!Array.isArray(otherScripts)) {
console.warn(`parseOnlyRamCalculate() not called with array of scripts`);
return { cost: RamCalculationErrorCode.ImportError };
}
let script = null;
const fn = nextModule.startsWith("./") ? nextModule.slice(2) : nextModule;
for (const s of otherScripts) {
if (areImportsEquals(s.filename, fn)) {
script = s;
break;
}
}
if (script == null) {
return { cost: RamCalculationErrorCode.ImportError }; // No such script on the server
}
code = script.code;
}
parseCode(code, nextModule);
if (script == null) {
return { cost: RamCalculationErrorCode.ImportError }; // No such script on the server
}
parseCode(script.code, nextModule);
}
// Finally, walk the reference map and generate a ram cost. The initial set of keys to scan
@ -406,13 +381,9 @@ function parseOnlyCalculateDeps(code: string, currentModule: string): ParseDepsR
* @param {Script[]} otherScripts - All other scripts on the server.
* Used to account for imported scripts
*/
export async function calculateRamUsage(
player: IPlayer,
codeCopy: string,
otherScripts: Script[],
): Promise<RamCalculation> {
export function calculateRamUsage(player: IPlayer, codeCopy: string, otherScripts: Script[]): RamCalculation {
try {
return await parseOnlyRamCalculate(player, otherScripts, codeCopy);
return parseOnlyRamCalculate(player, otherScripts, codeCopy);
} catch (e) {
console.error(`Failed to parse script for RAM calculations:`);
console.error(e);

@ -46,15 +46,16 @@ export class Script {
ramUsage = 0;
ramUsageEntries?: RamUsageEntry[];
// Used to deconflict multiple simultaneous compilations.
queueCompile = false;
// hostname of server that this script is on.
server = "";
constructor(player: IPlayer | null = null, fn = "", code = "", server = "", otherScripts: Script[] = []) {
this.filename = fn;
this.code = code;
this.ramUsage = 0;
this.server = server; // hostname of server this script is on
this.module = null;
this.moduleSequenceNumber = ++globalModuleSequenceNumber;
if (this.code !== "" && player !== null) {
this.updateRamUsage(player, otherScripts);
@ -105,7 +106,7 @@ export class Script {
const [dependentScript] = otherScripts.filter(
(s) => s.filename === dependent.filename && s.server == dependent.server,
);
if (dependentScript !== null) dependentScript.markUpdated();
dependentScript?.markUpdated();
}
}
@ -113,8 +114,8 @@ export class Script {
* Calculates and updates the script's RAM usage based on its code
* @param {Script[]} otherScripts - Other scripts on the server. Used to process imports
*/
async updateRamUsage(player: IPlayer, otherScripts: Script[]): Promise<void> {
const res = await calculateRamUsage(player, this.code, otherScripts);
updateRamUsage(player: IPlayer, otherScripts: Script[]): void {
const res = calculateRamUsage(player, this.code, otherScripts);
if (res.cost > 0) {
this.ramUsage = roundToTwo(res.cost);
this.ramUsageEntries = res.entries;

@ -257,7 +257,7 @@ export function Root(props: IProps): React.ReactElement {
}
setUpdatingRam(true);
const codeCopy = newCode + "";
const ramUsage = await calculateRamUsage(props.player, codeCopy, props.player.getCurrentServer().scripts);
const ramUsage = calculateRamUsage(props.player, codeCopy, props.player.getCurrentServer().scripts);
if (ramUsage.cost > 0) {
const entries = ramUsage.entries?.sort((a, b) => b.cost - a.cost) ?? [];
const entriesDisp = [];

@ -285,10 +285,8 @@ export async function determineAllPossibilitiesForTabCompletion(
return processFilepath(script.filename) === fn || script.filename === "/" + fn;
});
if (!script) return; // Doesn't exist.
if (!script.module) {
await compile(p, script, currServ.scripts);
}
const loadedModule = await script.module;
//Will return the already compiled module if recompilation not needed.
const loadedModule = await compile(p, script, currServ.scripts);
if (!loadedModule || !loadedModule.autocomplete) return; // Doesn't have an autocomplete function.
const runArgs = { "--tail": Boolean, "-t": Number };