MISC: Support compression of save data (#1162)

* Use Compression Streams API instead of jszip or other libraries.
* Remove usage of base64 in the new binary format.
* Do not convert binary data to string and back. The type of save data is SaveData, it's either string (old base64 format) or Uint8Array (new binary format).
* Proper support for interacting with electron-related code. Electron-related code assumes that save data is in the base64 format.
* Proper support for other tools (DevMenu, pretty-save.js). Full support for DevMenu will be added in a follow-up PR. Check the comments in src\DevMenu\ui\SaveFileDev.tsx for details.

electron/main.js

@@ -152,7 +152,7 @@ function setStopProcessHandler(app, window) {
       log.debug("Saving to Steam Cloud ...");
       try {
         const playerId = window.gameInfo.player.identifier;
-        await storage.pushGameSaveToSteamCloud(save, playerId);
+        await storage.pushSaveDataToSteamCloud(save, playerId);
         log.silly("Saved Game to Steam Cloud");
       } catch (error) {
         log.error(error);

electron/menu.js

@@ -103,8 +103,8 @@ function getMenu(window) {
           enabled: storage.isCloudEnabled(),
           click: async () => {
             try {
-              const saveGame = await storage.getSteamCloudSaveString();
+              const saveData = await storage.getSteamCloudSaveData();
-              await storage.pushSaveGameForImport(window, saveGame, false);
+              await storage.pushSaveGameForImport(window, saveData, false);
             } catch (error) {
               log.error(error);
               utils.writeToast(window, "Could not load from Steam Cloud", "error", 5000);
@@ -114,16 +114,6 @@ function getMenu(window) {
         {
           type: "separator",
         },
-        {
-          label: "Compress Disk Saves (.gz)",
-          type: "checkbox",
-          checked: storage.isSaveCompressionEnabled(),
-          click: (menuItem) => {
-            storage.setSaveCompressionConfig(menuItem.checked);
-            utils.writeToast(window, `${menuItem.checked ? "Enabled" : "Disabled"} Save Compression`, "info", 5000);
-            refreshMenu(window);
-          },
-        },
         {
           label: "Auto-Save to Disk",
           type: "checkbox",

electron/saveDataBinaryFormat.d.ts

@@ -0,0 +1 @@
+export declare const isBinaryFormat: (saveData: string | Uint8Array) => boolean;

electron/saveDataBinaryFormat.js

@@ -0,0 +1,13 @@
+// The 2 magic bytes of the gzip header plus the mandatory compression type of DEFLATE
+const magicBytes = [0x1f, 0x8b, 0x08];
+
+function isBinaryFormat(rawData) {
+  for (let i = 0; i < magicBytes.length; ++i) {
+    if (magicBytes[i] !== rawData[i]) {
+      return false;
+    }
+  }
+  return true;
+}
+
+module.exports = { isBinaryFormat };

electron/storage.js

@@ -11,6 +11,7 @@ const greenworks = require("./greenworks");
 const log = require("electron-log");
 const flatten = require("lodash/flatten");
 const Store = require("electron-store");
+const { isBinaryFormat } = require("./saveDataBinaryFormat");
 const store = new Store();
 
 // https://stackoverflow.com/a/69418940
@@ -86,14 +87,6 @@ function isAutosaveEnabled() {
   return store.get("autosave-enabled", true);
 }
 
-function setSaveCompressionConfig(value) {
-  store.set("save-compression-enabled", value);
-}
-
-function isSaveCompressionEnabled() {
-  return store.get("save-compression-enabled", true);
-}
-
 function setCloudEnabledConfig(value) {
   store.set("cloud-enabled", value);
 }
@@ -163,17 +156,22 @@ async function backupSteamDataToDisk(currentPlayerId) {
   const file = greenworks.getFileNameAndSize(0);
   const previousPlayerId = file.name.replace(".json.gz", "");
   if (previousPlayerId !== currentPlayerId) {
-    const backupSave = await getSteamCloudSaveString();
+    const backupSaveData = await getSteamCloudSaveData();
     const backupFile = path.join(app.getPath("userData"), "/saves/_backups", `${previousPlayerId}.json.gz`);
-    const buffer = Buffer.from(backupSave, "base64").toString("utf8");
+    await fs.writeFile(backupFile, backupSaveData, "utf8");
-    saveContent = await gzip(buffer);
-    await fs.writeFile(backupFile, saveContent, "utf8");
     log.debug(`Saved backup game to '${backupFile}`);
   }
 }
 
-async function pushGameSaveToSteamCloud(base64save, currentPlayerId) {
+/**
-  if (!isCloudEnabled) return Promise.reject("Steam Cloud is not Enabled");
+ * The name of save file is `${currentPlayerId}.json.gz`. The content of save file is weird: it's a base64 string of the
+ * binary data of compressed json save string. It's weird because the extension is .json.gz while the content is a
+ * base64 string. Check the comments in the implementation to see why it is like that.
+ */
+async function pushSaveDataToSteamCloud(saveData, currentPlayerId) {
+  if (!isCloudEnabled()) {
+    return Promise.reject("Steam Cloud is not Enabled");
+  }
 
   try {
     backupSteamDataToDisk(currentPlayerId);
@@ -183,12 +181,19 @@ async function pushGameSaveToSteamCloud(base64save, currentPlayerId) {
 
   const steamSaveName = `${currentPlayerId}.json.gz`;
 
-  // Let's decode the base64 string so GZIP is more efficient.
+  /**
-  const buffer = Buffer.from(base64save, "base64");
+   * When we push save file to Steam Cloud, we use greenworks.saveTextToFile. It seems that this method expects a string
-  const compressedBuffer = await gzip(buffer);
+   * as the file content. That is why saveData is encoded in base64 and pushed to Steam Cloud as a text file.
-  // We can't use utf8 for some reason, steamworks is unhappy.
+   *
-  const content = compressedBuffer.toString("base64");
+   * Encoding saveData in UTF-8 (with buffer.toString("utf8")) is not the proper way to convert binary data to string.
-  log.debug(`Uncompressed: ${base64save.length} bytes`);
+   * Quote from buffer's documentation: "If encoding is 'utf8' and a byte sequence in the input is not valid UTF-8, then
+   * each invalid byte is replaced with the replacement character U+FFFD.". The proper way to do it is to use
+   * String.fromCharCode or String.fromCodePoint.
+   *
+   * Instead of implementing it, the old code (encoding in base64) is used here for backward compatibility.
+   */
+  const content = saveData.toString("base64");
+  log.debug(`Uncompressed: ${saveData.length} bytes`);
   log.debug(`Compressed: ${content.length} bytes`);
   log.debug(`Saving to Steam Cloud as ${steamSaveName}`);
 
@@ -199,19 +204,22 @@ async function pushGameSaveToSteamCloud(base64save, currentPlayerId) {
   }
 }
 
-async function getSteamCloudSaveString() {
+/**
-  if (!isCloudEnabled()) return Promise.reject("Steam Cloud is not Enabled");
+ * This function processes the save file in Steam Cloud and returns the save data in the binary format.
+ */
+async function getSteamCloudSaveData() {
+  if (!isCloudEnabled()) {
+    return Promise.reject("Steam Cloud is not Enabled");
+  }
   log.debug(`Fetching Save in Steam Cloud`);
   const cloudString = await getCloudFile();
-  const gzippedBase64Buffer = Buffer.from(cloudString, "base64");
+  // Decode cloudString to get save data back.
-  const uncompressedBuffer = await gunzip(gzippedBase64Buffer);
+  const saveData = Buffer.from(cloudString, "base64");
-  const content = uncompressedBuffer.toString("base64");
+  log.debug(`SaveData: ${saveData.length} bytes`);
-  log.debug(`Compressed: ${cloudString.length} bytes`);
+  return saveData;
-  log.debug(`Uncompressed: ${content.length} bytes`);
-  return content;
 }
 
-async function saveGameToDisk(window, saveData) {
+async function saveGameToDisk(window, electronGameData) {
   const currentFolder = await getSaveFolder(window);
   let saveFolderSizeBytes = await getFolderSizeInBytes(currentFolder);
   const maxFolderSizeBytes = store.get("autosave-quota", 1e8); // 100Mb per playerIndentifier
@@ -221,19 +229,12 @@ async function saveGameToDisk(window, saveData) {
   log.debug(
     `Remaining: ${remainingSpaceBytes} bytes (${((saveFolderSizeBytes / maxFolderSizeBytes) * 100).toFixed(2)}% used)`,
   );
-  const shouldCompress = isSaveCompressionEnabled();
+  let saveData = electronGameData.save;
-  const fileName = saveData.fileName;
+  const file = path.join(currentFolder, electronGameData.fileName);
-  const file = path.join(currentFolder, fileName + (shouldCompress ? ".gz" : ""));
   try {
-    let saveContent = saveData.save;
+    await fs.writeFile(file, saveData, "utf8");
-    if (shouldCompress) {
-      // Let's decode the base64 string so GZIP is more efficient.
-      const buffer = Buffer.from(saveContent, "base64").toString("utf8");
-      saveContent = await gzip(buffer);
-    }
-    await fs.writeFile(file, saveContent, "utf8");
     log.debug(`Saved Game to '${file}'`);
-    log.debug(`Save Size: ${saveContent.length} bytes`);
+    log.debug(`Save Size: ${saveData.length} bytes`);
   } catch (error) {
     log.error(error);
   }
@@ -276,14 +277,14 @@ async function loadLastFromDisk(window) {
 async function loadFileFromDisk(path) {
   const buffer = await fs.readFile(path);
   let content;
-  if (path.endsWith(".gz")) {
+  if (isBinaryFormat(buffer)) {
-    const uncompressedBuffer = await gunzip(buffer);
+    // Save file is in the binary format.
-    content = uncompressedBuffer.toString("base64");
+    content = buffer;
-    log.debug(`Uncompressed file content (new size: ${content.length} bytes)`);
   } else {
+    // Save file is in the base64 format.
     content = buffer.toString("utf8");
-    log.debug(`Loaded file with ${content.length} bytes`);
   }
+  log.debug(`Loaded file with ${content.length} bytes`);
   return content;
 }
 
@@ -319,7 +320,7 @@ async function restoreIfNewerExists(window) {
   const disk = {};
 
   try {
-    steam.save = await getSteamCloudSaveString();
+    steam.save = await getSteamCloudSaveData();
     steam.data = await getSaveInformation(window, steam.save);
   } catch (error) {
     log.error("Could not retrieve steam file");
@@ -361,12 +362,12 @@ async function restoreIfNewerExists(window) {
       // We add a few seconds to the currentSave's lastSave to prioritize it
       log.info("Found newer data than the current's save file");
       log.silly(bestMatch.data);
-      await pushSaveGameForImport(window, bestMatch.save, true);
+      pushSaveGameForImport(window, bestMatch.save, true);
       return true;
     } else if (bestMatch.data.playtime > currentData.playtime && currentData.playtime < lowPlaytime) {
       log.info("Found older save, but with more playtime, and current less than 15 mins played");
       log.silly(bestMatch.data);
-      await pushSaveGameForImport(window, bestMatch.save, true);
+      pushSaveGameForImport(window, bestMatch.save, true);
       return true;
     } else {
       log.debug("Current save data is the freshest");
@@ -380,8 +381,8 @@ module.exports = {
   getSaveInformation,
   restoreIfNewerExists,
   pushSaveGameForImport,
-  pushGameSaveToSteamCloud,
+  pushSaveDataToSteamCloud,
-  getSteamCloudSaveString,
+  getSteamCloudSaveData,
   getSteamCloudQuota,
   deleteCloudFile,
   saveGameToDisk,
@@ -394,6 +395,4 @@ module.exports = {
   setCloudEnabledConfig,
   isAutosaveEnabled,
   setAutosaveConfig,
-  isSaveCompressionEnabled,
-  setSaveCompressionConfig,
 };

package-lock.json

@@ -24,7 +24,6 @@
         "arg": "^5.0.2",
         "bcryptjs": "^2.4.3",
         "better-react-mathjax": "^2.0.3",
-        "buffer": "^6.0.3",
         "clsx": "^1.2.1",
         "date-fns": "^2.30.0",
         "escodegen": "^2.1.0",
@@ -5884,6 +5883,7 @@
       "version": "1.5.1",
       "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
       "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
+      "dev": true,
       "funding": [
         {
           "type": "github",
@@ -6081,29 +6081,6 @@
         "node-int64": "^0.4.0"
       }
     },
-    "node_modules/buffer": {
-      "version": "6.0.3",
-      "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz",
-      "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/feross"
-        },
-        {
-          "type": "patreon",
-          "url": "https://www.patreon.com/feross"
-        },
-        {
-          "type": "consulting",
-          "url": "https://feross.org/support"
-        }
-      ],
-      "dependencies": {
-        "base64-js": "^1.3.1",
-        "ieee754": "^1.2.1"
-      }
-    },
     "node_modules/buffer-crc32": {
       "version": "0.2.13",
       "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
@@ -9858,25 +9835,6 @@
         "postcss": "^8.1.0"
       }
     },
-    "node_modules/ieee754": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
-      "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/feross"
-        },
-        {
-          "type": "patreon",
-          "url": "https://www.patreon.com/feross"
-        },
-        {
-          "type": "consulting",
-          "url": "https://feross.org/support"
-        }
-      ]
-    },
     "node_modules/ignore": {
       "version": "5.2.4",
       "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.4.tgz",

package.json

@@ -24,7 +24,6 @@
     "arg": "^5.0.2",
     "bcryptjs": "^2.4.3",
     "better-react-mathjax": "^2.0.3",
-    "buffer": "^6.0.3",
     "clsx": "^1.2.1",
     "date-fns": "^2.30.0",
     "escodegen": "^2.1.0",

src/DevMenu/ui/SaveFileDev.tsx

@@ -12,6 +12,7 @@ import { ToastVariant } from "@enums";
 import { Upload } from "@mui/icons-material";
 import { Button } from "@mui/material";
 import { OptionSwitch } from "../../ui/React/OptionSwitch";
+import { isBinaryFormat } from "../../../electron/saveDataBinaryFormat";
 
 export function SaveFileDev(): React.ReactElement {
   const importInput = useRef<HTMLInputElement>(null);
@@ -22,8 +23,14 @@ export function SaveFileDev(): React.ReactElement {
 
   async function onImport(event: React.ChangeEvent<HTMLInputElement>): Promise<void> {
     try {
-      const base64Save = await saveObject.getImportStringFromFile(event.target.files);
+      const saveData = await saveObject.getSaveDataFromFile(event.target.files);
-      const save = atob(base64Save);
+      // TODO Support binary format. This is low priority because this entire feature (SaveFileDev) is not fully
+      // implemented. "doRestore" does nothing.
+      if (isBinaryFormat(saveData)) {
+        SnackbarEvents.emit("We currently do not support binary format", ToastVariant.ERROR, 5000);
+        return;
+      }
+      const save = decodeURIComponent(escape(atob(saveData as string)));
       setSaveFile(save);
     } catch (e: unknown) {
       SnackbarEvents.emit(String(e), ToastVariant.ERROR, 5000);

src/Electron.tsx

@@ -4,13 +4,12 @@ import { Page } from "./ui/Router";
 import { Terminal } from "./Terminal";
 import { SnackbarEvents } from "./ui/React/Snackbar";
 import { ToastVariant } from "@enums";
-import { IReturnStatus } from "./types";
+import { IReturnStatus, SaveData } from "./types";
 import { GetServer } from "./Server/AllServers";
-import { ImportPlayerData, SaveData, saveObject } from "./SaveObject";
+import { ImportPlayerData, ElectronGameData, saveObject } from "./SaveObject";
 import { exportScripts } from "./Terminal/commands/download";
 import { CONSTANTS } from "./Constants";
 import { hash } from "./hash/hash";
-import { Buffer } from "buffer";
 import { resolveFilePath } from "./Paths/FilePath";
 import { hasScriptExtension } from "./Paths/ScriptFilePath";
 
@@ -28,9 +27,9 @@ declare global {
       triggerSave: () => Promise<void>;
       triggerGameExport: () => void;
       triggerScriptsExport: () => void;
-      getSaveData: () => { save: string; fileName: string };
+      getSaveData: () => Promise<{ save: SaveData; fileName: string }>;
-      getSaveInfo: (base64Save: string) => Promise<ImportPlayerData | undefined>;
+      getSaveInfo: (saveData: SaveData) => Promise<ImportPlayerData | undefined>;
-      pushSaveData: (base64Save: string, automatic?: boolean) => void;
+      pushSaveData: (saveData: SaveData, automatic?: boolean) => void;
     };
     electronBridge: {
       send: (channel: string, data?: unknown) => void;
@@ -85,7 +84,7 @@ function initWebserver(): void {
     if (!path) return { res: false, msg: "Invalid file path." };
     if (!hasScriptExtension(path)) return { res: false, msg: "Invalid file extension: must be a script" };
 
-    code = Buffer.from(code, "base64").toString();
+    code = decodeURIComponent(escape(atob(code)));
     const home = GetServer("home");
     if (!home) return { res: false, msg: "Home server does not exist." };
 
@@ -132,23 +131,23 @@ function initSaveFunctions(): void {
       }
     },
     triggerScriptsExport: (): void => exportScripts("*", Player.getHomeComputer()),
-    getSaveData: (): { save: string; fileName: string } => {
+    getSaveData: async (): Promise<{ save: SaveData; fileName: string }> => {
       return {
-        save: saveObject.getSaveString(),
+        save: await saveObject.getSaveData(),
         fileName: saveObject.getSaveFileName(),
       };
     },
-    getSaveInfo: async (base64Save: string): Promise<ImportPlayerData | undefined> => {
+    getSaveInfo: async (saveData: SaveData): Promise<ImportPlayerData | undefined> => {
       try {
-        const data = await saveObject.getImportDataFromString(base64Save);
+        const importData = await saveObject.getImportDataFromSaveData(saveData);
-        return data.playerData;
+        return importData.playerData;
       } catch (error) {
         console.error(error);
         return;
       }
     },
-    pushSaveData: (base64Save: string, automatic = false): void =>
+    pushSaveData: (saveData: SaveData, automatic = false): void =>
-      Router.toPage(Page.ImportSave, { base64Save, automatic }),
+      Router.toPage(Page.ImportSave, { saveData, automatic }),
   };
 
   // Will be consumed by the electron wrapper.
@@ -159,14 +158,16 @@ function initElectronBridge(): void {
   const bridge = window.electronBridge;
   if (!bridge) return;
 
-  bridge.receive("get-save-data-request", () => {
+  bridge.receive("get-save-data-request", async () => {
-    const data = window.appSaveFns.getSaveData();
+    const saveData = await window.appSaveFns.getSaveData();
-    bridge.send("get-save-data-response", data);
+    bridge.send("get-save-data-response", saveData);
   });
-  bridge.receive("get-save-info-request", async (save: unknown) => {
+  bridge.receive("get-save-info-request", async (saveData: unknown) => {
-    if (typeof save !== "string") throw new Error("Error while trying to get save info");
+    if (typeof saveData !== "string" && !(saveData instanceof Uint8Array)) {
-    const data = await window.appSaveFns.getSaveInfo(save);
+      throw new Error("Error while trying to get save info");
-    bridge.send("get-save-info-response", data);
+    }
+    const saveInfo = await window.appSaveFns.getSaveInfo(saveData);
+    bridge.send("get-save-info-response", saveInfo);
   });
   bridge.receive("push-save-request", (params: unknown) => {
     if (typeof params !== "object") throw new Error("Error trying to push save request");
@@ -202,7 +203,7 @@ function initElectronBridge(): void {
   });
 }
 
-export function pushGameSaved(data: SaveData): void {
+export function pushGameSaved(data: ElectronGameData): void {
   const bridge = window.electronBridge;
   if (!bridge) return;
 

src/GameOptions/ui/GameOptionsSidebar.tsx

@@ -71,8 +71,8 @@ export const GameOptionsSidebar = (props: IProps): React.ReactElement => {
 
   async function onImport(event: React.ChangeEvent<HTMLInputElement>): Promise<void> {
     try {
-      const base64Save = await saveObject.getImportStringFromFile(event.target.files);
+      const saveData = await saveObject.getSaveDataFromFile(event.target.files);
-      const data = await saveObject.getImportDataFromString(base64Save);
+      const data = await saveObject.getImportDataFromSaveData(saveData);
       setImportData(data);
       setImportSaveOpen(true);
     } catch (e: unknown) {
@@ -88,7 +88,7 @@ export const GameOptionsSidebar = (props: IProps): React.ReactElement => {
     if (!importData) return;
 
     try {
-      await saveObject.importGame(importData.base64);
+      await saveObject.importGame(importData.saveData);
     } catch (e: unknown) {
       SnackbarEvents.emit(String(e), ToastVariant.ERROR, 5000);
     }
@@ -99,7 +99,7 @@ export const GameOptionsSidebar = (props: IProps): React.ReactElement => {
 
   function compareSaveGame(): void {
     if (!importData) return;
-    Router.toPage(Page.ImportSave, { base64Save: importData.base64 });
+    Router.toPage(Page.ImportSave, { saveData: importData.saveData });
     setImportSaveOpen(false);
     setImportData(null);
   }

src/SaveObject.ts

@@ -38,20 +38,26 @@ import { Terminal } from "./Terminal";
 import { getRecordValues } from "./Types/Record";
 import { ExportMaterial } from "./Corporation/Actions";
 import { getGoSave, loadGo } from "./Go/SaveLoad";
+import { SaveData } from "./types";
+import { SaveDataError, canUseBinaryFormat, decodeSaveData, encodeJsonSaveString } from "./utils/SaveDataUtils";
+import { isBinaryFormat } from "../electron/saveDataBinaryFormat";
 
 /* SaveObject.js
  *  Defines the object used to save/load games
  */
 
-export interface SaveData {
+/**
+ * This interface is only for transferring game data to electron-related code.
+ */
+export interface ElectronGameData {
   playerIdentifier: string;
   fileName: string;
-  save: string;
+  save: SaveData;
   savedOn: number;
 }
 
 export interface ImportData {
-  base64: string;
+  saveData: SaveData;
   playerData?: ImportPlayerData;
 }
 
@@ -88,7 +94,7 @@ class BitburnerSaveObject {
   StaneksGiftSave = "";
   GoSave = "";
 
-  getSaveString(forceExcludeRunningScripts = false): string {
+  async getSaveData(forceExcludeRunningScripts = false): Promise<SaveData> {
     this.PlayerSave = JSON.stringify(Player);
 
     // For the servers save, overwrite the ExcludeRunningScripts setting if forced
@@ -110,115 +116,113 @@ class BitburnerSaveObject {
 
     if (Player.gang) this.AllGangsSave = JSON.stringify(AllGangs);
 
-    const saveString = btoa(unescape(encodeURIComponent(JSON.stringify(this))));
+    return await encodeJsonSaveString(JSON.stringify(this));
-    return saveString;
   }
 
-  saveGame(emitToastEvent = true): Promise<void> {
+  async saveGame(emitToastEvent = true): Promise<void> {
     const savedOn = new Date().getTime();
     Player.lastSave = savedOn;
-    const saveString = this.getSaveString();
+    const saveData = await this.getSaveData();
-    return new Promise((resolve, reject) => {
+    try {
-      save(saveString)
+      await save(saveData);
-        .then(() => {
+    } catch (error) {
-          const saveData: SaveData = {
+      console.error(error);
-            playerIdentifier: Player.identifier,
+      dialogBoxCreate(`Cannot save game: ${error}`);
-            fileName: this.getSaveFileName(),
+      return;
-            save: saveString,
+    }
-            savedOn,
+    const electronGameData: ElectronGameData = {
-          };
+      playerIdentifier: Player.identifier,
-          pushGameSaved(saveData);
+      fileName: this.getSaveFileName(),
+      save: saveData,
+      savedOn,
+    };
+    pushGameSaved(electronGameData);
 
-          if (emitToastEvent) {
+    if (emitToastEvent) {
-            SnackbarEvents.emit("Game Saved!", ToastVariant.INFO, 2000);
+      SnackbarEvents.emit("Game Saved!", ToastVariant.INFO, 2000);
-          }
+    }
-          return resolve();
-        })
-        .catch((err) => {
-          console.error(err);
-          return reject();
-        });
-    });
   }
 
-  getSaveFileName(isRecovery = false): string {
+  getSaveFileName(): string {
     // Save file name is based on current timestamp and BitNode
     const epochTime = Math.round(Date.now() / 1000);
     const bn = Player.bitNodeN;
-    let filename = `bitburnerSave_${epochTime}_BN${bn}x${Player.sourceFileLvl(bn) + 1}.json`;
+    /**
-    if (isRecovery) filename = "RECOVERY" + filename;
+     * - Binary format: save file uses .json.gz extension. Save data is the compressed json save string.
-    return filename;
+     * - Base64 format: save file uses .json extension. Save data is the base64-encoded json save string.
+     */
+    const extension = canUseBinaryFormat() ? "json.gz" : "json";
+    return `bitburnerSave_${epochTime}_BN${bn}x${Player.sourceFileLvl(bn) + 1}.${extension}`;
   }
 
-  exportGame(): void {
+  async exportGame(): Promise<void> {
-    const saveString = this.getSaveString();
+    const saveData = await this.getSaveData();
     const filename = this.getSaveFileName();
-    download(filename, saveString);
+    download(filename, saveData);
   }
 
-  importGame(base64Save: string, reload = true): Promise<void> {
+  async importGame(saveData: SaveData, reload = true): Promise<void> {
-    if (!base64Save || base64Save === "") throw new Error("Invalid import string");
+    if (!saveData || saveData.length === 0) {
-    return save(base64Save).then(() => {
+      throw new Error("Invalid import string");
-      if (reload) setTimeout(() => location.reload(), 1000);
+    }
-      return Promise.resolve();
+    try {
-    });
+      await save(saveData);
+    } catch (error) {
+      console.error(error);
+      dialogBoxCreate(`Cannot import save data: ${error}`);
+      return;
+    }
+    if (reload) {
+      setTimeout(() => location.reload(), 1000);
+    }
   }
 
-  getImportStringFromFile(files: FileList | null): Promise<string> {
+  async getSaveDataFromFile(files: FileList | null): Promise<SaveData> {
     if (files === null) return Promise.reject(new Error("No file selected"));
     const file = files[0];
     if (!file) return Promise.reject(new Error("Invalid file selected"));
 
-    const reader = new FileReader();
+    const rawData = new Uint8Array(await file.arrayBuffer());
-    const promise = new Promise<string>((resolve, reject) => {
+    if (isBinaryFormat(rawData)) {
-      reader.onload = function (this: FileReader, e: ProgressEvent<FileReader>) {
+      return rawData;
-        const target = e.target;
+    } else {
-        if (target === null) {
+      return new TextDecoder().decode(rawData);
-          return reject(new Error("Error importing file"));
+    }
-        }
-        const result = target.result;
-        if (typeof result !== "string") {
-          return reject(new Error("FileReader event was not type string"));
-        }
-        const contents = result;
-        resolve(contents);
-      };
-    });
-    reader.readAsText(file);
-    return promise;
   }
 
-  async getImportDataFromString(base64Save: string): Promise<ImportData> {
+  async getImportDataFromSaveData(saveData: SaveData): Promise<ImportData> {
-    if (!base64Save || base64Save === "") throw new Error("Invalid import string");
+    if (!saveData || saveData.length === 0) throw new Error("Invalid save data");
 
-    let newSave;
+    let decodedSaveData;
     try {
-      newSave = window.atob(base64Save);
+      decodedSaveData = await decodeSaveData(saveData);
-      newSave = newSave.trim();
+    } catch (error) {
+      console.error(error);
+      if (error instanceof SaveDataError) {
+        return Promise.reject(error);
+      }
+    }
+
+    if (!decodedSaveData || decodedSaveData === "") {
+      return Promise.reject(new Error("Save game is invalid"));
+    }
+
+    let parsedSaveData;
+    try {
+      parsedSaveData = JSON.parse(decodedSaveData);
     } catch (error) {
       console.error(error); // We'll handle below
     }
 
-    if (!newSave || newSave === "") {
+    if (!parsedSaveData || parsedSaveData.ctor !== "BitburnerSaveObject" || !parsedSaveData.data) {
-      return Promise.reject(new Error("Save game had not content or was not base64 encoded"));
-    }
-
-    let parsedSave;
-    try {
-      parsedSave = JSON.parse(newSave);
-    } catch (error) {
-      console.error(error); // We'll handle below
-    }
-
-    if (!parsedSave || parsedSave.ctor !== "BitburnerSaveObject" || !parsedSave.data) {
       return Promise.reject(new Error("Save game did not seem valid"));
     }
 
     const data: ImportData = {
-      base64: base64Save,
+      saveData: saveData,
     };
 
-    const importedPlayer = loadPlayer(parsedSave.data.PlayerSave);
+    const importedPlayer = loadPlayer(parsedSaveData.data.PlayerSave);
 
     const playerData: ImportPlayerData = {
       identifier: importedPlayer.identifier,
@@ -719,12 +723,12 @@ Error: ${e}`);
   }
 }
 
-function loadGame(saveString: string): boolean {
+async function loadGame(saveData: SaveData): Promise<boolean> {
   createScamUpdateText();
-  if (!saveString) return false;
+  if (!saveData) return false;
-  saveString = decodeURIComponent(escape(atob(saveString)));
+  const jsonSaveString = await decodeSaveData(saveData);
 
-  const saveObj = JSON.parse(saveString, Reviver);
+  const saveObj = JSON.parse(jsonSaveString, Reviver);
 
   setPlayer(loadPlayer(saveObj.PlayerSave));
   loadAllServers(saveObj.AllServersSave);
@@ -849,7 +853,7 @@ function createBetaUpdateText() {
   );
 }
 
-function download(filename: string, content: string): void {
+function download(filename: string, content: SaveData): void {
   const file = new Blob([content], { type: "text/plain" });
 
   const a = document.createElement("a"),

src/db.tsx

@@ -1,3 +1,5 @@
+import { SaveData } from "./types";
+
 function getDB(): Promise<IDBObjectStore> {
   return new Promise((resolve, reject) => {
     if (!window.indexedDB) {
@@ -32,30 +34,30 @@ function getDB(): Promise<IDBObjectStore> {
   });
 }
 
-export function load(): Promise<string> {
+export function load(): Promise<SaveData> {
   return new Promise((resolve, reject) => {
     getDB()
       .then((db) => {
-        return new Promise<string>((resolve, reject) => {
+        return new Promise<SaveData>((resolve, reject) => {
-          const request: IDBRequest<string> = db.get("save");
+          const request: IDBRequest<SaveData> = db.get("save");
-          request.onerror = function (this: IDBRequest<string>, ev: Event) {
+          request.onerror = function (this: IDBRequest<SaveData>, ev: Event) {
-            reject("Error in Database request to get savestring: " + ev);
+            reject("Error in Database request to get save data: " + ev);
           };
 
-          request.onsuccess = function (this: IDBRequest<string>) {
+          request.onsuccess = function (this: IDBRequest<SaveData>) {
             resolve(this.result);
           };
-        }).then((saveString) => resolve(saveString));
+        }).then((saveData) => resolve(saveData));
       })
       .catch((r) => reject(r));
   });
 }
 
-export function save(saveString: string): Promise<void> {
+export function save(saveData: SaveData): Promise<void> {
   return getDB().then((db) => {
     return new Promise<void>((resolve, reject) => {
-      // We'll save to both localstorage and indexedDb
+      // We'll save to IndexedDB
-      const request = db.put(saveString, "save");
+      const request = db.put(saveData, "save");
 
       request.onerror = function (e) {
         reject("Error saving game to IndexedDB: " + e);

src/engine.tsx

@@ -43,6 +43,7 @@ import React from "react";
 import { setupUncaughtPromiseHandler } from "./UncaughtPromiseHandler";
 import { Button, Typography } from "@mui/material";
 import { SnackbarEvents } from "./ui/React/Snackbar";
+import { SaveData } from "./types";
 
 /** Game engine. Handles the main game loop. */
 const Engine: {
@@ -66,7 +67,7 @@ const Engine: {
   };
   decrementAllCounters: (numCycles?: number) => void;
   checkCounters: () => void;
-  load: (saveString: string) => void;
+  load: (saveData: SaveData) => Promise<void>;
   start: () => void;
 } = {
   // Time variables (milliseconds unix epoch time)
@@ -218,7 +219,7 @@ const Engine: {
     }
   },
 
-  load: function (saveString) {
+  load: async function (saveData) {
     startExploits();
     setupUncaughtPromiseHandler();
     // Source files must be initialized early because save-game translation in
@@ -226,7 +227,7 @@ const Engine: {
     initSourceFiles();
     // Load game from save or create new game
 
-    if (loadGame(saveString)) {
+    if (await loadGame(saveData)) {
       FormatsNeedToChange.emit();
       initBitNodeMultipliers();
       if (Player.hasWseAccount) {

src/types.ts

@@ -44,3 +44,6 @@ export interface IMinMaxRange {
   /** The minimum bound of the range. */
   min: number;
 }
+
+// Type of save data. The base64 format is string, the binary format is Uint8Array.
+export type SaveData = string | Uint8Array;

src/ui/GameRoot.tsx

@@ -369,7 +369,7 @@ export function GameRoot(): React.ReactElement {
       break;
     }
     case Page.ImportSave: {
-      mainPage = <ImportSave importString={pageWithContext.base64Save} automatic={!!pageWithContext.automatic} />;
+      mainPage = <ImportSave saveData={pageWithContext.saveData} automatic={!!pageWithContext.automatic} />;
       withSidebar = false;
       withPopups = false;
       bypassGame = true;

src/ui/LoadingScreen.tsx

@@ -31,27 +31,20 @@ export function LoadingScreen(): React.ReactElement {
   });
 
   useEffect(() => {
-    async function doLoad(): Promise<void> {
+    load().then(async (saveData) => {
-      await load()
+      try {
-        .then((saveString) => {
+        await Engine.load(saveData);
-          try {
+      } catch (error) {
-            Engine.load(saveString);
+        console.error(error);
-          } catch (err: unknown) {
+        ActivateRecoveryMode(error);
-            ActivateRecoveryMode();
+        await Engine.load("");
-            setLoaded(true);
+        setLoaded(true);
-            throw err;
+        return;
-          }
+      }
 
-          pushGameReady();
+      pushGameReady();
-          setLoaded(true);
+      setLoaded(true);
-        })
+    });
-        .catch((reason) => {
-          console.error(reason);
-          Engine.load("");
-          setLoaded(true);
-        });
-    }
-    doLoad();
   }, []);
 
   return loaded ? (

src/ui/React/ImportSave/ImportSave.tsx

@@ -38,6 +38,7 @@ import { Page } from "../../Router";
 import { useBoolean } from "../hooks";
 
 import { ComparisonIcon } from "./ComparisonIcon";
+import { SaveData } from "../../../types";
 
 const useStyles = makeStyles((theme: Theme) =>
   createStyles({
@@ -89,7 +90,7 @@ const playerSkills: (keyof Skills)[] = ["hacking", "strength", "defense", "dexte
 
 let initialAutosave = 0;
 
-export const ImportSave = (props: { importString: string; automatic: boolean }): JSX.Element => {
+export const ImportSave = (props: { saveData: SaveData; automatic: boolean }): JSX.Element => {
   const classes = useStyles();
   const [importData, setImportData] = useState<ImportData | undefined>();
   const [currentData, setCurrentData] = useState<ImportData | undefined>();
@@ -105,7 +106,7 @@ export const ImportSave = (props: { importString: string; automatic: boolean }):
   };
 
   const handleImport = async (): Promise<void> => {
-    await saveObject.importGame(props.importString, true);
+    await saveObject.importGame(props.saveData, true);
     pushImportResult(true);
   };
 
@@ -122,16 +123,16 @@ export const ImportSave = (props: { importString: string; automatic: boolean }):
 
   useEffect(() => {
     async function fetchData(): Promise<void> {
-      const dataBeingImported = await saveObject.getImportDataFromString(props.importString);
+      const dataBeingImported = await saveObject.getImportDataFromSaveData(props.saveData);
-      const dataCurrentlyInGame = await saveObject.getImportDataFromString(saveObject.getSaveString(true));
+      const dataCurrentlyInGame = await saveObject.getImportDataFromSaveData(await saveObject.getSaveData(true));
 
       setImportData(dataBeingImported);
       setCurrentData(dataCurrentlyInGame);
 
       return Promise.resolve();
     }
-    if (props.importString) fetchData();
+    if (props.saveData) fetchData();
-  }, [props.importString]);
+  }, [props.saveData]);
 
   if (!importData || !currentData) return <></>;
 

src/ui/React/RecoveryRoot.tsx

@@ -12,11 +12,15 @@ import { SoftResetButton } from "./SoftResetButton";
 
 import DirectionsRunIcon from "@mui/icons-material/DirectionsRun";
 import GitHubIcon from "@mui/icons-material/GitHub";
+import { isBinaryFormat } from "../../../electron/saveDataBinaryFormat";
+import { InvalidSaveData, UnsupportedSaveData } from "../../utils/SaveDataUtils";
 
 export let RecoveryMode = false;
+let sourceError: unknown;
 
-export function ActivateRecoveryMode(): void {
+export function ActivateRecoveryMode(error: unknown): void {
   RecoveryMode = true;
+  sourceError = error;
 }
 
 interface IProps {
@@ -29,6 +33,7 @@ export function RecoveryRoot({ softReset, errorData, resetError }: IProps): Reac
   function recover(): void {
     if (resetError) resetError();
     RecoveryMode = false;
+    sourceError = undefined;
     Router.toPage(Page.Terminal);
   }
   Settings.AutosaveInterval = 0;
@@ -37,41 +42,65 @@ export function RecoveryRoot({ softReset, errorData, resetError }: IProps): Reac
     load()
       .then((content) => {
         const epochTime = Math.round(Date.now() / 1000);
-        const filename = `RECOVERY_BITBURNER_${epochTime}.json`;
+        const extension = isBinaryFormat(content) ? "json.gz" : "json";
+        const filename = `RECOVERY_BITBURNER_${epochTime}.${extension}`;
         download(filename, content);
       })
       .catch((err) => console.error(err));
   }, []);
 
+  let instructions;
+  if (sourceError instanceof UnsupportedSaveData) {
+    instructions = <Typography variant="h6">Please update your browser.</Typography>;
+  } else if (sourceError instanceof InvalidSaveData) {
+    instructions = (
+      <Typography variant="h6">Your save data is invalid. Please import a valid backup save file.</Typography>
+    );
+  } else {
+    instructions = (
+      <Box>
+        <Typography>It is recommended to alert a developer.</Typography>
+        <Typography>
+          <Link href={errorData?.issueUrl ?? newIssueUrl} target="_blank">
+            File an issue on github
+          </Link>
+        </Typography>
+        <Typography>
+          <Link href="https://www.reddit.com/r/Bitburner/" target="_blank">
+            Make a reddit post
+          </Link>
+        </Typography>
+        <Typography>
+          <Link href="https://discord.gg/TFc3hKD" target="_blank">
+            Post in the #bug-report channel on Discord.
+          </Link>
+        </Typography>
+        <Typography>Please include your save file.</Typography>
+      </Box>
+    );
+  }
+
   return (
     <Box sx={{ padding: "8px 16px", minHeight: "100vh", maxWidth: "1200px", boxSizing: "border-box" }}>
       <Typography variant="h3">RECOVERY MODE ACTIVATED</Typography>
       <Typography>
-        There was an error with your save file and the game went into recovery mode. In this mode saving is disabled and
+        There was an error with your save file and the game went into recovery mode. In this mode, saving is disabled
-        the game will automatically export your save file (to prevent corruption).
+        and the game will automatically export your save file to prevent corruption.
       </Typography>
-      <Typography>At this point it is recommended to alert a developer.</Typography>
-      <Typography>
-        <Link href={errorData?.issueUrl ?? newIssueUrl} target="_blank">
-          File an issue on github
-        </Link>
-      </Typography>
-      <Typography>
-        <Link href="https://www.reddit.com/r/Bitburner/" target="_blank">
-          Make a reddit post
-        </Link>
-      </Typography>
-      <Typography>
-        <Link href="https://discord.gg/TFc3hKD" target="_blank">
-          Post in the #bug-report channel on Discord.
-        </Link>
-      </Typography>
-      <Typography>Please include your save file.</Typography>
       <br />
+      {sourceError && (
+        <Box>
+          <Typography variant="h6" color={Settings.theme.error}>
+            Error: {sourceError.toString()}
+          </Typography>
+          <br />
+        </Box>
+      )}
+      {instructions}
       <br />
-      <Typography>You can disable recovery mode now. But chances are the game will not work correctly.</Typography>
+      <Typography>You can disable the recovery mode, but the game may not work correctly.</Typography>
       <ButtonGroup sx={{ my: 2 }}>
-        <Tooltip title="Disables the recovery mode & attempt to head back to the terminal page. This may or may not work. Ensure you have saved the recovery file.">
+        <Tooltip title="Disable the recovery mode and attempt to head back to the terminal page. This may or may not work. Ensure you saved the recovery file.">
           <Button onClick={recover} startIcon={<DirectionsRunIcon />}>
             Disable Recovery Mode
           </Button>
@@ -96,7 +125,7 @@ export function RecoveryRoot({ softReset, errorData, resetError }: IProps): Reac
               sx={{ "& .MuiOutlinedInput-root": { color: Settings.theme.secondary } }}
             />
           </Box>
-          <Tooltip title="Submitting an issue to GitHub really help us improve the game!">
+          <Tooltip title="Submitting an issue to GitHub really helps us improve the game!">
             <Button
               component={Link}
               startIcon={<GitHubIcon />}

src/ui/Router.ts

@@ -2,6 +2,7 @@ import type { ScriptFilePath } from "../Paths/ScriptFilePath";
 import type { TextFilePath } from "../Paths/TextFilePath";
 import type { Faction } from "../Faction/Faction";
 import type { Location } from "../Locations/Location";
+import { SaveData } from "../types";
 
 // This enum doesn't need enum helper support for now
 /**
@@ -70,7 +71,7 @@ export type PageContext<T extends Page> = T extends ComplexPage.BitVerse
   : T extends ComplexPage.Location
   ? { location: Location }
   : T extends ComplexPage.ImportSave
-  ? { base64Save: string; automatic?: boolean }
+  ? { saveData: SaveData; automatic?: boolean }
   : T extends ComplexPage.Documentation
   ? { docPage?: string }
   : never;

src/utils/SaveDataUtils.ts

@@ -0,0 +1,59 @@
+import { SaveData } from "../types";
+
+export abstract class SaveDataError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = this.constructor.name;
+  }
+}
+
+export class UnsupportedSaveData extends SaveDataError {}
+
+export class InvalidSaveData extends SaveDataError {}
+
+export function canUseBinaryFormat(): boolean {
+  return "CompressionStream" in globalThis;
+}
+
+async function compress(dataString: string): Promise<Uint8Array> {
+  const compressedReadableStream = new Blob([dataString]).stream().pipeThrough(new CompressionStream("gzip"));
+  return new Uint8Array(await new Response(compressedReadableStream).arrayBuffer());
+}
+
+async function decompress(binaryData: Uint8Array): Promise<string> {
+  const decompressedReadableStream = new Blob([binaryData]).stream().pipeThrough(new DecompressionStream("gzip"));
+  const reader = decompressedReadableStream.pipeThrough(new TextDecoderStream("utf-8", { fatal: true })).getReader();
+  let result = "";
+  try {
+    for (let { value, done } = await reader.read(); !done; { value, done } = await reader.read()) {
+      result += value;
+    }
+  } catch (error) {
+    throw new InvalidSaveData(String(error));
+  }
+  return result;
+}
+
+export async function encodeJsonSaveString(jsonSaveString: string): Promise<SaveData> {
+  // Fallback to the base64 format if player's browser does not support Compression Streams API.
+  if (canUseBinaryFormat()) {
+    return await compress(jsonSaveString);
+  } else {
+    // The unescape(encodeURIComponent()) pair encodes jsonSaveString into a "binary string" suitable for btoa, despite
+    // seeming like a no-op at first glance.
+    // Ref: https://stackoverflow.com/a/57713220
+    return btoa(unescape(encodeURIComponent(jsonSaveString)));
+  }
+}
+
+/** Return json save string */
+export async function decodeSaveData(saveData: SaveData): Promise<string> {
+  if (saveData instanceof Uint8Array) {
+    if (!canUseBinaryFormat()) {
+      throw new UnsupportedSaveData("Your browser does not support Compression Streams API");
+    }
+    return await decompress(saveData);
+  } else {
+    return decodeURIComponent(escape(atob(saveData)));
+  }
+}

test/jest/FullSave.test.ts

@@ -11,7 +11,7 @@ import { Companies } from "../../src/Company/Companies";
 describe("Check Save File Continuity", () => {
   establishInitialConditions();
   // Calling getSaveString forces save info to update
-  saveObject.getSaveString();
+  saveObject.getSaveData();
 
   const savesToTest = ["FactionsSave", "PlayerSave", "CompaniesSave", "GoSave"] as const;
   for (const saveToTest of savesToTest) {

tools/pretty-save.js

@@ -1,11 +1,20 @@
 /* eslint-disable @typescript-eslint/no-var-requires */
 const fs = require("fs").promises;
 const path = require("path");
+const { isBinaryFormat } = require("../electron/saveDataBinaryFormat");
 
 async function getSave(file) {
-  const data = await fs.readFile(file, "utf8");
+  const data = await fs.readFile(file);
 
-  const save = JSON.parse(decodeURIComponent(escape(atob(data))));
+  let jsonSaveString;
+  if (isBinaryFormat(data)) {
+    const decompressedReadableStream = new Blob([data]).stream().pipeThrough(new DecompressionStream("gzip"));
+    jsonSaveString = await new Response(decompressedReadableStream).text();
+  } else {
+    jsonSaveString = decodeURIComponent(escape(atob(data)));
+  }
+
+  const save = JSON.parse(jsonSaveString);
   const saveData = save.data;
   let gameSave = {
     PlayerSave: JSON.parse(saveData.PlayerSave),
@@ -13,7 +22,6 @@ async function getSave(file) {
     FactionsSave: JSON.parse(saveData.FactionsSave),
     AliasesSave: JSON.parse(saveData.AliasesSave),
     GlobalAliasesSave: JSON.parse(saveData.GlobalAliasesSave),
-    MessagesSave: JSON.parse(saveData.MessagesSave),
     StockMarketSave: JSON.parse(saveData.StockMarketSave),
     SettingsSave: JSON.parse(saveData.SettingsSave),
     VersionSave: JSON.parse(saveData.VersionSave),