Add saveStorage option to workflow

Author: tw4l

backend/btrixcloud/models.py

@@ -357,6 +357,8 @@ class RawCrawlConfig(BaseModel):
     selectLinks: List[str] = ["a[href]->href"]
     clickSelector: str = "a"
 
+    saveStorage: Optional[bool] = False
+
 
 # ============================================================================
 class CrawlConfigIn(BaseModel):

frontend/docs/docs/user-guide/workflow-setup.md

@@ -256,6 +256,10 @@ This setting will only be shown if multiple different release channels are avail
 
 Will prevent any content from the domains listed in [Steven Black's Unified Hosts file](https://github.com/StevenBlack/hosts) (ads & malware) from being captured by the crawler.
 
+### Save Local and Session Storage
+
+When enabled, instructs the crawler to save the browser's localStorage and sessionStorage data for each page in the web archive as part of the `WARC-JSON-Metadata` field. This option may be necessary to properly archive and replay certain websites. Use caution when sharing WACZ files created with this option enabled, as the saved browser storage may contain sensitive information.
+
 ### User Agent
 
 Sets the browser's [user agent](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent) in outgoing requests to the specified value. If left blank, the crawler will use the Brave browser's default user agent. For a list of common user agents see [useragents.me](https://www.useragents.me/).

frontend/src/components/ui/config-details.ts

@@ -262,6 +262,10 @@ export class ConfigDetails extends BtrixElement {
             msg("Block Ads by Domain"),
             seedsConfig?.blockAds,
           )}
+          ${this.renderSetting(
+            msg("Save Local and Session Storage"),
+            seedsConfig?.saveStorage,
+          )}
           ${this.renderSetting(
             msg("User Agent"),
             seedsConfig?.userAgent

frontend/src/features/crawl-workflows/workflow-editor.ts

@@ -1655,6 +1655,12 @@ https://archiveweb.page/images/${"logo.svg"}`}
         </sl-checkbox>
       `)}
       ${this.renderHelpTextCol(infoTextFor["blockAds"], false)}
+      ${inputCol(html`
+        <sl-checkbox name="saveStorage" ?checked=${this.formState.saveStorage}>
+          ${msg("Save local and session storage")}
+        </sl-checkbox>
+      `)}
+      ${this.renderHelpTextCol(infoTextFor["saveStorage"])}
       ${inputCol(html`
         <sl-input
           name="userAgent"
@@ -2631,7 +2637,12 @@ https://archiveweb.page/images/${"logo.svg"}`}
 
   private parseUrlListConfig(): Pick<
     CrawlConfigParams["config"],
-    "seeds" | "scopeType" | "extraHops" | "useSitemap" | "failOnFailedSeed"
+    | "seeds"
+    | "scopeType"
+    | "extraHops"
+    | "useSitemap"
+    | "failOnFailedSeed"
+    | "saveStorage"
   > {
     const config = {
       seeds: urlListToArray(this.formState.urlList).map((seedUrl) => {
@@ -2642,14 +2653,15 @@ https://archiveweb.page/images/${"logo.svg"}`}
       extraHops: this.formState.includeLinkedPages ? 1 : 0,
       useSitemap: false,
       failOnFailedSeed: this.formState.failOnFailedSeed,
+      saveStorage: this.formState.saveStorage,
     };
 
     return config;
   }
 
   private parseSeededConfig(): Pick<
     CrawlConfigParams["config"],
-    "seeds" | "scopeType" | "useSitemap" | "failOnFailedSeed"
+    "seeds" | "scopeType" | "useSitemap" | "failOnFailedSeed" | "saveStorage"
   > {
     const primarySeedUrl = this.formState.primarySeedUrl;
     const includeUrlList = this.formState.customIncludeUrlList
@@ -2680,6 +2692,7 @@ https://archiveweb.page/images/${"logo.svg"}`}
       scopeType: this.formState.scopeType as ScopeType,
       useSitemap: this.formState.useSitemap,
       failOnFailedSeed: false,
+      saveStorage: this.formState.saveStorage,
     };
     return config;
   }

frontend/src/strings/crawl-workflows/infoText.ts

@@ -75,6 +75,9 @@ export const infoTextFor = {
   customBehavior: msg(
     `Enable custom page actions with behavior scripts. You can specify any publicly accessible URL or public Git repository.`,
   ),
+  saveStorage: msg(
+    `Save the browser's localStorage and sessionStorage in resulting web archive. Use caution sharing WACZs created with this option.`,
+  ),
 } as const satisfies Partial<Record<Field, string | TemplateResult>>;
 
 export default infoTextFor;

frontend/src/types/crawler.ts

@@ -47,6 +47,7 @@ export type SeedConfig = Expand<
     selectLinks: string[];
     customBehaviors: string[];
     clickSelector: string;
+    saveStorage?: boolean;
   }
 >;
 

frontend/src/utils/workflow.ts

@@ -132,6 +132,7 @@ export type FormState = {
   proxyId: string | null;
   selectLinks: string[];
   clickSelector: string;
+  saveStorage: WorkflowParams["config"]["saveStorage"];
 };
 
 export type FormStateField = keyof FormState;
@@ -189,6 +190,7 @@ export const getDefaultFormState = (): FormState => ({
   selectLinks: DEFAULT_SELECT_LINKS,
   clickSelector: DEFAULT_AUTOCLICK_SELECTOR,
   customBehavior: false,
+  saveStorage: false,
 });
 
 export const mapSeedToUrl = (arr: Seed[]) =>
@@ -346,6 +348,7 @@ export function getInitialFormState(params: {
     crawlerChannel:
       params.initialWorkflow.crawlerChannel || defaultFormState.crawlerChannel,
     proxyId: params.initialWorkflow.proxyId || defaultFormState.proxyId,
+    saveStorage: params.initialWorkflow.config.saveStorage,
     ...formState,
   };
 }