Support for dynamic page loading for Multi-WACZs (#399)

Support querying pages dynamically via pages endpoint:
- If loaded pages < total pages, load additional pages via /pages
endpoint if canQueryPages is set to true on item
- If query provided, do `/pages?search=...` to search for pages. Use
pagination of 25 pages for now.

WACZ hash:
- if provided, store waczhash per page, and use for thumbnail, favicon
and top-level replay loading.

Embed: better support for force reloading, call delete with reload
directly in embed

Depends on webrecorder/wabac.js#220

package.json

@@ -1,7 +1,7 @@
 {
   "name": "replaywebpage",
   "productName": "ReplayWeb.page",
-  "version": "2.2.5",
+  "version": "2.3.0",
   "description": "Serverless Web Archive Replay",
   "repository": "https://github.com/webrecorder/replayweb.page",
   "homepage": "https://replayweb.page/",
@@ -18,7 +18,7 @@
   "dependencies": {
     "@fortawesome/fontawesome-free": "^5.15.4",
     "@shoelace-style/shoelace": "~2.15.1",
-    "@webrecorder/wabac": "^2.20.8",
+    "@webrecorder/wabac": "^2.21.0",
     "bulma": "^0.9.3",
     "electron-log": "^4.4.1",
     "electron-updater": "^6.3.9",

src/embed.ts

@@ -7,9 +7,9 @@ import {
 } from "lit";
 import { ifDefined } from "lit/directives/if-defined.js";
 
-import { wrapCss, updateFaviconLinks } from "./misc";
+import { wrapCss, updateFaviconLinks, apiPrefix } from "./misc";
 import { SWManager } from "./swmanager";
-import { property } from "lit/decorators.js";
+import { property, query } from "lit/decorators.js";
 import type { FavIconEventDetail } from "./types";
 import type { EmbedReplayData, EmbedReplayEvent } from "./item";
 
@@ -82,6 +82,9 @@ class Embed extends LitElement {
 
   @property({ type: Boolean }) useRuffle = false;
 
+  @query("iframe")
+  private readonly iframe?: HTMLIFrameElement | null;
+
   replayfile = defaultReplayFile;
   mainElementName = "replay-app-main";
   appName = "ReplayWeb.page";
@@ -126,17 +129,28 @@ class Embed extends LitElement {
     }
   }
 
-  fullReload() {
+  async fullReload() {
-    const iframe = this.renderRoot.querySelector("iframe");
+    if (!this.iframe?.contentWindow) {
-    if (iframe?.contentWindow) {
+      return false;
-      iframe.contentWindow.postMessage({ type: "fullReload" });
     }
+
+    const deleteURL = apiPrefix + "/c/" + this.coll + "?reload=1";
+
+    const resp = await this.iframe.contentWindow.fetch(deleteURL, {
+      method: "DELETE",
+    });
+
+    if (resp.status !== 200) {
+      return false;
+    }
+
+    this.iframe.contentWindow.location.reload();
+
+    return true;
   }
 
   handleMessage(event: IframeMessage) {
-    const iframe = this.renderRoot.querySelector("iframe");
+    if (this.iframe && event.source === this.iframe.contentWindow) {
-
-    if (iframe && event.source === iframe.contentWindow) {
       switch (event.data.type) {
         case "urlchange":
           if (this.deepLink) {

src/item.ts

@@ -85,6 +85,7 @@ export type EmbedReplayData = {
   ts?: string;
   title?: string;
   query?: string;
+  waczhash?: string;
 };
 
 export type EmbedReplayEvent = EmbedReplayData & {
@@ -229,16 +230,6 @@ class Item extends LitElement {
 
       this.observer.observe(this);
     }
-
-    if (this.embed) {
-      window.addEventListener("message", (event: MessageEvent) => {
-        if (event.source === window.parent) {
-          if (event.data.type === "fullReload") {
-            void this.deleteFully(true);
-          }
-        }
-      });
-    }
   }
 
   async runUpdateLoop() {
@@ -976,6 +967,7 @@ class Item extends LitElement {
                   sourceUrl="${this.sourceUrl || ""}"
                   url="${this.tabData.url || ""}"
                   ts="${this.tabData.ts || ""}"
+                  waczhash="${this.tabData.waczhash || ""}"
                   @coll-tab-nav="${this.onItemTabNav}"
                   id="replay"
                   @replay-loading="${this.onReplayLoading}"

src/pageentry.ts

@@ -5,7 +5,7 @@ import { property, state } from "lit/decorators.js";
 
 import "keyword-mark-element/lib/keyword-mark.js";
 
-import { getReplayLink } from "./pageutils";
+import { getPageDateTS, getReplayLink } from "./pageutils";
 
 import { wrapCss } from "./misc";
 import type { URLResource } from "./types";
@@ -192,7 +192,7 @@ class PageEntry extends LitElement {
 
   render() {
     const p = this.page!;
-    const date = this.page!.date;
+    const { date } = getPageDateTS(p);
 
     const hasSize = typeof p.size === "number";
 
@@ -245,6 +245,7 @@ class PageEntry extends LitElement {
                     "pages",
                     this.page!.url,
                     this.page!.timestamp!,
+                    this.page!.waczhash,
                   )}"
                 >
                   <p class="is-size-6 has-text-weight-bold has-text-link text">
@@ -298,6 +299,12 @@ class PageEntry extends LitElement {
     `;
   }
 
+  private getReplayPrefix() {
+    const waczhash = this.page!.waczhash ? `:${this.page!.waczhash}/` : "";
+    const ts = this.page!.timestamp || "";
+    return this.replayPrefix + "/" + waczhash + ts + "id_";
+  }
+
   private renderPageIcon() {
     if (!this.thumbnailValid) {
       return this.renderFavicon();
@@ -305,9 +312,7 @@ class PageEntry extends LitElement {
     return html`<img
       class="thumbnail"
       @error=${() => (this.thumbnailValid = false)}
-      src=${`${this.replayPrefix}/${this.page!.timestamp}id_/urn:thumbnail:${
+      src=${`${this.getReplayPrefix()}/urn:thumbnail:${this.page!.url}`}
-        this.page!.url
-      }`}
       loading="lazy"
     />`;
   }
@@ -319,9 +324,7 @@ class PageEntry extends LitElement {
     return html`<img
       class="favicon"
       @error=${() => (this.iconValid = false)}
-      src=${`${this.replayPrefix}/${this.page!.timestamp}id_/${
+      src=${`${this.getReplayPrefix()}/${this.page!.favIconUrl}`}
-        this.page!.favIconUrl
-      }`}
       loading="lazy"
     />`;
   }
@@ -377,6 +380,7 @@ class PageEntry extends LitElement {
     const data = {
       url: this.page!.url,
       ts: this.page!.timestamp,
+      waczhash: this.page!.waczhash,
     };
     this.sendChangeEvent(data, reload);
     return false;

src/pages.ts

@@ -20,6 +20,8 @@ import type { PageEntry } from "./pageentry";
 import type { Id, Index } from "flexsearch";
 import type { ItemType, URLResource } from "./types";
 
+const DYNAMIC_PAGE_SIZE = 25;
+
 // ===========================================================================
 class Pages extends LitElement {
   @property({ type: Array })
@@ -103,6 +105,18 @@ class Pages extends LitElement {
   @property({ type: String })
   defaultKey = "";
 
+  @property({ type: Boolean })
+  dynamicPagesQuery = false;
+
+  @property({ type: Number })
+  totalPages = 0;
+
+  @property({ type: Number })
+  dynamicPageCount = 1;
+
+  @property({ type: Boolean })
+  skipScrollMore = false;
+
   private _ival: number | undefined;
 
   static get sortKeys() {
@@ -214,10 +228,25 @@ class Pages extends LitElement {
       );
     } else if (this.showAllPages && this.hasExtraPages) {
       this.filteredPages = [...this.textPages!];
-    } else {
+    } else if (!this.dynamicPagesQuery) {
       this.filteredPages = [...this.collInfo!.pages];
     }
 
+    this.totalPages = this.filteredPages.length;
+
+    if (this.dynamicPagesQuery) {
+      if (!this.query) {
+        const seedPages = this.collInfo!.pages.filter((x) => x.isSeed);
+        this.filteredPages =
+          this.showAllPages || !seedPages.length
+            ? [...this.collInfo!.pages]
+            : seedPages;
+        this.hasExtraPages = seedPages.length !== this.collInfo!.pages.length;
+      }
+      this.dynamicPageCount = 1;
+      await this.addDynamicPages();
+    }
+
     if (this.currList !== 0) {
       await this.filterCurated();
     }
@@ -238,6 +267,80 @@ class Pages extends LitElement {
     this.sendChangeEvent(data);
   }
 
+  async addDynamicPages() {
+    const search = new URLSearchParams();
+    if (this.query) {
+      search.set("search", this.query);
+    }
+    search.set("page", this.dynamicPageCount + "");
+    search.set("pageSize", DYNAMIC_PAGE_SIZE + "");
+
+    const resp = await fetch(
+      `${this.collInfo!.apiPrefix}/pages?${search.toString()}`,
+    );
+    const json = await resp.json();
+    if (!json.pages) {
+      return;
+    }
+
+    const knownPages = new Set();
+    this.filteredPages.forEach((x) => knownPages.add(x.id));
+
+    const newPages = [];
+
+    for (const {
+      id,
+      url,
+      title,
+      mime,
+      status,
+      ts,
+      favIconUrl,
+      waczhash,
+      isSeed,
+    } of json.pages) {
+      if (knownPages.has(id)) {
+        continue;
+      }
+
+      if (!this.showAllPages && this.hasExtraPages && !this.query && !isSeed) {
+        continue;
+      }
+
+      let tsActual;
+
+      if (typeof ts === "string") {
+        tsActual = new Date(ts).getTime();
+      } else {
+        tsActual = ts;
+      }
+
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      const newPage: any = {
+        id,
+        url,
+        title,
+        mime,
+        status,
+        ts: tsActual,
+        favIconUrl,
+        waczhash,
+      };
+
+      newPages.push(newPage);
+    }
+
+    if (newPages.length) {
+      this.filteredPages = [...this.filteredPages, ...newPages];
+    }
+
+    if (json.total) {
+      this.totalPages = json.total;
+    } else {
+      this.totalPages = this.filteredPages.length;
+    }
+  }
+
   async filterCurated() {
     const resp = await fetch(
       `${this.collInfo!.apiPrefix}/curated/${this.currList}`,
@@ -263,11 +366,17 @@ class Pages extends LitElement {
     this.flex = flex;
     this.textPages = pages;
 
+    if (!this.dynamicPagesQuery) {
       this.hasExtraPages = Boolean(
         this.textPages &&
           this.collInfo?.pages &&
           this.textPages.length > this.collInfo.pages.length,
       );
+    }
+
+    if (this.collInfo) {
+      this.dynamicPagesQuery = this.collInfo.canQueryPages || false;
+    }
 
     return Promise.all(
       // @ts-expect-error [// TODO: Fix this the next time the file is edited.] - TS7006 - Parameter 'page' implicitly has an 'any' type. | TS7006 - Parameter 'index' implicitly has an 'any' type.
@@ -893,7 +1002,7 @@ class Pages extends LitElement {
           .sortDesc="${this.sortDesc}"
           .sortKeys="${Pages.sortKeys}"
           .data="${this.filteredPages}"
-          pageResults="100"
+          .pageResults="${this.dynamicPagesQuery ? this.totalPages : 100}"
           @sort-changed="${this.onSortChanged}"
           class="${this.filteredPages.length ? "" : "is-hidden"}"
         >
@@ -1064,6 +1173,8 @@ class Pages extends LitElement {
     this.sortedPages = event.detail.sortedData;
     this.sortKey = event.detail.sortKey;
     this.sortDesc = event.detail.sortDesc;
+
+    void this.updateComplete.then(() => (this.skipScrollMore = false));
   }
 
   // @ts-expect-error [// TODO: Fix this the next time the file is edited.] - TS7006 - Parameter 'event' implicitly has an 'any' type.
@@ -1184,9 +1295,7 @@ class Pages extends LitElement {
   }
 
   formatResults() {
-    // Default behavior: Count all available pages
+    const length = this.totalPages;
-    if (!this.query) {
-      const length = this.filteredPages.length;
     if (length === this.sortedPages.length) {
       return `${length} Page${length !== 1 ? "s" : ""}`;
     } else {
@@ -1194,14 +1303,6 @@ class Pages extends LitElement {
     }
   }
 
-    // ... unless they were filtered
-    if (this.sortedPages.length === 1) {
-      return "1 Page";
-    } else {
-      return `${this.sortedPages.length} Pages`;
-    }
-  }
-
   getNoResultsMessage() {
     if (!this.collInfo?.pages.length) {
       return html`<span class="fix-text-wrapping"
@@ -1229,14 +1330,23 @@ class Pages extends LitElement {
   }
 
   // @ts-expect-error [// TODO: Fix this the next time the file is edited.] - TS7006 - Parameter 'event' implicitly has an 'any' type.
-  onScroll(event) {
+  async onScroll(event) {
     const element = event.currentTarget;
     const diff =
       element.scrollHeight - element.scrollTop - element.clientHeight;
-    if (diff < 40) {
+    if (diff < 40 && !this.skipScrollMore) {
+      this.skipScrollMore = true;
+      if (
+        this.dynamicPagesQuery &&
+        this.filteredPages.length < this.totalPages
+      ) {
+        this.dynamicPageCount += 1;
+        await this.addDynamicPages();
+      }
+
       const sorter = this.renderRoot.querySelector<Sorter>("wr-sorter");
       if (sorter) {
-        sorter.getMore();
+        sorter.getMore(DYNAMIC_PAGE_SIZE);
       }
     }
   }

src/pageutils.ts

@@ -69,11 +69,19 @@ function getTS(iso: string) {
 }
 
 // ===========================================================================
-function getReplayLink(view: string, url: string, ts: string) {
+function getReplayLink(
+  view: string,
+  url: string,
+  ts: string,
+  waczhash?: string,
+) {
   const params = new URLSearchParams();
   params.set("view", view);
   params.set("url", url);
   params.set("ts", ts);
+  if (waczhash) {
+    params.set("waczhash", waczhash);
+  }
   return "#" + params.toString();
 }
 

src/replay.ts

@@ -20,6 +20,9 @@ class Replay extends LitElement {
   @property({ type: String })
   ts = "";
 
+  @property({ type: String })
+  waczhash = "";
+
   // actual replay url
   @property({ type: String })
   replayUrl = "";
@@ -97,7 +100,9 @@ class Replay extends LitElement {
   doSetIframeUrl() {
     this.iframeUrl =
       this.url && this.collInfo
-        ? `${this.collInfo.replayPrefix}/${this.ts || ""}mp_/${this.url}`
+        ? `${this.collInfo.replayPrefix}/${
+            this.waczhash ? `:${this.waczhash}/` : ""
+          }${this.ts || ""}mp_/${this.url}`
         : "";
   }
 

src/types.ts

@@ -11,9 +11,12 @@ export type URLResource = {
   size: number;
   favIconUrl: string;
   text?: string;
+  waczhash?: string;
 };
 
-export type Page = URLResource;
+export type Page = URLResource & {
+  isSeed?: boolean;
+};
 
 export type ItemType = {
   filename?: string;
@@ -43,12 +46,13 @@ export type ItemType = {
     software?: string;
   };
   onDemand?: boolean;
-  pages: URLResource[];
+  pages: Page[];
   curatedPages: (URLResource & { list: string })[];
   lists: URLResource[];
   ctime?: string;
   totalSize?: unknown;
   size?: number | string;
+  canQueryPages?: boolean;
 };
 
 export type FavIconEventDetail = {

yarn.lock

@@ -1028,16 +1028,16 @@
   resolved "https://registry.yarnpkg.com/@webpack-cli/serve/-/serve-2.0.5.tgz#325db42395cd49fe6c14057f9a900e427df8810e"
   integrity sha512-lqaoKnRYBdo1UgDX8uF24AfGMifWK19TxPmM5FHc2vAGxrJ/qtyUyFBWoY1tISZdelsQ5fBcOusifo5o5wSJxQ==
 
-"@webrecorder/wabac@^2.20.8":
+"@webrecorder/wabac@^2.21.0":
-  version "2.20.8"
+  version "2.21.0"
-  resolved "https://registry.yarnpkg.com/@webrecorder/wabac/-/wabac-2.20.8.tgz#37dd14f4cf821f36ecdf5719d5f334bb3445290e"
+  resolved "https://registry.yarnpkg.com/@webrecorder/wabac/-/wabac-2.21.0.tgz#cc3483f15cb3f1502fdef89e1c9a4ee5709bcaec"
-  integrity sha512-P6FxC17ZgPfwfp3HbdI9hkzMnqD5I3kBF0cMnue6m8wBVMCzcVGp9x43wT+pcq0zOv6ATGObxcmjJKCsDUus7A==
+  integrity sha512-28G3ci/m/qJXnLDoUP/Kt09PiTuORBZH0bw/JlKhPCeLL4IdYsODeXaImVwvTT12R9tV7Zp37w26jhkMfcur0w==
   dependencies:
     "@peculiar/asn1-ecc" "^2.3.4"
     "@peculiar/asn1-schema" "^2.3.3"
     "@peculiar/x509" "^1.9.2"
     "@types/js-levenshtein" "^1.1.3"
-    "@webrecorder/wombat" "^3.8.7"
+    "@webrecorder/wombat" "^3.8.8"
     acorn "^8.10.0"
     auto-js-ipfs "^2.1.1"
     base64-js "^1.5.1"
@@ -1058,10 +1058,10 @@
     stream-browserify "^3.0.0"
     warcio "^2.4.3"
 
-"@webrecorder/wombat@^3.8.7":
+"@webrecorder/wombat@^3.8.8":
-  version "3.8.7"
+  version "3.8.8"
-  resolved "https://registry.yarnpkg.com/@webrecorder/wombat/-/wombat-3.8.7.tgz#51c7465c589e0020be064121127c7c10a38ec21f"
+  resolved "https://registry.yarnpkg.com/@webrecorder/wombat/-/wombat-3.8.8.tgz#aab4dd8eea6d6cb17bfefb7ee1802e7b45b11ed7"
-  integrity sha512-bW5V7cBweTkTazOIN8oZZGwHLevsGNv1luY3t0RYdEZhs5BDpTmUHN33zEbrXDOiPUlY3N3I8+73VA+PuxihoQ==
+  integrity sha512-XkJOZAyHrdXNkAVoISQEh/NHzaBMekQZfWqes/k2vYkW6v9DmZ0wjP7Kf6MHCuajKX8uSH+caB2tv1kJgvnv3Q==
   dependencies:
     warcio "^2.4.0"