Support for dynamic page loading for Multi-WACZs (#399)

Support querying pages dynamically via pages endpoint: - If loaded pages < total pages, load additional pages via /pages endpoint if canQueryPages is set to true on item - If query provided, do `/pages?search=...` to search for pages. Use pagination of 25 pages for now. WACZ hash: - if provided, store waczhash per page, and use for thumbnail, favicon and top-level replay loading. Embed: better support for force reloading, call delete with reload directly in embed Depends on webrecorder/wabac.js#220
2025-02-24 14:35:03 -08:00
parent 94cbf5e43d
commit 6ef8bf4973
9 changed files with 203 additions and 66 deletions
@@ -7,9 +7,9 @@ import {
 } from "lit";
 import { ifDefined } from "lit/directives/if-defined.js";

-import { wrapCss, updateFaviconLinks } from "./misc";
+import { wrapCss, updateFaviconLinks, apiPrefix } from "./misc";
 import { SWManager } from "./swmanager";
-import { property } from "lit/decorators.js";
+import { property, query } from "lit/decorators.js";
 import type { FavIconEventDetail } from "./types";
 import type { EmbedReplayData, EmbedReplayEvent } from "./item";

@@ -82,6 +82,9 @@ class Embed extends LitElement {

  @property({ type: Boolean }) useRuffle = false;

+  @query("iframe")
+  private readonly iframe?: HTMLIFrameElement | null;
+
  replayfile = defaultReplayFile;
  mainElementName = "replay-app-main";
  appName = "ReplayWeb.page";
@@ -126,17 +129,28 @@ class Embed extends LitElement {
    }
  }

-  fullReload() {
-    const iframe = this.renderRoot.querySelector("iframe");
-    if (iframe?.contentWindow) {
-      iframe.contentWindow.postMessage({ type: "fullReload" });
+  async fullReload() {
+    if (!this.iframe?.contentWindow) {
+      return false;
    }
+
+    const deleteURL = apiPrefix + "/c/" + this.coll + "?reload=1";
+
+    const resp = await this.iframe.contentWindow.fetch(deleteURL, {
+      method: "DELETE",
+    });
+
+    if (resp.status !== 200) {
+      return false;
+    }
+
+    this.iframe.contentWindow.location.reload();
+
+    return true;
  }

  handleMessage(event: IframeMessage) {
-    const iframe = this.renderRoot.querySelector("iframe");
-
-    if (iframe && event.source === iframe.contentWindow) {
+    if (this.iframe && event.source === this.iframe.contentWindow) {
      switch (event.data.type) {
        case "urlchange":
          if (this.deepLink) {
@@ -85,6 +85,7 @@ export type EmbedReplayData = {
  ts?: string;
  title?: string;
  query?: string;
+  waczhash?: string;
 };

 export type EmbedReplayEvent = EmbedReplayData & {
@@ -229,16 +230,6 @@ class Item extends LitElement {

      this.observer.observe(this);
    }
-
-    if (this.embed) {
-      window.addEventListener("message", (event: MessageEvent) => {
-        if (event.source === window.parent) {
-          if (event.data.type === "fullReload") {
-            void this.deleteFully(true);
-          }
-        }
-      });
-    }
  }

  async runUpdateLoop() {
@@ -976,6 +967,7 @@ class Item extends LitElement {
                  sourceUrl="${this.sourceUrl || ""}"
                  url="${this.tabData.url || ""}"
                  ts="${this.tabData.ts || ""}"
+                  waczhash="${this.tabData.waczhash || ""}"
                  @coll-tab-nav="${this.onItemTabNav}"
                  id="replay"
                  @replay-loading="${this.onReplayLoading}"
@@ -5,7 +5,7 @@ import { property, state } from "lit/decorators.js";

 import "keyword-mark-element/lib/keyword-mark.js";

-import { getReplayLink } from "./pageutils";
+import { getPageDateTS, getReplayLink } from "./pageutils";

 import { wrapCss } from "./misc";
 import type { URLResource } from "./types";
@@ -192,7 +192,7 @@ class PageEntry extends LitElement {

  render() {
    const p = this.page!;
-    const date = this.page!.date;
+    const { date } = getPageDateTS(p);

    const hasSize = typeof p.size === "number";

@@ -245,6 +245,7 @@ class PageEntry extends LitElement {
                    "pages",
                    this.page!.url,
                    this.page!.timestamp!,
+                    this.page!.waczhash,
                  )}"
                >
                  <p class="is-size-6 has-text-weight-bold has-text-link text">
@@ -298,6 +299,12 @@ class PageEntry extends LitElement {
    `;
  }

+  private getReplayPrefix() {
+    const waczhash = this.page!.waczhash ? `:${this.page!.waczhash}/` : "";
+    const ts = this.page!.timestamp || "";
+    return this.replayPrefix + "/" + waczhash + ts + "id_";
+  }
+
  private renderPageIcon() {
    if (!this.thumbnailValid) {
      return this.renderFavicon();
@@ -305,9 +312,7 @@ class PageEntry extends LitElement {
    return html`<img
      class="thumbnail"
      @error=${() => (this.thumbnailValid = false)}
-      src=${`${this.replayPrefix}/${this.page!.timestamp}id_/urn:thumbnail:${
-        this.page!.url
-      }`}
+      src=${`${this.getReplayPrefix()}/urn:thumbnail:${this.page!.url}`}
      loading="lazy"
    />`;
  }
@@ -319,9 +324,7 @@ class PageEntry extends LitElement {
    return html`<img
      class="favicon"
      @error=${() => (this.iconValid = false)}
-      src=${`${this.replayPrefix}/${this.page!.timestamp}id_/${
-        this.page!.favIconUrl
-      }`}
+      src=${`${this.getReplayPrefix()}/${this.page!.favIconUrl}`}
      loading="lazy"
    />`;
  }
@@ -377,6 +380,7 @@ class PageEntry extends LitElement {
    const data = {
      url: this.page!.url,
      ts: this.page!.timestamp,
+      waczhash: this.page!.waczhash,
    };
    this.sendChangeEvent(data, reload);
    return false;
@@ -20,6 +20,8 @@ import type { PageEntry } from "./pageentry";
 import type { Id, Index } from "flexsearch";
 import type { ItemType, URLResource } from "./types";

+const DYNAMIC_PAGE_SIZE = 25;
+
 // ===========================================================================
 class Pages extends LitElement {
  @property({ type: Array })
@@ -103,6 +105,18 @@ class Pages extends LitElement {
  @property({ type: String })
  defaultKey = "";

+  @property({ type: Boolean })
+  dynamicPagesQuery = false;
+
+  @property({ type: Number })
+  totalPages = 0;
+
+  @property({ type: Number })
+  dynamicPageCount = 1;
+
+  @property({ type: Boolean })
+  skipScrollMore = false;
+
  private _ival: number | undefined;

  static get sortKeys() {
@@ -214,10 +228,25 @@ class Pages extends LitElement {
      );
    } else if (this.showAllPages && this.hasExtraPages) {
      this.filteredPages = [...this.textPages!];
-    } else {
+    } else if (!this.dynamicPagesQuery) {
      this.filteredPages = [...this.collInfo!.pages];
    }

+    this.totalPages = this.filteredPages.length;
+
+    if (this.dynamicPagesQuery) {
+      if (!this.query) {
+        const seedPages = this.collInfo!.pages.filter((x) => x.isSeed);
+        this.filteredPages =
+          this.showAllPages || !seedPages.length
+            ? [...this.collInfo!.pages]
+            : seedPages;
+        this.hasExtraPages = seedPages.length !== this.collInfo!.pages.length;
+      }
+      this.dynamicPageCount = 1;
+      await this.addDynamicPages();
+    }
+
    if (this.currList !== 0) {
      await this.filterCurated();
    }
@@ -238,6 +267,80 @@ class Pages extends LitElement {
    this.sendChangeEvent(data);
  }

+  async addDynamicPages() {
+    const search = new URLSearchParams();
+    if (this.query) {
+      search.set("search", this.query);
+    }
+    search.set("page", this.dynamicPageCount + "");
+    search.set("pageSize", DYNAMIC_PAGE_SIZE + "");
+
+    const resp = await fetch(
+      `${this.collInfo!.apiPrefix}/pages?${search.toString()}`,
+    );
+    const json = await resp.json();
+    if (!json.pages) {
+      return;
+    }
+
+    const knownPages = new Set();
+    this.filteredPages.forEach((x) => knownPages.add(x.id));
+
+    const newPages = [];
+
+    for (const {
+      id,
+      url,
+      title,
+      mime,
+      status,
+      ts,
+      favIconUrl,
+      waczhash,
+      isSeed,
+    } of json.pages) {
+      if (knownPages.has(id)) {
+        continue;
+      }
+
+      if (!this.showAllPages && this.hasExtraPages && !this.query && !isSeed) {
+        continue;
+      }
+
+      let tsActual;
+
+      if (typeof ts === "string") {
+        tsActual = new Date(ts).getTime();
+      } else {
+        tsActual = ts;
+      }
+
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      const newPage: any = {
+        id,
+        url,
+        title,
+        mime,
+        status,
+        ts: tsActual,
+        favIconUrl,
+        waczhash,
+      };
+
+      newPages.push(newPage);
+    }
+
+    if (newPages.length) {
+      this.filteredPages = [...this.filteredPages, ...newPages];
+    }
+
+    if (json.total) {
+      this.totalPages = json.total;
+    } else {
+      this.totalPages = this.filteredPages.length;
+    }
+  }
+
  async filterCurated() {
    const resp = await fetch(
      `${this.collInfo!.apiPrefix}/curated/${this.currList}`,
@@ -263,11 +366,17 @@ class Pages extends LitElement {
    this.flex = flex;
    this.textPages = pages;

-    this.hasExtraPages = Boolean(
-      this.textPages &&
-        this.collInfo?.pages &&
-        this.textPages.length > this.collInfo.pages.length,
-    );
+    if (!this.dynamicPagesQuery) {
+      this.hasExtraPages = Boolean(
+        this.textPages &&
+          this.collInfo?.pages &&
+          this.textPages.length > this.collInfo.pages.length,
+      );
+    }
+
+    if (this.collInfo) {
+      this.dynamicPagesQuery = this.collInfo.canQueryPages || false;
+    }

    return Promise.all(
      // @ts-expect-error [// TODO: Fix this the next time the file is edited.] - TS7006 - Parameter 'page' implicitly has an 'any' type. | TS7006 - Parameter 'index' implicitly has an 'any' type.
@@ -893,7 +1002,7 @@ class Pages extends LitElement {
          .sortDesc="${this.sortDesc}"
          .sortKeys="${Pages.sortKeys}"
          .data="${this.filteredPages}"
-          pageResults="100"
+          .pageResults="${this.dynamicPagesQuery ? this.totalPages : 100}"
          @sort-changed="${this.onSortChanged}"
          class="${this.filteredPages.length ? "" : "is-hidden"}"
        >
@@ -1064,6 +1173,8 @@ class Pages extends LitElement {
    this.sortedPages = event.detail.sortedData;
    this.sortKey = event.detail.sortKey;
    this.sortDesc = event.detail.sortDesc;
+
+    void this.updateComplete.then(() => (this.skipScrollMore = false));
  }

  // @ts-expect-error [// TODO: Fix this the next time the file is edited.] - TS7006 - Parameter 'event' implicitly has an 'any' type.
@@ -1184,21 +1295,11 @@ class Pages extends LitElement {
  }

  formatResults() {
-    // Default behavior: Count all available pages
-    if (!this.query) {
-      const length = this.filteredPages.length;
-      if (length === this.sortedPages.length) {
-        return `${length} Page${length !== 1 ? "s" : ""}`;
-      } else {
-        return `${this.sortedPages.length} of ${length} Pages Shown`;
-      }
-    }
-
-    // ... unless they were filtered
-    if (this.sortedPages.length === 1) {
-      return "1 Page";
+    const length = this.totalPages;
+    if (length === this.sortedPages.length) {
+      return `${length} Page${length !== 1 ? "s" : ""}`;
    } else {
-      return `${this.sortedPages.length} Pages`;
+      return `${this.sortedPages.length} of ${length} Pages Shown`;
    }
  }

@@ -1229,14 +1330,23 @@ class Pages extends LitElement {
  }

  // @ts-expect-error [// TODO: Fix this the next time the file is edited.] - TS7006 - Parameter 'event' implicitly has an 'any' type.
-  onScroll(event) {
+  async onScroll(event) {
    const element = event.currentTarget;
    const diff =
      element.scrollHeight - element.scrollTop - element.clientHeight;
-    if (diff < 40) {
+    if (diff < 40 && !this.skipScrollMore) {
+      this.skipScrollMore = true;
+      if (
+        this.dynamicPagesQuery &&
+        this.filteredPages.length < this.totalPages
+      ) {
+        this.dynamicPageCount += 1;
+        await this.addDynamicPages();
+      }
+
      const sorter = this.renderRoot.querySelector<Sorter>("wr-sorter");
      if (sorter) {
-        sorter.getMore();
+        sorter.getMore(DYNAMIC_PAGE_SIZE);
      }
    }
  }
@@ -69,11 +69,19 @@ function getTS(iso: string) {
 }

 // ===========================================================================
-function getReplayLink(view: string, url: string, ts: string) {
+function getReplayLink(
+  view: string,
+  url: string,
+  ts: string,
+  waczhash?: string,
+) {
  const params = new URLSearchParams();
  params.set("view", view);
  params.set("url", url);
  params.set("ts", ts);
+  if (waczhash) {
+    params.set("waczhash", waczhash);
+  }
  return "#" + params.toString();
 }

@@ -20,6 +20,9 @@ class Replay extends LitElement {
  @property({ type: String })
  ts = "";

+  @property({ type: String })
+  waczhash = "";
+
  // actual replay url
  @property({ type: String })
  replayUrl = "";
@@ -97,7 +100,9 @@ class Replay extends LitElement {
  doSetIframeUrl() {
    this.iframeUrl =
      this.url && this.collInfo
-        ? `${this.collInfo.replayPrefix}/${this.ts || ""}mp_/${this.url}`
+        ? `${this.collInfo.replayPrefix}/${
+            this.waczhash ? `:${this.waczhash}/` : ""
+          }${this.ts || ""}mp_/${this.url}`
        : "";
  }

@@ -11,9 +11,12 @@ export type URLResource = {
  size: number;
  favIconUrl: string;
  text?: string;
+  waczhash?: string;
 };

-export type Page = URLResource;
+export type Page = URLResource & {
+  isSeed?: boolean;
+};

 export type ItemType = {
  filename?: string;
@@ -43,12 +46,13 @@ export type ItemType = {
    software?: string;
  };
  onDemand?: boolean;
-  pages: URLResource[];
+  pages: Page[];
  curatedPages: (URLResource & { list: string })[];
  lists: URLResource[];
  ctime?: string;
  totalSize?: unknown;
  size?: number | string;
+  canQueryPages?: boolean;
 };

 export type FavIconEventDetail = {