Support for dynamic page loading for Multi-WACZs (#399)
Support querying pages dynamically via pages endpoint: - If loaded pages < total pages, load additional pages via /pages endpoint if canQueryPages is set to true on item - If query provided, do `/pages?search=...` to search for pages. Use pagination of 25 pages for now. WACZ hash: - if provided, store waczhash per page, and use for thumbnail, favicon and top-level replay loading. Embed: better support for force reloading, call delete with reload directly in embed Depends on webrecorder/wabac.js#220
This commit is contained in:
Ilya Kreymer
+2
-2
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "replaywebpage",
|
||||
"productName": "ReplayWeb.page",
|
||||
"version": "2.2.5",
|
||||
"version": "2.3.0",
|
||||
"description": "Serverless Web Archive Replay",
|
||||
"repository": "https://github.com/webrecorder/replayweb.page",
|
||||
"homepage": "https://replayweb.page/",
|
||||
@@ -18,7 +18,7 @@
|
||||
"dependencies": {
|
||||
"@fortawesome/fontawesome-free": "^5.15.4",
|
||||
"@shoelace-style/shoelace": "~2.15.1",
|
||||
"@webrecorder/wabac": "^2.20.8",
|
||||
"@webrecorder/wabac": "^2.21.0",
|
||||
"bulma": "^0.9.3",
|
||||
"electron-log": "^4.4.1",
|
||||
"electron-updater": "^6.3.9",
|
||||
|
||||
+23
-9
@@ -7,9 +7,9 @@ import {
|
||||
} from "lit";
|
||||
import { ifDefined } from "lit/directives/if-defined.js";
|
||||
|
||||
import { wrapCss, updateFaviconLinks } from "./misc";
|
||||
import { wrapCss, updateFaviconLinks, apiPrefix } from "./misc";
|
||||
import { SWManager } from "./swmanager";
|
||||
import { property } from "lit/decorators.js";
|
||||
import { property, query } from "lit/decorators.js";
|
||||
import type { FavIconEventDetail } from "./types";
|
||||
import type { EmbedReplayData, EmbedReplayEvent } from "./item";
|
||||
|
||||
@@ -82,6 +82,9 @@ class Embed extends LitElement {
|
||||
|
||||
@property({ type: Boolean }) useRuffle = false;
|
||||
|
||||
@query("iframe")
|
||||
private readonly iframe?: HTMLIFrameElement | null;
|
||||
|
||||
replayfile = defaultReplayFile;
|
||||
mainElementName = "replay-app-main";
|
||||
appName = "ReplayWeb.page";
|
||||
@@ -126,17 +129,28 @@ class Embed extends LitElement {
|
||||
}
|
||||
}
|
||||
|
||||
fullReload() {
|
||||
const iframe = this.renderRoot.querySelector("iframe");
|
||||
if (iframe?.contentWindow) {
|
||||
iframe.contentWindow.postMessage({ type: "fullReload" });
|
||||
async fullReload() {
|
||||
if (!this.iframe?.contentWindow) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const deleteURL = apiPrefix + "/c/" + this.coll + "?reload=1";
|
||||
|
||||
const resp = await this.iframe.contentWindow.fetch(deleteURL, {
|
||||
method: "DELETE",
|
||||
});
|
||||
|
||||
if (resp.status !== 200) {
|
||||
return false;
|
||||
}
|
||||
|
||||
this.iframe.contentWindow.location.reload();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
handleMessage(event: IframeMessage) {
|
||||
const iframe = this.renderRoot.querySelector("iframe");
|
||||
|
||||
if (iframe && event.source === iframe.contentWindow) {
|
||||
if (this.iframe && event.source === this.iframe.contentWindow) {
|
||||
switch (event.data.type) {
|
||||
case "urlchange":
|
||||
if (this.deepLink) {
|
||||
|
||||
+2
-10
@@ -85,6 +85,7 @@ export type EmbedReplayData = {
|
||||
ts?: string;
|
||||
title?: string;
|
||||
query?: string;
|
||||
waczhash?: string;
|
||||
};
|
||||
|
||||
export type EmbedReplayEvent = EmbedReplayData & {
|
||||
@@ -229,16 +230,6 @@ class Item extends LitElement {
|
||||
|
||||
this.observer.observe(this);
|
||||
}
|
||||
|
||||
if (this.embed) {
|
||||
window.addEventListener("message", (event: MessageEvent) => {
|
||||
if (event.source === window.parent) {
|
||||
if (event.data.type === "fullReload") {
|
||||
void this.deleteFully(true);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async runUpdateLoop() {
|
||||
@@ -976,6 +967,7 @@ class Item extends LitElement {
|
||||
sourceUrl="${this.sourceUrl || ""}"
|
||||
url="${this.tabData.url || ""}"
|
||||
ts="${this.tabData.ts || ""}"
|
||||
waczhash="${this.tabData.waczhash || ""}"
|
||||
@coll-tab-nav="${this.onItemTabNav}"
|
||||
id="replay"
|
||||
@replay-loading="${this.onReplayLoading}"
|
||||
|
||||
+12
-8
@@ -5,7 +5,7 @@ import { property, state } from "lit/decorators.js";
|
||||
|
||||
import "keyword-mark-element/lib/keyword-mark.js";
|
||||
|
||||
import { getReplayLink } from "./pageutils";
|
||||
import { getPageDateTS, getReplayLink } from "./pageutils";
|
||||
|
||||
import { wrapCss } from "./misc";
|
||||
import type { URLResource } from "./types";
|
||||
@@ -192,7 +192,7 @@ class PageEntry extends LitElement {
|
||||
|
||||
render() {
|
||||
const p = this.page!;
|
||||
const date = this.page!.date;
|
||||
const { date } = getPageDateTS(p);
|
||||
|
||||
const hasSize = typeof p.size === "number";
|
||||
|
||||
@@ -245,6 +245,7 @@ class PageEntry extends LitElement {
|
||||
"pages",
|
||||
this.page!.url,
|
||||
this.page!.timestamp!,
|
||||
this.page!.waczhash,
|
||||
)}"
|
||||
>
|
||||
<p class="is-size-6 has-text-weight-bold has-text-link text">
|
||||
@@ -298,6 +299,12 @@ class PageEntry extends LitElement {
|
||||
`;
|
||||
}
|
||||
|
||||
private getReplayPrefix() {
|
||||
const waczhash = this.page!.waczhash ? `:${this.page!.waczhash}/` : "";
|
||||
const ts = this.page!.timestamp || "";
|
||||
return this.replayPrefix + "/" + waczhash + ts + "id_";
|
||||
}
|
||||
|
||||
private renderPageIcon() {
|
||||
if (!this.thumbnailValid) {
|
||||
return this.renderFavicon();
|
||||
@@ -305,9 +312,7 @@ class PageEntry extends LitElement {
|
||||
return html`<img
|
||||
class="thumbnail"
|
||||
@error=${() => (this.thumbnailValid = false)}
|
||||
src=${`${this.replayPrefix}/${this.page!.timestamp}id_/urn:thumbnail:${
|
||||
this.page!.url
|
||||
}`}
|
||||
src=${`${this.getReplayPrefix()}/urn:thumbnail:${this.page!.url}`}
|
||||
loading="lazy"
|
||||
/>`;
|
||||
}
|
||||
@@ -319,9 +324,7 @@ class PageEntry extends LitElement {
|
||||
return html`<img
|
||||
class="favicon"
|
||||
@error=${() => (this.iconValid = false)}
|
||||
src=${`${this.replayPrefix}/${this.page!.timestamp}id_/${
|
||||
this.page!.favIconUrl
|
||||
}`}
|
||||
src=${`${this.getReplayPrefix()}/${this.page!.favIconUrl}`}
|
||||
loading="lazy"
|
||||
/>`;
|
||||
}
|
||||
@@ -377,6 +380,7 @@ class PageEntry extends LitElement {
|
||||
const data = {
|
||||
url: this.page!.url,
|
||||
ts: this.page!.timestamp,
|
||||
waczhash: this.page!.waczhash,
|
||||
};
|
||||
this.sendChangeEvent(data, reload);
|
||||
return false;
|
||||
|
||||
+126
-16
@@ -20,6 +20,8 @@ import type { PageEntry } from "./pageentry";
|
||||
import type { Id, Index } from "flexsearch";
|
||||
import type { ItemType, URLResource } from "./types";
|
||||
|
||||
const DYNAMIC_PAGE_SIZE = 25;
|
||||
|
||||
// ===========================================================================
|
||||
class Pages extends LitElement {
|
||||
@property({ type: Array })
|
||||
@@ -103,6 +105,18 @@ class Pages extends LitElement {
|
||||
@property({ type: String })
|
||||
defaultKey = "";
|
||||
|
||||
@property({ type: Boolean })
|
||||
dynamicPagesQuery = false;
|
||||
|
||||
@property({ type: Number })
|
||||
totalPages = 0;
|
||||
|
||||
@property({ type: Number })
|
||||
dynamicPageCount = 1;
|
||||
|
||||
@property({ type: Boolean })
|
||||
skipScrollMore = false;
|
||||
|
||||
private _ival: number | undefined;
|
||||
|
||||
static get sortKeys() {
|
||||
@@ -214,10 +228,25 @@ class Pages extends LitElement {
|
||||
);
|
||||
} else if (this.showAllPages && this.hasExtraPages) {
|
||||
this.filteredPages = [...this.textPages!];
|
||||
} else {
|
||||
} else if (!this.dynamicPagesQuery) {
|
||||
this.filteredPages = [...this.collInfo!.pages];
|
||||
}
|
||||
|
||||
this.totalPages = this.filteredPages.length;
|
||||
|
||||
if (this.dynamicPagesQuery) {
|
||||
if (!this.query) {
|
||||
const seedPages = this.collInfo!.pages.filter((x) => x.isSeed);
|
||||
this.filteredPages =
|
||||
this.showAllPages || !seedPages.length
|
||||
? [...this.collInfo!.pages]
|
||||
: seedPages;
|
||||
this.hasExtraPages = seedPages.length !== this.collInfo!.pages.length;
|
||||
}
|
||||
this.dynamicPageCount = 1;
|
||||
await this.addDynamicPages();
|
||||
}
|
||||
|
||||
if (this.currList !== 0) {
|
||||
await this.filterCurated();
|
||||
}
|
||||
@@ -238,6 +267,80 @@ class Pages extends LitElement {
|
||||
this.sendChangeEvent(data);
|
||||
}
|
||||
|
||||
async addDynamicPages() {
|
||||
const search = new URLSearchParams();
|
||||
if (this.query) {
|
||||
search.set("search", this.query);
|
||||
}
|
||||
search.set("page", this.dynamicPageCount + "");
|
||||
search.set("pageSize", DYNAMIC_PAGE_SIZE + "");
|
||||
|
||||
const resp = await fetch(
|
||||
`${this.collInfo!.apiPrefix}/pages?${search.toString()}`,
|
||||
);
|
||||
const json = await resp.json();
|
||||
if (!json.pages) {
|
||||
return;
|
||||
}
|
||||
|
||||
const knownPages = new Set();
|
||||
this.filteredPages.forEach((x) => knownPages.add(x.id));
|
||||
|
||||
const newPages = [];
|
||||
|
||||
for (const {
|
||||
id,
|
||||
url,
|
||||
title,
|
||||
mime,
|
||||
status,
|
||||
ts,
|
||||
favIconUrl,
|
||||
waczhash,
|
||||
isSeed,
|
||||
} of json.pages) {
|
||||
if (knownPages.has(id)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!this.showAllPages && this.hasExtraPages && !this.query && !isSeed) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let tsActual;
|
||||
|
||||
if (typeof ts === "string") {
|
||||
tsActual = new Date(ts).getTime();
|
||||
} else {
|
||||
tsActual = ts;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const newPage: any = {
|
||||
id,
|
||||
url,
|
||||
title,
|
||||
mime,
|
||||
status,
|
||||
ts: tsActual,
|
||||
favIconUrl,
|
||||
waczhash,
|
||||
};
|
||||
|
||||
newPages.push(newPage);
|
||||
}
|
||||
|
||||
if (newPages.length) {
|
||||
this.filteredPages = [...this.filteredPages, ...newPages];
|
||||
}
|
||||
|
||||
if (json.total) {
|
||||
this.totalPages = json.total;
|
||||
} else {
|
||||
this.totalPages = this.filteredPages.length;
|
||||
}
|
||||
}
|
||||
|
||||
async filterCurated() {
|
||||
const resp = await fetch(
|
||||
`${this.collInfo!.apiPrefix}/curated/${this.currList}`,
|
||||
@@ -263,11 +366,17 @@ class Pages extends LitElement {
|
||||
this.flex = flex;
|
||||
this.textPages = pages;
|
||||
|
||||
if (!this.dynamicPagesQuery) {
|
||||
this.hasExtraPages = Boolean(
|
||||
this.textPages &&
|
||||
this.collInfo?.pages &&
|
||||
this.textPages.length > this.collInfo.pages.length,
|
||||
);
|
||||
}
|
||||
|
||||
if (this.collInfo) {
|
||||
this.dynamicPagesQuery = this.collInfo.canQueryPages || false;
|
||||
}
|
||||
|
||||
return Promise.all(
|
||||
// @ts-expect-error [// TODO: Fix this the next time the file is edited.] - TS7006 - Parameter 'page' implicitly has an 'any' type. | TS7006 - Parameter 'index' implicitly has an 'any' type.
|
||||
@@ -893,7 +1002,7 @@ class Pages extends LitElement {
|
||||
.sortDesc="${this.sortDesc}"
|
||||
.sortKeys="${Pages.sortKeys}"
|
||||
.data="${this.filteredPages}"
|
||||
pageResults="100"
|
||||
.pageResults="${this.dynamicPagesQuery ? this.totalPages : 100}"
|
||||
@sort-changed="${this.onSortChanged}"
|
||||
class="${this.filteredPages.length ? "" : "is-hidden"}"
|
||||
>
|
||||
@@ -1064,6 +1173,8 @@ class Pages extends LitElement {
|
||||
this.sortedPages = event.detail.sortedData;
|
||||
this.sortKey = event.detail.sortKey;
|
||||
this.sortDesc = event.detail.sortDesc;
|
||||
|
||||
void this.updateComplete.then(() => (this.skipScrollMore = false));
|
||||
}
|
||||
|
||||
// @ts-expect-error [// TODO: Fix this the next time the file is edited.] - TS7006 - Parameter 'event' implicitly has an 'any' type.
|
||||
@@ -1184,9 +1295,7 @@ class Pages extends LitElement {
|
||||
}
|
||||
|
||||
formatResults() {
|
||||
// Default behavior: Count all available pages
|
||||
if (!this.query) {
|
||||
const length = this.filteredPages.length;
|
||||
const length = this.totalPages;
|
||||
if (length === this.sortedPages.length) {
|
||||
return `${length} Page${length !== 1 ? "s" : ""}`;
|
||||
} else {
|
||||
@@ -1194,14 +1303,6 @@ class Pages extends LitElement {
|
||||
}
|
||||
}
|
||||
|
||||
// ... unless they were filtered
|
||||
if (this.sortedPages.length === 1) {
|
||||
return "1 Page";
|
||||
} else {
|
||||
return `${this.sortedPages.length} Pages`;
|
||||
}
|
||||
}
|
||||
|
||||
getNoResultsMessage() {
|
||||
if (!this.collInfo?.pages.length) {
|
||||
return html`<span class="fix-text-wrapping"
|
||||
@@ -1229,14 +1330,23 @@ class Pages extends LitElement {
|
||||
}
|
||||
|
||||
// @ts-expect-error [// TODO: Fix this the next time the file is edited.] - TS7006 - Parameter 'event' implicitly has an 'any' type.
|
||||
onScroll(event) {
|
||||
async onScroll(event) {
|
||||
const element = event.currentTarget;
|
||||
const diff =
|
||||
element.scrollHeight - element.scrollTop - element.clientHeight;
|
||||
if (diff < 40) {
|
||||
if (diff < 40 && !this.skipScrollMore) {
|
||||
this.skipScrollMore = true;
|
||||
if (
|
||||
this.dynamicPagesQuery &&
|
||||
this.filteredPages.length < this.totalPages
|
||||
) {
|
||||
this.dynamicPageCount += 1;
|
||||
await this.addDynamicPages();
|
||||
}
|
||||
|
||||
const sorter = this.renderRoot.querySelector<Sorter>("wr-sorter");
|
||||
if (sorter) {
|
||||
sorter.getMore();
|
||||
sorter.getMore(DYNAMIC_PAGE_SIZE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+9
-1
@@ -69,11 +69,19 @@ function getTS(iso: string) {
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
function getReplayLink(view: string, url: string, ts: string) {
|
||||
function getReplayLink(
|
||||
view: string,
|
||||
url: string,
|
||||
ts: string,
|
||||
waczhash?: string,
|
||||
) {
|
||||
const params = new URLSearchParams();
|
||||
params.set("view", view);
|
||||
params.set("url", url);
|
||||
params.set("ts", ts);
|
||||
if (waczhash) {
|
||||
params.set("waczhash", waczhash);
|
||||
}
|
||||
return "#" + params.toString();
|
||||
}
|
||||
|
||||
|
||||
+6
-1
@@ -20,6 +20,9 @@ class Replay extends LitElement {
|
||||
@property({ type: String })
|
||||
ts = "";
|
||||
|
||||
@property({ type: String })
|
||||
waczhash = "";
|
||||
|
||||
// actual replay url
|
||||
@property({ type: String })
|
||||
replayUrl = "";
|
||||
@@ -97,7 +100,9 @@ class Replay extends LitElement {
|
||||
doSetIframeUrl() {
|
||||
this.iframeUrl =
|
||||
this.url && this.collInfo
|
||||
? `${this.collInfo.replayPrefix}/${this.ts || ""}mp_/${this.url}`
|
||||
? `${this.collInfo.replayPrefix}/${
|
||||
this.waczhash ? `:${this.waczhash}/` : ""
|
||||
}${this.ts || ""}mp_/${this.url}`
|
||||
: "";
|
||||
}
|
||||
|
||||
|
||||
+6
-2
@@ -11,9 +11,12 @@ export type URLResource = {
|
||||
size: number;
|
||||
favIconUrl: string;
|
||||
text?: string;
|
||||
waczhash?: string;
|
||||
};
|
||||
|
||||
export type Page = URLResource;
|
||||
export type Page = URLResource & {
|
||||
isSeed?: boolean;
|
||||
};
|
||||
|
||||
export type ItemType = {
|
||||
filename?: string;
|
||||
@@ -43,12 +46,13 @@ export type ItemType = {
|
||||
software?: string;
|
||||
};
|
||||
onDemand?: boolean;
|
||||
pages: URLResource[];
|
||||
pages: Page[];
|
||||
curatedPages: (URLResource & { list: string })[];
|
||||
lists: URLResource[];
|
||||
ctime?: string;
|
||||
totalSize?: unknown;
|
||||
size?: number | string;
|
||||
canQueryPages?: boolean;
|
||||
};
|
||||
|
||||
export type FavIconEventDetail = {
|
||||
|
||||
@@ -1028,16 +1028,16 @@
|
||||
resolved "https://registry.yarnpkg.com/@webpack-cli/serve/-/serve-2.0.5.tgz#325db42395cd49fe6c14057f9a900e427df8810e"
|
||||
integrity sha512-lqaoKnRYBdo1UgDX8uF24AfGMifWK19TxPmM5FHc2vAGxrJ/qtyUyFBWoY1tISZdelsQ5fBcOusifo5o5wSJxQ==
|
||||
|
||||
"@webrecorder/wabac@^2.20.8":
|
||||
version "2.20.8"
|
||||
resolved "https://registry.yarnpkg.com/@webrecorder/wabac/-/wabac-2.20.8.tgz#37dd14f4cf821f36ecdf5719d5f334bb3445290e"
|
||||
integrity sha512-P6FxC17ZgPfwfp3HbdI9hkzMnqD5I3kBF0cMnue6m8wBVMCzcVGp9x43wT+pcq0zOv6ATGObxcmjJKCsDUus7A==
|
||||
"@webrecorder/wabac@^2.21.0":
|
||||
version "2.21.0"
|
||||
resolved "https://registry.yarnpkg.com/@webrecorder/wabac/-/wabac-2.21.0.tgz#cc3483f15cb3f1502fdef89e1c9a4ee5709bcaec"
|
||||
integrity sha512-28G3ci/m/qJXnLDoUP/Kt09PiTuORBZH0bw/JlKhPCeLL4IdYsODeXaImVwvTT12R9tV7Zp37w26jhkMfcur0w==
|
||||
dependencies:
|
||||
"@peculiar/asn1-ecc" "^2.3.4"
|
||||
"@peculiar/asn1-schema" "^2.3.3"
|
||||
"@peculiar/x509" "^1.9.2"
|
||||
"@types/js-levenshtein" "^1.1.3"
|
||||
"@webrecorder/wombat" "^3.8.7"
|
||||
"@webrecorder/wombat" "^3.8.8"
|
||||
acorn "^8.10.0"
|
||||
auto-js-ipfs "^2.1.1"
|
||||
base64-js "^1.5.1"
|
||||
@@ -1058,10 +1058,10 @@
|
||||
stream-browserify "^3.0.0"
|
||||
warcio "^2.4.3"
|
||||
|
||||
"@webrecorder/wombat@^3.8.7":
|
||||
version "3.8.7"
|
||||
resolved "https://registry.yarnpkg.com/@webrecorder/wombat/-/wombat-3.8.7.tgz#51c7465c589e0020be064121127c7c10a38ec21f"
|
||||
integrity sha512-bW5V7cBweTkTazOIN8oZZGwHLevsGNv1luY3t0RYdEZhs5BDpTmUHN33zEbrXDOiPUlY3N3I8+73VA+PuxihoQ==
|
||||
"@webrecorder/wombat@^3.8.8":
|
||||
version "3.8.8"
|
||||
resolved "https://registry.yarnpkg.com/@webrecorder/wombat/-/wombat-3.8.8.tgz#aab4dd8eea6d6cb17bfefb7ee1802e7b45b11ed7"
|
||||
integrity sha512-XkJOZAyHrdXNkAVoISQEh/NHzaBMekQZfWqes/k2vYkW6v9DmZ0wjP7Kf6MHCuajKX8uSH+caB2tv1kJgvnv3Q==
|
||||
dependencies:
|
||||
warcio "^2.4.0"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user