diff --git a/package.json b/package.json index 81a5d67..28f5f95 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "replaywebpage", "productName": "ReplayWeb.page", - "version": "2.2.5", + "version": "2.3.0", "description": "Serverless Web Archive Replay", "repository": "https://github.com/webrecorder/replayweb.page", "homepage": "https://replayweb.page/", @@ -18,7 +18,7 @@ "dependencies": { "@fortawesome/fontawesome-free": "^5.15.4", "@shoelace-style/shoelace": "~2.15.1", - "@webrecorder/wabac": "^2.20.8", + "@webrecorder/wabac": "^2.21.0", "bulma": "^0.9.3", "electron-log": "^4.4.1", "electron-updater": "^6.3.9", diff --git a/src/embed.ts b/src/embed.ts index 6594aba..0799436 100644 --- a/src/embed.ts +++ b/src/embed.ts @@ -7,9 +7,9 @@ import { } from "lit"; import { ifDefined } from "lit/directives/if-defined.js"; -import { wrapCss, updateFaviconLinks } from "./misc"; +import { wrapCss, updateFaviconLinks, apiPrefix } from "./misc"; import { SWManager } from "./swmanager"; -import { property } from "lit/decorators.js"; +import { property, query } from "lit/decorators.js"; import type { FavIconEventDetail } from "./types"; import type { EmbedReplayData, EmbedReplayEvent } from "./item"; @@ -82,6 +82,9 @@ class Embed extends LitElement { @property({ type: Boolean }) useRuffle = false; + @query("iframe") + private readonly iframe?: HTMLIFrameElement | null; + replayfile = defaultReplayFile; mainElementName = "replay-app-main"; appName = "ReplayWeb.page"; @@ -126,17 +129,28 @@ class Embed extends LitElement { } } - fullReload() { - const iframe = this.renderRoot.querySelector("iframe"); - if (iframe?.contentWindow) { - iframe.contentWindow.postMessage({ type: "fullReload" }); + async fullReload() { + if (!this.iframe?.contentWindow) { + return false; } + + const deleteURL = apiPrefix + "/c/" + this.coll + "?reload=1"; + + const resp = await this.iframe.contentWindow.fetch(deleteURL, { + method: "DELETE", + }); + + if (resp.status !== 200) { + return false; + } + + this.iframe.contentWindow.location.reload(); + + return true; } handleMessage(event: IframeMessage) { - const iframe = this.renderRoot.querySelector("iframe"); - - if (iframe && event.source === iframe.contentWindow) { + if (this.iframe && event.source === this.iframe.contentWindow) { switch (event.data.type) { case "urlchange": if (this.deepLink) { diff --git a/src/item.ts b/src/item.ts index f5c6f14..e50216c 100644 --- a/src/item.ts +++ b/src/item.ts @@ -85,6 +85,7 @@ export type EmbedReplayData = { ts?: string; title?: string; query?: string; + waczhash?: string; }; export type EmbedReplayEvent = EmbedReplayData & { @@ -229,16 +230,6 @@ class Item extends LitElement { this.observer.observe(this); } - - if (this.embed) { - window.addEventListener("message", (event: MessageEvent) => { - if (event.source === window.parent) { - if (event.data.type === "fullReload") { - void this.deleteFully(true); - } - } - }); - } } async runUpdateLoop() { @@ -976,6 +967,7 @@ class Item extends LitElement { sourceUrl="${this.sourceUrl || ""}" url="${this.tabData.url || ""}" ts="${this.tabData.ts || ""}" + waczhash="${this.tabData.waczhash || ""}" @coll-tab-nav="${this.onItemTabNav}" id="replay" @replay-loading="${this.onReplayLoading}" diff --git a/src/pageentry.ts b/src/pageentry.ts index 9df724e..0350474 100644 --- a/src/pageentry.ts +++ b/src/pageentry.ts @@ -5,7 +5,7 @@ import { property, state } from "lit/decorators.js"; import "keyword-mark-element/lib/keyword-mark.js"; -import { getReplayLink } from "./pageutils"; +import { getPageDateTS, getReplayLink } from "./pageutils"; import { wrapCss } from "./misc"; import type { URLResource } from "./types"; @@ -192,7 +192,7 @@ class PageEntry extends LitElement { render() { const p = this.page!; - const date = this.page!.date; + const { date } = getPageDateTS(p); const hasSize = typeof p.size === "number"; @@ -245,6 +245,7 @@ class PageEntry extends LitElement { "pages", this.page!.url, this.page!.timestamp!, + this.page!.waczhash, )}" >

@@ -298,6 +299,12 @@ class PageEntry extends LitElement { `; } + private getReplayPrefix() { + const waczhash = this.page!.waczhash ? `:${this.page!.waczhash}/` : ""; + const ts = this.page!.timestamp || ""; + return this.replayPrefix + "/" + waczhash + ts + "id_"; + } + private renderPageIcon() { if (!this.thumbnailValid) { return this.renderFavicon(); @@ -305,9 +312,7 @@ class PageEntry extends LitElement { return html` (this.thumbnailValid = false)} - src=${`${this.replayPrefix}/${this.page!.timestamp}id_/urn:thumbnail:${ - this.page!.url - }`} + src=${`${this.getReplayPrefix()}/urn:thumbnail:${this.page!.url}`} loading="lazy" />`; } @@ -319,9 +324,7 @@ class PageEntry extends LitElement { return html` (this.iconValid = false)} - src=${`${this.replayPrefix}/${this.page!.timestamp}id_/${ - this.page!.favIconUrl - }`} + src=${`${this.getReplayPrefix()}/${this.page!.favIconUrl}`} loading="lazy" />`; } @@ -377,6 +380,7 @@ class PageEntry extends LitElement { const data = { url: this.page!.url, ts: this.page!.timestamp, + waczhash: this.page!.waczhash, }; this.sendChangeEvent(data, reload); return false; diff --git a/src/pages.ts b/src/pages.ts index cb0cec4..6c7e3cd 100644 --- a/src/pages.ts +++ b/src/pages.ts @@ -20,6 +20,8 @@ import type { PageEntry } from "./pageentry"; import type { Id, Index } from "flexsearch"; import type { ItemType, URLResource } from "./types"; +const DYNAMIC_PAGE_SIZE = 25; + // =========================================================================== class Pages extends LitElement { @property({ type: Array }) @@ -103,6 +105,18 @@ class Pages extends LitElement { @property({ type: String }) defaultKey = ""; + @property({ type: Boolean }) + dynamicPagesQuery = false; + + @property({ type: Number }) + totalPages = 0; + + @property({ type: Number }) + dynamicPageCount = 1; + + @property({ type: Boolean }) + skipScrollMore = false; + private _ival: number | undefined; static get sortKeys() { @@ -214,10 +228,25 @@ class Pages extends LitElement { ); } else if (this.showAllPages && this.hasExtraPages) { this.filteredPages = [...this.textPages!]; - } else { + } else if (!this.dynamicPagesQuery) { this.filteredPages = [...this.collInfo!.pages]; } + this.totalPages = this.filteredPages.length; + + if (this.dynamicPagesQuery) { + if (!this.query) { + const seedPages = this.collInfo!.pages.filter((x) => x.isSeed); + this.filteredPages = + this.showAllPages || !seedPages.length + ? [...this.collInfo!.pages] + : seedPages; + this.hasExtraPages = seedPages.length !== this.collInfo!.pages.length; + } + this.dynamicPageCount = 1; + await this.addDynamicPages(); + } + if (this.currList !== 0) { await this.filterCurated(); } @@ -238,6 +267,80 @@ class Pages extends LitElement { this.sendChangeEvent(data); } + async addDynamicPages() { + const search = new URLSearchParams(); + if (this.query) { + search.set("search", this.query); + } + search.set("page", this.dynamicPageCount + ""); + search.set("pageSize", DYNAMIC_PAGE_SIZE + ""); + + const resp = await fetch( + `${this.collInfo!.apiPrefix}/pages?${search.toString()}`, + ); + const json = await resp.json(); + if (!json.pages) { + return; + } + + const knownPages = new Set(); + this.filteredPages.forEach((x) => knownPages.add(x.id)); + + const newPages = []; + + for (const { + id, + url, + title, + mime, + status, + ts, + favIconUrl, + waczhash, + isSeed, + } of json.pages) { + if (knownPages.has(id)) { + continue; + } + + if (!this.showAllPages && this.hasExtraPages && !this.query && !isSeed) { + continue; + } + + let tsActual; + + if (typeof ts === "string") { + tsActual = new Date(ts).getTime(); + } else { + tsActual = ts; + } + + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const newPage: any = { + id, + url, + title, + mime, + status, + ts: tsActual, + favIconUrl, + waczhash, + }; + + newPages.push(newPage); + } + + if (newPages.length) { + this.filteredPages = [...this.filteredPages, ...newPages]; + } + + if (json.total) { + this.totalPages = json.total; + } else { + this.totalPages = this.filteredPages.length; + } + } + async filterCurated() { const resp = await fetch( `${this.collInfo!.apiPrefix}/curated/${this.currList}`, @@ -263,11 +366,17 @@ class Pages extends LitElement { this.flex = flex; this.textPages = pages; - this.hasExtraPages = Boolean( - this.textPages && - this.collInfo?.pages && - this.textPages.length > this.collInfo.pages.length, - ); + if (!this.dynamicPagesQuery) { + this.hasExtraPages = Boolean( + this.textPages && + this.collInfo?.pages && + this.textPages.length > this.collInfo.pages.length, + ); + } + + if (this.collInfo) { + this.dynamicPagesQuery = this.collInfo.canQueryPages || false; + } return Promise.all( // @ts-expect-error [// TODO: Fix this the next time the file is edited.] - TS7006 - Parameter 'page' implicitly has an 'any' type. | TS7006 - Parameter 'index' implicitly has an 'any' type. @@ -893,7 +1002,7 @@ class Pages extends LitElement { .sortDesc="${this.sortDesc}" .sortKeys="${Pages.sortKeys}" .data="${this.filteredPages}" - pageResults="100" + .pageResults="${this.dynamicPagesQuery ? this.totalPages : 100}" @sort-changed="${this.onSortChanged}" class="${this.filteredPages.length ? "" : "is-hidden"}" > @@ -1064,6 +1173,8 @@ class Pages extends LitElement { this.sortedPages = event.detail.sortedData; this.sortKey = event.detail.sortKey; this.sortDesc = event.detail.sortDesc; + + void this.updateComplete.then(() => (this.skipScrollMore = false)); } // @ts-expect-error [// TODO: Fix this the next time the file is edited.] - TS7006 - Parameter 'event' implicitly has an 'any' type. @@ -1184,21 +1295,11 @@ class Pages extends LitElement { } formatResults() { - // Default behavior: Count all available pages - if (!this.query) { - const length = this.filteredPages.length; - if (length === this.sortedPages.length) { - return `${length} Page${length !== 1 ? "s" : ""}`; - } else { - return `${this.sortedPages.length} of ${length} Pages Shown`; - } - } - - // ... unless they were filtered - if (this.sortedPages.length === 1) { - return "1 Page"; + const length = this.totalPages; + if (length === this.sortedPages.length) { + return `${length} Page${length !== 1 ? "s" : ""}`; } else { - return `${this.sortedPages.length} Pages`; + return `${this.sortedPages.length} of ${length} Pages Shown`; } } @@ -1229,14 +1330,23 @@ class Pages extends LitElement { } // @ts-expect-error [// TODO: Fix this the next time the file is edited.] - TS7006 - Parameter 'event' implicitly has an 'any' type. - onScroll(event) { + async onScroll(event) { const element = event.currentTarget; const diff = element.scrollHeight - element.scrollTop - element.clientHeight; - if (diff < 40) { + if (diff < 40 && !this.skipScrollMore) { + this.skipScrollMore = true; + if ( + this.dynamicPagesQuery && + this.filteredPages.length < this.totalPages + ) { + this.dynamicPageCount += 1; + await this.addDynamicPages(); + } + const sorter = this.renderRoot.querySelector("wr-sorter"); if (sorter) { - sorter.getMore(); + sorter.getMore(DYNAMIC_PAGE_SIZE); } } } diff --git a/src/pageutils.ts b/src/pageutils.ts index cf28403..795ea64 100644 --- a/src/pageutils.ts +++ b/src/pageutils.ts @@ -69,11 +69,19 @@ function getTS(iso: string) { } // =========================================================================== -function getReplayLink(view: string, url: string, ts: string) { +function getReplayLink( + view: string, + url: string, + ts: string, + waczhash?: string, +) { const params = new URLSearchParams(); params.set("view", view); params.set("url", url); params.set("ts", ts); + if (waczhash) { + params.set("waczhash", waczhash); + } return "#" + params.toString(); } diff --git a/src/replay.ts b/src/replay.ts index ab905a2..c1edeb2 100644 --- a/src/replay.ts +++ b/src/replay.ts @@ -20,6 +20,9 @@ class Replay extends LitElement { @property({ type: String }) ts = ""; + @property({ type: String }) + waczhash = ""; + // actual replay url @property({ type: String }) replayUrl = ""; @@ -97,7 +100,9 @@ class Replay extends LitElement { doSetIframeUrl() { this.iframeUrl = this.url && this.collInfo - ? `${this.collInfo.replayPrefix}/${this.ts || ""}mp_/${this.url}` + ? `${this.collInfo.replayPrefix}/${ + this.waczhash ? `:${this.waczhash}/` : "" + }${this.ts || ""}mp_/${this.url}` : ""; } diff --git a/src/types.ts b/src/types.ts index 91d0770..9578e03 100644 --- a/src/types.ts +++ b/src/types.ts @@ -11,9 +11,12 @@ export type URLResource = { size: number; favIconUrl: string; text?: string; + waczhash?: string; }; -export type Page = URLResource; +export type Page = URLResource & { + isSeed?: boolean; +}; export type ItemType = { filename?: string; @@ -43,12 +46,13 @@ export type ItemType = { software?: string; }; onDemand?: boolean; - pages: URLResource[]; + pages: Page[]; curatedPages: (URLResource & { list: string })[]; lists: URLResource[]; ctime?: string; totalSize?: unknown; size?: number | string; + canQueryPages?: boolean; }; export type FavIconEventDetail = { diff --git a/yarn.lock b/yarn.lock index 9fe5632..1ebdd23 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1028,16 +1028,16 @@ resolved "https://registry.yarnpkg.com/@webpack-cli/serve/-/serve-2.0.5.tgz#325db42395cd49fe6c14057f9a900e427df8810e" integrity sha512-lqaoKnRYBdo1UgDX8uF24AfGMifWK19TxPmM5FHc2vAGxrJ/qtyUyFBWoY1tISZdelsQ5fBcOusifo5o5wSJxQ== -"@webrecorder/wabac@^2.20.8": - version "2.20.8" - resolved "https://registry.yarnpkg.com/@webrecorder/wabac/-/wabac-2.20.8.tgz#37dd14f4cf821f36ecdf5719d5f334bb3445290e" - integrity sha512-P6FxC17ZgPfwfp3HbdI9hkzMnqD5I3kBF0cMnue6m8wBVMCzcVGp9x43wT+pcq0zOv6ATGObxcmjJKCsDUus7A== +"@webrecorder/wabac@^2.21.0": + version "2.21.0" + resolved "https://registry.yarnpkg.com/@webrecorder/wabac/-/wabac-2.21.0.tgz#cc3483f15cb3f1502fdef89e1c9a4ee5709bcaec" + integrity sha512-28G3ci/m/qJXnLDoUP/Kt09PiTuORBZH0bw/JlKhPCeLL4IdYsODeXaImVwvTT12R9tV7Zp37w26jhkMfcur0w== dependencies: "@peculiar/asn1-ecc" "^2.3.4" "@peculiar/asn1-schema" "^2.3.3" "@peculiar/x509" "^1.9.2" "@types/js-levenshtein" "^1.1.3" - "@webrecorder/wombat" "^3.8.7" + "@webrecorder/wombat" "^3.8.8" acorn "^8.10.0" auto-js-ipfs "^2.1.1" base64-js "^1.5.1" @@ -1058,10 +1058,10 @@ stream-browserify "^3.0.0" warcio "^2.4.3" -"@webrecorder/wombat@^3.8.7": - version "3.8.7" - resolved "https://registry.yarnpkg.com/@webrecorder/wombat/-/wombat-3.8.7.tgz#51c7465c589e0020be064121127c7c10a38ec21f" - integrity sha512-bW5V7cBweTkTazOIN8oZZGwHLevsGNv1luY3t0RYdEZhs5BDpTmUHN33zEbrXDOiPUlY3N3I8+73VA+PuxihoQ== +"@webrecorder/wombat@^3.8.8": + version "3.8.8" + resolved "https://registry.yarnpkg.com/@webrecorder/wombat/-/wombat-3.8.8.tgz#aab4dd8eea6d6cb17bfefb7ee1802e7b45b11ed7" + integrity sha512-XkJOZAyHrdXNkAVoISQEh/NHzaBMekQZfWqes/k2vYkW6v9DmZ0wjP7Kf6MHCuajKX8uSH+caB2tv1kJgvnv3Q== dependencies: warcio "^2.4.0"