text search and ui fixes:

- text support via <coll>/textIndex api, fallback to existing pageList text data
- page sorting: add score column for page search results with query
- safari: add min-height: fit-content for safari
- refresh: ensure replay refresh triggers refresh icon
- embeds: ensure deep linking working for search view pages (map view and query params)

package.json

@@ -11,6 +11,7 @@
     "@fortawesome/fontawesome-free": "^5.13.0",
     "@webrecorder/wabac": "github:webrecorder/wabac.js",
     "bulma": "^0.8.2",
+    "fetch-ndjson": "^1.1.0",
     "flexsearch": "^0.6.32",
     "keyword-mark-element": "^0.1.2",
     "lit-element": "^2.3.1",

src/coll.js

@@ -31,6 +31,7 @@ class Coll extends LitElement
     super();
     this.sourceUrl = null;
     this.inited = false;
+    this.isLoading = false;
 
     this.baseApiPrefix = "./wabac/api";
     this.baseReplayPrefix = "./wabac";
@@ -68,7 +69,9 @@ class Coll extends LitElement
 
       collInfo: { type: Object, attribute: false },
       coll: { type: String },
+
       hasStory: { type: Boolean },
+      isLoading: { type: Boolean },
 
       tabData: { type: Object, attribute: false },
 
@@ -129,6 +132,9 @@ class Coll extends LitElement
         } else {
           window.location.hash = this._locationHash;
         }
+        if (this.embed && window.parent !== window) {
+          window.parent.postMessage(this.tabData, '*');
+        }
       }
       this._locUpdateNeeded = false;
     }
@@ -372,7 +378,7 @@ class Coll extends LitElement
   }
 
   renderTabHeader() {
-    if (this.tabData.view === "replay" || (this.embed && this.embed !== "full")) {
+    if (this.tabData.view === "replay") {
       return "";
     }
 
@@ -548,10 +554,10 @@ class Coll extends LitElement
     ${isReplay ? html`
     <wr-coll-replay .collInfo="${this.collInfo}"
     sourceUrl="${this.sourceUrl}"
-    embed="${this.embed}"
     url="${this.tabData.url || ""}"
     ts="${this.tabData.ts || ""}"
     @coll-tab-nav="${this.onCollTabNav}" id="replay"
+    @replay-loading="${(e) => this.isLoading = e.detail.loading}"
     class="${isReplay ? '' : 'is-hidden'}">
     </wr-coll-replay>
     ` : ``}
@@ -560,7 +566,7 @@ class Coll extends LitElement
 
   onKeyDown(event) {
     if (event.key === "Esc" || event.key === "Escape") {
-      event.target.value = this.replayUrl;
+      event.target.value = this.url;
     }
   }
 

src/embed.js

@@ -18,6 +18,7 @@ class Embed extends LitElement
     this.view = "replay";
     this.ts = "";
     this.url = "";
+    this.query = "";
     this.config = "";
     this.coll = "";
     this.paramString = null;
@@ -30,6 +31,8 @@ class Embed extends LitElement
     return {
       url: { type: String },
       ts: { type: String },
+      query: { type: String },
+
       source: { type: String },
       view: { type: String },
       embed: { type: String },
@@ -74,16 +77,11 @@ class Embed extends LitElement
           this.title = event.data.title;
         }
 
-        console.log(event.data);
-
         if (!this.deepLink) {
           return;
         }
 
-        const currHash = new URLSearchParams({
+        const currHash = new URLSearchParams(event.data);
-          url: event.data.url,
-          ts: event.data.ts,
-        });
         const url = new URL(window.location.href);
         url.hash = "#" + currHash.toString();
         window.history.replaceState({}, "", url);
@@ -105,11 +103,19 @@ class Embed extends LitElement
     if (qs.has("ts")) {
       this.ts = qs.get("ts");
     }
+    if (qs.has("query")) {
+      this.query = qs.get("query");
+    }
+    if (qs.has("view")) {
+      this.view = qs.get("view");
+    }
   }
 
   updated(changedProperties) {
     if (changedProperties.has("url") || 
         changedProperties.has("ts") ||
+        changedProperties.has("query") ||
+        changedProperties.has("view") || 
         changedProperties.has("source")) {
 
       if (this.embed === null) {
@@ -127,6 +133,7 @@ class Embed extends LitElement
       this.hashString = new URLSearchParams({
         url: this.url,
         ts: this.ts,
+        query: this.query,
         view: this.view
       }).toString();
     }
@@ -180,9 +187,6 @@ Please try a different browser.\n
   }
 
   onLoad(event) {
-    if (this.injected) {
-      return;
-    }
     const win = event.target.contentWindow;
     const doc = event.target.contentDocument;
 
@@ -199,7 +203,6 @@ Please try a different browser.\n
     //const script = event.target.contentDocument.querySelector("script");
     script.src = scriptSrc;
     doc.head.appendChild(script);
-    this.injected = true;
   }
 }
 

src/pages.js

@@ -2,6 +2,7 @@
 
 import { LitElement, html, css } from 'lit-element';
 import { wrapCss } from './misc';
+import ndjson from 'fetch-ndjson';
 
 import prettyBytes from 'pretty-bytes';
 
@@ -25,6 +26,7 @@ class Pages extends LitElement
     this.flex = null;
     this.newQuery = null;
     this.loading = false;
+    this.updatingSearch = false;
 
     this.currList = 0;
 
@@ -42,6 +44,10 @@ class Pages extends LitElement
 
   static get sortKeys() {
     return [
+      {
+        "key": "",
+        "name": "Score",
+      },
       {
         "key": "title",
         "name": "Title"
@@ -64,6 +70,7 @@ class Pages extends LitElement
       query: { type: String },
 
       loading: { type: Boolean },
+      updatingSearch: { type: Boolean },
       editable: { type: Boolean },
 
       selectedPages: { type: Set },
@@ -87,8 +94,14 @@ class Pages extends LitElement
   updated(changedProperties) {
     if (changedProperties.has("collInfo")) {
       this.updateTextSearch();
+    } else if (changedProperties.has("query")) {
       this.filter();
-    } else if (changedProperties.has("query") || changedProperties.has("currList")) {
+
+      // default sort to score if has query, otherwise timestamp
+      this.sortKey = this.query ? "" : "ts";
+      this.sortDesc = this.query ? false : true;
+
+    } else if (changedProperties.has("currList")) {
       this.filter();
     }
     if (changedProperties.has("active") && this.active) {
@@ -116,7 +129,8 @@ class Pages extends LitElement
     }
     this.loading = true;
     if (this.flex && this.query) {
-      const result = await this.flex.search(this.query);
+      const result = await this.flex.search(this.query, {limit: 25});
+
       this.filteredPages = result;
     } else {
       this.filteredPages = [...this.collInfo.pages];
@@ -152,8 +166,16 @@ class Pages extends LitElement
     this.dispatchEvent(new CustomEvent("coll-tab-nav", {detail: {data}}));
   }
 
-  updateTextSearch() {
+  async updateTextSearch() {
-    this.flex = new FlexSearch({
+    if (this.updatingSearch) {
+      return;
+    }
+
+    this.updatingSearch = true;
+    let count = 0;
+
+    try {
+      const flex = new FlexSearch({
         doc: {
           id: "id",
           field: ["url", "title", "text"],
@@ -161,9 +183,56 @@ class Pages extends LitElement
         async: true
       });
 
+      this.flex = flex;
+
+      const cache = await caches.open("cache:" + this.collInfo.coll);
+
+      const indexUrl = `${this.collInfo.apiPrefix}/textIndex`;
+
+      let resp = await cache.match(indexUrl);
+
+      if (!resp) {
+        resp = await fetch(`${this.collInfo.apiPrefix}/textIndex`);
+        if (resp.status === 200) {
+          cache.put(indexUrl, resp.clone());
+        }
+      }
+
+      let lines = [];
+
+      async function flush() {
+        let curr = lines;
+        lines = [];
+        await flex.add(curr);
+        console.log("added " + count + " " + curr.length);
+        console.log(flex.info());
+      }
+
+      for await (const line of ndjson(resp.body.getReader())) {
+        if (!line.text) {
+          continue;
+        }
+
+        line.id = ++count;
+        lines.push(line);
+        if ((count % 100) === 0) {
+          await flush();
+        }
+      }
+
+      await flush();
+
+    } finally {
+      if (count === 0) {
         this.flex.add(this.collInfo.pages);
       }
 
+      this.updatingSearch = false;
+    }
+
+    await this.filter();
+  }
+
   static get styles() {
     return wrapCss(css`
       :host {
@@ -313,6 +382,7 @@ class Pages extends LitElement
         display: flex;
         flex-direction: row;
         width: 100%;
+        min-height: fit-content;
 
         margin-bottom: 1.0em;
         border-bottom: 3px solid rgb(237, 237, 237);
@@ -428,8 +498,11 @@ class Pages extends LitElement
     </div>` : html``}
 
     <div class="header columns is-hidden-mobile">
-      <a @click="${this.onSort}" data-key="title" class="column is-2 ${this.sortKey === "title" ? (this.sortDesc ? "desc" : "asc") : ''}">Date</a>
+      ${this.query ? html`
-      <a @click="${this.onSort}" data-key="ts" class="column is-8 pagetitle ${this.sortKey === "ts" ? (this.sortDesc ? "desc" : "asc") : ''}">Page Title</a>
+      <a @click="${this.onSort}" data-key="" class="column is-1 ${this.sortKey === "" ? (this.sortDesc ? "desc" : "asc") : ''}">Score</a>` : ``}
+
+      <a @click="${this.onSort}" data-key="ts" class="column is-2 ${this.sortKey === "ts" ? (this.sortDesc ? "desc" : "asc") : ''}">Date</a>
+      <a @click="${this.onSort}" data-key="title" class="column is-6 pagetitle ${this.sortKey === "title" ? (this.sortDesc ? "desc" : "asc") : ''}">Page Title</a>
     </div>
 
     <div class="is-hidden-tablet mobile-header">
@@ -453,9 +526,9 @@ class Pages extends LitElement
       ${this.renderPageHeader()}
       </div>
       <div class="scroller" @scroll="${this.onScroll}">
-        ${this.sortedPages.map((p) => html`
+        ${this.sortedPages.map((p, i) => html`
         <div class="content ${this.selectedPages.has(p.id) ? 'selected' : ''}">
-          <wr-page-entry .editable="${this.editable}" .selected="${this.selectedPages.has(p.id)}" @sel-page="${this.onSelectToggle}" @delete-page="${this.onDeletePage}" replayPrefix="${this.collInfo.replayPrefix}" query="${this.query}" .page="${p}">
+          <wr-page-entry .index="${this.query ? (this.sortDesc ? this.sortedPages.length - i : i + 1) : 0}" .editable="${this.editable}" .selected="${this.selectedPages.has(p.id)}" @sel-page="${this.onSelectToggle}" @delete-page="${this.onDeletePage}" replayPrefix="${this.collInfo.replayPrefix}" query="${this.query}" .page="${p}">
           </wr-page-entry>
         </div>`)}
       </div>
@@ -476,7 +549,7 @@ class Pages extends LitElement
   onSort(event) {
     event.preventDefault();
 
-    const key = event.currentTarget.getAttribute("data-key");
+    const key = event.currentTarget.getAttribute("data-key") || "";
     if (key === this.sortKey) {
       this.sortDesc = !this.sortDesc;
     } else {
@@ -546,7 +619,6 @@ class Pages extends LitElement
     //}
 
     this.updateTextSearch();
-    this.filter();
     this.requestUpdate();
   }
 
@@ -555,6 +627,10 @@ class Pages extends LitElement
       return html`No Pages defined this archive. Check out&nbsp;<a href="#view=resources">Page Resources</a>&nbsp;to search by URL.`;
     }
 
+    if (this.updatingSearch) {
+      return "Initializing Search..."
+    }
+
     if (this.loading) {
       return "Searching...";
     }
@@ -592,6 +668,7 @@ class PageEntry extends LitElement
     this.deleting = false;
     this.editable = false;
     this.iconValid = false;
+    this.index = 0;
   }
 
   static get properties() {
@@ -603,7 +680,8 @@ class PageEntry extends LitElement
       deleting: { type: Boolean },
       selected: { type: Boolean },
       editable: { type: Boolean },
-      iconValid: { type: Boolean }
+      iconValid: { type: Boolean },
+      index: { type: Number },
     }
   }
 
@@ -705,11 +783,14 @@ class PageEntry extends LitElement
     </div>` : ``}
 
     <div class="columns">
+      ${this.index ? html`
+      <div class="column col-index is-1">${this.index}.</div>
+      ` : ``}
       <div class="column col-date is-2">
         <div>${date ? date.toLocaleDateString() : ""}</div>
         <div>${date ? date.toLocaleTimeString() : ""}</div>
       </div>
-      <div class="column is-9">
+      <div class="column">
         <article class="media">
           <figure class="media-left">
             <p class="">

src/replay.js

@@ -8,7 +8,6 @@ class Replay extends LitElement
 {
   constructor() {
     super();
-    this.isLoading = false;
     this.replayUrl = "";
     this.replayTS = "";
     this.url = "";
@@ -34,7 +33,6 @@ class Replay extends LitElement
       title: { type: String },
 
       iframeUrl: { type: String },
-      isLoading: { type: Boolean },
 
       showAuth: { type: Boolean },
     }
@@ -86,14 +84,6 @@ class Replay extends LitElement
   
       this.dispatchEvent(new CustomEvent("coll-tab-nav", {detail: {replaceLoc: true, data}}));
     }
-
-    if (this.embed && window.parent !== window && changedProperties.has("title")) {
-      window.parent.postMessage({
-        title: this.title,
-        url: this.replayUrl,
-        ts: this.replayTS
-      }, '*');
-    }
   }
 
   onReplayMessage(event) {
@@ -117,7 +107,7 @@ class Replay extends LitElement
   }
 
   waitForLoad() {
-    this.isLoading = true;
+    this.setLoading();
     this._loadPoll = window.setInterval(() => {
       const iframe = this.renderRoot.querySelector("iframe");
       if (!iframe || !iframe.contentDocument || !iframe.contentWindow || 
@@ -128,7 +118,8 @@ class Replay extends LitElement
   }
 
   clearLoading(iframeWin) {
-    this.isLoading = false;
+    this.dispatchEvent(new CustomEvent("replay-loading", {detail: {loading: false}}));
+
     if (this._loadPoll) {
       window.clearInterval(this._loadPoll);
       this._loadPoll = null;
@@ -136,16 +127,16 @@ class Replay extends LitElement
 
     if (iframeWin) {
       iframeWin.addEventListener("beforeunload", () => {
-        this.isLoading = true;
+        this.setLoading();
       });
     }
   }
 
-  refresh() {
+  setLoading() {
-    if (this.isLoading && !forceReload) {
+    this.dispatchEvent(new CustomEvent("replay-loading", {detail: {loading: true}}));
-      return;
   }
 
+  refresh() {
     const iframe = this.renderRoot.querySelector("iframe");
 
     if (iframe) {

src/sorter.js

@@ -16,6 +16,12 @@ class Sorter extends LitElement
 
     this.numResults = DEFAULT_RESULTS;
     this.defaultDesc = false;
+
+    this.sortKey = "url";
+    this.sortDesc = false;
+
+    this.defaultKey = "";
+    this.defaultDesc = false;
   }
 
   static get properties() {
@@ -68,6 +74,11 @@ class Sorter extends LitElement
     this.sortedData = [...this.data];
     this.numResults = DEFAULT_RESULTS;
 
+    if (this.sortKey === "") {
+      if (this.sortDesc) {
+        this.sortedData.reverse();
+      }
+    } else {
       this.sortedData.sort((first, second) => {
         if (first[this.sortKey] === second[this.sortKey]) {
           return 0;
@@ -75,6 +86,7 @@ class Sorter extends LitElement
 
         return (this.sortDesc == (first[this.sortKey] < second[this.sortKey])) ? 1 : -1;
       });
+    }
 
     this.sendSortChanged();
   }

src/url-resources.js

@@ -235,10 +235,12 @@ class URLResources extends LitElement
     }   
     .result {
       border-bottom: 1px #dbdbdb solid;
+      min-height: fit-content;
     }
     .results-head {
       border-bottom: 2px #dbdbdb solid;
       margin-right: 16px;
+      min-height: fit-content;
     }
     .results-head a {
       color: black;

yarn.lock

@@ -292,8 +292,8 @@
     "@xtuc/long" "4.2.2"
 
 "@webrecorder/wabac@github:webrecorder/wabac.js":
-  version "2.0.3"
+  version "2.0.4"
-  resolved "https://codeload.github.com/webrecorder/wabac.js/tar.gz/39c73bd44a0d0659d0438cb10a29f46c2d58a671"
+  resolved "https://codeload.github.com/webrecorder/wabac.js/tar.gz/a286e76e705a8686c124e755cb22fad9618f4f23"
   dependencies:
     brotli "github:foliojs/brotli.js"
     fast-xml-parser "^3.15.1"
@@ -2152,6 +2152,11 @@ fd-slicer@~1.1.0:
   dependencies:
     pend "~1.2.0"
 
+fetch-ndjson@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/fetch-ndjson/-/fetch-ndjson-1.1.0.tgz#dff9f70493c2a8a61b46d4c0a6d2c04255c01618"
+  integrity sha512-/2xKQdp7MMPMReDXvWiU5BHsvcBjrGGO8s3jJBfEWL/dJXG3/u++BG1WNBM/AGFNvK/1HdrM6H8jFWFCBa9Mdw==
+
 figgy-pudding@^3.5.1:
   version "3.5.2"
   resolved "https://registry.yarnpkg.com/figgy-pudding/-/figgy-pudding-3.5.2.tgz#b4eee8148abb01dcf1d1ac34367d59e12fa61d6e"