text search and ui fixes:

- text support via <coll>/textIndex api, fallback to existing pageList text data - page sorting: add score column for page search results with query - safari: add min-height: fit-content for safari - refresh: ensure replay refresh triggers refresh icon - embeds: ensure deep linking working for search view pages (map view and query params)
2020-08-07 16:48:02 -07:00
parent 69e7982d31
commit 4efce70a39
10 changed files with 303 additions and 193 deletions
@@ -31,6 +31,7 @@ class Coll extends LitElement
    super();
    this.sourceUrl = null;
    this.inited = false;
+    this.isLoading = false;

    this.baseApiPrefix = "./wabac/api";
    this.baseReplayPrefix = "./wabac";
@@ -68,7 +69,9 @@ class Coll extends LitElement

      collInfo: { type: Object, attribute: false },
      coll: { type: String },
+
      hasStory: { type: Boolean },
+      isLoading: { type: Boolean },

      tabData: { type: Object, attribute: false },

@@ -129,6 +132,9 @@ class Coll extends LitElement
        } else {
          window.location.hash = this._locationHash;
        }
+        if (this.embed && window.parent !== window) {
+          window.parent.postMessage(this.tabData, '*');
+        }
      }
      this._locUpdateNeeded = false;
    }
@@ -372,7 +378,7 @@ class Coll extends LitElement
  }

  renderTabHeader() {
-    if (this.tabData.view === "replay" || (this.embed && this.embed !== "full")) {
+    if (this.tabData.view === "replay") {
      return "";
    }

@@ -548,10 +554,10 @@ class Coll extends LitElement
    ${isReplay ? html`
    <wr-coll-replay .collInfo="${this.collInfo}"
    sourceUrl="${this.sourceUrl}"
-    embed="${this.embed}"
    url="${this.tabData.url || ""}"
    ts="${this.tabData.ts || ""}"
    @coll-tab-nav="${this.onCollTabNav}" id="replay"
+    @replay-loading="${(e) => this.isLoading = e.detail.loading}"
    class="${isReplay ? '' : 'is-hidden'}">
    </wr-coll-replay>
    ` : ``}
@@ -560,7 +566,7 @@ class Coll extends LitElement

  onKeyDown(event) {
    if (event.key === "Esc" || event.key === "Escape") {
-      event.target.value = this.replayUrl;
+      event.target.value = this.url;
    }
  }

@@ -18,6 +18,7 @@ class Embed extends LitElement
    this.view = "replay";
    this.ts = "";
    this.url = "";
+    this.query = "";
    this.config = "";
    this.coll = "";
    this.paramString = null;
@@ -30,6 +31,8 @@ class Embed extends LitElement
    return {
      url: { type: String },
      ts: { type: String },
+      query: { type: String },
+
      source: { type: String },
      view: { type: String },
      embed: { type: String },
@@ -74,16 +77,11 @@ class Embed extends LitElement
          this.title = event.data.title;
        }

-        console.log(event.data);
-
        if (!this.deepLink) {
          return;
        }

-        const currHash = new URLSearchParams({
-          url: event.data.url,
-          ts: event.data.ts,
-        });
+        const currHash = new URLSearchParams(event.data);
        const url = new URL(window.location.href);
        url.hash = "#" + currHash.toString();
        window.history.replaceState({}, "", url);
@@ -105,11 +103,19 @@ class Embed extends LitElement
    if (qs.has("ts")) {
      this.ts = qs.get("ts");
    }
+    if (qs.has("query")) {
+      this.query = qs.get("query");
+    }
+    if (qs.has("view")) {
+      this.view = qs.get("view");
+    }
  }

  updated(changedProperties) {
    if (changedProperties.has("url") || 
        changedProperties.has("ts") ||
+        changedProperties.has("query") ||
+        changedProperties.has("view") || 
        changedProperties.has("source")) {

      if (this.embed === null) {
@@ -127,6 +133,7 @@ class Embed extends LitElement
      this.hashString = new URLSearchParams({
        url: this.url,
        ts: this.ts,
+        query: this.query,
        view: this.view
      }).toString();
    }
@@ -180,9 +187,6 @@ Please try a different browser.\n
  }

  onLoad(event) {
-    if (this.injected) {
-      return;
-    }
    const win = event.target.contentWindow;
    const doc = event.target.contentDocument;

@@ -199,7 +203,6 @@ Please try a different browser.\n
    //const script = event.target.contentDocument.querySelector("script");
    script.src = scriptSrc;
    doc.head.appendChild(script);
-    this.injected = true;
  }
 }

@@ -2,6 +2,7 @@

 import { LitElement, html, css } from 'lit-element';
 import { wrapCss } from './misc';
+import ndjson from 'fetch-ndjson';

 import prettyBytes from 'pretty-bytes';

@@ -25,6 +26,7 @@ class Pages extends LitElement
    this.flex = null;
    this.newQuery = null;
    this.loading = false;
+    this.updatingSearch = false;

    this.currList = 0;

@@ -42,6 +44,10 @@ class Pages extends LitElement

  static get sortKeys() {
    return [
+      {
+        "key": "",
+        "name": "Score",
+      },
      {
        "key": "title",
        "name": "Title"
@@ -64,6 +70,7 @@ class Pages extends LitElement
      query: { type: String },

      loading: { type: Boolean },
+      updatingSearch: { type: Boolean },
      editable: { type: Boolean },

      selectedPages: { type: Set },
@@ -87,8 +94,14 @@ class Pages extends LitElement
  updated(changedProperties) {
    if (changedProperties.has("collInfo")) {
      this.updateTextSearch();
+    } else if (changedProperties.has("query")) {
      this.filter();
-    } else if (changedProperties.has("query") || changedProperties.has("currList")) {
+
+      // default sort to score if has query, otherwise timestamp
+      this.sortKey = this.query ? "" : "ts";
+      this.sortDesc = this.query ? false : true;
+
+    } else if (changedProperties.has("currList")) {
      this.filter();
    }
    if (changedProperties.has("active") && this.active) {
@@ -116,7 +129,8 @@ class Pages extends LitElement
    }
    this.loading = true;
    if (this.flex && this.query) {
-      const result = await this.flex.search(this.query);
+      const result = await this.flex.search(this.query, {limit: 25});
+
      this.filteredPages = result;
    } else {
      this.filteredPages = [...this.collInfo.pages];
@@ -152,16 +166,71 @@ class Pages extends LitElement
    this.dispatchEvent(new CustomEvent("coll-tab-nav", {detail: {data}}));
  }

-  updateTextSearch() {
-    this.flex = new FlexSearch({
-      doc: {
-        id: "id",
-        field: ["url", "title", "text"],
-      },
-      async: true
-    });
+  async updateTextSearch() {
+    if (this.updatingSearch) {
+      return;
+    }

-    this.flex.add(this.collInfo.pages);
+    this.updatingSearch = true;
+    let count = 0;
+
+    try {
+      const flex = new FlexSearch({
+        doc: {
+          id: "id",
+          field: ["url", "title", "text"],
+        },
+        async: true
+      });
+
+      this.flex = flex;
+
+      const cache = await caches.open("cache:" + this.collInfo.coll);
+
+      const indexUrl = `${this.collInfo.apiPrefix}/textIndex`;
+
+      let resp = await cache.match(indexUrl);
+
+      if (!resp) {
+        resp = await fetch(`${this.collInfo.apiPrefix}/textIndex`);
+        if (resp.status === 200) {
+          cache.put(indexUrl, resp.clone());
+        }
+      }
+
+      let lines = [];
+
+      async function flush() {
+        let curr = lines;
+        lines = [];
+        await flex.add(curr);
+        console.log("added " + count + " " + curr.length);
+        console.log(flex.info());
+      }
+
+      for await (const line of ndjson(resp.body.getReader())) {
+        if (!line.text) {
+          continue;
+        }
+
+        line.id = ++count;
+        lines.push(line);
+        if ((count % 100) === 0) {
+          await flush();
+        }
+      }
+
+      await flush();
+
+    } finally {
+      if (count === 0) {
+        this.flex.add(this.collInfo.pages);
+      }
+
+      this.updatingSearch = false;
+    }
+
+    await this.filter();
  }

  static get styles() {
@@ -313,6 +382,7 @@ class Pages extends LitElement
        display: flex;
        flex-direction: row;
        width: 100%;
+        min-height: fit-content;

        margin-bottom: 1.0em;
        border-bottom: 3px solid rgb(237, 237, 237);
@@ -428,8 +498,11 @@ class Pages extends LitElement
    </div>` : html``}

    <div class="header columns is-hidden-mobile">
-      <a @click="${this.onSort}" data-key="title" class="column is-2 ${this.sortKey === "title" ? (this.sortDesc ? "desc" : "asc") : ''}">Date</a>
-      <a @click="${this.onSort}" data-key="ts" class="column is-8 pagetitle ${this.sortKey === "ts" ? (this.sortDesc ? "desc" : "asc") : ''}">Page Title</a>
+      ${this.query ? html`
+      <a @click="${this.onSort}" data-key="" class="column is-1 ${this.sortKey === "" ? (this.sortDesc ? "desc" : "asc") : ''}">Score</a>` : ``}
+
+      <a @click="${this.onSort}" data-key="ts" class="column is-2 ${this.sortKey === "ts" ? (this.sortDesc ? "desc" : "asc") : ''}">Date</a>
+      <a @click="${this.onSort}" data-key="title" class="column is-6 pagetitle ${this.sortKey === "title" ? (this.sortDesc ? "desc" : "asc") : ''}">Page Title</a>
    </div>

    <div class="is-hidden-tablet mobile-header">
@@ -453,9 +526,9 @@ class Pages extends LitElement
      ${this.renderPageHeader()}
      </div>
      <div class="scroller" @scroll="${this.onScroll}">
-        ${this.sortedPages.map((p) => html`
+        ${this.sortedPages.map((p, i) => html`
        <div class="content ${this.selectedPages.has(p.id) ? 'selected' : ''}">
-          <wr-page-entry .editable="${this.editable}" .selected="${this.selectedPages.has(p.id)}" @sel-page="${this.onSelectToggle}" @delete-page="${this.onDeletePage}" replayPrefix="${this.collInfo.replayPrefix}" query="${this.query}" .page="${p}">
+          <wr-page-entry .index="${this.query ? (this.sortDesc ? this.sortedPages.length - i : i + 1) : 0}" .editable="${this.editable}" .selected="${this.selectedPages.has(p.id)}" @sel-page="${this.onSelectToggle}" @delete-page="${this.onDeletePage}" replayPrefix="${this.collInfo.replayPrefix}" query="${this.query}" .page="${p}">
          </wr-page-entry>
        </div>`)}
      </div>
@@ -476,7 +549,7 @@ class Pages extends LitElement
  onSort(event) {
    event.preventDefault();

-    const key = event.currentTarget.getAttribute("data-key");
+    const key = event.currentTarget.getAttribute("data-key") || "";
    if (key === this.sortKey) {
      this.sortDesc = !this.sortDesc;
    } else {
@@ -546,7 +619,6 @@ class Pages extends LitElement
    //}

    this.updateTextSearch();
-    this.filter();
    this.requestUpdate();
  }

@@ -555,6 +627,10 @@ class Pages extends LitElement
      return html`No Pages defined this archive. Check out&nbsp;<a href="#view=resources">Page Resources</a>&nbsp;to search by URL.`;
    }

+    if (this.updatingSearch) {
+      return "Initializing Search..."
+    }
+
    if (this.loading) {
      return "Searching...";
    }
@@ -592,6 +668,7 @@ class PageEntry extends LitElement
    this.deleting = false;
    this.editable = false;
    this.iconValid = false;
+    this.index = 0;
  }

  static get properties() {
@@ -603,7 +680,8 @@ class PageEntry extends LitElement
      deleting: { type: Boolean },
      selected: { type: Boolean },
      editable: { type: Boolean },
-      iconValid: { type: Boolean }
+      iconValid: { type: Boolean },
+      index: { type: Number },
    }
  }

@@ -705,11 +783,14 @@ class PageEntry extends LitElement
    </div>` : ``}

    <div class="columns">
+      ${this.index ? html`
+      <div class="column col-index is-1">${this.index}.</div>
+      ` : ``}
      <div class="column col-date is-2">
-      <div>${date ? date.toLocaleDateString() : ""}</div>
-      <div>${date ? date.toLocaleTimeString() : ""}</div>
+        <div>${date ? date.toLocaleDateString() : ""}</div>
+        <div>${date ? date.toLocaleTimeString() : ""}</div>
      </div>
-      <div class="column is-9">
+      <div class="column">
        <article class="media">
          <figure class="media-left">
            <p class="">
@@ -8,7 +8,6 @@ class Replay extends LitElement
 {
  constructor() {
    super();
-    this.isLoading = false;
    this.replayUrl = "";
    this.replayTS = "";
    this.url = "";
@@ -34,7 +33,6 @@ class Replay extends LitElement
      title: { type: String },

      iframeUrl: { type: String },
-      isLoading: { type: Boolean },

      showAuth: { type: Boolean },
    }
@@ -86,14 +84,6 @@ class Replay extends LitElement
  
      this.dispatchEvent(new CustomEvent("coll-tab-nav", {detail: {replaceLoc: true, data}}));
    }
-
-    if (this.embed && window.parent !== window && changedProperties.has("title")) {
-      window.parent.postMessage({
-        title: this.title,
-        url: this.replayUrl,
-        ts: this.replayTS
-      }, '*');
-    }
  }

  onReplayMessage(event) {
@@ -117,7 +107,7 @@ class Replay extends LitElement
  }

  waitForLoad() {
-    this.isLoading = true;
+    this.setLoading();
    this._loadPoll = window.setInterval(() => {
      const iframe = this.renderRoot.querySelector("iframe");
      if (!iframe || !iframe.contentDocument || !iframe.contentWindow || 
@@ -128,7 +118,8 @@ class Replay extends LitElement
  }

  clearLoading(iframeWin) {
-    this.isLoading = false;
+    this.dispatchEvent(new CustomEvent("replay-loading", {detail: {loading: false}}));
+
    if (this._loadPoll) {
      window.clearInterval(this._loadPoll);
      this._loadPoll = null;
@@ -136,16 +127,16 @@ class Replay extends LitElement

    if (iframeWin) {
      iframeWin.addEventListener("beforeunload", () => {
-        this.isLoading = true;
+        this.setLoading();
      });
    }
  }

-  refresh() {
-    if (this.isLoading && !forceReload) {
-      return;
-    }
+  setLoading() {
+    this.dispatchEvent(new CustomEvent("replay-loading", {detail: {loading: true}}));
+  }

+  refresh() {
    const iframe = this.renderRoot.querySelector("iframe");

    if (iframe) {
@@ -16,6 +16,12 @@ class Sorter extends LitElement

    this.numResults = DEFAULT_RESULTS;
    this.defaultDesc = false;
+
+    this.sortKey = "url";
+    this.sortDesc = false;
+
+    this.defaultKey = "";
+    this.defaultDesc = false;
  }

  static get properties() {
@@ -68,13 +74,19 @@ class Sorter extends LitElement
    this.sortedData = [...this.data];
    this.numResults = DEFAULT_RESULTS;

-    this.sortedData.sort((first, second) => {
-      if (first[this.sortKey] === second[this.sortKey]) {
-        return 0;
+    if (this.sortKey === "") {
+      if (this.sortDesc) {
+        this.sortedData.reverse();
      }
+    } else {
+      this.sortedData.sort((first, second) => {
+        if (first[this.sortKey] === second[this.sortKey]) {
+          return 0;
+        }

-      return (this.sortDesc == (first[this.sortKey] < second[this.sortKey])) ? 1 : -1;
-    });
+        return (this.sortDesc == (first[this.sortKey] < second[this.sortKey])) ? 1 : -1;
+      });
+    }

    this.sendSortChanged();
  }
@@ -235,10 +235,12 @@ class URLResources extends LitElement
    }   
    .result {
      border-bottom: 1px #dbdbdb solid;
+      min-height: fit-content;
    }
    .results-head {
      border-bottom: 2px #dbdbdb solid;
      margin-right: 16px;
+      min-height: fit-content;
    }
    .results-head a {
      color: black;