fidelity: fix eval edge case (via wombat), enable http->https check for wacz sources (via wabac.js)

dependency updates:
- switch to webpack 5 and related build dependencies
- update litelement -> 'lit'
- wabac.js 2.9.2, wombat 1.3.4

search:
- update to latest flexsearch
- optimize search by using single-field index

bump to 1.5.3

CHANGES.md

@@ -1,8 +1,13 @@
 ## CHANGES
 
+v1.5.3
+- Fidelity: fix for some eval replay (via wombat), include http->https check for wacz sources
+- Dependencies: update to webpack 5 build, wabac.js 2.9.2, wombat 3.3.4
+- Search: update to latest flexsearch, faster/improved searching
+
 v1.5.2
 - App: IPFS: Configure js-ipfs repo directory to be in existing profile directory
-- App: IPFS: Fix IPFS support for OSX M1 builds by including leveldown prebuilds
+- Loading: IPFS: Fix IPFS support for OSX M1 builds by including leveldown prebuilds
 - Dependencies: update to wabac.js 2.9.1, wombat 3.3.3
 
 v1.5.1

_data/package.json

@@ -1,47 +1,49 @@
 {
   "name": "replaywebpage",
   "productName": "ReplayWeb.page",
-  "version": "1.5.2",
+  "version": "1.5.3",
   "description": "Serverless Web Archive Replay",
   "repository": "https://github.com/webrecorder/replayweb.page",
   "homepage": "https://replayweb.page/",
   "author": "Webrecorder Software",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@fortawesome/fontawesome-free": "^5.13.0",
+    "@fortawesome/fontawesome-free": "^5.15.4",
-    "@webrecorder/wabac": "^2.9.1",
+    "@webrecorder/wabac": "^2.9.2",
     "bulma": "^0.9.3",
-    "electron-log": "^4.3.0",
+    "electron-log": "^4.4.1",
-    "electron-updater": "^4.3.5",
+    "electron-updater": "^4.3.9",
-    "fetch-ndjson": "^1.1.0",
+    "fetch-ndjson": "^1.1.6",
-    "flexsearch": "^0.6.32",
+    "flexsearch": "^0.7.21",
-    "ipfs-core": "^0.10.6",
+    "ipfs-core": "^0.10.8",
-    "keyword-mark-element": "^0.1.2",
+    "keyword-mark-element": "^0.2.0",
-    "lit-element": "^2.4.0",
+    "lit": "^2.0.0-rc.4",
-    "marked": "^1.0.0",
+    "marked": "^3.0.3",
-    "mime-types": "^2.1.27",
+    "mime-types": "^2.1.32",
     "minimist": "^1.2.5",
-    "pretty-bytes": "^5.3.0",
+    "pretty-bytes": "^5.6.0",
-    "register-service-worker": "^1.7.1",
+    "process": "^0.11.10",
-    "split.js": "^1.6.2"
+    "querystring-es3": "^0.2.1",
+    "register-service-worker": "^1.7.2",
+    "split.js": "^1.6.4",
+    "stream-browserify": "^3.0.0"
   },
   "devDependencies": {
-    "copy-webpack-plugin": "^6.3.0",
+    "copy-webpack-plugin": "^9.0.1",
-    "css-loader": "^3.5.3",
+    "css-loader": "^6.2.0",
-    "electron": "^14.0.0",
+    "electron": "^14.0.1",
-    "electron-builder": "^22.13.1",
+    "electron-builder": "^22.14.2",
-    "electron-notarize": "^1.0.0",
+    "electron-notarize": "^1.1.1",
-    "http-server": "^0.12.3",
+    "http-server": "^13.0.2",
-    "mini-css-extract-plugin": "^0.9.0",
+    "mini-css-extract-plugin": "^2.3.0",
-    "node-sass": "^4.14.0",
+    "node-sass": "^6.0.1",
-    "raw-loader": "^4.0.1",
+    "raw-loader": "^4.0.2",
-    "sass-loader": "^8.0.2",
+    "sass-loader": "^12.1.0",
-    "style-loader": "^1.2.1",
+    "style-loader": "^3.2.1",
     "svg-inline-loader": "^0.8.2",
-    "to-string-loader": "^1.1.6",
+    "webpack": "^5.53.0",
-    "webpack": "^4.44.2",
+    "webpack-cli": "^4.8.0",
-    "webpack-cli": "^3.3.11",
+    "webpack-dev-server": "^4.2.1"
-    "webpack-dev-server": "^3.10.3"
   },
   "files": [
     "/assets",
@@ -55,7 +57,7 @@
     "build": "webpack --mode production",
     "build-dev": "webpack --mode development",
     "build-docs": "bundle install; bundle exec jekyll build",
-    "start-dev": "webpack-dev-server --mode development",
+    "start-dev": "webpack serve --mode development",
     "start-prod": "http-server -p 9990",
     "pack": "CSC_IDENTITY_AUTO_DISCOVERY=false electron-builder --publish never",
     "pack-signed": "electron-builder",

package.json

@@ -1,47 +1,49 @@
 {
   "name": "replaywebpage",
   "productName": "ReplayWeb.page",
-  "version": "1.5.2",
+  "version": "1.5.3",
   "description": "Serverless Web Archive Replay",
   "repository": "https://github.com/webrecorder/replayweb.page",
   "homepage": "https://replayweb.page/",
   "author": "Webrecorder Software",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@fortawesome/fontawesome-free": "^5.13.0",
+    "@fortawesome/fontawesome-free": "^5.15.4",
-    "@webrecorder/wabac": "^2.9.1",
+    "@webrecorder/wabac": "^2.9.2",
     "bulma": "^0.9.3",
-    "electron-log": "^4.3.0",
+    "electron-log": "^4.4.1",
-    "electron-updater": "^4.3.5",
+    "electron-updater": "^4.3.9",
-    "fetch-ndjson": "^1.1.0",
+    "fetch-ndjson": "^1.1.6",
-    "flexsearch": "^0.6.32",
+    "flexsearch": "^0.7.21",
-    "ipfs-core": "^0.10.6",
+    "ipfs-core": "^0.10.8",
-    "keyword-mark-element": "^0.1.2",
+    "keyword-mark-element": "^0.2.0",
-    "lit-element": "^2.4.0",
+    "lit": "^2.0.0-rc.4",
-    "marked": "^1.0.0",
+    "marked": "^3.0.3",
-    "mime-types": "^2.1.27",
+    "mime-types": "^2.1.32",
     "minimist": "^1.2.5",
-    "pretty-bytes": "^5.3.0",
+    "pretty-bytes": "^5.6.0",
-    "register-service-worker": "^1.7.1",
+    "process": "^0.11.10",
-    "split.js": "^1.6.2"
+    "querystring-es3": "^0.2.1",
+    "register-service-worker": "^1.7.2",
+    "split.js": "^1.6.4",
+    "stream-browserify": "^3.0.0"
   },
   "devDependencies": {
-    "copy-webpack-plugin": "^6.3.0",
+    "copy-webpack-plugin": "^9.0.1",
-    "css-loader": "^3.5.3",
+    "css-loader": "^6.2.0",
-    "electron": "^14.0.0",
+    "electron": "^14.0.1",
-    "electron-builder": "^22.13.1",
+    "electron-builder": "^22.14.2",
-    "electron-notarize": "^1.0.0",
+    "electron-notarize": "^1.1.1",
-    "http-server": "^0.12.3",
+    "http-server": "^13.0.2",
-    "mini-css-extract-plugin": "^0.9.0",
+    "mini-css-extract-plugin": "^2.3.0",
-    "node-sass": "^4.14.0",
+    "node-sass": "^6.0.1",
-    "raw-loader": "^4.0.1",
+    "raw-loader": "^4.0.2",
-    "sass-loader": "^8.0.2",
+    "sass-loader": "^12.1.0",
-    "style-loader": "^1.2.1",
+    "style-loader": "^3.2.1",
     "svg-inline-loader": "^0.8.2",
-    "to-string-loader": "^1.1.6",
+    "webpack": "^5.53.0",
-    "webpack": "^4.44.2",
+    "webpack-cli": "^4.8.0",
-    "webpack-cli": "^3.3.11",
+    "webpack-dev-server": "^4.2.1"
-    "webpack-dev-server": "^3.10.3"
   },
   "files": [
     "/assets",
@@ -55,7 +57,7 @@
     "build": "webpack --mode production",
     "build-dev": "webpack --mode development",
     "build-docs": "bundle install; bundle exec jekyll build",
-    "start-dev": "webpack-dev-server --mode development",
+    "start-dev": "webpack serve --mode development",
     "start-prod": "http-server -p 9990",
     "pack": "CSC_IDENTITY_AUTO_DISCOVERY=false electron-builder --publish never",
     "pack-signed": "electron-builder",

src/appmain.js

@@ -1,4 +1,4 @@
-import { LitElement, html, css } from "lit-element";
+import { LitElement, html, css } from "lit";
 import { wrapCss, rwpLogo, IS_APP, VERSION, clickOnSpacebarPress } from "./misc";
 
 import { registerSW } from "./pageutils";

src/chooser.js

@@ -1,4 +1,4 @@
-import { LitElement, html, css } from "lit-element";
+import { LitElement, html, css } from "lit";
 import { IS_APP, wrapCss } from "./misc";
 
 import fasUpload from "@fortawesome/fontawesome-free/svgs/solid/upload.svg";

src/coll-index.js

@@ -1,4 +1,4 @@
-import { LitElement, html, css } from "lit-element";
+import { LitElement, html, css } from "lit";
 import { wrapCss, apiPrefix } from "./misc";
 
 import prettyBytes from "pretty-bytes";

src/coll.js

@@ -1,4 +1,4 @@
-import { LitElement, html, css } from "lit-element";
+import { LitElement, html, css } from "lit";
 import { wrapCss, rwpLogo, IS_APP, VERSION, clickOnSpacebarPress, apiPrefix, replayPrefix } from "./misc";
 
 import { sourceToId, tsToDate, getPageDateTS } from "./pageutils";

src/embed.js

@@ -1,5 +1,5 @@
-import { LitElement, html, css } from "lit-element";
+import { LitElement, html, css } from "lit";
-import { ifDefined } from "lit-html/directives/if-defined.js";
+import { ifDefined } from "lit/directives/if-defined.js";
 
 import { registerSW } from "./pageutils";
 

src/gdrive.js

@@ -1,4 +1,4 @@
-import { LitElement, html, css } from "lit-element";
+import { LitElement, html, css } from "lit";
 import { wrapCss } from "./misc";
 
 import fabGoogleDrive from "@fortawesome/fontawesome-free/svgs/brands/google-drive.svg";

src/loader.js

@@ -1,4 +1,4 @@
-import { LitElement, html, css } from "lit-element";
+import { LitElement, html, css } from "lit";
 import { wrapCss, IS_APP } from "./misc";
 
 import prettyBytes from "pretty-bytes";

src/misc.js

@@ -1,8 +1,8 @@
 "use strict";
 
-import { LitElement, html, css, unsafeCSS } from "lit-element";
+import { LitElement, html, css, unsafeCSS } from "lit";
-import { unsafeSVG } from "lit-html/directives/unsafe-svg";
+import { unsafeSVG } from "lit/directives/unsafe-svg.js";
-import { styleMap } from "lit-html/directives/style-map";
+import { styleMap } from "lit/directives/style-map.js";
 
 import allCssRaw from "../assets/main.scss";
 

src/pageentry.js

@@ -1,6 +1,6 @@
 import prettyBytes from "pretty-bytes";
 
-import { LitElement, html, css, unsafeCSS } from "lit-element";
+import { LitElement, html, css, unsafeCSS } from "lit";
 
 import "keyword-mark-element/lib/keyword-mark.js";
 

src/pages.js

@@ -1,10 +1,10 @@
 "use strict";
 
-import { LitElement, html, css, unsafeCSS } from "lit-element";
+import { LitElement, html, css, unsafeCSS } from "lit";
 import { wrapCss, clickOnSpacebarPress } from "./misc";
 import ndjson from "fetch-ndjson";
 
-import FlexSearch from "flexsearch";
+import { Index as FlexIndex } from "flexsearch";
 import prettyBytes from "pretty-bytes";
 
 import { getTS, getPageDateTS } from "./pageutils";
@@ -24,6 +24,7 @@ class Pages extends LitElement
     this.sortedPages = [];
     this.query = "";
     this.flex = null;
+    this.textPages = null;
     this.newQuery = null;
     this.loading = false;
     this.updatingSearch = false;
@@ -169,10 +170,9 @@ class Pages extends LitElement
       this.changeNeeded = true;
     }
     this.loading = true;
-    if (this.flex && this.query) {
+    if (this.flex && this.query && this.textPages) {
-      const result = await this.flex.search(this.query, {limit: 25});
+      const results = await this.flex.searchAsync(this.query, 25);
-
+      this.filteredPages = results.map(inx => this.textPages[inx]);
-      this.filteredPages = result;
     } else {
       this.filteredPages = [...this.collInfo.pages];
     }
@@ -207,6 +207,24 @@ class Pages extends LitElement
     this.dispatchEvent(new CustomEvent("coll-tab-nav", {detail: {data}}));
   }
 
+  addPages(pages) {
+    const flex = new FlexIndex();
+
+    this.flex = flex;
+    this.textPages = pages;
+
+    return Promise.all(pages.map((page, index) => {
+      let text = page.url;
+      if (page.title) {
+        text += " " + page.title;
+      }
+      if (page.text) {
+        text += " " + page.text;
+      }
+      return flex.addAsync(index, text);
+    }));
+  };
+
   async updateTextSearch() {
     if (this.updatingSearch) {
       return;
@@ -216,16 +234,6 @@ class Pages extends LitElement
     let count = 0;
 
     try {
-      const flex = new FlexSearch({
-        doc: {
-          id: "id",
-          field: ["url", "title", "text"],
-        },
-        async: true
-      });
-
-      this.flex = flex;
-
       const cache = await caches.open("cache:" + this.collInfo.coll);
 
       const indexUrl = `${this.collInfo.apiPrefix}/textIndex`;
@@ -239,15 +247,7 @@ class Pages extends LitElement
         }
       }
 
-      let lines = [];
+      const lines = [];
-
-      const flush = async () => {
-        let curr = lines;
-        lines = [];
-        await flex.add(curr);
-        //console.log("added " + count + " " + curr.length);
-        //console.log(flex.info());
-      };
 
       for await (const line of ndjson(resp.body.getReader())) {
         if (!line.text) {
@@ -256,16 +256,16 @@ class Pages extends LitElement
 
         line.id = ++count;
         lines.push(line);
-        if ((count % 100) === 0) {
-          await flush();
-        }
       }
 
-      await flush();
+      await this.addPages(lines);
+
+    } catch (e) {
+      console.warn(e);
 
     } finally {
       if (count === 0) {
-        this.flex.add(this.collInfo.pages);
+        await this.addPages(this.collInfo.pages);
       }
 
       this.updatingSearch = false;

src/replay.js

@@ -1,4 +1,4 @@
-import { LitElement, html, css } from "lit-element";
+import { LitElement, html, css } from "lit";
 import { wrapCss, rwpLogo } from "./misc";
 
 

src/sorter.js

@@ -1,4 +1,4 @@
-import { LitElement, html, css } from "lit-element";
+import { LitElement, html, css } from "lit";
 import { wrapCss } from "./misc";
 
 import fasSortDown from "@fortawesome/fontawesome-free/svgs/solid/sort-down.svg";

src/story.js

@@ -1,7 +1,7 @@
-import { LitElement, html, css, unsafeCSS } from "lit-element";
+import { LitElement, html, css, unsafeCSS } from "lit";
 import { wrapCss } from "./misc";
 
-import { unsafeHTML } from "lit-html/directives/unsafe-html";
+import { unsafeHTML } from "lit/directives/unsafe-html.js";
 import marked from "marked";
 
 import { getTS, getReplayLink } from "./pageutils";
@@ -397,4 +397,4 @@ class Story extends LitElement
 
 customElements.define("wr-coll-story", Story);
 
-export { Story };
+export { Story };

src/url-resources.js

@@ -1,4 +1,4 @@
-import { LitElement, html, css } from "lit-element";
+import { LitElement, html, css } from "lit";
 import { wrapCss, clickOnSpacebarPress } from "./misc";
 
 import { getReplayLink } from "./pageutils";

webpack.config.js

@@ -4,6 +4,7 @@ const path = require("path");
 const webpack = require("webpack");
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
 const CopyPlugin = require("copy-webpack-plugin");
+const TerserPlugin = require("terser-webpack-plugin");
 const package_json = require("./package.json");
 
 // fake url used in app to serve files
@@ -25,6 +26,23 @@ const BANNER_TEXT = "'[name].js is part of ReplayWeb.page (https://replayweb.pag
 const IPFS_CORE_URL = `https://cdn.jsdelivr.net/npm/ipfs-core@${package_json.dependencies["ipfs-core"].replace("^", "")}/dist/index.min.js`;
 
 
+const fallback = {
+  "stream": require.resolve("stream-browserify"),
+  "querystring": require.resolve("querystring-es3"),
+  "url": require.resolve("url/"),
+  "buffer": false,
+};
+
+const optimization = {
+  minimize: true,
+  minimizer: [
+    new TerserPlugin({
+      extractComments: false,
+    }),
+  ],
+};
+
+
 const electronMainConfig = (/*env, argv*/) => {
   return {
     target: "electron-main",
@@ -32,12 +50,13 @@ const electronMainConfig = (/*env, argv*/) => {
     entry: {
       "electron": "./src/electron-main.js", 
     },
+    optimization,
     resolve: {
       alias: {
         "abort-controller": "abort-controller/dist/abort-controller.js",
         "dlv": "dlv/dist/dlv.js",
         "bignumber.js": "bignumber.js/bignumber.js",
-        "multiformats/hashes/sha2": "multiformats/cjs/src/hashes/sha2.js"
+        //"multiformats/hashes/sha2": "multiformats/cjs/src/hashes/sha2.js"
       }
     },
     output: {
@@ -77,6 +96,7 @@ const electronPreloadConfig = (/*env, argv*/) => {
     entry: {
       "preload": "./src/electron-preload.js", 
     },
+    optimization,
     plugins: [
       new webpack.BannerPlugin(BANNER_TEXT),
 
@@ -100,6 +120,9 @@ const browserConfig = (/*env, argv*/) => {
       "sw": "./src/sw.js"
     },
 
+    optimization,
+    resolve: {fallback},
+
     output: {
       path: path.join(__dirname),
       filename: "[name].js",
@@ -116,14 +139,17 @@ const browserConfig = (/*env, argv*/) => {
       compress: true,
       port: 9990,
       open: false,
-      contentBase:  path.join(__dirname),
+      static:  path.join(__dirname),
-      publicPath: "/"
+      //publicPath: "/"
     },
 
     plugins: [
       new webpack.optimize.LimitChunkCountPlugin({
         maxChunks: 1,
       }),
+      new webpack.ProvidePlugin({
+        process: "process/browser",
+      }),
       new MiniCssExtractPlugin(),
       new webpack.DefinePlugin({
         __SW_NAME__: JSON.stringify("sw.js"),