archivalo-oss/replayweb.page
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fidelity: fix eval edge case (via wombat), enable http->https check for wacz sources (via wabac.js)

Browse Source 
dependency updates:
- switch to webpack 5 and related build dependencies
- update litelement -> 'lit'
- wabac.js 2.9.2, wombat 1.3.4

search:
- update to latest flexsearch
- optimize search by using single-field index

bump to 1.5.3
This commit is contained in:
Ilya Kreymer
2021-09-18 12:32:00 -07:00
parent d341c03350
commit a261a61721
22 changed files with 1386 additions and 4310 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGES.md
+6 -1
View File
@@ -1,8 +1,13 @@
## CHANGES
v1.5.3
- Fidelity: fix for some eval replay (via wombat), include http->https check for wacz sources
- Dependencies: update to webpack 5 build, wabac.js 2.9.2, wombat 3.3.4
- Search: update to latest flexsearch, faster/improved searching
v1.5.2
- App: IPFS: Configure js-ipfs repo directory to be in existing profile directory
- App: IPFS: Fix IPFS support for OSX M1 builds by including leveldown prebuilds
- Loading: IPFS: Fix IPFS support for OSX M1 builds by including leveldown prebuilds
- Dependencies: update to wabac.js 2.9.1, wombat 3.3.3
v1.5.1
_data/package.json
+33 -31
View File
@@ -1,47 +1,49 @@
{
"name": "replaywebpage",
"productName": "ReplayWeb.page",
"version": "1.5.2",
"version": "1.5.3",
"description": "Serverless Web Archive Replay",
"repository": "https://github.com/webrecorder/replayweb.page",
"homepage": "https://replayweb.page/",
"author": "Webrecorder Software",
"license": "AGPL-3.0-or-later",
"dependencies": {
"@fortawesome/fontawesome-free": "^5.13.0",
"@webrecorder/wabac": "^2.9.1",
"@fortawesome/fontawesome-free": "^5.15.4",
"@webrecorder/wabac": "^2.9.2",
"bulma": "^0.9.3",
"electron-log": "^4.3.0",
"electron-updater": "^4.3.5",
"fetch-ndjson": "^1.1.0",
"flexsearch": "^0.6.32",
"ipfs-core": "^0.10.6",
"keyword-mark-element": "^0.1.2",
"lit-element": "^2.4.0",
"marked": "^1.0.0",
"mime-types": "^2.1.27",
"electron-log": "^4.4.1",
"electron-updater": "^4.3.9",
"fetch-ndjson": "^1.1.6",
"flexsearch": "^0.7.21",
"ipfs-core": "^0.10.8",
"keyword-mark-element": "^0.2.0",
"lit": "^2.0.0-rc.4",
"marked": "^3.0.3",
"mime-types": "^2.1.32",
"minimist": "^1.2.5",
"pretty-bytes": "^5.3.0",
"register-service-worker": "^1.7.1",
"split.js": "^1.6.2"
"pretty-bytes": "^5.6.0",
"process": "^0.11.10",
"querystring-es3": "^0.2.1",
"register-service-worker": "^1.7.2",
"split.js": "^1.6.4",
"stream-browserify": "^3.0.0"
},
"devDependencies": {
"copy-webpack-plugin": "^6.3.0",
"css-loader": "^3.5.3",
"electron": "^14.0.0",
"electron-builder": "^22.13.1",
"electron-notarize": "^1.0.0",
"http-server": "^0.12.3",
"mini-css-extract-plugin": "^0.9.0",
"node-sass": "^4.14.0",
"raw-loader": "^4.0.1",
"sass-loader": "^8.0.2",
"style-loader": "^1.2.1",
"copy-webpack-plugin": "^9.0.1",
"css-loader": "^6.2.0",
"electron": "^14.0.1",
"electron-builder": "^22.14.2",
"electron-notarize": "^1.1.1",
"http-server": "^13.0.2",
"mini-css-extract-plugin": "^2.3.0",
"node-sass": "^6.0.1",
"raw-loader": "^4.0.2",
"sass-loader": "^12.1.0",
"style-loader": "^3.2.1",
"svg-inline-loader": "^0.8.2",
"to-string-loader": "^1.1.6",
"webpack": "^4.44.2",
"webpack-cli": "^3.3.11",
"webpack-dev-server": "^3.10.3"
"webpack": "^5.53.0",
"webpack-cli": "^4.8.0",
"webpack-dev-server": "^4.2.1"
},
"files": [
"/assets",
@@ -55,7 +57,7 @@
"build": "webpack --mode production",
"build-dev": "webpack --mode development",
"build-docs": "bundle install; bundle exec jekyll build",
"start-dev": "webpack-dev-server --mode development",
"start-dev": "webpack serve --mode development",
"start-prod": "http-server -p 9990",
"pack": "CSC_IDENTITY_AUTO_DISCOVERY=false electron-builder --publish never",
"pack-signed": "electron-builder",
frontend.js
-1097
View File
File diff suppressed because one or more lines are too long
package.json
+33 -31
View File
@@ -1,47 +1,49 @@
{
"name": "replaywebpage",
"productName": "ReplayWeb.page",
"version": "1.5.2",
"version": "1.5.3",
"description": "Serverless Web Archive Replay",
"repository": "https://github.com/webrecorder/replayweb.page",
"homepage": "https://replayweb.page/",
"author": "Webrecorder Software",
"license": "AGPL-3.0-or-later",
"dependencies": {
"@fortawesome/fontawesome-free": "^5.13.0",
"@webrecorder/wabac": "^2.9.1",
"@fortawesome/fontawesome-free": "^5.15.4",
"@webrecorder/wabac": "^2.9.2",
"bulma": "^0.9.3",
"electron-log": "^4.3.0",
"electron-updater": "^4.3.5",
"fetch-ndjson": "^1.1.0",
"flexsearch": "^0.6.32",
"ipfs-core": "^0.10.6",
"keyword-mark-element": "^0.1.2",
"lit-element": "^2.4.0",
"marked": "^1.0.0",
"mime-types": "^2.1.27",
"electron-log": "^4.4.1",
"electron-updater": "^4.3.9",
"fetch-ndjson": "^1.1.6",
"flexsearch": "^0.7.21",
"ipfs-core": "^0.10.8",
"keyword-mark-element": "^0.2.0",
"lit": "^2.0.0-rc.4",
"marked": "^3.0.3",
"mime-types": "^2.1.32",
"minimist": "^1.2.5",
"pretty-bytes": "^5.3.0",
"register-service-worker": "^1.7.1",
"split.js": "^1.6.2"
"pretty-bytes": "^5.6.0",
"process": "^0.11.10",
"querystring-es3": "^0.2.1",
"register-service-worker": "^1.7.2",
"split.js": "^1.6.4",
"stream-browserify": "^3.0.0"
},
"devDependencies": {
"copy-webpack-plugin": "^6.3.0",
"css-loader": "^3.5.3",
"electron": "^14.0.0",
"electron-builder": "^22.13.1",
"electron-notarize": "^1.0.0",
"http-server": "^0.12.3",
"mini-css-extract-plugin": "^0.9.0",
"node-sass": "^4.14.0",
"raw-loader": "^4.0.1",
"sass-loader": "^8.0.2",
"style-loader": "^1.2.1",
"copy-webpack-plugin": "^9.0.1",
"css-loader": "^6.2.0",
"electron": "^14.0.1",
"electron-builder": "^22.14.2",
"electron-notarize": "^1.1.1",
"http-server": "^13.0.2",
"mini-css-extract-plugin": "^2.3.0",
"node-sass": "^6.0.1",
"raw-loader": "^4.0.2",
"sass-loader": "^12.1.0",
"style-loader": "^3.2.1",
"svg-inline-loader": "^0.8.2",
"to-string-loader": "^1.1.6",
"webpack": "^4.44.2",
"webpack-cli": "^3.3.11",
"webpack-dev-server": "^3.10.3"
"webpack": "^5.53.0",
"webpack-cli": "^4.8.0",
"webpack-dev-server": "^4.2.1"
},
"files": [
"/assets",
@@ -55,7 +57,7 @@
"build": "webpack --mode production",
"build-dev": "webpack --mode development",
"build-docs": "bundle install; bundle exec jekyll build",
"start-dev": "webpack-dev-server --mode development",
"start-dev": "webpack serve --mode development",
"start-prod": "http-server -p 9990",
"pack": "CSC_IDENTITY_AUTO_DISCOVERY=false electron-builder --publish never",
"pack-signed": "electron-builder",
src/appmain.js
+1 -1
View File
@@ -1,4 +1,4 @@
import { LitElement, html, css } from "lit-element";
import { LitElement, html, css } from "lit";
import { wrapCss, rwpLogo, IS_APP, VERSION, clickOnSpacebarPress } from "./misc";
import { registerSW } from "./pageutils";
src/chooser.js
+1 -1
View File
@@ -1,4 +1,4 @@
import { LitElement, html, css } from "lit-element";
import { LitElement, html, css } from "lit";
import { IS_APP, wrapCss } from "./misc";
import fasUpload from "@fortawesome/fontawesome-free/svgs/solid/upload.svg";
src/coll-index.js
+1 -1
View File
@@ -1,4 +1,4 @@
import { LitElement, html, css } from "lit-element";
import { LitElement, html, css } from "lit";
import { wrapCss, apiPrefix } from "./misc";
import prettyBytes from "pretty-bytes";
src/coll.js
+1 -1
View File
@@ -1,4 +1,4 @@
import { LitElement, html, css } from "lit-element";
import { LitElement, html, css } from "lit";
import { wrapCss, rwpLogo, IS_APP, VERSION, clickOnSpacebarPress, apiPrefix, replayPrefix } from "./misc";
import { sourceToId, tsToDate, getPageDateTS } from "./pageutils";
src/embed.js
+2 -2
View File
@@ -1,5 +1,5 @@
import { LitElement, html, css } from "lit-element";
import { ifDefined } from "lit-html/directives/if-defined.js";
import { LitElement, html, css } from "lit";
import { ifDefined } from "lit/directives/if-defined.js";
import { registerSW } from "./pageutils";
src/gdrive.js
+1 -1
View File
@@ -1,4 +1,4 @@
import { LitElement, html, css } from "lit-element";
import { LitElement, html, css } from "lit";
import { wrapCss } from "./misc";
import fabGoogleDrive from "@fortawesome/fontawesome-free/svgs/brands/google-drive.svg";
src/loader.js
+1 -1
View File
@@ -1,4 +1,4 @@
import { LitElement, html, css } from "lit-element";
import { LitElement, html, css } from "lit";
import { wrapCss, IS_APP } from "./misc";
import prettyBytes from "pretty-bytes";
src/misc.js
+3 -3
View File
@@ -1,8 +1,8 @@
"use strict";
import { LitElement, html, css, unsafeCSS } from "lit-element";
import { unsafeSVG } from "lit-html/directives/unsafe-svg";
import { styleMap } from "lit-html/directives/style-map";
import { LitElement, html, css, unsafeCSS } from "lit";
import { unsafeSVG } from "lit/directives/unsafe-svg.js";
import { styleMap } from "lit/directives/style-map.js";
import allCssRaw from "../assets/main.scss";
src/pageentry.js
+1 -1
View File
@@ -1,6 +1,6 @@
import prettyBytes from "pretty-bytes";
import { LitElement, html, css, unsafeCSS } from "lit-element";
import { LitElement, html, css, unsafeCSS } from "lit";
import "keyword-mark-element/lib/keyword-mark.js";
src/pages.js
+30 -30
View File
@@ -1,10 +1,10 @@
"use strict";
import { LitElement, html, css, unsafeCSS } from "lit-element";
import { LitElement, html, css, unsafeCSS } from "lit";
import { wrapCss, clickOnSpacebarPress } from "./misc";
import ndjson from "fetch-ndjson";
import FlexSearch from "flexsearch";
import { Index as FlexIndex } from "flexsearch";
import prettyBytes from "pretty-bytes";
import { getTS, getPageDateTS } from "./pageutils";
@@ -24,6 +24,7 @@ class Pages extends LitElement
this.sortedPages = [];
this.query = "";
this.flex = null;
this.textPages = null;
this.newQuery = null;
this.loading = false;
this.updatingSearch = false;
@@ -169,10 +170,9 @@ class Pages extends LitElement
this.changeNeeded = true;
}
this.loading = true;
if (this.flex && this.query) {
const result = await this.flex.search(this.query, {limit: 25});
this.filteredPages = result;
if (this.flex && this.query && this.textPages) {
const results = await this.flex.searchAsync(this.query, 25);
this.filteredPages = results.map(inx => this.textPages[inx]);
} else {
this.filteredPages = [...this.collInfo.pages];
}
@@ -207,6 +207,24 @@ class Pages extends LitElement
this.dispatchEvent(new CustomEvent("coll-tab-nav", {detail: {data}}));
}
addPages(pages) {
const flex = new FlexIndex();
this.flex = flex;
this.textPages = pages;
return Promise.all(pages.map((page, index) => {
let text = page.url;
if (page.title) {
text += " " + page.title;
}
if (page.text) {
text += " " + page.text;
}
return flex.addAsync(index, text);
}));
};
async updateTextSearch() {
if (this.updatingSearch) {
return;
@@ -216,16 +234,6 @@ class Pages extends LitElement
let count = 0;
try {
const flex = new FlexSearch({
doc: {
id: "id",
field: ["url", "title", "text"],
},
async: true
});
this.flex = flex;
const cache = await caches.open("cache:" + this.collInfo.coll);
const indexUrl = `${this.collInfo.apiPrefix}/textIndex`;
@@ -239,15 +247,7 @@ class Pages extends LitElement
}
}
let lines = [];
const flush = async () => {
let curr = lines;
lines = [];
await flex.add(curr);
//console.log("added " + count + " " + curr.length);
//console.log(flex.info());
};
const lines = [];
for await (const line of ndjson(resp.body.getReader())) {
if (!line.text) {
@@ -256,16 +256,16 @@ class Pages extends LitElement
line.id = ++count;
lines.push(line);
if ((count % 100) === 0) {
await flush();
}
}
await flush();
await this.addPages(lines);
} catch (e) {
console.warn(e);
} finally {
if (count === 0) {
this.flex.add(this.collInfo.pages);
await this.addPages(this.collInfo.pages);
}
this.updatingSearch = false;
src/replay.js
+1 -1
View File
@@ -1,4 +1,4 @@
import { LitElement, html, css } from "lit-element";
import { LitElement, html, css } from "lit";
import { wrapCss, rwpLogo } from "./misc";
src/sorter.js
+1 -1
View File
@@ -1,4 +1,4 @@
import { LitElement, html, css } from "lit-element";
import { LitElement, html, css } from "lit";
import { wrapCss } from "./misc";
import fasSortDown from "@fortawesome/fontawesome-free/svgs/solid/sort-down.svg";
src/story.js
+3 -3
View File
@@ -1,7 +1,7 @@
import { LitElement, html, css, unsafeCSS } from "lit-element";
import { LitElement, html, css, unsafeCSS } from "lit";
import { wrapCss } from "./misc";
import { unsafeHTML } from "lit-html/directives/unsafe-html";
import { unsafeHTML } from "lit/directives/unsafe-html.js";
import marked from "marked";
import { getTS, getReplayLink } from "./pageutils";
@@ -397,4 +397,4 @@ class Story extends LitElement
customElements.define("wr-coll-story", Story);
export { Story };
export { Story };
src/url-resources.js
+1 -1
View File
@@ -1,4 +1,4 @@
import { LitElement, html, css } from "lit-element";
import { LitElement, html, css } from "lit";
import { wrapCss, clickOnSpacebarPress } from "./misc";
import { getReplayLink } from "./pageutils";
sw.js
+9 -23
View File
File diff suppressed because one or more lines are too long
ui.js
+262 -402
View File
File diff suppressed because one or more lines are too long
webpack.config.js
+29 -3
View File
@@ -4,6 +4,7 @@ const path = require("path");
const webpack = require("webpack");
const MiniCssExtractPlugin = require("mini-css-extract-plugin");
const CopyPlugin = require("copy-webpack-plugin");
const TerserPlugin = require("terser-webpack-plugin");
const package_json = require("./package.json");
// fake url used in app to serve files
@@ -25,6 +26,23 @@ const BANNER_TEXT = "'[name].js is part of ReplayWeb.page (https://replayweb.pag
const IPFS_CORE_URL = `https://cdn.jsdelivr.net/npm/ipfs-core@${package_json.dependencies["ipfs-core"].replace("^", "")}/dist/index.min.js`;
const fallback = {
"stream": require.resolve("stream-browserify"),
"querystring": require.resolve("querystring-es3"),
"url": require.resolve("url/"),
"buffer": false,
};
const optimization = {
minimize: true,
minimizer: [
new TerserPlugin({
extractComments: false,
}),
],
};
const electronMainConfig = (/*env, argv*/) => {
return {
target: "electron-main",
@@ -32,12 +50,13 @@ const electronMainConfig = (/*env, argv*/) => {
entry: {
"electron": "./src/electron-main.js",
},
optimization,
resolve: {
alias: {
"abort-controller": "abort-controller/dist/abort-controller.js",
"dlv": "dlv/dist/dlv.js",
"bignumber.js": "bignumber.js/bignumber.js",
"multiformats/hashes/sha2": "multiformats/cjs/src/hashes/sha2.js"
//"multiformats/hashes/sha2": "multiformats/cjs/src/hashes/sha2.js"
}
},
output: {
@@ -77,6 +96,7 @@ const electronPreloadConfig = (/*env, argv*/) => {
entry: {
"preload": "./src/electron-preload.js",
},
optimization,
plugins: [
new webpack.BannerPlugin(BANNER_TEXT),
@@ -100,6 +120,9 @@ const browserConfig = (/*env, argv*/) => {
"sw": "./src/sw.js"
},
optimization,
resolve: {fallback},
output: {
path: path.join(__dirname),
filename: "[name].js",
@@ -116,14 +139,17 @@ const browserConfig = (/*env, argv*/) => {
compress: true,
port: 9990,
open: false,
contentBase: path.join(__dirname),
publicPath: "/"
static: path.join(__dirname),
//publicPath: "/"
},
plugins: [
new webpack.optimize.LimitChunkCountPlugin({
maxChunks: 1,
}),
new webpack.ProvidePlugin({
process: "process/browser",
}),
new MiniCssExtractPlugin(),
new webpack.DefinePlugin({
__SW_NAME__: JSON.stringify("sw.js"),
yarn.lock
+966 -2674
View File
File diff suppressed because it is too large Load Diff