archivalo-oss/replayweb.page
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

remove js-ipfs in electron app, eslint to dev deps (#123)

Browse Source 
- add ipfs:// protocol handlers, not enabled for now
- use browser-side auto-js-ipfs for loading from ipfs even in electron app
- deps: bump wabac.js to 2.13.0
- update CHANGES

Co-authored-by: Ilya Kreymer <ikreymer@gmail.com>
This commit is contained in:
RangerMauve
2022-10-04 15:54:43 -07:00
committed by GitHub
co-authored by Ilya Kreymer
parent 43122425e5
commit f1df02ea3c
10 changed files with 562 additions and 2801 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGES.md
+1
View File
@@ -11,6 +11,7 @@ v1.7.0
- Fidelity/Embeds: Redirect new windows from links / `window.open` to existing replay frame (only possible w/o 'sandbox').
- Embeds: Add 'requireSubdomainIframe' embed attribute to only allow embed if loaded from a cross-origin / subdomain iframe.
- UI: improved formatting on 404 not found page.
- Loading: Fix IPFS loading, use auto-js-ipfs to load from gateway or existing local instance (don't run local node for now)
- Dependencies: update to wabac.js 2.13.0 to support verification, latest wombat (3.3.9)
v1.6.5
_data/package.json
+4 -3
View File
@@ -9,13 +9,13 @@
"license": "AGPL-3.0-or-later",
"dependencies": {
"@fortawesome/fontawesome-free": "^5.15.4",
"@webrecorder/wabac": "github:webrecorder/wabac.js",
"@webrecorder/wabac": "^2.13.0",
"auto-js-ipfs": "^1.3.9",
"bulma": "^0.9.3",
"electron-log": "^4.4.1",
"electron-updater": "^4.3.9",
"fetch-ndjson": "^1.1.6",
"flexsearch": "^0.7.21",
"ipfs-core": "^0.10.8",
"keyword-mark-element": "^0.2.0",
"lit": "^2.1.1",
"marked": "^4.0.10",
@@ -34,6 +34,7 @@
"electron": "^19.0.4",
"electron-builder": "^23.0.3",
"electron-notarize": "^1.1.1",
"eslint": "^8.23.1",
"http-server": "^13.0.2",
"mini-css-extract-plugin": "^2.3.0",
"node-sass": "^7.0.1",
@@ -117,7 +118,7 @@
"extraResources": [
"plugins-mac"
],
"singleArchFiles": "dist/prebuilds/**/*"
"singleArchFiles": "{dist/prebuilds,dist/prebuilds/**}"
},
"linux": {
"category": "Archiving;Utility;",
package.json
+4 -3
View File
@@ -9,13 +9,13 @@
"license": "AGPL-3.0-or-later",
"dependencies": {
"@fortawesome/fontawesome-free": "^5.15.4",
"@webrecorder/wabac": "github:webrecorder/wabac.js",
"@webrecorder/wabac": "^2.13.0",
"auto-js-ipfs": "^1.3.9",
"bulma": "^0.9.3",
"electron-log": "^4.4.1",
"electron-updater": "^4.3.9",
"fetch-ndjson": "^1.1.6",
"flexsearch": "^0.7.21",
"ipfs-core": "^0.10.8",
"keyword-mark-element": "^0.2.0",
"lit": "^2.1.1",
"marked": "^4.0.10",
@@ -34,6 +34,7 @@
"electron": "^19.0.4",
"electron-builder": "^23.0.3",
"electron-notarize": "^1.1.1",
"eslint": "^8.23.1",
"http-server": "^13.0.2",
"mini-css-extract-plugin": "^2.3.0",
"node-sass": "^7.0.1",
@@ -117,7 +118,7 @@
"extraResources": [
"plugins-mac"
],
"singleArchFiles": "dist/prebuilds/**/*"
"singleArchFiles": "{dist/prebuilds,dist/prebuilds/**}"
},
"linux": {
"category": "Archiving;Utility;",
src/electron-ipfs-handler.js
+44
View File
@@ -0,0 +1,44 @@
import { protocol, session } from "electron";
import fetchToHandler from "fetch-to-electron-protocol-handler";
import makeIPFSFetch from "js-ipfs-fetch";
import * as IPFS from "ipfs-core";
export const PROTOCOL_PERMISSIONS = {
standard: true,
secure: true,
allowServiceWorkers: true,
supportFetchAPI: true,
bypassCSP: false,
corsEnabled: true,
stream: true
};
export function registerPrivileges(privileges = PROTOCOL_PERMISSIONS) {
protocol.registerSchemesAsPrivileged([
{scheme: "ipfs", privileges},
{scheme: "ipns", privileges},
{scheme: "ipld", privileges}
]);
}
export function registerHandler(ipfsRepoPath) {
const handler = createHandler(ipfsRepoPath);
protocol.registerStreamProtocol("ipfs", handler);
protocol.registerStreamProtocol("ipns", handler);
protocol.registerStreamProtocol("ipld", handler);
}
export function createHandler (ipfsRepoPath) {
const {handler} = fetchToHandler(async () => {
const ipfs = await IPFS.create({
repo: ipfsRepoPath
});
const fetch = await makeIPFSFetch({ ipfs });
return fetch;
}, session.defaultSession);
return handler;
}
src/electron-replay-app.js
+13 -36
View File
@@ -9,7 +9,7 @@ import path from "path";
import fs from "fs";
import { ArchiveResponse, Rewriter } from "@webrecorder/wabac/src/rewrite";
import { IPFSClient } from "@webrecorder/wabac/src/ipfs";
import {create as createIPFS} from "auto-js-ipfs";
import { PassThrough, Readable } from "stream";
@@ -27,33 +27,6 @@ const REPLAY_PREFIX = STATIC_PREFIX + "w/";
const URL_RX = /([^/]+)\/([\d]+)(?:\w\w_)?\/(.*)$/;
let IPFS = null;
// ============================================================================
class NativeIPFSClient extends IPFSClient
{
constructor(repoPath) {
super();
this.repoPath = repoPath;
}
async _doInitIPFS() {
if (!IPFS) {
IPFS = require("ipfs-core");
}
this.ipfs = await IPFS.create({
repo: this.repoPath,
init: {emptyRepo: true},
//preload: {enabled: false},
});
this.resetGC();
}
}
// ============================================================================
class ElectronReplayApp
{
@@ -145,11 +118,6 @@ class ElectronReplayApp
app.setPath("userData", path.join(app.getPath("appData"), this.profileName));
}
const ipfsRepoPath = path.join(app.getPath("userData"), "js-ipfs");
console.log("ipfs path", ipfsRepoPath);
this.ipfsClient = new NativeIPFSClient(ipfsRepoPath);
app.on("will-finish-launching", () => {
app.on("open-file", (event, filePath) => {
this.openNextFile = filePath;
@@ -185,6 +153,9 @@ class ElectronReplayApp
}
onAppReady() {
const ipfsRepoPath = path.join(app.getPath("userData"), "js-ipfs");
console.log("ipfs path", ipfsRepoPath);
this.checkUpdates();
this.screenSize = screen.getPrimaryDisplay().workAreaSize;
@@ -293,13 +264,16 @@ class ElectronReplayApp
return;
} else if (ipfsCID) {
const ipfsURL = `ipfs://${ipfsCID}}`;
console.log("ipfs serve:", ipfsCID);
await this.ipfsClient.initIPFS();
// TODO: Pass in config?
this.ipfsClient = await createIPFS();
console.log("inited");
let size = await this.ipfsClient.getFileSize(ipfsCID);
let size = await this.ipfsClient.getSize(ipfsURL);
console.log("got size", size);
@@ -314,7 +288,10 @@ class ElectronReplayApp
if (request.method === "GET") {
const offset = start || 0;
const length = end ? end - start + 1 : size;
data = Readable.from(await this.ipfsClient.cat(ipfsCID, {offset, length}));
data = Readable.from(await this.ipfsClient.get(ipfsURL, {
start: offset,
end: offset +length
}));
}
callback({statusCode, headers, data});
src/loader.js
+1 -11
View File
@@ -1,5 +1,5 @@
import { LitElement, html, css } from "lit";
import { wrapCss, IS_APP } from "./misc";
import { wrapCss } from "./misc";
import prettyBytes from "pretty-bytes";
@@ -163,16 +163,6 @@ You can select a file to upload from the main page by clicking the 'Choose File.
source = this.loadInfo;
break;
case "ipfs":
if (IS_APP) {
// eslint-disable-next-line no-undef
const url = new URL(__APP_FILE_SERVE_PREFIX__);
const hash = sourceUrl.split("#", 1)[0];
url.searchParams.set("ipfs", hash.slice("ipfs://".length));
source = {sourceUrl, loadUrl: url.href};
}
break;
case "proxy":
sourceUrl = "proxy:" + sourceUrl.slice("proxy://".length);
break;
sw.js
+10 -10
View File
File diff suppressed because one or more lines are too long
ui.js
+4 -3
View File
@@ -865,13 +865,14 @@ class se extends oe{}se.directiveName="unsafeSVG",se.resultType=2;const le=ae(se
</div>
${n?P`
<hr class="dropdown-divider">
<a href="${n}" class="button" @keyup="${ge}">
<h2 mt-4">Get A Copy!</h2>
<p class="mt-2">After downloading, this web archive can be loaded and viewed directly in your browser via <a style="white-space: nowrap;" target="_blank" href="https://replayweb.page">replayweb.page</a>.</p>
<a href="${n}" class="button mt-4" @keyup="${ge}">
<span class="icon is-small">
<fa-icon size="1.0em" class="has-text-grey" aria-hidden="true" .svg="${tt()}"></fa-icon>
</span>
<span>Download Archive</span>
</a>
<p class="mt-2">This archive can then be viewed directly in your browser by loading it via <a target="_blank" href="https://replayweb.page">replayweb.page</a></p>
`:""}
<p class="is-size-7 is-flex is-justify-content-space-between" style="margin-top: 40px">
<span>
@@ -1562,7 +1563,7 @@ class se extends oe{}se.directiveName="unsafeSVG",se.resultType=2;const le=ae(se
<span class="icon"><fa-icon .svg="${Ui()}"></fa-icon></span>
<span>Authorize Google Drive</span>
</button>
`}`}script(){if("trypublic"===this.state||this.scriptLoaded)return P``;const e=document.createElement("script");return e.onload=()=>this.onLoad(),e.src="https://apis.google.com/js/platform.js",e}gauth(e,t){self.gapi.load("auth2",(()=>{self.gapi.auth2.authorize({client_id:"160798412227-tko4c82uopud11q105b2lvbogsj77hlg.apps.googleusercontent.com",scope:"https://www.googleapis.com/auth/drive.file",response_type:"token",prompt:e},t)}))}}customElements.define("wr-gdrive",Ri);class qi extends X{constructor(){super(),this.progress=0,this.total=0,this.percent=0,this.coll="",this.state="waiting",this.loadInfo=null,this.currentSize=0,this.totalSize=0,this.tryFileHandle=!!window.showOpenFilePicker,this.fileHandle=null,this.errorAllowRetry=!1,this.pingInterval=0,this.noWebWorker=!1}static get properties(){return{sourceUrl:{type:String},loadInfo:{type:Object},state:{type:String},progress:{type:Number},percent:{type:Number},currentSize:{type:Number},totalSize:{type:Number},error:{type:String},total:{type:Number},status:{type:String},coll:{type:String},embed:{type:String},tryFileHandle:{type:Boolean},errorAllowRetry:{type:Boolean},extraMsg:{type:String}}}firstUpdated(){this.initMessages()}initMessages(){if(this.noWebWorker=this.loadInfo&&this.loadInfo.noWebWorker,this.noWebWorker){if(!navigator.serviceWorker)return;this.worker=navigator.serviceWorker}else this.worker=new Worker("sw.js");this.worker.addEventListener("message",(e=>{switch(e.data.msg_type){case"collProgress":if(e.data.name===this.coll){if(this.percent=e.data.percent,e.data.error)if(this.error=e.data.error,this.state="errored",this.errorAllowRetry=!0,this.fileHandle=e.data.fileHandle,"missing_local_file"===this.error)this.tryFileHandle=!1;else if("permission_needed"===this.error&&e.data.fileHandle){this.state="permission_needed";break}e.data.currentSize&&e.data.totalSize&&(this.currentSize=e.data.currentSize,this.totalSize=e.data.totalSize),this.extraMsg=e.data.extraMsg}break;case"collAdded":e.data.name===this.coll&&(this.total||(this.total=100),this.progress=this.total,this.percent=100,this.dispatchEvent(new CustomEvent("coll-loaded",{detail:e.data})),this.noWebWorker?this.pingInterval&&clearInterval(this.pingInterval):this.worker.terminate(),this.worker=null)}}))}async doLoad(){let e,t,i=this.sourceUrl,a=null;if(this.percent=this.currentSize=this.totalSize=0,this.loadInfo&&this.loadInfo.swError)return this.state="errored",this.error=this.loadInfo.swError,void(this.errorAllowRetry=!1);try{const{scheme:e,host:t,path:r}=function(e){let t=e.indexOf("://"),i=0,a="",r="",o="";return t>=0?(a=e.slice(0,t),t+=3):(t++,e.startsWith("//")&&(t+=2)),i=e.indexOf("/",t),i>0?(r=e.slice(t,i),o=e.slice(i)):(r=e.slice(t),o=""),{scheme:a,host:r,path:o}}(i);switch(e){case"googledrive":this.state="googledrive",a=await this.googledriveInit();break;case"s3":a={sourceUrl:i,loadUrl:`https://${t}.s3.amazonaws.com${r}`,name:this.sourceUrl};break;case"file":if(!this.loadInfo&&!this.tryFileHandle)return this.state="errored",this.error="File URLs can not be entered directly or shared.\nYou can select a file to upload from the main page by clicking the 'Choose File...' button.",void(this.errorAllowRetry=!1);a=this.loadInfo;break;case"ipfs":if(pe){const e=new URL("http://files.replayweb.page/"),t=i.split("#",1)[0];e.searchParams.set("ipfs",t.slice("ipfs://".length)),a={sourceUrl:i,loadUrl:e.href}}break;case"proxy":i="proxy:"+i.slice("proxy://".length)}}catch(e){console.log(e)}a||(a={sourceUrl:i}),this.state="started",this.loadInfo&&(a.newFullImport=this.loadInfo.newFullImport,a.noCache=this.loadInfo.noCache,this.loadInfo.extraConfig&&(t=this.loadInfo.extraConfig),i.startsWith("proxy:")&&t&&t.recording&&(e="recordingproxy"));const r={msg_type:"addColl",name:this.coll,extraConfig:t,type:e,skipExisting:!0,file:a};navigator.serviceWorker.controller||await new Promise((e=>{navigator.serviceWorker.addEventListener("controllerchange",(()=>e()))})),this.worker&&(this.noWebWorker?(navigator.serviceWorker.controller.postMessage(r),this.pingInterval=setInterval((()=>{navigator.serviceWorker.controller.postMessage({msg_type:"ping"})}),15e3)):this.worker.postMessage(r))}googledriveInit(){return this._gdWait=new Promise((e=>this._gdResolve=e)),this._gdWait}async onLoadReady(e){this._gdResolve&&this._gdResolve(e.detail)}onCancel(){if(!this.worker)return;const e={msg_type:"cancelLoad",name:this.coll};this.noWebWorker?navigator.serviceWorker&&navigator.serviceWorker.controller&&(navigator.serviceWorker.controller.postMessage(e),this.pingInterval&&clearInterval(this.pingInterval)):this.worker.postMessage(e)}updated(e){(this.sourceUrl&&e.has("sourceUrl")||e.has("tryFileHandle"))&&this.doLoad()}static get styles(){return ue(o`
`}`}script(){if("trypublic"===this.state||this.scriptLoaded)return P``;const e=document.createElement("script");return e.onload=()=>this.onLoad(),e.src="https://apis.google.com/js/platform.js",e}gauth(e,t){self.gapi.load("auth2",(()=>{self.gapi.auth2.authorize({client_id:"160798412227-tko4c82uopud11q105b2lvbogsj77hlg.apps.googleusercontent.com",scope:"https://www.googleapis.com/auth/drive.file",response_type:"token",prompt:e},t)}))}}customElements.define("wr-gdrive",Ri);class qi extends X{constructor(){super(),this.progress=0,this.total=0,this.percent=0,this.coll="",this.state="waiting",this.loadInfo=null,this.currentSize=0,this.totalSize=0,this.tryFileHandle=!!window.showOpenFilePicker,this.fileHandle=null,this.errorAllowRetry=!1,this.pingInterval=0,this.noWebWorker=!1}static get properties(){return{sourceUrl:{type:String},loadInfo:{type:Object},state:{type:String},progress:{type:Number},percent:{type:Number},currentSize:{type:Number},totalSize:{type:Number},error:{type:String},total:{type:Number},status:{type:String},coll:{type:String},embed:{type:String},tryFileHandle:{type:Boolean},errorAllowRetry:{type:Boolean},extraMsg:{type:String}}}firstUpdated(){this.initMessages()}initMessages(){if(this.noWebWorker=this.loadInfo&&this.loadInfo.noWebWorker,this.noWebWorker){if(!navigator.serviceWorker)return;this.worker=navigator.serviceWorker}else this.worker=new Worker("sw.js");this.worker.addEventListener("message",(e=>{switch(e.data.msg_type){case"collProgress":if(e.data.name===this.coll){if(this.percent=e.data.percent,e.data.error)if(this.error=e.data.error,this.state="errored",this.errorAllowRetry=!0,this.fileHandle=e.data.fileHandle,"missing_local_file"===this.error)this.tryFileHandle=!1;else if("permission_needed"===this.error&&e.data.fileHandle){this.state="permission_needed";break}e.data.currentSize&&e.data.totalSize&&(this.currentSize=e.data.currentSize,this.totalSize=e.data.totalSize),this.extraMsg=e.data.extraMsg}break;case"collAdded":e.data.name===this.coll&&(this.total||(this.total=100),this.progress=this.total,this.percent=100,this.dispatchEvent(new CustomEvent("coll-loaded",{detail:e.data})),this.noWebWorker?this.pingInterval&&clearInterval(this.pingInterval):this.worker.terminate(),this.worker=null)}}))}async doLoad(){let e,t,i=this.sourceUrl,a=null;if(this.percent=this.currentSize=this.totalSize=0,this.loadInfo&&this.loadInfo.swError)return this.state="errored",this.error=this.loadInfo.swError,void(this.errorAllowRetry=!1);try{const{scheme:e,host:t,path:r}=function(e){let t=e.indexOf("://"),i=0,a="",r="",o="";return t>=0?(a=e.slice(0,t),t+=3):(t++,e.startsWith("//")&&(t+=2)),i=e.indexOf("/",t),i>0?(r=e.slice(t,i),o=e.slice(i)):(r=e.slice(t),o=""),{scheme:a,host:r,path:o}}(i);switch(e){case"googledrive":this.state="googledrive",a=await this.googledriveInit();break;case"s3":a={sourceUrl:i,loadUrl:`https://${t}.s3.amazonaws.com${r}`,name:this.sourceUrl};break;case"file":if(!this.loadInfo&&!this.tryFileHandle)return this.state="errored",this.error="File URLs can not be entered directly or shared.\nYou can select a file to upload from the main page by clicking the 'Choose File...' button.",void(this.errorAllowRetry=!1);a=this.loadInfo;break;case"proxy":i="proxy:"+i.slice("proxy://".length)}}catch(e){console.log(e)}a||(a={sourceUrl:i}),this.state="started",this.loadInfo&&(a.newFullImport=this.loadInfo.newFullImport,a.noCache=this.loadInfo.noCache,this.loadInfo.extraConfig&&(t=this.loadInfo.extraConfig),i.startsWith("proxy:")&&t&&t.recording&&(e="recordingproxy"));const r={msg_type:"addColl",name:this.coll,extraConfig:t,type:e,skipExisting:!0,file:a};navigator.serviceWorker.controller||await new Promise((e=>{navigator.serviceWorker.addEventListener("controllerchange",(()=>e()))})),this.worker&&(this.noWebWorker?(navigator.serviceWorker.controller.postMessage(r),this.pingInterval=setInterval((()=>{navigator.serviceWorker.controller.postMessage({msg_type:"ping"})}),15e3)):this.worker.postMessage(r))}googledriveInit(){return this._gdWait=new Promise((e=>this._gdResolve=e)),this._gdWait}async onLoadReady(e){this._gdResolve&&this._gdResolve(e.detail)}onCancel(){if(!this.worker)return;const e={msg_type:"cancelLoad",name:this.coll};this.noWebWorker?navigator.serviceWorker&&navigator.serviceWorker.controller&&(navigator.serviceWorker.controller.postMessage(e),this.pingInterval&&clearInterval(this.pingInterval)):this.worker.postMessage(e)}updated(e){(this.sourceUrl&&e.has("sourceUrl")||e.has("tryFileHandle"))&&this.doLoad()}static get styles(){return ue(o`
:host {
height: 100%;
display: flex;
webpack.config.js
+5 -9
View File
@@ -22,10 +22,6 @@ const GDRIVE_CLIENT_ID = "160798412227-tko4c82uopud11q105b2lvbogsj77hlg.apps.goo
// Copyright banner text
const BANNER_TEXT = "'[name].js is part of ReplayWeb.page (https://replayweb.page) Copyright (C) 2020-2021, Webrecorder Software. Licensed under the Affero General Public License v3.'";
// IPFS ipfs-core lib url
const IPFS_CORE_URL = `https://cdn.jsdelivr.net/npm/ipfs-core@${package_json.dependencies["ipfs-core"].replace("^", "")}/dist/index.min.js`;
const fallback = {
"stream": require.resolve("stream-browserify"),
"querystring": require.resolve("querystring-es3"),
@@ -77,14 +73,16 @@ const electronMainConfig = (/*env, argv*/) => {
new webpack.BannerPlugin(BANNER_TEXT),
new CopyPlugin({
patterns: [
{ from: "node_modules/leveldown/prebuilds/", to: "prebuilds" },
{ from: "build/extra_prebuilds/", to: "prebuilds" },
// { from: "node_modules/classic-level/prebuilds/", to: "prebuilds" },
{ from: "build/extra_prebuilds/", to: "prebuilds" }
],
}),
//new webpack.NormalModuleReplacementPlugin(/\.\/http\/fetch/, "./http/fetch.node"),
//new webpack.NormalModuleReplacementPlugin(/\.\.\/fetch$/, "electron-fetch"),
],
externals: {
"bufferutil": "bufferutil",
"utf-8-validate": "utf-8-validate",
"utf-8-validate": "utf-8-validate"
}
};
};
@@ -158,12 +156,10 @@ const browserConfig = (/*env, argv*/) => {
__HELPER_PROXY__ : JSON.stringify(HELPER_PROXY),
__GDRIVE_CLIENT_ID__ : JSON.stringify(GDRIVE_CLIENT_ID),
__VERSION__: JSON.stringify(package_json.version),
__IPFS_CORE_URL__: JSON.stringify(IPFS_CORE_URL)
}),
new webpack.BannerPlugin(BANNER_TEXT),
new CopyPlugin({
patterns: [
{ from: "node_modules/ipfs-core/dist/index.min.js", to: "ipfs-core.min.js" },
{ from: "package.json", to: "_data/package.json" }
]
}),
yarn.lock
+476 -2726
View File
File diff suppressed because it is too large Load Diff