remove js-ipfs in electron app, eslint to dev deps (#123)

- add ipfs:// protocol handlers, not enabled for now
- use browser-side auto-js-ipfs for loading from ipfs even in electron app
- deps: bump wabac.js to 2.13.0
- update CHANGES

Co-authored-by: Ilya Kreymer <ikreymer@gmail.com>

CHANGES.md

@@ -11,6 +11,7 @@ v1.7.0
 - Fidelity/Embeds: Redirect new windows from links / `window.open` to existing replay frame (only possible w/o 'sandbox').
 - Embeds: Add 'requireSubdomainIframe' embed attribute to only allow embed if loaded from a cross-origin / subdomain iframe.
 - UI: improved formatting on 404 not found page.
+- Loading: Fix IPFS loading, use auto-js-ipfs to load from gateway or existing local instance (don't run local node for now)
 - Dependencies: update to wabac.js 2.13.0 to support verification, latest wombat (3.3.9)
 
 v1.6.5

_data/package.json

@@ -9,13 +9,13 @@
   "license": "AGPL-3.0-or-later",
   "dependencies": {
     "@fortawesome/fontawesome-free": "^5.15.4",
-    "@webrecorder/wabac": "github:webrecorder/wabac.js",
+    "@webrecorder/wabac": "^2.13.0",
+    "auto-js-ipfs": "^1.3.9",
     "bulma": "^0.9.3",
     "electron-log": "^4.4.1",
     "electron-updater": "^4.3.9",
     "fetch-ndjson": "^1.1.6",
     "flexsearch": "^0.7.21",
-    "ipfs-core": "^0.10.8",
     "keyword-mark-element": "^0.2.0",
     "lit": "^2.1.1",
     "marked": "^4.0.10",
@@ -34,6 +34,7 @@
     "electron": "^19.0.4",
     "electron-builder": "^23.0.3",
     "electron-notarize": "^1.1.1",
+    "eslint": "^8.23.1",
     "http-server": "^13.0.2",
     "mini-css-extract-plugin": "^2.3.0",
     "node-sass": "^7.0.1",
@@ -117,7 +118,7 @@
       "extraResources": [
         "plugins-mac"
       ],
-      "singleArchFiles": "dist/prebuilds/**/*"
+      "singleArchFiles": "{dist/prebuilds,dist/prebuilds/**}"
     },
     "linux": {
       "category": "Archiving;Utility;",

package.json

@@ -9,13 +9,13 @@
   "license": "AGPL-3.0-or-later",
   "dependencies": {
     "@fortawesome/fontawesome-free": "^5.15.4",
-    "@webrecorder/wabac": "github:webrecorder/wabac.js",
+    "@webrecorder/wabac": "^2.13.0",
+    "auto-js-ipfs": "^1.3.9",
     "bulma": "^0.9.3",
     "electron-log": "^4.4.1",
     "electron-updater": "^4.3.9",
     "fetch-ndjson": "^1.1.6",
     "flexsearch": "^0.7.21",
-    "ipfs-core": "^0.10.8",
     "keyword-mark-element": "^0.2.0",
     "lit": "^2.1.1",
     "marked": "^4.0.10",
@@ -34,6 +34,7 @@
     "electron": "^19.0.4",
     "electron-builder": "^23.0.3",
     "electron-notarize": "^1.1.1",
+    "eslint": "^8.23.1",
     "http-server": "^13.0.2",
     "mini-css-extract-plugin": "^2.3.0",
     "node-sass": "^7.0.1",
@@ -117,7 +118,7 @@
       "extraResources": [
         "plugins-mac"
       ],
-      "singleArchFiles": "dist/prebuilds/**/*"
+      "singleArchFiles": "{dist/prebuilds,dist/prebuilds/**}"
     },
     "linux": {
       "category": "Archiving;Utility;",

src/electron-ipfs-handler.js

@@ -0,0 +1,44 @@
+import { protocol, session } from "electron";
+
+import fetchToHandler from "fetch-to-electron-protocol-handler";
+import makeIPFSFetch from "js-ipfs-fetch";
+import * as IPFS from "ipfs-core";
+
+export const PROTOCOL_PERMISSIONS = {
+  standard: true,
+  secure: true,
+  allowServiceWorkers: true,
+  supportFetchAPI: true,
+  bypassCSP: false,
+  corsEnabled: true,
+  stream: true
+};
+
+export function registerPrivileges(privileges = PROTOCOL_PERMISSIONS) {
+  protocol.registerSchemesAsPrivileged([
+    {scheme: "ipfs", privileges},
+    {scheme: "ipns", privileges},
+    {scheme: "ipld", privileges}
+  ]);
+}
+
+export function registerHandler(ipfsRepoPath) {
+  const handler = createHandler(ipfsRepoPath);
+  protocol.registerStreamProtocol("ipfs", handler);
+  protocol.registerStreamProtocol("ipns", handler);
+  protocol.registerStreamProtocol("ipld", handler);
+}
+
+export function createHandler (ipfsRepoPath) {
+  const {handler} = fetchToHandler(async () => {
+    const ipfs = await IPFS.create({
+      repo: ipfsRepoPath
+    });
+
+    const fetch = await makeIPFSFetch({ ipfs });
+
+    return fetch;
+  }, session.defaultSession);
+
+  return handler;
+}

src/electron-replay-app.js

@@ -9,7 +9,7 @@ import path from "path";
 import fs from "fs";
 
 import { ArchiveResponse, Rewriter } from "@webrecorder/wabac/src/rewrite";
-import { IPFSClient } from "@webrecorder/wabac/src/ipfs";
+import {create as createIPFS} from "auto-js-ipfs";
 
 import { PassThrough, Readable } from "stream";
 
@@ -27,33 +27,6 @@ const REPLAY_PREFIX = STATIC_PREFIX + "w/";
 
 const URL_RX = /([^/]+)\/([\d]+)(?:\w\w_)?\/(.*)$/;
 
-
-let IPFS = null;
-
-// ============================================================================
-class NativeIPFSClient extends IPFSClient
-{
-  constructor(repoPath) {
-    super();
-    this.repoPath = repoPath;
-  }
-
-  async _doInitIPFS() {
-    if (!IPFS) {
-      IPFS = require("ipfs-core");
-    }
-
-    this.ipfs = await IPFS.create({
-      repo: this.repoPath,
-      init: {emptyRepo: true},
-      //preload: {enabled: false},
-    });
-
-    this.resetGC();
-  }
-}
-
-
 // ============================================================================
 class ElectronReplayApp
 {
@@ -145,11 +118,6 @@ class ElectronReplayApp
       app.setPath("userData", path.join(app.getPath("appData"), this.profileName));
     }
 
-    const ipfsRepoPath = path.join(app.getPath("userData"), "js-ipfs");
-    console.log("ipfs path", ipfsRepoPath);
-
-    this.ipfsClient = new NativeIPFSClient(ipfsRepoPath);
-
     app.on("will-finish-launching", () => {
       app.on("open-file", (event, filePath) => {
         this.openNextFile = filePath;
@@ -185,6 +153,9 @@ class ElectronReplayApp
   }
 
   onAppReady() {
+    const ipfsRepoPath = path.join(app.getPath("userData"), "js-ipfs");
+    console.log("ipfs path", ipfsRepoPath);
+
     this.checkUpdates();
 
     this.screenSize = screen.getPrimaryDisplay().workAreaSize;
@@ -293,13 +264,16 @@ class ElectronReplayApp
         return;
 
       } else if (ipfsCID) {
+        const ipfsURL = `ipfs://${ipfsCID}}`;
+
         console.log("ipfs serve:", ipfsCID);
 
-        await this.ipfsClient.initIPFS();
+        // TODO: Pass in config?
+        this.ipfsClient = await createIPFS();
 
         console.log("inited");
 
-        let size = await this.ipfsClient.getFileSize(ipfsCID);
+        let size = await this.ipfsClient.getSize(ipfsURL);
 
         console.log("got size", size);
 
@@ -314,7 +288,10 @@ class ElectronReplayApp
         if (request.method === "GET") {
           const offset = start || 0;
           const length = end ? end - start + 1 : size;
-          data = Readable.from(await this.ipfsClient.cat(ipfsCID, {offset, length}));
+          data = Readable.from(await this.ipfsClient.get(ipfsURL, {
+            start: offset,
+            end: offset +length
+          }));
         }
 
         callback({statusCode, headers, data});

src/loader.js

@@ -1,5 +1,5 @@
 import { LitElement, html, css } from "lit";
-import { wrapCss, IS_APP } from "./misc";
+import { wrapCss } from "./misc";
 
 import prettyBytes from "pretty-bytes";
 
@@ -163,16 +163,6 @@ You can select a file to upload from the main page by clicking the 'Choose File.
         source = this.loadInfo;
         break;
 
-      case "ipfs":
-        if (IS_APP) {
-          // eslint-disable-next-line no-undef
-          const url = new URL(__APP_FILE_SERVE_PREFIX__);
-          const hash = sourceUrl.split("#", 1)[0];
-          url.searchParams.set("ipfs", hash.slice("ipfs://".length));
-          source = {sourceUrl, loadUrl: url.href};
-        }
-        break;
-
       case "proxy":
         sourceUrl = "proxy:" + sourceUrl.slice("proxy://".length);
         break;

ui.js

@@ -865,13 +865,14 @@ class se extends oe{}se.directiveName="unsafeSVG",se.resultType=2;const le=ae(se
           </div>
           ${n?P`
           <hr class="dropdown-divider">
-          <a href="${n}" class="button" @keyup="${ge}">
+          <h2 mt-4">Get A Copy!</h2>
+          <p class="mt-2">After downloading, this web archive can be loaded and viewed directly in your browser via <a style="white-space: nowrap;" target="_blank" href="https://replayweb.page">replayweb.page</a>.</p>
+          <a href="${n}" class="button mt-4" @keyup="${ge}">
             <span class="icon is-small">
               <fa-icon size="1.0em" class="has-text-grey" aria-hidden="true" .svg="${tt()}"></fa-icon>
             </span>
             <span>Download Archive</span>
           </a>
-          <p class="mt-2">This archive can then be viewed directly in your browser by loading it via <a target="_blank" href="https://replayweb.page">replayweb.page</a></p>
           `:""}
           <p class="is-size-7 is-flex is-justify-content-space-between" style="margin-top: 40px">
             <span>
@@ -1562,7 +1563,7 @@ class se extends oe{}se.directiveName="unsafeSVG",se.resultType=2;const le=ae(se
     <span class="icon"><fa-icon .svg="${Ui()}"></fa-icon></span>
     <span>Authorize Google Drive</span>
     </button>
-    `}`}script(){if("trypublic"===this.state||this.scriptLoaded)return P``;const e=document.createElement("script");return e.onload=()=>this.onLoad(),e.src="https://apis.google.com/js/platform.js",e}gauth(e,t){self.gapi.load("auth2",(()=>{self.gapi.auth2.authorize({client_id:"160798412227-tko4c82uopud11q105b2lvbogsj77hlg.apps.googleusercontent.com",scope:"https://www.googleapis.com/auth/drive.file",response_type:"token",prompt:e},t)}))}}customElements.define("wr-gdrive",Ri);class qi extends X{constructor(){super(),this.progress=0,this.total=0,this.percent=0,this.coll="",this.state="waiting",this.loadInfo=null,this.currentSize=0,this.totalSize=0,this.tryFileHandle=!!window.showOpenFilePicker,this.fileHandle=null,this.errorAllowRetry=!1,this.pingInterval=0,this.noWebWorker=!1}static get properties(){return{sourceUrl:{type:String},loadInfo:{type:Object},state:{type:String},progress:{type:Number},percent:{type:Number},currentSize:{type:Number},totalSize:{type:Number},error:{type:String},total:{type:Number},status:{type:String},coll:{type:String},embed:{type:String},tryFileHandle:{type:Boolean},errorAllowRetry:{type:Boolean},extraMsg:{type:String}}}firstUpdated(){this.initMessages()}initMessages(){if(this.noWebWorker=this.loadInfo&&this.loadInfo.noWebWorker,this.noWebWorker){if(!navigator.serviceWorker)return;this.worker=navigator.serviceWorker}else this.worker=new Worker("sw.js");this.worker.addEventListener("message",(e=>{switch(e.data.msg_type){case"collProgress":if(e.data.name===this.coll){if(this.percent=e.data.percent,e.data.error)if(this.error=e.data.error,this.state="errored",this.errorAllowRetry=!0,this.fileHandle=e.data.fileHandle,"missing_local_file"===this.error)this.tryFileHandle=!1;else if("permission_needed"===this.error&&e.data.fileHandle){this.state="permission_needed";break}e.data.currentSize&&e.data.totalSize&&(this.currentSize=e.data.currentSize,this.totalSize=e.data.totalSize),this.extraMsg=e.data.extraMsg}break;case"collAdded":e.data.name===this.coll&&(this.total||(this.total=100),this.progress=this.total,this.percent=100,this.dispatchEvent(new CustomEvent("coll-loaded",{detail:e.data})),this.noWebWorker?this.pingInterval&&clearInterval(this.pingInterval):this.worker.terminate(),this.worker=null)}}))}async doLoad(){let e,t,i=this.sourceUrl,a=null;if(this.percent=this.currentSize=this.totalSize=0,this.loadInfo&&this.loadInfo.swError)return this.state="errored",this.error=this.loadInfo.swError,void(this.errorAllowRetry=!1);try{const{scheme:e,host:t,path:r}=function(e){let t=e.indexOf("://"),i=0,a="",r="",o="";return t>=0?(a=e.slice(0,t),t+=3):(t++,e.startsWith("//")&&(t+=2)),i=e.indexOf("/",t),i>0?(r=e.slice(t,i),o=e.slice(i)):(r=e.slice(t),o=""),{scheme:a,host:r,path:o}}(i);switch(e){case"googledrive":this.state="googledrive",a=await this.googledriveInit();break;case"s3":a={sourceUrl:i,loadUrl:`https://${t}.s3.amazonaws.com${r}`,name:this.sourceUrl};break;case"file":if(!this.loadInfo&&!this.tryFileHandle)return this.state="errored",this.error="File URLs can not be entered directly or shared.\nYou can select a file to upload from the main page by clicking the 'Choose File...' button.",void(this.errorAllowRetry=!1);a=this.loadInfo;break;case"ipfs":if(pe){const e=new URL("http://files.replayweb.page/"),t=i.split("#",1)[0];e.searchParams.set("ipfs",t.slice("ipfs://".length)),a={sourceUrl:i,loadUrl:e.href}}break;case"proxy":i="proxy:"+i.slice("proxy://".length)}}catch(e){console.log(e)}a||(a={sourceUrl:i}),this.state="started",this.loadInfo&&(a.newFullImport=this.loadInfo.newFullImport,a.noCache=this.loadInfo.noCache,this.loadInfo.extraConfig&&(t=this.loadInfo.extraConfig),i.startsWith("proxy:")&&t&&t.recording&&(e="recordingproxy"));const r={msg_type:"addColl",name:this.coll,extraConfig:t,type:e,skipExisting:!0,file:a};navigator.serviceWorker.controller||await new Promise((e=>{navigator.serviceWorker.addEventListener("controllerchange",(()=>e()))})),this.worker&&(this.noWebWorker?(navigator.serviceWorker.controller.postMessage(r),this.pingInterval=setInterval((()=>{navigator.serviceWorker.controller.postMessage({msg_type:"ping"})}),15e3)):this.worker.postMessage(r))}googledriveInit(){return this._gdWait=new Promise((e=>this._gdResolve=e)),this._gdWait}async onLoadReady(e){this._gdResolve&&this._gdResolve(e.detail)}onCancel(){if(!this.worker)return;const e={msg_type:"cancelLoad",name:this.coll};this.noWebWorker?navigator.serviceWorker&&navigator.serviceWorker.controller&&(navigator.serviceWorker.controller.postMessage(e),this.pingInterval&&clearInterval(this.pingInterval)):this.worker.postMessage(e)}updated(e){(this.sourceUrl&&e.has("sourceUrl")||e.has("tryFileHandle"))&&this.doLoad()}static get styles(){return ue(o`
+    `}`}script(){if("trypublic"===this.state||this.scriptLoaded)return P``;const e=document.createElement("script");return e.onload=()=>this.onLoad(),e.src="https://apis.google.com/js/platform.js",e}gauth(e,t){self.gapi.load("auth2",(()=>{self.gapi.auth2.authorize({client_id:"160798412227-tko4c82uopud11q105b2lvbogsj77hlg.apps.googleusercontent.com",scope:"https://www.googleapis.com/auth/drive.file",response_type:"token",prompt:e},t)}))}}customElements.define("wr-gdrive",Ri);class qi extends X{constructor(){super(),this.progress=0,this.total=0,this.percent=0,this.coll="",this.state="waiting",this.loadInfo=null,this.currentSize=0,this.totalSize=0,this.tryFileHandle=!!window.showOpenFilePicker,this.fileHandle=null,this.errorAllowRetry=!1,this.pingInterval=0,this.noWebWorker=!1}static get properties(){return{sourceUrl:{type:String},loadInfo:{type:Object},state:{type:String},progress:{type:Number},percent:{type:Number},currentSize:{type:Number},totalSize:{type:Number},error:{type:String},total:{type:Number},status:{type:String},coll:{type:String},embed:{type:String},tryFileHandle:{type:Boolean},errorAllowRetry:{type:Boolean},extraMsg:{type:String}}}firstUpdated(){this.initMessages()}initMessages(){if(this.noWebWorker=this.loadInfo&&this.loadInfo.noWebWorker,this.noWebWorker){if(!navigator.serviceWorker)return;this.worker=navigator.serviceWorker}else this.worker=new Worker("sw.js");this.worker.addEventListener("message",(e=>{switch(e.data.msg_type){case"collProgress":if(e.data.name===this.coll){if(this.percent=e.data.percent,e.data.error)if(this.error=e.data.error,this.state="errored",this.errorAllowRetry=!0,this.fileHandle=e.data.fileHandle,"missing_local_file"===this.error)this.tryFileHandle=!1;else if("permission_needed"===this.error&&e.data.fileHandle){this.state="permission_needed";break}e.data.currentSize&&e.data.totalSize&&(this.currentSize=e.data.currentSize,this.totalSize=e.data.totalSize),this.extraMsg=e.data.extraMsg}break;case"collAdded":e.data.name===this.coll&&(this.total||(this.total=100),this.progress=this.total,this.percent=100,this.dispatchEvent(new CustomEvent("coll-loaded",{detail:e.data})),this.noWebWorker?this.pingInterval&&clearInterval(this.pingInterval):this.worker.terminate(),this.worker=null)}}))}async doLoad(){let e,t,i=this.sourceUrl,a=null;if(this.percent=this.currentSize=this.totalSize=0,this.loadInfo&&this.loadInfo.swError)return this.state="errored",this.error=this.loadInfo.swError,void(this.errorAllowRetry=!1);try{const{scheme:e,host:t,path:r}=function(e){let t=e.indexOf("://"),i=0,a="",r="",o="";return t>=0?(a=e.slice(0,t),t+=3):(t++,e.startsWith("//")&&(t+=2)),i=e.indexOf("/",t),i>0?(r=e.slice(t,i),o=e.slice(i)):(r=e.slice(t),o=""),{scheme:a,host:r,path:o}}(i);switch(e){case"googledrive":this.state="googledrive",a=await this.googledriveInit();break;case"s3":a={sourceUrl:i,loadUrl:`https://${t}.s3.amazonaws.com${r}`,name:this.sourceUrl};break;case"file":if(!this.loadInfo&&!this.tryFileHandle)return this.state="errored",this.error="File URLs can not be entered directly or shared.\nYou can select a file to upload from the main page by clicking the 'Choose File...' button.",void(this.errorAllowRetry=!1);a=this.loadInfo;break;case"proxy":i="proxy:"+i.slice("proxy://".length)}}catch(e){console.log(e)}a||(a={sourceUrl:i}),this.state="started",this.loadInfo&&(a.newFullImport=this.loadInfo.newFullImport,a.noCache=this.loadInfo.noCache,this.loadInfo.extraConfig&&(t=this.loadInfo.extraConfig),i.startsWith("proxy:")&&t&&t.recording&&(e="recordingproxy"));const r={msg_type:"addColl",name:this.coll,extraConfig:t,type:e,skipExisting:!0,file:a};navigator.serviceWorker.controller||await new Promise((e=>{navigator.serviceWorker.addEventListener("controllerchange",(()=>e()))})),this.worker&&(this.noWebWorker?(navigator.serviceWorker.controller.postMessage(r),this.pingInterval=setInterval((()=>{navigator.serviceWorker.controller.postMessage({msg_type:"ping"})}),15e3)):this.worker.postMessage(r))}googledriveInit(){return this._gdWait=new Promise((e=>this._gdResolve=e)),this._gdWait}async onLoadReady(e){this._gdResolve&&this._gdResolve(e.detail)}onCancel(){if(!this.worker)return;const e={msg_type:"cancelLoad",name:this.coll};this.noWebWorker?navigator.serviceWorker&&navigator.serviceWorker.controller&&(navigator.serviceWorker.controller.postMessage(e),this.pingInterval&&clearInterval(this.pingInterval)):this.worker.postMessage(e)}updated(e){(this.sourceUrl&&e.has("sourceUrl")||e.has("tryFileHandle"))&&this.doLoad()}static get styles(){return ue(o`
       :host {
         height: 100%;
         display: flex;

webpack.config.js

@@ -22,10 +22,6 @@ const GDRIVE_CLIENT_ID = "160798412227-tko4c82uopud11q105b2lvbogsj77hlg.apps.goo
 // Copyright banner text
 const BANNER_TEXT = "'[name].js is part of ReplayWeb.page (https://replayweb.page) Copyright (C) 2020-2021, Webrecorder Software. Licensed under the Affero General Public License v3.'";
 
-// IPFS ipfs-core lib url
-const IPFS_CORE_URL = `https://cdn.jsdelivr.net/npm/ipfs-core@${package_json.dependencies["ipfs-core"].replace("^", "")}/dist/index.min.js`;
-
-
 const fallback = {
   "stream": require.resolve("stream-browserify"),
   "querystring": require.resolve("querystring-es3"),
@@ -77,14 +73,16 @@ const electronMainConfig = (/*env, argv*/) => {
       new webpack.BannerPlugin(BANNER_TEXT),
       new CopyPlugin({
         patterns: [
-          { from: "node_modules/leveldown/prebuilds/", to: "prebuilds" },
+          // { from: "node_modules/classic-level/prebuilds/", to: "prebuilds" },
-          { from: "build/extra_prebuilds/", to: "prebuilds" },
+          { from: "build/extra_prebuilds/", to: "prebuilds" }
         ],
       }),
+      //new webpack.NormalModuleReplacementPlugin(/\.\/http\/fetch/, "./http/fetch.node"),
+      //new webpack.NormalModuleReplacementPlugin(/\.\.\/fetch$/, "electron-fetch"),
     ],
     externals: {
       "bufferutil": "bufferutil",
-      "utf-8-validate": "utf-8-validate",
+      "utf-8-validate": "utf-8-validate"
     }
   };
 };
@@ -158,12 +156,10 @@ const browserConfig = (/*env, argv*/) => {
         __HELPER_PROXY__ : JSON.stringify(HELPER_PROXY),
         __GDRIVE_CLIENT_ID__ : JSON.stringify(GDRIVE_CLIENT_ID),
         __VERSION__: JSON.stringify(package_json.version),
-        __IPFS_CORE_URL__: JSON.stringify(IPFS_CORE_URL)
       }),
       new webpack.BannerPlugin(BANNER_TEXT),
       new CopyPlugin({
         patterns: [
-          { from: "node_modules/ipfs-core/dist/index.min.js", to: "ipfs-core.min.js" },
           { from: "package.json", to: "_data/package.json" }
         ]
       }),