archivalo-oss/replayweb.page
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

* large WARC loading optimizations (attempted to fix #67)

Browse Source 
- use dedicated web worker for loading, remove when done
- update to warcio 1.5.0 for faster loading of large WARCs
- show WARC records loading for better progress update

* fidelity: fix potentially invalid location redirect via wombat 3.3.5 (see: webrecorder/pywb#684)

* dependencies: wabac.js 2.10.0-beta.0 (includes warcio 1.5.0, wombat 3.3.5)

* bump to 1.5.6, update CHANGES
This commit is contained in:
Ilya Kreymer
2021-11-27 17:06:37 -08:00
parent dfc059042d
commit 178d0507eb
9 changed files with 99 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ui.js
+11 -3
View File
@@ -48,7 +48,7 @@ class ie extends te{constructor(e){if(super(e),this.it=I,e.type!==X)throw Error(
* Copyright 2017 Google LLC
* SPDX-License-Identifier: BSD-3-Clause
*/
class re extends ie{}re.directiveName="unsafeSVG",re.resultType=2;const oe=ee(re),ne=ee(class extends te{constructor(e){var t;if(super(e),e.type!==Y||"style"!==e.name||(null===(t=e.strings)||void 0===t?void 0:t.length)>2)throw Error("The `styleMap` directive must be used in the `style` attribute and must be the only part in the attribute.")}render(e){return Object.keys(e).reduce(((t,i)=>{const a=e[i];return null==a?t:t+`${i=i.replace(/(?:^(webkit|moz|ms|o)|)(?=[A-Z])/g,"-$&").toLowerCase()}:${a};`}),"")}update(e,[t]){const{style:i}=e.element;if(void 0===this.ut){this.ut=new Set;for(const e in t)this.ut.add(e);return this.render(t)}this.ut.forEach((e=>{null==t[e]&&(this.ut.delete(e),e.includes("-")?i.removeProperty(e):i[e]="")}));for(const e in t){const a=t[e];null!=a&&(this.ut.add(e),e.includes("-")?i.setProperty(e,a):i[e]=a)}return T}});var se=__webpack_require__(6274),le=__webpack_require__(3029),ce=__webpack_require__.n(le);const de=r(se.Z);function ue(e){return[de,e]}const he=window.IS_APP||window.electron&&window.electron.IS_APP||window.matchMedia("(display-mode: standalone)").matches,fe="1.5.5";function be(e){" "==e.key&&(e.preventDefault(),e.target.click())}class pe extends J{constructor(){super(),this.size="1.1em",this.width=null,this.height=null}static get properties(){return{svg:{type:String},size:{type:String},width:{type:String},height:{type:String}}}static get styles(){return o`
class re extends ie{}re.directiveName="unsafeSVG",re.resultType=2;const oe=ee(re),ne=ee(class extends te{constructor(e){var t;if(super(e),e.type!==Y||"style"!==e.name||(null===(t=e.strings)||void 0===t?void 0:t.length)>2)throw Error("The `styleMap` directive must be used in the `style` attribute and must be the only part in the attribute.")}render(e){return Object.keys(e).reduce(((t,i)=>{const a=e[i];return null==a?t:t+`${i=i.replace(/(?:^(webkit|moz|ms|o)|)(?=[A-Z])/g,"-$&").toLowerCase()}:${a};`}),"")}update(e,[t]){const{style:i}=e.element;if(void 0===this.ut){this.ut=new Set;for(const e in t)this.ut.add(e);return this.render(t)}this.ut.forEach((e=>{null==t[e]&&(this.ut.delete(e),e.includes("-")?i.removeProperty(e):i[e]="")}));for(const e in t){const a=t[e];null!=a&&(this.ut.add(e),e.includes("-")?i.setProperty(e,a):i[e]=a)}return T}});var se=__webpack_require__(6274),le=__webpack_require__(3029),ce=__webpack_require__.n(le);const de=r(se.Z);function ue(e){return[de,e]}const he=window.IS_APP||window.electron&&window.electron.IS_APP||window.matchMedia("(display-mode: standalone)").matches,fe="1.5.6";function be(e){" "==e.key&&(e.preventDefault(),e.target.click())}class pe extends J{constructor(){super(),this.size="1.1em",this.width=null,this.height=null}static get properties(){return{svg:{type:String},size:{type:String},width:{type:String},height:{type:String}}}static get styles(){return o`
:host {
display: inline-block;
padding: 0;
@@ -1312,7 +1312,7 @@ class re extends ie{}re.directiveName="unsafeSVG",re.resultType=2;const oe=ee(re
<span class="icon"><fa-icon .svg="${Mt()}"></fa-icon></span>
<span>Authorize Google Drive</span>
</button>
`}`}script(){if("trypublic"===this.state||this.scriptLoaded)return L``;const e=document.createElement("script");return e.onload=()=>this.onLoad(),e.src="https://apis.google.com/js/platform.js",e}gauth(e,t){self.gapi.load("auth2",(()=>{self.gapi.auth2.authorize({client_id:"160798412227-tko4c82uopud11q105b2lvbogsj77hlg.apps.googleusercontent.com",scope:"https://www.googleapis.com/auth/drive.file",response_type:"token",prompt:e},t)}))}}customElements.define("wr-gdrive",jt);class Ht extends J{constructor(){super(),this.progress=0,this.total=0,this.percent=0,this.coll="",this.state="waiting",this.loadInfo=null,this.currentSize=0,this.totalSize=0,this.tryFileHandle=!!window.showOpenFilePicker,this.fileHandle=null,this.errorAllowRetry=!1,this.pingInterval=""}static get properties(){return{sourceUrl:{type:String},loadInfo:{type:Object},state:{type:String},progress:{type:Number},percent:{type:Number},currentSize:{type:Number},totalSize:{type:Number},error:{type:String},total:{type:Number},status:{type:String},coll:{type:String},embed:{type:String},tryFileHandle:{type:Boolean},errorAllowRetry:{type:Boolean}}}firstUpdated(){this.initMessages()}initMessages(){navigator.serviceWorker&&navigator.serviceWorker.addEventListener("message",(e=>{switch(e.data.msg_type){case"collProgress":if(e.data.name===this.coll){if(this.percent=e.data.percent,e.data.error)if(this.error=e.data.error,this.state="errored",this.errorAllowRetry=!0,this.fileHandle=e.data.fileHandle,"missing_local_file"===this.error)this.tryFileHandle=!1;else if("permission_needed"===this.error&&e.data.fileHandle){this.state="permission_needed";break}e.data.currentSize&&e.data.totalSize&&(this.currentSize=e.data.currentSize,this.totalSize=e.data.totalSize)}break;case"collAdded":e.data.name===this.coll&&(this.total||(this.total=100),this.progress=this.total,this.percent=100,this.pingInterval&&clearInterval(this.pingInterval),this.dispatchEvent(new CustomEvent("coll-loaded",{detail:e.data})))}}))}async doLoad(){let e=this.sourceUrl,t=null;if(this.percent=this.currentSize=this.totalSize=0,!navigator.serviceWorker)return this.state="errored","http:"===window.location.protocol?this.error=`Sorry, the ReplayWeb.page system must be loaded from an HTTPS URL, but was loaded from: ${window.location.host}.\nPlease try loading this page from an HTTPS URL`:this.error="Sorry, this browser is not supported. Please try a different browser\n(If you're using Firefox, try without Private Mode)",void(this.errorAllowRetry=!1);try{const{scheme:i,host:a,path:r}=function(e){let t=e.indexOf("://"),i=0,a="",r="",o="";return t>=0?(a=e.slice(0,t),t+=3):(t++,e.startsWith("//")&&(t+=2)),i=e.indexOf("/",t),i>0?(r=e.slice(t,i),o=e.slice(i)):(r=e.slice(t),o=""),{scheme:a,host:r,path:o}}(e);switch(i){case"googledrive":this.state="googledrive",t=await this.googledriveInit();break;case"s3":t={sourceUrl:e,loadUrl:`https://${a}.s3.amazonaws.com${r}`,name:this.sourceUrl};break;case"file":if(!this.loadInfo&&!this.tryFileHandle)return this.state="errored",this.error="File URLs can not be entered directly or shared.\nYou can select a file to upload from the main page by clicking the 'Choose File...' button.",void(this.errorAllowRetry=!1);t=this.loadInfo;break;case"ipfs":if(he){const i=new URL("http://files.replayweb.page/"),a=e.split("#",1)[0];i.searchParams.set("ipfs",a.slice("ipfs://".length)),t={sourceUrl:e,loadUrl:i.href}}}}catch(e){console.log(e)}t||(t={sourceUrl:e}),t.newFullImport=this.loadInfo&&this.loadInfo.newFullImport,this.state="started";const i={msg_type:"addColl",name:this.coll,skipExisting:!0,file:t};this.loadInfo&&this.loadInfo.extraConfig&&(i.extraConfig=this.loadInfo.extraConfig),navigator.serviceWorker.controller||await new Promise((e=>{navigator.serviceWorker.addEventListener("controllerchange",(()=>e()))})),navigator.serviceWorker.controller.postMessage(i),this.pingInterval=setInterval((()=>{navigator.serviceWorker.controller.postMessage({msg_type:"ping"})}),15e3)}googledriveInit(){return this._gdWait=new Promise((e=>this._gdResolve=e)),this._gdWait}async onLoadReady(e){this._gdResolve&&this._gdResolve(e.detail)}onCancel(){navigator.serviceWorker&&navigator.serviceWorker.controller&&(navigator.serviceWorker.controller.postMessage({msg_type:"cancelLoad",name:this.coll}),this.pingInterval&&clearInterval(this.pingInterval))}updated(e){(this.sourceUrl&&e.has("sourceUrl")||e.has("tryFileHandle"))&&this.doLoad()}static get styles(){return ue(o`
`}`}script(){if("trypublic"===this.state||this.scriptLoaded)return L``;const e=document.createElement("script");return e.onload=()=>this.onLoad(),e.src="https://apis.google.com/js/platform.js",e}gauth(e,t){self.gapi.load("auth2",(()=>{self.gapi.auth2.authorize({client_id:"160798412227-tko4c82uopud11q105b2lvbogsj77hlg.apps.googleusercontent.com",scope:"https://www.googleapis.com/auth/drive.file",response_type:"token",prompt:e},t)}))}}customElements.define("wr-gdrive",jt);class Ht extends J{constructor(){super(),this.progress=0,this.total=0,this.percent=0,this.coll="",this.state="waiting",this.loadInfo=null,this.currentSize=0,this.totalSize=0,this.tryFileHandle=!!window.showOpenFilePicker,this.fileHandle=null,this.errorAllowRetry=!1,this.pingInterval=""}static get properties(){return{sourceUrl:{type:String},loadInfo:{type:Object},state:{type:String},progress:{type:Number},percent:{type:Number},currentSize:{type:Number},totalSize:{type:Number},error:{type:String},total:{type:Number},status:{type:String},coll:{type:String},embed:{type:String},tryFileHandle:{type:Boolean},errorAllowRetry:{type:Boolean},extraMsg:{type:String}}}firstUpdated(){this.initMessages()}initMessages(){this.worker=new Worker("sw.js"),this.worker.addEventListener("message",(e=>{switch(e.data.msg_type){case"collProgress":if(e.data.name===this.coll){if(this.percent=e.data.percent,e.data.error)if(this.error=e.data.error,this.state="errored",this.errorAllowRetry=!0,this.fileHandle=e.data.fileHandle,"missing_local_file"===this.error)this.tryFileHandle=!1;else if("permission_needed"===this.error&&e.data.fileHandle){this.state="permission_needed";break}e.data.currentSize&&e.data.totalSize&&(this.currentSize=e.data.currentSize,this.totalSize=e.data.totalSize),this.extraMsg=e.data.extraMsg}break;case"collAdded":e.data.name===this.coll&&(this.total||(this.total=100),this.progress=this.total,this.percent=100,this.pingInterval&&clearInterval(this.pingInterval),this.dispatchEvent(new CustomEvent("coll-loaded",{detail:e.data})),this.worker.terminate())}}))}async doLoad(){let e=this.sourceUrl,t=null;if(this.percent=this.currentSize=this.totalSize=0,!navigator.serviceWorker)return this.state="errored","http:"===window.location.protocol?this.error=`Sorry, the ReplayWeb.page system must be loaded from an HTTPS URL, but was loaded from: ${window.location.host}.\nPlease try loading this page from an HTTPS URL`:this.error="Sorry, this browser is not supported. Please try a different browser\n(If you're using Firefox, try without Private Mode)",void(this.errorAllowRetry=!1);try{const{scheme:i,host:a,path:r}=function(e){let t=e.indexOf("://"),i=0,a="",r="",o="";return t>=0?(a=e.slice(0,t),t+=3):(t++,e.startsWith("//")&&(t+=2)),i=e.indexOf("/",t),i>0?(r=e.slice(t,i),o=e.slice(i)):(r=e.slice(t),o=""),{scheme:a,host:r,path:o}}(e);switch(i){case"googledrive":this.state="googledrive",t=await this.googledriveInit();break;case"s3":t={sourceUrl:e,loadUrl:`https://${a}.s3.amazonaws.com${r}`,name:this.sourceUrl};break;case"file":if(!this.loadInfo&&!this.tryFileHandle)return this.state="errored",this.error="File URLs can not be entered directly or shared.\nYou can select a file to upload from the main page by clicking the 'Choose File...' button.",void(this.errorAllowRetry=!1);t=this.loadInfo;break;case"ipfs":if(he){const i=new URL("http://files.replayweb.page/"),a=e.split("#",1)[0];i.searchParams.set("ipfs",a.slice("ipfs://".length)),t={sourceUrl:e,loadUrl:i.href}}}}catch(e){console.log(e)}t||(t={sourceUrl:e}),t.newFullImport=this.loadInfo&&this.loadInfo.newFullImport,this.state="started";const i={msg_type:"addColl",name:this.coll,skipExisting:!0,file:t};this.loadInfo&&this.loadInfo.extraConfig&&(i.extraConfig=this.loadInfo.extraConfig),navigator.serviceWorker.controller||await new Promise((e=>{navigator.serviceWorker.addEventListener("controllerchange",(()=>e()))})),this.worker.postMessage(i)}googledriveInit(){return this._gdWait=new Promise((e=>this._gdResolve=e)),this._gdWait}async onLoadReady(e){this._gdResolve&&this._gdResolve(e.detail)}onCancel(){this.worker&&this.worker.controller.postMessage({msg_type:"cancelLoad",name:this.coll})}updated(e){(this.sourceUrl&&e.has("sourceUrl")||e.has("tryFileHandle"))&&this.doLoad()}static get styles(){return ue(o`
:host {
height: 100%;
display: flex;
@@ -1351,6 +1351,10 @@ class re extends ie{}re.directiveName="unsafeSVG",re.resultType=2;const oe=ee(re
section.container {
margin: auto;
}
.extra-msg {
font-size: 0.8rem;
}
`)}render(){return L`
<section class="container">
<div class="has-text-centered is-flex">
@@ -1372,7 +1376,11 @@ class re extends ie{}re.directiveName="unsafeSVG",re.resultType=2;const oe=ee(re
value="${this.percent}" max="100"></progress>
<label class="progress-label" for="progress">${this.percent}%</label>
${this.currentSize&&this.totalSize?L`
<p class="loaded-prog">Loaded <b>${Ue()(this.currentSize)}</b> of <b>${Ue()(this.totalSize)}</b></p>`:L``}
<div class="loaded-prog">Loaded <b>${Ue()(this.currentSize)}</b> of <b>${Ue()(this.totalSize)}</b>
${this.extraMsg&&L`
<p class="extra-msg">(${this.extraMsg})</p>
`}
</div>`:L``}
${this.embed?"":L`
<button @click="${this.onCancel}" class="button is-danger">Cancel</button>`}