archivalo-oss/replayweb.page
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

jekyll: move jekyll back to root, but simpler setup, exclude via _config.yml

Browse Source 
app: bundle jekyll docs into app, add 'yarn run docs' to build
ui: add back/forward arrows for app only, adjust responsiveness for app, add IS_APP flag injected via preload
ui: add animated logo, friendly message on empty replay frame
app: improve serve logic:
- serve static files from static root
- serve replay from http localhost instead to avoid proxying https for remote requests
- serve local lookup from http prefix also
- catch any fallthrough from replay as 404
- add timestamp to response channels to ensure uniqness
This commit is contained in:
Ilya Kreymer
2020-05-21 18:49:26 -07:00
parent da1e5c03df
commit 220d14aea3
29 changed files with 528 additions and 304 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/electron-main.js
+84 -68
View File
@@ -1,27 +1,28 @@
"use strict";
const {app, session, BrowserWindow, protocol, ipcMain} = require('electron');
const path = require('path');
const fs = require('fs');
const qs = require('querystring');
import fetch from 'node-fetch';
import { Headers } from 'node-fetch';
const { ArchiveResponse, Rewriter } = require('wabac/src/rewrite');
import {app, session, BrowserWindow, ipcMain} from 'electron';
const { Headers } = require('node-fetch');
import path from 'path';
import fs from 'fs';
import qs from 'querystring';
import { ArchiveResponse, Rewriter } from 'wabac/src/rewrite';
import { PassThrough } from 'stream';
import mime from 'mime-types';
global.Headers = Headers;
const { PassThrough } = require('stream');
const STATIC_PREFIX = `https://replayweb.page/`;
const STATIC_PREFIX = "http://localhost:5471/";
const REPLAY_PREFIX = STATIC_PREFIX + "wabac/";
const STATIC_PATHS = {
'index.html': 'text/html',
'frontend.js': 'application/javascript',
'sw.js': 'application/javascript'
};
const HTML_TYPE = 'text/html; charset="utf-8"';
const JS_TYPE = 'application/javascript; charset="utf-8"';
const URL_RX = /([^\/]+)\/([\d]+)(?:\w\w_)?\/(.*)$/;
@@ -29,7 +30,7 @@ const appPath = app.getAppPath();
const projPath = path.join(appPath, "../");
const staticContentPath = path.join(__dirname, "../", "docs");
const staticContentPath = path.join(__dirname, "../");
const pluginPath = path.join(projPath, "plugins", "PepperFlashPlayer.plugin");
@@ -51,7 +52,7 @@ app.commandLine.appendSwitch('ppapi-flash-path', pluginPath);
var proxyColl;
var proxyTS;
var mainContents;
var mainWindow;
function bufferToStream(data) {
@@ -65,42 +66,50 @@ function bufferToStream(data) {
async function doIntercept(request, callback) {
console.log("request", request.url);
console.log(`${request.method} ${request.url} from ${request.referrer}`);
// if local server
if (request.url.startsWith(STATIC_PREFIX)) {
let filename = request.url.slice(STATIC_PREFIX.length).split("?", 1)[0];
if (filename === "") {
filename = "index.html";
}
const type = STATIC_PATHS[filename];
if (type) {
const fullPath = path.join(staticContentPath, filename);
console.log("fullPath: " + fullPath);
const data = fs.createReadStream(fullPath);
callback({statusCode: 200, headers: {"content-type": type}, data});
return;
} else if (request.url.startsWith(REPLAY_PREFIX)) {
//if replay prefix
if (request.url.startsWith(REPLAY_PREFIX)) {
const m = request.url.slice(REPLAY_PREFIX.length).match(URL_RX);
if (m) {
proxyColl = m[1];
proxyTS = m[2];
//const data = bufferToStream(`<html><head><script>window.location.href = "${m[3]}";</script></head></html>`);
//const statusCode = 200;
//const headers = {"location": m[3], "cache-control": "no-cache"};
//callback({statusCode, headers, data});
request.url = m[3];
await resolveResponse(request, callback);
return;
return await resolveArchiveResponse(request, callback);
}
} else {
// try serve static file from app dir
let filename = request.url.slice(STATIC_PREFIX.length).split("?", 1)[0];
if (filename === "") {
filename = "index.html";
} else if (filename === "docs") {
filename = "docs/index.html";
}
let ext = path.extname(filename);
if (!ext) {
ext = ".html";
filename += ext;
}
const mimeType = mime.contentType(ext);
if (mimeType) {
const fullPath = path.join(staticContentPath, filename);
console.log("fullPath: " + fullPath);
const data = fs.createReadStream(fullPath);
return callback({statusCode: 200, headers: {"content-type": mimeType}, data});
}
}
return notFound(request.url, callback);
}
@@ -122,12 +131,30 @@ async function doIntercept(request, callback) {
const {statusCode, start, end} = parseRange(reqHeaders, headers, size);
const data = fs.createReadStream(filename, {start, end});
const data = request.method === "HEAD" ? null : fs.createReadStream(filename, {start, end});
callback({statusCode, headers, data});
return;
}
await resolveResponse(request, callback);
// possible 'live leak' attempt, just return not found
if (request.referrer && request.referrer.startsWith(REPLAY_PREFIX)) {
return notFound(request.url, callback);
}
await proxyLive(request, callback);
}
async function proxyLive(request, callback) {
let headers = request.headers;
const method = request.method;
const response = await fetch(request.url, {method, headers});
const data = method === "HEAD" ? null : response.body;
const statusCode = response.status;
headers = Object.fromEntries(response.headers.entries());
callback({statusCode, headers, data});
}
function notFound(url, callback) {
@@ -136,9 +163,10 @@ function notFound(url, callback) {
callback({statusCode: 404, headers: {"Content-Type": 'text/html; charset="utf-8"'}, data});
}
async function resolveResponse(request, callback) {
//mainContents.once("ipc-message", async (event, channel, status, headers, payload) => {
ipcMain.once("req:" + request.url, async (event, status, headers, payload) => {
async function resolveArchiveResponse(request, callback) {
const channel = `req:${new Date().getTime()}:${request.url}`;
ipcMain.once(channel, async (event, status, headers, payload) => {
const url = request.url;
if (status === 404 && !payload) {
@@ -161,7 +189,6 @@ async function resolveResponse(request, callback) {
useBaseRules: true
});
//const timestamp = proxyTS;
request.headers = new Headers(request.headers);
try {
@@ -188,7 +215,7 @@ async function resolveResponse(request, callback) {
});
mainContents.send("getresponse", request, proxyColl, proxyTS);
mainWindow.webContents.send("getresponse", request, proxyColl, proxyTS, channel);
}
function parseRange(reqHeaders, headers, size) {
@@ -215,28 +242,14 @@ function parseRange(reqHeaders, headers, size) {
function createWindow () {
//createServer();
const sesh = session.defaultSession;
protocol.registerStreamProtocol("filex", async (request, callback) => {
const filename = qs.unescape(request.url.slice("filex://-".length));
console.log("filex", filename);
if (filename.endsWith("?redir")) {
const newLoc = request.url.replace("?redir", "");
console.log("redir to: " + newLoc);
callback({statusCode: 304, headers: {"Location": newLoc}, data: bufferToStream(NOT_FOUND)});
return;
}
});
//sesh.protocol.interceptStreamProtocol("file", doIntercept);
sesh.protocol.interceptStreamProtocol("http", doIntercept);
sesh.protocol.interceptStreamProtocol("https", doIntercept);
//sesh.protocol.interceptStreamProtocol("https", doIntercept);
// Create the browser window.
const mainWindow = new BrowserWindow({
mainWindow = new BrowserWindow({
width: 1024,
height: 768,
isMaximized: true,
@@ -244,6 +257,8 @@ function createWindow () {
webPreferences: {
plugins: true,
preload: path.join(__dirname, 'preload.js'),
nativeWindowOpen: true,
contextIsolation: true
}
}).once('ready-to-show', () => {
@@ -289,11 +304,12 @@ app.on('window-all-closed', function () {
app.quit();
});
/*
app.on('web-contents-created', (event, contents) => {
if (contents.getType() === "webview") {
contents.openDevTools();
} else if (contents.getType() === "window") {
console.log("main contents set")
mainContents = contents;
}
});
});*/
src/electron-preload.js
+4 -4
View File
@@ -2,7 +2,9 @@
import { CollectionLoader } from 'wabac/src/loaders';
const { ipcRenderer } = require('electron');
const { ipcRenderer, contextBridge } = require('electron');
contextBridge.exposeInMainWorld('electron', {'IS_APP': true});
const dbs = {};
@@ -17,7 +19,7 @@ async function getDB(name) {
return dbs[name].store;
}
async function getResponse(event, request, coll, ts) {
async function getResponse(event, request, coll, ts, channel) {
const db = await getDB(coll);
await db.initing;
@@ -25,8 +27,6 @@ async function getResponse(event, request, coll, ts) {
const result = await db.getResource(req, "", {request});
const channel = "req:" + request.url;
if (!result) {
ipcRenderer.send(channel, 404);
return;
src/frontend.js
+139 -62
View File
@@ -11,8 +11,13 @@ import marked from 'marked';
import allCssRaw from '../assets/main.scss';
import rwpLogo from '../assets/logo.svg';
import rwpAnimLogo from '../assets/logoa.svg';
import fasArrowLeft from '@fortawesome/fontawesome-free/svgs/solid/arrow-left.svg';
import fasArrowRight from '@fortawesome/fontawesome-free/svgs/solid/arrow-right.svg';
import fasInfoIcon from '@fortawesome/fontawesome-free/svgs/solid/info-circle.svg';
import fasArchiveIcon from '@fortawesome/fontawesome-free/svgs/solid/info-circle.svg';
import fasRefresh from '@fortawesome/fontawesome-free/svgs/solid/redo-alt.svg';
import fasFullscreen from '@fortawesome/fontawesome-free/svgs/solid/desktop.svg';
import fasUnfullscreen from '@fortawesome/fontawesome-free/svgs/solid/compress-arrows-alt.svg';
@@ -31,10 +36,7 @@ function wrapCss(custom) {
return [allCss, custom];
}
const GDRIVE_CLIENT_ID = "160798412227-tko4c82uopud11q105b2lvbogsj77hlg.apps.googleusercontent.com";
const HELPER_PROXY = "https://helper-proxy.webrecorder.workers.dev";
const IS_APP = window.electron && window.electron.IS_APP;
const dbworker = new Worker(__SW_PATH__);
@@ -87,9 +89,32 @@ class AppMain extends LitElement
.navbar {
padding: 0 0.5em;
}
.info-menu {
padding: 0 1.0em;
div.navbar-menu fa-icon {
vertical-align: sub;
}
@media screen and (min-width: 800px) {
.menu-only {
display: none;
}
a.arrow-button {
padding-left: 4px;
padding-right: 4px;
}
.info-menu {
padding: 0 1.0em;
}
}
@media screen and (max-width: 800px) {
.wide-only {
display: none;
}
}
`);
}
@@ -111,11 +136,21 @@ class AppMain extends LitElement
</div>
<div class="navbar-menu ${this.navMenuShown ? 'is-active' : ''}">
<div class="navbar-start">
${IS_APP ? html`
<a class="navbar-item arrow-button" title="Go Back" @click="${(e) => window.history.back()}">
<fa-icon .svg="${fasArrowLeft}"></fa-icon><span class="menu-only">&nbsp;Go Back</span>
</a>
<a class="navbar-item arrow-button" title="Go Forward" @click="${(e) => window.history.forward()}">
<fa-icon .svg="${fasArrowRight}"></fa-icon><span class="menu-only">&nbsp;Go Forward</span>
</a>
` : ``}
${this.sourceUrl && this.collInfo ? html`
<div class="navbar-item has-dropdown is-hoverable">
<a class="info-menu navbar-link is-arrowless"><fa-icon size="1.2em" .svg="${fasArchiveIcon}"></fa-icon></a>
<a class="info-menu navbar-link is-arrowless"><fa-icon size="1.2em" .svg="${fasInfoIcon}"></fa-icon>
<span class="menu-only">About this Archive</span>
</a>
<div class="navbar-dropdown">
<div class="navbar-item"><i>About this Archive</i></div>
<div class="navbar-item wide-only"><i>About this Archive</i></div>
<hr class="navbar-divider">
<div class="navbar-item">
Source:&nbsp;<b>${this.sourceUrl}</b>
@@ -138,7 +173,7 @@ class AppMain extends LitElement
</div>
<div class="navbar-end">
<a href="/docs" target="_blank" class="navbar-item">
<fa-icon .svg="${fasHelp}"></fa-icon>&nbsp;How it Works
<fa-icon .svg="${fasHelp}"></fa-icon><span>&nbsp;How it Works</span>
</a>
<a href="?terms" @click="${(e) => { e.preventDefault(); this.showTerms = true} }"class="navbar-item">Terms</a>
</div>
@@ -193,6 +228,9 @@ class AppMain extends LitElement
firstUpdated() {
this.initRoute();
window.addEventListener("popstate", (event) => {
this.initRoute();
});
}
updated(changedProperties) {
@@ -268,10 +306,15 @@ class AppMain extends LitElement
onStartLoad(event) {
// just redirect right away?
this.pageParams.set("source", event.detail.sourceUrl);
if (!event.detail.isFile) {
this.pageParams.set("source", event.detail.sourceUrl);
window.location.search = this.pageParams.toString();
return;
} else {
const url = new URL(window.location.href);
url.search = this.pageParams.toString();
window.history.pushState({}, "", url.toString());
}
this.sourceUrl = event.detail.sourceUrl;
@@ -289,7 +332,7 @@ class AppMain extends LitElement
return;
}
this.initRoute();
//this.initRoute();
if (event.detail.sourceUrl !== this.sourceUrl) {
this.pageParams.set("source", event.detail.sourceUrl);
@@ -468,7 +511,7 @@ You can select a file to upload from the main page by clicking the \'Choose File
return html`
<section class="container">
<div class="has-text-centered is-flex">
<fa-icon class="logo" size="96px" .svg=${rwpLogo}></fa-icon>
<fa-icon class="logo" size="96px" .svg=${rwpAnimLogo}></fa-icon>
</div>
<div class="level">
<p class="level-item">Loading&nbsp;<b>${this.sourceUrl}</b>...</p>
@@ -620,6 +663,9 @@ class WrIndex extends LitElement
div.field.has-addons {
flex: auto;
}
form {
flex-grow: 1;
}
button.is-loading {
line-height: 1.5em;
height: 1.5em;
@@ -645,46 +691,48 @@ class WrIndex extends LitElement
render() {
return html`
<section class="section no-top-padding">
<div class="container">
<nav class="panel is-warning">
<p class="panel-heading">Load Web Archive</p>
<div class="extra-padding panel-block file has-name">
<form class="container is-flex" @submit="${this.onStartLoad}">
<label class="file-label">
<input class="file-input"
@click="${(e) => e.currentTarget.value = null}"
@change=${this.onChooseFile} type="file" id="fileupload" name="fileupload">
<span class="file-cta">
<span class="file-icon">
<fa-icon size="0.9em" .svg=${fasUpload}></fa-icon>
</span>
<span class="file-label">
Choose File...
</span>
${!IS_APP ? html`
<p class="subtitle has-text-centered">Explore and Replay Interactive Archived Webpages Directly in your Browser.</p>
` : ``}
<nav class="panel is-warning">
<p class="panel-heading">Load Web Archive</p>
<div class="extra-padding panel-block file has-name">
<form class="content is-flex" @submit="${this.onStartLoad}">
<label class="file-label">
<input class="file-input"
@click="${(e) => e.currentTarget.value = null}"
@change=${this.onChooseFile} type="file" id="fileupload" name="fileupload">
<span class="file-cta">
<span class="file-icon">
<fa-icon size="0.9em" .svg=${fasUpload}></fa-icon>
</span>
</label>
<span class="file-label">
Choose File...
</span>
</span>
</label>
<div class="field has-addons">
<p class="control is-expanded">
<input class="file-name input" type="text"
name="filename" id="filename"
pattern="^((file|http|https|s3):\/\/.*\.(warc|warc.gz|har|zip|waz))|(googledrive:\/\/[^\/\s]+)$"
.value="${this.fileDisplayName}"
@input="${this.onInput}"
autocomplete="off"
placeholder="Choose a local file or enter a URL for a (WARC, HAR, WBN, or ZIP) archive">
</p>
<div class="control">
<button type="submit" class="button is-primary">Load</button>
</div>
<div class="field has-addons">
<p class="control is-expanded">
<input class="file-name input" type="text"
name="filename" id="filename"
pattern="((file|http|https|s3):\/\/.*\.(warc|warc.gz|har|zip|wacz))|(googledrive:\/\/.+)"
.value="${this.fileDisplayName}"
@input="${this.onInput}"
autocomplete="off"
placeholder="Choose a local file or enter a URL for a (WARC, HAR, WBN, or WACZ) archive">
</p>
<div class="control">
<button type="submit" class="button is-primary">Load</button>
</div>
</div>
</form>
</div>
</nav>
</div>
</form>
</div>
</nav>
</section>
<section class="container">
<section class="section no-top-padding">
<nav class="panel is-light">
<p class="panel-heading">Loaded Archives</p>
<div class="coll-list">
@@ -789,7 +837,7 @@ class WrColl extends LitElement
const newHash = "#" + new URLSearchParams(this.tabData).toString();
if (newHash !== this._locationHash) {
this._locationHash = newHash;
if (this._replaceLoc) {
if (this._replaceLoc || Object.keys(changedProperties.get("tabData")).length === 0) {
const newLoc = new URL(window.location.href);
newLoc.hash = this._locationHash;
window.history.replaceState({}, "", newLoc.href);
@@ -900,13 +948,13 @@ class WrColl extends LitElement
display: none;
}
@media screen and (min-width: 1024px) {
@media screen and (min-width: ${!IS_APP ? css`1024px` : css`1134px`}) {
.tab-label {
display: inline;
}
}
@media screen and (min-width: 680px) {
@media screen and (min-width: ${!IS_APP ? css`680px` : css`850px`}) {
.main.tabs {
position: absolute;
top: 0px;
@@ -997,7 +1045,7 @@ class WrColl extends LitElement
<wr-coll-resources .collInfo="${this.collInfo}"
urlSearch="${this.tabData.urlSearch || ""}"
urlSearchType="${this.tabData.urlSearchType || ""}"
.currMime="${this.tabData.currMime}"
.currMime="${this.tabData.currMime || ""}"
@coll-tab-nav="${this.onCollTabNav}" id="resources"
class="is-paddingless ${this.tabData.view === 'resources' ? '' : 'is-hidden'}">
</wr-coll-resources>
@@ -1250,7 +1298,7 @@ class WrResources extends LitElement
{
static get filters() {
return [
{name: "HTML", filter: "text/html,text/xhtml"},
{name: "HTML", filter: ""}, //default
{name: "Images", filter: "image/"},
{name: "Audio/Video", filter: "audio/,video/"},
{name: "PDF", filter: "application/pdf"},
@@ -1306,10 +1354,6 @@ class WrResources extends LitElement
changedProperties.has("urlSearchType") ||
changedProperties.has("currMime")) {
if (!this.currMime) {
this.currMime = "text/html,text/xhtml";
}
this.doLoadResources();
const data = {
urlSearch: this.urlSearch,
@@ -1345,7 +1389,20 @@ class WrResources extends LitElement
url = "https://" + url;
}
const mime = this.currMime === "all" ? "" : this.currMime;
let mime;
switch (this.currMime) {
case "all":
mime = "";
break;
case "":
mime = "text/html,text/xhtml";
break;
default:
mime = this.currMime;
}
const params = new URLSearchParams({
mime,
@@ -1753,6 +1810,21 @@ class WrReplayPage extends LitElement
border-radius: 4px;
}
.intro-panel {
min-width: 40%;
display: flex;
flex-direction: column;
margin: auto;
}
.intro-panel .panel-heading {
font-size: 1.0em;
}
.intro-panel .panel-block {
padding: 1.0em;
}
#datetime {
position: absolute;
right: 1em;
@@ -1776,7 +1848,7 @@ class WrReplayPage extends LitElement
render() {
return html`
${this.embed !== "replayonly" ? html`
<div class="container replay-bar">
<div class="replay-bar">
<form @submit="${this.onSubmit}">
<div class="field has-addons">
<a id="fullscreen" class="button is-borderless" @click="${this.onFullscreenToggle}">
@@ -1803,7 +1875,12 @@ class WrReplayPage extends LitElement
${this.iframeUrl ? html`
<iframe @message="${this.onReplayMessage}" allow="autoplay 'self'; fullscreen"
src="${this.iframeUrl}"></iframe>
` : ``}
` : html`
<nav class="panel intro-panel">
<p class="panel-heading">Replay Web Page</p>
<p class="panel-block">Enter a URL above to replay it from the web archive!</p>
</nav>
`}
${this.isAuthable ? html`
<div class="modal ${this.showAuth ? 'is-active' : ''}">
@@ -1878,7 +1955,7 @@ class WrGdrive extends LitElement
try {
const sourceUrl = this.sourceUrl;
const fileId = sourceUrl.slice("googledrive://".length);
const publicCheckUrl = `${HELPER_PROXY}/g/${fileId}`;
const publicCheckUrl = `${__HELPER_PROXY__}/g/${fileId}`;
let resp = null;
try {
@@ -2007,7 +2084,7 @@ class WrGdrive extends LitElement
gauth(prompt, callback) {
gapi.load('auth2', () => {
gapi.auth2.authorize({
client_id: GDRIVE_CLIENT_ID,
client_id: __GDRIVE_CLIENT_ID__,
scope: "https://www.googleapis.com/auth/drive.file",
response_type: "token",
prompt