diff --git a/.gitignore b/.gitignore index 0a8416e..9f4a803 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,7 @@ dist/sw.js **/node_modules yarn-error.log coverage/ +local-examples/ .DS_Store +.sass-cache/ +_site/ diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..d88a476 --- /dev/null +++ b/Gemfile @@ -0,0 +1 @@ +gem "just-the-docs" diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 0000000..14b7bac --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,67 @@ +GEM + specs: + addressable (2.7.0) + public_suffix (>= 2.0.2, < 5.0) + colorator (1.1.0) + concurrent-ruby (1.1.6) + em-websocket (0.5.1) + eventmachine (>= 0.12.9) + http_parser.rb (~> 0.6.0) + eventmachine (1.2.7) + ffi (1.12.2) + forwardable-extended (2.6.0) + http_parser.rb (0.6.0) + i18n (0.9.5) + concurrent-ruby (~> 1.0) + jekyll (3.8.6) + addressable (~> 2.4) + colorator (~> 1.0) + em-websocket (~> 0.5) + i18n (~> 0.7) + jekyll-sass-converter (~> 1.0) + jekyll-watch (~> 2.0) + kramdown (~> 1.14) + liquid (~> 4.0) + mercenary (~> 0.3.3) + pathutil (~> 0.9) + rouge (>= 1.7, < 4) + safe_yaml (~> 1.0) + jekyll-sass-converter (1.5.2) + sass (~> 3.4) + jekyll-seo-tag (2.6.1) + jekyll (>= 3.3, < 5.0) + jekyll-watch (2.2.1) + listen (~> 3.0) + just-the-docs (0.2.7) + jekyll (~> 3.8.5) + jekyll-seo-tag (~> 2.0) + rake (~> 12.3.1) + kramdown (1.17.0) + liquid (4.0.3) + listen (3.2.1) + rb-fsevent (~> 0.10, >= 0.10.3) + rb-inotify (~> 0.9, >= 0.9.10) + mercenary (0.3.6) + pathutil (0.16.2) + forwardable-extended (~> 2.6) + public_suffix (4.0.3) + rake (12.3.3) + rb-fsevent (0.10.3) + rb-inotify (0.10.1) + ffi (~> 1.0) + rouge (3.17.0) + safe_yaml (1.0.5) + sass (3.7.4) + sass-listen (~> 4.0.0) + sass-listen (4.0.0) + rb-fsevent (~> 0.9, >= 0.9.4) + rb-inotify (~> 0.9, >= 0.9.7) + +PLATFORMS + ruby + +DEPENDENCIES + just-the-docs + +BUNDLED WITH + 2.1.4 diff --git a/_config.yml b/_config.yml new file mode 100644 index 0000000..ef7f900 --- /dev/null +++ b/_config.yml @@ -0,0 +1,22 @@ +remote_theme: pmarsceill/just-the-docs +theme: "just-the-docs" + +title: ReplayWeb.Page + +# Enable or disable the site search +search_enabled: false + +# Set the search token separator for hyphenated-word search: +search_tokenizer_separator: /[\s/]+/ + +# Enable or disable heading anchors +heading_anchors: true + +# Aux links for the upper right navigation +aux_links: + "View replayweb.page on GitHub": + - "//github.com/ikreymer/replayweb.page" + + +logo: "/static/logo.svg" + diff --git a/_includes/title.html b/_includes/title.html new file mode 100644 index 0000000..428514e --- /dev/null +++ b/_includes/title.html @@ -0,0 +1,5 @@ +{% if site.logo %} + +{% endif %} +replay +web.page diff --git a/_sass/custom/custom.scss b/_sass/custom/custom.scss new file mode 100644 index 0000000..6bb7f78 --- /dev/null +++ b/_sass/custom/custom.scss @@ -0,0 +1,33 @@ +$rwp-green: #64e986; +$rwp-yellow: #f5c71a; +//$link-color: #27a0ff; +$link-color: #4876ff; + +//$content-width: 960px; +$nav-width-md: $nav-width; + +.side-bar { + @include mq(md) { + //max-width: calc(0.5rem + #{$nav-width}); + max-width: $nav-width; + } +} + +.site-title { + padding-right: 1.5rem !important; +} + +.rwp-logo-font { + font-variant-caps: all-small-caps; + font-size: 20px; + font-weight: bold; +} + +.rwp-green { + color: $rwp-green; +} + +.rwp-yellow { + color: $rwp-yellow; +} + diff --git a/docs/browsing.md b/docs/browsing.md new file mode 100644 index 0000000..3ab4a7d --- /dev/null +++ b/docs/browsing.md @@ -0,0 +1,79 @@ +--- +layout: default +title: Browsing Web Archives +nav_order: 1 +parent: Usage +permalink: /docs/browsing +--- + +# Browsing Web Archives via the ReplayWeb.page +{: .no_toc } + +## Table of contents +{: .no_toc .text-delta } + +1. TOC +{:toc} + + +## Loading Web Archives + +Using [https://replayweb.page](https://replayweb.page), you can select a web archive file from a local +file or remote URL to load into the site. + +Archives loaded by ReplayWeb.page are identified by their unique URL, which can be entered directly or selected from a local file browser. + + +### Remote Archives + +To load a remote archive, simply enter the URL of the archive and click `Replay!`. +(See: [Supported Locations](locations.md) for details on where archives can be loaded from). + +The archive will be downloaded, either fully or [only as needed (if possible)](streaming-archives.md) and presented on the archive page. + +The system supports WARC files, as well as several other formats (See: [Supported Formats](formats.md) for more details on supported formats) + +Once loaded, the page is redirected to the archive collection page. + + +### Local Archives + +Using the `Choose File` button, you can pick any local archive file to load for replay. +The click `Replay` to load the file. + +The archive is laoded in the browser directly and *not* uploaded anywhere -- it never leaves your computer! + +(This process is similar to how [Webrecorder Player](https://github.com/webrecorder/webrecorder-player) works, but happens entirely in the browser. + + +It will be identified by a `file://` URL and is not accessible outside your browser. + + +## Archive View + +Once an archive is loaded, you are taken to the archive view, and you are ready to explore and replay the web archive! + +This view has several tabs: + +- **Curated Pages** - This tab exists if the archive has curated lists of pages. Currently this is available only for Webrecorder WARCs, but may be possible +with other formats (see: [Web Archive Zip Format](webarchive-zip-format.md) + +- **URL Resources** - This tab allows searching the archive by URLs, as well as by MIME type. For many archives with no page or curatorial metadata, this is +a way to explore the archive data in more detail. + +Once a URL is selected, you will be taken to the Replay tab which renders the archived page directly in your browser. + + +## Index of Archives + +The ReplayWeb.page homepage lists an index of all archives loaded in your browser. +Once loaded, the archive remains cached in the browser for quick access. This view will be unique to your browser. +To remove the archive from your browser, click on the 'X'. (Of course, this does not delete the archive from its original location, only your local copy of it) + + +
+Next: [Sharing Links to Archived Pages](linking.md) + + + + diff --git a/docs/linking.md b/docs/linking.md new file mode 100644 index 0000000..c0c9e5b --- /dev/null +++ b/docs/linking.md @@ -0,0 +1,8 @@ +--- +layout: default +title: Linking to Web Archives +nav_order: 2 +parent: Usage +permalink: /docs/linking +--- + diff --git a/docs/reference.md b/docs/reference.md new file mode 100644 index 0000000..2ccc421 --- /dev/null +++ b/docs/reference.md @@ -0,0 +1,11 @@ +--- +layout: default +title: Reference +nav_order: 2 +has_children: true +permalink: /docs/reference +--- + +## Reference + +A guide to the file formats, source locations and query arguments supported by ReplayWeb.Page diff --git a/docs/supported-formats.md b/docs/supported-formats.md new file mode 100644 index 0000000..74a2bd2 --- /dev/null +++ b/docs/supported-formats.md @@ -0,0 +1,26 @@ +--- +layout: default +title: 'Supported Formats' +nav_order: 2 +description: 'Supported Formats' +permalink: /docs/formats +parent: Reference +--- + +## Supported Formats + +ReplayWeb.Page supports the following archive formats. +Format is currently determined based on the extension, though more extensive detection may be added later. + + +| Format | Extensions | Status | +|:--------|:--------------------|:--------------| +| WARC | `.warc`, `.warc.gz` | Supported | +| HAR | `.har` | Supported | +| WBN | `.wbn` | Experimental | +| ARC | `.arc` | Not Supported | +| CDX | `.cdx`, `.cdxj` | Supported | +| ZIP | `.zip`, `.waz` | In Progress | + + + diff --git a/docs/supported-locations.md b/docs/supported-locations.md new file mode 100644 index 0000000..dfaba76 --- /dev/null +++ b/docs/supported-locations.md @@ -0,0 +1,77 @@ +--- +layout: default +title: 'Supported Locations' +nav_order: 1 +description: 'Supported Locations' +parent: Reference +permalink: /docs/locations +--- + +## Supported Location URLs + +ReplayWeb.Page uses URLs to uniquely indentify web archives. + +In addition HTTP/S, the system supports loading from additional cloud providers +and via peer-to-peer decentralized services. The goal is to support loading +web archives from any source a web browser can connect to. + +The following is a list of currently supported URL schemes and details below: + + +| Scheme | Status | +|:-----------------|:------------| +| `https://` | Supported | +| `http://` | Supported | +| `file://` | Supported | +| `s3://` | Supported | +| `googledrive://` | In progress | +| `ipfs://` | Partial | +| `dat://` | Planned | + + +### HTTP and HTTPS + +While `https://replayweb.page` can load from any url, a site must allow access by enabling [CORS](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS) +for the web archive. HTTP URLs may not be accessible from `https://replayweb.page` since it requires HTTPS. + +If you need to provide access to web archives stored on your own domain or over HTTP, consider +deploying a copy on your server. See: Running your Own. + +### Local Files + +Local files selected via the UI file chooser, and will be given a `file://` URL. +Unlike other web archive URLs, file URLs can not be shared with others. + + +### Amazon S3 + +The `s3://` url scheme is provided as a convenience to reference S3 URLs. The URL is converted to an HTTPS URL for access. +The S3 Bucket must be public and also provide access to `https://replayweb.page` via CORS. +Downloading via S3 credentials is not currently supported. + + +### Google Drive + +The `googledrive://` url scheme is used when accessing a web archive shared over google drive. It provides a unique identifier +provided via Google Drive. When accessing a google drive URL, the user will be asked to authenticate to be able to access the file. + + +### IPFS +{: .d-inline-block } + +Help Wanted +{: .label .label-yellow } + + +The `ipfs://` url scheme is provide as a convenience and resolves to using the Cloudflare IPFS Gateway to download archives from IPFS. +A better implementation with IPFS directly is also possible. + + +### DAT +{: .d-inline-block } + +Help Wanted +{: .label .label-yellow } + +Support for `dat://` is planned to allow true peer-to-peer transfer of web archives. + diff --git a/docs/usage.md b/docs/usage.md new file mode 100644 index 0000000..8d41061 --- /dev/null +++ b/docs/usage.md @@ -0,0 +1,20 @@ +--- +layout: default +title: Usage +nav_order: 1 +description: 'Usage' +has_children: true +permalink: /docs +--- + +# ReplayWeb.Page +## Serverless / Client-Side Web Archive Replay + +A browser-based system to replay archived web pages as accurately as possible, with all interactive elements preserved. +{: .fs-6 .fw-300 } + +No server necessary. Archives can be loaded as static files from anywhere a browser can connect to. +{: .fs-6 .fw-300 } + + +