Site reorg: Remove site/ subpath (#306)

Unfortunately, for proper package pathing, need to remove the `site/`
subpath. This includes:
- Moving everything back into root
- Adding a `copy-site.sh` script to copy resources into `mkdocs/site/`
before building the site with mkdocs.
- Also update examples to ensure all are loading the local sw.js / ui.js
- Consolidate examples in `docs/examples`
- Consolidate assets under src/assets/
- Move `favicons` to root
- Reformat and relint
- Tests: update playwright tests to use mkdocs output, add test for
embed in docs
- bump to 2.0.0-beta.2

.eslintrc.js

@@ -103,10 +103,11 @@ module.exports = {
   ignorePatterns: [
     "__generated__",
     "__mocks__",
+    "mkdocs/_genhtml",
     "dist",
     "ruffle",
-    "ui.js",
+    "**/ui.js",
-    "sw.js",
+    "**/sw.js",
   ],
   overrides: [
     {

.github/workflows/integration.yaml

@@ -29,5 +29,10 @@ jobs:
       - name: Playwright Install Browsers
         run: yarn playwright install
 
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.x
+      - run: pip install -r mkdocs/requirements.txt
+
       - name: Playwright Run Tests
         run: yarn run playwright test

.github/workflows/site-publish.yaml

@@ -26,11 +26,8 @@ jobs:
       - name: Yarn Install
         run: yarn install --frozen-lockfile
 
-      - name: Yarn Build
-        run: yarn build
-
       - uses: actions/setup-python@v4
         with:
           python-version: 3.x
-      - run: pip install mkdocs-macros-plugin mkdocs-redirects mkdocs-material
+      - run: pip install -r mkdocs/requirements.txt
-      - run: cd mkdocs && mkdocs gh-deploy --force
+      - run: yarn run publish-docs

.gitignore

@@ -1,7 +1,8 @@
+**/sw.js
+**/ui.js
 *.har
 *.warc
 *.warc.gz
-!site/examples/*.warc
 **/node_modules
 yarn-error.log
 coverage/
@@ -9,12 +10,13 @@ coverage/
 **/.DS_Store
 **/.sass-cache/
 **/__pycache__/
-mkdocs/_genhtml/
 dist/
 plugins-*/
-site/sw.js
+!mkdocs/site/docs/examples/*.warc
-site/ui.js
+mkdocs/_genhtml/
-site/favicons/
+mkdocs/site/index.html
-.jekyll-cache
+mkdocs/site/embed.html
-
+mkdocs/site/webmanifest.json
+mkdocs/site/favicons
+mkdocs/site/ruffle
 ipfs-core.min.js

CHANGES.md

@@ -3,27 +3,33 @@
 ## CHANGES
 
 v1.8.17
+
 - Dependencies: update to wabac.js 2.17.3, wombat 3.7.3
 - Fidelity: Fix resolving relative URL by avoiding not-http/https URL as base URL, fixes #300 (via wombat 3.7.3)
 
 v1.8.16
+
 - Dependencies: update to wabac.js 2.17.2, electron 29.2.0
 - Fidelity: More accurate rewriting of 'import(' statement (via wabac.js 2.17.2)
 
 v1.8.15
+
 - Dependencies: update to wabac.js 2.17.1, wombat 3.7.2, electron 29.0.1
 - Build: update to latest builder process
 
 v1.8.14
+
 - Loading: fix loading revisit records that may have incorrect content-length (via wabac.js)
 - Fidelity: improved fuzzy matching rules (via wabac.js)
 - Misc: github issue templates, fix typo in CORS config docs
 - Dependencies: Update to wabac.js 2.16.13
 
 v1.8.13
+
 - Loading: Ensure default index.html is served when not in embed mode, don't add '?serveIndex=1' for app itself
 
 v1.8.12
+
 - Loading: Fixes embed 'Purge Cache + Reload', ensure '?serveIndex=1' is always set on all service worker loading
 
 v1.8.11

README.md

@@ -1,6 +1,6 @@
 <h1>
     <div align="center">
-        <img alt="ReplayWebpage" src="assets/replaywebpage-lockup-color-dynamic.svg" width="90%">
+        <img alt="ReplayWebpage" src="src/assets/brand/replaywebpage-lockup-color-dynamic.svg" width="90%">
     </div>
 </h1>
 

embed.html

@@ -6,15 +6,15 @@
         background-color: lightgrey;
       }
     </style>
-    <link rel="icon" href="/favicons/favicon.ico" sizes="32x32">
+    <link rel="icon" href="/favicons/favicon.ico" sizes="32x32" />
-    <link rel="icon" href="/favicons.favicon.svg" type="image/svg+xml">
+    <link rel="icon" href="/favicons.favicon.svg" type="image/svg+xml" />
-    <link rel="apple-touch-icon" href="/favicons/apple-touch-icon.png">
+    <link rel="apple-touch-icon" href="/favicons/apple-touch-icon.png" />
   </head>
   <body>
     <replay-web-page
       replaybase="./"
       embed="replay-with-info"
-      src="./docs/assets/tweet-example.wacz"
+      src="./docs/examples/tweet-example.wacz"
       url="page:0"
     ></replay-web-page>
   </body>

index.html

@@ -5,9 +5,9 @@
     <title>ReplayWeb.page</title>
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <script src="./ui.js"></script>
-    <link rel="icon" href="/favicons/favicon.ico" sizes="32x32">
+    <link rel="icon" href="/favicons/favicon.ico" sizes="32x32" />
-    <link rel="icon" href="/favicons/favicon.svg" type="image/svg+xml">
+    <link rel="icon" href="/favicons/favicon.svg" type="image/svg+xml" />
-    <link rel="apple-touch-icon" href="/favicons/apple-touch-icon.png">
+    <link rel="apple-touch-icon" href="/favicons/apple-touch-icon.png" />
   </head>
   <body>
     <replay-app-main></replay-app-main>

mkdocs/copy-site.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+curr=$(dirname "${BASH_SOURCE[0]}")
+src=${curr}/..
+dest=${curr}/site/
+for file in $(cat $curr/sitefiles.txt); do cp -r "$src/$file" $dest; done

mkdocs/mkdocs.yml

@@ -1,18 +1,16 @@
 site_name: ReplayWeb.page Docs
-docs_dir: ../site/
+docs_dir: ./site/
 watch:
-  - ../site/
+  - ./site/
 site_dir: _genhtml/
 repo_url: https://github.com/webrecorder/replayweb.page
 repo_name: ReplayWeb.page
 edit_uri: edit/main/site/docs/
 extra_css:
-    - docs/stylesheets/extra.css
+  - docs/stylesheets/extra.css
-extra_javascript:
-    - docs/js/sw.js
 theme:
   name: material
-  custom_dir: ../site/docs/overrides
+  custom_dir: ./site/docs/overrides
   features:
     - navigation.sections
     - navigation.tabs
@@ -52,18 +50,18 @@ theme:
 nav:
   - docs/index.md
   - Embedding:
-    - docs/embedding/index.md
+      - docs/embedding/index.md
-    - docs/embedding/cors-settings.md
+      - docs/embedding/cors-settings.md
   - User Guide:
-    - docs/user-guide/index.md
+      - docs/user-guide/index.md
-    - docs/user-guide/loading.md
+      - docs/user-guide/loading.md
-    - docs/user-guide/locations.md
+      - docs/user-guide/locations.md
-    - docs/user-guide/exploring.md
+      - docs/user-guide/exploring.md
-    - docs/user-guide/sharing.md
+      - docs/user-guide/sharing.md
-    - docs/user-guide/offline-use.md
+      - docs/user-guide/offline-use.md
 
 markdown_extensions:
-  - toc: 
+  - toc:
       toc_depth: 3
       permalink: true
   - pymdownx.highlight:
@@ -73,7 +71,7 @@ markdown_extensions:
       emoji_generator: !!python/name:material.extensions.emoji.to_svg
       options:
         custom_icons:
-          - ../site/docs/overrides/.icons
+          - ./site/docs/overrides/.icons
   - admonition
   - pymdownx.inlinehilite
   - pymdownx.details
@@ -100,11 +98,11 @@ plugins:
   - search
   - redirects:
       redirect_maps:
-        'docs/cors-settings.md': 'docs/embedding/cors-settings.md'
+        "docs/cors-settings.md": "docs/embedding/cors-settings.md"
-        'docs/usage.md': 'docs/user-guide/index.md'
+        "docs/usage.md": "docs/user-guide/index.md"
-        'docs/loading_online.md': docs/user-guide/loading.md
+        "docs/loading_online.md": docs/user-guide/loading.md
-        'docs/loading_offline.md': docs/user-guide/offline-use.md
+        "docs/loading_offline.md": docs/user-guide/offline-use.md
-        'docs/exploring.md': 'docs/user-guide/exploring.md'
+        "docs/exploring.md": "docs/user-guide/exploring.md"
-        'docs/sharing.md': 'docs/user-guide/sharing.md'
+        "docs/sharing.md": "docs/user-guide/sharing.md"
-        'docs/locations.md': 'docs/user-guide/locations.md'
+        "docs/locations.md": "docs/user-guide/locations.md"
-        'docs/formats.md': 'docs/user-guide/index.md#supported-formats'
+        "docs/formats.md": "docs/user-guide/index.md#supported-formats"

mkdocs/requirements.txt

@@ -0,0 +1,3 @@
+mkdocs-macros-plugin
+mkdocs-redirects
+mkdocs-material

mkdocs/site/docs/embedding/index.md

@@ -15,12 +15,12 @@ The `#!html <replay-web-page>` web component consists of a "backend" service wor
 
 ### Loading the ReplayWeb.page UI (Frontend)
 
-To embed a WACZ stored at `https://replayweb.page/docs/assets/tweet-example.wacz`, add the following `#!html <script>` tag to your HTML page to load the user interface from the jsDelivr CDN, and use the `#!html <replay-web-page>` component to point to the WACZ:
+To embed a WACZ stored at `https://replayweb.page/docs/examples/tweet-example.wacz`, add the following `#!html <script>` tag to your HTML page to load the user interface from the jsDelivr CDN, and use the `#!html <replay-web-page>` component to point to the WACZ:
 
 ```html
 <script src="https://cdn.jsdelivr.net/npm/replaywebpage@{{ rwp_version() }}/ui.js"></script>
 
-<replay-web-page source="https://replayweb.page/docs/assets/tweet-example.wacz"
+<replay-web-page source="https://replayweb.page/docs/examples/tweet-example.wacz"
 url="https://oembed.link/https://twitter.com/webrecorder_io/status/1565881026215219200"></replay-web-page>
 ```
 
@@ -45,7 +45,7 @@ allowing it to render web archives on the `/replay/` path.
 
 That's it! Loading `https://example.com/path/my-web-archive-embed.html` should now load the web archive.
 
-Be sure to add width and height styles to the `#!html <replay-web-page>` tag as needed to scale the embed, and replace `https://replayweb.page/docs/assets/example.wacz` with any web archive hosted on your site.
+Be sure to add width and height styles to the `#!html <replay-web-page>` tag as needed to scale the embed, and replace `https://replayweb.page/docs/examples/example.wacz` with any web archive hosted on your site.
 
 ??? Info "Info: Loading files from different servers"
     If the file is loaded from a different origin, your site must have Cross Origin Resource Sharing (CORS) access to download the web archive.
@@ -124,8 +124,8 @@ ReplayWeb.page offers four different ways to embed the archived content, includi
 - `replay-with-info`: Show the `replayonly` mode, but also add an archive info dropdown, which shows an archival 'receipt' with provenance and verification information (new in 1.7.0)
 
 ??? example "Example: Embed with `replay-with-info` enabled"
-    <script src="https://cdn.jsdelivr.net/npm/replaywebpage/ui.js"></script>
+    <script src="/ui.js"></script>
-    <replay-web-page style="height: 600px" loading="eager" embed="replay-with-info" replaybase="../../../" source="../../assets/tweet-example.wacz"url="https://oembed.link/https://twitter.com/webrecorder_io/status/1565881026215219200"></replay-web-page>
+    <replay-web-page style="height: 600px" loading="eager" embed="replay-with-info" replaybase="/" source="/docs/examples/tweet-example.wacz"url="https://oembed.link/https://twitter.com/webrecorder_io/status/1565881026215219200"></replay-web-page>
 
 ## Embedding Options
 

mkdocs/site/docs/user-guide/exploring.md

@@ -16,7 +16,7 @@ Pages can be searched for by their title, URLs, or extracted text if available.
 
 ReplayWeb.page will load pre-generated full-text search data when loading WACZ files, and will also attempt to generate full-text index from HTML files contained in WARC files upon loading a WARC.
 
-[See an example of searching for pages containing "WARC".](https://replayweb.page/?source=examples/netpreserve-twitter.warc#view=pages&query=WARC)
+[See an example of searching for pages containing "WARC".](/?source=docs/examples/netpreserve-twitter.warc#view=pages&query=WARC)
 
 ### Resources
 
@@ -24,7 +24,7 @@ The resources view displays all the content within a loaded archived item, liste
 
 This view is available for all archives that only store raw data.
 
-[See an example of searching for image resources.](https://replayweb.page/?source=examples/netpreserve-twitter.warc#view=resources&currMime=image%2F&urlSearchType=prefix)
+[See an example of searching for image resources.](/?source=docs/examples/netpreserve-twitter.warc#view=resources&currMime=image%2F&urlSearchType=prefix)
 
 #### Searching For Resources
 

mkdocs/sitefiles.txt

@@ -0,0 +1,7 @@
+sw.js
+ui.js
+index.html
+embed.html
+webmanifest.json
+favicons
+ruffle

package.json

@@ -1,7 +1,7 @@
 {
   "name": "replaywebpage",
   "productName": "ReplayWeb.page",
-  "version": "2.0.0-beta.1",
+  "version": "2.0.0-beta.2",
   "description": "Serverless Web Archive Replay",
   "repository": "https://github.com/webrecorder/replayweb.page",
   "homepage": "https://replayweb.page/",
@@ -65,21 +65,22 @@
     "webpack-merge": "^5.10.0"
   },
   "files": [
-    "/assets",
     "/dist",
     "/src",
-    "site/ruffle",
+    "/ruffle",
-    "site/sw.js",
+    "/sw.js",
-    "site/ui.js",
+    "/ui.js",
-    "site/index.html"
+    "/index.html"
   ],
   "scripts": {
     "build": "webpack --mode production",
     "build-dev": "webpack --mode development",
-    "build-docs": "yarn run build && cd mkdocs && mkdocs build",
+    "build-docs": "yarn run build && cd mkdocs && ./copy-site.sh && mkdocs build",
-    "start-prod": "cd site && http-server -p 9990 --cors",
+    "publish-docs": "yarn run build && cd mkdocs && ./copy-site.sh && mkdocs gh-deploy --force",
+    "start-prod": "http-server -p 9990 --cors",
+    "start-docs-prod": "yarn run build-docs && cd mkdocs/_genhtml && http-server -p 9990 --cors",
     "start-dev": "webpack serve --mode development",
-    "start-docs": "yarn run build-dev && cd mkdocs && mkdocs serve",
+    "start-docs": "yarn run build-dev && cd mkdocs && ./copy-site.sh && mkdocs serve",
     "start-electron": "NODE_ENV=development electron ./dist/electron.js $1",
     "test-start-embed": "cd tests/embed; http-server -p 8020",
     "test-start-sandbox": "cd tests/embed/sandbox; http-server -p 8030",
@@ -106,11 +107,11 @@
       "dist/*.js",
       "dist/prebuilds/${platform}-${arch}/*",
       "dist/build/*",
-      "site/ui.js",
+      "ui.js",
-      "site/sw.js",
+      "sw.js",
-      "site/index.html",
+      "index.html",
-      "site/ruffle",
+      "ruffle",
-      "site/webmanifest.json"
+      "webmanifest.json"
     ],
     "dmg": {
       "title": "ReplayWeb.page"

playwright.config.js

@@ -4,7 +4,7 @@ export default defineConfig({
   reporter: process.env.CI ? "github" : "list",
   webServer: [
     {
-      command: "yarn run start-prod",
+      command: "yarn run start-docs-prod",
       url: "http://127.0.0.1:9990",
       timeout: 120 * 1000,
       reuseExistingServer: !process.env.CI,

src/embed.ts

@@ -118,8 +118,6 @@ class Embed extends LitElement {
       await this.swmanager.register();
       this.inited = true;
     } catch (e) {
-      // TODO: Fix this the next time the file is edited.
-      // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
       this.errorMessage = this.swmanager.renderErrorReport();
     }
   }

src/misc.ts

@@ -4,7 +4,7 @@ import { LitElement, html, css, unsafeCSS, type CSSResultGroup } from "lit";
 import { unsafeSVG } from "lit/directives/unsafe-svg.js";
 import { styleMap } from "lit/directives/style-map.js";
 
-import allCssRaw from "../assets/main.scss";
+import allCssRaw from "~assets/main.scss";
 
 import type { FavIconEventDetail } from "./types";
 

tests/embed/index-sandbox.html

@@ -1,19 +1,24 @@
 <!doctype html>
 <html>
-<head>
+  <head>
-<script src="http://localhost:9990/ui.js"></script>
+    <script src="http://localhost:9990/ui.js"></script>
-<style>
+    <style>
-html, body {
+      html,
-  background-color: lightgrey;
+      body {
-  height: 100%;
+        background-color: lightgrey;
-  width: 100%;
+        height: 100%;
-  overflow: hidden;
+        width: 100%;
-}
+        overflow: hidden;
-</style>
+      }
-</head>
+    </style>
-<body>
+  </head>
-
+  <body>
-<replay-web-page embed="replay" src="http://localhost:9990/docs/assets/tweet-example.wacz" url="page:0" sandbox requiresubdomainiframe></replay-web-page>
+    <replay-web-page
-
+      embed="replay"
-</body>
+      src="http://localhost:9990/docs/examples/tweet-example.wacz"
+      url="page:0"
+      sandbox
+      requiresubdomainiframe
+    ></replay-web-page>
+  </body>
 </html>

tests/embed/index.html

@@ -1,17 +1,19 @@
 <!doctype html>
 <html>
-<head>
+  <head>
-<script src="http://localhost:9990/ui.js"></script>
+    <script src="http://localhost:9990/ui.js"></script>
-<style>
+    <style>
-body {
+      body {
-  background-color: lightgrey;
+        background-color: lightgrey;
-  height: 600px;
+        height: 600px;
-}
+      }
-</style>
+    </style>
-</head>
+  </head>
-<body>
+  <body>
-
+    <replay-web-page
-<replay-web-page embed="replay-with-info" src="http://localhost:9990/docs/assets/tweet-example.wacz" url="page:0"></replay-web-page>
+      embed="replay-with-info"
-
+      src="http://localhost:9990/docs/examples/tweet-example.wacz"
-</body>
+      url="page:0"
+    ></replay-web-page>
+  </body>
 </html>

tests/embed/sandbox/index.html

@@ -1,22 +1,24 @@
 <!doctype html>
 <html>
-<head>
+  <head>
-<style>
+    <style>
-html, body {
+      html,
-  width: 100%;
+      body {
-  height: 100%;
+        width: 100%;
-}
+        height: 100%;
-iframe {
+      }
-  border: 1px black solid;
+      iframe {
-  display: flex;
+        border: 1px black solid;
-  width: 90%;
+        display: flex;
-  height: 600px;
+        width: 90%;
-}
+        height: 600px;
-</style>
+      }
-</head>
+    </style>
-<body>
+  </head>
-
+  <body>
-<iframe src="http://localhost:8020/index-sandbox.html" sandbox="allow-scripts allow-modals allow-forms allow-same-origin allow-downloads"></iframe>
+    <iframe
-
+      src="http://localhost:8020/index-sandbox.html"
-</body>
+      sandbox="allow-scripts allow-modals allow-forms allow-same-origin allow-downloads"
+    ></iframe>
+  </body>
 </html>

tests/embeds.spec.js

@@ -14,6 +14,20 @@ test("same-domain embed is loading", async ({ page }) => {
   await expect(res).toContainText("Want to help");
 });
 
+test("docs page embed is loading", async ({ page }) => {
+  await page.goto("http://localhost:9990/docs/embedding/");
+
+  const res = page
+    .locator("replay-web-page")
+    .frameLocator("iframe")
+    .locator("replay-app-main wr-item wr-coll-replay")
+    .frameLocator("iframe")
+    .frameLocator("iframe#twitter-widget-0")
+    .locator("body");
+
+  await expect(res).toContainText("Want to help");
+});
+
 test("cross-domain embed is loading", async ({ page }) => {
   await page.goto("http://localhost:8020/");
 

tsconfig.json

@@ -29,7 +29,7 @@
     "incremental": true,
     "paths": {
       "flexsearch": ["./node_modules/@types/flexsearch"],
-      "~assets/*": ["./src/assets/*"],
+      "~assets/*": ["./src/assets/*"]
     }
   },
   "include": ["src/**/*.ts", "src/**/*.js", "src/**/*.d.ts"]

webmanifest.json

@@ -3,7 +3,11 @@
   "short_name": "ReplayWeb.page",
   "description": "A tool to explore and replay interactive archived web pages directly in the browser",
   "icons": [
-    { "src": "/favicons/icon-192.png", "type": "image/png", "sizes": "192x192" },
+    {
+      "src": "/favicons/icon-192.png",
+      "type": "image/png",
+      "sizes": "192x192"
+    },
     { "src": "/favicons/icon-512.png", "type": "image/png", "sizes": "512x512" }
   ],
   "start_url": "/",

webpack.config.js

@@ -72,10 +72,7 @@ const electronMainConfig = (/*env, argv*/) => {
       }),
       new webpack.BannerPlugin(BANNER_TEXT),
       new CopyPlugin({
-        patterns: [
+        patterns: [{ from: "build/extra_prebuilds/", to: "prebuilds" }],
-          // { from: "node_modules/classic-level/prebuilds/", to: "prebuilds" },
-          { from: "build/extra_prebuilds/", to: "prebuilds" },
-        ],
       }),
     ],
   };
@@ -105,22 +102,17 @@ const browserConfig = (/*env, argv*/) => {
     ui: "./src/index.ts",
   };
 
-  //const patterns = [{ from: "package.json", to: "_data/package.json" }];
+  const extraPlugins = [];
-  let patterns = [{
-    from: "src/assets/favicons",
-    to: "site/favicons",
-  }];
 
   if (isDevServer) {
     entry["sw"] = "@webrecorder/wabac/src/sw.js";
   } else {
-    patterns.push({
+    const patterns = [
-      from: "node_modules/@webrecorder/wabac/dist/sw.js",
+      { from: "node_modules/@webrecorder/wabac/dist/sw.js", to: "sw.js" },
-      to: "site/sw.js",
+    ];
-    });
+    extraPlugins.push(new CopyPlugin({ patterns }));
   }
 
-
   /** @type {import('webpack').Configuration} */
   const config = {
     target: "web",
@@ -136,7 +128,7 @@ const browserConfig = (/*env, argv*/) => {
 
     output: {
       path: path.join(__dirname),
-      filename: "site/[name].js",
+      filename: "[name].js",
       libraryTarget: "self",
       globalObject: "self",
       publicPath: "/",
@@ -146,7 +138,7 @@ const browserConfig = (/*env, argv*/) => {
       compress: true,
       port: 9990,
       open: false,
-      static: path.join(__dirname, "site"),
+      static: __dirname,
       //publicPath: "/"
     },
 
@@ -173,7 +165,7 @@ const browserConfig = (/*env, argv*/) => {
         __VERSION__: JSON.stringify(package_json.version),
       }),
       new webpack.BannerPlugin(BANNER_TEXT),
-      new CopyPlugin({ patterns }),
+      ...extraPlugins,
     ],
 
     module: {