ndjson parsing: ingore invalid JSON lines (#544)

use parse function from https://github.com/mash/fetch-ndjson but add
try/catch to ignore pages with invalid JSON

package.json

@@ -26,7 +26,6 @@
     "bulma": "^0.9.3",
     "electron-log": "^4.4.1",
     "electron-updater": "^6.8.9",
-    "fetch-ndjson": "^1.1.6",
     "flexsearch": "^0.7.31",
     "keyword-mark-element": "^0.2.0",
     "lit": "3.1.1",

src/pages.ts

@@ -11,7 +11,6 @@ import {
 import { property, state } from "lit/decorators.js";
 import { ifDefined } from "lit/directives/if-defined.js";
 import { wrapCss, clickOnSpacebarPress } from "./misc";
-import ndjson from "fetch-ndjson";
 
 import { Index as FlexIndex } from "flexsearch";
 import prettyBytes from "pretty-bytes";
@@ -414,6 +413,51 @@ class Pages extends LitElement {
     );
   }
 
+  // Adapted from: https://github.com/mash/fetch-ndjson but with try/catch to ignore invalid JSON lines
+  // Commit: https://github.com/mash/fetch-ndjson/commit/c82586bfa51492edd88cc4104c9e00c5a568e307, MIT License
+  async *ndjson(
+    reader: ReadableStreamDefaultReader<Uint8Array>,
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  ): AsyncGenerator<any, void> {
+    const matcher = /\r?\n/;
+    const decoder = new TextDecoder();
+    let buf = "";
+
+    let next = reader.read();
+    while (true) {
+      const { done, value } = await next;
+
+      if (done) {
+        if (buf.length > 0) {
+          try {
+            yield JSON.parse(buf);
+          } catch (e) {
+            console.warn("Ignoring page with invalid JSON", buf);
+          }
+        }
+        return;
+      }
+
+      const chunk = decoder.decode(value, { stream: true });
+      // debug(`chunk=${chunk}`);
+      buf += chunk;
+
+      const parts = buf.split(matcher);
+      buf = parts.pop()!;
+      for (const i of parts) {
+        if (!i.match(/^\s*$/)) {
+          try {
+            yield JSON.parse(i);
+          } catch (e) {
+            console.warn("Ignoring page with invalid JSON", buf);
+          }
+        }
+      }
+
+      next = reader.read();
+    }
+  }
+
   async updateTextSearch() {
     if (this.updatingSearch) {
       return;
@@ -440,7 +484,7 @@ class Pages extends LitElement {
 
       const lines: Page[] = [];
 
-      for await (const line of ndjson(resp.body!.getReader())) {
+      for await (const line of this.ndjson(resp.body!.getReader())) {
         if (!line.url) {
           continue;
         }

yarn.lock

@@ -3371,13 +3371,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"fetch-ndjson@npm:^1.1.6":
-  version: 1.1.6
-  resolution: "fetch-ndjson@npm:1.1.6"
-  checksum: 10c0/d05b1df13295cf18560e90c028d105f01936a66019a5789b83c3c7f5437870e7f09ad65430cc8f9a84af788e5d3406190e0ab03a4f651bf1e4aa840c18ce5927
-  languageName: node
-  linkType: hard
-
 "file-entry-cache@npm:^6.0.1":
   version: 6.0.1
   resolution: "file-entry-cache@npm:6.0.1"
@@ -6206,7 +6199,6 @@ __metadata:
     eslint-config-prettier: "npm:^9.1.0"
     eslint-plugin-lit: "npm:^1.11.0"
     eslint-plugin-wc: "npm:^2.0.4"
-    fetch-ndjson: "npm:^1.1.6"
     flexsearch: "npm:^0.7.31"
     husky: "npm:^8.0.0"
     keyword-mark-element: "npm:^0.2.0"