handle HTML markdown tokens better when rendering with svelte (#638)

modules/desktop/src/routes/packages/[slug]/+page.svelte

@@ -36,7 +36,7 @@
     readme?.data !== "" && {
       label: $t("common.details"),
       component: Markdown,
-      props: { pkg, source: readme, hook: useDefaultBrowser }
+      props: { source: readme, hook: useDefaultBrowser }
     },
     bottles?.length && {
       label: `${$t("common.versions")} (${versions.length || 0})`,

modules/ui/package.json

@@ -58,6 +58,7 @@
     "@types/he": "^1.2.0",
     "@types/prismjs": "^1.26.0",
     "he": "^1.2.0",
+    "marked": "^5.0.4",
     "prismjs": "^1.29.0",
     "restructured": "0.0.11",
     "svelte-markdown": "^0.2.3",

modules/ui/src/markdown/markdown.svelte

@@ -4,6 +4,7 @@
   import rst2html from "./rst2html";
   import "./styles.css";
   import { onMount } from "svelte";
+  import { tokenizeMarkdown } from "./md";
 
   export let source: { data: string; type: "md" | "rst" };
 
@@ -46,7 +47,7 @@
 <section use:hook class="markdown-body py-4">
   {#if source.type === "md"}
     <div bind:this={markDownRoot}>
-      <SvelteMarkdown source={source.data} {renderers} />
+      <SvelteMarkdown source={tokenizeMarkdown(source.data)} {renderers} />
     </div>
   {:else if source.type === "rst"}
     {@html html}

modules/ui/src/markdown/md.ts

@@ -0,0 +1,69 @@
+import { marked } from "marked";
+
+const defaultOptions = {
+  breaks: false,
+  gfm: true,
+  headerIds: true,
+  headerPrefix: "",
+  langPrefix: "language-",
+  mangle: false,
+  pedantic: false,
+  sanitize: false,
+  silent: false,
+  smartLists: false,
+  smartypants: false,
+  xhtml: false
+};
+
+export const tokenizeMarkdown = (data: string) => {
+  const tokens = marked.lexer(data, defaultOptions);
+
+  try {
+    const newTokens = preprocessHtmlTokens([...tokens]);
+    return newTokens;
+  } catch (err) {
+    console.error("Failed to preprocess markdown html tokens", err);
+    return tokens;
+  }
+};
+
+const preprocessHtmlTokens = (tokens: marked.Token[]): marked.Token[] => {
+  const processedTokens: marked.Token[] = [];
+  let htmlTokens: marked.Tokens.HTML[] = [];
+
+  // collapse all contiguous sibling html tokens.
+  // fortunately html tokens cannot have child tokens so combining them is easier
+  tokens.forEach((token: marked.Token) => {
+    if (token.type === "html") {
+      htmlTokens.push(token as marked.Tokens.HTML);
+    } else {
+      if (htmlTokens.length > 0) {
+        processedTokens.push(combineHtmlTokens(htmlTokens));
+        htmlTokens = [];
+      }
+      processedTokens.push(token);
+    }
+  });
+
+  if (htmlTokens.length > 0) {
+    processedTokens.push(combineHtmlTokens(htmlTokens));
+    htmlTokens = [];
+  }
+
+  // after the list of tokens has been collapsed, delve into any nodes with children and preprocess them as well
+  processedTokens.forEach((token: marked.Token) => {
+    if ("tokens" in token && token.tokens?.length) {
+      token.tokens = preprocessHtmlTokens(token.tokens);
+    }
+  });
+
+  return processedTokens;
+};
+
+const combineHtmlTokens = (htmlTokens: marked.Tokens.HTML[]) => {
+  return htmlTokens.reduce((acc, token) => {
+    const raw = acc.raw + token.raw;
+    const text = acc.text + token.text;
+    return { ...acc, raw, text };
+  });
+};

pnpm-lock.yaml

@@ -225,6 +225,7 @@ importers:
       eslint-plugin-storybook: ^0.6.7
       eslint-plugin-svelte3: ^4.0.0
       he: ^1.2.0
+      marked: ^5.0.4
       postcss: ^8.4.19
       prettier: ^2.8.8
       prettier-plugin-svelte: ^2.10.0
@@ -247,6 +248,7 @@ importers:
       '@types/he': 1.2.0
       '@types/prismjs': 1.26.0
       he: 1.2.0
+      marked: 5.0.4
       prismjs: 1.29.0
       restructured: 0.0.11
       svelte-markdown: 0.2.3_svelte@3.59.1
@@ -9225,6 +9227,12 @@ packages:
     hasBin: true
     dev: false
 
+  /marked/5.0.4:
+    resolution: {integrity: sha512-r0W8/DK56fAkV0qfUCO9cEt/VlFWUzoJOqEigvijmsVkTuPOHckh7ZutNJepRO1AxHhK96/9txonHg4bWd/aLA==}
+    engines: {node: '>= 18'}
+    hasBin: true
+    dev: false
+
   /marky/1.2.5:
     resolution: {integrity: sha512-q9JtQJKjpsVxCRVgQ+WapguSbKC3SQ5HEzFGPAJMStgh3QjCawp00UKv3MTTAArTmGmmPUvllHZoNbZ3gs0I+Q==}
     dev: true