fix(ui): search citation rendering issues (#2484)

* fix(ui): search citation invalid character

* citation popup and render inside li

* default favicon

* clean

* update

* key issue

ee/tabby-ui/app/search/components/search.tsx

@@ -9,9 +9,12 @@ import {
   useRef,
   useState
 } from 'react'
+import Image from 'next/image'
 import { useRouter } from 'next/navigation'
+import defaultFavicon from '@/assets/default-favicon.png'
 import { Message } from 'ai'
 import DOMPurify from 'dompurify'
+import he from 'he'
 import { marked } from 'marked'
 import { nanoid } from 'nanoid'
 import remarkGfm from 'remark-gfm'
@@ -520,10 +523,9 @@ function AnswerBlock({
           >
             {answer.relevant_documents.map((source, index) => (
               <SourceCard
-                key={source.link}
+                key={source.link + index}
                 source={source}
                 showMore={showMore}
-                index={index + 1}
               />
             ))}
           </div>
@@ -556,7 +558,6 @@ function AnswerBlock({
         {answer.isLoading && !answer.content && (
           <Skeleton className="mt-1 h-40 w-full" />
         )}
-
         <MessageMarkdown
           message={answer.content}
           sources={answer.relevant_documents}
@@ -614,29 +615,26 @@ function AnswerBlock({
   )
 }
 
+// Remove HTML and Markdown format
+const normalizedText = (input: string) => {
+  const sanitizedHtml = DOMPurify.sanitize(input, {
+    ALLOWED_TAGS: [],
+    ALLOWED_ATTR: []
+  })
+  const parsed = marked.parse(sanitizedHtml) as string
+  const decoded = he.decode(parsed)
+  const plainText = decoded.replace(/<\/?[^>]+(>|$)/g, '')
+  return plainText
+}
+
 function SourceCard({
   source,
-  index,
   showMore
 }: {
   source: Source
-  index: number
   showMore: boolean
 }) {
   const { hostname } = new URL(source.link)
-
-  // Remove HTML and Markdown format
-  const normalizedText = (input: string) => {
-    const sanitizedHtml = DOMPurify.sanitize(input, {
-      ALLOWED_TAGS: [],
-      ALLOWED_ATTR: []
-    })
-    const parsed = marked.parse(sanitizedHtml) as string
-    const plainText = parsed.replace(/<\/?[^>]+(>|$)/g, '')
-
-    return plainText
-  }
-
   return (
     <div
       className="flex cursor-pointer flex-col justify-between gap-y-1 rounded-lg border bg-card p-3 hover:bg-card/60"
@@ -685,6 +683,65 @@ function MessageMarkdown({
   headline?: boolean
   sources?: Source[]
 }) {
+  const renderTextWithCitation = (nodeStr: string, index: number) => {
+    const citationMatchRegex = /\[\[?citation:\s*\d+\]?\]/g
+    const textList = nodeStr.split(citationMatchRegex)
+    const citationList = nodeStr.match(citationMatchRegex)
+    return (
+      <span key={index}>
+        {textList.map((text, index) => {
+          const citation = citationList?.[index]
+          const citationNumberMatch = citation?.match(/\d+/)
+          const citationIndex = citationNumberMatch
+            ? parseInt(citationNumberMatch[0], 10)
+            : null
+          const source =
+            citationIndex !== null ? sources?.[citationIndex - 1] : null
+          const sourceUrl = source ? new URL(source.link) : null
+          return (
+            <span key={index}>
+              {text && <span>{text}</span>}
+              {source && (
+                <HoverCard>
+                  <HoverCardTrigger>
+                    <span
+                      className="relative -top-2 mr-0.5 inline-block h-4 w-4 cursor-pointer rounded-full bg-muted text-center text-xs"
+                      onClick={() => window.open(source.link)}
+                    >
+                      {citationIndex}
+                    </span>
+                  </HoverCardTrigger>
+                  <HoverCardContent className="w-96 text-sm">
+                    <div className="flex w-full flex-col gap-y-1">
+                      <div className="m-0 flex items-center space-x-1 text-xs leading-none text-muted-foreground">
+                        <SiteFavicon
+                          hostname={sourceUrl!.hostname}
+                          className="m-0 mr-1 leading-none"
+                        />
+                        <p className="m-0 leading-none">
+                          {sourceUrl!.hostname}
+                        </p>
+                      </div>
+                      <p
+                        className="m-0 cursor-pointer font-bold leading-none transition-opacity hover:opacity-70"
+                        onClick={() => window.open(source.link)}
+                      >
+                        {source.title}
+                      </p>
+                      <p className="m-0 line-clamp-4 leading-none">
+                        {normalizedText(source.snippet)}
+                      </p>
+                    </div>
+                  </HoverCardContent>
+                </HoverCard>
+              )}
+            </span>
+          )
+        })}
+      </span>
+    )
+  }
+
   return (
     <MemoizedReactMarkdown
       className="prose max-w-none break-words dark:prose-invert prose-p:leading-relaxed prose-pre:mt-1 prose-pre:p-0"
@@ -704,64 +761,7 @@ function MessageMarkdown({
               <div className="mb-2 inline-block leading-relaxed last:mb-0">
                 {children.map((childrenItem, index) => {
                   if (typeof childrenItem === 'string') {
-                    const citationMatchRegex = /\[\[?citation:\s*\d+\]?\]/g
-                    const textList = childrenItem.split(citationMatchRegex)
-                    const citationList = childrenItem.match(citationMatchRegex)
-                    return (
-                      <span key={index}>
-                        {textList.map((text, index) => {
-                          const citation = citationList?.[index]
-                          const citationNumberMatch = citation?.match(/\d+/)
-                          const citationIndex = citationNumberMatch
-                            ? parseInt(citationNumberMatch[0], 10)
-                            : null
-                          const source =
-                            citationIndex !== null
-                              ? sources?.[citationIndex - 1]
-                              : null
-                          const sourceUrl = source ? new URL(source.link) : null
-                          return (
-                            <span key={index}>
-                              {text && <span>{text}</span>}
-                              {source && (
-                                <HoverCard>
-                                  <HoverCardTrigger>
-                                    <span
-                                      className="relative -top-2 mr-0.5 inline-block h-4 w-4 cursor-pointer rounded-full bg-muted text-center text-xs"
-                                      onClick={() => window.open(source.link)}
-                                    >
-                                      {citationIndex}
-                                    </span>
-                                  </HoverCardTrigger>
-                                  <HoverCardContent className="w-96 text-sm">
-                                    <div className="flex w-full flex-col gap-y-1">
-                                      <div className="m-0 flex items-center space-x-1 text-xs leading-none text-muted-foreground">
-                                        <SiteFavicon
-                                          hostname={sourceUrl!.hostname}
-                                          className="m-0 mr-1 leading-none"
-                                        />
-                                        <p className="m-0 leading-none">
-                                          {sourceUrl!.hostname}
-                                        </p>
-                                      </div>
-                                      <p
-                                        className="m-0 cursor-pointer font-bold leading-none transition-opacity hover:opacity-70"
-                                        onClick={() => window.open(source.link)}
-                                      >
-                                        {source.title}
-                                      </p>
-                                      <p className="m-0 leading-none">
-                                        {source.snippet}
-                                      </p>
-                                    </div>
-                                  </HoverCardContent>
-                                </HoverCard>
-                              )}
-                            </span>
-                          )
-                        })}
-                      </span>
-                    )
+                    return renderTextWithCitation(childrenItem, index)
                   }
 
                   return <span key={index}>{childrenItem}</span>
@@ -772,6 +772,22 @@ function MessageMarkdown({
 
           return <p className="mb-2 last:mb-0">{children}</p>
         },
+        li({ children }) {
+          if (children && children.length) {
+            return (
+              <li>
+                {children.map((childrenItem, index) => {
+                  if (typeof childrenItem === 'string') {
+                    return renderTextWithCitation(childrenItem, index)
+                  }
+
+                  return <span key={index}>{childrenItem}</span>
+                })}
+              </li>
+            )
+          }
+          return <li>{children}</li>
+        },
         code({ node, inline, className, children, ...props }) {
           if (children.length) {
             if (children[0] == '▍') {
@@ -816,12 +832,39 @@ function SiteFavicon({
   hostname: string
   className?: string
 }) {
+  const [isLoaded, setIsLoaded] = useState(false)
+
+  const handleImageLoad = () => {
+    setIsLoaded(true)
+  }
+
   return (
-    <img
-      src={`https://s2.googleusercontent.com/s2/favicons?sz=128&domain_url=${hostname}`}
-      alt={hostname}
-      className={cn('h-3.5 w-3.5 rounded-full leading-none', className)}
-    />
+    <div className="relative h-3.5 w-3.5">
+      <Image
+        src={defaultFavicon}
+        alt={hostname}
+        width={14}
+        height={14}
+        className={cn(
+          'absolute left-0 top-0 z-0 h-3.5 w-3.5 rounded-full leading-none',
+          className
+        )}
+      />
+      <Image
+        src={`https://s2.googleusercontent.com/s2/favicons?sz=128&domain_url=${hostname}`}
+        alt={hostname}
+        width={14}
+        height={14}
+        className={cn(
+          'relative z-10 h-3.5 w-3.5 rounded-full bg-card leading-none',
+          className,
+          {
+            'opacity-0': !isLoaded
+          }
+        )}
+        onLoad={handleImageLoad}
+      />
+    </div>
   )
 }
 

ee/tabby-ui/package.json

@@ -119,6 +119,7 @@
     "@types/aos": "^3.0.7",
     "@types/color": "^3.0.6",
     "@types/dompurify": "^3.0.5",
+    "@types/he": "^1.2.3",
     "@types/humanize-duration": "^3.27.4",
     "@types/lodash-es": "^4.17.10",
     "@types/node": "^17.0.12",
@@ -136,10 +137,13 @@
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-tailwindcss": "^3.12.0",
     "eslint-plugin-unused-imports": "^3.0.0",
+    "from": "^0.1.7",
+    "he": "^1.2.0",
+    "import": "^0.0.6",
     "npm-run-all": "^4.1.5",
     "postcss": "^8.4.21",
     "prettier": "^2.7.1",
-    "tabby-openapi": "workspace:*",
+    "tabby-openapi": "workspace:0.12.0-dev",
     "tailwind-merge": "^1.12.0",
     "tailwindcss": "^3.3.1",
     "tailwindcss-animate": "^1.0.5",

pnpm-lock.yaml

@@ -695,6 +695,9 @@ importers:
       '@types/dompurify':
         specifier: ^3.0.5
         version: 3.0.5
+      '@types/he':
+        specifier: ^1.2.3
+        version: 1.2.3
       '@types/humanize-duration':
         specifier: ^3.27.4
         version: 3.27.4
@@ -746,6 +749,15 @@ importers:
       eslint-plugin-unused-imports:
         specifier: ^3.0.0
         version: 3.0.0(@typescript-eslint/eslint-plugin@7.4.0(@typescript-eslint/parser@5.62.0(eslint@8.50.0)(typescript@5.2.2))(eslint@8.50.0)(typescript@5.2.2))(eslint@8.50.0)
+      from:
+        specifier: ^0.1.7
+        version: 0.1.7
+      he:
+        specifier: ^1.2.0
+        version: 1.2.0
+      import:
+        specifier: ^0.0.6
+        version: 0.0.6
       npm-run-all:
         specifier: ^4.1.5
         version: 4.1.5
@@ -756,7 +768,7 @@ importers:
         specifier: ^2.7.1
         version: 2.8.8
       tabby-openapi:
-        specifier: workspace:*
+        specifier: workspace:0.12.0-dev
         version: link:../../clients/tabby-openapi
       tailwind-merge:
         specifier: ^1.12.0
@@ -3603,6 +3615,9 @@ packages:
   '@types/hast@3.0.4':
     resolution: {integrity: sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==}
 
+  '@types/he@1.2.3':
+    resolution: {integrity: sha512-q67/qwlxblDzEDvzHhVkwc1gzVWxaNxeyHUBF4xElrvjL11O+Ytze+1fGpBHlr/H9myiBUaUXNnNPmBHxxfAcA==}
+
   '@types/humanize-duration@3.27.4':
     resolution: {integrity: sha512-yaf7kan2Sq0goxpbcwTQ+8E9RP6HutFBPv74T/IA/ojcHKhuKVlk2YFYyHhWZeLvZPzzLE3aatuQB4h0iqyyUA==}
 
@@ -6424,6 +6439,11 @@ packages:
   import-meta-resolve@3.1.1:
     resolution: {integrity: sha512-qeywsE/KC3w9Fd2ORrRDUw6nS/nLwZpXgfrOc2IILvZYnCaEMd+D56Vfg9k4G29gIeVi3XKql1RQatME8iYsiw==}
 
+  import@0.0.6:
+    resolution: {integrity: sha512-QPhTdjy9J4wUzmWSG7APkSgMFuPGPw+iJTYUblcfc2AfpqaatbwgCldK1HoLYx+v/+lWvab63GWZtNkcnj9JcQ==}
+    engines: {node: '>=0.10.0'}
+    hasBin: true
+
   imurmurhash@0.1.4:
     resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==}
     engines: {node: '>=0.8.19'}
@@ -7778,6 +7798,9 @@ packages:
     resolution: {integrity: sha512-c/hfooPx+RBIOPM09GSxABOZhYPblDoyaGhqBkD/59vtpN21jEuWKDlM0KYTvqJVlSYjKs0tBcIdeXKChlSPtw==}
     hasBin: true
 
+  optimist@0.3.7:
+    resolution: {integrity: sha512-TCx0dXQzVtSCg2OgY/bO9hjM9cV4XYx09TVK+s3+FhkjT6LovsLe+pPMzpWf+6yXK/hUizs2gUoTw3jHM0VaTQ==}
+
   optionator@0.9.3:
     resolution: {integrity: sha512-JjCoypp+jKn1ttEFExxhetCKeJt9zhAgAve5FXHixTvFDW/5aEktX9bufBKLRRMdU7bNtpLfcGu94B3cdEJgjg==}
     engines: {node: '>= 0.8.0'}
@@ -10086,6 +10109,10 @@ packages:
     resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==}
     engines: {node: '>=0.10.0'}
 
+  wordwrap@0.0.3:
+    resolution: {integrity: sha512-1tMA907+V4QmxV7dbRvb4/8MaRALK6q9Abid3ndMYnbyo8piisCmeONVqVSXqQA3KaP4SLt5b7ud6E2sqP8TFw==}
+    engines: {node: '>=0.4.0'}
+
   workerpool@6.2.1:
     resolution: {integrity: sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw==}
 
@@ -13623,6 +13650,8 @@ snapshots:
     dependencies:
       '@types/unist': 3.0.2
 
+  '@types/he@1.2.3': {}
+
   '@types/humanize-duration@3.27.4': {}
 
   '@types/js-yaml@4.0.9': {}
@@ -17369,6 +17398,10 @@ snapshots:
 
   import-meta-resolve@3.1.1: {}
 
+  import@0.0.6:
+    dependencies:
+      optimist: 0.3.7
+
   imurmurhash@0.1.4: {}
 
   indent-string@4.0.0: {}
@@ -18972,6 +19005,10 @@ snapshots:
       undici: 5.28.4
       yargs-parser: 21.1.1
 
+  optimist@0.3.7:
+    dependencies:
+      wordwrap: 0.0.3
+
   optionator@0.9.3:
     dependencies:
       '@aashutoshrathi/word-wrap': 1.2.6
@@ -21788,6 +21825,8 @@ snapshots:
 
   word-wrap@1.2.5: {}
 
+  wordwrap@0.0.3: {}
+
   workerpool@6.2.1: {}
 
   wrap-ansi@6.2.0: