mirrors/tabby
1
0
Fork 0
mirror of https://github.com/TabbyML/tabby synced 2024-11-21 16:03:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(index): skip web documents if body is empty (#2831)

Browse Source 
* chore(index): skip web documents if body is empty

* Update .changes/unreleased/Fixed and Improvements-20240811-124728.yaml
This commit is contained in:
Meng Zhang 2024-08-11 12:58:24 -07:00 committed by GitHub
parent 4d3f889c92
commit 6c60ca4a17
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 10 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.changes/unreleased/Fixed and Improvements-20240811-124728.yaml Normal file
View File

@ -0,0 +1,3 @@
kind: Fixed and Improvements
body: Skip web documents if body is empty
time: 2024-08-11T12:47:28.678694-07:00

11
crates/tabby-index/src/doc/public.rs
View File

@ -47,15 +47,18 @@ impl DocIndexer {
};
stream! {
let is_document_empty = document.body.trim().is_empty();
let (id, s) = self.builder.build(document).await;
self.indexer.delete(&id);
for await doc in s.buffer_unordered(std::cmp::max(std::thread::available_parallelism().unwrap().get() * 2, 32)) {
if let Ok(Some(doc)) = doc {
self.indexer.add(doc).await;
if !is_document_empty {
for await doc in s.buffer_unordered(std::cmp::max(std::thread::available_parallelism().unwrap().get() * 2, 32)) {
if let Ok(Some(doc)) = doc {
self.indexer.add(doc).await;
}
}
}
}.count().await;
true
}