From 894a4c6c7c695cc1bf178debb0ce2d043a4cd5c2 Mon Sep 17 00:00:00 2001
From: Meng Zhang <meng@tabbyml.com>
Date: Fri, 28 Jun 2024 15:26:17 +0800
Subject: [PATCH] chore: handle 429 Too Many Requests status code in crawl_url
 function (#2539)

---
 crates/tabby-scheduler/src/crawl/mod.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/crates/tabby-scheduler/src/crawl/mod.rs b/crates/tabby-scheduler/src/crawl/mod.rs
index 9ae3f0537..8f1b0df16 100644
--- a/crates/tabby-scheduler/src/crawl/mod.rs
+++ b/crates/tabby-scheduler/src/crawl/mod.rs
@@ -54,6 +54,10 @@ async fn crawl_url(start_url: &str) -> anyhow::Result<impl Stream<Item = KatanaR
                 }
             };
 
+            if data.response.status_code == Some(429) {
+                logkit::warn!("429 Too Many Requests, consider adjust your rate limit settings...");
+            }
+
             // Skip if the status code is not 200
             if data.response.status_code != Some(200) {
                 continue;