Skip to content

Commit bd6de7b

Browse files
author
jhengy
committed
Fix infinite looping when handling rate limit to retry at most once per article
1 parent 4239f38 commit bd6de7b

File tree

1 file changed

+8
-6
lines changed

1 file changed

+8
-6
lines changed

src/content_aggregator/__init__.py

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -148,16 +148,18 @@ async def process_articles(self, articles: List[Article]) -> List[Article]:
148148

149149
print(f"✅ Successfully processed: {url}")
150150

151+
# TODO: exponential backoff in retry?
151152
except RateLimitExceededError as e:
152-
retry_after = e.retry_after
153-
if retry_after:
153+
if url not in already_retried:
154+
retry_after = e.retry_after or self.config['retry_delay_seconds']
154155
print(f"🔄 Retrying {url} in {retry_after} seconds")
155156
time.sleep(retry_after)
157+
158+
already_retried.add(url)
159+
articles.append(article)
156160
else:
157-
print(f"🔄 Retrying {url} in {self.config['retry_delay_seconds']} seconds")
158-
time.sleep(self.config['retry_delay_seconds'])
159-
already_retried.add(url)
160-
articles.append(article)
161+
print(f"⚠️ Already retried {url} - skipping")
162+
continue
161163

162164
except Exception as e:
163165
print(f"❌ Error processing {url} - {str(e)}")

0 commit comments

Comments
 (0)