Skip to content

Commit 1607641

Browse files
committed
Fixed token, inconsistent discovery issue
1 parent c905532 commit 1607641

2 files changed

Lines changed: 26 additions & 5 deletions

File tree

.DS_Store

2 KB
Binary file not shown.

scripts/fetch_all_categories.py

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -937,16 +937,37 @@ async def main():
937937
("most-popular", fetch_most_popular),
938938
]
939939

940-
# Each category gets its own client with its own token (dedicated rate limit)
941-
for cat_name, cat_fn in categories:
942-
token = get_token(cat_name)
940+
# Detect shared tokens so we can split the budget fairly across categories
941+
tokens = [get_token(name) for name, _ in categories]
942+
num_categories = len(categories)
943+
shared_token = len(set(tokens)) < num_categories
944+
945+
if shared_token:
946+
print(f"\n⚠ Some categories share the same token — budget will be split evenly")
947+
print(f" TIP: Set GH_TOKEN_TRENDING, GH_TOKEN_NEW_RELEASES, GH_TOKEN_MOST_POPULAR "
948+
f"to 3 separate PATs for 3× the rate limit")
949+
950+
for cat_idx, (cat_name, cat_fn) in enumerate(categories):
951+
token = tokens[cat_idx]
952+
943953
async with GitHubClient(token) as client:
944-
# Check rate limit for this token
954+
# Check actual rate limit for this token
945955
data, _ = await client.get("https://api.github.com/rate_limit")
946956
if data:
947957
remaining = data.get("resources", {}).get("core", {}).get("remaining", 0)
948958
limit = data.get("resources", {}).get("core", {}).get("limit", 0)
949-
print(f"\n[{cat_name}] API budget: {remaining}/{limit} requests remaining")
959+
960+
if shared_token:
961+
# Token is shared — only use this category's fair share
962+
categories_left = num_categories - cat_idx
963+
category_budget = (remaining - RATE_LIMIT_FLOOR) // categories_left
964+
# Cap _rate_remaining so process_category divides only our share
965+
client._rate_remaining = category_budget + RATE_LIMIT_FLOOR
966+
print(f"\n[{cat_name}] API budget: {remaining}/{limit} remaining, "
967+
f"category share: ~{category_budget} (1/{categories_left} of shared token)")
968+
else:
969+
print(f"\n[{cat_name}] API budget: {remaining}/{limit} requests remaining (dedicated token)")
970+
950971
if remaining < 500:
951972
print(f"WARNING: Low rate limit for {cat_name}!", file=sys.stderr)
952973

0 commit comments

Comments
 (0)