feat: delay auto onboarding to weekend and prevent them during weekdays [CM-1011] (#3886)

Author: mbani01

services/apps/git_integration/src/crowdgit/database/crud.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import datetime, timezone
 
 from loguru import logger
 from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed
@@ -151,6 +151,7 @@ async def acquire_recurrent_repo() -> Repository | None:
         RepositoryState.PENDING,
         RepositoryState.PROCESSING,
         RepositoryState.STUCK,
+        RepositoryState.PENDING_REONBOARD,
     )
     return await acquire_repository(
         recurrent_repo_sql_query,
@@ -173,6 +174,39 @@ async def can_onboard_more():
         return False  # if query failed mostly due to timeout then db is already under high load
 
 
+async def acquire_pending_reonboard_repo() -> Repository | None:
+    """Acquire a pending_reonboard repo for re-onboarding (only called on weekends)."""
+    pending_reonboard_sql_query = f"""
+    WITH selected_repo AS (
+        SELECT r.id
+        FROM public.repositories r
+        JOIN git."repositoryProcessing" rp ON rp."repositoryId" = r.id
+        WHERE rp.state = $1
+            AND rp."lockedAt" IS NULL
+            AND r."deletedAt" IS NULL
+        ORDER BY rp.priority ASC, rp."lastProcessedAt" ASC
+        LIMIT 1
+        FOR UPDATE OF rp SKIP LOCKED
+    )
+    UPDATE git."repositoryProcessing" rp
+    SET "lockedAt" = NOW(),
+        state = $2,
+        "lastProcessedCommit" = NULL,
+        branch = NULL,
+        "reOnboardingCount" = rp."reOnboardingCount" + 1,
+        "updatedAt" = NOW()
+    FROM public.repositories r
+    CROSS JOIN selected_repo
+    WHERE rp."repositoryId" = r.id
+        AND rp."repositoryId" = selected_repo.id
+    RETURNING {REPO_SELECT_COLUMNS}
+    """
+    return await acquire_repository(
+        pending_reonboard_sql_query,
+        (RepositoryState.PENDING_REONBOARD, RepositoryState.PROCESSING),
+    )
+
+
 async def acquire_repo_for_processing() -> Repository | None:
     """
     Acquire the next repository to process based on priority and system load.
@@ -182,6 +216,8 @@ async def acquire_repo_for_processing() -> Repository | None:
        current onboarding count is below MAX_CONCURRENT_ONBOARDINGS
     2. Recurrent repos (non-PENDING/non-PROCESSING) - fallback when onboarding
        is unavailable or skipped due to high load
+    3. Pending reonboard repos (PENDING_REONBOARD state) - weekend-only, lowest priority.
+       These are repos needing re-onboarding that were deferred until the weekend.
 
     Onboarding is delayed when integration.results exceeds MAX_INTEGRATION_RESULTS
     to prevent overloading the system during high activity periods.
@@ -192,6 +228,11 @@ async def acquire_repo_for_processing() -> Repository | None:
     else:
         logger.info("Skipping onboarding due to high load on integration.results")
 
+    if not repo_to_process:
+        is_weekend = datetime.now(timezone.utc).weekday() >= 5
+        if is_weekend:
+            repo_to_process = await acquire_pending_reonboard_repo()
+
     if not repo_to_process:
         repo_to_process = await acquire_recurrent_repo()
 
@@ -227,16 +268,6 @@ async def update_last_processed_commit(repo_id: str, commit_hash: str, branch: s
     return str(result)
 
 
-async def increase_re_onboarding_count(repo_id: str):
-    sql_query = """
-    UPDATE git."repositoryProcessing"
-        SET "reOnboardingCount" = "reOnboardingCount" + 1,
-            "updatedAt" = NOW()
-    WHERE "repositoryId" = $1
-    """
-    return await execute(sql_query, (repo_id,))
-
-
 async def mark_repo_as_processed(repo_id: str, repo_state: RepositoryState):
     sql_query = """
     UPDATE git."repositoryProcessing"

services/apps/git_integration/src/crowdgit/enums.py

@@ -33,6 +33,7 @@ class RepositoryState(str, Enum):
     FAILED = "failed"
     REQUIRES_PARENT = "requires_parent"  # fork repo without valid parent repo in out system
     STUCK = "stuck"  # requires manual resolution
+    PENDING_REONBOARD = "pending_reonboard"  # re-onboarding deferred until weekend
 
 
 class RepositoryPriority(int):

services/apps/git_integration/src/crowdgit/worker/repository_worker.py

@@ -4,7 +4,6 @@
 from crowdgit.database.crud import (
     acquire_repo_for_processing,
     get_recently_processed_repository_by_url,
-    increase_re_onboarding_count,
     mark_repo_as_processed,
     release_repo,
     update_last_processed_commit,
@@ -256,14 +255,8 @@ async def _process_single_repository(self, repository: Repository):
             )
             processing_state = RepositoryState.STUCK
         except ReOnboardingRequiredError:
-            logger.info(f"Resetting and queueing {repository.url} for re-onboarding")
-            await update_last_processed_commit(
-                repo_id=repository.id,
-                commit_hash=None,
-                branch=None,
-            )
-            processing_state = RepositoryState.PENDING
-            await increase_re_onboarding_count(repository.id)
+            logger.info(f"Repo {repository.url} needs re-onboarding, deferring until weekend")
+            processing_state = RepositoryState.PENDING_REONBOARD
         except ParentRepoInvalidError as e:
             logger.error(f"Parent repo validation failed: {repr(e)}")
             processing_state = RepositoryState.REQUIRES_PARENT