Skip to content

Commit 236d4e8

Browse files
kanwarujjavalkanwarujjaval
committed
Introduce index-aware orchestration and progress controls
- Added index initialization and status tracking in Collection Orchestrator. - Improved scheduling logic to wait for index readiness before processing collections. - Enhanced migration dashboard with index build progress and theme toggle. - Updated schema defaults for larger batch sizes and refined batch runner states.
1 parent 15dd295 commit 236d4e8

6 files changed

Lines changed: 320 additions & 65 deletions

File tree

‎src/config/schema.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,9 @@ export const configSchema = z.object({
5555
readConcern: z.string().default("majority"),
5656
retryReads: booleanFromEnv.default(true),
5757
appName: z.string().optional(),
58-
batchRowsTarget: positiveIntFromEnv.default(10_000),
58+
batchRowsTarget: positiveIntFromEnv.default(50_000),
5959
mongoPageSize: positiveIntFromEnv.default(2_000),
60-
cursorBatchSize: positiveIntFromEnv.default(2_000),
60+
cursorBatchSize: positiveIntFromEnv.default(10_000),
6161
maxTimeMs: positiveIntFromEnv.default(600_000),
6262
}),
6363

‎src/http/stats-route.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ export function registerStatsRoute(app: FastifyInstance, deps: StatsDeps): void
7676
const runnerStatus = orchestrator.getStatus();
7777
const currentBatchSeq = orchestrator.getCurrentBatchSeq();
7878
const estimatedCounts = orchestrator.getEstimatedCounts();
79+
const indexStatus = orchestrator.getIndexStatus();
7980

8081
// Find the current run ID from the orchestrator's progress
8182
const currentResult = progress.results.find(
@@ -173,6 +174,8 @@ export function registerStatsRoute(app: FastifyInstance, deps: StatsDeps): void
173174
? `${((totalDocsSkipped / currentCollDocsRead) * 100).toFixed(1)}%` : '0%',
174175
} : null,
175176

177+
indexStatus,
178+
176179
service: {
177180
name: config.service.name,
178181
version,

‎src/http/viz-route.ts‎

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,16 @@ const DASHBOARD_HTML = /* html */ `<!DOCTYPE html>
5656
.status-failed { background: rgba(239,68,68,0.15); color: var(--red); }
5757
.status-stopped { background: rgba(234,179,8,0.15); color: var(--yellow); }
5858
.status-idle { background: rgba(92,96,116,0.15); color: var(--text3); }
59+
.status-waiting_for_index { background: rgba(249,115,22,0.15); color: var(--orange); }
60+
61+
/* Light theme */
62+
body.light {
63+
--bg: #f5f5f7; --bg2: #ffffff; --bg3: #e8e8ed; --border: #d1d1d6;
64+
--text: #1d1d1f; --text2: #6e6e73; --text3: #aeaeb2;
65+
--bar-bg: #e8e8ed;
66+
}
67+
body.light .btn { background: #f0f0f5; }
68+
body.light .btn:hover:not(:disabled) { background: #e0e0e5; }
5969
6070
/* Progress bars */
6171
.progress-outer { background: var(--bar-bg); border-radius: 6px; height: 24px; overflow: hidden; position: relative; }
@@ -145,6 +155,7 @@ const DASHBOARD_HTML = /* html */ `<!DOCTYPE html>
145155
<span id="svc-host" class="muted"></span>
146156
<span id="svc-uptime"></span>
147157
<span>v<span id="svc-version">-</span></span>
158+
<button class="btn" id="btn-theme" style="padding:4px 8px;font-size:11px">Theme</button>
148159
<div class="heartbeat" id="heartbeat"></div>
149160
</div>
150161
</div>
@@ -251,6 +262,12 @@ const DASHBOARD_HTML = /* html */ `<!DOCTYPE html>
251262
</div>
252263
</div>
253264
265+
<!-- Index Status -->
266+
<div class="card full" style="margin-top:12px">
267+
<h3>Index Status <span id="idx-summary" style="font-weight:400;color:var(--text2)"></span></h3>
268+
<div id="idx-details" style="font-size:12px"></div>
269+
</div>
270+
254271
<!-- Collections Table -->
255272
<div class="card full" style="margin-top:12px">
256273
<h3>Collections <span id="coll-summary" style="font-weight:400;color:var(--text2)"></span></h3>
@@ -278,7 +295,15 @@ const DASHBOARD_HTML = /* html */ `<!DOCTYPE html>
278295
function $(id) { return document.getElementById(id); }
279296
function fmt(n) { return n == null ? '-' : Number(n).toLocaleString('en-US'); }
280297
function mb(b) { return b ? (b / 1024 / 1024).toFixed(0) + ' MB' : '-'; }
281-
function esc(s) { var d = document.createElement('div'); d.textContent = s; return d.textContent; }
298+
299+
// Theme toggle
300+
var theme = localStorage.getItem('viz-theme') || 'dark';
301+
if (theme === 'light') document.body.classList.add('light');
302+
$('btn-theme').addEventListener('click', function() {
303+
document.body.classList.toggle('light');
304+
theme = document.body.classList.contains('light') ? 'light' : 'dark';
305+
localStorage.setItem('viz-theme', theme);
306+
});
282307
283308
// Tab switching
284309
$('tab-bar').addEventListener('click', function(e) {
@@ -299,6 +324,7 @@ const DASHBOARD_HTML = /* html */ `<!DOCTYPE html>
299324
300325
function statusClass(s) {
301326
if (s === 'running' || s === 'stopping') return 'status-running';
327+
if (s === 'waiting_for_index') return 'status-waiting_for_index';
302328
if (s === 'paused') return 'status-paused';
303329
if (s === 'completed') return 'status-completed';
304330
if (s === 'failed') return 'status-failed';
@@ -575,6 +601,39 @@ const DASHBOARD_HTML = /* html */ `<!DOCTYPE html>
575601
['Redis Error', redis.lastError || 'none', redis.lastError ? 'var(--red)' : 'var(--text3)'],
576602
]);
577603
604+
// Index status
605+
var idx = d.indexStatus || { ready: 0, building: 0, failed: 0, details: [] };
606+
$('idx-summary').textContent = '(' + idx.ready + ' ready, ' + idx.building + ' building' + (idx.failed > 0 ? ', ' + idx.failed + ' failed' : '') + ')';
607+
var idxContainer = $('idx-details');
608+
while (idxContainer.firstChild) idxContainer.removeChild(idxContainer.firstChild);
609+
if (idx.building === 0 && idx.failed === 0) {
610+
var allReady = document.createElement('span');
611+
allReady.style.color = 'var(--green)';
612+
allReady.textContent = 'All indexes ready';
613+
idxContainer.appendChild(allReady);
614+
} else {
615+
for (var ii = 0; ii < idx.details.length; ii++) {
616+
var det = idx.details[ii];
617+
var detRow = document.createElement('div');
618+
detRow.style.cssText = 'display:flex;justify-content:space-between;padding:2px 0';
619+
var detName = document.createElement('span');
620+
detName.style.cssText = 'max-width:60%;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;font-size:11px';
621+
detName.textContent = det.collection.length > 50 ? det.collection.slice(0, 20) + '...' + det.collection.slice(-8) : det.collection;
622+
detRow.appendChild(detName);
623+
var detStatus = document.createElement('span');
624+
detStatus.style.fontWeight = '600';
625+
if (det.status === 'building') {
626+
detStatus.style.color = 'var(--orange)';
627+
detStatus.textContent = 'building' + (det.elapsedSec ? ' (' + Math.round(det.elapsedSec / 60) + 'm)' : '');
628+
} else {
629+
detStatus.style.color = 'var(--red)';
630+
detStatus.textContent = 'failed';
631+
}
632+
detRow.appendChild(detStatus);
633+
idxContainer.appendChild(detRow);
634+
}
635+
}
636+
578637
// Collections
579638
var orch = d.orchestrator || {};
580639
$('coll-summary').textContent = '(' + (orch.totalCollections || 0) + ' total: '

‎src/runtime/batch-runner.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ export interface BatchRunnerDeps {
5050
export type RunnerStatus =
5151
| "idle"
5252
| "running"
53+
| "waiting_for_index"
5354
| "paused"
5455
| "stopping"
5556
| "stopped"

0 commit comments

Comments
 (0)