fix(bq): split BigQuery modules.sql when exceeding 1MB query limit (#599)

Author: vdelacruzb

clouds/bigquery/Makefile

@@ -61,7 +61,7 @@ endif
 build-modules:
 	mkdir -p $(BUILD_DIR)
 	$(MAKE) -C modules build
-	cp modules/build/modules.sql $(BUILD_DIR)
+	cp modules/build/modules*.sql $(BUILD_DIR)
 
 deploy:
 	$(MAKE) deploy-libraries
@@ -105,7 +105,7 @@ create-package:
 
 	rm -rf $(DIST_DIR)
 	mkdir -p $(DIST_DIR)/$(PACKAGE_NAME)
-	cp $(BUILD_DIR)/modules.sql $(DIST_DIR)/$(PACKAGE_NAME)/
+	cp $(BUILD_DIR)/modules*.sql $(DIST_DIR)/$(PACKAGE_NAME)/
 	cp -r $(BUILD_DIR)/libs $(DIST_DIR)/$(PACKAGE_NAME)/libs
 
 	$(MAKE) extra-package

clouds/bigquery/common/build_modules.js

@@ -151,13 +151,9 @@ function add (f, include) {
 functions.forEach(f => add(f));
 
 // Replace environment variables
-let separator;
-if (argv.production) {
-    separator = '\n';
-} else {
-    separator = '\n-->\n';  // marker to future SQL split
-}
-let content = output.map(f => f.content).join(separator);
+const internalSeparator = '\n-->\n';  // marker to split SQL statements
+const outputSeparator = argv.production ? '\n' : internalSeparator;
+let content = output.map(f => f.content).join(internalSeparator);
 
 function apply_replacements (text) {
     const libraries = [... new Set(content.match(new RegExp('@@BQ_LIBRARY_[^@]*?_BUCKET@@', 'g')))];
@@ -190,14 +186,54 @@ function apply_replacements (text) {
 
 if (argv.dropfirst) {
     const header = fs.readFileSync(path.resolve(__dirname, 'DROP_FUNCTIONS.sql')).toString();
-    content = header + separator + content
+    content = header + internalSeparator + content
 }
 
 const footer = fs.readFileSync(path.resolve(__dirname, 'VERSION.sql')).toString();
-content += separator + footer;
+content += internalSeparator + footer;
 
 content = apply_replacements(content);
 
-// Write modules.sql file
-fs.writeFileSync(path.join(outputDir, 'modules.sql'), content);
-console.log(`Write ${outputDir}/modules.sql`);
+// Split into individual statements and write output files,
+// chunking into multiple files if the BigQuery query size limit is exceeded
+// BigQuery hard limit is 1,024,000 characters per query.
+// Using 768KB to keep files well under the limit and reduce future splits.
+const BQ_QUERY_CHAR_LIMIT = 1024 * 1000; // 1,024,000 characters
+const SAFE_LIMIT = 768 * 1000; // 768,000 characters
+
+const statements = content.split(internalSeparator).filter(q => q.trim());
+const singleContent = statements.join(outputSeparator);
+
+if (singleContent.length <= SAFE_LIMIT) {
+    fs.writeFileSync(path.join(outputDir, 'modules.sql'), singleContent);
+    console.log(`Write ${outputDir}/modules.sql`);
+} else {
+    const chunks = [];
+    let currentStatements = [];
+    let currentSize = 0;
+
+    for (const stmt of statements) {
+        if (stmt.length > BQ_QUERY_CHAR_LIMIT) {
+            console.log(`ERROR: Single statement exceeds BigQuery limit (${stmt.length} chars)`);
+            process.exit(1);
+        }
+        const addedSize = stmt.length + outputSeparator.length;
+        if (currentSize + addedSize > SAFE_LIMIT && currentStatements.length > 0) {
+            chunks.push(currentStatements.join(outputSeparator));
+            currentStatements = [stmt];
+            currentSize = addedSize;
+        } else {
+            currentStatements.push(stmt);
+            currentSize += addedSize;
+        }
+    }
+    if (currentStatements.length > 0) {
+        chunks.push(currentStatements.join(outputSeparator));
+    }
+
+    for (let i = 0; i < chunks.length; i++) {
+        const filename = `modules_${String(i + 1).padStart(2, '0')}.sql`;
+        fs.writeFileSync(path.join(outputDir, filename), chunks[i]);
+        console.log(`Write ${outputDir}/${filename}`);
+    }
+}

clouds/bigquery/modules/Makefile

@@ -58,8 +58,10 @@ build: $(NODE_MODULES_DEV)
 deploy: check build
 	echo "Deploying modules..."
 	$(MAKE) dataset-create
-	GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) \
-	$(COMMON_DIR)/run-script.js $(BUILD_DIR)/modules.sql
+	for f in $(BUILD_DIR)/modules*.sql; do \
+		GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) \
+		$(COMMON_DIR)/run-script.js $$f || exit 1; \
+	done
 ifdef BQ_PERMISSIONS
 	BQ_PERMISSIONS_TARGET_DATASET=$(BQ_DEPLOY_DATASET) $(COMMON_DIR)/$(MODULE_PERMISSIONS_BASH)
 endif