Dest-S3: Enable ObjectLoader Interface (#56935)

Co-authored-by: Cole Snodgrass <cole@airbyte.io>

Author: johnny-schmidt

.github/workflows/connectors_pre_release_checks.yml

@@ -21,7 +21,7 @@ on:
 jobs:
   connectors_ci:
     name: Connector Pre-Release Checks
-    runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings
+    runs-on: linux-24.04-large # Custom runner, defined in GitHub org settings
     if: >
       github.event.pull_request.head.repo.fork != true &&
       github.event.pull_request.draft == false

.github/workflows/connectors_tests.yml

@@ -78,7 +78,7 @@ jobs:
     # If the condition is not met the job will be skipped (it will not fail)
     if: (github.event_name == 'pull_request' && needs.changes.outputs.connectors == 'true' && github.event.pull_request.head.repo.fork != true) || github.event_name == 'workflow_dispatch'
     name: Connectors CI
-    runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings
+    runs-on: linux-24.04-large # Custom runner, defined in GitHub org settings
     timeout-minutes: 360 # 6 hours
     steps:
       - name: Checkout Airbyte

.github/workflows/gradle.yml

@@ -49,7 +49,7 @@ jobs:
     # Any revision upwards should be based on a performance analysis of gradle scans.
     # See https://github.com/airbytehq/airbyte/pull/36055 for an example of this,
     # which explains why which we went down from 64 cores to 16.
-    runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings
+    runs-on: linux-24.04-large # Custom runner, defined in GitHub org settings
     name: Gradle Check
     timeout-minutes: 60
     steps:

.github/workflows/live_tests.yml

@@ -48,7 +48,7 @@ on:
 jobs:
   live_tests:
     name: Live Tests
-    runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings
+    runs-on: linux-24.04-large # Custom runner, defined in GitHub org settings
     timeout-minutes: 360 # 6 hours
     steps:
       - name: Checkout Airbyte

.github/workflows/publish-bulk-cdk.yml

@@ -25,7 +25,7 @@ env:
 jobs:
   publish-bulk-cdk:
     name: Publish Bulk CDK
-    runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings
+    runs-on: linux-24.04-large # Custom runner, defined in GitHub org settings
     timeout-minutes: 30
     steps:
       - name: Checkout Airbyte

.github/workflows/publish-java-cdk-command.yml

@@ -61,7 +61,7 @@ env:
 jobs:
   publish-cdk:
     name: Publish Java CDK
-    runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings
+    runs-on: linux-24.04-large # Custom runner, defined in GitHub org settings
     timeout-minutes: 30
     steps:
       - name: Link comment to Workflow Run

.github/workflows/regression_tests.yml

@@ -48,7 +48,7 @@ on:
 jobs:
   regression_tests:
     name: Regression Tests
-    runs-on: linux-20.04-large # Custom runner, defined in GitHub org settings
+    runs-on: linux-24.04-large # Custom runner, defined in GitHub org settings
     timeout-minutes: 360 # 6 hours
     steps:
       - name: Install Python

airbyte-integrations/connectors/destination-s3/metadata.yaml

@@ -2,7 +2,7 @@ data:
   connectorSubtype: file
   connectorType: destination
   definitionId: 4816b78f-1489-44c1-9060-4b19d5fa9362
-  dockerImageTag: 1.6.0
+  dockerImageTag: 1.7.0-rc.1
   dockerRepository: airbyte/destination-s3
   githubIssueLabel: destination-s3
   icon: s3.svg
@@ -23,7 +23,7 @@ data:
           **This release includes breaking changes, including major revisions to the schema of stored data. Do not upgrade without reviewing the migration guide.**
         upgradeDeadline: "2024-10-08"
     rolloutConfiguration:
-      enableProgressiveRollout: false
+      enableProgressiveRollout: true
   resourceRequirements:
     jobSpecific:
       - jobType: sync

airbyte-integrations/connectors/destination-s3/src/main/kotlin/S3V2Configuration.kt

@@ -40,7 +40,7 @@ data class S3V2Configuration<T : OutputStream>(
     override val estimatedRecordMemoryOverheadRatio: Double = 5.0,
     override val processEmptyFiles: Boolean = true,
 
-    /** Below has no effect until [S3V2ObjectLoader] is enabled. */
+    // ObjectLoader-specific configuration
     val numPartWorkers: Int = 2,
     val numUploadWorkers: Int = 5,
     val maxMemoryRatioReservedForParts: Double = 0.4,

airbyte-integrations/connectors/destination-s3/src/main/kotlin/S3V2ObjectLoader.kt

@@ -6,12 +6,11 @@ package io.airbyte.integrations.destination.s3_v2
 
 import io.airbyte.cdk.load.pipeline.RoundRobinInputPartitioner
 import io.airbyte.cdk.load.write.object_storage.ObjectLoader
+import io.micronaut.context.annotation.Requires
+import jakarta.inject.Singleton
 
-/**
- * These are temporarily disabled so that we can merge the underlying changes without doing a full
- * S3 release. I will do a separate PR to re-enable this and roll it out gradually.
- */
-// @Singleton
+@Singleton
+@Requires(property = "airbyte.destination.core.file-transfer.enabled", value = "false")
 class S3V2ObjectLoader(config: S3V2Configuration<*>) : ObjectLoader {
     override val numPartWorkers: Int = config.numPartWorkers
     override val numUploadWorkers: Int = config.numUploadWorkers
@@ -20,5 +19,6 @@ class S3V2ObjectLoader(config: S3V2Configuration<*>) : ObjectLoader {
     override val partSizeBytes: Long = config.partSizeBytes
 }
 
-// @Singleton
+@Singleton
+@Requires(bean = S3V2ObjectLoader::class)
 class S3V2RoundRobinInputPartitioner : RoundRobinInputPartitioner()