refactor: Use boil instead of bake (#79)

* feat(setup-tools): Support installing latest version of tools

* refactor!(shard): Use boil

* chore(run-integration-test): Add comments

* refactor!(build-product-image): Use boil

* chore: Update refs to local actions

* chore: Add release instructions

* temp: Temporarily use PR branch for actions pointing to main

* chore: Add boil config, remove old bake configs

* chore(shard): Adjust version extraction command

* fix(shard): Add missing double quote

* fix(shard): Use --compact-output for jq instead of pretty (default)

* ci: Use Ubuntu 24.04 for Ubicloud ARM runner

* fix(build-product-image): Use correct boil command and args

* ci(smoke): Add boil config

* fix(build-product-image): Use --target-platform arg

* fix(build-product-image): Remove superfluous double quote

Aaaaaaaargh

* fix(build-product-image): Load container image

* fix: Add registry-namespace input

* ci: Use correct source image URI

* chore: Use main branch again, add follow-up PR comments

Note: This will break CI again. It will be fixed in a follow-up
PR.

* ci: Temporarily pin boil version

Author: Techassi

.github/workflows/smoke-build.yaml

@@ -28,7 +28,7 @@ jobs:
         uses: ./shard
         with:
           product-name: smoke
-          config-file: smoke/conf.py
+          boil-version: 0.1.0
     outputs:
       versions: ${{ steps.shard.outputs.versions }}
 
@@ -41,8 +41,10 @@ jobs:
     strategy:
       matrix:
         runner:
+          # Github's ubuntu-latest runners use 24.04. By default Ubicloud's runners use 22.04.
+          # We therefore explicitly request 24.04 for them as well.
           - {name: "ubuntu-latest", arch: "amd64"}
-          - {name: "ubicloud-standard-8-arm", arch: "arm64"}
+          - {name: "ubicloud-standard-8-arm-ubuntu-2404", arch: "arm64"}
         versions: ${{ fromJson(needs.generate_matrix.outputs.versions) }}
     steps:
       - name: Checkout Repository
@@ -57,10 +59,12 @@ jobs:
         id: build
         uses: ./build-product-image
         with:
-          product-name: smoke
           product-version: ${{ matrix.versions }}
-          bake-config-file: smoke/conf.py
+          boil-config-file: smoke/boil.toml
+          boil-version: 0.1.0
+          registry-namespace: stackable
           extra-tag-data: pr-321
+          product-name: smoke
 
       - name: Publish Container Image on oci.stackable.tech
         uses: ./publish-image
@@ -70,7 +74,7 @@ jobs:
           image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }}
           image-repository: stackable/smoke
           image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
-          source-image-uri: localhost/smoke:${{ steps.build.outputs.image-manifest-tag }}
+          source-image-uri: localhost/stackable/smoke:${{ steps.build.outputs.image-manifest-tag }}
 
   publish_manifests:
     name: Build/Publish ${{ matrix.versions }} Index Manifest

.scripts/actions/shard.py

@@ -0,0 +1,9 @@
+# Release
+
+To release a new set of actions follow these steps:
+
+- Choose an appropriate new semantic version based on the latest changes
+- Update any references in actions to other local actions to the chosen version
+- Raise a PR with the appropriate changes
+- Once merged, tag on `main` using `git tag -s v<X.Y.Z>`
+- Manually create a GitHub release

RELEASE.md

@@ -8,27 +8,17 @@
 > a name. If the name does not contain a registry, `hub.docker.com` (?) is implied. Therefore,
 > `localhost` will be used as the registry so as to avoid accidental interactions with an unintended
 > registry.
->
-> Ideally, bake should be refactored to use `localhost` as the registry for the previously mentioned
-> reason (whether or not that is behind some option).
 
-This action builds a *single* container image using `bake`. It does the following work:
+This action builds a *single* container image using `boil`. It does the following work:
 
 1. Free disk space to avoid running out of disk space during larger builds.
-2. Build the image using `bake` which internally uses `docker buildx`.
-3. Temporarily retag the image to use `localhost` instead of `oci.stackable.tech/sdp`.
-4. Produce output values to be used in later steps.
+2. Build the image using `boil` which internally uses `docker buildx`.
+3. Produce output values to be used in later steps.
 
 This action is considered to be the **single** source of truth regarding image index tag and image
 manifest tag. All subsequent tasks must use these values to ensure consistency.
 
-Currently, bake provides the following ouput in the `bake-target-tags` file:
-
-```plain
-oci.stackable.tech/sdp/kafka:3.4.1-stackable0.0.0-dev-amd64
-```
-
-Until bake supports the ability to specify the registry, this action will retag the image as:
+boil provides the following output in the `boil-target-tags` file:
 
 ```plain
 localhost/kafka:3.4.1-stackable0.0.0-dev-amd64
@@ -41,16 +31,21 @@ localhost/kafka:3.4.1-stackable0.0.0-dev-amd64
 
 ### Inputs
 
-- `product-name` (eg: `kafka`)
-- `product-version` (eg: `3.4.1`)
-- `image-tools-version` (eg: `0.0.14`)
-- `bake-config-file` (defaults to `./conf.py`)
-- `sdp-version` (defaults to: `0.0.0-dev`)
-- `extra-tag-data` (optional, eg. `pr321`)
+| Input                | Required (Default) | Description                                                  |
+| -------------------- | ------------------ | ------------------------------------------------------------ |
+| `product-name`       | Yes                | The name of the product image to build (eg: `kafka`)         |
+| `product-version`    | Yes                | The version of the product image to build (eg: `3.4.1`)      |
+| `registry-namespace` | Yes                | The path of the registry namespace, eg. `sdp` or `stackable` |
+| `boil-version`       | No (`latest`)      | The version of boil used to build the image                  |
+| `boil-config-file`   | No (`./boil.toml`) | The path to the boil config file                             |
+| `sdp-version`        | No (`0.0.0-dev`)   | The SDP version of the image                                 |
+| `extra-tag-data`     | No                 | Extra data to be included in the tag, eg. `pr321`            |
 
 ### Outputs
 
-- `image-manifest-tag` (eg: `3.4.1-stackable0.0.0-dev-amd64`)
-- `suggested-image-index-manifest-tag` (eg: `3.4.1-stackable0.0.0-dev`)
+| Output                               | Example                          | Description                                     |
+| ------------------------------------ | -------------------------------- | ----------------------------------------------- |
+| `image-manifest-tag`                 | `3.4.1-stackable0.0.0-dev-amd64` | The image manifest tag (including architecture) |
+| `suggested-image-index-manifest-tag` | `3.4.1-stackable0.0.0-dev`       | The suggested image index manifest tag          |
 
 [build-product-image]: ./action.yaml

build-product-image/README.md

@@ -1,26 +1,29 @@
 ---
 name: Build Product Image
-description: This action builds a product Docker image with a specific version
+description: This action builds a product container image with a specific version
 inputs:
   product-name:
-    description: The name of the product to build via bake (directory name)
+    description: The name of the product to build via boil (directory name)
     required: true
   product-version:
-    description: The version of the product to build via bake
+    description: The version of the product to build via boil
     required: true
-  image-tools-version:
-    description: The Stackable image-tools version
-    default: 0.0.17
-  bake-config-file:
-    description: Path to the bake config file, defaults to `./conf.py`
-    default: ./conf.py
+  boil-version:
+    description: The boil version
+    default: latest
+  boil-config-file:
+    description: Path to the boil config file, defaults to `./boil.toml`
+    default: ./boil.toml
   sdp-version:
     description: |
       Stackable Data Platform version (eg: `24.7.0`)
     default: 0.0.0-dev
   extra-tag-data:
     description: |
       Extra data to include in the final image manifest tag (eg: `pr321`)
+  registry-namespace:
+    description: Path of the registry namespace, eg. `sdp` or `stackable`
+    required: true
 outputs:
   image-manifest-tag:
     description: |
@@ -38,22 +41,18 @@ runs:
     - name: Setup Docker Buildx
       uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
 
-      # NOTE (@Techassi): Why do we install python via apt and not the setup-python action?
-    - name: Setup Python
-      shell: bash
-      run: |
-        set -euo pipefail
-        sudo apt update
-        sudo apt install -y python3
+    - name: Setup boil (${{ inputs.boil-version }})
+      # Using other "local" actions is super painful. As such, a follow-up PR will make this way
+      # more comfortable again.
+      uses: stackabletech/actions/setup-tools@main # TODO: Replace with latest released version
+      with:
+        boil-version: ${{ inputs.boil-version }}
 
-    - name: Install image-tools-stackabletech
-      shell: bash
-      run: pip install image-tools-stackabletech==${{ inputs.image-tools-version }}
-
-    - name: Building ${{ inputs.product-name }} Container Image using bake
+    - name: Building ${{ inputs.product-name }} Container Image using boil
       env:
-        BAKE_PRODUCT_VERSION: ${{ inputs.product-version }}
-        BAKE_CONFIG_FILE: ${{ inputs.bake-config-file }}
+        REGISTRY_NAMESPACE: ${{ inputs.registry-namespace }}
+        BOIL_PRODUCT_VERSION: ${{ inputs.product-version }}
+        BOIL_CONFIG_FILE: ${{ inputs.boil-config-file }}
         IMAGE_REPOSITORY: ${{ inputs.product-name }}
         EXTRA_TAG_DATA: ${{ inputs.extra-tag-data }}
         SDP_VERSION: ${{ inputs.sdp-version }}
@@ -63,9 +62,9 @@ runs:
         IMAGE_ARCH=$("$GITHUB_ACTION_PATH/../.scripts/actions/get_architecture.sh")
 
         # Will be either:
-        # - 3.9.2-stackable0.0.0-dev-arm64 or
-        # - 3.9.2-stackable0.0.0-dev-pr321-arm64
-        IMAGE_INDEX_MANIFEST_TAG="${SDP_VERSION}${EXTRA_TAG_DATA:+-$EXTRA_TAG_DATA}-${IMAGE_ARCH}"
+        # - 3.9.2-stackable0.0.0-dev or
+        # - 3.9.2-stackable0.0.0-dev-pr321
+        IMAGE_INDEX_MANIFEST_TAG="${SDP_VERSION}${EXTRA_TAG_DATA:+-$EXTRA_TAG_DATA}"
         echo "IMAGE_INDEX_MANIFEST_TAG=$IMAGE_INDEX_MANIFEST_TAG" | tee -a "$GITHUB_ENV"
 
         # Validate that final tag is valid according to
@@ -75,40 +74,27 @@ runs:
           exit 1
         fi
 
-        echo "::group::bake"
-        bake \
-          --product "$IMAGE_REPOSITORY=$BAKE_PRODUCT_VERSION" \
+        echo "::group::boil"
+        boil build \
           --image-version "$IMAGE_INDEX_MANIFEST_TAG" \
-          --release "$SDP_VERSION" \
-          --architecture "linux/${IMAGE_ARCH}" \
-          --export-tags-file bake-target-tags \
-          --configuration "$BAKE_CONFIG_FILE"
+          --registry-namespace "$REGISTRY_NAMESPACE" \
+          --target-platform "linux/${IMAGE_ARCH}" \
+          --configuration "$BOIL_CONFIG_FILE" \
+          --write-image-manifest-uris \
+          --use-localhost-registry \
+          --load \
+          "$IMAGE_REPOSITORY=$BOIL_PRODUCT_VERSION"
         echo "::endgroup::"
 
-    - name: Re-tag Image (Temporary)
-      shell: bash
-      run: |
-        set -euo pipefail
-
-        # Extract the image uri and replace 'oci.stackable.tech/sdp'
-        # with 'localhost' until bake does the right thing
-        OLD_IMAGE_URI="$(< bake-target-tags)"
-
-        # Replace the image uri in the bake file
-        sed -i -e 's/oci\.stackable\.tech\/sdp/localhost/' bake-target-tags
-
-        # Finally, re-tag image
-        docker tag "$OLD_IMAGE_URI" "$(< bake-target-tags)"
-
     - name: Extract Environment Variables
       id: image_info
       shell: bash
       run: |
         set -euo pipefail
-        echo "bake-target-tags: "$(< bake-target-tags)
+        echo "boil-target-tags: "$(< boil-target-tags)
 
-        # Extract the image manifest tag from the bake-target-tags file
-        IMAGE_MANIFEST_TAG=$(cut -d : -f 2 < bake-target-tags)
+        # Extract the image manifest tag from the boil-target-tags file
+        IMAGE_MANIFEST_TAG=$(cut -d : -f 2 < boil-target-tags)
         [[ -n "$IMAGE_MANIFEST_TAG" ]]
         [[ -n "$IMAGE_INDEX_MANIFEST_TAG" ]]
 

build-product-image/action.yaml

@@ -36,7 +36,9 @@ runs:
     - name: Set up Helm
       # We cannot use a path to point to the local action
       # See https://github.com/orgs/community/discussions/74934
-      uses: stackabletech/actions/setup-k8s-tools@7279ffd852a8013df52028de0d99eef68700f803 # TODO: Pin this to the latest tag
+      # Using other "local" actions is super painful. As such, a follow-up PR will make this way
+      # more comfortable again.
+      uses: stackabletech/actions/setup-k8s-tools@main # TODO: Replace with latest released version
       with:
         helm-version: ${{ inputs.helm-version }}
 

publish-helm-chart/action.yaml

@@ -60,7 +60,9 @@ runs:
   using: composite
   steps:
     - name: Install interu
-      uses: stackabletech/actions/setup-tools@7279ffd852a8013df52028de0d99eef68700f803 # TODO: Pin this to the latest tag
+      # Using other "local" actions is super painful. As such, a follow-up PR will make this way
+      # more comfortable again.
+      uses: stackabletech/actions/setup-tools@main # TODO: Replace with latest released version
       with:
         interu-version: ${{ inputs.interu-version }}
     - name: Extract Test and Instance Configuration
@@ -97,7 +99,9 @@ runs:
     # and are therefore not available, there is no need to create the cluster or run the tests,
     # because the tests can never run in the first place.
     - name: Install kubectl, kubectl-kuttl, and helm
-      uses: stackablectl/actions/setup-k8s-tools@7279ffd852a8013df52028de0d99eef68700f803 # TODO: Pin this to the latest tag
+      # Using other "local" actions is super painful. As such, a follow-up PR will make this way
+      # more comfortable again.
+      uses: stackablectl/actions/setup-k8s-tools@main # TODO: Replace with latest released version
       with:
         kubectl-version: ${{ inputs.kubectl-version }}
         kuttl-version: ${{ inputs.kuttl-version }}
@@ -110,7 +114,9 @@ runs:
     # mikefarah/yq is already installed on the runner
     # See https://github.com/actions/runner-images/blob/main/images/ubuntu/scripts/build/install-yq.sh
     - name: Install stackablectl and beku
-      uses: stackabletech/actions/setup-tools@7279ffd852a8013df52028de0d99eef68700f803 # TODO: Pin this to the latest tag
+      # Using other "local" actions is super painful. As such, a follow-up PR will make this way
+      # more comfortable again.
+      uses: stackabletech/actions/setup-tools@main # TODO: Replace with latest released version
       with:
         stackablectl-version: ${{ inputs.stackablectl-version }}
         beku-version: ${{ inputs.beku-version }}

run-integration-test/action.yaml

@@ -25,7 +25,12 @@ runs:
 
         ARCH=$(uname -m)
 
-        curl -fsSL -o /tmp/boil "https://github.com/stackabletech/docker-images/releases/download/boil-${BOIL_VERSION}/boil-${ARCH}-unknown-linux-gnu"
+        if [ "$BOIL_VERSION" == "latest" ]; then
+          curl -fsSL -o /tmp/boil "https://github.com/stackabletech/docker-images/releases/latest/download/boil-${ARCH}-unknown-linux-gnu"
+        else
+          curl -fsSL -o /tmp/boil "https://github.com/stackabletech/docker-images/releases/download/boil-${BOIL_VERSION}/boil-${ARCH}-unknown-linux-gnu"
+        fi
+
         sudo install -m 755 -t /usr/local/bin /tmp/boil
 
     - name: Install stackablectl
@@ -40,7 +45,12 @@ runs:
 
         ARCH=$(uname -m)
 
-        curl -fsSL -o /tmp/stackablectl "https://github.com/stackabletech/stackable-cockpit/releases/download/stackablectl-${STACKABLECTL_VERSION}/stackablectl-${ARCH}-unknown-linux-gnu"
+        if [ "$STACKABLECTL_VERSION" == "latest" ]; then
+          curl -fsSL -o /tmp/stackablectl "https://github.com/stackabletech/stackable-cockpit/releases/latest/download/stackablectl-${ARCH}-unknown-linux-gnu"
+        else
+          curl -fsSL -o /tmp/stackablectl "https://github.com/stackabletech/stackable-cockpit/releases/download/stackablectl-${STACKABLECTL_VERSION}/stackablectl-${ARCH}-unknown-linux-gnu"
+        fi
+
         sudo install -m 755 -t /usr/local/bin /tmp/stackablectl
 
     - name: Install interu
@@ -55,7 +65,12 @@ runs:
 
         ARCH=$(uname -m)
 
-        curl -fsSL -o /tmp/interu "https://github.com/stackabletech/actions/releases/download/interu-${INTERU_VERSION}/interu-${ARCH}-unknown-linux-gnu"
+        if [ "$INTERU_VERSION" == "latest" ]; then
+          curl -fsSL -o /tmp/interu "https://github.com/stackabletech/actions/releases/latest/download/interu-${ARCH}-unknown-linux-gnu"
+        else
+          curl -fsSL -o /tmp/interu "https://github.com/stackabletech/actions/releases/download/interu-${INTERU_VERSION}/interu-${ARCH}-unknown-linux-gnu"
+        fi
+
         sudo install -m 755 -t /usr/local/bin /tmp/interu
 
     - name: Install beku