Automated Deploy #201
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Automated Deploy | |
| on: | |
| workflow_run: | |
| workflows: | |
| - Build Hosts | |
| types: | |
| - completed | |
| branches: | |
| - main | |
| jobs: | |
| deploy: | |
| runs-on: ci-ocf-nix-deploy | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup hosts | |
| run: | | |
| HOSTS=$(nix eval .#autoDeploy.nodes --apply 'builtins.concatStringsSep " "' --raw) | |
| HOST_MACS=$(nix eval .#autoDeploy.MACs --apply 'builtins.concatStringsSep " "' --raw) | |
| echo "HOSTS=$HOSTS" >> $GITHUB_ENV | |
| echo "HOST_MACS=$HOST_MACS" >> $GITHUB_ENV | |
| - name: Setup SSH | |
| run: | | |
| echo '${{ secrets.COLMENA_APPLY_SSH_KEY }}' > ${{ github.workspace }}/id_ed25519 | |
| chmod 400 id_ed25519 | |
| for host in ${HOSTS}; do | |
| cat secrets/host-keys/$host.pub | sed "s/^/$host.ocf.berkeley.edu /" >> known_hosts | |
| done | |
| echo " | |
| CanonicalizeHostname yes | |
| CanonicalDomains ocf.berkeley.edu | |
| Identityfile $GITHUB_WORKSPACE/id_ed25519 | |
| UserKnownHostsFile $GITHUB_WORKSPACE/known_hosts | |
| StrictHostKeyChecking yes | |
| UpdateHostKeys no | |
| Host * | |
| User ocf-nix-deploy-user | |
| " >> ${{ github.workspace }}/ssh_config | |
| - name: Wake up hosts | |
| run: | | |
| # not using quotes here so that each mac address is a separate argument | |
| if ! nix develop .#deploy -c wol $HOST_MACS; then | |
| echo '❌ Failed to wakeup nodes' | |
| fi | |
| - name: Deploy with colmena | |
| env: | |
| SSH_CONFIG_FILE: ${{ github.workspace }}/ssh_config | |
| run: | | |
| echo "$HOSTS" | tr ' ' '\n' | \ | |
| xargs -P 10 -n 1 -I {} \ | |
| nix develop .#deploy -c sh -c " | |
| # wait for host to be online with a timeout of 5s per attempt and 12 | |
| # total tries, but fail immediately if the error is auth related | |
| if ! ssh -F \$SSH_CONFIG_FILE \ | |
| -o BatchMode=yes \ | |
| -o ConnectTimeout=5 \ | |
| -o ConnectionAttempts=12 \ | |
| {} echo 'continuous deployment ping!' > {}.log 2>&1; then | |
| echo '❌ Failed to deploy to {} (ssh timeout)' | |
| echo 'FAILURE:{}' >> results.txt | |
| exit | |
| fi | |
| if colmena apply --on {} switch > {}.log 2>&1; then | |
| echo '✅ Successfully deployed to {}' | |
| echo 'SUCCESS:{}' >> results.txt | |
| else | |
| echo '❌ Failed to deploy to {}' | |
| echo 'FAILURE:{}' >> results.txt | |
| fi | |
| " | |
| - name: Display successful logs | |
| if: always() | |
| run: | | |
| grep "SUCCESS:" results.txt | cut -d':' -f2 | while read host; do | |
| echo "--- $host ---" | |
| cat "${host}.log" | |
| done | |
| - name: Display failed logs | |
| if: always() | |
| run: | | |
| grep "FAILURE:" results.txt | cut -d':' -f2 | while read host; do | |
| echo "--- $host ---" | |
| cat "${host}.log" | |
| done | |
| - name: Check status | |
| if: always() | |
| run: | | |
| if grep -q "FAILURE:" results.txt; then | |
| exit 1 | |
| fi |