[ML] Upgrade the CI image used for macOS builds

.buildkite/pipelines/build_macos.json.py

@@ -33,7 +33,7 @@
 agents = {
    "aarch64": {
       "provider": "orka",
-      "image": "ml-macos-13-arm-005.orkasi"
+      "image": "ml-macos-14-arm-003.orkasi"
    }
 }
 envs = {

.ci/orka/README.md

@@ -9,15 +9,15 @@
 ## Files
 
 - `install.sh` The script that does the software installs on the image
-- `orka-macos-13-arm.pkr.hcl` The packer definition for a MacOS 13 ARM builder image
+- `orka-macos-14-arm.pkr.hcl` The packer definition for a MacOS 14 ARM builder image
 
 
 ## Set Up Packer
 
 If you haven't run these before, run the following once so packer downloads the `vault` integration:
 
 ```
-packer init orka-macos-13-arm.pkr.hcl
+packer init orka-macos-14-arm.pkr.hcl
 ```
 
 ## Build
@@ -26,10 +26,10 @@ Make sure you are connected to the Orka VM.
 
 Packer requires access to secrets in vault, where VAULT_ADDR=https://vault-ci-prod.elastic.dev and VAULT_TOKEN must be set appropriately in the environment.
 
-Run the following to create the image (MacOS 13 ARM in this example):
+Run the following to create the image (MacOS 14 ARM in this example):
 
 ```
-packer build orka-macos-13-arm.pkr.hcl
+packer build orka-macos-14-arm.pkr.hcl
 ```
 
 ## Versioning
@@ -38,28 +38,28 @@ The name of the resulting images are hard-coded (currently), and end in a sequen
 
 ## Source Images
 
-We make use of an image - `generic-13-ventura-arm-002.orkasi` - that is configured such that it:
+We make use of an image - `generic-14-sonoma-arm-001.orkasi` - that is configured such that it:
 
     * Adds passwordless `sudo` for the default `admin` user
     * Configures `the admin` user to be automatically logged in
     * Installs Xcode command line tools version 14 (by running `clang++ --version` and clicking through the dialogues)
 
 The generic image has the following packages installed:
 
+ * Google Cloud SDK into `~admin/google-cloud-sdk/`
+
+## Packer Install Steps
+
+The ML packer scripts do the following:
  * brew `4.0.28`
  * vault `1.14.0`
  * python3 `3.10.8`
  * jq `1.6`
  * orka-vm-tools
- * Google Cloud SDK into `~admin/google-cloud-sdk/`
  * `gobld-bootstrap.sh` script to run at system startup
     * This script pulls down and runs another script from a static location to do the following:
       * Unseal one-time vault token from gobld
       * Install and run the latest `buildkite-agent`
-
-## Packer Install Steps
-
-The ML packer scripts do the following:
  * Install JDK `11.0.25`
  * Install CMake `3.30.5`
  * Install Boost `1.86.0` from source

.ci/orka/install.sh

@@ -15,6 +15,31 @@ else
     exit 1
 fi
 
+if ! command -v brew 2> /dev/null ; then
+    /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+fi
+
+eval "$(brew shellenv)"
+echo 'export PATH="$HOMEBREW_PREFIX/bin:$PATH"' >> ~/.zshrc
+echo 'export PATH="$HOMEBREW_PREFIX/bin:$PATH"' >> ~/.bash_profile
+export PATH="$HOMEBREW_PREFIX/bin:$PATH"
+
+if ! command -v vault 2> /dev/null ; then
+    echo "install vault"
+    brew tap hashicorp/tap
+    brew install hashicorp/tap/vault
+fi
+
+if ! command -v jq 2> /dev/null ; then
+    echo "install jq"
+    brew install jq
+fi
+
+if ! command -v orka-vm-tools 2> /dev/null ; then
+    echo "install orka-vm-tools"
+    brew install orka-vm-tools
+fi
+
 echo "export PATH=$PATH" >> .zshrc
 
 if ! java --version 2> /dev/null ; then
@@ -31,5 +56,13 @@ echo "Install CMake"
 curl -v -L https://github.com/Kitware/CMake/releases/download/v3.30.5/cmake-3.30.5-macos-universal.tar.gz | tar xvzf - --strip-components 1 -C /Applications
 sudo ln -sf /Applications/CMake.app/Contents/bin/cmake /usr/local/bin/cmake
 
+# Install the gobld-bootstrap.sh
+sudo mkdir -p /usr/local/bin
+sudo cp /tmp/gobld-bootstrap.sh /usr/local/bin/gobld-bootstrap.sh
+sudo chmod +x /usr/local/bin/gobld-bootstrap.sh
+sudo cp /tmp/gobld-bootstrap.plist /Library/LaunchDaemons/gobld-bootstrap.plist
+sudo launchctl bootstrap system /Library/LaunchDaemons/gobld-bootstrap.plist
+sudo cp /tmp/gobld-bootstrap.plist /Users/admin
+
 # Make sure all changes are written to disk
 sync

.ci/orka/orka-macos-14-arm.pkr.hcl

@@ -0,0 +1,63 @@
+packer {
+  required_plugins {
+    macstadium-orka = {
+      version = "= 2.3.0"
+      source  = "github.com/macstadium/macstadium-orka"
+    }
+  }
+}
+
+locals {
+  orka_endpoint = vault("secret/ci/elastic-ml-cpp/orka", "orka_endpoint")
+  orka_user     = vault("secret/ci/elastic-ml-cpp/orka", "orka_user")
+  orka_password = vault("secret/ci/elastic-ml-cpp/orka", "orka_password")
+  ssh_username  = vault("secret/ci/elastic-ml-cpp/orka", "ssh_username")
+  ssh_password  = vault("secret/ci/elastic-ml-cpp/orka", "ssh_password")
+  sensitive     = true
+}
+
+source "macstadium-orka" "image" {
+  source_image     = "generic-14-sonoma-arm-001.orkasi"
+  image_name       = "ml-macos-14-arm-003.orkasi"
+  orka_endpoint    = local.orka_endpoint
+  orka_user        = local.orka_user
+  orka_password    = local.orka_password
+  ssh_username     = local.ssh_username
+  ssh_password     = local.ssh_password
+  orka_vm_cpu_core = 4
+  no_delete_vm     = false
+}
+
+build {
+  sources = [
+    "macstadium-orka.image"
+  ]
+  provisioner "file" {
+    source = "install.sh"
+    destination = "/tmp/install.sh"
+  }
+  provisioner "file" {
+    source = "third_party_deps.sh"
+    destination = "/tmp/third_party_deps.sh"
+  }
+  provisioner "file" {
+    source = "gobld-bootstrap.sh"
+    destination = "/tmp/gobld-bootstrap.sh"
+  }
+  provisioner "file" {
+    source = "gobld-bootstrap.plist"
+    destination = "/tmp/gobld-bootstrap.plist"
+  }
+  provisioner "shell" {
+    inline = [
+      "chmod u+x /tmp/install.sh",
+      "/tmp/install.sh",
+    ]
+  }
+  provisioner "shell" {
+    inline = [
+      "chmod u+x /tmp/third_party_deps.sh",
+      "/tmp/third_party_deps.sh",
+    ]
+  }
+}