diff --git a/.buildkite/pipelines/build_macos.json.py b/.buildkite/pipelines/build_macos.json.py index 4b72b6964..65bd0e435 100755 --- a/.buildkite/pipelines/build_macos.json.py +++ b/.buildkite/pipelines/build_macos.json.py @@ -33,7 +33,7 @@ agents = { "aarch64": { "provider": "orka", - "image": "ml-macos-13-arm-005.orkasi" + "image": "ml-macos-14-arm-003.orkasi" } } envs = { diff --git a/.ci/orka/README.md b/.ci/orka/README.md index 6b9d39c36..4664a047c 100644 --- a/.ci/orka/README.md +++ b/.ci/orka/README.md @@ -9,7 +9,7 @@ ## Files - `install.sh` The script that does the software installs on the image -- `orka-macos-13-arm.pkr.hcl` The packer definition for a MacOS 13 ARM builder image +- `orka-macos-14-arm.pkr.hcl` The packer definition for a MacOS 14 ARM builder image ## Set Up Packer @@ -17,7 +17,7 @@ If you haven't run these before, run the following once so packer downloads the `vault` integration: ``` -packer init orka-macos-13-arm.pkr.hcl +packer init orka-macos-14-arm.pkr.hcl ``` ## Build @@ -26,10 +26,10 @@ Make sure you are connected to the Orka VM. Packer requires access to secrets in vault, where VAULT_ADDR=https://vault-ci-prod.elastic.dev and VAULT_TOKEN must be set appropriately in the environment. -Run the following to create the image (MacOS 13 ARM in this example): +Run the following to create the image (MacOS 14 ARM in this example): ``` -packer build orka-macos-13-arm.pkr.hcl +packer build orka-macos-14-arm.pkr.hcl ``` ## Versioning @@ -38,7 +38,7 @@ The name of the resulting images are hard-coded (currently), and end in a sequen ## Source Images -We make use of an image - `generic-13-ventura-arm-002.orkasi` - that is configured such that it: +We make use of an image - `generic-14-sonoma-arm-001.orkasi` - that is configured such that it: * Adds passwordless `sudo` for the default `admin` user * Configures `the admin` user to be automatically logged in @@ -46,20 +46,20 @@ We make use of an image - `generic-13-ventura-arm-002.orkasi` - that is configur The generic image has the following packages installed: + * Google Cloud SDK into `~admin/google-cloud-sdk/` + +## Packer Install Steps + +The ML packer scripts do the following: * brew `4.0.28` * vault `1.14.0` * python3 `3.10.8` * jq `1.6` * orka-vm-tools - * Google Cloud SDK into `~admin/google-cloud-sdk/` * `gobld-bootstrap.sh` script to run at system startup * This script pulls down and runs another script from a static location to do the following: * Unseal one-time vault token from gobld * Install and run the latest `buildkite-agent` - -## Packer Install Steps - -The ML packer scripts do the following: * Install JDK `11.0.25` * Install CMake `3.30.5` * Install Boost `1.86.0` from source diff --git a/.ci/orka/install.sh b/.ci/orka/install.sh index 40d17297a..444262fde 100644 --- a/.ci/orka/install.sh +++ b/.ci/orka/install.sh @@ -15,6 +15,31 @@ else exit 1 fi +if ! command -v brew 2> /dev/null ; then + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" +fi + +eval "$(brew shellenv)" +echo 'export PATH="$HOMEBREW_PREFIX/bin:$PATH"' >> ~/.zshrc +echo 'export PATH="$HOMEBREW_PREFIX/bin:$PATH"' >> ~/.bash_profile +export PATH="$HOMEBREW_PREFIX/bin:$PATH" + +if ! command -v vault 2> /dev/null ; then + echo "install vault" + brew tap hashicorp/tap + brew install hashicorp/tap/vault +fi + +if ! command -v jq 2> /dev/null ; then + echo "install jq" + brew install jq +fi + +if ! command -v orka-vm-tools 2> /dev/null ; then + echo "install orka-vm-tools" + brew install orka-vm-tools +fi + echo "export PATH=$PATH" >> .zshrc if ! java --version 2> /dev/null ; then @@ -31,5 +56,13 @@ echo "Install CMake" curl -v -L https://github.com/Kitware/CMake/releases/download/v3.30.5/cmake-3.30.5-macos-universal.tar.gz | tar xvzf - --strip-components 1 -C /Applications sudo ln -sf /Applications/CMake.app/Contents/bin/cmake /usr/local/bin/cmake +# Install the gobld-bootstrap.sh +sudo mkdir -p /usr/local/bin +sudo cp /tmp/gobld-bootstrap.sh /usr/local/bin/gobld-bootstrap.sh +sudo chmod +x /usr/local/bin/gobld-bootstrap.sh +sudo cp /tmp/gobld-bootstrap.plist /Library/LaunchDaemons/gobld-bootstrap.plist +sudo launchctl bootstrap system /Library/LaunchDaemons/gobld-bootstrap.plist +sudo cp /tmp/gobld-bootstrap.plist /Users/admin + # Make sure all changes are written to disk sync diff --git a/.ci/orka/orka-macos-14-arm.pkr.hcl b/.ci/orka/orka-macos-14-arm.pkr.hcl new file mode 100644 index 000000000..9d58e4903 --- /dev/null +++ b/.ci/orka/orka-macos-14-arm.pkr.hcl @@ -0,0 +1,63 @@ +packer { + required_plugins { + macstadium-orka = { + version = "= 2.3.0" + source = "github.com/macstadium/macstadium-orka" + } + } +} + +locals { + orka_endpoint = vault("secret/ci/elastic-ml-cpp/orka", "orka_endpoint") + orka_user = vault("secret/ci/elastic-ml-cpp/orka", "orka_user") + orka_password = vault("secret/ci/elastic-ml-cpp/orka", "orka_password") + ssh_username = vault("secret/ci/elastic-ml-cpp/orka", "ssh_username") + ssh_password = vault("secret/ci/elastic-ml-cpp/orka", "ssh_password") + sensitive = true +} + +source "macstadium-orka" "image" { + source_image = "generic-14-sonoma-arm-001.orkasi" + image_name = "ml-macos-14-arm-003.orkasi" + orka_endpoint = local.orka_endpoint + orka_user = local.orka_user + orka_password = local.orka_password + ssh_username = local.ssh_username + ssh_password = local.ssh_password + orka_vm_cpu_core = 4 + no_delete_vm = false +} + +build { + sources = [ + "macstadium-orka.image" + ] + provisioner "file" { + source = "install.sh" + destination = "/tmp/install.sh" + } + provisioner "file" { + source = "third_party_deps.sh" + destination = "/tmp/third_party_deps.sh" + } + provisioner "file" { + source = "gobld-bootstrap.sh" + destination = "/tmp/gobld-bootstrap.sh" + } + provisioner "file" { + source = "gobld-bootstrap.plist" + destination = "/tmp/gobld-bootstrap.plist" + } + provisioner "shell" { + inline = [ + "chmod u+x /tmp/install.sh", + "/tmp/install.sh", + ] + } + provisioner "shell" { + inline = [ + "chmod u+x /tmp/third_party_deps.sh", + "/tmp/third_party_deps.sh", + ] + } +}