From 65ec8e1620b3da70fb84727361ad67fe8793756e Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Sun, 25 Feb 2024 19:52:13 -0800 Subject: [PATCH 01/17] Added u-boot submodule --- .gitmodules | 3 +++ u-boot | 1 + 2 files changed, 4 insertions(+) create mode 160000 u-boot diff --git a/.gitmodules b/.gitmodules index 7eab220..7159958 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ path = linux url = https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/ branch = master +[submodule "u-boot"] + path = u-boot + url = https://github.com/u-boot/u-boot diff --git a/u-boot b/u-boot new file mode 160000 index 0000000..1a66a77 --- /dev/null +++ b/u-boot @@ -0,0 +1 @@ +Subproject commit 1a66a7768af7e8106c2cd93a19f4013877fb85ae From 188a1574d906e9df75d9fd72cf22150124059189 Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Sun, 25 Feb 2024 19:52:46 -0800 Subject: [PATCH 02/17] Add u-boot build targets --- Makefile | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/Makefile b/Makefile index 0b5857a..eb2a0a2 100644 --- a/Makefile +++ b/Makefile @@ -344,6 +344,46 @@ rootfs_clean: $(SUDO) rm -f $(CPIO_FILE) $(SUDO) rm -f $(ROOTFS) +## +## U-Boot +## + +# Note: We're reusing the TARGET variable from the Linux build section + +UBOOT_SRC := $(ROOT_DIR)/u-boot +UBOOT_OUT := $(OUT_DIR)/uboot/$(ARCH) +UBOOT_CONFIG := $(UBOOT_OUT)/.config +UBOOT_BIN := $(UBOOT_OUT)/u-boot.bin + +ifeq ($(ARCH),x86_64) + UBOOT_DEFCONFIG=qemu-x86_64_defconfig +else ifeq ($(ARCH),i386) + UBOOT_DEFCONFIG=qemu-x86_defconfig +else ifeq ($(ARCH),arm64) + UBOOT_DEFCONFIG=qemu_arm64_defconfig +endif + +UBOOT_MAKE := \ + PATH=$(CLANG_DIR)/bin:$(PATH) \ + $(MAKE) \ + -C $(UBOOT_SRC) \ + HOSTCC=clang \ + O=$(UBOOT_OUT) \ + -j `nproc` + +.PHONY: uboot_defconfig +uboot_defconfig $(UBOOT_CONFIG): | $(CLANG_DIR) + + $(UBOOT_MAKE) $(UBOOT_DEFCONFIG) + +.PHONY: uboot +uboot $(UBOOT_BIN): $(UBOOT_CONFIG) | $(CLANG_DIR) + + $(UBOOT_MAKE) CROSS_COMPILE=$(TARGET)- CC=clang + cd $(UBOOT_SRC) && ./scripts/gen_compile_commands.py -d $(UBOOT_OUT) + +.PHONY: uboot_clean +uboot_clean: + + $(UBOOT_MAKE) mrproper + ## ## Run QEMU ## From c6f08a64cbcbf43cbf9407cf31ba332adeb57b46 Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Sun, 25 Feb 2024 19:53:42 -0800 Subject: [PATCH 03/17] Run qemu with u-boot depending on UBOOT flag --- Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Makefile b/Makefile index eb2a0a2..08f1d7a 100644 --- a/Makefile +++ b/Makefile @@ -400,6 +400,7 @@ ECHR ?= 1 ROOT ?= /dev/vda RW ?= rw KASLR ?= 0 +UBOOT ?= 0 QEMU_KERNEL_CMDLINE := selinux=0 @@ -415,6 +416,10 @@ QEMU_ARGS := \ -echr $(ECHR) \ $(QEMU_EXTRA_ARGS) +ifeq ($(UBOOT),1) + QEMU_ARGS += -bios $(UBOOT_BIN) +endif + ifneq ($(INITRD),) ifeq ($(INITRD),1) INITRD := $(CPIO_FILE) @@ -469,6 +474,9 @@ endif QEMU_ARGS += -append "$(QEMU_KERNEL_CMDLINE) $(QEMU_EXTRA_KERNEL_CMDLINE)" RUN_DEPS := $(QEMU_KERNEL_IMAGE) +ifeq ($(UBOOT),1) + RUN_DEPS += $(UBOOT_BIN) +endif # Make sure the modules directory exists, even if it's empty. Otherwise mount # will fail. From 5b7be18233ca873091f119ed12e6427ff4bb35d9 Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Sun, 25 Feb 2024 19:55:20 -0800 Subject: [PATCH 04/17] Make some u-boot variables configurable --- Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 08f1d7a..4b5aa40 100644 --- a/Makefile +++ b/Makefile @@ -350,17 +350,17 @@ rootfs_clean: # Note: We're reusing the TARGET variable from the Linux build section -UBOOT_SRC := $(ROOT_DIR)/u-boot -UBOOT_OUT := $(OUT_DIR)/uboot/$(ARCH) +UBOOT_SRC ?= $(ROOT_DIR)/u-boot +UBOOT_OUT ?= $(OUT_DIR)/uboot/$(ARCH) UBOOT_CONFIG := $(UBOOT_OUT)/.config UBOOT_BIN := $(UBOOT_OUT)/u-boot.bin ifeq ($(ARCH),x86_64) - UBOOT_DEFCONFIG=qemu-x86_64_defconfig + UBOOT_DEFCONFIG ?= qemu-x86_64_defconfig else ifeq ($(ARCH),i386) - UBOOT_DEFCONFIG=qemu-x86_defconfig + UBOOT_DEFCONFIG ?= qemu-x86_defconfig else ifeq ($(ARCH),arm64) - UBOOT_DEFCONFIG=qemu_arm64_defconfig + UBOOT_DEFCONFIG ?= qemu_arm64_defconfig endif UBOOT_MAKE := \ From 20ab084b9ae083afe5f73df518cfa86964122f09 Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Sat, 24 Feb 2024 19:18:54 -0800 Subject: [PATCH 05/17] WIP --- .gitignore | 2 ++ Makefile | 88 ++++++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 81 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 12b10c1..d31e9d7 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ compile_commands.json /external/* !/external/external.mk !/external/README.md + +/trusty diff --git a/Makefile b/Makefile index 4b5aa40..f5cb71a 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,14 @@ else ifeq ($(filter x86_64 arm64 i386,$(ARCH)),) $(error Invalid architecture $(ARCH)) endif +ifneq ($(ARCH),arm64) + # We currently only support Trusty with arm64, as we rely on ARM TF-A + ifneq ($(filter trusty,$(MAKECMDGOALS)),) + $(error Building Trusty is only supported on arm64) + endif +endif + + .PHONY: default default: linux linux_modules tools-vm @@ -384,6 +392,39 @@ uboot $(UBOOT_BIN): $(UBOOT_CONFIG) | $(CLANG_DIR) uboot_clean: + $(UBOOT_MAKE) mrproper +## +## Trusty +## + +TRUSTY_SRC ?= $(ROOT_DIR)/trusty +TRUSTY_TARGET ?= qemu-generic-arm64-test-debug +TRUSTY_BUILD_ROOT ?= $(OUT_DIR)/trusty +TRUSTY_OUT := $(TRUSTY_BUILD_ROOT)/build-$(TRUSTY_TARGET) + +ATF_DIR := $(TRUSTY_OUT)/atf/qemu/debug +ATF_BL1 := $(ATF_DIR)/bl1.bin +ATF_BL33 := $(ATF_DIR)/bl33.bin + +.PHONY: trusty-init +trusty-init: + mkdir -p $(TRUSTY_SRC) + cd $(TRUSTY_SRC) && repo init -u https://android.googlesource.com/trusty/manifest -b main + cd $(TRUSTY_SRC) && repo sync -j`nproc` -c --no-tags + +.PHONY: trusty +trusty $(ATF_BL1): | $(TRUSTY_SRC) + $(TRUSTY_SRC)/trusty/vendor/google/aosp/scripts/build.py --build-root $(TRUSTY_BUILD_ROOT) --skip-tests $(TRUSTY_TARGET) + $(MAKE) trusty_bl33 + +.PHONY: trusty_bl33 +trusty_bl33 $(ATF_BL33): $(UBOOT_BIN) + rm $(ATF_BL33) + cp $(UBOOT_BIN) $(ATF_BL33) + +.PHONY: trusty_clean +trusty_clean: + rm -rf $(TRUSTY_OUT) + ## ## Run QEMU ## @@ -400,7 +441,9 @@ ECHR ?= 1 ROOT ?= /dev/vda RW ?= rw KASLR ?= 0 + UBOOT ?= 0 +TRUSTY ?= 0 QEMU_KERNEL_CMDLINE := selinux=0 @@ -410,16 +453,23 @@ QEMU_ARGS := \ -nographic \ -no-reboot \ -kernel $(QEMU_KERNEL_IMAGE) \ - -netdev user,id=eth0,hostfwd=tcp::7777-:7777,hostfwd=tcp::2222-:22,hostfwd=tcp::2223-:23 -device virtio-net-pci,netdev=eth0 \ - -virtfs local,security_model=mapped-xattr,path=$(SHARED_DIR),mount_tag=shared \ - -virtfs local,security_model=mapped-xattr,path=$(LINUX_MODULES_INSTALL_PATH)/lib/modules,mount_tag=modules \ -echr $(ECHR) \ $(QEMU_EXTRA_ARGS) -ifeq ($(UBOOT),1) +ifeq ($(TRUSTY),1) + QEMU_ARGS += -bios $(ATF_BL1) +else ifeq ($(UBOOT),1) QEMU_ARGS += -bios $(UBOOT_BIN) endif +ifneq ($(TRUSTY),1) + # Trusty currently only works with a very old version of QEMU, these flags + # don't seem to work with it + QEMU_ARGS += -netdev user,id=eth0,hostfwd=tcp::7777-:7777,hostfwd=tcp::2222-:22,hostfwd=tcp::2223-:23 -device virtio-net-pci,netdev=eth0 + QEMU_ARGS += -virtfs local,security_model=mapped-xattr,path=$(SHARED_DIR),mount_tag=shared + QEMU_ARGS += -virtfs local,security_model=mapped-xattr,path=$(LINUX_MODULES_INSTALL_PATH)/lib/modules,mount_tag=modules +endif + ifneq ($(INITRD),) ifeq ($(INITRD),1) INITRD := $(CPIO_FILE) @@ -436,7 +486,7 @@ ifeq ($(GDB),1) endif ifeq ($(ARCH),x86_64) - QEMU_BIN := qemu-system-x86_64 + QEMU_BIN ?= qemu-system-x86_64 # 8250.nr_uarts=1 is needed because some Android kernels set # `CONFIG_SERIAL_8250_RUNTIME_UARTS` to zero @@ -449,14 +499,26 @@ ifeq ($(ARCH),x86_64) QEMU_ARGS += -accel kvm endif else ifeq ($(ARCH),i386) - QEMU_BIN := qemu-system-i386 + QEMU_BIN ?= qemu-system-i386 QEMU_KERNEL_CMDLINE += console=ttyS0 else - QEMU_BIN := qemu-system-aarch64 + ifeq ($(TRUSTY),1) + # Trusty currently only runs with its patched build of QEMU + QEMU_BIN ?= $(TRUSTY_OUT)/qemu-build/aarch64-softmmu/qemu-system-aarch64 + else + QEMU_BIN ?= qemu-system-aarch64 + endif + QEMU_KERNEL_CMDLINE += console=ttyAMA0 + ifeq ($(TRUSTY),1) + MACHINE := -machine virt,secure=on,virtualization=on + else + MACHINE := -machine virt,virtualization=on + endif + QEMU_ARGS += \ - -M virt \ + $(MACHINE) \ -cpu cortex-a53 \ -semihosting-config enable=on,target=native endif @@ -474,10 +536,18 @@ endif QEMU_ARGS += -append "$(QEMU_KERNEL_CMDLINE) $(QEMU_EXTRA_KERNEL_CMDLINE)" RUN_DEPS := $(QEMU_KERNEL_IMAGE) +RUN_DIR := $(ROOT_DIR) + ifeq ($(UBOOT),1) RUN_DEPS += $(UBOOT_BIN) endif +ifeq ($(TRUSTY),1) + RUN_DEPS += $(ATF_BL1) + # TODO: Is there a QEMU flag we can use to make this not necessary? + RUN_DIR := $(ATF_DIR) +endif + # Make sure the modules directory exists, even if it's empty. Otherwise mount # will fail. $(LINUX_MODULES_INSTALL_PATH)/lib/modules: @@ -496,7 +566,7 @@ endif endif @echo '' - $(QEMU_BIN) $(QEMU_ARGS) + cd $(RUN_DIR) && $(QEMU_BIN) $(QEMU_ARGS) .PHONY: run-ack run-ack: run From 4b8ac4b6b67a56273ad7dbbf95ebc3e1875b5f6c Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Mon, 26 Feb 2024 19:36:44 -0800 Subject: [PATCH 06/17] Add Trusty GDB support --- Makefile | 2 +- scripts/gdb.sh | 11 +++++++++-- scripts/gdbinit.gdb | 9 +++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index f5cb71a..d1cce49 100644 --- a/Makefile +++ b/Makefile @@ -557,7 +557,7 @@ $(LINUX_MODULES_INSTALL_PATH)/lib/modules: run: $(RUN_DEPS) | $(SHARED_DIR) $(LINUX_MODULES_INSTALL_PATH)/lib/modules @echo "$(GREEN)Running QEMU, press 'ctrl-a x' to quit $(NC)" ifeq ($(GDB),1) - @echo "$(ARCH) $(ACK)" > $(OUT_DIR)/.gdb + @echo "$(ARCH) $(ACK) $(TRUSTY) $(TRUSTY_TARGET)" > $(OUT_DIR)/.gdb @echo "$(GREEN)Waiting for GDB, attach with \`scripts/gdb.sh\` $(NC)" ifdef TERMINAL_CMD diff --git a/scripts/gdb.sh b/scripts/gdb.sh index 3631098..4bd9a48 100755 --- a/scripts/gdb.sh +++ b/scripts/gdb.sh @@ -18,7 +18,7 @@ alternatively run the following command: EOF fi -IFS=" " read -r ARCH ACK < $GDB_FILE +IFS=" " read -r ARCH ACK TRUSTY TRUSTY_TARGET < $GDB_FILE if [[ "$ACK" -eq 1 ]]; then LINUX_OUT=$OUT_DIR/ack/common/$ARCH @@ -26,8 +26,15 @@ else LINUX_OUT=$OUT_DIR/linux/$ARCH fi +TRUSTY_OUT=$OUT_DIR/trusty/build-$TRUSTY_TARGET + OUTPUT=$(mktemp) -sed "s|##LINUX_OUT##|${LINUX_OUT}|g" "$GDBINIT" > "$OUTPUT" +cp "$GDBINIT" "$OUTPUT" +echo "Using gdbscript in $OUTPUT" + +sed -i "s|##LINUX_OUT##|${LINUX_OUT}|g" "$OUTPUT" +sed -i "s|##TRUSTY_OUT##|${TRUSTY_OUT}|g" "$OUTPUT" +sed -i "s|##TRUSTY##|${TRUSTY}|g" "$OUTPUT" if [[ $ARCH == "x86_64" ]]; then GDB=gdb diff --git a/scripts/gdbinit.gdb b/scripts/gdbinit.gdb index e8cfa17..a1cc0ac 100644 --- a/scripts/gdbinit.gdb +++ b/scripts/gdbinit.gdb @@ -1,4 +1,13 @@ file ##LINUX_OUT##/vmlinux source ##LINUX_OUT##/vmlinux-gdb.py + target remote :1234 + # add-symbol-file ##LINUX_OUT##/modules_install/lib/modules/5.10.107/extra/my_module.ko -s .text 0xffffffc0091b0800 + +if ##TRUSTY## == 1 + add-symbol-file ##TRUSTY_OUT##/atf/qemu/debug/bl1/bl1.elf 0x0 + add-symbol-file ##TRUSTY_OUT##/atf/qemu/debug/bl2/bl2.elf 0x0 + add-symbol-file ##TRUSTY_OUT##/atf/qemu/debug/bl31/bl31.elf 0xe0a0000 + add-symbol-file ##TRUSTY_OUT##/lk.elf 0xe200000 +endif From d072ba95e1efb528a874c05d91e7eeecf40338de Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Mon, 26 Feb 2024 19:50:04 -0800 Subject: [PATCH 07/17] Support building Trusty with bear --- Makefile | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d1cce49..885f304 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,8 @@ VERBOSE ?= 0 ARCH ?= x86_64 +BEAR ?= 0 + GREEN := $(shell tput setaf 2) YELLOW := $(shell tput setaf 3) NC := $(shell tput sgr0) @@ -405,6 +407,11 @@ ATF_DIR := $(TRUSTY_OUT)/atf/qemu/debug ATF_BL1 := $(ATF_DIR)/bl1.bin ATF_BL33 := $(ATF_DIR)/bl33.bin +BEAR_CMD := +ifeq ($(BEAR),1) + BEAR_CMD := bear -- +endif + .PHONY: trusty-init trusty-init: mkdir -p $(TRUSTY_SRC) @@ -413,7 +420,9 @@ trusty-init: .PHONY: trusty trusty $(ATF_BL1): | $(TRUSTY_SRC) - $(TRUSTY_SRC)/trusty/vendor/google/aosp/scripts/build.py --build-root $(TRUSTY_BUILD_ROOT) --skip-tests $(TRUSTY_TARGET) + $(BEAR_CMD) $(TRUSTY_SRC)/trusty/vendor/google/aosp/scripts/build.py \ + --build-root $(TRUSTY_BUILD_ROOT) \ + --skip-tests $(TRUSTY_TARGET) $(MAKE) trusty_bl33 .PHONY: trusty_bl33 From c517ab8a6c2bb73e91c7946f9a2753893bb9ae05 Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Tue, 27 Feb 2024 22:56:16 -0800 Subject: [PATCH 08/17] Use custom QEMU build file --- Makefile | 33 +++++++++++-- config/trusty/qemu-qemu-inc.mk | 89 ++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+), 5 deletions(-) create mode 100644 config/trusty/qemu-qemu-inc.mk diff --git a/Makefile b/Makefile index 885f304..bf01099 100644 --- a/Makefile +++ b/Makefile @@ -409,15 +409,36 @@ ATF_BL33 := $(ATF_DIR)/bl33.bin BEAR_CMD := ifeq ($(BEAR),1) - BEAR_CMD := bear -- + BEAR_CMD := bear --output $(TRUSTY_SRC) -- endif -.PHONY: trusty-init -trusty-init: +# Trusty is not a submodule, we need to sync it manually +$(TRUSTY_SRC): mkdir -p $(TRUSTY_SRC) + +.PHONY: trusty-init +trusty-init: | $(TRUSTY_SRC) cd $(TRUSTY_SRC) && repo init -u https://android.googlesource.com/trusty/manifest -b main cd $(TRUSTY_SRC) && repo sync -j`nproc` -c --no-tags + $(MAKE) trusty-qemu-init + +# QEMU 3.0 is used by trusty by default. Let's use something newer, we just +# need to apply some Android-specific patches +.PHONY: trusty-qemu-init +trusty-qemu-init: + # TODO: Make this a loop, put commit message of each in a comment + # TODO: Hardcode a stable version commit instead + # TODO: Can we skip the third one + cd $(TRUSTY_SRC)/external/qemu \ + && git checkout aosp/upstream-master \ + && git cherry-pick a4d024b2fdcc478402d00890965eeacb5542c12e \ + && git cherry-pick f060068503259b661be8bd8c803291ff6412d2d6 \ + && git cherry-pick 8a933fbb9c6fb8add1c74f5b523ecb44da7372fa \ + && git cherry-pick 0bfea6599b8a3ebd2c3f98bf6e0d2705e5cb609c + + sed -i 's|include project/qemu-qemu-inc.mk|include $(CONFIG_DIR)/trusty/qemu-qemu-inc.mk|g' $(TRUSTY_SRC)/trusty/device/arm/generic-arm64/project/qemu-inc.mk + .PHONY: trusty trusty $(ATF_BL1): | $(TRUSTY_SRC) $(BEAR_CMD) $(TRUSTY_SRC)/trusty/vendor/google/aosp/scripts/build.py \ @@ -463,6 +484,8 @@ QEMU_ARGS := \ -no-reboot \ -kernel $(QEMU_KERNEL_IMAGE) \ -echr $(ECHR) \ + -netdev user,id=eth0,hostfwd=tcp::7777-:7777,hostfwd=tcp::2222-:22,hostfwd=tcp::2223-:23 -device virtio-net-pci,netdev=eth0 \ + -virtfs local,security_model=mapped-xattr,path=$(SHARED_DIR),mount_tag=shared \ $(QEMU_EXTRA_ARGS) ifeq ($(TRUSTY),1) @@ -512,8 +535,8 @@ else ifeq ($(ARCH),i386) QEMU_KERNEL_CMDLINE += console=ttyS0 else ifeq ($(TRUSTY),1) - # Trusty currently only runs with its patched build of QEMU - QEMU_BIN ?= $(TRUSTY_OUT)/qemu-build/aarch64-softmmu/qemu-system-aarch64 + # Trusty needs to use its own build of QEMU which has some custom patches + QEMU_BIN ?= $(TRUSTY_OUT)/qemu-build/qemu-system-aarch64 else QEMU_BIN ?= qemu-system-aarch64 endif diff --git a/config/trusty/qemu-qemu-inc.mk b/config/trusty/qemu-qemu-inc.mk new file mode 100644 index 0000000..0cee562 --- /dev/null +++ b/config/trusty/qemu-qemu-inc.mk @@ -0,0 +1,89 @@ +# +# Copyright (c) 2018, Google, Inc. All rights reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# This makefile contains rules for building QEMU for running Trusty. +# It is expected that it will be included by the project that uses QEMU +# and the caller will configure the following variables: +# +# QEMU_ROOT - Root of qemu project +# QEMU_BUILD_BASE - location that will be used to store temp files and +# build results. +# QEMU_ARCH - qemu arch to build +# QEMU_TARGET - targets to build, use comma to separate targets +# if multiple targets are specified. +# +# The following variable is returned to the caller: +# QEMU_BIN - resulting qemu image +# QEMU_BUILD_BASE - location that will be used to store temp files and +# build results. +# +# + +QEMU_BIN:=$(QEMU_BUILD_BASE)/$(QEMU_ARCH)-softmmu/qemu-system-$(QEMU_ARCH) +QEMU_MAKEFILE:=$(QEMU_BUILD_BASE)/Makefile + +# Set of features disabled by the AOSP emulator. We don't need these features +# either, and we want minimal dependencies. +QEMU_AOSP_DISABLES := \ + --disable-curses \ + --disable-docs \ + --disable-glusterfs \ + --disable-gtk \ + --disable-spice \ + --disable-opengl \ + +# Warnings which pollute the build output and which can make us miss +# warnings in non-external code that we should be paying attention to. +QEMU_EXTRA_CFLAGS := \ + -Wno-address-of-packed-member \ + -Wno-format-truncation \ + -Wno-stringop-truncation \ + -Wno-array-bounds \ + +# Newer capstone releases have the headers under include/capstone +QEMU_EXTRA_CFLAGS += -I$(TRUSTY_TOP)/$(QEMU_ROOT)/capstone/include/capstone + +$(QEMU_MAKEFILE): QEMU_ROOT:=$(QEMU_ROOT) +$(QEMU_MAKEFILE): QEMU_BUILD_BASE:=$(QEMU_BUILD_BASE) +$(QEMU_MAKEFILE): QEMU_TARGET:=$(QEMU_TARGET) +$(QEMU_MAKEFILE): QEMU_AOSP_DISABLES:=$(QEMU_AOSP_DISABLES) +$(QEMU_MAKEFILE): QEMU_EXTRA_CFLAGS:=$(QEMU_EXTRA_CFLAGS) +$(QEMU_MAKEFILE): + mkdir -p $(QEMU_BUILD_BASE) + #--with-git=true sets the "git" program to /bin/true - it essentially disables git + #--disable-git-update may look like what we want, but it requests manual intervention, not disables git + # TODO(b/148904400): Our prebuilt Clang can't build QEMU yet, and there is no + # prebuilts GCC, i.e. currently we can only build QEMU with host toolchain. On + # some hosts compiler will complain about stringop truncation. + cd $(QEMU_BUILD_BASE) && $(abspath $(QEMU_ROOT)/configure) \ + --target-list=$(QEMU_TARGET) --disable-werror \ + --extra-cflags="$(QEMU_EXTRA_CFLAGS)" \ + --disable-gcrypt $(QEMU_AOSP_DISABLES) \ + --enable-slirp + +$(QEMU_BIN): QEMU_BUILD_BASE:=$(QEMU_BUILD_BASE) +$(QEMU_BIN): $(QEMU_MAKEFILE) .PHONY + $(MAKE) -C $(QEMU_BUILD_BASE) + +# Add QEMU_BIN to the list of project dependencies +EXTRA_BUILDDEPS += $(QEMU_BIN) + +QEMU_ARCH:= +QEMU_ROOT:= +QEMU_TARGET:= +QEMU_AOSP_DISABLES:= +QEMU_EXTRA_CFLAGS:= From a89d4810ac4c1e907797a801f75e557203fa9342 Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Wed, 28 Feb 2024 12:30:20 -0800 Subject: [PATCH 09/17] Add QEMU upstream remote and checkout stable branch --- Makefile | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index bf01099..3eb3331 100644 --- a/Makefile +++ b/Makefile @@ -403,6 +403,8 @@ TRUSTY_TARGET ?= qemu-generic-arm64-test-debug TRUSTY_BUILD_ROOT ?= $(OUT_DIR)/trusty TRUSTY_OUT := $(TRUSTY_BUILD_ROOT)/build-$(TRUSTY_TARGET) +QEMU_BRANCH := stable-7.2 + ATF_DIR := $(TRUSTY_OUT)/atf/qemu/debug ATF_BL1 := $(ATF_DIR)/bl1.bin ATF_BL33 := $(ATF_DIR)/bl33.bin @@ -424,14 +426,19 @@ trusty-init: | $(TRUSTY_SRC) $(MAKE) trusty-qemu-init # QEMU 3.0 is used by trusty by default. Let's use something newer, we just -# need to apply some Android-specific patches +# need to apply some Android-specific patches: +# a4d024b2 arm_gic: Implement GICC_AIAR, GICC_AEOIR and GICC_AHPPIR +# f0600685 Fix GIC model for aliased interrupts +# 8a933fbb hw/virt/arm: double the amount of secure memory +# 0bfea659 hw/arm/virt: Commandeer most of PCIE_MMIO region for secure memory .PHONY: trusty-qemu-init trusty-qemu-init: - # TODO: Make this a loop, put commit message of each in a comment - # TODO: Hardcode a stable version commit instead - # TODO: Can we skip the third one + - cd $(TRUSTY_SRC)/external/qemu \ + && git remote add upstream https://gitlab.com/qemu-project/qemu.git + cd $(TRUSTY_SRC)/external/qemu \ - && git checkout aosp/upstream-master \ + && git fetch upstream $(QEMU_BRANCH) \ + && git checkout upstream/$(QEMU_BRANCH) \ && git cherry-pick a4d024b2fdcc478402d00890965eeacb5542c12e \ && git cherry-pick f060068503259b661be8bd8c803291ff6412d2d6 \ && git cherry-pick 8a933fbb9c6fb8add1c74f5b523ecb44da7372fa \ From 5f03a29c7ffa99fc6a1b374c465901bb1a4cc1c8 Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Thu, 29 Feb 2024 12:49:40 -0800 Subject: [PATCH 10/17] WIP: android userspace --- Makefile | 79 +++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 69 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 3eb3331..c226817 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ SCRIPT_DIR := $(ROOT_DIR)/scripts CONFIG_DIR := $(ROOT_DIR)/config CLANG_DIR ?= $(ROOT_DIR)/toolchain/clang -CLANG_URL := https://github.com/llvm/llvm-project/releases/download/llvmorg-15.0.6/clang+llvm-15.0.6-x86_64-linux-gnu-ubuntu-18.04.tar.xz +CLANG_URL := https://github.com/llvm/llvm-project/releases/download/llvmorg-17.0.6/clang+llvm-17.0.6-x86_64-linux-gnu-ubuntu-22.04.tar.xz SHARED_DIR := $(ROOT_DIR)/shared @@ -42,6 +42,7 @@ endif .PHONY: default default: linux linux_modules tools-vm +default: linux linux_modules tools-vm .PHONY: clean clean: linux_clean tools-vm_clean @@ -151,6 +152,8 @@ else LINUX_DEFCONFIG ?= defconfig endif +TRUSTY_LINUX_CONFIG_FRAGMENT := $(ROOT_DIR)/trusty/external/linux/arch/arm64/configs/trusty_qemu_defconfig.fragment + LINUX_CONFIG_FRAGMENT ?= $(CONFIG_DIR)/config.fragment LINUX_OUT_MODULES_DEP := $(LINUX_OUT)/modules_install.stamp LINUX_MODULES_INSTALL_PATH := $(LINUX_OUT)/modules_install @@ -451,10 +454,10 @@ trusty $(ATF_BL1): | $(TRUSTY_SRC) $(BEAR_CMD) $(TRUSTY_SRC)/trusty/vendor/google/aosp/scripts/build.py \ --build-root $(TRUSTY_BUILD_ROOT) \ --skip-tests $(TRUSTY_TARGET) - $(MAKE) trusty_bl33 + $(MAKE) trusty-bl33 -.PHONY: trusty_bl33 -trusty_bl33 $(ATF_BL33): $(UBOOT_BIN) +.PHONY: trusty-bl33 +trusty-bl33 $(ATF_BL33): $(UBOOT_BIN) rm $(ATF_BL33) cp $(UBOOT_BIN) $(ATF_BL33) @@ -462,6 +465,32 @@ trusty_bl33 $(ATF_BL33): $(UBOOT_BIN) trusty_clean: rm -rf $(TRUSTY_OUT) +## +## Android +## + +# TODO: We shouldn't depend on Trusty for this +TRUSTY_PREBUILT_IMAGE_DIR := $(TRUSTY_SRC)/trusty/prebuilts/aosp/android/out/target/product/trusty/ +SYSTEM_IMG := $(TRUSTY_PREBUILT_IMAGE_DIR)/system.img +VENDOR_IMG := $(TRUSTY_PREBUILT_IMAGE_DIR)/vendor.img +USERDATA_IMG := $(TRUSTY_PREBUILT_IMAGE_DIR)/userdata.img + +ANDROID_USERSPACE ?= 0 + +DTC := dtc + +ANDROID_DTS := $(OUT_DIR)/android.dts +ANDROID_DTB := $(OUT_DIR)/android.dtb +QEMU_DTB := $(OUT_DIR)/qemu.dtb +QEMU_DTS := $(OUT_DIR)/qemu.dts + +.PHONY: android_dtb +android-dtb $(ANDROID_DTB): + QEMU_EXTRA_ARGS="-M dumpdtb=$(QEMU_DTB)" $(MAKE) run + $(DTC) -I dtb -O dts $(QEMU_DTB) > $(QEMU_DTS) + cat $(QEMU_DTS) $(ATF_DIR)/firmware.android.dts > $(ANDROID_DTS) + $(DTC) -I dts -O dtb $(ANDROID_DTS) > $(ANDROID_DTB) + ## ## Run QEMU ## @@ -482,7 +511,13 @@ KASLR ?= 0 UBOOT ?= 0 TRUSTY ?= 0 -QEMU_KERNEL_CMDLINE := selinux=0 +ifeq ($(ANDROID_USERSPACE),1) + SELINUX ?= 1 +else + SELINUX ?= 0 +endif + +QEMU_KERNEL_CMDLINE := selinux=$(SELINUX) QEMU_ARGS := \ -m $(MEM) \ @@ -491,10 +526,14 @@ QEMU_ARGS := \ -no-reboot \ -kernel $(QEMU_KERNEL_IMAGE) \ -echr $(ECHR) \ - -netdev user,id=eth0,hostfwd=tcp::7777-:7777,hostfwd=tcp::2222-:22,hostfwd=tcp::2223-:23 -device virtio-net-pci,netdev=eth0 \ - -virtfs local,security_model=mapped-xattr,path=$(SHARED_DIR),mount_tag=shared \ $(QEMU_EXTRA_ARGS) +ifneq ($(TRUSTY),1) + QEMU_ARGS += -netdev user,id=eth0,hostfwd=tcp::7777-:7777,hostfwd=tcp::2222-:22,hostfwd=tcp::2223-:23 -device virtio-net-pci,netdev=eth0 + QEMU_ARGS += -virtfs local,security_model=mapped-xattr,path=$(SHARED_DIR),mount_tag=shared + #QEMU_ARGS += -virtfs local,security_model=mapped-xattr,path=$(LINUX_MODULES_INSTALL_PATH)/lib/modules,mount_tag=modules +endif + ifeq ($(TRUSTY),1) QEMU_ARGS += -bios $(ATF_BL1) else ifeq ($(UBOOT),1) @@ -509,7 +548,16 @@ ifneq ($(TRUSTY),1) QEMU_ARGS += -virtfs local,security_model=mapped-xattr,path=$(LINUX_MODULES_INSTALL_PATH)/lib/modules,mount_tag=modules endif -ifneq ($(INITRD),) +# TODO: check that ROOTFS and ANDROID_USERSPACE aren't both passed at the same time? +ifeq ($(ANDROID_USERSPACE),1) + QEMU_ARGS += \ + -device virtio-blk,drive=vda -drive file=$(SYSTEM_IMG),index=0,if=none,id=vda,format=raw \ + -device virtio-blk,drive=vdb -drive file=$(VENDOR_IMG),index=1,if=none,id=vdb,format=raw \ + -device virtio-blk,drive=vdc -drive file=$(USERDATA_IMG),index=2,if=none,id=vdc,format=raw \ + -device virtio-net,netdev=adbnet0 -netdev user,id=adbnet0,hostfwd=tcp::5554-:5554,hostfwd=tcp::5555-:5555 + QEMU_KERNEL_CMDLINE += root=$(ROOT) $(RW) kvm-arm.mode=protected earlyprintk androidboot.hardware=qemu_trusty trusty-log.log_ratelimit_interval=0 trusty-log.log_to_dmesg=always + QEMU_ARGS += -dtb $(ANDROID_DTB) +else ifneq ($(INITRD),) ifeq ($(INITRD),1) INITRD := $(CPIO_FILE) endif @@ -543,9 +591,10 @@ else ifeq ($(ARCH),i386) else ifeq ($(TRUSTY),1) # Trusty needs to use its own build of QEMU which has some custom patches - QEMU_BIN ?= $(TRUSTY_OUT)/qemu-build/qemu-system-aarch64 + # QEMU_BIN ?= $(TRUSTY_OUT)/qemu-build/qemu-system-aarch64 + QEMU_BIN ?= $(TRUSTY_OUT)/qemu-build/aarch64-softmmu/qemu-system-aarch64 else - QEMU_BIN ?= qemu-system-aarch64 + QEMU_BIN ?= qemu-system-aarch64 endif QEMU_KERNEL_CMDLINE += console=ttyAMA0 @@ -574,6 +623,11 @@ endif QEMU_ARGS += -append "$(QEMU_KERNEL_CMDLINE) $(QEMU_EXTRA_KERNEL_CMDLINE)" +# Force using the Trusty kernel image for now, there are boot errors with other images +ifeq ($(TRUSTY),1) + QEMU_KERNEL_IMAGE := $(TRUSTY_OUT)/linux-build/arch/arm64/boot/Image +endif + RUN_DEPS := $(QEMU_KERNEL_IMAGE) RUN_DIR := $(ROOT_DIR) @@ -587,6 +641,11 @@ ifeq ($(TRUSTY),1) RUN_DIR := $(ATF_DIR) endif +ifeq ($(ANDROID_USERSPACE),1) + # We need this device tree blob to mount /vendor + RUN_DEPS += $(ANDROID_DTB) +endif + # Make sure the modules directory exists, even if it's empty. Otherwise mount # will fail. $(LINUX_MODULES_INSTALL_PATH)/lib/modules: From 52587d37b76e9ab5eb4f9f88d08e444e5e461294 Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Thu, 7 Mar 2024 14:11:52 -0800 Subject: [PATCH 11/17] Add note about libslirp-dev --- config/trusty/qemu-qemu-inc.mk | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/config/trusty/qemu-qemu-inc.mk b/config/trusty/qemu-qemu-inc.mk index 0cee562..f5f55ad 100644 --- a/config/trusty/qemu-qemu-inc.mk +++ b/config/trusty/qemu-qemu-inc.mk @@ -64,11 +64,7 @@ $(QEMU_MAKEFILE): QEMU_AOSP_DISABLES:=$(QEMU_AOSP_DISABLES) $(QEMU_MAKEFILE): QEMU_EXTRA_CFLAGS:=$(QEMU_EXTRA_CFLAGS) $(QEMU_MAKEFILE): mkdir -p $(QEMU_BUILD_BASE) - #--with-git=true sets the "git" program to /bin/true - it essentially disables git - #--disable-git-update may look like what we want, but it requests manual intervention, not disables git - # TODO(b/148904400): Our prebuilt Clang can't build QEMU yet, and there is no - # prebuilts GCC, i.e. currently we can only build QEMU with host toolchain. On - # some hosts compiler will complain about stringop truncation. + # Note: `libslirp-dev` must be installed before running this command cd $(QEMU_BUILD_BASE) && $(abspath $(QEMU_ROOT)/configure) \ --target-list=$(QEMU_TARGET) --disable-werror \ --extra-cflags="$(QEMU_EXTRA_CFLAGS)" \ From b9ba02091af946d0190ceb908bd48bfbea380aef Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Thu, 7 Mar 2024 14:29:33 -0800 Subject: [PATCH 12/17] Use correct kernel image with Trusty --- Makefile | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index c226817..3264483 100644 --- a/Makefile +++ b/Makefile @@ -405,6 +405,7 @@ TRUSTY_SRC ?= $(ROOT_DIR)/trusty TRUSTY_TARGET ?= qemu-generic-arm64-test-debug TRUSTY_BUILD_ROOT ?= $(OUT_DIR)/trusty TRUSTY_OUT := $(TRUSTY_BUILD_ROOT)/build-$(TRUSTY_TARGET) +TRUSTY_KERNEL_IMAGE := $(TRUSTY_OUT)/linux-build/arch/arm64/boot/Image QEMU_BRANCH := stable-7.2 @@ -450,7 +451,7 @@ trusty-qemu-init: sed -i 's|include project/qemu-qemu-inc.mk|include $(CONFIG_DIR)/trusty/qemu-qemu-inc.mk|g' $(TRUSTY_SRC)/trusty/device/arm/generic-arm64/project/qemu-inc.mk .PHONY: trusty -trusty $(ATF_BL1): | $(TRUSTY_SRC) +trusty $(ATF_BL1) $(TRUSTY_KERNEL_IMAGE): | $(TRUSTY_SRC) $(BEAR_CMD) $(TRUSTY_SRC)/trusty/vendor/google/aosp/scripts/build.py \ --build-root $(TRUSTY_BUILD_ROOT) \ --skip-tests $(TRUSTY_TARGET) @@ -465,6 +466,11 @@ trusty-bl33 $(ATF_BL33): $(UBOOT_BIN) trusty_clean: rm -rf $(TRUSTY_OUT) +# When Trusty is enabled, use the kernel image built by Trusty +ifeq ($(TRUSTY),1) + KERNEL_IMAGE := $(TRUSTY_KERNEL_IMAGE) +endif + ## ## Android ## @@ -623,11 +629,6 @@ endif QEMU_ARGS += -append "$(QEMU_KERNEL_CMDLINE) $(QEMU_EXTRA_KERNEL_CMDLINE)" -# Force using the Trusty kernel image for now, there are boot errors with other images -ifeq ($(TRUSTY),1) - QEMU_KERNEL_IMAGE := $(TRUSTY_OUT)/linux-build/arch/arm64/boot/Image -endif - RUN_DEPS := $(QEMU_KERNEL_IMAGE) RUN_DIR := $(ROOT_DIR) From 67d34193c0063845037a14d1a50420a3ad765f50 Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Thu, 7 Mar 2024 14:37:49 -0800 Subject: [PATCH 13/17] Warn if attempting to use ACK with TRUSTY --- Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Makefile b/Makefile index 3264483..3cdb0ce 100644 --- a/Makefile +++ b/Makefile @@ -468,6 +468,12 @@ trusty_clean: # When Trusty is enabled, use the kernel image built by Trusty ifeq ($(TRUSTY),1) + ifeq ($(ACK),1) + ifndef QEMU_KERNEL_IMAGE + $(warning $(YELLOW)ACK was enabled, but Trusty runs with its own kernel. To force using a specific kernel image, add `QEMU_KERNEL_IMAGE=$(KERNEL_IMAGE)` to the command line $(NC)) + endif + endif + KERNEL_IMAGE := $(TRUSTY_KERNEL_IMAGE) endif From acbd5733ecb78b3db4a46f94c4b69f792b4fddfe Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Fri, 8 Mar 2024 15:54:56 -0800 Subject: [PATCH 14/17] Only run android-dtb for arm64 --- Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3cdb0ce..cf127d3 100644 --- a/Makefile +++ b/Makefile @@ -496,8 +496,11 @@ ANDROID_DTB := $(OUT_DIR)/android.dtb QEMU_DTB := $(OUT_DIR)/qemu.dtb QEMU_DTS := $(OUT_DIR)/qemu.dts -.PHONY: android_dtb +.PHONY: android-dtb android-dtb $(ANDROID_DTB): +ifneq ($(ARCH),arm64) + $(error android-dtb is only supported from arm64) +endif QEMU_EXTRA_ARGS="-M dumpdtb=$(QEMU_DTB)" $(MAKE) run $(DTC) -I dtb -O dts $(QEMU_DTB) > $(QEMU_DTS) cat $(QEMU_DTS) $(ATF_DIR)/firmware.android.dts > $(ANDROID_DTS) From d7965e2b2e7b7c7dcfde80c87bc201963c08d0ab Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Fri, 8 Mar 2024 15:55:45 -0800 Subject: [PATCH 15/17] Symlink to old QEMU binary --- Makefile | 4 ++-- config/trusty/qemu-qemu-inc.mk | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index cf127d3..2ebf24d 100644 --- a/Makefile +++ b/Makefile @@ -499,8 +499,9 @@ QEMU_DTS := $(OUT_DIR)/qemu.dts .PHONY: android-dtb android-dtb $(ANDROID_DTB): ifneq ($(ARCH),arm64) - $(error android-dtb is only supported from arm64) + $(error android-dtb is only supported from arm64) endif + QEMU_EXTRA_ARGS="-M dumpdtb=$(QEMU_DTB)" $(MAKE) run $(DTC) -I dtb -O dts $(QEMU_DTB) > $(QEMU_DTS) cat $(QEMU_DTS) $(ATF_DIR)/firmware.android.dts > $(ANDROID_DTS) @@ -606,7 +607,6 @@ else ifeq ($(ARCH),i386) else ifeq ($(TRUSTY),1) # Trusty needs to use its own build of QEMU which has some custom patches - # QEMU_BIN ?= $(TRUSTY_OUT)/qemu-build/qemu-system-aarch64 QEMU_BIN ?= $(TRUSTY_OUT)/qemu-build/aarch64-softmmu/qemu-system-aarch64 else QEMU_BIN ?= qemu-system-aarch64 diff --git a/config/trusty/qemu-qemu-inc.mk b/config/trusty/qemu-qemu-inc.mk index f5f55ad..88f0384 100644 --- a/config/trusty/qemu-qemu-inc.mk +++ b/config/trusty/qemu-qemu-inc.mk @@ -57,6 +57,7 @@ QEMU_EXTRA_CFLAGS := \ # Newer capstone releases have the headers under include/capstone QEMU_EXTRA_CFLAGS += -I$(TRUSTY_TOP)/$(QEMU_ROOT)/capstone/include/capstone +$(QEMU_MAKEFILE): QEMU_ARCH:=$(QEMU_ARCH) $(QEMU_MAKEFILE): QEMU_ROOT:=$(QEMU_ROOT) $(QEMU_MAKEFILE): QEMU_BUILD_BASE:=$(QEMU_BUILD_BASE) $(QEMU_MAKEFILE): QEMU_TARGET:=$(QEMU_TARGET) @@ -71,6 +72,9 @@ $(QEMU_MAKEFILE): --disable-gcrypt $(QEMU_AOSP_DISABLES) \ --enable-slirp + # Symlink to the old QEMU path so we can always find it at the same place + ln -s $(QEMU_BUILD_BASE)/qemu-system-$(QEMU_ARCH) $(QEMU_BIN) + $(QEMU_BIN): QEMU_BUILD_BASE:=$(QEMU_BUILD_BASE) $(QEMU_BIN): $(QEMU_MAKEFILE) .PHONY $(MAKE) -C $(QEMU_BUILD_BASE) From 234dfce9ffe71f2ea568f7492c7b40e4bcb7a399 Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Fri, 8 Mar 2024 17:37:51 -0800 Subject: [PATCH 16/17] Regenerate DTB if QEMU_BIN changes --- Makefile | 48 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 2ebf24d..b6617e6 100644 --- a/Makefile +++ b/Makefile @@ -409,6 +409,11 @@ TRUSTY_KERNEL_IMAGE := $(TRUSTY_OUT)/linux-build/arch/arm64/boot/Image QEMU_BRANCH := stable-7.2 +ifeq ($(TRUSTY),1) + # Trusty needs to use its own build of QEMU which has some custom patches + QEMU_BIN ?= $(TRUSTY_OUT)/qemu-build/aarch64-softmmu/qemu-system-aarch64 +endif + ATF_DIR := $(TRUSTY_OUT)/atf/qemu/debug ATF_BL1 := $(ATF_DIR)/bl1.bin ATF_BL33 := $(ATF_DIR)/bl33.bin @@ -496,13 +501,21 @@ ANDROID_DTB := $(OUT_DIR)/android.dtb QEMU_DTB := $(OUT_DIR)/qemu.dtb QEMU_DTS := $(OUT_DIR)/qemu.dts +# We use DUMPING_DTB to avoid infinite recursion +DUMPING_DTB := 0 + +# We depend on QEMU_BIN because it's best if we generate this every time (as it +# the DTB can change if run with a different version QEMU and we want to make +# sure it exactly matches the DTB for the current QEMU binary being used) .PHONY: android-dtb android-dtb $(ANDROID_DTB): ifneq ($(ARCH),arm64) $(error android-dtb is only supported from arm64) endif - QEMU_EXTRA_ARGS="-M dumpdtb=$(QEMU_DTB)" $(MAKE) run +# DUMPING_DTB=1 prevents infinite recursion. It must be set as a `make` argument, +# not an environment variable + QEMU_EXTRA_ARGS="-M dumpdtb=$(QEMU_DTB)" $(MAKE) run DUMPING_DTB=1 $(DTC) -I dtb -O dts $(QEMU_DTB) > $(QEMU_DTS) cat $(QEMU_DTS) $(ATF_DIR)/firmware.android.dts > $(ANDROID_DTS) $(DTC) -I dts -O dtb $(ANDROID_DTS) > $(ANDROID_DTB) @@ -572,7 +585,12 @@ ifeq ($(ANDROID_USERSPACE),1) -device virtio-blk,drive=vdc -drive file=$(USERDATA_IMG),index=2,if=none,id=vdc,format=raw \ -device virtio-net,netdev=adbnet0 -netdev user,id=adbnet0,hostfwd=tcp::5554-:5554,hostfwd=tcp::5555-:5555 QEMU_KERNEL_CMDLINE += root=$(ROOT) $(RW) kvm-arm.mode=protected earlyprintk androidboot.hardware=qemu_trusty trusty-log.log_ratelimit_interval=0 trusty-log.log_to_dmesg=always + +# Don't add the DTB as an argument if we're in the process of dumping it +ifneq ($(DUMPING_DTB),1) QEMU_ARGS += -dtb $(ANDROID_DTB) +endif + else ifneq ($(INITRD),) ifeq ($(INITRD),1) INITRD := $(CPIO_FILE) @@ -605,13 +623,7 @@ else ifeq ($(ARCH),i386) QEMU_BIN ?= qemu-system-i386 QEMU_KERNEL_CMDLINE += console=ttyS0 else - ifeq ($(TRUSTY),1) - # Trusty needs to use its own build of QEMU which has some custom patches - QEMU_BIN ?= $(TRUSTY_OUT)/qemu-build/aarch64-softmmu/qemu-system-aarch64 - else - QEMU_BIN ?= qemu-system-aarch64 - endif - + QEMU_BIN ?= qemu-system-aarch64 QEMU_KERNEL_CMDLINE += console=ttyAMA0 ifeq ($(TRUSTY),1) @@ -652,8 +664,15 @@ ifeq ($(TRUSTY),1) endif ifeq ($(ANDROID_USERSPACE),1) - # We need this device tree blob to mount /vendor - RUN_DEPS += $(ANDROID_DTB) + # We need a device tree blob to mount /vendor. If DUMPING_DTB is set, that + # means we're already in the process of dumping the DTB and shouldn't add it + # as a run dependency, otherwise we'll run into infinite recursion + ifneq ($(DUMPING_DTB),1) + # Add `android-dtb` instead of ANDROID_DTB so that this target is forced to + # be run every time (to prevent accidentally using a DTB generated from a + # different QEMU binary) + RUN_DEPS += android-dtb + endif endif # Make sure the modules directory exists, even if it's empty. Otherwise mount @@ -663,8 +682,13 @@ $(LINUX_MODULES_INSTALL_PATH)/lib/modules: .PHONY: run run: $(RUN_DEPS) | $(SHARED_DIR) $(LINUX_MODULES_INSTALL_PATH)/lib/modules +ifneq ($(DUMPING_DTB), 1) @echo "$(GREEN)Running QEMU, press 'ctrl-a x' to quit $(NC)" +endif + ifeq ($(GDB),1) + +ifneq ($(DUMPING_DTB),1) @echo "$(ARCH) $(ACK) $(TRUSTY) $(TRUSTY_TARGET)" > $(OUT_DIR)/.gdb @echo "$(GREEN)Waiting for GDB, attach with \`scripts/gdb.sh\` $(NC)" @@ -672,7 +696,9 @@ ifdef TERMINAL_CMD $(TERMINAL_CMD) $(SCRIPT_DIR)/gdb.sh endif -endif +endif # DUMPING_DTB +endif # GDB + @echo '' cd $(RUN_DIR) && $(QEMU_BIN) $(QEMU_ARGS) From 112f824df70f0bbbcb64533bde12111f45c94224 Mon Sep 17 00:00:00 2001 From: Gulshan Singh Date: Sun, 10 Mar 2024 18:47:26 -0700 Subject: [PATCH 17/17] Configure with Trusty config fragment if Trusty is enabled --- Makefile | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index b6617e6..2b91d1b 100644 --- a/Makefile +++ b/Makefile @@ -159,6 +159,11 @@ LINUX_OUT_MODULES_DEP := $(LINUX_OUT)/modules_install.stamp LINUX_MODULES_INSTALL_PATH := $(LINUX_OUT)/modules_install LINUX_CONFIG := $(LINUX_OUT)/.config +LINUX_CONFIG_FRAGMENTS := $(LINUX_CONFIG_FRAGMENT) +ifeq ($(TRUSTY),1) + LINUX_CONFIG_FRAGMENTS += $(TRUSTY_LINUX_CONFIG_FRAGMENT) +endif + ifeq ($(ARCH),x86_64) TARGET := x86_64-pc-linux-gnu KERNEL_IMAGE := $(LINUX_OUT)/arch/$(ARCH)/boot/bzImage @@ -194,9 +199,14 @@ linux_defconfig $(LINUX_CONFIG): $(LINUX_CONFIG_FRAGMENT) | $(CLANG_DIR) $(LINUX_SRC)/scripts/kconfig/merge_config.sh \ -m \ $(LINUX_CONFIG) \ - $(LINUX_CONFIG_FRAGMENT) - + $(LINUX_MAKE) olddefconfig - $(SCRIPT_DIR)/check_merged_config.sh $(LINUX_CONFIG) $(LINUX_CONFIG_FRAGMENT) + $(LINUX_CONFIG_FRAGMENTS) + + + $(LINUX_MAKE) olddefconfig + + for fragment in $(LINUX_CONFIG_FRAGMENTS); do \ + $(SCRIPT_DIR)/check_merged_config.sh $(LINUX_CONFIG) $$fragment ; \ + done + .PHONY: linux_menuconfig linux_menuconfig: