diff --git a/.github/actions/build-kernel/action.yml b/.github/actions/build-kernel/action.yml index 893df81..97a0e45 100644 --- a/.github/actions/build-kernel/action.yml +++ b/.github/actions/build-kernel/action.yml @@ -31,6 +31,7 @@ runs: build-essential \ bc \ bison \ + debhelper \ flex \ libelf-dev \ libssl-dev \ @@ -81,3 +82,11 @@ runs: ARCH: ${{ inputs.arch }} run: | make -j$(nproc) linux + + - name: Build debpkg + shell: bash + env: + ACK: ${{ inputs.ack }} + ARCH: ${{ inputs.arch }} + run: | + make -j$(nproc) linux_bindebpkg diff --git a/.github/workflows/build-and-test-all.yml b/.github/workflows/build-and-test-all.yml new file mode 100644 index 0000000..aeb4a42 --- /dev/null +++ b/.github/workflows/build-and-test-all.yml @@ -0,0 +1,47 @@ +name: Build and Test All +on: + push: + branches: + - main + path: + - '*' + - '!**/*.md' + pull_request: + workflow_call: + workflow_dispatch: + +jobs: + build: + uses: ./.github/workflows/build.yml + strategy: + matrix: + kernel: [{ack: 0, version: 5.10.y}, {ack: 0, version: 5.15.y}, {ack: 0, version: 6.1.y}, {ack: 0, version: 6.6.y}, {ack: 1, version: android13-5.10-lts}, {ack: 1, version: android14-5.15-lts}] + arch: [arm64, x86_64] + with: + host-os: ubuntu-20.04 + ack: ${{ matrix.kernel.ack }} + arch: ${{ matrix.arch }} + version: ${{ matrix.kernel.version }} + + rootfs: + uses: ./.github/workflows/rootfs.yml + strategy: + matrix: + arch: [arm64, x86_64] + with: + host-os: ubuntu-22.04 + arch: ${{ matrix.arch }} + use-cached: true + + build-and-test: + uses: ./.github/workflows/build-and-test.yml + needs: rootfs + strategy: + matrix: + kernel: [{ack: 0, version: 5.10.y}, {ack: 0, version: 5.15.y}, {ack: 0, version: 6.1.y}, {ack: 0, version: 6.6.y}, {ack: 1, version: android13-5.10-lts}, {ack: 1, version: android14-5.15-lts}] + arch: [arm64, x86_64] + with: + host-os: ubuntu-22.04 + ack: ${{ matrix.kernel.ack }} + arch: ${{ matrix.arch }} + version: ${{ matrix.kernel.version }} diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml new file mode 100644 index 0000000..7349b5c --- /dev/null +++ b/.github/workflows/build-and-test.yml @@ -0,0 +1,70 @@ +name: Build And Test +on: + workflow_dispatch: + inputs: + host-os: + required: false + type: choice + options: + - ubuntu-22.04 + - ubuntu-20.04 + default: ubuntu-22.04 + ack: + required: true + type: choice + options: + - '0' + - '1' + arch: + required: true + type: choice + options: + - x86_64 + - arm64 + version: + required: true + type: string + use-cached: + required: false + type: boolean + default: false + workflow_call: + inputs: + host-os: + required: false + type: string + default: ubuntu-22.04 + ack: + required: true + type: string + arch: + required: true + type: string + version: + required: true + type: string + use-cached: + required: false + type: boolean + default: false + +jobs: + build: + uses: ./.github/workflows/build.yml + with: + host-os: ${{ inputs.host-os }} + ack: ${{ inputs.ack }} + arch: ${{ inputs.arch }} + version: ${{ inputs.version }} + use-cached: ${{ inputs.use-cached }} + + test: + uses: ./.github/workflows/test.yml + needs: build + with: + ack: ${{ inputs.ack }} + arch: ${{ inputs.arch }} + version: ${{ inputs.version }} + artifact-dir: ${{ needs.build.outputs.artifact-dir }} + kernel-image-name: ${{ needs.build.outputs.kernel-image-name }} + suffix: ${{ needs.build.outputs.suffix }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f7b2f71..ea20dad 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,32 +1,108 @@ name: Build -on: [workflow_dispatch] +on: + workflow_dispatch: + inputs: + host-os: + required: true + type: choice + options: + - ubuntu-22.04 + - ubuntu-20.04 + ack: + required: true + type: choice + options: + - "0" + - "1" + arch: + required: true + type: choice + options: + - x86_64 + - arm64 + version: + required: true + type: string + use-cached: + required: false + type: boolean + default: false + workflow_call: + inputs: + host-os: + required: false + type: string + default: ubuntu-22.04 + ack: + required: true + type: string + arch: + required: true + type: string + version: + required: true + type: string + use-cached: + required: false + type: boolean + default: false + outputs: + artifact-dir: + value: ${{ jobs.build.outputs.artifact-dir }} + kernel-image-name: + value: ${{ jobs.build.outputs.kernel-image-name }} + suffix: + value: ${{ jobs.build.outputs.suffix }} + version: + value: ${{ jobs.build.outputs.version }} jobs: build: - strategy: - fail-fast: false - matrix: - os: [ubuntu-22.04, ubuntu-20.04] - kernel: [{ack: 0, version: 5.10.y}, {ack: 0, version: 5.15.y}, {ack: 0, version: 6.1.y}, {ack: 1, version: android13-5.10-lts}] - arch: [arm64, x86_64] - runs-on: ${{ matrix.os }} + name: Build (ack=${{ inputs.ack }}, ${{ inputs.version }}, ${{ inputs.arch }}, ${{ inputs.host-os }}) + runs-on: ${{ inputs.host-os }} timeout-minutes: 100 env: - ACK: ${{ matrix.kernel.ack }} - ARCH: ${{ matrix.arch }} + ACK: ${{ inputs.ack }} + ARCH: ${{ inputs.arch }} outputs: - SHORT_HASH: ${{ steps.vars.outputs.SHORT_HASH }} + artifact-dir: ${{ steps.vars.outputs.ARTIFACT_DIR }} + kernel-image-name: ${{ steps.vars.outputs.KERNEL_IMAGE_NAME }} + suffix: ${{ steps.vars.outputs.SUFFIX }} + version: ${{ steps.version.outputs.VERSION }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + + - name: Get real kernel version + id: version + shell: bash + run: | + LINUX_URL=https://cdn.kernel.org/pub/linux/kernel + VERSION="${{ inputs.version }}" + first_char=${VERSION:0:1} + last_char=${VERSION: -1:1} + if [ $last_char == y ]; then + major_minor=${VERSION: 0:-1} + + # We do this in multiple steps instead of a chain of pipes to make debugging easier + TMP=$(curl -s $LINUX_URL/v${first_char}.x/) + TMP=$(echo "$TMP" | sed -e 's/<[^>]*>//g') + TMP=$(echo "$TMP" | grep -oP "linux-${major_minor}[0-9]+") + TMP=$(echo "$TMP" | sort -r -V) + TMP=$(echo "$TMP" | head -n1) + TMP=$(echo "$TMP" | cut -d '-' -f2) + + VERSION=$TMP + fi + + echo "VERSION=$VERSION" >> $GITHUB_OUTPUT - name: Set output variables id: vars + shell: bash run: | set -x - SHORT_HASH=$(git rev-parse --short HEAD) - if [ $ACK -eq 0 ]; then KERNEL_TYPE=linux OUT_DIR=out/linux/${ARCH} @@ -38,41 +114,35 @@ jobs: if [ $ARCH == x86_64 ]; then # We need to use the real image in the x86 folder because caching # doesn't work with symlinks - KERNEL_IMAGE="${OUT_DIR}/arch/x86/boot/bzImage" + KERNEL_IMAGE="arch/x86/boot/bzImage" elif [ $ARCH == arm64 ]; then - KERNEL_IMAGE="${OUT_DIR}/arch/${ARCH}/boot/Image" + KERNEL_IMAGE="arch/arm64/boot/Image" fi - echo "SHORT_HASH=$SHORT_HASH" >> $GITHUB_OUTPUT + SUFFIX=${KERNEL_TYPE}-${{ steps.version.outputs.VERSION }}-${{ inputs.arch }} + + echo "SUFFIX=$SUFFIX" >> $GITHUB_OUTPUT echo "OUT_DIR=$OUT_DIR" >> $GITHUB_OUTPUT echo "KERNEL_TYPE=$KERNEL_TYPE" >> $GITHUB_OUTPUT echo "KERNEL_IMAGE=$KERNEL_IMAGE" >> $GITHUB_OUTPUT + echo "KERNEL_IMAGE_NAME=$(basename $KERNEL_IMAGE)" >> $GITHUB_OUTPUT - - name: Get real kernel version - run: | - LINUX_URL=https://cdn.kernel.org/pub/linux/kernel - VERSION="${{ matrix.kernel.version }}" - first_char=${VERSION:0:1} - last_char=${VERSION: -1:1} - if [ $last_char == y ]; then - major_minor=${VERSION: 0:-1} - VERSION=$(curl -s $LINUX_URL/v${first_char}.x/ \ - | sed -e 's/<[^>]*>//g' \ - | grep -oP "linux-${major_minor}[0-9]+" \ - | sort -r -V \ - | head -n1 \ - | cut -d '-' -f2) - fi - echo "VERSION=$VERSION" >> $GITHUB_ENV + echo "FINAL_KERNEL_IMAGE_PATH=${OUT_DIR}/$(basename ${KERNEL_IMAGE})-${SUFFIX}" >> $GITHUB_OUTPUT + echo "FINAL_VMLINUX_PATH=${OUT_DIR}/vmlinux-${SUFFIX}" >> $GITHUB_OUTPUT + echo "FINAL_LINUX_HEADERS_PATH=${OUT_DIR}/linux-headers-${SUFFIX}.deb" >> $GITHUB_OUTPUT + + echo "ARTIFACT_DIR=kernel-artifacts-${SUFFIX}" >> $GITHUB_OUTPUT - name: Cache kernel image + if: ${{ inputs.use-cached == true }} id: cache-kernel - uses: actions/cache@v3 + uses: actions/cache@v4 with: - key: ${{ matrix.os }}-${{ env.VERSION }}-${{ matrix.arch }} + key: ${{ inputs.host-os }}-${{ steps.vars.outputs.SUFFIX }} path: | - ${{ steps.vars.outputs.KERNEL_IMAGE }} - ${{ steps.vars.outputs.OUT_DIR }}/vmlinux + ${{ steps.vars.outputs.FINAL_KERNEL_IMAGE_PATH }} + ${{ steps.vars.outputs.FINAL_VMLINUX_PATH }} + ${{ steps.vars.outputs.FINAL_LINUX_HEADERS_PATH }} - run: ls -lR if: ${{ steps.cache-kernel.outputs.cache-hit == 'true' }} @@ -81,164 +151,24 @@ jobs: if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }} uses: ./.github/actions/build-kernel with: - os: ${{ matrix.os }} - arch: ${{ matrix.arch }} - ack: ${{ matrix.kernel.ack }} - kernel-version: ${{ env.VERSION }} + os: ${{ inputs.host-os }} + arch: ${{ inputs.arch }} + ack: ${{ inputs.ack }} + kernel-version: ${{ steps.version.outputs.VERSION }} - name: Rename kernel images + if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }} run: | - SUFFIX=${{ steps.vars.outputs.KERNEL_TYPE }}-${{ env.VERSION }}-${{ matrix.arch }} - cp ${{ steps.vars.outputs.KERNEL_IMAGE }} ${{ steps.vars.outputs.KERNEL_IMAGE }}-$SUFFIX - cp ${{ steps.vars.outputs.OUT_DIR }}/vmlinux ${{ steps.vars.outputs.OUT_DIR }}/vmlinux-$SUFFIX - - echo "SUFFIX=$SUFFIX" >> $GITHUB_ENV + mv ${{ steps.vars.outputs.OUT_DIR }}/${{ steps.vars.outputs.KERNEL_IMAGE }} ${{ steps.vars.outputs.FINAL_KERNEL_IMAGE_PATH }} + mv ${{ steps.vars.outputs.OUT_DIR }}/vmlinux ${{ steps.vars.outputs.FINAL_VMLINUX_PATH }} + mv ${{ steps.vars.outputs.OUT_DIR }}/../linux-headers-*.deb ${{ steps.vars.outputs.FINAL_LINUX_HEADERS_PATH }} - name: Upload kernel image artifact - if: ${{ matrix.os == 'ubuntu-22.04' }} - uses: actions/upload-artifact@v3 + if: ${{ inputs.host-os == 'ubuntu-22.04' }} + uses: actions/upload-artifact@v4 with: - name: kernel-image - path: ${{ steps.vars.outputs.KERNEL_IMAGE }}-${{ env.SUFFIX }} - - - name: Upload vmlinux artifact - if: ${{ matrix.os == 'ubuntu-22.04' }} - uses: actions/upload-artifact@v3 - with: - name: vmlinux - path: ${{ steps.vars.outputs.OUT_DIR }}/vmlinux-${{ env.SUFFIX }} - - run: - needs: build - strategy: - fail-fast: false - matrix: - kernel: [{ack: 0, version: 5.10.y}, {ack: 0, version: 5.15.y}, {ack: 0, version: 6.1.y}, {ack: 1, version: android13-5.10-lts}] - arch: [arm64, x86_64] - timeout-minutes: 20 - runs-on: ubuntu-22.04 - env: - ACK: ${{ matrix.kernel.ack }} - ARCH: ${{ matrix.arch }} - steps: - - uses: actions/checkout@v3 - - - name: Download all workflow run artifacts - uses: actions/download-artifact@v3 - - - name: Install common dependencies - shell: bash - run: sudo apt update && sudo apt install -y debootstrap - - - name: Install x86_64 dependencies - if: ${{ matrix.arch == 'x86_64' }} - shell: bash - run: sudo apt install -y qemu-system-x86 - - - name: Install arm64 dependencies - if: ${{ matrix.arch == 'arm64' }} - shell: bash - run: sudo apt install -y qemu-system-arm qemu-user-static binfmt-support - - - name: Get real kernel version - run: | - LINUX_URL=https://cdn.kernel.org/pub/linux/kernel - VERSION="${{ matrix.kernel.version }}" - first_char=${VERSION:0:1} - last_char=${VERSION: -1:1} - if [ $last_char == y ]; then - major_minor=${VERSION: 0:-1} - VERSION=$(curl -s $LINUX_URL/v${first_char}.x/ \ - | sed -e 's/<[^>]*>//g' \ - | grep -oP "linux-${major_minor}[0-9]+" \ - | sort -r -V \ - | head -n1 \ - | cut -d '-' -f2) - fi - echo "VERSION=$VERSION" >> $GITHUB_ENV - - - name: Set output variables - id: vars - run: | - set -x - - if [ $ACK -eq 0 ]; then - KERNEL_TYPE=linux - else - KERNEL_TYPE=ack - fi - - SUFFIX=$KERNEL_TYPE-${{ env.VERSION }}-${{ matrix.arch }} - - if [ $ARCH == x86_64 ]; then - IMAGE_NAME=bzImage-$SUFFIX - else - IMAGE_NAME=Image-$SUFFIX - fi - - echo "IMAGE_NAME=$IMAGE_NAME" >> $GITHUB_ENV - - - name: Initialize rootfs and initramfs - run: | - set -x - make rootfs-init - ROOTFS=./alpine-${{ matrix.arch }}.img make rootfs-init - - scripts/ubuntu_debootstrap.sh jammy ${{ matrix.arch }} - SUDO=1 ROOTFS_DIR=./rootfs/ubuntu-jammy-${{ matrix.arch }} CPIO_FILE=ubuntu-jammy-${{ matrix.arch }}.cpio.gz make rootfs-overlay - - - name: Setup shared/init.sh - run: | - mkdir shared - echo poweroff > shared/init.sh - chmod +x shared/init.sh - - - name: Run kernel - run: | - QEMU_KERNEL_IMAGE=./kernel-image/$IMAGE_NAME make run - QEMU_KERNEL_IMAGE=./kernel-image/$IMAGE_NAME ROOTFS=./rootfs/ubuntu-jammy-${{ matrix.arch }}.img make run - QEMU_KERNEL_IMAGE=./kernel-image/$IMAGE_NAME INITRD=1 make run - QEMU_KERNEL_IMAGE=./kernel-image/$IMAGE_NAME INITRD=./ubuntu-jammy-${{ matrix.arch }}.cpio.gz make run - QEMU_KERNEL_IMAGE=./kernel-image/$IMAGE_NAME CPU=2 MEM=2048M QEMU_EXTRA_ARGS="" QEMU_EXTRA_KERNEL_CMDLINE="nokaslr" ROOTFS=./alpine-${{ matrix.arch }}.img make run - - - name: Upload rootfs artifact - uses: actions/upload-artifact@v3 - with: - name: rootfs - path: rootfs/alpine-${{ matrix.arch }}.img - - release: - runs-on: ubuntu-22.04 - needs: [run, build] - permissions: - contents: write - steps: - - - name: Download all workflow run artifacts - uses: actions/download-artifact@v3 - - - name: Compute hashsums and create hashsums.txt - run: | - (cd vmlinux && sha256sum *) > hashsums.txt - (cd kernel-image && sha256sum *) >> hashsums.txt - (cd rootfs && sha256sum *) >> hashsums.txt - - - run: ls -lR - - - name: Set output variables - id: vars - run: | - set -x - - echo "DATE=$(date +'%Y.%m.%d')" >> $GITHUB_OUTPUT - - - name: Publish release - uses: softprops/action-gh-release@v1 - with: - prerelease: true - tag_name: ${{ steps.vars.outputs.DATE }}-${{ needs.build.outputs.SHORT_HASH }} - files: | - vmlinux/* - kernel-image/* - rootfs/* - hashsums.txt + name: ${{ steps.vars.outputs.ARTIFACT_DIR }} + path: | + ${{ steps.vars.outputs.FINAL_KERNEL_IMAGE_PATH }} + ${{ steps.vars.outputs.FINAL_VMLINUX_PATH }} + ${{ steps.vars.outputs.FINAL_LINUX_HEADERS_PATH }} diff --git a/.github/workflows/qemu.yml b/.github/workflows/qemu.yml new file mode 100644 index 0000000..2ed23e4 --- /dev/null +++ b/.github/workflows/qemu.yml @@ -0,0 +1,66 @@ +name: Build QEMU +on: + workflow_dispatch: + inputs: + version: + required: false + type: string + default: '8.2.2' + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + arch: [aarch64, x86_64] + steps: + + - name: Install dependencies + run: sudo apt install -y libglib2.0-dev libfdt-dev libpixman-1-dev zlib1g-dev ninja-build libslirp-dev + + - name: Cache QEMU source + id: cache + uses: actions/cache@v4 + with: + path: qemu-${{ inputs.version }} + key: qemu-${{ inputs.version }} + + - name: Download QEMU source + if: steps.cache.outputs.cache-hit != 'true' + run: | + wget --no-verbose https://download.qemu.org/qemu-${{ inputs.version }}.tar.xz + tar -xf qemu-${{ inputs.version }}.tar.xz + + - name: Build QEMU + run: | + cd qemu-${{ inputs.version }} + ./configure --enable-slirp --target-list=${{ matrix.arch }}-softmmu + make -j$(nproc) + + strip build/qemu-system-${{ matrix.arch }} + + - name: Upload QEMU binary + uses: actions/upload-artifact@v4 + with: + name: qemu-${{ matrix.arch }} + path: | + qemu-${{ inputs.version }}/build/qemu-system-${{ matrix.arch }} + + release: + runs-on: ubuntu-latest + needs: build + permissions: + contents: write + steps: + + - name: Download QEMU binaries + uses: actions/download-artifact@v4 + + - name: Publish release + uses: softprops/action-gh-release@v2 + with: + prerelease: true + tag_name: qemu-${{ inputs.version }}-${{ github.workflow_sha }} + files: | + qemu-x86_64/qemu-system-x86_64 + qemu-aarch64/qemu-system-aarch64 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..b1bca5d --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,65 @@ +name: Release +on: [workflow_dispatch] + +jobs: + build-and-test-all: + uses: ./.github/workflows/build-and-test-all.yml + + release: + runs-on: ubuntu-22.04 + needs: [build-and-test-all] + permissions: + contents: write + steps: + + - name: Download kernel artifacts + uses: actions/download-artifact@v4 + with: + pattern: kernel-artifacts-* + merge-multiple: true + path: kernel-artifacts + + - name: Download rootfs artifacts + uses: actions/download-artifact@v4 + with: + pattern: rootfs-* + merge-multiple: true + path: rootfs + + - run: ls -lR + + - name: Compute hashsums and create hashsums.txt + shell: bash + run: | + shopt -s extglob + (cd kernel-artifacts && sha256sum !(*.deb)) > hashsums.txt + (cd rootfs && sha256sum *) >> hashsums.txt + + - name: Set output variables + id: vars + run: | + set -x + + SHORT_HASH=${GITHUB_SHA:0:7} + DATE=$(date +'%Y.%m.%d') + + echo "SUFFIX=${DATE}-${SHORT_HASH}" >> $GITHUB_OUTPUT + + - name: Publish release + uses: softprops/action-gh-release@v2 + with: + prerelease: true + tag_name: ${{ steps.vars.outputs.SUFFIX }} + files: | + kernel-artifacts/vmlinux* + kernel-artifacts/Image* + kernel-artifacts/bzImage* + rootfs/* + hashsums.txt + + - name: Publish header release + uses: softprops/action-gh-release@v2 + with: + prerelease: true + tag_name: linux-headers-${{ steps.vars.outputs.SUFFIX }} + files: kernel-artifacts/linux-headers-*.deb diff --git a/.github/workflows/rootfs.yml b/.github/workflows/rootfs.yml new file mode 100644 index 0000000..588cd6f --- /dev/null +++ b/.github/workflows/rootfs.yml @@ -0,0 +1,126 @@ +name: Build Rootfs +on: + workflow_dispatch: + inputs: + host-os: + required: false + type: string + default: ubuntu-22.04 + arch: + required: true + type: choice + options: + - x86_64 + - arm64 + use-cached: + required: false + type: boolean + default: false + workflow_call: + inputs: + host-os: + required: false + type: string + default: ubuntu-22.04 + arch: + required: true + type: string + use-cached: + required: false + type: boolean + default: true + +jobs: + check-cache: + name: Check cache (${{ inputs.arch }}) + runs-on: ${{ inputs.host-os }} + outputs: + cache-hit: ${{ steps.cache-check.outputs.cache-hit }} + env: + ALPINE_ROOTFS: alpine-${{ inputs.arch }}.img + ALPINE_CPIO: alpine-${{ inputs.arch }}.cpio.gz + UBUNTU_ROOTFS: ubuntu-jammy-${{ inputs.arch }}.img + UBUNTU_CPIO: ubuntu-jammy-${{ inputs.arch }}.cpio.gz + steps: + + - name: Restore cache for rootfs + if: ${{ inputs.use-cached == true }} + id: cache-check + uses: actions/cache/restore@v4 + with: + key: rootfs-${{ inputs.arch }} + path: | + rootfs/${{ env.ALPINE_ROOTFS }} + rootfs/${{ env.UBUNTU_ROOTFS }} + rootfs/${{ env.ALPINE_CPIO }} + rootfs/${{ env.UBUNTU_CPIO }} + + - name: Upload rootfs artifact + if: steps.cache-check.outputs.cache-hit == 'true' + uses: actions/upload-artifact@v4 + with: + name: rootfs-${{ inputs.arch }} + path: | + rootfs/${{ env.ALPINE_ROOTFS }} + rootfs/${{ env.UBUNTU_ROOTFS }} + rootfs/${{ env.ALPINE_CPIO }} + rootfs/${{ env.UBUNTU_CPIO }} + + rootfs: + name: rootfs (${{ inputs.arch }}) + needs: check-cache + if: needs.check-cache.outputs.cache-hit != 'true' + runs-on: ${{ inputs.host-os }} + env: + # ARCH isn't directly used by the workflow, but it is used by the `make` commands + ARCH: ${{ inputs.arch }} + ALPINE_ROOTFS: alpine-${{ inputs.arch }}.img + ALPINE_CPIO: alpine-${{ inputs.arch }}.cpio.gz + UBUNTU_ROOTFS: ubuntu-jammy-${{ inputs.arch }}.img + UBUNTU_CPIO: ubuntu-jammy-${{ inputs.arch }}.cpio.gz + steps: + + - uses: actions/checkout@v4 + + - name: Install common dependencies + run: sudo apt update && sudo apt install -y debootstrap qemu-utils + + - name: Install arm64 dependencies + if: ${{ inputs.arch == 'arm64' }} + run: sudo apt install -y qemu-user-static binfmt-support + + - name: Initialize rootfs and initramfs + shell: bash + run: | + set -x + + make rootfs-init + + # Test generating rootfs in non-standard location + ROOTFS=${ALPINE_ROOTFS} make rootfs-init + + # Generate ubuntu rootfs and cpio (with CPIO at a non-standard location) + scripts/ubuntu_debootstrap.sh jammy ${{ inputs.arch }} + SUDO=1 ROOTFS_DIR=./rootfs/ubuntu-jammy-${{ inputs.arch }} CPIO_FILE=${UBUNTU_CPIO} make rootfs-overlay + + mv ./${UBUNTU_CPIO} rootfs/${UBUNTU_CPIO} + + - name: Cache rootfs artifacts + uses: actions/cache/save@v4 + with: + key: rootfs-${{ inputs.arch }} + path: | + rootfs/${{ env.ALPINE_ROOTFS }} + rootfs/${{ env.UBUNTU_ROOTFS }} + rootfs/${{ env.ALPINE_CPIO }} + rootfs/${{ env.UBUNTU_CPIO }} + + - name: Upload rootfs artifact + uses: actions/upload-artifact@v4 + with: + name: rootfs-${{ inputs.arch }} + path: | + rootfs/${{ env.ALPINE_ROOTFS }} + rootfs/${{ env.UBUNTU_ROOTFS }} + rootfs/${{ env.ALPINE_CPIO }} + rootfs/${{ env.UBUNTU_CPIO }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..81b0613 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,91 @@ +name: Test +on: + workflow_call: + inputs: + host-os: + required: false + type: string + default: ubuntu-22.04 + ack: + required: true + type: string + arch: + required: true + type: string + version: + required: true + type: string + artifact-dir: + required: true + type: string + kernel-image-name: + required: true + type: string + suffix: + required: true + type: string + +jobs: + test: + name: Test (ack=${{ inputs.ack }}, ${{ inputs.version }}, ${{ inputs.arch }}) + runs-on: ubuntu-22.04 + # Put ACK and ARCH in the environment so it's picked up by the `make` commands + env: + ACK: ${{ inputs.ack }} + ARCH: ${{ inputs.arch }} + steps: + - uses: actions/checkout@v4 + + - name: Download kernel artifacts + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.artifact-dir }} + path: ${{ inputs.artifact-dir }} + + - name: Download rootfs artifacts + uses: actions/download-artifact@v4 + with: + name: rootfs-${{ inputs.arch }} + path: rootfs + + - run: ls -lR + + - name: Install x86_64 dependencies + if: ${{ inputs.arch == 'x86_64' }} + run: sudo apt update && sudo apt install -y qemu-system-x86 + + - name: Install arm64 dependencies + if: ${{ inputs.arch == 'arm64' }} + run: sudo apt update && sudo apt install -y qemu-system-arm + + - name: Setup shared/init.sh + run: | + mkdir shared + echo poweroff > shared/init.sh + chmod +x shared/init.sh + + - name: Run kernel + run: | + export QEMU_KERNEL_IMAGE=${{ inputs.artifact-dir }}/${{ inputs.kernel-image-name }}-${{ inputs.suffix }} + + ALPINE_ROOTFS=alpine-${{ inputs.arch }}.img + UBUNTU_ROOTFS=ubuntu-jammy-${{ inputs.arch }}.img + UBUNTU_CPIO=ubuntu-jammy-${{ inputs.arch }}.cpio.gz + + cp ./rootfs/${ALPINE_ROOTFS} ./${ALPINE_ROOTFS} + cp ./rootfs/${UBUNTU_CPIO} ./${UBUNTU_CPIO} + + # Run with default alpine rootfs + make run + + # Run with ubuntu rootfs + ROOTFS=./rootfs/${UBUNTU_ROOTFS} make run + + # Run with default alpine cpio + INITRD=1 make run + + # Run with ubuntu cpio in non-standard location + INITRD=./${UBUNTU_CPIO} make run + + # Run with alpine rootfs in non-standard location, with additional options + CPU=2 MEM=2048M QEMU_EXTRA_ARGS="" QEMU_EXTRA_KERNEL_CMDLINE="nokaslr" ROOTFS=./${ALPINE_ROOTFS} make run diff --git a/.gitignore b/.gitignore index 58dcc52..12b10c1 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,9 @@ /shared /out /.cache +/.env compile_commands.json + +/external/* +!/external/external.mk +!/external/README.md diff --git a/.gitmodules b/.gitmodules index 283cca4..7eab220 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,11 +1,11 @@ [submodule "ack/common-modules/virtual-device"] path = ack/common-modules/virtual-device url = https://android.googlesource.com/kernel/common-modules/virtual-device - branch = android13-5.10 + branch = android14-5.15 [submodule "ack/common"] path = ack/common url = https://android.googlesource.com/kernel/common - branch = android13-5.10 + branch = android14-5.15 [submodule "linux"] path = linux url = https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/ diff --git a/Makefile b/Makefile index d74aab7..4fdac28 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ SCRIPT_DIR := $(ROOT_DIR)/scripts CONFIG_DIR := $(ROOT_DIR)/config CLANG_DIR ?= $(ROOT_DIR)/toolchain/clang -CLANG_URL := https://github.com/llvm/llvm-project/releases/download/llvmorg-15.0.6/clang+llvm-15.0.6-x86_64-linux-gnu-ubuntu-18.04.tar.xz +CLANG_URL := https://mirrors.edge.kernel.org/pub/tools/llvm/files/llvm-18.1.3-x86_64.tar.xz SHARED_DIR := $(ROOT_DIR)/shared @@ -31,7 +31,7 @@ else ifeq ($(filter x86_64 arm64 i386,$(ARCH)),) endif .PHONY: default -default: linux tools-vm +default: linux linux_modules tools-vm .PHONY: clean clean: linux_clean tools-vm_clean @@ -40,60 +40,70 @@ clean: linux_clean tools-vm_clean help: @echo '$(GREEN)General Environment Variables:$(NC)' @echo ' ARCH - Specify one of the supported architectures: x86_64, i386, arm64 (default: x86_64)' - @echo ' ACK - Set to 1 if building ACK instead of the Linux kernel. Does not need to be set for `ack` and `run-ack` targets (default: 0)' + @echo ' ACK - Set to 1 to build ACK instead of the Linux kernel. Does not need to be set for `ack` and `run-ack` targets (default: 0)' @echo ' VERBOSE - Set to 1 to enable verbose output (default: 0)' @echo '' @echo '$(GREEN)Build/Config:$(NC)' @echo ' Targets:' - @echo ' linux (default) - Builds the Linux kernel' - @echo ' linux_defconfig - Builds the Linux kernel' - @echo ' linux_modules - Builds the Linux kernel modules' + @echo ' linux (default) - Build the Linux kernel' + @echo ' linux_defconfig - Run `make defconfig`' + @echo ' linux_menuconfig - Run `make menuconfig`' + @echo ' linux_modules - Build the Linux kernel modules' @echo ' linux_debpkg - Creates a Debian package for the kernel' - @echo ' ack - Builds the Android Common Kernel' - @echo ' tools-vm - Builds linux/tools/vm' + @echo ' ack - Build the Android Common Kernel' + @echo ' tools-vm - Build linux/tools/vm' @echo '' @echo ' Environment Variables:' @echo ' LINUX_DEFCONFIG - The defconfig to use when building the kernel (default: defconfig, ACK default: gki_defconfig)' @echo ' LINUX_SRC - The path to the kernel source directory (default: linux, ACK default: ack/common)' @echo ' LINUX_OUT - The path where the kernel build output should be stored (default: out/linux/$$ARCH, ACK default: out/ack/common/$$ARCH)' - @echo ' LINUX_CONFIG_FRAGMENT - A kernel config fragment to merge with the defconfig (default: config/config.fragment, ACK default: config/config.fragment)' + @echo ' LINUX_CONFIG_FRAGMENT - A kernel config fragment to merge with the defconfig (default: config/config.fragment)' @echo '' @echo '$(GREEN)Clean:$(NC)' @echo ' Targets:' - @echo ' clean - Cleans output from default build targets' - @echo ' _clean - Cleans output for ' + @echo ' clean - Clean output from default build targets' + @echo ' _clean - Clean output for , where is one of: ack, linux, tools-vm, rootfs' @echo '' @echo '$(GREEN)Run/Debug:$(NC)' @echo ' Targets:' - @echo ' run - Run QEMU with the built kernel, bootloader, and rootfs image' + @echo ' run - Run QEMU with the built kernel and rootfs image' @echo ' run-ack - Same as `run` but runs ACK instead' @echo '' @echo ' Environment Variables:' @echo ' GDB - Set to 1 to start a gdbserver and wait for GDB when running QEMU (default: 0)' - @echo ' CPU - Specify the number of CPUs to use when running QEMU (default: 4)' - @echo ' MEM - Specify the memory size in MB to use when running QEMU (default: 1024)' - @echo ' QEMU_EXTRA_ARGS - Specify additional arguments to pass to QEMU (default: "")' - @echo ' QEMU_EXTRA_KERNEL_CMDLINE - Specify additional arguments to pass to the kernel (default: "")' - @echo ' QEMU_KERNEL_IMAGE - The path to the kernel image to run (x86_64 default: $$LINUX_OUT/arch/$$ARCH/boot/bzImage, arm64 default: $$LINUX_OUT/arch/$$ARCH/boot/Image)' - @echo ' ROOTFS - The path to the rootfs image file (default: rootfs/rootfs-$$ARCH.img)' - @echo ' ROOTFS_FORMAT - The format of the rootfs image file (default: qcow2)' - @echo ' INITRD - The path to a gziped initramfs CPIO file to use instead of a rootfs image (default: "")' - @echo ' RDINIT - The value of the `rdinit` kernel command line paramter (default: "", default if INITRD is set: /sbin/init)' + @echo ' CPU - The number of CPUs to use when running QEMU (default: 4)' + @echo ' MEM - The memory size in MB to use when running QEMU (default: 1024)' + @echo ' QEMU_EXTRA_ARGS - Additional arguments to pass to QEMU (default: "")' + @echo ' QEMU_EXTRA_KERNEL_CMDLINE - Additional arguments to pass to the kernel (default: "")' + @echo ' QEMU_KERNEL_IMAGE - The path to the kernel image to run (x86_64/i386 default: $$LINUX_OUT/arch/$$ARCH/boot/bzImage, arm64 default: $$LINUX_OUT/arch/$$ARCH/boot/Image)' + @echo ' ROOTFS - The path to the rootfs image file (default: rootfs/alpine-$$ARCH.img)' + @echo ' ROOTFS_FORMAT - The format of the rootfs image file: raw, qcow2 (default: qcow2)' + @echo ' INITRD - Set to 1 to use the $$CPIO_FILE initramfs instead of the $$ROOTFS image as the rootfs, or specify the path to an alternative CPIO file (default: "")' + @echo ' RDINIT - The value of the `rdinit` kernel command line parameter (default: "", default if INITRD is set: /sbin/init)' + @echo ' ECHR - The value of the QEMU `-echr` flag (default: 1)' + @echo ' ROOT - The value of the `root` kernel command line parameter (default: /dev/vda)' + @echo ' RW - Whether to mount the rootfs as read-write or read-only: ro, rw (default: rw)' + @echo ' KASLR - Set to 1 to enable KASLR (default: 0)' @echo '' @echo '$(GREEN)rootfs:$(NC)' @echo ' Targets:' - @echo ' rootfs-init - Extracts the Alpine Linux rootfs to rootfs/alpine-$$ARCH, makes arch-specific changes, and builds a rootfs image at rootfs/alpine-$$ARCH.img' - @echo ' rootfs - An alias for the `ext4` target' - @echo ' ext4 - Builds a QCOW2 rootfs image at rootfs/alpine-$$ARCH.img with an ext4 filesystem' - @echo ' cpio - Builds a gziped initramfs CPIO file from the $$INITRAMFS_DIR directory' - @echo ' rootfs-mount - Mount rootfs image at /tmp/rootfs' - @echo ' rootfs-unmount - Unmount rootfs image' + @echo ' rootfs-init - Extract the Alpine Linux rootfs to $$ROOTFS_DIR and then run the `rootfs-overlay` target' + @echo ' rootfs-overlay - Apply arch-specific changes to $$ROOTFS_DIR and run the `rootfs` target' + @echo ' rootfs - Run the `ext4` and `cpio` targets' + @echo ' ext4 - Build a $$ROOTFS $$ROOTFS_FORMAT (default qcow2) image with an ext4 filesystem from $$ROOTFS_DIR' + @echo ' cpio - Build a $$CPIO_FILE gzipped initramfs CPIO file from $$ROOTFS_DIR' + @echo ' initramfs - An alias for the `cpio` target' + @echo ' uncpio - Extract $$CPIO_FILE to $$ROOTFS_DIR' + @echo ' rootfs-mount - Mount $$ROOTFS image at /tmp/rootfs' + @echo ' rootfs-unmount - Unmount rootfs image from /tmp/rootfs' + @echo ' chroot - chroot into $$ROOTFS_DIR' @echo '' @echo ' Environment Variables:' @echo ' EXT4_SIZE - The disk size of the rootfs image to build' @echo ' ROOTFS_DIR - The directory to create the ext4 rootfs image and initramfs CPIO from (default: rootfs/alpine-$$ARCH)' - @echo ' ROOTFS - The path to the rootfs image file (default: rootfs/rootfs-$$ARCH.img)' - @echo ' ROOTFS_FORMAT - The format of the rootfs image file (default: qcow2)' + @echo ' ROOTFS - The path to the rootfs image file (default: rootfs/alpine-$$ARCH.img)' + @echo ' ROOTFS_FORMAT - The format of the rootfs image file: raw, qcow2 (default: qcow2)' + @echo ' CPIO_FILE - The path to the CPIO file to create (default: rootfs/alpine-$$ARCH.cpio.gz)' @echo '' @echo '$(GREEN)Miscellaneous:$(NC)' @echo ' Targets:' @@ -155,11 +165,12 @@ LINUX_MAKE := \ CROSS_COMPILE=$(TARGET)- \ LLVM=1 LLVM_IAS=1 \ V=$(VERBOSE) \ - O=$(LINUX_OUT) + O=$(LINUX_OUT) \ + -j `nproc` $(CLANG_DIR): $(warning $(YELLOW)Clang directory $(CLANG_DIR) does not exist, downloading prebuilt binaries $(NC)) - wget --no-verbose --show-progress $(CLANG_URL) -O clang.tar.xz + wget --no-verbose --show-progress --progress=bar:force $(CLANG_URL) -O clang.tar.xz mkdir -p $(CLANG_DIR) tar -xf clang.tar.xz -C $(CLANG_DIR) --strip-components=1 @@ -183,11 +194,10 @@ linux_menuconfig: linux $(KERNEL_IMAGE): $(LINUX_CONFIG) | $(CLANG_DIR) + $(LINUX_MAKE) # Older versions of Linux don't have this script -ifneq (,$(wildcard ./.env)) - $(LINUX_SRC)/scripts/clang-tools/gen_compile_commands.py -d $(LINUX_OUT) +ifneq (,$(wildcard $(LINUX_SRC)/scripts/clang-tools/gen_compile_commands.py)) + cd $(LINUX_SRC) && ./scripts/clang-tools/gen_compile_commands.py -d $(LINUX_OUT) endif - .PHONY: linux_modules linux_modules $(LINUX_OUT_MODULES_DEP): $(KERNEL_IMAGE) + $(LINUX_MAKE) modules @@ -197,6 +207,10 @@ linux_modules $(LINUX_OUT_MODULES_DEP): $(KERNEL_IMAGE) @cmp $(LINUX_OUT_MODULES_DEP).tmp $(LINUX_OUT_MODULES_DEP) || \ mv $(LINUX_OUT_MODULES_DEP).tmp $(LINUX_OUT_MODULES_DEP) +.PHONY: linux_bindebpkg +linux_bindebpkg: $(KERNEL_IMAGE) + + $(LINUX_MAKE) bindeb-pkg + .PHONY: linux_clean linux_clean: + $(LINUX_MAKE) mrproper @@ -357,6 +371,7 @@ QEMU_ARGS := \ -kernel $(QEMU_KERNEL_IMAGE) \ -netdev user,id=eth0,hostfwd=tcp::7777-:7777,hostfwd=tcp::2222-:22,hostfwd=tcp::2223-:23 -device virtio-net-pci,netdev=eth0 \ -virtfs local,security_model=mapped-xattr,path=$(SHARED_DIR),mount_tag=shared \ + -virtfs local,security_model=mapped-xattr,path=$(LINUX_MODULES_INSTALL_PATH)/lib/modules,mount_tag=modules \ -echr $(ECHR) \ $(QEMU_EXTRA_ARGS) @@ -377,7 +392,10 @@ endif ifeq ($(ARCH),x86_64) QEMU_BIN := qemu-system-x86_64 - QEMU_KERNEL_CMDLINE += console=ttyS0 kpti no5lvl + + # 8250.nr_uarts=1 is needed because some Android kernels set + # `CONFIG_SERIAL_8250_RUNTIME_UARTS` to zero + QEMU_KERNEL_CMDLINE += console=ttyS0 kpti no5lvl 8250.nr_uarts=1 QEMU_ARGS += -cpu kvm64,+smep,+smap @@ -402,12 +420,23 @@ ifeq ($(KASLR),0) QEMU_KERNEL_CMDLINE += nokaslr endif +# This is not an actual kernel parameter, but it allows us to check from within +# QEMU if this is an ACK kernel or not. +ifeq ($(ACK),1) + QEMU_KERNEL_CMDLINE += lede.ack=1 +endif + QEMU_ARGS += -append "$(QEMU_KERNEL_CMDLINE) $(QEMU_EXTRA_KERNEL_CMDLINE)" RUN_DEPS := $(QEMU_KERNEL_IMAGE) +# Make sure the modules directory exists, even if it's empty. Otherwise mount +# will fail. +$(LINUX_MODULES_INSTALL_PATH)/lib/modules: + mkdir -p $@ + .PHONY: run -run: $(RUN_DEPS) | $(SHARED_DIR) +run: $(RUN_DEPS) | $(SHARED_DIR) $(LINUX_MODULES_INSTALL_PATH)/lib/modules @echo "$(GREEN)Running QEMU, press 'ctrl-a x' to quit $(NC)" ifeq ($(GDB),1) @echo "$(ARCH) $(ACK)" > $(OUT_DIR)/.gdb @@ -423,3 +452,8 @@ endif .PHONY: run-ack run-ack: run + +EXTERNAL_DIR := $(ROOT_DIR)/external +ifneq (,$(wildcard $(EXTERNAL_DIR)/external.mk)) + include $(EXTERNAL_DIR)/external.mk +endif diff --git a/README.md b/README.md index c60087b..f086073 100644 --- a/README.md +++ b/README.md @@ -52,61 +52,71 @@ ARCH=arm64 make run ``` $ make help General Environment Variables: - ARCH - Specify one of the supported architectures: x86_64, arm64 (default: x86_64) - ACK - Set to 1 if building ACK instead of the Linux kernel. Does not need to be set for `ack` and `run-ack` targets (default: 0) + ARCH - Specify one of the supported architectures: x86_64, i386, arm64 (default: x86_64) + ACK - Set to 1 to build ACK instead of the Linux kernel. Does not need to be set for `ack` and `run-ack` targets (default: 0) VERBOSE - Set to 1 to enable verbose output (default: 0) Build/Config: Targets: - linux (default) - Builds the Linux kernel - linux_defconfig - Builds the Linux kernel - linux_modules - Builds the Linux kernel modules - ack - Builds the Android Common Kernel - tools-vm - Builds linux/tools/vm + linux (default) - Build the Linux kernel + linux_defconfig - Run `make defconfig` + linux_menuconfig - Run `make menuconfig` + linux_modules - Build the Linux kernel modules + linux_debpkg - Creates a Debian package for the kernel + ack - Build the Android Common Kernel + tools-vm - Build linux/tools/vm Environment Variables: LINUX_DEFCONFIG - The defconfig to use when building the kernel (default: defconfig, ACK default: gki_defconfig) LINUX_SRC - The path to the kernel source directory (default: linux, ACK default: ack/common) LINUX_OUT - The path where the kernel build output should be stored (default: out/linux/$ARCH, ACK default: out/ack/common/$ARCH) - LINUX_CONFIG_FRAGMENT - A kernel config fragment to merge with the defconfig (default: config/config.fragment, ACK default: config/config.fragment) + LINUX_CONFIG_FRAGMENT - A kernel config fragment to merge with the defconfig (default: config/config.fragment) Clean: Targets: - clean - Cleans output from default build targets - _clean - Cleans output for + clean - Clean output from default build targets + _clean - Clean output for , where is one of: ack, linux, tools-vm, rootfs Run/Debug: Targets: - run - Run QEMU with the built kernel, bootloader, and rootfs image + run - Run QEMU with the built kernel and rootfs image run-ack - Same as `run` but runs ACK instead Environment Variables: GDB - Set to 1 to start a gdbserver and wait for GDB when running QEMU (default: 0) - CPU - Specify the number of CPUs to use when running QEMU (default: 4) - MEM - Specify the memory size in MB to use when running QEMU (default: 1024) - QEMU_EXTRA_ARGS - Specify additional arguments to pass to QEMU (default: "") - QEMU_EXTRA_KERNEL_CMDLINE - Specify additional arguments to pass to the kernel (default: "") - KERNEL_IMAGE - The path to the kernel image to run (x86_64 default: $LINUX_OUT/arch/$ARCH/boot/bzImage, arm64 default: $LINUX_OUT/arch/$ARCH/boot/Image) - ROOTFS - The path to the rootfs image file (default: rootfs/rootfs-$ARCH.img) - ROOTFS_FORMAT - The format of the rootfs image file (default: qcow2) - INITRD - The path to a gziped initramfs CPIO file to use instead of a rootfs image (default: "") - RDINIT - The value of the `rdinit` kernel command line paramter (default: "", default if INITRD is set: /sbin/init) + CPU - The number of CPUs to use when running QEMU (default: 4) + MEM - The memory size in MB to use when running QEMU (default: 1024) + QEMU_EXTRA_ARGS - Additional arguments to pass to QEMU (default: "") + QEMU_EXTRA_KERNEL_CMDLINE - Additional arguments to pass to the kernel (default: "") + QEMU_KERNEL_IMAGE - The path to the kernel image to run (x86_64/i386 default: $LINUX_OUT/arch/$ARCH/boot/bzImage, arm64 default: $LINUX_OUT/arch/$ARCH/boot/Image) + ROOTFS - The path to the rootfs image file (default: rootfs/alpine-$ARCH.img) + ROOTFS_FORMAT - The format of the rootfs image file: raw, qcow2 (default: qcow2) + INITRD - Set to 1 to use the $CPIO_FILE initramfs instead of the $ROOTFS image as the rootfs, or specify the path to an alternative CPIO file (default: "") + RDINIT - The value of the `rdinit` kernel command line parameter (default: "", default if INITRD is set: /sbin/init) + ECHR - The value of the QEMU `-echr` flag (default: 1) + ROOT - The value of the `root` kernel command line parameter (default: /dev/vda) + RW - Whether to mount the rootfs as read-write or read-only: ro, rw (default: rw) + KASLR - Set to 1 to enable KASLR (default: 0) rootfs: Targets: - rootfs-init - Extracts the Alpine Linux rootfs to rootfs/alpine-$ARCH, makes arch-specific changes, and builds a rootfs image at rootfs/alpine-$ARCH.img - rootfs - An alias for the `ext4` target - ext4 - Builds a QCOW2 rootfs image at rootfs/alpine-$ARCH.img with an ext4 filesystem - cpio - Builds a gziped initramfs CPIO file from the $INITRAMFS_DIR directory - rootfs-mount - Mount rootfs image at /tmp/rootfs - rootfs-unmount - Unmount rootfs image + rootfs-init - Extract the Alpine Linux rootfs to $ROOTFS_DIR and then run the `rootfs-overlay` target + rootfs-overlay - Apply arch-specific changes to $ROOTFS_DIR and run the `rootfs` target + rootfs - Run the `ext4` and `cpio` targets + ext4 - Build a $ROOTFS $ROOTFS_FORMAT (default qcow2) image with an ext4 filesystem from $ROOTFS_DIR + cpio - Build a $CPIO_FILE gzipped initramfs CPIO file from $ROOTFS_DIR + initramfs - An alias for the `cpio` target + uncpio - Extract $CPIO_FILE to $ROOTFS_DIR + rootfs-mount - Mount $ROOTFS image at /tmp/rootfs + rootfs-unmount - Unmount rootfs image from /tmp/rootfs + chroot - chroot into $ROOTFS_DIR Environment Variables: EXT4_SIZE - The disk size of the rootfs image to build - INITRAMFS_DIR - The directory to create the initramfs from (default: rootfs/alpine-$ARCH) - ROOTFS_DIR - The directory to create the ext4/rootfs image from (default: rootfs/alpine-$ARCH) - ROOTFS - The path to the rootfs image file (default: rootfs/rootfs-$ARCH.img) - ROOTFS_FORMAT - The format of the rootfs image file (default: qcow2) + ROOTFS_DIR - The directory to create the ext4 rootfs image and initramfs CPIO from (default: rootfs/alpine-$ARCH) + ROOTFS - The path to the rootfs image file (default: rootfs/alpine-$ARCH.img) + ROOTFS_FORMAT - The format of the rootfs image file: raw, qcow2 (default: qcow2) + CPIO_FILE - The path to the CPIO file to create (default: rootfs/alpine-$ARCH.cpio.gz) Miscellaneous: Targets: @@ -114,7 +124,7 @@ Miscellaneous: linux_checkout - Checks out the version specified by $VERSION of the linux kernel in $LINUX_SRC Environment Variables: - VERSION - The version to download or checkout. If the patch version is a 'y', the latest version of the kernel with that major and minor version is used. Examples: 5.10, 5.10.107, v5.10, 5.10.y, linux-5.10.y, android13-5.10 + VERSION - The version to download or checkout. For checkout only, if the third number in the version string is a 'y', the latest version of the kernel with that major and minor version is used. Examples: 5.10, 5.10.107, v5.10, 5.10.y, linux-5.10.y ``` ## Building @@ -195,3 +205,11 @@ VERSION=5.10.y make linux_checkout The `LINUX_SRC` variable can be used to customize the source directory for `linux_checkout`. These `make` targets are just wrappers around the `./scripts/download_linux.sh` and `./scripts/checkout_linux.sh` scripts, so if you prefer those can be used instead. + +## Miscellaneous Features + +If a `.env` file exists in root of the repository, it will be sourced at the beginning of the Makefile. If you are often passing in the same environment variables on every invocation (i.e. `ARCH=arm64`), you can add this into `.env` instead of passing it on the command line. If you want to put your most commonly used defaults in `.env` but still use other values for some invocations of `make`, you can override any variables by putting them after the `make` invocation. For example, if `.env` contains `ARCH=arm64`, then `make linux ARCH=x86_64` would set this variable to `x86_64` when building. Setting the variable before the `make` invocation (i.e. `ARCH=x86_64 make linux`) will not override the values defined in `.env`. + +If the `TERMINAL_CMD` environment variable is defined when executing `make run`, it will be used to execute `scripts/gdb.sh` automatically before starting QEMU. You can use this to open GDB in a new window or frame. For example, when running `make run` in a `tmux` session, setting `TERMINAL_CMD="tmux splitw -h"` will launch GDB in a new `tmux` pane. + +You can add your own `make` targets without modifying the core `makefile`. See the README in the [external](./external) directory for more information and [this project](https://github.com/gsingh93/linux-exploit-dev-env-art-kt) for an example. diff --git a/ack/common b/ack/common index 7b52958..0add0e5 160000 --- a/ack/common +++ b/ack/common @@ -1 +1 @@ -Subproject commit 7b529581b92f406755d5dd2a7fcc907586b81049 +Subproject commit 0add0e52ef2bf1f20c1afef6566a0133a77401ea diff --git a/ack/common-modules/virtual-device b/ack/common-modules/virtual-device index 5213b71..bc1ecbc 160000 --- a/ack/common-modules/virtual-device +++ b/ack/common-modules/virtual-device @@ -1 +1 @@ -Subproject commit 5213b7155b81b7ae80bffd34546dfc581c62fec7 +Subproject commit bc1ecbc0ef31e570d3f7d26a8ae93855d2531bfb diff --git a/config/config.fragment b/config/config.fragment index 2d28ef2..81122c9 100644 --- a/config/config.fragment +++ b/config/config.fragment @@ -92,3 +92,8 @@ CONFIG_TRACING=y # Allow lvl5 paging CONFIG_X86_5LEVEL=y + +# Some ACK kernels set this to zero. Technically we don't need it because we +# also set it in the kernel command line, but it helps if someone wants to use +# our prebuilt kernels and run them manually +CONFIG_SERIAL_8250_RUNTIME_UARTS=4 diff --git a/config/rootfs-overlay/etc/init.d/setup b/config/rootfs-overlay/etc/init.d/setup index 615d7b8..196e0d5 100755 --- a/config/rootfs-overlay/etc/init.d/setup +++ b/config/rootfs-overlay/etc/init.d/setup @@ -26,6 +26,9 @@ case "$1" in mkdir -p /shared mount -t 9p -o trans=virtio,version=9p2000.L shared /shared + mkdir -p /lib/modules + mount -t 9p -o trans=virtio,version=9p2000.L modules /lib/modules + mount -t debugfs none /sys/kernel/debug [ ! -e /d ] && ln -s /sys/kernel/debug /d diff --git a/external/README.md b/external/README.md new file mode 100644 index 0000000..331f797 --- /dev/null +++ b/external/README.md @@ -0,0 +1,4 @@ +All files called `rules.mk` contained within subfolders of this directory will +be sourced at the end of the main `Makefile`. You can use this to add your own +`make` targets. `rules.mk` will have access to all of the variables defined in +the main `Makefile`. diff --git a/external/external.mk b/external/external.mk new file mode 100644 index 0000000..b24ed8a --- /dev/null +++ b/external/external.mk @@ -0,0 +1 @@ +include $(wildcard $(EXTERNAL_DIR)/*/rules.mk) diff --git a/linux b/linux index c9c3395..2cc14f5 160000 --- a/linux +++ b/linux @@ -1 +1 @@ -Subproject commit c9c3395d5e3dcc6daee66c6908354d47bf98cb0c +Subproject commit 2cc14f52aeb78ce3f29677c2de1f06c0e91471ab diff --git a/scripts/download_linux.sh b/scripts/download_linux.sh index 32fc57a..7c8856c 100755 --- a/scripts/download_linux.sh +++ b/scripts/download_linux.sh @@ -39,14 +39,14 @@ if [ $ACK -eq 0 ]; then echo "Downloading kernel version $VERSION" if [ $first_char -ge $MIN_VERSION ] && [ $first_char -le $MAX_VERSION ]; then - wget --no-verbose --show-progress $LINUX_URL/v${first_char}.x/linux-$VERSION.tar.xz + wget --no-verbose --show-progress --progress=bar:force $LINUX_URL/v${first_char}.x/linux-$VERSION.tar.xz tar -xf linux-$VERSION.tar.xz else echo "$first_char is not a valid kernel major version number" exit 1 fi else - wget $ACK_URL/$VERSION.tar.gz + wget --no-verbose --show-progress --progress=bar:force $ACK_URL/$VERSION.tar.gz mkdir -p $VERSION tar -xf $VERSION.tar.gz -C $VERSION fi diff --git a/scripts/gdbinit.gdb b/scripts/gdbinit.gdb index ef226e6..e8cfa17 100644 --- a/scripts/gdbinit.gdb +++ b/scripts/gdbinit.gdb @@ -1,4 +1,4 @@ file ##LINUX_OUT##/vmlinux source ##LINUX_OUT##/vmlinux-gdb.py target remote :1234 -add-symbol-file ##LINUX_OUT##/modules_install/lib/modules/5.10.107/extra/art_rootkit.ko -s .text 0xffffffc0091b0800 +# add-symbol-file ##LINUX_OUT##/modules_install/lib/modules/5.10.107/extra/my_module.ko -s .text 0xffffffc0091b0800