Open
Description
Name and Version
version: b5320
built with macOS Sonoma, Android Studio Meerkat 2024.3.1 Patch 2 and Android NDK 27.012077973
Operating systems
Other? (Please let us know in description), Mac
Which llama.cpp modules do you know to be affected?
libllama (core library)
Command line
Problem description & steps to reproduce
I'm trying to use llama.cpp on Android with local inference using NDK with JNI. When I try to load a model (nomic_embed_text_v1_5_q4_0.gguf) with the "llama_model_load_from_file" method, it does not load and returns null.
CMakeLists.txt
cmake_minimum_required(VERSION 3.22.1)
project(llama_jni)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17")
# Path to llama.cpp folder is in the root folder of the project
set(LLAMA_CPP_DIR "${CMAKE_SOURCE_DIR}/../../../../llama.cpp")
set(LLAMA_CPP_SRC_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../../../llama.cpp/src")
set(LLAMA_CPP_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../../../llama.cpp/include")
set(LLAMA_GGML_SRC_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../../../llama.cpp/ggml/src")
set(LLAMA_GGML_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../../../llama.cpp/ggml/include")
add_definitions(
-DANDROID_ABI=arm64-v8a
-DANDROID_PLATFORM=android-28
-DCMAKE_C_FLAGS="-march=armv8.7a"
-DCMAKE_CXX_FLAGS="-march=armv8.7a"
-DGGML_OPENMP=OFF
-DGGML_LLAMAFILE=OFF
-B build-android
-DLLAMA_CURL=OFF
)
include_directories(
${CMAKE_SOURCE_DIR}
${LLAMA_CPP_DIR}
${LLAMA_CPP_INCLUDE_DIR}
${LLAMA_GGML_SRC_DIR}
${LLAMA_GGML_INCLUDE_DIR}
)
# Collect llama.cpp source files
file(GLOB LLAMA_SOURCES
${LLAMA_CPP_SRC_DIR}/*.cpp
${LLAMA_GGML_SRC_DIR}/*.cpp
${LLAMA_GGML_SRC_DIR}/*.c
${LLAMA_GGML_INCLUDE_DIR}
)
add_library(llama_jni SHARED
llama_jni.cpp
llama_embed.cpp
${LLAMA_SOURCES}
)
# Required Android libraries
find_library(log-lib log)
target_include_directories(llama_jni PRIVATE
${LLAMA_CPP_DIR}
${LLAMA_CPP_INCLUDE_DIR}
${LLAMA_GGML_SRC_DIR}
${LLAMA_GGML_INCLUDE_DIR}
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_SOURCE_DIR}/../c_interop/include # location of llama.h
${LLAMA_SOURCES}
)
target_link_libraries(
llama_jni
${log-lib}
)
llama_embed.cpp
static struct llama_model *model = nullptr;
static struct llama_context *ctx = nullptr;
static int embedding_size = 0;
bool llama_embed_init(const char *model_path) {
__android_log_print(ANDROID_LOG_INFO, "llama_jni", "Initializing llama...");
llama_backend_init();
__android_log_print(ANDROID_LOG_INFO, "llama_jni", "Model loading...");
llama_model_params model_params = llama_model_default_params();
__android_log_print(ANDROID_LOG_INFO, "llama_jni", "File Path: %s", model_path);
if (std::filesystem::exists(model_path)) {
__android_log_print(ANDROID_LOG_INFO, "llama_jni", "Model file exists: %s", model_path);
}
model = llama_model_load_from_file(model_path, model_params);
if (!model) return false;
__android_log_print(ANDROID_LOG_INFO, "llama_jni", "Model loaded successfully.");
__android_log_print(ANDROID_LOG_INFO, "llama_jni", "Context creating...");
llama_context_params ctx_params = llama_context_default_params();
ctx_params.embeddings = true;
ctx = llama_init_from_model(model, ctx_params);
if (!ctx) return false;
__android_log_print(ANDROID_LOG_INFO, "llama_jni", "Context created successfully.");
embedding_size = llama_model_n_embd(model);
return true;
}
First Bad Commit
No response
Relevant log output
No log raised in Logcat