[BFCL] Add Support for Fully Offline Model Inference via --local-model-path (ShishirPatil#985)

Resolve ShishirPatil#968
- **New CLI flag**: `--local-model-path` added to generate command
- **Validation logic**: Checks for required files (`config.json`,
`tokenizer_config.json`) in the model path
- Updated model loading:
- `AutoTokenizer` and `AutoConfig` now use `local_files_only=True` when
loading local models
- Server subprocesses (vLLM / SGLang) now reference `model_path_or_id`
- Inference calls to VLLM use `model_path_or_id` instead of
`model_name_huggingface` to avoid mismatches

---------

Co-authored-by: Huanzhi Mao <huanzhimao@gmail.com>

Author: catherineruoxiwu

berkeley-function-call-leaderboard/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 All notable changes to the Berkeley Function Calling Leaderboard will be documented in this file.
 
+- [Apr 14, 2025] [#985](https://github.com/ShishirPatil/gorilla/pull/985): Support fully offline inference via the `--local-model-path` flag. Point it to a directory that already holds the model’s files (`config.json`, tokenizer, weights, etc.); use this flag only when the model has been pre‑downloaded outside the default $HF_HOME cache.
 - [Apr 13, 2025] [#980](https://github.com/ShishirPatil/gorilla/pull/980): Integrate Novita AI as a third-party inference provider for the following open-source models:
   - `Llama-4-Maverick-17B-128E-Instruct-FP8` (Prompt & FC)
   - `Llama-4-Scout-17B-16E-Instruct` (Prompt & FC)

berkeley-function-call-leaderboard/README.md

@@ -127,11 +127,18 @@ bfcl generate --model MODEL_NAME --test-category TEST_CATEGORY --num-threads 1
 #### For Locally-hosted OSS Models
 
 ```bash
-bfcl generate --model MODEL_NAME --test-category TEST_CATEGORY --backend {vllm|sglang} --num-gpus 1 --gpu-memory-utilization 0.9
+bfcl generate \
+  --model MODEL_NAME \
+  --test-category TEST_CATEGORY \
+  --backend {vllm|sglang} \
+  --num-gpus 1 \
+  --gpu-memory-utilization 0.9 \
+  --local-model-path /path/to/local/model   # ← optional
 ```
 
 - Choose your backend using `--backend vllm` or `--backend sglang`. The default backend is `vllm`.
 - Control GPU usage by adjusting `--num-gpus` (default `1`, relevant for multi-GPU tensor parallelism) and `--gpu-memory-utilization` (default `0.9`), which can help avoid out-of-memory errors.
+- `--local-model-path` (optional): Point this flag at a directory that already contains the model’s files (`config.json`, tokenizer, weights, etc.). Use it only when you’ve pre‑downloaded the model and the weights live somewhere other than the default `$HF_HOME` cache.
 
 ##### For Pre-existing OpenAI-compatible Endpoints
 

berkeley-function-call-leaderboard/bfcl/__main__.py

@@ -1,7 +1,8 @@
 import csv
 from datetime import datetime
+import os
 from types import SimpleNamespace
-from typing import List
+from typing import List, Optional
 
 import typer
 from bfcl._llm_response_generation import main as generation_main
@@ -118,6 +119,11 @@ def generate(
         "--skip-server-setup",
         help="Skip vLLM/SGLang server setup and use existing endpoint specified by the VLLM_ENDPOINT and VLLM_PORT environment variables.",
     ),
+    local_model_path: Optional[str] = typer.Option(
+        None,
+        "--local-model-path",
+        help="Specify the path to a local directory containing the model's config/tokenizer/weights for fully offline inference. Use this only if the model weights are stored in a location other than the default HF_HOME directory.",
+    ),
     result_dir: str = typer.Option(
         RESULT_PATH,
         "--result-dir",
@@ -150,6 +156,7 @@ def generate(
         gpu_memory_utilization=gpu_memory_utilization,
         backend=backend,
         skip_server_setup=skip_server_setup,
+        local_model_path=local_model_path,
         result_dir=result_dir,
         allow_overwrite=allow_overwrite,
         run_ids=run_ids,

berkeley-function-call-leaderboard/bfcl/_llm_response_generation.py

@@ -18,11 +18,7 @@
 from bfcl.eval_checker.eval_runner_helper import load_file
 from bfcl.model_handler.handler_map import HANDLER_MAP
 from bfcl.model_handler.model_style import ModelStyle
-from bfcl.utils import (
-    is_multi_turn,
-    parse_test_category_argument,
-    sort_key,
-)
+from bfcl.utils import is_multi_turn, parse_test_category_argument, sort_key
 from tqdm import tqdm
 
 RETRY_LIMIT = 3
@@ -55,7 +51,15 @@ def get_args():
         default=False,
         help="Skip vLLM/SGLang server setup and use existing endpoint specified by the VLLM_ENDPOINT and VLLM_PORT environment variables."
     )
+    # Optional local model path
+    parser.add_argument(
+        "--local-model-path",
+        type=str,
+        default=None,
+        help="Specify the path to a local directory containing the model's config/tokenizer/weights for fully offline inference. Use this only if the model weights are stored in a location other than the default HF_HOME directory.",
+    )
     args = parser.parse_args()
+
     return args
 
 
@@ -226,6 +230,7 @@ def generate_results(args, model_name, test_cases_total):
             gpu_memory_utilization=args.gpu_memory_utilization,
             backend=args.backend,
             skip_server_setup=args.skip_server_setup,
+            local_model_path=args.local_model_path,
             include_input_log=args.include_input_log,
             exclude_state_log=args.exclude_state_log,
             result_dir=args.result_dir,

berkeley-function-call-leaderboard/bfcl/model_handler/local_inference/base_oss_handler.py

@@ -1,14 +1,12 @@
+import os
 import subprocess
 import threading
 import time
-import os
 from concurrent.futures import ThreadPoolExecutor
+from typing import Optional
 
 import requests
-from bfcl.constants.eval_config import (
-    RESULT_PATH,
-    VLLM_PORT,
-)
+from bfcl.constants.eval_config import RESULT_PATH, VLLM_PORT
 from bfcl.model_handler.base_handler import BaseHandler
 from bfcl.model_handler.model_style import ModelStyle
 from bfcl.model_handler.utils import (
@@ -28,6 +26,11 @@ def __init__(self, model_name, temperature, dtype="bfloat16") -> None:
         self.model_name_huggingface = model_name
         self.model_style = ModelStyle.OSSMODEL
         self.dtype = dtype
+
+        # Might be overridden in batch_inference method if local_model_path is provided
+        # Used to indicate where the tokenizer and config should be loaded from
+        self.model_path_or_id = self.model_name_huggingface
+
         # Read from env vars with fallbacks
         self.vllm_host = os.getenv("VLLM_ENDPOINT", "localhost")
         self.vllm_port = os.getenv("VLLM_PORT", VLLM_PORT)
@@ -63,6 +66,7 @@ def batch_inference(
         gpu_memory_utilization: float,
         backend: str,
         skip_server_setup: bool,
+        local_model_path: Optional[str],
         include_input_log: bool,
         exclude_state_log: bool,
         update_mode: bool,
@@ -73,13 +77,36 @@ def batch_inference(
         """
         from transformers import AutoConfig, AutoTokenizer
 
-        self.tokenizer = AutoTokenizer.from_pretrained(
-            self.model_name_huggingface, trust_remote_code=True
-        )
+        # Determine the model source
+        if local_model_path is not None:
+            # Validate the local_model_path
+            if not os.path.isdir(local_model_path):
+                raise ValueError(
+                    f"local_model_path '{local_model_path}' does not exist or is not a directory."
+                )
+
+            required_files = ["config.json", "tokenizer_config.json"]
+            for file_name in required_files:
+                if not os.path.exists(os.path.join(local_model_path, file_name)):
+                    raise ValueError(
+                        f"Required file '{file_name}' not found in local_model_path '{local_model_path}'."
+                    )
+
+            self.model_path_or_id = local_model_path
+            load_kwargs = {
+                "pretrained_model_name_or_path": self.model_path_or_id,
+                "local_files_only": True,
+                "trust_remote_code": True,
+            }
+        else:
+            load_kwargs = {
+                "pretrained_model_name_or_path": self.model_path_or_id,
+                "trust_remote_code": True,
+            }
+
+        self.tokenizer = AutoTokenizer.from_pretrained(**load_kwargs)
+        config = AutoConfig.from_pretrained(**load_kwargs)
 
-        config = AutoConfig.from_pretrained(
-            self.model_name_huggingface, trust_remote_code=True
-        )
         if hasattr(config, "max_position_embeddings"):
             self.max_context_length = config.max_position_embeddings
         elif self.tokenizer.model_max_length is not None:
@@ -97,7 +124,7 @@ def batch_inference(
                     [
                         "vllm",
                         "serve",
-                        str(self.model_name_huggingface),
+                        str(self.model_path_or_id),
                         "--port",
                         str(self.vllm_port),
                         "--dtype",
@@ -120,7 +147,7 @@ def batch_inference(
                         "-m",
                         "sglang.launch_server",
                         "--model-path",
-                        str(self.model_name_huggingface),
+                        str(self.model_path_or_id),
                         "--port",
                         str(self.vllm_port),
                         "--dtype",
@@ -311,7 +338,7 @@ def _query_prompting(self, inference_data: dict):
         start_time = time.time()
         if len(extra_body) > 0:
             api_response = self.client.completions.create(
-                model=self.model_name_huggingface,
+                model=self.model_path_or_id,
                 temperature=self.temperature,
                 prompt=formatted_prompt,
                 max_tokens=leftover_tokens_count,
@@ -320,7 +347,7 @@ def _query_prompting(self, inference_data: dict):
             )
         else:
             api_response = self.client.completions.create(
-                model=self.model_name_huggingface,
+                model=self.model_path_or_id,
                 temperature=self.temperature,
                 prompt=formatted_prompt,
                 max_tokens=leftover_tokens_count,