在本教程中，我们使用 Underdog Saluki 27B，这是 Qwen3.8-27B 的一个 7.9 GB 版本，据其开发者称，它在大约每权重两比特的情况下仍能保持工具调用能力，并且在他们自己的 120 个任务基准测试中甚至略微胜过完整的 54 GB 模型。Saluki 基于同一模型的 ISTA-DASLab GSQ-RCO 量化构建，因此我们在原生 llama.cpp 中将这两个文件并排比较。我们从文件本身入手：先对两个 GGUF 头部进行逐张量 diff，再进行字节级比较，以判断 Saluki 究竟是 ISTA 比特的重新混编还是一个经过修改的模型。然后，我们在单个 GPU 上分别部署每个文件，读取其内存和速度，并在来自 Berkeley Function Calling Leaderboard 的同一批固定随机种子的 120 个任务上对两者评分，使用 BFCL 自带的检查器进行严格和宽松两种解析。
Copy Codeimport os
import re
import ast
import sys
import json
import math
import copy
import enum
import glob
import time
import types
import random
import shutil
import struct
import hashlib
import tarfile
import platform
import traceback
import subprocess
import collections
import urllib.request
import urllib.error
RESULTS = {} # step -> one-line result, for the summary table
CLAIMS = [] # (step, claim, held) rows; held is True, False or None (not decided in this run)
CURRENT = ["1"]
def banner(title):
print("\n" + "=" * 78)
print(title)
print("=" * 78)
def section(name):
def wrap(fn):
def run(*a, **kw):
CURRENT[0] = name.split(".")[0]
banner(name)
t0 = time.time()
try:
out = fn(*a, **kw)
RESULTS[name] = out if isinstance(out, str) else "ok"
return out
except Exception as e:
RESULTS[name] = f"SKIPPED / FAILED -> {type(e).__name__}: {e}"
print(f"\n[!] {name} did not complete: {type(e).__name__}: {e}")
traceback.print_exc(limit=3)
return None
finally:
print(f"\n[{time.time() - t0:.1f} s]")
json.dump({"results": RESULTS, "claims": CLAIMS}, open(os.path.join(WORK, "results.json"), "w"), indent=1)
return run
return wrap
def claim(text, held):
CLAIMS.append((CURRENT[0], text, held))
return held
banner("1. Platform, pins and helpers")
SYSTEM, MACHINE = platform.system(), platform.machine()
ON_CUDA = SYSTEM == "Linux" and shutil.which("nvidia-smi") is not None
assert ON_CUDA or (SYSTEM == "Darwin" and MACHINE == "arm64"), "needs an NVIDIA GPU on Linux (Colab T4) or an Apple-silicon Mac"
WORK = os.environ.get("SALUKI_WORK") or ("/content/saluki" if os.path.isdir("/content") else os.path.abspath("saluki_work"))
for d in ("models", "llama", "bfcl", "runs"):
os.makedirs(os.path.join(WORK, d), exist_ok=True)
# Everything the notebook downloads is pinned by size and SHA-256.
LLAMA_TAG = "b11565" # llama.cpp release of 2026-10-12, commit f9a95aeb2
LLAMA_ASSET = {"Linux": ("llama-b11565-bin-ubuntu-cuda-12.8-x64.tar.gz", 171559844,
"61b8cf4e49739afe4008b06a92390dff999e706e9e7e9749c015544a9f241627"),
"Darwin": ("llama-b11565-bin-macos-arm64.tar.gz", 12105509,
"19289548468babcd5131cecccf3a9bc7c67030cc7f0a978dafc0db1ba7891068")}[SYSTEM]
MODELS = {
"Saluki": dict(repo="ConwayResearch/Underdog-Saluki-27B-1.0", rev="4f60eba09f4b6db63c72915d328d97e1d93edb8c",
file="Underdog-Saluki-27B-1.0-IQ2-mix.gguf", size=7898369152,
sha256="4a673518b11b1c4445f9b9a9d3c40356f5ba39f6a6260dcb25f3fbfd475d9efb"),
"ISTA": dict(repo="ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF", rev="d562806dbafae37109975e970aae91b43e73b440",
file="Qwen3.8-27B-GSQ-RCO-IQ2_XS.gguf", size=8422841472,
sha256="f0ae5006da0ce6225935339e4e989369f94de95d2263cf969519f8420c9ae02c"),
}
# BFCL v4 data and BFCL's own checker code, at one gorilla commit.
GORILLA = ("https://raw.githubusercontent.com/ShishirPatil/gorilla/6ea57973c7a6097fd7c5915698c54c17c5b1b6c8/"
"berkeley-function-call-leaderboard/bfcl_eval/")
BFCL_FILES = {
"data/BFCL_v4_simple_python.json": "82dd63ba502eb2520c6b5d1d9a5c4b590e03ff261565175561f6228a367d1991",
"data/BFCL_v4_multiple.json": "aef168155ebd74b7ac2401198b201343bc7d16d7a3d7e0d4e6d8ee82c6969b2a",
"data/BFCL_v4_irrelevance.json": "2b6ed4c2e992cdcf5f1678a701851f944bef7550ee026ed1ddb89efed5be01a6",
"data/BFCL_v4_parallel.json": "19f51a82eff42e5d62541aa500115a056eb78f437c2ba1f10415fd7c8e5dda84",
"data/BFCL_v4_parallel_multiple.json": "8863ea8433239f55c5f016154cf0830853c89f693c6ea270396a2fa121960579",
"data/possible_answer/BFCL_v4_simple_python.json": "90cd5bc653690ee8e459b5b3f3fc9458606f7f3fcbf795bb51b7dc581f8c86dc",
"data/possible_answer/BFCL_v4_multiple.json": "244e00ce9395df948bcafc7bee64e8f9c87ef70887587d83cae45b13699f3047",
"data/possible_answer/BFCL_v4_parallel.json": "8a6aa19c1adddc6a5a2f7e40f9dbf30cc7e95815e7b830c90589ab318229e0f0",
"data/possible_answer/BFCL_v4_parallel_multiple.json": "5ebf24f458c1f16300c05505d83d6f0a1b68b79be273a033febd0d4f840507e3",
"eval_checker/ast_eval/ast_checker.py": "2aae7a68461a8f76c0be3894c8901b66b56967a1989d3ab066051e3fb97f1538",
"constants/enums.py": "2182becfa2a1d071ee1db30db593b4758c6bf866aa12d2d4b8daf09175ea518a",
"constants/type_mappings.py": "1702fb67afbe2c492608e58e2b7d02e46381f50166b47f3c952f76e34c7cd3bd",
"utils.py": "3703b9bb63f83581c60b8e0b82aac74c360c9ca781e99e7d4eb28b0cce2285dd",
"model_handler/utils.py": "f78fd3edce603b333dc9a88ee2c041dc547d51f71aa449ffebc044c4b1e353f3",
}
# Underdog Bench's five kinds of tasks, 24 each; we draw our own seeded sample because its task list is not published.
KINDS = [("simple_python", "Call the one right function"), ("multiple", "Pick the right function from several"),
("irrelevance", "Know when no tool fits"), ("parallel", "Same function, several calls"),
("parallel_multiple", "Several different functions")]
N_PER_KIND, SEED, CTX, PORT = 24, 2026, 16384, 8080
# llama-server's own defaults are temperature 0.8 and top_k 40, so every request states its sampler.
GREEDY = dict(temperature=0, top_k=0, top_p=1.0, min_p=0)
def sh(cmd, **kw):
return subprocess.run(cmd, capture_output=True, text=True, **kw)
def sha256_file(path, chunk=1 << 22):
h = hashlib.sha256()
with open(path, "rb") as f:
for b in iter(lambda: f.read(chunk), b""):
h.update(b)
return h.hexdigest()
def fetch(url, dest, size=None, sha=None, quiet=False):
"""Resumable download with curl, then a size and SHA-256 check. Returns seconds spent downloading."""
t0 = time.time()
if not (os.path.exists(dest) and (size is None or os.path.getsize(dest) == size)):
for attempt in range(5):
r = sh(["curl", "-fsSL", "-C", "-", "--retry", "5", "-o", dest, url])
if r.returncode == 0 and (size is None or os.path.getsize(dest) == size):
break
time.sleep(5 * (attempt + 1))
got = os.path.getsize(dest)
assert size is None or got == size, f"{os.path.basename(dest)}: {got} bytes, expected {size}"
if sha:
assert sha256_file(dest) == sha, f"{os.path.basename(dest)}: SHA-256 mismatch"
return time.time() - t0
print(f"platform : {SYSTEM} {MACHINE}, Python {platform.python_version()}, glibc {platform.libc_ver()[1] or 'n/a'}")
if SYSTEM == "Linux":
print("os :", open("/etc/os-release").read().split('PRETTY_NAME="')[1].split('"')[0])
gpu = sh(["nvidia-smi", "--query-gpu=name,memory.total,driver_version", "--format=csv,noheader"]).stdout.strip()
print("gpu :", gpu)
# The CUDA 12.8 build of llama.cpp ships without the CUDA runtime; Colab's image has it as pip wheels.
import importlib.util
if importlib.util.find_spec("nvidia.cuda_runtime") is None or importlib.util.find_spec("nvidia.cublas") is None:
subprocess.run([sys.executable, "-m", "pip", "install", "-q", "nvidia-cuda-runtime-cu12==12.9.79",
"nvidia-cublas-cu12==12.9.2.10"], check=True)
import nvidia.cuda_runtime, nvidia.cublas
CUDA_LIBS = [os.path.join(list(m.__path__)[0], "lib") for m in (nvidia.cuda_runtime, nvidia.cublas)]
else:
chip = sh(["sysctl", "-n", "machdep.cpu.brand_string"]).stdout.strip()
mem = int(sh(["sysctl", "-n", "hw.memsize"]).stdout) / 2**30
print(f"gpu : {chip}, {mem:.0f} GB unified memory (Metal)")
CUDA_LIBS = []
HW = gpu.split(",")[0] if SYSTEM == "Linux" else chip
print(f"work dir : {WORK} ({shutil.disk_usage(WORK).free / 1e9:.0f} GB free)")
笔记本所触及的一切都在任何程序运行之前被固定下来。设置单元按大小和 SHA-256 记录 llama.cpp 版本、两个模型文件以及十四个 BFCL 文件，并为每个请求固定采样器，因为 llama-server 否则会回退到我们没有选择的默认值。它会检测平台（Linux 上的 NVIDIA GPU 或 Apple 芯片 Mac），并在 Colab 上确保预构建 llama.cpp 所需的 CUDA 12 运行时库可导入。
Copy Code@section("2. Download llama.cpp b11565 and both 2-bit files, and check every byte against its pin")
def download_step():
global LLAMA_DIR, SERVER_BIN, SERVER_ENV
name, size, sha = LLAMA_ASSET
tar = os.path.join(WORK, "llama", name)
fetch(f"https://github.com/ggml-org/llama.cpp/releases/download/{LLAMA_TAG}/{name}", tar, size, sha)
with tarfile.open(tar) as t:
t.extractall(os.path.join(WORK, "llama"), filter="data")
SERVER_BIN = glob.glob(os.path.join(WORK, "llama", "**", "llama-server"), recursive=True)[0]
LLAMA_DIR = os.path.dirname(SERVER_BIN)
for b in glob.glob(os.path.join(LLAMA_DIR, "llama-*")):
os.chmod(b, 0o755)
SERVER_ENV = dict(os.environ)
if CUDA_LIBS:
SERVER_ENV["LD_LIBRARY_PATH"] = ":".join(CUDA_LIBS + [LLAMA_DIR, SERVER_ENV.get("LD_LIBRARY_PATH", "")])
missing = [l for l in sh(["ldd", SERVER_BIN], env=SERVER_ENV).stdout.splitlines() if "not found" in l]
assert not missing, f"unresolved libraries: {missing}"
r = sh([SERVER_BIN, "--version"], env=SERVER_ENV)
ver = re.search(r"version: .*", r.stdout + r.stderr).group(0)
print(f"llama.cpp: {name} (SHA-256 checked) -> {ver}")
for rel, digest in BFCL_FILES.items():
dest = os.path.join(WORK, "bfcl", rel.replace("/", "__"))
fetch(GORILLA + rel, dest, sha=digest)
print(f"BFCL v4: {len(BFCL_FILES)} files from gorilla@6ea5797 (5 task files, 4 answer files, 5 checker sources), all SHA-256 checked")
for tag, m in MODELS.items():
path = os.path.join(WORK, "models", m["file"])
had = os.path.exists(path) and os.path.getsize(path) == m["size"]
url = f"https://huggingface.co/{m['repo']}/resolve/{m['rev']}/{m['file']}"
dl = fetch(url, path, m["size"]) # size check now, hash below
t0 = time.time()
ok = sha256_file(path) == m["sha256"]
how = "already on disk" if had else f"downloaded in {dl:.0f} s ({m['size'] / dl / 1e6:.0f} MB/s)"
print(f"{tag:7s}: {m['file']} {m['size']:,} bytes, {how}; SHA-256 {'matches' if ok else 'DOES NOT MATCH'} "
f"the Hub's LFS pin ({time.time() - t0:.0f} s to hash)")
claim(f"{m['file']} is byte-identical to the file pinned at {m['repo']}@{m['rev'][:7]}", ok)
assert ok
return f"llama.cpp {LLAMA_TAG}; Saluki 7.90 GB and ISTA IQ2_XS 8.42 GB verified by SHA-256"
download_step()
我们下载 llama.cpp b11565 的 CUDA 构建、两个 2-bit 文件以及 BFCL 数据，并对照其固定值检查每一个字节。Hugging Face 下载是可断点续传的，哈希值是 Hub 在固定修订版本上的 LFS 摘要，因此被悄悄替换的文件会使笔记本停止而不是改变其结果。BFCL 的任务文件、答案文件和检查器代码全部来自同一个 gorilla 提交。该单元会打印下载所花的时间，这是 Colab 上运行中最慢的部分。
Copy CodeGGML_TYPES = {0: ("F32", 1, 4), 1: ("F16", 1, 2), 8: ("Q8_0", 32, 34), 10: ("Q2_K", 256, 84), 12: ("Q4_K", 256, 144),
16: ("IQ2_XXS", 256, 66), 17: ("IQ2_XS", 256, 74), 18: ("IQ3_XXS", 256, 98), 19: ("IQ1_S", 256, 50),
21: ("IQ3_S", 256, 110), 22: ("IQ2_S", 256, 82), 23: ("IQ4_XS", 256, 136), 29: ("IQ1_M", 256, 56),
30: ("BF16", 1, 2)} # name, values per block, bytes per block (ggml-common.h)
def read_gguf(path):
"""GGUF v3 header: metadata, tensor table (name, dims, type, offset) and where the tensor data starts."""
f = open(path, "rb")
u = lambda fmt: struct.unpack("<" + fmt, f.read(struct.calcsize(fmt)))[0]
s = lambda: f.read(u("Q")).decode("utf-8", "replace")
scalar = {0: "B", 1: "b", 2: "H", 3: "h", 4: "I", 5: "i", 6: "f", 7: "?", 10: "Q", 11: "q", 12: "d"}
def val(t):
if t == 8:
return s()
if t == 9:
et, n = u("I"), u("Q")
return [val(et) for _ in range(n)]
return u(scalar[t])
assert f.read(4) == b"GGUF"
version, n_tensors, n_kv = u("I"), u("Q"), u("Q")
kv = {}
for _ in range(n_kv):
k = s()
kv[k] = val(u("I"))
tensors = []
for _ in range(n_tensors):
name = s()
dims = [u("Q") for _ in range(u("I"))]
ttype, off = u("I"), u("Q")
n = math.prod(dims)
_, per, nbytes = GGML_TYPES[ttype]
tensors.append(dict(name=name, dims=dims, type=GGML_TYPES[ttype][0], n=n, bytes=n // per * nbytes, off=off))
align = kv.get("general.alignment", 32)
data_start = (f.tell() + align - 1) // align * align
return kv, tensors, data_start
def tensor_class(name):
if not name.startswith("blk."):
return name.split(".")[0] # token_embd, output, output_norm
kind = name.split(".")[2]
if kind in ("attn_q", "attn_k", "attn_v", "attn_output"):
return "full-attention " + kind[5:]
if kind in ("attn_qkv", "attn_gate", "ssm_out", "ssm_conv1d"):
return "linear-attention " + kind
if kind in ("ssm_alpha", "ssm_beta"):
return "linear-attention gates (alpha, beta)"
if kind in ("ffn_gate", "ffn_up", "ffn_down"):
return kind
return "norms and small vectors"
@section("3. What Saluki changes: a tensor-by-tensor diff of the two GGUF headers")
def header_diff():
global HDR
HDR = {tag: read_gguf(os.path.join(WORK, "models", m["file"])) for tag, m in MODELS.items()}
(kv_s, t_s, start_s), (kv_i, t_i, start_i) = HDR["Saluki"], HDR["ISTA"]
assert [t["name"] for t in t_s] == [t["name"] for t in t_i] and [t["dims"] for t in t_s] == [t["dims"] for t in t_i]
n_params = sum(t["n"] for t in t_s)
print(f"both files: {len(t_s)} tensors with the same names and shapes, {n_params:,} weights, "
f"architecture '{kv_s['general.architecture']}', {kv_s['qwen35.block_count']} blocks")
for tag, (kv, ts, start) in HDR.items():
data = sum(t["bytes"] for t in ts)
size = MODELS[tag]["size"]
print(f" {tag:7s}: header {start:,} B + tensor data {data:,} B = {start + data:,} B "
f"({'equals' if start + data == size else 'DIFFERS FROM'} the file size); {8 * data / n_params:.3f} bits per weight")
claim(f"{tag}: header plus tensor bytes computed from the type table equals the file size exactly", start + data == size)
rows = collections.OrderedDict()
moves = collections.Counter()
up = down = 0
for a, b in zip(t_i, t_s):
r = rows.setdefault(tensor_class(a["name"]), dict(k=0, n=0, bi=0, bs=0))
r["k"] += 1; r["n"] += a["n"]; r["bi"] += a["bytes"]; r["bs"] += b["bytes"]
if a["type"] != b["type"]:
moves[(a["type"], b["type"])] += 1
up += b["bytes"] > a["bytes"]; down += b["bytes"] < a["bytes"]
print(f"\n{'tensor class':34s} {'count':>5s} {'weights':>9s} {'ISTA bpw':>9s} {'Saluki bpw':>10s} {'change MB':>10s}")
for c, r in sorted(rows.items(), key=lambda kv: -kv[1]["n"]):
if r["n"] < 1e6:
continue
print(f"{c:34s} {r['k']:5d} {r['n'] / 1e9:8.3f}B {8 * r['bi'] / r['n']:9.3f} {8 * r['bs'] / r['n']:10.3f} {(r['bs'] - r['bi']) / 1e6:+10.1f}")
changed = sum(moves.values())
print(f"\n{changed} of {len(t_s)} tensors change type: " + ", ".join(f"{n} {a}->{b}" for (a, b), n in moves.most_common()))
print(f" {down} get fewer bytes, {up} get more")
claim("every tensor whose type changes gets fewer bits; none gets more", up == 0 and down == changed)
untouched = [c for c, r in rows.items() if r["bi"] == r["bs"] and r["n"] >= 1e6]
print(f" classes with no change at all: {', '.join(sorted(untouched))}")
proj = lambda c: c.startswith(("full-attention", "linear-attention")) and "gates" not in c
claim("every attention and DeltaNet projection matrix keeps its type (only the small alpha/beta gates change)",
all(a["type"] == b["type"] for a, b in zip(t_i, t_s) if proj(tensor_class(a["name"]))))
keys = sorted(k for k in set(kv_s) | set(kv_i) if kv_s.get(k) != kv_i.get(k) and not k.startswith("tokenizer.ggml.t"))
print(f"\nmetadata keys that differ: {keys}")
print(f"Saluki's header still reads: general.name = {kv_s['general.name']!r}, quantized_by = {kv_s['general.quantized_by']!r}")
print(f"sampling stored in both headers: temp {kv_s['general.sampling.temp']}, top_k {kv_s['general.sampling.top_k']}, "
f"top_p {kv_s['general.sampling.top_p']:.2f} (these are Qwen's thinking-mode settings; step 5 shows what the server does with them)")
saved = (MODELS["ISTA"]["size"] - MODELS["Saluki"]["size"]) / 1e6
return f"{changed} of {len(t_s)} tensor types changed, all downgrades; {saved:.0f} MB smaller"
header_diff()
用纯 Python 读取两个 GGUF 头部，可以准确显示 Saluki 改变了什么。两个文件都拥有相同形状的相同 851 个张量，类型表可以预测每个文件的字节大小。Saluki 改变了 146 个张量的类型，而且每一处改动都削减比特：大多数落在前馈下投影和输出层上，而小的 DeltaNet 门控向量从 BF16 降至 Q8_0。没有注意力或 DeltaNet 投影矩阵被改动，头部仍然将 ISTA DASLab 列为量化者。
Copy Code@section("4. Byte-level provenance: are the unchanged tensors ISTA's own bytes?")
def provenance():
import numpy as np
(_, t_s, start_s), (_, t_i, start_i) = HDR["Saluki"], HDR["ISTA"]
ms = np.memmap(os.path.join(WORK, "models", MODELS["Saluki"]["file"]), dtype=np.uint8, mode="r")
mi = np.memmap(os.path.join(WORK, "models", MODELS["ISTA"]["file"]), dtype=np.uint8, mode="r")
q = dict(k=0, same=0, eq_bytes=0, bytes=0)
f32 = collections.defaultdict(lambda: [0, 0, []]) # kind -> [identical, differ, relative L2 changes]
for a, b in zip(t_i, t_s):
if a["type"] != b["type"]:
continue
x = mi[start_i + a["off"]: start_i + a["off"] + a["bytes"]]
y = ms[start_s + b["off"]: start_s + b["off"] + b["bytes"]]
if a["type"] == "F32":
kind = re.sub(r"blk\.\d+\.", "", a["name"])
u, v = x.view(np.float32).astype(np.float64), y.view(np.float32).astype(np.float64)
same = bool(np.array_equal(u, v))
f32[kind][0 if same else 1] += 1
if not same:
f32[kind][2].append(np.linalg.norm(u - v) / np.linalg.norm(u))
else:
eq = int(np.count_nonzero(x == y))
q["k"] += 1; q["same"] += eq == a["bytes"]; q["eq_bytes"] += eq; q["bytes"] += a["bytes"]
print(f"quantised matrices that keep their type: {q['same']} of {q['k']} byte-identical, but "
f"{100 * q['eq_bytes'] / q['bytes']:.1f}% of their {q['bytes'] / 1e9:.2f} GB of bytes are equal")
print(f"\nunquantised F32 vectors (never re-encoded, so any change means the weights changed):")
print(f" {'kind':28s} {'identical':>9s} {'changed':>8s} median relative change")
for kind, (same, diff, rel) in sorted(f32.items(), key=lambda kv: -kv[1][1]):
med = f"{sorted(rel)[len(rel) // 2]:.1e}" if rel else "-"
print(f" {kind:28s} {same:9d} {diff:8d} {med}")
changed = sum(v[1] for v in f32.values())
total = sum(v[0] + v[1] for v in f32.values())
claim("Saluki's unchanged-type tensors are byte-for-byte ISTA's (a pure type re-mix)", q["same"] == q["k"] and changed == 0)
claim("Saluki's F32 norm and gate vectors differ from ISTA's, so its weights were changed, not only re-encoded", changed > 0)
return (f"{q['same']}/{q['k']} same-type matrices byte-identical, {100 * q['eq_bytes'] / q['bytes']:.1f}% of their bytes equal; "
f"{changed}/{total} F32 vectors changed")
provenance()
类型相同并不意味着权重相同，因此我们比较保持类型的 705 个张量的字节。352 个量化矩阵中只有一个是字节完全相同的，但它们 98.4% 的字节匹配，这与对底层权重进行小幅编辑然后在相同网格上重新量化相一致。未量化的归一化和门控向量给出了定论：大多数变化的中位数为万分之几。Saluki 是一个经过轻微修改的模型，而不是 ISTA 文件的重新编码。
Copy CodeSERVER = {}
def start_server(tag):
"""One llama-server per model, one slot, all layers on the GPU, debug log kept for unparsed replies."""
stop_server()
log_path = os.path.join(WORK, "runs", f"server_{tag}.log")
cmd = [SERVER_BIN, "-m", os.path.join(WORK, "models", MODELS[tag]["file"]), "--host", "127.0.0.1", "--port", str(PORT),
"-c", str(CTX), "-np", "1", "-ngl", "99", "-fa", "on", "--jinja", "--reasoning-format", "deepseek", "-lv", "5"]
t0 = time.time()
proc = subprocess.Popen(cmd, stdout=open(log_path, "w"), stderr=subprocess.STDOUT, env=SERVER_ENV)
while True:
try:
if urllib.request.urlopen(f"http://127.0.0.1:{PORT}/health", timeout=5).status == 200:
break
except urllib.error.HTTPError as e:
assert e.code == 503, e
except urllib.error.URLError:
pass
assert proc.poll() is None, f"llama-server exited; see {log_path}"
assert time.time() - t0 < 900, "server did not become healthy in 15 minutes"
time.sleep(2)
SERVER.update(tag=tag, proc=proc, log=log_path, load_s=time.time() - t0)
return SERVER["load_s"]
def stop_server():
if SERVER.get("proc") and SERVER["proc"].poll() is None:
SERVER["proc"].terminate()
SERVER["proc"].wait(timeout=60)
SERVER.clear()
def api(path, body=None, timeout=1800):
req = urllib.request.Request(f"http://127.0.0.1:{PORT}{path}", None if body is None else json.dumps(body).encode(),
{"Content-Type": "application/json"})
with urllib.request.urlopen(req, timeout=timeout) as r:
return json.loads(r.read())
def memory_report():
"""Buffer sizes from the server's load log, plus what the GPU reports."""
text = open(SERVER["log"]).read()
out = {}
for label, what in (("weights", "model"), ("KV cache", "KV"), ("recurrent state", "RS"), ("compute", "compute")):
last = {} # llama.cpp sizes every buffer in a dry run first; keep the final value per device
for dev, mib in re.findall(rf"(\S+) +{what} buffer size = +([\d.]+) MiB", text):
last[dev] = float(mib)
out[label] = sum(last.values())
if ON_CUDA:
out["GPU in use (nvidia-smi)"] = float(sh(["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"]).stdout.split()[0])
return out
@section("5. A hybrid model on one GPU: start the server, read its memory, and look at the prompt it builds")
def architecture_and_server():
kv = HDR["Saluki"][0]
L, every = kv["qwen35.block_count"], kv["qwen35.full_attention_interval"]
full = [i for i in range(L) if (i + 1) % every == 0]
heads_kv, dk = kv["qwen35.attention.head_count_kv"], kv["qwen35.attention.key_length"]
kv_tok = len(full) * heads_kv * dk * 2 * 2 # K and V, f16
conv = (kv["qwen35.ssm.conv_kernel"] - 1) * (kv["qwen35.ssm.inner_size"] + 2 * kv["qwen35.ssm.group_count"] * kv["qwen35.ssm.state_size"])
state = kv["qwen35.ssm.state_size"] * kv["qwen35.ssm.inner_size"]
rs = (L - len(full)) * (conv + state) * 4
print(f"{L} blocks: {L - len(full)} Gated DeltaNet (linear attention) and {len(full)} full-attention layers (3, 7, ..., {full[-1]})")
print(f"KV cache: {kv_tok // 1024} KiB per token for the {len(full)} attention layers; a model with attention in all "
f"{L} layers would need {L * heads_kv * dk * 4 // 1024} KiB")
print(f"recurrent state: {rs / 2**20:.1f} MiB per sequence, fixed, whatever the context length")
print(f"expected at -c {CTX}: KV {kv_tok * CTX / 2**20:.0f} MiB + state {rs / 2**20:.0f} MiB")
load_s = start_server("Saluki")
mem = memory_report()
print(f"\nllama-server up in {load_s:.0f} s (-c {CTX}, -np 1, -ngl 99, -fa on); buffers from its log:")
for k, v in mem.items():
print(f" {k:24s} {v:8.0f} MiB")
claim(f"the server's KV buffer at -c {CTX} matches 64 KiB per token within 2%",
abs(mem["KV cache"] - kv_tok * CTX / 2**20) <= 0.02 * kv_tok * CTX / 2**20)
SERVER_MEM["Saluki"] = mem
props = api("/props")
p = props["default_generation_settings"]["params"]
g = {k: kv[f"general.sampling.{k}"] for k in ("temp", "top_k", "top_p", "min_p")}
from_file = abs(p["temperature"] - g["temp"]) < 1e-6 and p["top_k"] == g["top_k"] and abs(p["top_p"] - g["top_p"]) < 1e-6
print(f"\nserver defaults: temperature {p['temperature']:.2f}, top_k {p['top_k']}, top_p {p['top_p']:.2f}, min_p {p['min_p']:.2f}"
f" -> {'taken from the GGUF header' if from_file else 'llama.cpp built-ins, not the header'}; the card suggests 0.6 for thinking,"
f" so every request below states its own sampler")
claim("llama-server takes its default sampler from the GGUF's general.sampling keys", from_file)
caps = props.get("chat_template_caps", {})
print("template supports: " + ", ".join(k.replace("supports_", "") for k, v in caps.items() if v))
msgs = [{"role": "user", "content": "What is the weather in Oslo?"}]
tool = [{"type": "function", "function": {"name": "get_weather", "description": "Current weather for a city",
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}}}]
for label, kw in (("thinking off", {"enable_thinking": False}), ("thinking on, effort low", {"reasoning_effort": "low"}),
("thinking on, default", {})):
prompt = api("/apply-template", {"messages": msgs, "tools": tool, "chat_template_kwargs": kw})["prompt"]
n = len(api("/tokenize", {"content": prompt, "add_special": False})["tokens"])
print(f" {label:24s}: {n:3d} prompt tokens, ends {prompt[-28:]!r}")
return f"KV {kv_tok // 1024} KiB/token, state {rs / 2**20:.0f} MiB; server buffers {sum(mem[k] for k in ('weights', 'KV cache', 'recurrent state', 'compute')) / 1024:.1f} GiB"
SERVER_MEM = {}
architecture_and_server()
Qwen3.8-27B 是一种混合架构：其 64 个块中有 48 个是带有固定循环状态的 Gated DeltaNet 线性注意力层，只有每第四个块保留完整注意力。因此 KV 缓存每 token 花费 64 KiB 而非 256 KiB，外加每个序列约 150 MiB 的状态。我们以单槽位启动 llama-server，从日志中读取其缓冲区，检查其默认采样器的来源，并查看聊天模板在思考关闭、开启和低力度时构建的提示词。
Copy CodeSPEED = {}
def speed_probe(tag):
"""Prefill and decode rates from the server's own timings: two prompt lengths, 128 forced output tokens."""
rng = random.Random(SEED)
words = "def class return import yield lambda tensor gradient matrix kernel cache token buffer layer".split()
rows = []
for target in (256, 1024):
text = " ".join(rng.choice(words) for _ in range(target))
ids = api("/tokenize", {"content": text, "add_special": False})["tokens"][:target]
r = api("/completion", {"prompt": ids, "n_predict": 128, "ignore_eos": True, "cache_prompt": False, **GREEDY})
t = r["timings"]
rows.append((t["prompt_n"], t["prompt_per_second"], t["predicted_n"], t["predicted_per_second"]))
print(f" {tag:7s} prompt {t['prompt_n']:5d} tok at {t['prompt_per_second']:7.1f} tok/s | "
f"decode {t['predicted_n']} tok at {t['predicted_per_second']:5.2f} tok/s")
SPEED[tag] = rows
return rows
@section("6. Prefill and decode speed, measured by the server")
def speed_step():
print(f"hardware: {HW}")
rows = speed_probe("Saluki")
return f"Saluki on {HW}: prefill {rows[1][1]:.0f} tok/s at {rows[1][0]} tokens, decode {rows[1][3]:.1f} tok/s"
speed_step()
速度来自服务器自身的计时，而不是围绕请求的秒表。我们发送一个 256 token 和一个 1,024 token 的提示词，关闭提示词缓存并强制生成 128 个输出 token，从而得到每个长度下的预填充率和解码率。稠密的 27B 模型为每个生成的 token 读取其完整权重文件，因此解码速度主要取决于内存带宽，稍后我们还会在 ISTA 的文件上重复同样的探测，以便进行同等条件比较。
Copy Codedef load_bfcl_checker():
"""BFCL's own checker functions, executed from the pinned source with their package imports replaced."""
src = lambda rel: open(os.path.join(WORK, "bfcl", rel.replace("/", "__"))).read()
def run(rel, ns, only=None):
tree = ast.parse(src(rel))
body = [n for n in tree.body if not isinstance(n, (ast.Import, ast.ImportFrom))]
if only:
body = [n for n in body if getattr(n, "name", None) in only]
exec(compile(ast.Module(body=body, type_ignores=[]), rel, "exec"), ns)
return ns
enums = run("constants/enums.py", {"Enum": enum.Enum})
tmap = run("constants/type_mappings.py", {})
model_cfg = collections.defaultdict(lambda: types.SimpleNamespace(underscore_to_dot=True)) # OpenAI-style FC model
chk = run("eval_checker/ast_eval/ast_checker.py", {"re": re, "Language": enums["Language"], "MODEL_CONFIG_MAPPING": model_cfg,
"JAVA_TYPE_CONVERSION": tmap["JAVA_TYPE_CONVERSION"], "JS_TYPE_CONVERSION": tmap["JS_TYPE_CONVERSION"],
"java_type_converter": None, "js_type_converter": None})
utl = run("utils.py", {}, {"is_function_calling_format_output", "is_empty_output"})
conv = run("model_handler/utils.py", {"copy": copy, "re": re, "json": json, "ModelStyle": enums["ModelStyle"],
"GORILLA_TO_OPENAPI": tmap["GORILLA_TO_OPENAPI"]}, {"_cast_to_openai_type", "convert_to_tool"})
return types.SimpleNamespace(
ast_checker=lambda funcs, out, gt, cat: chk["ast_checker"](funcs, out, gt, enums["Language"].PYTHON, cat, "fc-model"),
is_empty=utl["is_empty_output"],
to_tools=lambda funcs: conv["convert_to_tool"](funcs, tmap["GORILLA_TO_OPENAPI"], enums["ModelStyle"].OPENAI_COMPLETIONS))
def load_tasks():
tasks = []
for kind, label in KINDS:
rows = [json.loads(l) for l in open(os.path.join(WORK, "bfcl", f"data__BFCL_v4_{kind}.json")) if l.strip()]
gt = {}
if kind != "irrelevance":
for l in open(os.path.join(WORK, "bfcl", f"data__possible_answer__BFCL_v4_{kind}.json")):
if l.strip():
d = json.loads(l); gt[d["id"]] = d["ground_truth"]
for r in sorted(random.Random(f"{SEED}-{kind}").sample(rows, N_PER_KIND), key=lambda r: int(r["id"].rsplit("_", 1)[1])):
tasks.append(dict(id=r["id"], kind=kind, messages=r["question"][0], functions=r["function"], gt=gt.get(r["id"])))
return tasks
TOOL_BLOCK = re.compile(r"<tool_call>(.*?)(?:</tool_call>|$)", re.S)
FUNC_XML = re.compile(r"<function=([^>\s]+)\s*>(.*?)(?:</function>|$)", re.S)
PARAM_XML = re.compile(r"<parameter=([^>\s]+)\s*>\n?(.*?)\n?</parameter>", re.S)
def lenient_calls(raw, functions):
"""Forgiving reading of the model's raw text: any <function=...> block, closed or not, inside or outside <tool_call>.
Values are typed with the function's own schema (BFCL types). Not part of BFCL; used only for the 'lenient' column."""
raw = raw.split("</think>")[-1] # with thinking on, the reasoning precedes </think>
types_ = {re.sub(r"\.", "_", f["name"]): {k: v.get("type", "string") for k, v in f["parameters"]["properties"].items()}
for f in functions} # tool names were sent with dots turned into underscores
calls = []
for fname, body in FUNC_XML.findall(raw):
fname = fname.strip().strip('"')
args = {}
for p, v in PARAM_XML.findall(body):
t = types_.get(re.sub(r"\.", "_", fname), {}).get(p, "string")
v = v.strip() if t != "string" else v
try:
args[p] = (int(v) if t == "integer" else float(v) if t in ("float", "number") else
v.lower() == "true" if t == "boolean" and v.lower() in ("true", "false") else
json.loads(v) if t in ("array", "tuple", "dict", "object") else v)
except ValueError:
try:
args[p] = ast.literal_eval(v)
except Exception:
args[p] = v
calls.append({fname: args})
return calls
def score(task, calls):
"""BFCL's rule: irrelevance passes with no call; the other kinds go through BFCL's ast_checker."""
if task["kind"] == "irrelevance":
return CHECK.is_empty(calls), "made a call" if calls else None
if not calls:
return False, "no call"
try:
r = CHECK.ast_checker(task["functions"], calls, task["gt"], task["kind"])
except Exception as e: # malformed arguments the checker cannot walk
return False, f"checker_error:{type(e).__name__}"
return bool(r["valid"]), r.get("error_type")
def run_task(task, think=False, max_tokens=512):
body = {"messages": task["messages"], "tools": CHECK.to_tools(task["functions"]), "max_tokens": max_tokens,
"cache_prompt": False, "verbose": True, **GREEDY,
"chat_template_kwargs": {"reasoning_effort": "low"} if think else {"enable_thinking": False}}
log_pos = os.path.getsize(SERVER["log"])
t0 = time.time()
rec = dict(id=task["id"], kind=task["kind"], think=think)
try:
r = api("/v1/chat/completions", body)
msg = r["choices"][0]["message"]
rec.update(status=200, finish=r["choices"][0]["finish_reason"], raw=r["__verbose"]["content"],
prompt_n=r["timings"]["prompt_n"], out_n=r["timings"]["predicted_n"],
reasoning_chars=len(msg.get("reasoning_content") or ""))
rec["calls"] = [{tc["function"]["name"]: json.loads(tc["function"]["arguments"])} for tc in msg.get("tool_calls") or []]
except urllib.error.HTTPError as e: # stock llama.cpp answers 500 when its parser cannot read the reply
time.sleep(0.5)
with open(SERVER["log"], errors="replace") as f:
f.seek(log_pos)
new = f.read()
m = re.findall(r"=== BEGIN ===\n(.*?)\n=== END ===", new, re.S)
rec.update(status=e.code, finish="parse error", raw=m[-1] if m else "", calls=None, prompt_n=None, out_n=None)
rec["seconds"] = time.time() - t0
rec["strict"], rec["strict_why"] = score(task, rec["calls"]) if rec["calls"] is not None else (False, "unparsed (HTTP 500)")
if rec["strict"]:
rec["lenient"] = True
else:
rec["lenient"], _ = score(task, lenient_calls(rec["raw"], task["functions"]))
return rec
def bench(tag, tasks):
out, t0 = [], time.time()
for i, task in enumerate(tasks):
out.append(run_task(task))
if (i + 1) % 30 == 0:
print(f" {tag}: {i + 1}/{len(tasks)} done, {time.time() - t0:.0f} s")
json.dump(out, open(os.path.join(WORK, "runs", f"bfcl_{tag}.json"), "w"), indent=1)
return out
def ci(k, n, alpha=0.05):
"""Clopper-Pearson interval by bisection on the binomial tail."""
def cdf(x, p):
return sum(math.comb(n, i) * p ** i * (1 - p) ** (n - i) for i in range(x + 1))
def solve(f, target, rising):
lo, hi = 0.0, 1.0
for _ in range(60):
mid = (lo + hi) / 2
lo, hi = (mid, hi) if (f(mid) < target) == rising else (lo, mid)
return (lo + hi) / 2
lo = 0.0 if k == 0 else solve(lambda p: 1 - cdf(k - 1, p), alpha / 2, True)
hi = 1.0 if k == n else solve(lambda p: cdf(k, p), alpha / 2, False)
return lo, hi
def kind_table(tag, recs):
print(f"\n{tag}: {'kind':38s} {'strict':>8s} {'lenient':>8s} replies: structured / plain text / unparsed (500) / cut off")
for kind, label in KINDS:
rs = [r for r in recs if r["kind"] == kind]
st = collections.Counter("unparsed" if r["status"] != 200 else "cut off" if r["finish"] == "length"
else "structured" if r["calls"] else "plain text" for r in rs)
print(f" {label:44s} {sum(r['strict'] for r in rs):3d}/{len(rs):<4d} {sum(r['lenient'] for r in rs):3d}/{len(rs):<4d}"
f" {st['structured']:3d} / {st['plain text']:3d} / {st['unparsed']:3d} / {st['cut off']:3d}")
k, n = sum(r["strict"] for r in recs), len(recs)
lo, hi = ci(k, n)
print(f" {'total':44s} {k:3d}/{n:<4d} {sum(r['lenient'] for r in recs):3d}/{n:<4d} strict 95% CI {100 * lo:.0f}-{100 * hi:.0f}%")
@section("7. Tool calling on 120 BFCL v4 tasks, thinking off: Saluki")
def bench_saluki():
global CHECK, TASKS, RUNS
CHECK = load_bfcl_checker()
TASKS = load_tasks()
print(f"sample: {N_PER_KIND} tasks per kind, drawn with seed {SEED}, {len(TASKS)} in all; scored with BFCL's ast_checker "
f"(gorilla@6ea5797); greedy, thinking off, max_tokens 512")
print(f"first ids per kind: " + ", ".join(t["id"] for t in TASKS[::N_PER_KIND]))
RUNS = {"Saluki": bench("Saluki", TASKS)}
kind_table("Saluki", RUNS["Saluki"])
sec = [r["seconds"] for r in RUNS["Saluki"]]
print(f"\nmedian {sorted(sec)[len(sec) // 2]:.1f} s per task on {HW}; median output {sorted(r['out_n'] or 0 for r in RUNS['Saluki'])[len(sec) // 2]} tokens")
again = [run_task(t) for t in TASKS[:: len(TASKS) // 5][:5]]
same = sum(a["calls"] == b["calls"] and a["raw"] == b["raw"] for a, b in zip(again, RUNS["Saluki"][:: len(TASKS) // 5][:5]))
print(f"greedy re-run of 5 tasks: {same} of 5 identical raw replies")
claim("greedy decoding with the prompt cache off gives the same reply on a re-run (5 tasks)", same == 5)
s = sum(r["strict"] for r in RUNS["Saluki"])
return f"Saluki {s}/{len(TASKS)} strict, {sum(r['lenient'] for r in RUNS['Saluki'])}/{len(TASKS)} lenient"
bench_saluki()
Underdog Bench 的 120 个任务并未公开，因此我们自行抽取：以固定随机种子从相同的五个 BFCL v4 类别中每类抽取 24 个。我们将每个任务作为 OpenAI 工具请求发送到服务器，使用 BFCL 自带的函数进行转换，关闭思考并使用贪心解码。严格评分仅使用 llama.cpp 解析出的调用和 BFCL 的 ast_checker，而宽松评分则用宽容的解析器重新读取原始回复，这正是我们测试厂商所称的严格与宽容之间差距的方法。
Copy Code@section("8. Does thinking help? Six multi-call tasks with reasoning on, effort low")
def thinking_step():
pick = [t for t in TASKS if t["kind"] in ("parallel", "parallel_multiple")]
pick = random.Random(SEED).sample(pick, min(6, len(pick)))
off = {r["id"]: r for r in RUNS["Saluki"]}
rows = []
for t in pick:
r = run_task(t, think=True, max_tokens=2048)
o = off[t["id"]]
rows.append((t["id"], o, r))
print(f" {t['id']:22s} off: {'pass' if o['strict'] else 'fail'} {o['out_n'] or 0:4d} tok {o['seconds']:5.1f} s | "
f"on: {'pass' if r['strict'] else 'fail'} {r['out_n'] or 0:4d} tok {r['seconds']:6.1f} s ({r['finish']})")
p_off, p_on = sum(o["strict"] for _, o, _ in rows), sum(r["strict"] for _, _, r in rows)
tok = sum(r["out_n"] or 0 for *_, r in rows) / max(1, sum(o["out_n"] or 0 for _, o, _ in rows))
sec = sum(r["seconds"] for *_, r in rows) / sum(o["seconds"] for _, o, _ in rows)
print(f"\nthinking off {p_off}/{len(rows)}, thinking on {p_on}/{len(rows)}; thinking used {tok:.1f}x the output tokens and {sec:.1f}x the time")
return f"thinking on (effort low): {p_on}/{len(rows)} vs {p_off}/{len(rows)} off, {tok:.1f}x tokens, {sec:.1f}x time"
thinking_step()
开启思考是这张卡针对一般用途的建议,但在 27B 规模下,每个推理 token 都会消耗时间。我们以最低努力程度开启推理,重新运行六个带种子的多次调用任务,并与关闭思考运行的相同任务进行比较:通过或失败、输出 token 数和秒数。六个任务无法确定思考是否改善工具调用,因此该单元格聚焦于成本,并展示当模型先思考时,单次工具调用会多消耗多少 token。
复制代码@section("9. The parent file: ISTA's IQ2_XS through the same server, the same sample and the same checker")
def bench_ista():
load_s = start_server("ISTA")
mem = memory_report()
SERVER_MEM["ISTA"] = mem
print(f"llama-server up in {load_s:.0f} s; " + ", ".join(f"{k} {v:.0f} MiB" for k, v in mem.items()))
print(f"weights: ISTA {mem['weights']:.0f} MiB vs Saluki {SERVER_MEM['Saluki']['weights']:.0f} MiB")
speed_probe("ISTA")
s, i = SPEED["Saluki"][1], SPEED["ISTA"][1]
print(f"decode at {s[0]} prompt tokens: Saluki {s[3]:.2f} vs ISTA {i[3]:.2f} tok/s; prefill {s[1]:.0f} vs {i[1]:.0f} tok/s")
RUNS["ISTA"] = bench("ISTA", TASKS)
kind_table("ISTA", RUNS["ISTA"])
stop_server()
k = sum(r["strict"] for r in RUNS["ISTA"])
return f"ISTA IQ2_XS {k}/{len(TASKS)} strict, {sum(r['lenient'] for r in RUNS['ISTA'])}/{len(TASKS)} lenient"
bench_ista()
父文件遵循相同的路径。我们停止 Saluki 的服务器,使用相同标志在 ISTA 的 IQ2_XS 文件上启动一个服务器,读取其内存,重复速度探测,并在相同的 120 个任务上用相同的检查器和解析规则为其评分。一次只运行一个模型使每个模型都完全处于 GPU 内存范围内,而对两者使用同一个测试框架可从比较中消除提示、采样或解析方面的差异。
复制代码def mcnemar(a, b):
"""Exact two-sided McNemar test on paired pass/fail lists."""
b10 = sum(x and not y for x, y in zip(a, b))
b01 = sum(y and not x for x, y in zip(a, b))
n, k = b10 + b01, min(b10, b01)
p = min(1.0, 2 * sum(math.comb(n, i) for i in range(k + 1)) / 2 ** n) if n else 1.0
return b10, b01, p
@section("10. Head to head: the same 120 tasks, paired, next to the vendor's numbers")
def head_to_head():
S, I = RUNS["Saluki"], RUNS["ISTA"]
assert [r["id"] for r in S] == [r["id"] for r in I]
vendor = {"simple_python": 24, "multiple": 23, "irrelevance": 18, "parallel": 12, "parallel_multiple": 11}
print(f"{'kind':38s} {'Saluki':>7s} {'ISTA':>6s} | {'lenient':>7s} {'':>5s} | vendor's Saluki (of 24)")
for kind, label in KINDS:
s = [r for r in S if r["kind"] == kind]; i = [r for r in I if r["kind"] == kind]
print(f"{label:38s} {sum(r['strict'] for r in s):7d} {sum(r['strict'] for r in i):6d} | "
f"{sum(r['lenient'] for r in s):7d} {sum(r['lenient'] for r in i):5d} | {vendor[kind]:3d}")
out = {}
for mode in ("strict", "lenient"):
a, b = [r[mode] for r in S], [r[mode] for r in I]
b10, b01, p = mcnemar(a, b)
out[mode] = (sum(a), sum(b), b10, b01, p)
print(f"{mode:8s}: Saluki {sum(a)}/120, ISTA {sum(b)}/120; Saluki-only passes {b10}, ISTA-only {b01}, exact McNemar p = {p:.3f}")
print("vendor's Underdog Bench (different 120 tasks, unpublished): Saluki 88, ISTA IQ2_XS 76, full Qwen3.8-27B 84")
sa, ia, b10, b01, p = out["strict"]
claim("Saluki passes more of our 120 tasks than its parent (strict scoring)", sa > ia)
claim("the difference is significant at 5% (exact McNemar)", p < 0.05)
pick_s = sum(r["strict"] for r in S if r["kind"] in ("simple_python", "multiple"))
pick_i = sum(r["strict"] for r in I if r["kind"] in ("simple_python", "multiple"))
claim("Saluki is at least as good as its parent at picking the right single function (first two kinds)", pick_s >= pick_i)
return f"strict: Saluki {sa}/120 vs ISTA {ia}/120 (McNemar p = {p:.2f}); lenient: {out['lenient'][0]} vs {out['lenient'][1]}"
head_to_head()
由于两个模型回答的是相同的任务,该比较是配对的。该单元格将各类型得分与厂商公布的 Saluki 各类型数字并列,并在两个文件不一致的任务上运行精确 McNemar 检验,分别采用严格和宽松评分。在 120 个任务中出现几个任务的差距处于运行间噪声之内,正如厂商页面自己也承认的,因此该检验告诉我们我们的样本是否显示出真实差异。
复制代码@section("11. Where the failures come from")
def failures():
for tag in ("Saluki", "ISTA"):
recs = RUNS[tag]
why = collections.Counter((r["strict_why"] or "?").split(":")[-1] for r in recs if not r["strict"])
print(f"{tag}: {sum(not r['strict'] for r in recs)} failures -> " + ", ".join(f"{k} {v}" for k, v in why.most_common()))
rescued = [r for r in recs if r["lenient"] and not r["strict"]]
print(f" lenient reading rescues {len(rescued)}: " + ", ".join(r["id"] for r in rescued[:8]))
ex = next((r for r in RUNS["Saluki"] if not r["strict"] and r["kind"] != "irrelevance"), None)
if ex:
task = next(t for t in TASKS if t["id"] == ex["id"])
print(f"\nexample, {ex['id']} ({ex['strict_why']}):")
print(" expected:", json.dumps(task["gt"])[:300])
print(" model :", (json.dumps(ex["calls"]) if ex["calls"] is not None else ex["raw"])[:300])
irr = [r for r in RUNS["Saluki"] if r["kind"] == "irrelevance" and not r["strict"]]
if irr:
print(f"\nirrelevance: Saluki made a call on {len(irr)} of {N_PER_KIND} prompts that no tool fits, e.g. {irr[0]['id']}: "
f"{json.dumps(irr[0]['calls'])[:160]}")
un = sum(r["status"] != 200 for tag in RUNS for r in RUNS[tag])
return f"{un} replies across both models were unparseable by the stock server"
failures()
得分会掩盖调用在哪里出错,因此该单元格按 BFCL 自身的错误类型对失败进行分解:调用次数错误、参数错误或缺失、在没有合适工具时进行的调用,以及标准服务器无法解析的回复。它还列出了宽松解读所挽救的任务,并打印一个失败任务及其期望答案,这是区分模型错误与评分伪影的最快方式。
复制代码banner("SUMMARY")
for name, res in RESULTS.items():
print(f" {name:<100s} {res}")
print("\nClaims decided in this run:")
for step, text, held in CLAIMS:
print(f" [{'held' if held else 'DID NOT HOLD' if held is False else 'undecided'}] step {step}: {text}")
failed = [k for k, v in RESULTS.items() if str(v).startswith("SKIPPED")]
print(f"\n{sum(h is True for *_, h in CLAIMS)} of {len(CLAIMS)} claims held; steps failed or skipped: {failed or 'none'}")
print(f"run on: {HW}; llama.cpp {LLAMA_TAG}; Python {platform.python_version()}")
print("not tested here: the full-size Qwen3.8-27B; Underdog Bench's own 120 tasks and the 100-task parallel set (unpublished); "
"SWE-bench, WebWalkerQA, IFEval and AIME; the vision add-on; Mia and Ternary Bonsai 2; thinking at the default effort.")
摘要打印每个步骤的单行结果和声明账本(成立、不成立或未定),以及硬件和 llama.cpp 版本.cpp 版本。它以笔记本未测试的内容作结:完整尺寸的 Qwen3.8-27B、厂商自己的任务列表、智能体和数学基准,以及视觉附加组件。
总而言之,Saluki 并不完全像其文件所显示的那样。其头部保留了 ISTA 的 851 张量布局,并仅在 ISTA 花费更多位的地方节省位,但其字节和未量化的向量显示这是一个权重被更改过的模型,这与 Underdog 称其为预览检查点相一致。这使得与其父文件的工具调用比较成为对该训练的测试,而非仅对量化的测试。两个文件都在原版 llama.cpp 中逐个运行,而 BFCL 自身的检查器在相同任务上提供了配对比较。小样本、贪婪解码和单次运行限制了任何一个笔记本所能声明的内容,因此区间与得分同等重要。
请查看 完整代码。所有功劳归于该项目的研究者。另外,欢迎在 Twitter 上关注我们,并且别忘了加入我们的 150k+ ML SubReddit 并订阅 我们的通讯。等等!你在用 telegram 吗? 现在你也可以在 telegram 上加入我们。
需要与我们合作推广你的 GitHub 仓库或 Hugging Face 页面或产品发布或网络研讨会等吗? 与我们联系
本文 《如何运行 Underdog Saluki 27B:2 比特 Qwen3.8-27B 工具调用指南》 首发于 MarkTechPost.