في هذا الدرس التعليمي، نعمل مع Underdog Saluki 27B، وهو نسخة بحجم 7.9 GB من Qwen3.8-27B يقول صانعوها إنها تحافظ على استدعاء الأدوات سليماً عند نحو بتّين لكل وزن، بل تتفوق حتى على النموذج الكامل البالغ 54 GB على معيارهم الخاص المكوّن من 120 مهمة. Saluki مبني على تحويل GSQ-RCO الكمي من ISTA-DASLab للنموذج نفسه، لذلك وضعنا الملفين جنباً إلى جنب في llama.cpp الأصلي. نبدأ بالملفات نفسها: مقارنة tensor-by-tensor لترويستي GGUF، ثم مقارنة على مستوى البايت تكشف ما إذا كان Saluki إعادة مزجاً لبتات ISTA أم نموذجاً متغيّراً. ثم نخدم كل ملف على وحدة معالجة رسومات واحدة، ونقرأ استهلاكه للذاكرة وسرعته، ونقيّم كليهما على المهام الـ 120 ذاتها المزروعة البذرة من Berkeley Function Calling Leaderboard، باستخدام أداة التحقق الخاصة بـ BFCL مع تحليل صارم ومتساهل.
نسخ الكودimport os
import re
import ast
import sys
import json
import math
import copy
import enum
import glob
import time
import types
import random
import shutil
import struct
import hashlib
import tarfile
import platform
import traceback
import subprocess
import collections
import urllib.request
import urllib.error
RESULTS = {} # step -> one-line result, for the summary table
CLAIMS = [] # (step, claim, held) rows; held is True, False or None (not decided in this run)
CURRENT = ["1"]
def banner(title):
print("\n" + "=" * 78)
print(title)
print("=" * 78)
def section(name):
def wrap(fn):
def run(*a, **kw):
CURRENT[0] = name.split(".")[0]
banner(name)
t0 = time.time()
try:
out = fn(*a, **kw)
RESULTS[name] = out if isinstance(out, str) else "ok"
return out
except Exception as e:
RESULTS[name] = f"SKIPPED / FAILED -> {type(e).__name__}: {e}"
print(f"\n[!] {name} did not complete: {type(e).__name__}: {e}")
traceback.print_exc(limit=3)
return None
finally:
print(f"\n[{time.time() - t0:.1f} s]")
json.dump({"results": RESULTS, "claims": CLAIMS}, open(os.path.join(WORK, "results.json"), "w"), indent=1)
return run
return wrap
def claim(text, held):
CLAIMS.append((CURRENT[0], text, held))
return held
banner("1. Platform, pins and helpers")
SYSTEM, MACHINE = platform.system(), platform.machine()
ON_CUDA = SYSTEM == "Linux" and shutil.which("nvidia-smi") is not None
assert ON_CUDA or (SYSTEM == "Darwin" and MACHINE == "arm64"), "needs an NVIDIA GPU on Linux (Colab T4) or an Apple-silicon Mac"
WORK = os.environ.get("SALUKI_WORK") or ("/content/saluki" if os.path.isdir("/content") else os.path.abspath("saluki_work"))
for d in ("models", "llama", "bfcl", "runs"):
os.makedirs(os.path.join(WORK, d), exist_ok=True)
# Everything the notebook downloads is pinned by size and SHA-256.
LLAMA_TAG = "b11565" # llama.cpp release of 2026-10-12, commit f9a95aeb2
LLAMA_ASSET = {"Linux": ("llama-b11565-bin-ubuntu-cuda-12.8-x64.tar.gz", 171559844,
"61b8cf4e49739afe4008b06a92390dff999e706e9e7e9749c015544a9f241627"),
"Darwin": ("llama-b11565-bin-macos-arm64.tar.gz", 12105509,
"19289548468babcd5131cecccf3a9bc7c67030cc7f0a978dafc0db1ba7891068")}[SYSTEM]
MODELS = {
"Saluki": dict(repo="ConwayResearch/Underdog-Saluki-27B-1.0", rev="4f60eba09f4b6db63c72915d328d97e1d93edb8c",
file="Underdog-Saluki-27B-1.0-IQ2-mix.gguf", size=7898369152,
sha256="4a673518b11b1c4445f9b9a9d3c40356f5ba39f6a6260dcb25f3fbfd475d9efb"),
"ISTA": dict(repo="ISTA-DASLab/Qwen3.8-27B-GSQ-RCO-GGUF", rev="d562806dbafae37109975e970aae91b43e73b440",
file="Qwen3.8-27B-GSQ-RCO-IQ2_XS.gguf", size=8422841472,
sha256="f0ae5006da0ce6225935339e4e989369f94de95d2263cf969519f8420c9ae02c"),
}
# BFCL v4 data and BFCL's own checker code, at one gorilla commit.
GORILLA = ("https://raw.githubusercontent.com/ShishirPatil/gorilla/6ea57973c7a6097fd7c5915698c54c17c5b1b6c8/"
"berkeley-function-call-leaderboard/bfcl_eval/")
BFCL_FILES = {
"data/BFCL_v4_simple_python.json": "82dd63ba502eb2520c6b5d1d9a5c4b590e03ff261565175561f6228a367d1991",
"data/BFCL_v4_multiple.json": "aef168155ebd74b7ac2401198b201343bc7d16d7a3d7e0d4e6d8ee82c6969b2a",
"data/BFCL_v4_irrelevance.json": "2b6ed4c2e992cdcf5f1678a701851f944bef7550ee026ed1ddb89efed5be01a6",
"data/BFCL_v4_parallel.json": "19f51a82eff42e5d62541aa500115a056eb78f437c2ba1f10415fd7c8e5dda84",
"data/BFCL_v4_parallel_multiple.json": "8863ea8433239f55c5f016154cf0830853c89f693c6ea270396a2fa121960579",
"data/possible_answer/BFCL_v4_simple_python.json": "90cd5bc653690ee8e459b5b3f3fc9458606f7f3fcbf795bb51b7dc581f8c86dc",
"data/possible_answer/BFCL_v4_multiple.json": "244e00ce9395df948bcafc7bee64e8f9c87ef70887587d83cae45b13699f3047",
"data/possible_answer/BFCL_v4_parallel.json": "8a6aa19c1adddc6a5a2f7e40f9dbf30cc7e95815e7b830c90589ab318229e0f0",
"data/possible_answer/BFCL_v4_parallel_multiple.json": "5ebf24f458c1f16300c05505d83d6f0a1b68b79be273a033febd0d4f840507e3",
"eval_checker/ast_eval/ast_checker.py": "2aae7a68461a8f76c0be3894c8901b66b56967a1989d3ab066051e3fb97f1538",
"constants/enums.py": "2182becfa2a1d071ee1db30db593b4758c6bf866aa12d2d4b8daf09175ea518a",
"constants/type_mappings.py": "1702fb67afbe2c492608e58e2b7d02e46381f50166b47f3c952f76e34c7cd3bd",
"utils.py": "3703b9bb63f83581c60b8e0b82aac74c360c9ca781e99e7d4eb28b0cce2285dd",
"model_handler/utils.py": "f78fd3edce603b333dc9a88ee2c041dc547d51f71aa449ffebc044c4b1e353f3",
}
# Underdog Bench's five kinds of tasks, 24 each; we draw our own seeded sample because its task list is not published.
KINDS = [("simple_python", "Call the one right function"), ("multiple", "Pick the right function from several"),
("irrelevance", "Know when no tool fits"), ("parallel", "Same function, several calls"),
("parallel_multiple", "Several different functions")]
N_PER_KIND, SEED, CTX, PORT = 24, 2026, 16384, 8080
# llama-server's own defaults are temperature 0.8 and top_k 40, so every request states its sampler.
GREEDY = dict(temperature=0, top_k=0, top_p=1.0, min_p=0)
def sh(cmd, **kw):
return subprocess.run(cmd, capture_output=True, text=True, **kw)
def sha256_file(path, chunk=1 << 22):
h = hashlib.sha256()
with open(path, "rb") as f:
for b in iter(lambda: f.read(chunk), b""):
h.update(b)
return h.hexdigest()
def fetch(url, dest, size=None, sha=None, quiet=False):
"""Resumable download with curl, then a size and SHA-256 check. Returns seconds spent downloading."""
t0 = time.time()
if not (os.path.exists(dest) and (size is None or os.path.getsize(dest) == size)):
for attempt in range(5):
r = sh(["curl", "-fsSL", "-C", "-", "--retry", "5", "-o", dest, url])
if r.returncode == 0 and (size is None or os.path.getsize(dest) == size):
break
time.sleep(5 * (attempt + 1))
got = os.path.getsize(dest)
assert size is None or got == size, f"{os.path.basename(dest)}: {got} bytes, expected {size}"
if sha:
assert sha256_file(dest) == sha, f"{os.path.basename(dest)}: SHA-256 mismatch"
return time.time() - t0
print(f"platform : {SYSTEM} {MACHINE}, Python {platform.python_version()}, glibc {platform.libc_ver()[1] or 'n/a'}")
if SYSTEM == "Linux":
print("os :", open("/etc/os-release").read().split('PRETTY_NAME="')[1].split('"')[0])
gpu = sh(["nvidia-smi", "--query-gpu=name,memory.total,driver_version", "--format=csv,noheader"]).stdout.strip()
print("gpu :", gpu)
# The CUDA 12.8 build of llama.cpp ships without the CUDA runtime; Colab's image has it as pip wheels.
import importlib.util
if importlib.util.find_spec("nvidia.cuda_runtime") is None or importlib.util.find_spec("nvidia.cublas") is None:
subprocess.run([sys.executable, "-m", "pip", "install", "-q", "nvidia-cuda-runtime-cu12==12.9.79",
"nvidia-cublas-cu12==12.9.2.10"], check=True)
import nvidia.cuda_runtime, nvidia.cublas
CUDA_LIBS = [os.path.join(list(m.__path__)[0], "lib") for m in (nvidia.cuda_runtime, nvidia.cublas)]
else:
chip = sh(["sysctl", "-n", "machdep.cpu.brand_string"]).stdout.strip()
mem = int(sh(["sysctl", "-n", "hw.memsize"]).stdout) / 2**30
print(f"gpu : {chip}, {mem:.0f} GB unified memory (Metal)")
CUDA_LIBS = []
HW = gpu.split(",")[0] if SYSTEM == "Linux" else chip
print(f"work dir : {WORK} ({shutil.disk_usage(WORK).free / 1e9:.0f} GB free)")
كل ما يمسه الدفتر مثبّت قبل تشغيل أي شيء. خلية الإعداد تسجّل إصدار llama.cpp، وملفّي النموذج، وأربعة عشر ملفاً من BFCL حسب الحجم وSHA-256، وتثبّت أداة أخذ العينات لكل طلب، لأن llama-server يعود وإلا إلى الافتراضيات التي لم نخترها. فهي تكشف المنصة، وحدة معالجة رسومات NVIDIA على لينكس أو ماك بمعالج Apple-silicon، وعلى Colab تضمن استيراد مكتبات تشغيل CUDA 12 التي يحتاجها llama. cpp المجمّع مسبقاً.
نسخ الكود@section("2. Download llama.cpp b11565 and both 2-bit files, and check every byte against its pin")
def download_step():
global LLAMA_DIR, SERVER_BIN, SERVER_ENV
name, size, sha = LLAMA_ASSET
tar = os.path.join(WORK, "llama", name)
fetch(f"https://github.com/ggml-org/llama.cpp/releases/download/{LLAMA_TAG}/{name}", tar, size, sha)
with tarfile.open(tar) as t:
t.extractall(os.path.join(WORK, "llama"), filter="data")
SERVER_BIN = glob.glob(os.path.join(WORK, "llama", "**", "llama-server"), recursive=True)[0]
LLAMA_DIR = os.path.dirname(SERVER_BIN)
for b in glob.glob(os.path.join(LLAMA_DIR, "llama-*")):
os.chmod(b, 0o755)
SERVER_ENV = dict(os.environ)
if CUDA_LIBS:
SERVER_ENV["LD_LIBRARY_PATH"] = ":".join(CUDA_LIBS + [LLAMA_DIR, SERVER_ENV.get("LD_LIBRARY_PATH", "")])
missing = [l for l in sh(["ldd", SERVER_BIN], env=SERVER_ENV).stdout.splitlines() if "not found" in l]
assert not missing, f"unresolved libraries: {missing}"
r = sh([SERVER_BIN, "--version"], env=SERVER_ENV)
ver = re.search(r"version: .*", r.stdout + r.stderr).group(0)
print(f"llama.cpp: {name} (SHA-256 checked) -> {ver}")
for rel, digest in BFCL_FILES.items():
dest = os.path.join(WORK, "bfcl", rel.replace("/", "__"))
fetch(GORILLA + rel, dest, sha=digest)
print(f"BFCL v4: {len(BFCL_FILES)} files from gorilla@6ea5797 (5 task files, 4 answer files, 5 checker sources), all SHA-256 checked")
for tag, m in MODELS.items():
path = os.path.join(WORK, "models", m["file"])
had = os.path.exists(path) and os.path.getsize(path) == m["size"]
url = f"https://huggingface.co/{m['repo']}/resolve/{m['rev']}/{m['file']}"
dl = fetch(url, path, m["size"]) # size check now, hash below
t0 = time.time()
ok = sha256_file(path) == m["sha256"]
how = "already on disk" if had else f"downloaded in {dl:.0f} s ({m['size'] / dl / 1e6:.0f} MB/s)"
print(f"{tag:7s}: {m['file']} {m['size']:,} bytes, {how}; SHA-256 {'matches' if ok else 'DOES NOT MATCH'} "
f"the Hub's LFS pin ({time.time() - t0:.0f} s to hash)")
claim(f"{m['file']} is byte-identical to the file pinned at {m['repo']}@{m['rev'][:7]}", ok)
assert ok
return f"llama.cpp {LLAMA_TAG}; Saluki 7.90 GB and ISTA IQ2_XS 8.42 GB verified by SHA-256"
download_step()
ننزّل نسخة CUDA من llama.cpp b11565، وكلا الملفين بدقة 2 بت، وبيانات BFCL، ونتحقق من كل بايت مقابل مرجعه الثابت. التنزيلات من Hugging Face قابلة للاستئناف، والتجزئات هي بصمات LFS الخاصة بالمركز نفسه عند مراجعة ثابتة، بحيث يتوقف الدفتر عند استبدال أي ملف صامتاً بدلاً من تغيير نتائجه. وملفات المهام وملفات الإجابات وكود المدقق في BFCL كلها تأتي من كوميت واحد في gorilla. تطبع الخلية مدة التنزيلات، وهي أبطأ جزء في التشغيل على Colab.
نسخ الكودGGML_TYPES = {0: ("F32", 1, 4), 1: ("F16", 1, 2), 8: ("Q8_0", 32, 34), 10: ("Q2_K", 256, 84), 12: ("Q4_K", 256, 144),
16: ("IQ2_XXS", 256, 66), 17: ("IQ2_XS", 256, 74), 18: ("IQ3_XXS", 256, 98), 19: ("IQ1_S", 256, 50),
21: ("IQ3_S", 256, 110), 22: ("IQ2_S", 256, 82), 23: ("IQ4_XS", 256, 136), 29: ("IQ1_M", 256, 56),
30: ("BF16", 1, 2)} # name, values per block, bytes per block (ggml-common.h)
def read_gguf(path):
"""GGUF v3 header: metadata, tensor table (name, dims, type, offset) and where the tensor data starts."""
f = open(path, "rb")
u = lambda fmt: struct.unpack("<" + fmt, f.read(struct.calcsize(fmt)))[0]
s = lambda: f.read(u("Q")).decode("utf-8", "replace")
scalar = {0: "B", 1: "b", 2: "H", 3: "h", 4: "I", 5: "i", 6: "f", 7: "?", 10: "Q", 11: "q", 12: "d"}
def val(t):
if t == 8:
return s()
if t == 9:
et, n = u("I"), u("Q")
return [val(et) for _ in range(n)]
return u(scalar[t])
assert f.read(4) == b"GGUF"
version, n_tensors, n_kv = u("I"), u("Q"), u("Q")
kv = {}
for _ in range(n_kv):
k = s()
kv[k] = val(u("I"))
tensors = []
for _ in range(n_tensors):
name = s()
dims = [u("Q") for _ in range(u("I"))]
ttype, off = u("I"), u("Q")
n = math.prod(dims)
_, per, nbytes = GGML_TYPES[ttype]
tensors.append(dict(name=name, dims=dims, type=GGML_TYPES[ttype][0], n=n, bytes=n // per * nbytes, off=off))
align = kv.get("general.alignment", 32)
data_start = (f.tell() + align - 1) // align * align
return kv, tensors, data_start
def tensor_class(name):
if not name.startswith("blk."):
return name.split(".")[0] # token_embd, output, output_norm
kind = name.split(".")[2]
if kind in ("attn_q", "attn_k", "attn_v", "attn_output"):
return "full-attention " + kind[5:]
if kind in ("attn_qkv", "attn_gate", "ssm_out", "ssm_conv1d"):
return "linear-attention " + kind
if kind in ("ssm_alpha", "ssm_beta"):
return "linear-attention gates (alpha, beta)"
if kind in ("ffn_gate", "ffn_up", "ffn_down"):
return kind
return "norms and small vectors"
@section("3. What Saluki changes: a tensor-by-tensor diff of the two GGUF headers")
def header_diff():
global HDR
HDR = {tag: read_gguf(os.path.join(WORK, "models", m["file"])) for tag, m in MODELS.items()}
(kv_s, t_s, start_s), (kv_i, t_i, start_i) = HDR["Saluki"], HDR["ISTA"]
assert [t["name"] for t in t_s] == [t["name"] for t in t_i] and [t["dims"] for t in t_s] == [t["dims"] for t in t_i]
n_params = sum(t["n"] for t in t_s)
print(f"both files: {len(t_s)} tensors with the same names and shapes, {n_params:,} weights, "
f"architecture '{kv_s['general.architecture']}', {kv_s['qwen35.block_count']} blocks")
for tag, (kv, ts, start) in HDR.items():
data = sum(t["bytes"] for t in ts)
size = MODELS[tag]["size"]
print(f" {tag:7s}: header {start:,} B + tensor data {data:,} B = {start + data:,} B "
f"({'equals' if start + data == size else 'DIFFERS FROM'} the file size); {8 * data / n_params:.3f} bits per weight")
claim(f"{tag}: header plus tensor bytes computed from the type table equals the file size exactly", start + data == size)
rows = collections.OrderedDict()
moves = collections.Counter()
up = down = 0
for a, b in zip(t_i, t_s):
r = rows.setdefault(tensor_class(a["name"]), dict(k=0, n=0, bi=0, bs=0))
r["k"] += 1; r["n"] += a["n"]; r["bi"] += a["bytes"]; r["bs"] += b["bytes"]
if a["type"] != b["type"]:
moves[(a["type"], b["type"])] += 1
up += b["bytes"] > a["bytes"]; down += b["bytes"] < a["bytes"]
print(f"\n{'tensor class':34s} {'count':>5s} {'weights':>9s} {'ISTA bpw':>9s} {'Saluki bpw':>10s} {'change MB':>10s}")
for c, r in sorted(rows.items(), key=lambda kv: -kv[1]["n"]):
if r["n"] < 1e6:
continue
print(f"{c:34s} {r['k']:5d} {r['n'] / 1e9:8.3f}B {8 * r['bi'] / r['n']:9.3f} {8 * r['bs'] / r['n']:10.3f} {(r['bs'] - r['bi']) / 1e6:+10.1f}")
changed = sum(moves.values())
print(f"\n{changed} of {len(t_s)} tensors change type: " + ", ".join(f"{n} {a}->{b}" for (a, b), n in moves.most_common()))
print(f" {down} get fewer bytes, {up} get more")
claim("every tensor whose type changes gets fewer bits; none gets more", up == 0 and down == changed)
untouched = [c for c, r in rows.items() if r["bi"] == r["bs"] and r["n"] >= 1e6]
print(f" classes with no change at all: {', '.join(sorted(untouched))}")
proj = lambda c: c.startswith(("full-attention", "linear-attention")) and "gates" not in c
claim("every attention and DeltaNet projection matrix keeps its type (only the small alpha/beta gates change)",
all(a["type"] == b["type"] for a, b in zip(t_i, t_s) if proj(tensor_class(a["name"]))))
keys = sorted(k for k in set(kv_s) | set(kv_i) if kv_s.get(k) != kv_i.get(k) and not k.startswith("tokenizer.ggml.t"))
print(f"\nmetadata keys that differ: {keys}")
print(f"Saluki's header still reads: general.name = {kv_s['general.name']!r}, quantized_by = {kv_s['general.quantized_by']!r}")
print(f"sampling stored in both headers: temp {kv_s['general.sampling.temp']}, top_k {kv_s['general.sampling.top_k']}, "
f"top_p {kv_s['general.sampling.top_p']:.2f} (these are Qwen's thinking-mode settings; step 5 shows what the server does with them)")
saved = (MODELS["ISTA"]["size"] - MODELS["Saluki"]["size"]) / 1e6
return f"{changed} of {len(t_s)} tensor types changed, all downgrades; {saved:.0f} MB smaller"
header_diff()
قراءة ترويستي GGUF في بايثون العادية تُظهر بالضبط ما غيّره Saluki. كلا الملفين يحتوي على التنسورات الـ 851 نفسها بالأشكال نفسها، وجدول الأنواع يتنبأ بحجم كل ملف بالبايت. يغيّر Saluki نوع 146 تنسوراً، وكل تغيير يقلّل البتات: معظمها يقع على مسارات الإسقاط النازلة للتغذية الأمامية وطبقة الإخراج، ومتجهات بوابة DeltaNet الصغيرة تنخفض من BF16 إلى Q8_0. لا تتغير أي مصفوفات انتباه أو إسقاط DeltaNet، ولا تزال الترويسة تسمّي ISTA DASLab كأداة التحويل الكمي.
نسخ الكود@section("4. Byte-level provenance: are the unchanged tensors ISTA's own bytes?")
def provenance():
import numpy as np
(_, t_s, start_s), (_, t_i, start_i) = HDR["Saluki"], HDR["ISTA"]
ms = np.memmap(os.path.join(WORK, "models", MODELS["Saluki"]["file"]), dtype=np.uint8, mode="r")
mi = np.memmap(os.path.join(WORK, "models", MODELS["ISTA"]["file"]), dtype=np.uint8, mode="r")
q = dict(k=0, same=0, eq_bytes=0, bytes=0)
f32 = collections.defaultdict(lambda: [0, 0, []]) # kind -> [identical, differ, relative L2 changes]
for a, b in zip(t_i, t_s):
if a["type"] != b["type"]:
continue
x = mi[start_i + a["off"]: start_i + a["off"] + a["bytes"]]
y = ms[start_s + b["off"]: start_s + b["off"] + b["bytes"]]
if a["type"] == "F32":
kind = re.sub(r"blk\.\d+\.", "", a["name"])
u, v = x.view(np.float32).astype(np.float64), y.view(np.float32).astype(np.float64)
same = bool(np.array_equal(u, v))
f32[kind][0 if same else 1] += 1
if not same:
f32[kind][2].append(np.linalg.norm(u - v) / np.linalg.norm(u))
else:
eq = int(np.count_nonzero(x == y))
q["k"] += 1; q["same"] += eq == a["bytes"]; q["eq_bytes"] += eq; q["bytes"] += a["bytes"]
print(f"quantised matrices that keep their type: {q['same']} of {q['k']} byte-identical, but "
f"{100 * q['eq_bytes'] / q['bytes']:.1f}% of their {q['bytes'] / 1e9:.2f} GB of bytes are equal")
print(f"\nunquantised F32 vectors (never re-encoded, so any change means the weights changed):")
print(f" {'kind':28s} {'identical':>9s} {'changed':>8s} median relative change")
for kind, (same, diff, rel) in sorted(f32.items(), key=lambda kv: -kv[1][1]):
med = f"{sorted(rel)[len(rel) // 2]:.1e}" if rel else "-"
print(f" {kind:28s} {same:9d} {diff:8d} {med}")
changed = sum(v[1] for v in f32.values())
total = sum(v[0] + v[1] for v in f32.values())
claim("Saluki's unchanged-type tensors are byte-for-byte ISTA's (a pure type re-mix)", q["same"] == q["k"] and changed == 0)
claim("Saluki's F32 norm and gate vectors differ from ISTA's, so its weights were changed, not only re-encoded", changed > 0)
return (f"{q['same']}/{q['k']} same-type matrices byte-identical, {100 * q['eq_bytes'] / q['bytes']:.1f}% of their bytes equal; "
f"{changed}/{total} F32 vectors changed")
provenance()
الأنواع نفسها لا تعني الأوزان نفسها، لذلك نقارن بايتات التنسورات الـ 705 التي حافظت على نوعها. مصفوفة واحدة فقط من أصل 352 مصفوفة كمّية متطابقة بايتاً ببايت، ومع ذلك تتطابق 98.4% من بايتاتها، وهو ما يتوافق مع تعديلات طفيفة على الأوزان الأساسية أعقبها إعادة تحويل كمي على الشبكات نفسها. متجهات المعايرة غير المكمّية وبوابات تحسم الأمر: تغيّر معظمها بوسيط يبلغ بضع أجزاء من عشرة آلاف. Saluki نموذج متغيّر بتغيير طفيف، وليس إعادة ترميز لملف ISTA.
نسخ الكودSERVER = {}
def start_server(tag):
"""One llama-server per model, one slot, all layers on the GPU, debug log kept for unparsed replies."""
stop_server()
log_path = os.path.join(WORK, "runs", f"server_{tag}.log")
cmd = [SERVER_BIN, "-m", os.path.join(WORK, "models", MODELS[tag]["file"]), "--host", "127.0.0.1", "--port", str(PORT),
"-c", str(CTX), "-np", "1", "-ngl", "99", "-fa", "on", "--jinja", "--reasoning-format", "deepseek", "-lv", "5"]
t0 = time.time()
proc = subprocess.Popen(cmd, stdout=open(log_path, "w"), stderr=subprocess.STDOUT, env=SERVER_ENV)
while True:
try:
if urllib.request.urlopen(f"http://127.0.0.1:{PORT}/health", timeout=5).status == 200:
break
except urllib.error.HTTPError as e:
assert e.code == 503, e
except urllib.error.URLError:
pass
assert proc.poll() is None, f"llama-server exited; see {log_path}"
assert time.time() - t0 < 900, "server did not become healthy in 15 minutes"
time.sleep(2)
SERVER.update(tag=tag, proc=proc, log=log_path, load_s=time.time() - t0)
return SERVER["load_s"]
def stop_server():
if SERVER.get("proc") and SERVER["proc"].poll() is None:
SERVER["proc"].terminate()
SERVER["proc"].wait(timeout=60)
SERVER.clear()
def api(path, body=None, timeout=1800):
req = urllib.request.Request(f"http://127.0.0.1:{PORT}{path}", None if body is None else json.dumps(body).encode(),
{"Content-Type": "application/json"})
with urllib.request.urlopen(req, timeout=timeout) as r:
return json.loads(r.read())
def memory_report():
"""Buffer sizes from the server's load log, plus what the GPU reports."""
text = open(SERVER["log"]).read()
out = {}
for label, what in (("weights", "model"), ("KV cache", "KV"), ("recurrent state", "RS"), ("compute", "compute")):
last = {} # llama.cpp sizes every buffer in a dry run first; keep the final value per device
for dev, mib in re.findall(rf"(\S+) +{what} buffer size = +([\d.]+) MiB", text):
last[dev] = float(mib)
out[label] = sum(last.values())
if ON_CUDA:
out["GPU in use (nvidia-smi)"] = float(sh(["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"]).stdout.split()[0])
return out
@section("5. A hybrid model on one GPU: start the server, read its memory, and look at the prompt it builds")
def architecture_and_server():
kv = HDR["Saluki"][0]
L, every = kv["qwen35.block_count"], kv["qwen35.full_attention_interval"]
full = [i for i in range(L) if (i + 1) % every == 0]
heads_kv, dk = kv["qwen35.attention.head_count_kv"], kv["qwen35.attention.key_length"]
kv_tok = len(full) * heads_kv * dk * 2 * 2 # K and V, f16
conv = (kv["qwen35.ssm.conv_kernel"] - 1) * (kv["qwen35.ssm.inner_size"] + 2 * kv["qwen35.ssm.group_count"] * kv["qwen35.ssm.state_size"])
state = kv["qwen35.ssm.state_size"] * kv["qwen35.ssm.inner_size"]
rs = (L - len(full)) * (conv + state) * 4
print(f"{L} blocks: {L - len(full)} Gated DeltaNet (linear attention) and {len(full)} full-attention layers (3, 7, ..., {full[-1]})")
print(f"KV cache: {kv_tok // 1024} KiB per token for the {len(full)} attention layers; a model with attention in all "
f"{L} layers would need {L * heads_kv * dk * 4 // 1024} KiB")
print(f"recurrent state: {rs / 2**20:.1f} MiB per sequence, fixed, whatever the context length")
print(f"expected at -c {CTX}: KV {kv_tok * CTX / 2**20:.0f} MiB + state {rs / 2**20:.0f} MiB")
load_s = start_server("Saluki")
mem = memory_report()
print(f"\nllama-server up in {load_s:.0f} s (-c {CTX}, -np 1, -ngl 99, -fa on); buffers from its log:")
for k, v in mem.items():
print(f" {k:24s} {v:8.0f} MiB")
claim(f"the server's KV buffer at -c {CTX} matches 64 KiB per token within 2%",
abs(mem["KV cache"] - kv_tok * CTX / 2**20) <= 0.02 * kv_tok * CTX / 2**20)
SERVER_MEM["Saluki"] = mem
props = api("/props")
p = props["default_generation_settings"]["params"]
g = {k: kv[f"general.sampling.{k}"] for k in ("temp", "top_k", "top_p", "min_p")}
from_file = abs(p["temperature"] - g["temp"]) < 1e-6 and p["top_k"] == g["top_k"] and abs(p["top_p"] - g["top_p"]) < 1e-6
print(f"\nserver defaults: temperature {p['temperature']:.2f}, top_k {p['top_k']}, top_p {p['top_p']:.2f}, min_p {p['min_p']:.2f}"
f" -> {'taken from the GGUF header' if from_file else 'llama.cpp built-ins, not the header'}; the card suggests 0.6 for thinking,"
f" so every request below states its own sampler")
claim("llama-server takes its default sampler from the GGUF's general.sampling keys", from_file)
caps = props.get("chat_template_caps", {})
print("template supports: " + ", ".join(k.replace("supports_", "") for k, v in caps.items() if v))
msgs = [{"role": "user", "content": "What is the weather in Oslo?"}]
tool = [{"type": "function", "function": {"name": "get_weather", "description": "Current weather for a city",
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}}}]
for label, kw in (("thinking off", {"enable_thinking": False}), ("thinking on, effort low", {"reasoning_effort": "low"}),
("thinking on, default", {})):
prompt = api("/apply-template", {"messages": msgs, "tools": tool, "chat_template_kwargs": kw})["prompt"]
n = len(api("/tokenize", {"content": prompt, "add_special": False})["tokens"])
print(f" {label:24s}: {n:3d} prompt tokens, ends {prompt[-28:]!r}")
return f"KV {kv_tok // 1024} KiB/token, state {rs / 2**20:.0f} MiB; server buffers {sum(mem[k] for k in ('weights', 'KV cache', 'recurrent state', 'compute')) / 1024:.1f} GiB"
SERVER_MEM = {}
architecture_and_server()
Qwen3.8-27B نموذج هجين: 48 من كتلتيه الـ 64 هي طبقات انتباه خطي Gated DeltaNet بحالة تكرارية ثابتة، وكل كتلة رابعة فقط تحتفظ بالانتباه الكامل. لذا تكلّف ذاكرة KV التخزينية المؤقتة 64 KiB لكل رمز بدلاً من 256 KiB، بالإضافة إلى نحو 150 MiB من الحالة لكل تسلسل. نشغّل llama-server بفتحة واحدة، ونقرأ مخازنه المؤقتة من السجل، ونتحقق من مصدر أداة أخذ العينات الافتراضية، وننظر إلى المطالبات التي يبنيها قالب الدردشة مع إيقاف التفكير وتشغيله وعند جهد منخفض.
نسخ الكودSPEED = {}
def speed_probe(tag):
"""Prefill and decode rates from the server's own timings: two prompt lengths, 128 forced output tokens."""
rng = random.Random(SEED)
words = "def class return import yield lambda tensor gradient matrix kernel cache token buffer layer".split()
rows = []
for target in (256, 1024):
text = " ".join(rng.choice(words) for _ in range(target))
ids = api("/tokenize", {"content": text, "add_special": False})["tokens"][:target]
r = api("/completion", {"prompt": ids, "n_predict": 128, "ignore_eos": True, "cache_prompt": False, **GREEDY})
t = r["timings"]
rows.append((t["prompt_n"], t["prompt_per_second"], t["predicted_n"], t["predicted_per_second"]))
print(f" {tag:7s} prompt {t['prompt_n']:5d} tok at {t['prompt_per_second']:7.1f} tok/s | "
f"decode {t['predicted_n']} tok at {t['predicted_per_second']:5.2f} tok/s")
SPEED[tag] = rows
return rows
@section("6. Prefill and decode speed, measured by the server")
def speed_step():
print(f"hardware: {HW}")
rows = speed_probe("Saluki")
return f"Saluki on {HW}: prefill {rows[1][1]:.0f} tok/s at {rows[1][0]} tokens, decode {rows[1][3]:.1f} tok/s"
speed_step()
السرعة تأتي من توقيتات الخادم نفسه وليس من ساعة إيقاف حول الطلب. نرسل مطالبة من 256 رمزاً وأخرى من 1,024 رمزاً مع إيقاف ذاكرة المطالبات المؤقتة وفرض 128 رمزاً إخراجياً، وهو ما يعطي معدل prefill ومعدل decode عند كل طول. يقرأ النموذج الكثيف بحجم 27B ملف أوزانه الكامل لكل رمز مولّد، لذا تعتمد سرعة decode في معظمها على نطاق الذاكرة، ويُجرى الفحص نفسه لاحقاً على ملف ISTA للمقارنة على قدم المساواة.
نسخ الكودdef load_bfcl_checker():
"""BFCL's own checker functions, executed from the pinned source with their package imports replaced."""
src = lambda rel: open(os.path.join(WORK, "bfcl", rel.replace("/", "__"))).read()
def run(rel, ns, only=None):
tree = ast.parse(src(rel))
body = [n for n in tree.body if not isinstance(n, (ast.Import, ast.ImportFrom))]
if only:
body = [n for n in body if getattr(n, "name", None) in only]
exec(compile(ast.Module(body=body, type_ignores=[]), rel, "exec"), ns)
return ns
enums = run("constants/enums.py", {"Enum": enum.Enum})
tmap = run("constants/type_mappings.py", {})
model_cfg = collections.defaultdict(lambda: types.SimpleNamespace(underscore_to_dot=True)) # OpenAI-style FC model
chk = run("eval_checker/ast_eval/ast_checker.py", {"re": re, "Language": enums["Language"], "MODEL_CONFIG_MAPPING": model_cfg,
"JAVA_TYPE_CONVERSION": tmap["JAVA_TYPE_CONVERSION"], "JS_TYPE_CONVERSION": tmap["JS_TYPE_CONVERSION"],
"java_type_converter": None, "js_type_converter": None})
utl = run("utils.py", {}, {"is_function_calling_format_output", "is_empty_output"})
conv = run("model_handler/utils.py", {"copy": copy, "re": re, "json": json, "ModelStyle": enums["ModelStyle"],
"GORILLA_TO_OPENAPI": tmap["GORILLA_TO_OPENAPI"]}, {"_cast_to_openai_type", "convert_to_tool"})
return types.SimpleNamespace(
ast_checker=lambda funcs, out, gt, cat: chk["ast_checker"](funcs, out, gt, enums["Language"].PYTHON, cat, "fc-model"),
is_empty=utl["is_empty_output"],
to_tools=lambda funcs: conv["convert_to_tool"](funcs, tmap["GORILLA_TO_OPENAPI"], enums["ModelStyle"].OPENAI_COMPLETIONS))
def load_tasks():
tasks = []
for kind, label in KINDS:
rows = [json.loads(l) for l in open(os.path.join(WORK, "bfcl", f"data__BFCL_v4_{kind}.json")) if l.strip()]
gt = {}
if kind != "irrelevance":
for l in open(os.path.join(WORK, "bfcl", f"data__possible_answer__BFCL_v4_{kind}.json")):
if l.strip():
d = json.loads(l); gt[d["id"]] = d["ground_truth"]
for r in sorted(random.Random(f"{SEED}-{kind}").sample(rows, N_PER_KIND), key=lambda r: int(r["id"].rsplit("_", 1)[1])):
tasks.append(dict(id=r["id"], kind=kind, messages=r["question"][0], functions=r["function"], gt=gt.get(r["id"])))
return tasks
TOOL_BLOCK = re.compile(r"<tool_call>(.*?)(?:</tool_call>|$)", re.S)
FUNC_XML = re.compile(r"<function=([^>\s]+)\s*>(.*?)(?:</function>|$)", re.S)
PARAM_XML = re.compile(r"<parameter=([^>\s]+)\s*>\n?(.*?)\n?</parameter>", re.S)
def lenient_calls(raw, functions):
"""Forgiving reading of the model's raw text: any <function=...> block, closed or not, inside or outside <tool_call>.
Values are typed with the function's own schema (BFCL types). Not part of BFCL; used only for the 'lenient' column."""
raw = raw.split("</think>")[-1] # with thinking on, the reasoning precedes </think>
types_ = {re.sub(r"\.", "_", f["name"]): {k: v.get("type", "string") for k, v in f["parameters"]["properties"].items()}
for f in functions} # tool names were sent with dots turned into underscores
calls = []
for fname, body in FUNC_XML.findall(raw):
fname = fname.strip().strip('"')
args = {}
for p, v in PARAM_XML.findall(body):
t = types_.get(re.sub(r"\.", "_", fname), {}).get(p, "string")
v = v.strip() if t != "string" else v
try:
args[p] = (int(v) if t == "integer" else float(v) if t in ("float", "number") else
v.lower() == "true" if t == "boolean" and v.lower() in ("true", "false") else
json.loads(v) if t in ("array", "tuple", "dict", "object") else v)
except ValueError:
try:
args[p] = ast.literal_eval(v)
except Exception:
args[p] = v
calls.append({fname: args})
return calls
def score(task, calls):
"""BFCL's rule: irrelevance passes with no call; the other kinds go through BFCL's ast_checker."""
if task["kind"] == "irrelevance":
return CHECK.is_empty(calls), "made a call" if calls else None
if not calls:
return False, "no call"
try:
r = CHECK.ast_checker(task["functions"], calls, task["gt"], task["kind"])
except Exception as e: # malformed arguments the checker cannot walk
return False, f"checker_error:{type(e).__name__}"
return bool(r["valid"]), r.get("error_type")
def run_task(task, think=False, max_tokens=512):
body = {"messages": task["messages"], "tools": CHECK.to_tools(task["functions"]), "max_tokens": max_tokens,
"cache_prompt": False, "verbose": True, **GREEDY,
"chat_template_kwargs": {"reasoning_effort": "low"} if think else {"enable_thinking": False}}
log_pos = os.path.getsize(SERVER["log"])
t0 = time.time()
rec = dict(id=task["id"], kind=task["kind"], think=think)
try:
r = api("/v1/chat/completions", body)
msg = r["choices"][0]["message"]
rec.update(status=200, finish=r["choices"][0]["finish_reason"], raw=r["__verbose"]["content"],
prompt_n=r["timings"]["prompt_n"], out_n=r["timings"]["predicted_n"],
reasoning_chars=len(msg.get("reasoning_content") or ""))
rec["calls"] = [{tc["function"]["name"]: json.loads(tc["function"]["arguments"])} for tc in msg.get("tool_calls") or []]
except urllib.error.HTTPError as e: # stock llama.cpp answers 500 when its parser cannot read the reply
time.sleep(0.5)
with open(SERVER["log"], errors="replace") as f:
f.seek(log_pos)
new = f.read()
m = re.findall(r"=== BEGIN ===\n(.*?)\n=== END ===", new, re.S)
rec.update(status=e.code, finish="parse error", raw=m[-1] if m else "", calls=None, prompt_n=None, out_n=None)
rec["seconds"] = time.time() - t0
rec["strict"], rec["strict_why"] = score(task, rec["calls"]) if rec["calls"] is not None else (False, "unparsed (HTTP 500)")
if rec["strict"]:
rec["lenient"] = True
else:
rec["lenient"], _ = score(task, lenient_calls(rec["raw"], task["functions"]))
return rec
def bench(tag, tasks):
out, t0 = [], time.time()
for i, task in enumerate(tasks):
out.append(run_task(task))
if (i + 1) % 30 == 0:
print(f" {tag}: {i + 1}/{len(tasks)} done, {time.time() - t0:.0f} s")
json.dump(out, open(os.path.join(WORK, "runs", f"bfcl_{tag}.json"), "w"), indent=1)
return out
def ci(k, n, alpha=0.05):
"""Clopper-Pearson interval by bisection on the binomial tail."""
def cdf(x, p):
return sum(math.comb(n, i) * p ** i * (1 - p) ** (n - i) for i in range(x + 1))
def solve(f, target, rising):
lo, hi = 0.0, 1.0
for _ in range(60):
mid = (lo + hi) / 2
lo, hi = (mid, hi) if (f(mid) < target) == rising else (lo, mid)
return (lo + hi) / 2
lo = 0.0 if k == 0 else solve(lambda p: 1 - cdf(k - 1, p), alpha / 2, True)
hi = 1.0 if k == n else solve(lambda p: cdf(k, p), alpha / 2, False)
return lo, hi
def kind_table(tag, recs):
print(f"\n{tag}: {'kind':38s} {'strict':>8s} {'lenient':>8s} replies: structured / plain text / unparsed (500) / cut off")
for kind, label in KINDS:
rs = [r for r in recs if r["kind"] == kind]
st = collections.Counter("unparsed" if r["status"] != 200 else "cut off" if r["finish"] == "length"
else "structured" if r["calls"] else "plain text" for r in rs)
print(f" {label:44s} {sum(r['strict'] for r in rs):3d}/{len(rs):<4d} {sum(r['lenient'] for r in rs):3d}/{len(rs):<4d}"
f" {st['structured']:3d} / {st['plain text']:3d} / {st['unparsed']:3d} / {st['cut off']:3d}")
k, n = sum(r["strict"] for r in recs), len(recs)
lo, hi = ci(k, n)
print(f" {'total':44s} {k:3d}/{n:<4d} {sum(r['lenient'] for r in recs):3d}/{n:<4d} strict 95% CI {100 * lo:.0f}-{100 * hi:.0f}%")
@section("7. Tool calling on 120 BFCL v4 tasks, thinking off: Saluki")
def bench_saluki():
global CHECK, TASKS, RUNS
CHECK = load_bfcl_checker()
TASKS = load_tasks()
print(f"sample: {N_PER_KIND} tasks per kind, drawn with seed {SEED}, {len(TASKS)} in all; scored with BFCL's ast_checker "
f"(gorilla@6ea5797); greedy, thinking off, max_tokens 512")
print(f"first ids per kind: " + ", ".join(t["id"] for t in TASKS[::N_PER_KIND]))
RUNS = {"Saluki": bench("Saluki", TASKS)}
kind_table("Saluki", RUNS["Saluki"])
sec = [r["seconds"] for r in RUNS["Saluki"]]
print(f"\nmedian {sorted(sec)[len(sec) // 2]:.1f} s per task on {HW}; median output {sorted(r['out_n'] or 0 for r in RUNS['Saluki'])[len(sec) // 2]} tokens")
again = [run_task(t) for t in TASKS[:: len(TASKS) // 5][:5]]
same = sum(a["calls"] == b["calls"] and a["raw"] == b["raw"] for a, b in zip(again, RUNS["Saluki"][:: len(TASKS) // 5][:5]))
print(f"greedy re-run of 5 tasks: {same} of 5 identical raw replies")
claim("greedy decoding with the prompt cache off gives the same reply on a re-run (5 tasks)", same == 5)
s = sum(r["strict"] for r in RUNS["Saluki"])
return f"Saluki {s}/{len(TASKS)} strict, {sum(r['lenient'] for r in RUNS['Saluki'])}/{len(TASKS)} lenient"
bench_saluki()
مهام Underdog Bench الـ 120 غير منشورة، لذا نرسم مهامنا الخاصة: 24 لكل نوع من الفئات الخمس نفسها في BFCL v4، ببذرة ثابتة. نرسل كل مهمة إلى الخادم كطلب أدوات بصيغة OpenAI، محوّلاً بدالة BFCL الخاصة، مع إيقاف التفكير وفك ترميز جشع. يستخدم التقييم الصارم فقط الاستدعاءات التي يحللها llama.cpp وأداة ast_checker الخاصة بـ BFCL، بينما يعيد التقييم المتساهل قراءة الرد الخام بمحلل متسامح، وهو كيف نختبر الفجوة بين الصرامة والتسامح لدى المزوّد.
نسخ الكود@section("8. Does thinking help? Six multi-call tasks with reasoning on, effort low")
def thinking_step():
pick = [t for t in TASKS if t["kind"] in ("parallel", "parallel_multiple")]
pick = random.Random(SEED).sample(pick, min(6, len(pick)))
off = {r["id"]: r for r in RUNS["Saluki"]}
rows = []
for t in pick:
r = run_task(t, think=True, max_tokens=2048)
o = off[t["id"]]
rows.append((t["id"], o, r))
print(f" {t['id']:22s} off: {'pass' if o['strict'] else 'fail'} {o['out_n'] or 0:4d} tok {o['seconds']:5.1f} s | "
f"on: {'pass' if r['strict'] else 'fail'} {r['out_n'] or 0:4d} tok {r['seconds']:6.1f} s ({r['finish']})")
p_off, p_on = sum(o["strict"] for _, o, _ in rows), sum(r["strict"] for _, _, r in rows)
tok = sum(r["out_n"] or 0 for *_, r in rows) / max(1, sum(o["out_n"] or 0 for _, o, _ in rows))
sec = sum(r["seconds"] for *_, r in rows) / sum(o["seconds"] for _, o, _ in rows)
print(f"\nthinking off {p_off}/{len(rows)}, thinking on {p_on}/{len(rows)}; thinking used {tok:.1f}x the output tokens and {sec:.1f}x the time")
return f"thinking on (effort low): {p_on}/{len(rows)} vs {p_off}/{len(rows)} off, {tok:.1f}x tokens, {sec:.1f}x time"
thinking_step()
تشغيل التفكير هو نصيحة البطاقة للاستخدام العام، لكن عند 27B كل رمز استدلالي يكلف وقتًا. أعدنا تشغيل ست مهام متعددة الاستدعاءات مزروعة البذور مع تشغيل التفكير عند أدنى جهد، وقارنّاها بنفس المهام مع إيقاف التفكير: نجاح أو فشل، رموز الإخراج والثواني. ست مهام لا يمكنها حسم ما إذا كان التفكير يحسّن استدعاءات الأدوات، لذا تركز هذه الخلية على التكلفة وتُظهر كم رمزًا إضافيًا يشتريه استدعاء أداة واحد عندما يفكر النموذج أولًا.
Copy Code@section("9. The parent file: ISTA's IQ2_XS through the same server, the same sample and the same checker")
def bench_ista():
load_s = start_server("ISTA")
mem = memory_report()
SERVER_MEM["ISTA"] = mem
print(f"llama-server up in {load_s:.0f} s; " + ", ".join(f"{k} {v:.0f} MiB" for k, v in mem.items()))
print(f"weights: ISTA {mem['weights']:.0f} MiB vs Saluki {SERVER_MEM['Saluki']['weights']:.0f} MiB")
speed_probe("ISTA")
s, i = SPEED["Saluki"][1], SPEED["ISTA"][1]
print(f"decode at {s[0]} prompt tokens: Saluki {s[3]:.2f} vs ISTA {i[3]:.2f} tok/s; prefill {s[1]:.0f} vs {i[1]:.0f} tok/s")
RUNS["ISTA"] = bench("ISTA", TASKS)
kind_table("ISTA", RUNS["ISTA"])
stop_server()
k = sum(r["strict"] for r in RUNS["ISTA"])
return f"ISTA IQ2_XS {k}/{len(TASKS)} strict, {sum(r['lenient'] for r in RUNS['ISTA'])}/{len(TASKS)} lenient"
bench_ista()
يتبع الملف الأصل المسار نفسه. نوقف خادم Saluki، ونشغّل واحدًا على ملف ISTA بصيغة IQ2_XS بالأعلام نفسها، ونقرأ ذاكرته، ونعيد اختبار السرعة، ونسجّله على المهام الـ 120 نفسها بقواعد التدقيق والتحليل نفسها. تشغيل نموذج واحد في كل مرة يجعل كلًّا منهما ضمن ذاكرة الـ GPU بأمان، واستخدام حزمة اختبار واحدة لكليهما يزيل اختلافات المطالبات وأخذ العينات والتحليل من المقارنة.
Copy Codedef mcnemar(a, b):
"""Exact two-sided McNemar test on paired pass/fail lists."""
b10 = sum(x and not y for x, y in zip(a, b))
b01 = sum(y and not x for x, y in zip(a, b))
n, k = b10 + b01, min(b10, b01)
p = min(1.0, 2 * sum(math.comb(n, i) for i in range(k + 1)) / 2 ** n) if n else 1.0
return b10, b01, p
@section("10. Head to head: the same 120 tasks, paired, next to the vendor's numbers")
def head_to_head():
S, I = RUNS["Saluki"], RUNS["ISTA"]
assert [r["id"] for r in S] == [r["id"] for r in I]
vendor = {"simple_python": 24, "multiple": 23, "irrelevance": 18, "parallel": 12, "parallel_multiple": 11}
print(f"{'kind':38s} {'Saluki':>7s} {'ISTA':>6s} | {'lenient':>7s} {'':>5s} | vendor's Saluki (of 24)")
for kind, label in KINDS:
s = [r for r in S if r["kind"] == kind]; i = [r for r in I if r["kind"] == kind]
print(f"{label:38s} {sum(r['strict'] for r in s):7d} {sum(r['strict'] for r in i):6d} | "
f"{sum(r['lenient'] for r in s):7d} {sum(r['lenient'] for r in i):5d} | {vendor[kind]:3d}")
out = {}
for mode in ("strict", "lenient"):
a, b = [r[mode] for r in S], [r[mode] for r in I]
b10, b01, p = mcnemar(a, b)
out[mode] = (sum(a), sum(b), b10, b01, p)
print(f"{mode:8s}: Saluki {sum(a)}/120, ISTA {sum(b)}/120; Saluki-only passes {b10}, ISTA-only {b01}, exact McNemar p = {p:.3f}")
print("vendor's Underdog Bench (different 120 tasks, unpublished): Saluki 88, ISTA IQ2_XS 76, full Qwen3.8-27B 84")
sa, ia, b10, b01, p = out["strict"]
claim("Saluki passes more of our 120 tasks than its parent (strict scoring)", sa > ia)
claim("the difference is significant at 5% (exact McNemar)", p < 0.05)
pick_s = sum(r["strict"] for r in S if r["kind"] in ("simple_python", "multiple"))
pick_i = sum(r["strict"] for r in I if r["kind"] in ("simple_python", "multiple"))
claim("Saluki is at least as good as its parent at picking the right single function (first two kinds)", pick_s >= pick_i)
return f"strict: Saluki {sa}/120 vs ISTA {ia}/120 (McNemar p = {p:.2f}); lenient: {out['lenient'][0]} vs {out['lenient'][1]}"
head_to_head()
لأن كلا النموذجين أجاب على المهام نفسها، فإن المقارنة مقترنة. تطبع الخلية درجات كل نوع بجوار أرقام الشركة المصنّعة لكل نوع الخاصة بـ Saluki، وتجري اختبار McNemar الدقيق على المهام التي يختلف فيها الملفان، للتدقيق الصارم والمتساهل. فجوة بضع مهام من أصل 120 تقع ضمن الضجيج بين التشغيلات، كما تعترف بذلك صفحة الشركة نفسها، لذا يخبرنا الاختبار ما إذا كان عيّنتنا تُظهر فرقًا حقيقيًا.
Copy Code@section("11. Where the failures come from")
def failures():
for tag in ("Saluki", "ISTA"):
recs = RUNS[tag]
why = collections.Counter((r["strict_why"] or "?").split(":")[-1] for r in recs if not r["strict"])
print(f"{tag}: {sum(not r['strict'] for r in recs)} failures -> " + ", ".join(f"{k} {v}" for k, v in why.most_common()))
rescued = [r for r in recs if r["lenient"] and not r["strict"]]
print(f" lenient reading rescues {len(rescued)}: " + ", ".join(r["id"] for r in rescued[:8]))
ex = next((r for r in RUNS["Saluki"] if not r["strict"] and r["kind"] != "irrelevance"), None)
if ex:
task = next(t for t in TASKS if t["id"] == ex["id"])
print(f"\nexample, {ex['id']} ({ex['strict_why']}):")
print(" expected:", json.dumps(task["gt"])[:300])
print(" model :", (json.dumps(ex["calls"]) if ex["calls"] is not None else ex["raw"])[:300])
irr = [r for r in RUNS["Saluki"] if r["kind"] == "irrelevance" and not r["strict"]]
if irr:
print(f"\nirrelevance: Saluki made a call on {len(irr)} of {N_PER_KIND} prompts that no tool fits, e.g. {irr[0]['id']}: "
f"{json.dumps(irr[0]['calls'])[:160]}")
un = sum(r["status"] != 200 for tag in RUNS for r in RUNS[tag])
return f"{un} replies across both models were unparseable by the stock server"
failures()
الدرجات تخفي أين تخفق الاستدعاءات، لذا تفصّل هذه الخلية الإخفاقات حسب أنواع أخطاء BFCL نفسها: عدد خاطئ من الاستدعاءات، وسائط خاطئة أو مفقودة، واستدعاء أُجري عندما لا تناسبه أي أداة، وردود لم يتمكن الخادم الافتراضي من تحليلها. كما تسرد المهام التي ينقذها القراءة المتساهلة، وتطبع مهمة فاشلة واحدة بجوار إجابتها المتوقعة، وهي أسرع طريقة للتمييز بين خطأ النموذج وأثر خاطئ في التقييم.
Copy Codebanner("SUMMARY")
for name, res in RESULTS.items():
print(f" {name:<100s} {res}")
print("\nClaims decided in this run:")
for step, text, held in CLAIMS:
print(f" [{'held' if held else 'DID NOT HOLD' if held is False else 'undecided'}] step {step}: {text}")
failed = [k for k, v in RESULTS.items() if str(v).startswith("SKIPPED")]
print(f"\n{sum(h is True for *_, h in CLAIMS)} of {len(CLAIMS)} claims held; steps failed or skipped: {failed or 'none'}")
print(f"run on: {HW}; llama.cpp {LLAMA_TAG}; Python {platform.python_version()}")
print("not tested here: the full-size Qwen3.8-27B; Underdog Bench's own 120 tasks and the 100-task parallel set (unpublished); "
"SWE-bench, WebWalkerQA, IFEval and AIME; the vision add-on; Mia and Ternary Bonsai 2; thinking at the default effort.")
يطبع الملخص نتيجة سطر واحد لكل خطوة وسجل الادعاءات (متحقق، لم يتحقق، أو غير محسوم)، إلى جانب العتاد وإصدار llama.cpp version.cpp version. ويختم بما لا يختبره الدفتر: نموذج Qwen3.8-27B بالحجم الكامل، وقوائم مهام الشركة نفسها، ومعايير الوكلاء والرياضيات، وإضافة الرؤية.
وفي الختام، Saluki ليس تمامًا ما يقترحه ملفه. يحافظ ترويسته على تخطيط ISTا ذي الموترات الـ 851 ويوفّر البتات فقط حيث أنفقت ISTA أكثر، لكن بايتاته ومتجهاته غير المكمّاة تُظهر نموذجًا عُدّلت أوزانه، بما يتسق مع وصف Underdog له كنقطة فحص معاينة. ذلك يحوّل مقارنة استدعاء الأدوات مع ملفه الأصل إلى اختبار لتلك العملية التدريبية وليس للمكممة وحدها. كلا الملفين يعملان على llama.cpp الافتراضي، واحدًا تلو الآخر، ومدقّق BFCL نفسه يوفر مقارنة مقترنة على مهام متطابقة. العينات الصغيرة، وفك الترميز الجشع، والتشغيل الواحد تحدّ مما يمكن لأي دفتر أن يدّعيه، لذا فإن الفترات لا تقل أهمية عن الدرجات.
اطلع على الأكواد الكاملة هنا. كل الفضل للباحث في هذا المشروع. كما أنك مدعو لمتابعتنا على تويتر ولا تنسَ الانضمام إلى سابرديت الماكينات التعلمية الذي يضم أكثر من 150k+ والاشتراك في نشرتنا البريدية. مهلًا! هل أنت على تيليجرام؟ يمكنك الآن الانضمام إلينا على تيليجرام أيضًا.
هل تحتاج إلى الشراكة معنا للترويج لمستودع GitHub الخاص بك أو صفحة Hugging Face أو إطلاق منتج أو ندوة عبر الإنترنت وغيرها؟ تواصل معنا
ظهر المقال كيفية تشغيل Underdog Saluki 27B: دليل استدعاء الأدوات لنموذج Qwen3.8-27B بـ 2 بت أولًا على MarkTechPost.