Salvatore Sanfilippo
63b5dca93e
Collect references from Alibaba through OpenRouter with an explicit empty system message and no provider fallback. Validate token byte alignment and reject lossy Unicode metadata for API agreement while retaining continuation likelihood scoring. Track 100 short and 12 long reference cases, Q2/Q4 default and quality controls, continued-prefill results, and independent long-context answers. Include collector and scorer regressions and document the hosted-checkpoint and precision limitations.
84 lines
4.5 KiB
Python
84 lines
4.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Collector requests must record the distribution actually requested."""
|
|
import importlib.util
|
|
import io
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
spec = importlib.util.spec_from_file_location(
|
|
"collector", ROOT / "gguf-tools/quality-testing/collect_official.py")
|
|
collector = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(collector)
|
|
|
|
|
|
class CollectorTests(unittest.TestCase):
|
|
def test_temperature_and_record(self):
|
|
for temperature in (None, "1"):
|
|
with self.subTest(temperature=temperature), tempfile.TemporaryDirectory() as td:
|
|
prompt = Path(td) / "input.jsonl"
|
|
prompt.write_text('{"prompt": "Why?"}\n')
|
|
out = Path(td) / "output"
|
|
argv = ["collect", "--out", str(out), "--prompts", str(prompt),
|
|
"--count", "1", "--api-key-env", "TEST_API_KEY"]
|
|
if temperature:
|
|
argv += ["--temperature", temperature]
|
|
response = {"model": "test-model", "choices": [{
|
|
"message": {"content": "Because."},
|
|
"logprobs": {"content": [{"token": "Because", "logprob": -0.5}]}}]}
|
|
with patch("sys.argv", argv), patch.dict("os.environ", {"TEST_API_KEY": "test"}), \
|
|
patch.object(collector.urllib.request, "urlopen",
|
|
return_value=io.BytesIO(json.dumps(response).encode())) as request, \
|
|
patch.object(collector.time, "sleep"):
|
|
self.assertEqual(collector.main(), 0)
|
|
sent = json.loads(request.call_args.args[0].data)
|
|
recorded = json.loads((out / "collection.json").read_text())
|
|
expected = float(temperature or 0)
|
|
self.assertEqual(sent["temperature"], expected)
|
|
self.assertEqual(recorded["temperature"], expected)
|
|
self.assertTrue(sent["logprobs"])
|
|
self.assertEqual(json.loads((out / "responses/case_000.json").read_text()), response)
|
|
|
|
def test_system_message_and_resume(self):
|
|
for system in (None, "", "You are a helpful assistant."):
|
|
with self.subTest(system=system), tempfile.TemporaryDirectory() as td:
|
|
argv = ["collect", "--out", td, "--count", "1", "--api-key-env", "TEST_API_KEY", "--resume"]
|
|
if system is not None:
|
|
argv += ["--system", system]
|
|
response = {"choices": [{"message": {"content": "Answer."},
|
|
"logprobs": {"content": [{"token": "Answer", "logprob": -0.5}]}}]}
|
|
with patch("sys.argv", argv), patch.dict("os.environ", {"TEST_API_KEY": "test"}), \
|
|
patch.object(collector.urllib.request, "urlopen",
|
|
return_value=io.BytesIO(json.dumps(response).encode())) as request, \
|
|
patch.object(collector.time, "sleep"):
|
|
self.assertEqual(collector.main(), 0)
|
|
messages = json.loads(request.call_args.args[0].data)["messages"]
|
|
self.assertEqual([m["role"] for m in messages],
|
|
["user"] if system is None else ["system", "user"])
|
|
if system is not None:
|
|
self.assertEqual(messages[0]["content"], system)
|
|
self.assertEqual(json.loads((Path(td) / "collection.json").read_text())["system"], system)
|
|
request.reset_mock()
|
|
with patch("sys.argv", argv + ["--resume"]):
|
|
self.assertEqual(collector.main(), 0)
|
|
request.assert_not_called()
|
|
with patch("sys.argv", argv + ["--resume", "--system", "Different."]):
|
|
with self.assertRaisesRegex(RuntimeError, "saved system message differs"):
|
|
collector.main()
|
|
request.assert_not_called()
|
|
|
|
def test_invalid_temperature(self):
|
|
for value in ("-0.1", "2.1", "nan", "inf", "-inf"):
|
|
with self.subTest(value=value), patch("sys.argv", ["collect", "--temperature=" + value]), \
|
|
patch.object(collector.urllib.request, "urlopen") as request:
|
|
with self.assertRaisesRegex(SystemExit, "--temperature must"):
|
|
collector.main()
|
|
request.assert_not_called()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|