mtfyun/ds4
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
4f806387f71d3f0e8517650d5cd13b559003cdfd
ds4/gguf-tools/quality-testing/data/flash/responses/case_009.json
T

5567 lines
128 KiB
JSON
Raw Blame History

This file contains invisible Unicode characters
This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
{
"id": "c5e1eb1a-8cb8-44a4-b501-8421eb86714f",
"object": "chat.completion",
"created": 1785576283,
"model": "deepseek-v4-flash",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": "Here are three invariants for a tokenizer special-token table:\n\n## 1. **Uniqueness Invariant"
},
"logprobs": {
"content": [
{
"token": "Here",
"logprob": 0.0,
"bytes": [
72,
101,
114,
101
],
"top_logprobs": [
{
"token": "Here",
"logprob": 0.0,
"bytes": [
72,
101,
114,
101
]
},
{
"token": "For",
"logprob": -9999.0,
"bytes": [
70,
111,
114
]
},
{
"token": "I",
"logprob": -9999.0,
"bytes": [
73
]
},
{
"token": "A",
"logprob": -9999.0,
"bytes": [
65
]
},
{
"token": "Inv",
"logprob": -9999.0,
"bytes": [
73,
110,
118
]
},
{
"token": "An",
"logprob": -9999.0,
"bytes": [
65,
110
]
},
{
"token": " Here",
"logprob": -9999.0,
"bytes": [
32,
72,
101,
114,
101
]
},
{
"token": "Three",
"logprob": -9999.0,
"bytes": [
84,
104,
114,
101,
101
]
},
{
"token": "Writing",
"logprob": -9999.0,
"bytes": [
87,
114,
105,
116,
105,
110,
103
]
},
{
"token": "#",
"logprob": -9999.0,
"bytes": [
35
]
},
{
"token": "To",
"logprob": -9999.0,
"bytes": [
84,
111
]
},
{
"token": "The",
"logprob": -9999.0,
"bytes": [
84,
104,
101
]
},
{
"token": "When",
"logprob": -9999.0,
"bytes": [
87,
104,
101,
110
]
},
{
"token": "In",
"logprob": -9999.0,
"bytes": [
73,
110
]
},
{
"token": "##",
"logprob": -9999.0,
"bytes": [
35,
35
]
},
{
"token": "**",
"logprob": -9999.0,
"bytes": [
42,
42
]
},
{
"token": "Great",
"logprob": -9999.0,
"bytes": [
71,
114,
101,
97,
116
]
},
{
"token": "Okay",
"logprob": -9999.0,
"bytes": [
79,
107,
97,
121
]
},
{
"token": "<",
"logprob": -9999.0,
"bytes": [
60
]
},
{
"token": "Write",
"logprob": -9999.0,
"bytes": [
87,
114,
105,
116,
101
]
}
]
},
{
"token": " are",
"logprob": 0.0,
"bytes": [
32,
97,
114,
101
],
"top_logprobs": [
{
"token": " are",
"logprob": 0.0,
"bytes": [
32,
97,
114,
101
]
},
{
"token": "'s",
"logprob": -9999.0,
"bytes": [
39,
115
]
},
{
"token": " is",
"logprob": -9999.0,
"bytes": [
32,
105,
115
]
},
{
"token": "are",
"logprob": -9999.0,
"bytes": [
97,
114,
101
]
},
{
"token": ",",
"logprob": -9999.0,
"bytes": [
44
]
},
{
"token": "’",
"logprob": -9999.0,
"bytes": [
226,
128,
153
]
},
{
"token": " ar",
"logprob": -9999.0,
"bytes": [
32,
97,
114
]
},
{
"token": " were",
"logprob": -9999.0,
"bytes": [
32,
119,
101,
114,
101
]
},
{
"token": " three",
"logprob": -9999.0,
"bytes": [
32,
116,
104,
114,
101,
101
]
},
{
"token": " a",
"logprob": -9999.0,
"bytes": [
32,
97
]
},
{
"token": "<｜｜end▁of▁sentence｜｜>",
"logprob": -9999.0,
"bytes": [
60,
239,
189,
156,
239,
189,
156,
101,
110,
100,
226,
150,
129,
111,
102,
226,
150,
129,
115,
101,
110,
116,
101,
110,
99,
101,
239,
189,
156,
239,
189,
156,
62
]
},
{
"token": " ",
"logprob": -9999.0,
"bytes": [
32
]
},
{
"token": ".",
"logprob": -9999.0,
"bytes": [
46
]
},
{
"token": " sont",
"logprob": -9999.0,
"bytes": [
32,
115,
111,
110,
116
]
},
{
"token": " Are",
"logprob": -9999.0,
"bytes": [
32,
65,
114,
101
]
},
{
"token": " have",
"logprob": -9999.0,
"bytes": [
32,
104,
97,
118,
101
]
},
{
"token": "'re",
"logprob": -9999.0,
"bytes": [
39,
114,
101
]
},
{
"token": "'",
"logprob": -9999.0,
"bytes": [
39
]
},
{
"token": " be",
"logprob": -9999.0,
"bytes": [
32,
98,
101
]
},
{
"token": " and",
"logprob": -9999.0,
"bytes": [
32,
97,
110,
100
]
}
]
},
{
"token": " three",
"logprob": 0.0,
"bytes": [
32,
116,
104,
114,
101,
101
],
"top_logprobs": [
{
"token": " three",
"logprob": 0.0,
"bytes": [
32,
116,
104,
114,
101,
101
]
},
{
"token": " **",
"logprob": -9999.0,
"bytes": [
32,
42,
42
]
},
{
"token": " ",
"logprob": -9999.0,
"bytes": [
32
]
},
{
"token": " four",
"logprob": -9999.0,
"bytes": [
32,
102,
111,
117,
114
]
},
{
"token": " five",
"logprob": -9999.0,
"bytes": [
32,
102,
105,
118,
101
]
},
{
"token": " the",
"logprob": -9999.0,
"bytes": [
32,
116,
104,
101
]
},
{
"token": " six",
"logprob": -9999.0,
"bytes": [
32,
115,
105,
120
]
},
{
"token": " two",
"logprob": -9999.0,
"bytes": [
32,
116,
119,
111
]
},
{
"token": "three",
"logprob": -9999.0,
"bytes": [
116,
104,
114,
101,
101
]
},
{
"token": " invari",
"logprob": -9999.0,
"bytes": [
32,
105,
110,
118,
97,
114,
105
]
},
{
"token": " th",
"logprob": -9999.0,
"bytes": [
32,
116,
104
]
},
{
"token": " drei",
"logprob": -9999.0,
"bytes": [
32,
100,
114,
101,
105
]
},
{
"token": " eight",
"logprob": -9999.0,
"bytes": [
32,
101,
105,
103,
104,
116
]
},
{
"token": "**",
"logprob": -9999.0,
"bytes": [
42,
42
]
},
{
"token": " seven",
"logprob": -9999.0,
"bytes": [
32,
115,
101,
118,
101,
110
]
},
{
"token": " THREE",
"logprob": -9999.0,
"bytes": [
32,
84,
72,
82,
69,
69
]
},
{
"token": " Three",
"logprob": -9999.0,
"bytes": [
32,
84,
104,
114,
101,
101
]
},
{
"token": " a",
"logprob": -9999.0,
"bytes": [
32,
97
]
},
{
"token": " several",
"logprob": -9999.0,
"bytes": [
32,
115,
101,
118,
101,
114,
97,
108
]
},
{
"token": "\\xe2\\x80",
"logprob": -9999.0,
"bytes": [
226,
128
]
}
]
},
{
"token": " invari",
"logprob": 0.0,
"bytes": [
32,
105,
110,
118,
97,
114,
105
],
"top_logprobs": [
{
"token": " invari",
"logprob": 0.0,
"bytes": [
32,
105,
110,
118,
97,
114,
105
]
},
{
"token": " important",
"logprob": -9999.0,
"bytes": [
32,
105,
109,
112,
111,
114,
116,
97,
110,
116
]
},
{
"token": " key",
"logprob": -9999.0,
"bytes": [
32,
107,
101,
121
]
},
{
"token": " useful",
"logprob": -9999.0,
"bytes": [
32,
117,
115,
101,
102,
117,
108
]
},
{
"token": " meaningful",
"logprob": -9999.0,
"bytes": [
32,
109,
101,
97,
110,
105,
110,
103,
102,
117,
108
]
},
{
"token": " invariant",
"logprob": -9999.0,
"bytes": [
32,
105,
110,
118,
97,
114,
105,
97,
110,
116
]
},
{
"token": " practical",
"logprob": -9999.0,
"bytes": [
32,
112,
114,
97,
99,
116,
105,
99,
97,
108
]
},
{
"token": " essential",
"logprob": -9999.0,
"bytes": [
32,
101,
115,
115,
101,
110,
116,
105,
97,
108
]
},
{
"token": " formal",
"logprob": -9999.0,
"bytes": [
32,
102,
111,
114,
109,
97,
108
]
},
{
"token": " critical",
"logprob": -9999.0,
"bytes": [
32,
99,
114,
105,
116,
105,
99,
97,
108
]
},
{
"token": " robust",
"logprob": -9999.0,
"bytes": [
32,
114,
111,
98,
117,
115,
116
]
},
{
"token": " distinct",
"logprob": -9999.0,
"bytes": [
32,
100,
105,
115,
116,
105,
110,
99,
116
]
},
{
"token": " structural",
"logprob": -9999.0,
"bytes": [
32,
115,
116,
114,
117,
99,
116,
117,
114,
97,
108
]
},
{
"token": " inv",
"logprob": -9999.0,
"bytes": [
32,
105,
110,
118
]
},
{
"token": " crucial",
"logprob": -9999.0,
"bytes": [
32,
99,
114,
117,
99,
105,
97,
108
]
},
{
"token": " strong",
"logprob": -9999.0,
"bytes": [
32,
115,
116,
114,
111,
110,
103
]
},
{
"token": " fundamental",
"logprob": -9999.0,
"bytes": [
32,
102,
117,
110,
100,
97,
109,
101,
110,
116,
97,
108
]
},
{
"token": " mathematical",
"logprob": -9999.0,
"bytes": [
32,
109,
97,
116,
104,
101,
109,
97,
116,
105,
99,
97,
108
]
},
{
"token": " solid",
"logprob": -9999.0,
"bytes": [
32,
115,
111,
108,
105,
100
]
},
{
"token": " relevant",
"logprob": -9999.0,
"bytes": [
32,
114,
101,
108,
101,
118,
97,
110,
116
]
}
]
},
{
"token": "ants",
"logprob": 0.0,
"bytes": [
97,
110,
116,
115
],
"top_logprobs": [
{
"token": "ants",
"logprob": 0.0,
"bytes": [
97,
110,
116,
115
]
},
{
"token": "ents",
"logprob": -9999.0,
"bytes": [
101,
110,
116,
115
]
},
{
"token": "ari",
"logprob": -9999.0,
"bytes": [
97,
114,
105
]
},
{
"token": "ains",
"logprob": -9999.0,
"bytes": [
97,
105,
110,
115
]
},
{
"token": "ients",
"logprob": -9999.0,
"bytes": [
105,
101,
110,
116,
115
]
},
{
"token": "antes",
"logprob": -9999.0,
"bytes": [
97,
110,
116,
101,
115
]
},
{
"token": " ants",
"logprob": -9999.0,
"bytes": [
32,
97,
110,
116,
115
]
},
{
"token": "arts",
"logprob": -9999.0,
"bytes": [
97,
114,
116,
115
]
},
{
"token": "ante",
"logprob": -9999.0,
"bytes": [
97,
110,
116,
101
]
},
{
"token": "aints",
"logprob": -9999.0,
"bytes": [
97,
105,
110,
116,
115
]
},
{
"token": "aries",
"logprob": -9999.0,
"bytes": [
97,
114,
105,
101,
115
]
},
{
"token": "ats",
"logprob": -9999.0,
"bytes": [
97,
116,
115
]
},
{
"token": "orts",
"logprob": -9999.0,
"bytes": [
111,
114,
116,
115
]
},
{
"token": "ands",
"logprob": -9999.0,
"bytes": [
97,
110,
100,
115
]
},
{
"token": "raints",
"logprob": -9999.0,
"bytes": [
114,
97,
105,
110,
116,
115
]
},
{
"token": "ables",
"logprob": -9999.0,
"bytes": [
97,
98,
108,
101,
115
]
},
{
"token": "ets",
"logprob": -9999.0,
"bytes": [
101,
116,
115
]
},
{
"token": "ant",
"logprob": -9999.0,
"bytes": [
97,
110,
116
]
},
{
"token": "ations",
"logprob": -9999.0,
"bytes": [
97,
116,
105,
111,
110,
115
]
},
{
"token": "rants",
"logprob": -9999.0,
"bytes": [
114,
97,
110,
116,
115
]
}
]
},
{
"token": " for",
"logprob": 0.0,
"bytes": [
32,
102,
111,
114
],
"top_logprobs": [
{
"token": " for",
"logprob": 0.0,
"bytes": [
32,
102,
111,
114
]
},
{
"token": " that",
"logprob": -9999.0,
"bytes": [
32,
116,
104,
97,
116
]
},
{
"token": " (",
"logprob": -9999.0,
"bytes": [
32,
40
]
},
{
"token": ",",
"logprob": -9999.0,
"bytes": [
44
]
},
{
"token": " to",
"logprob": -9999.0,
"bytes": [
32,
116,
111
]
},
{
"token": " specifically",
"logprob": -9999.0,
"bytes": [
32,
115,
112,
101,
99,
105,
102,
105,
99,
97,
108,
108,
121
]
},
{
"token": " typically",
"logprob": -9999.0,
"bytes": [
32,
116,
121,
112,
105,
99,
97,
108,
108,
121
]
},
{
"token": " commonly",
"logprob": -9999.0,
"bytes": [
32,
99,
111,
109,
109,
111,
110,
108,
121
]
},
{
"token": " suitable",
"logprob": -9999.0,
"bytes": [
32,
115,
117,
105,
116,
97,
98,
108,
101
]
},
{
"token": " designed",
"logprob": -9999.0,
"bytes": [
32,
100,
101,
115,
105,
103,
110,
101,
100
]
},
{
"token": " and",
"logprob": -9999.0,
"bytes": [
32,
97,
110,
100
]
},
{
"token": " with",
"logprob": -9999.0,
"bytes": [
32,
119,
105,
116,
104
]
},
{
"token": " appropriate",
"logprob": -9999.0,
"bytes": [
32,
97,
112,
112,
114,
111,
112,
114,
105,
97,
116,
101
]
},
{
"token": " a",
"logprob": -9999.0,
"bytes": [
32,
97
]
},
{
"token": " I",
"logprob": -9999.0,
"bytes": [
32,
73
]
},
{
"token": " or",
"logprob": -9999.0,
"bytes": [
32,
111,
114
]
},
{
"token": "—",
"logprob": -9999.0,
"bytes": [
226,
128,
148
]
},
{
"token": " you",
"logprob": -9999.0,
"bytes": [
32,
121,
111,
117
]
},
{
"token": " essential",
"logprob": -9999.0,
"bytes": [
32,
101,
115,
115,
101,
110,
116,
105,
97,
108
]
},
{
"token": " governing",
"logprob": -9999.0,
"bytes": [
32,
103,
111,
118,
101,
114,
110,
105,
110,
103
]
}
]
},
{
"token": " a",
"logprob": 0.0,
"bytes": [
32,
97
],
"top_logprobs": [
{
"token": " a",
"logprob": 0.0,
"bytes": [
32,
97
]
},
{
"token": " an",
"logprob": -9999.0,
"bytes": [
32,
97,
110
]
},
{
"token": " maintaining",
"logprob": -9999.0,
"bytes": [
32,
109,
97,
105,
110,
116,
97,
105,
110,
105,
110,
103
]
},
{
"token": " the",
"logprob": -9999.0,
"bytes": [
32,
116,
104,
101
]
},
{
"token": " token",
"logprob": -9999.0,
"bytes": [
32,
116,
111,
107,
101,
110
]
},
{
"token": " special",
"logprob": -9999.0,
"bytes": [
32,
115,
112,
101,
99,
105,
97,
108
]
},
{
"token": " ensuring",
"logprob": -9999.0,
"bytes": [
32,
101,
110,
115,
117,
114,
105,
110,
103
]
},
{
"token": " evaluating",
"logprob": -9999.0,
"bytes": [
32,
101,
118,
97,
108,
117,
97,
116,
105,
110,
103
]
},
{
"token": " managing",
"logprob": -9999.0,
"bytes": [
32,
109,
97,
110,
97,
103,
105,
110,
103
]
},
{
"token": " that",
"logprob": -9999.0,
"bytes": [
32,
116,
104,
97,
116
]
},
{
"token": " any",
"logprob": -9999.0,
"bytes": [
32,
97,
110,
121
]
},
{
"token": " implementing",
"logprob": -9999.0,
"bytes": [
32,
105,
109,
112,
108,
101,
109,
101,
110,
116,
105,
110,
103
]
},
{
"token": " maintain",
"logprob": -9999.0,
"bytes": [
32,
109,
97,
105,
110,
116,
97,
105,
110
]
},
{
"token": " constructing",
"logprob": -9999.0,
"bytes": [
32,
99,
111,
110,
115,
116,
114,
117,
99,
116,
105,
110,
103
]
},
{
"token": " how",
"logprob": -9999.0,
"bytes": [
32,
104,
111,
119
]
},
{
"token": " (",
"logprob": -9999.0,
"bytes": [
32,
40
]
},
{
"token": " analyzing",
"logprob": -9999.0,
"bytes": [
32,
97,
110,
97,
108,
121,
122,
105,
110,
103
]
},
{
"token": "a",
"logprob": -9999.0,
"bytes": [
97
]
},
{
"token": " designing",
"logprob": -9999.0,
"bytes": [
32,
100,
101,
115,
105,
103,
110,
105,
110,
103
]
},
{
"token": " validating",
"logprob": -9999.0,
"bytes": [
32,
118,
97,
108,
105,
100,
97,
116,
105,
110,
103
]
}
]
},
{
"token": " token",
"logprob": 0.0,
"bytes": [
32,
116,
111,
107,
101,
110
],
"top_logprobs": [
{
"token": " token",
"logprob": 0.0,
"bytes": [
32,
116,
111,
107,
101,
110
]
},
{
"token": " special",
"logprob": -9999.0,
"bytes": [
32,
115,
112,
101,
99,
105,
97,
108
]
},
{
"token": " specialized",
"logprob": -9999.0,
"bytes": [
32,
115,
112,
101,
99,
105,
97,
108,
105,
122,
101,
100
]
},
{
"token": " **",
"logprob": -9999.0,
"bytes": [
32,
42,
42
]
},
{
"token": " character",
"logprob": -9999.0,
"bytes": [
32,
99,
104,
97,
114,
97,
99,
116,
101,
114
]
},
{
"token": " specialization",
"logprob": -9999.0,
"bytes": [
32,
115,
112,
101,
99,
105,
97,
108,
105,
122,
97,
116,
105,
111,
110
]
},
{
"token": " `",
"logprob": -9999.0,
"bytes": [
32,
96
]
},
{
"token": " lexical",
"logprob": -9999.0,
"bytes": [
32,
108,
101,
120,
105,
99,
97,
108
]
},
{
"token": " typical",
"logprob": -9999.0,
"bytes": [
32,
116,
121,
112,
105,
99,
97,
108
]
},
{
"token": " compiler",
"logprob": -9999.0,
"bytes": [
32,
99,
111,
109,
112,
105,
108,
101,
114
]
},
{
"token": " tokens",
"logprob": -9999.0,
"bytes": [
32,
116,
111,
107,
101,
110,
115
]
},
{
"token": " specially",
"logprob": -9999.0,
"bytes": [
32,
115,
112,
101,
99,
105,
97,
108,
108,
121
]
},
{
"token": " valid",
"logprob": -9999.0,
"bytes": [
32,
118,
97,
108,
105,
100
]
},
{
"token": " spec",
"logprob": -9999.0,
"bytes": [
32,
115,
112,
101,
99
]
},
{
"token": " proper",
"logprob": -9999.0,
"bytes": [
32,
112,
114,
111,
112,
101,
114
]
},
{
"token": " tok",
"logprob": -9999.0,
"bytes": [
32,
116,
111,
107
]
},
{
"token": " parser",
"logprob": -9999.0,
"bytes": [
32,
112,
97,
114,
115,
101,
114
]
},
{
"token": " Token",
"logprob": -9999.0,
"bytes": [
32,
84,
111,
107,
101,
110
]
},
{
"token": " specialty",
"logprob": -9999.0,
"bytes": [
32,
115,
112,
101,
99,
105,
97,
108,
116,
121
]
},
{
"token": " standard",
"logprob": -9999.0,
"bytes": [
32,
115,
116,
97,
110,
100,
97,
114,
100
]
}
]
},
{
"token": "izer",
"logprob": 0.0,
"bytes": [
105,
122,
101,
114
],
"top_logprobs": [
{
"token": "izer",
"logprob": 0.0,
"bytes": [
105,
122,
101,
114
]
},
{
"token": "zier",
"logprob": -9999.0,
"bytes": [
122,
105,
101,
114
]
},
{
"token": "ization",
"logprob": -9999.0,
"bytes": [
105,
122,
97,
116,
105,
111,
110
]
},
{
"token": "izers",
"logprob": -9999.0,
"bytes": [
105,
122,
101,
114,
115
]
},
{
"token": "zer",
"logprob": -9999.0,
"bytes": [
122,
101,
114
]
},
{
"token": "iser",
"logprob": -9999.0,
"bytes": [
105,
115,
101,
114
]
},
{
"token": "ized",
"logprob": -9999.0,
"bytes": [
105,
122,
101,
100
]
},
{
"token": " special",
"logprob": -9999.0,
"bytes": [
32,
115,
112,
101,
99,
105,
97,
108
]
},
{
"token": "ize",
"logprob": -9999.0,
"bytes": [
105,
122,
101
]
},
{
"token": "iz",
"logprob": -9999.0,
"bytes": [
105,
122
]
},
{
"token": "'s",
"logprob": -9999.0,
"bytes": [
39,
115
]
},
{
"token": "ier",
"logprob": -9999.0,
"bytes": [
105,
101,
114
]
},
{
"token": "er",
"logprob": -9999.0,
"bytes": [
101,
114
]
},
{
"token": "-based",
"logprob": -9999.0,
"bytes": [
45,
98,
97,
115,
101,
100
]
},
{
"token": " table",
"logprob": -9999.0,
"bytes": [
32,
116,
97,
98,
108,
101
]
},
{
"token": "‑",
"logprob": -9999.0,
"bytes": [
226,
128,
145
]
},
{
"token": " generator",
"logprob": -9999.0,
"bytes": [
32,
103,
101,
110,
101,
114,
97,
116,
111,
114
]
},
{
"token": " token",
"logprob": -9999.0,
"bytes": [
32,
116,
111,
107,
101,
110
]
},
{
"token": "izing",
"logprob": -9999.0,
"bytes": [
105,
122,
105,
110,
103
]
},
{
"token": " **",
"logprob": -9999.0,
"bytes": [
32,
42,
42
]
}
]
},
{
"token": " special",
"logprob": 0.0,
"bytes": [
32,
115,
112,
101,
99,
105,
97,
108
],
"top_logprobs": [
{
"token": " special",
"logprob": 0.0,
"bytes": [
32,
115,
112,
101,
99,
105,
97,
108
]
},
{
"token": "'s",
"logprob": -9999.0,
"bytes": [
39,
115
]
},
{
"token": "’",
"logprob": -9999.0,
"bytes": [
226,
128,
153
]
},
{
"token": " **",
"logprob": -9999.0,
"bytes": [
32,
42,
42
]
},
{
"token": " specialized",
"logprob": -9999.0,
"bytes": [
32,
115,
112,
101,
99,
105,
97,
108,
105,
122,
101,
100
]
},
{
"token": "\\'",
"logprob": -9999.0,
"bytes": [
92,
39
]
},
{
"token": " spec",
"logprob": -9999.0,
"bytes": [
32,
115,
112,
101,
99
]
},
{
"token": "'",
"logprob": -9999.0,
"bytes": [
39
]
},
{
"token": "\\xe2\\x80",
"logprob": -9999.0,
"bytes": [
226,
128
]
},
{
"token": " token",
"logprob": -9999.0,
"bytes": [
32,
116,
111,
107,
101,
110
]
},
{
"token": " `",
"logprob": -9999.0,
"bytes": [
32,
96
]
},
{
"token": " specially",
"logprob": -9999.0,
"bytes": [
32,
115,
112,
101,
99,
105,
97,
108,
108,
121
]
},
{
"token": "**",
"logprob": -9999.0,
"bytes": [
42,
42
]
},
{
"token": " (",
"logprob": -9999.0,
"bytes": [
32,
40
]
},
{
"token": "-specific",
"logprob": -9999.0,
"bytes": [
45,
115,
112,
101,
99,
105,
102,
105,
99
]
},
{
"token": "&#",
"logprob": -9999.0,
"bytes": [
38,
35
]
},
{
"token": " specialty",
"logprob": -9999.0,
"bytes": [
32,
115,
112,
101,
99,
105,
97,
108,
116,
121
]
},
{
"token": " that",
"logprob": -9999.0,
"bytes": [
32,
116,
104,
97,
116
]
},
{
"token": "-s",
"logprob": -9999.0,
"bytes": [
45,
115
]
},
{
"token": "ʼ",
"logprob": -9999.0,
"bytes": [
202,
188
]
}
]
},
{
"token": "-t",
"logprob": 0.0,
"bytes": [
45,
116
],
"top_logprobs": [
{
"token": "-t",
"logprob": 0.0,
"bytes": [
45,
116
]
},
{
"token": "‑",
"logprob": -9999.0,
"bytes": [
226,
128,
145
]
},
{
"token": "\\-",
"logprob": -9999.0,
"bytes": [
92,
45
]
},
{
"token": " token",
"logprob": -9999.0,
"bytes": [
32,
116,
111,
107,
101,
110
]
},
{
"token": "-",
"logprob": -9999.0,
"bytes": [
45
]
},
{
"token": "–",
"logprob": -9999.0,
"bytes": [
226,
128,
147
]
},
{
"token": "‐",
"logprob": -9999.0,
"bytes": [
226,
128,
144
]
},
{
"token": "\\xe2\\x80",
"logprob": -9999.0,
"bytes": [
226,
128
]
},
{
"token": "-table",
"logprob": -9999.0,
"bytes": [
45,
116,
97,
98,
108,
101
]
},
{
"token": "­",
"logprob": -9999.0,
"bytes": [
194,
173
]
},
{
"token": "-to",
"logprob": -9999.0,
"bytes": [
45,
116,
111
]
},
{
"token": "-s",
"logprob": -9999.0,
"bytes": [
45,
115
]
},
{
"token": "token",
"logprob": -9999.0,
"bytes": [
116,
111,
107,
101,
110
]
},
{
"token": "_token",
"logprob": -9999.0,
"bytes": [
95,
116,
111,
107,
101,
110
]
},
{
"token": "-task",
"logprob": -9999.0,
"bytes": [
45,
116,
97,
115,
107
]
},
{
"token": "−",
"logprob": -9999.0,
"bytes": [
226,
136,
146
]
},
{
"token": "-ton",
"logprob": -9999.0,
"bytes": [
45,
116,
111,
110
]
},
{
"token": " -",
"logprob": -9999.0,
"bytes": [
32,
45
]
},
{
"token": "-character",
"logprob": -9999.0,
"bytes": [
45,
99,
104,
97,
114,
97,
99,
116,
101,
114
]
},
{
"token": "—",
"logprob": -9999.0,
"bytes": [
226,
128,
148
]
}
]
},
{
"token": "oken",
"logprob": 0.0,
"bytes": [
111,
107,
101,
110
],
"top_logprobs": [
{
"token": "oken",
"logprob": 0.0,
"bytes": [
111,
107,
101,
110
]
},
{
"token": "okens",
"logprob": -9999.0,
"bytes": [
111,
107,
101,
110,
115
]
},
{
"token": "otal",
"logprob": -9999.0,
"bytes": [
111,
116,
97,
108
]
},
{
"token": " token",
"logprob": -9999.0,
"bytes": [
32,
116,
111,
107,
101,
110
]
},
{
"token": "okan",
"logprob": -9999.0,
"bytes": [
111,
107,
97,
110
]
},
{
"token": "ok",
"logprob": -9999.0,
"bytes": [
111,
107
]
},
{
"token": "ool",
"logprob": -9999.0,
"bytes": [
111,
111,
108
]
},
{
"token": "aken",
"logprob": -9999.0,
"bytes": [
97,
107,
101,
110
]
},
{
"token": "oker",
"logprob": -9999.0,
"bytes": [
111,
107,
101,
114
]
},
{
"token": "Token",
"logprob": -9999.0,
"bytes": [
84,
111,
107,
101,
110
]
},
{
"token": "oke",
"logprob": -9999.0,
"bytes": [
111,
107,
101
]
},
{
"token": "oten",
"logprob": -9999.0,
"bytes": [
111,
116,
101,
110
]
},
{
"token": "ocken",
"logprob": -9999.0,
"bytes": [
111,
99,
107,
101,
110
]
},
{
"token": "token",
"logprob": -9999.0,
"bytes": [
116,
111,
107,
101,
110
]
},
{
"token": "icket",
"logprob": -9999.0,
"bytes": [
105,
99,
107,
101,
116
]
},
{
"token": "opic",
"logprob": -9999.0,
"bytes": [
111,
112,
105,
99
]
},
{
"token": "ank",
"logprob": -9999.0,
"bytes": [
97,
110,
107
]
},
{
"token": "ken",
"logprob": -9999.0,
"bytes": [
107,
101,
110
]
},
{
"token": "ot",
"logprob": -9999.0,
"bytes": [
111,
116
]
},
{
"token": "ock",
"logprob": -9999.0,
"bytes": [
111,
99,
107
]
}
]
},
{
"token": " table",
"logprob": 0.0,
"bytes": [
32,
116,
97,
98,
108,
101
],
"top_logprobs": [
{
"token": " table",
"logprob": 0.0,
"bytes": [
32,
116,
97,
98,
108,
101
]
},
{
"token": " token",
"logprob": -9999.0,
"bytes": [
32,
116,
111,
107,
101,
110
]
},
{
"token": " (",
"logprob": -9999.0,
"bytes": [
32,
40
]
},
{
"token": " **",
"logprob": -9999.0,
"bytes": [
32,
42,
42
]
},
{
"token": " dictionary",
"logprob": -9999.0,
"bytes": [
32,
100,
105,
99,
116,
105,
111,
110,
97,
114,
121
]
},
{
"token": " lookup",
"logprob": -9999.0,
"bytes": [
32,
108,
111,
111,
107,
117,
112
]
},
{
"token": " vocabulary",
"logprob": -9999.0,
"bytes": [
32,
118,
111,
99,
97,
98,
117,
108,
97,
114,
121
]
},
{
"token": "-table",
"logprob": -9999.0,
"bytes": [
45,
116,
97,
98,
108,
101
]
},
{
"token": " list",
"logprob": -9999.0,
"bytes": [
32,
108,
105,
115,
116
]
},
{
"token": " collection",
"logprob": -9999.0,
"bytes": [
32,
99,
111,
108,
108,
101,
99,
116,
105,
111,
110
]
},
{
"token": " mapping",
"logprob": -9999.0,
"bytes": [
32,
109,
97,
112,
112,
105,
110,
103
]
},
{
"token": " tables",
"logprob": -9999.0,
"bytes": [
32,
116,
97,
98,
108,
101,
115
]
},
{
"token": " registry",
"logprob": -9999.0,
"bytes": [
32,
114,
101,
103,
105,
115,
116,
114,
121
]
},
{
"token": " Table",
"logprob": -9999.0,
"bytes": [
32,
84,
97,
98,
108,
101
]
},
{
"token": " database",
"logprob": -9999.0,
"bytes": [
32,
100,
97,
116,
97,
98,
97,
115,
101
]
},
{
"token": " storage",
"logprob": -9999.0,
"bytes": [
32,
115,
116,
111,
114,
97,
103,
101
]
},
{
"token": " TABLE",
"logprob": -9999.0,
"bytes": [
32,
84,
65,
66,
76,
69
]
},
{
"token": " system",
"logprob": -9999.0,
"bytes": [
32,
115,
121,
115,
116,
101,
109
]
},
{
"token": " tablet",
"logprob": -9999.0,
"bytes": [
32,
116,
97,
98,
108,
101,
116
]
},
{
"token": "\\xe2\\x80",
"logprob": -9999.0,
"bytes": [
226,
128
]
}
]
},
{
"token": ":\n\n",
"logprob": 0.0,
"bytes": [
58,
10,
10
],
"top_logprobs": [
{
"token": ":\n\n",
"logprob": 0.0,
"bytes": [
58,
10,
10
]
},
{
"token": ",",
"logprob": -9999.0,
"bytes": [
44
]
},
{
"token": " (",
"logprob": -9999.0,
"bytes": [
32,
40
]
},
{
"token": " that",
"logprob": -9999.0,
"bytes": [
32,
116,
104,
97,
116
]
},
{
"token": ".",
"logprob": -9999.0,
"bytes": [
46
]
},
{
"token": " along",
"logprob": -9999.0,
"bytes": [
32,
97,
108,
111,
110,
103
]
},
{
"token": " with",
"logprob": -9999.0,
"bytes": [
32,
119,
105,
116,
104
]
},
{
"token": ":",
"logprob": -9999.0,
"bytes": [
58
]
},
{
"token": " in",
"logprob": -9999.0,
"bytes": [
32,
105,
110
]
},
{
"token": " designed",
"logprob": -9999.0,
"bytes": [
32,
100,
101,
115,
105,
103,
110,
101,
100
]
},
{
"token": " based",
"logprob": -9999.0,
"bytes": [
32,
98,
97,
115,
101,
100
]
},
{
"token": " system",
"logprob": -9999.0,
"bytes": [
32,
115,
121,
115,
116,
101,
109
]
},
{
"token": " —",
"logprob": -9999.0,
"bytes": [
32,
226,
128,
148
]
},
{
"token": ".\n\n",
"logprob": -9999.0,
"bytes": [
46,
10,
10
]
},
{
"token": " (`",
"logprob": -9999.0,
"bytes": [
32,
40,
96
]
},
{
"token": " design",
"logprob": -9999.0,
"bytes": [
32,
100,
101,
115,
105,
103,
110
]
},
{
"token": "—",
"logprob": -9999.0,
"bytes": [
226,
128,
148
]
},
{
"token": " to",
"logprob": -9999.0,
"bytes": [
32,
116,
111
]
},
{
"token": "：\n\n",
"logprob": -9999.0,
"bytes": [
239,
188,
154,
10,
10
]
},
{
"token": " written",
"logprob": -9999.0,
"bytes": [
32,
119,
114,
105,
116,
116,
101,
110
]
}
]
},
{
"token": "##",
"logprob": 0.0,
"bytes": [
35,
35
],
"top_logprobs": [
{
"token": "##",
"logprob": 0.0,
"bytes": [
35,
35
]
},
{
"token": "**",
"logprob": -9999.0,
"bytes": [
42,
42
]
},
{
"token": "1",
"logprob": -9999.0,
"bytes": [
49
]
},
{
"token": "---\n\n",
"logprob": -9999.0,
"bytes": [
45,
45,
45,
10,
10
]
},
{
"token": "```",
"logprob": -9999.0,
"bytes": [
96,
96,
96
]
},
{
"token": "###",
"logprob": -9999.0,
"bytes": [
35,
35,
35
]
},
{
"token": "```\n",
"logprob": -9999.0,
"bytes": [
96,
96,
96,
10
]
},
{
"token": "#",
"logprob": -9999.0,
"bytes": [
35
]
},
{
"token": "**(",
"logprob": -9999.0,
"bytes": [
42,
42,
40
]
},
{
"token": "---\n",
"logprob": -9999.0,
"bytes": [
45,
45,
45,
10
]
},
{
"token": "<!--",
"logprob": -9999.0,
"bytes": [
60,
33,
45,
45
]
},
{
"token": "|",
"logprob": -9999.0,
"bytes": [
124
]
},
{
"token": ">",
"logprob": -9999.0,
"bytes": [
62
]
},
{
"token": "Inv",
"logprob": -9999.0,
"bytes": [
73,
110,
118
]
},
{
"token": "####",
"logprob": -9999.0,
"bytes": [
35,
35,
35,
35
]
},
{
"token": "```\n\n",
"logprob": -9999.0,
"bytes": [
96,
96,
96,
10,
10
]
},
{
"token": "``",
"logprob": -9999.0,
"bytes": [
96,
96
]
},
{
"token": " **",
"logprob": -9999.0,
"bytes": [
32,
42,
42
]
},
{
"token": "-",
"logprob": -9999.0,
"bytes": [
45
]
},
{
"token": "\\[",
"logprob": -9999.0,
"bytes": [
92,
91
]
}
]
},
{
"token": " ",
"logprob": 0.0,
"bytes": [
32
],
"top_logprobs": [
{
"token": " ",
"logprob": 0.0,
"bytes": [
32
]
},
{
"token": " Inv",
"logprob": -9999.0,
"bytes": [
32,
73,
110,
118
]
},
{
"token": " **",
"logprob": -9999.0,
"bytes": [
32,
42,
42
]
},
{
"token": " In",
"logprob": -9999.0,
"bytes": [
32,
73,
110
]
},
{
"token": " invariant",
"logprob": -9999.0,
"bytes": [
32,
105,
110,
118,
97,
114,
105,
97,
110,
116
]
},
{
"token": " I",
"logprob": -9999.0,
"bytes": [
32,
73
]
},
{
"token": " Key",
"logprob": -9999.0,
"bytes": [
32,
75,
101,
121
]
},
{
"token": " INV",
"logprob": -9999.0,
"bytes": [
32,
73,
78,
86
]
},
{
"token": " Table",
"logprob": -9999.0,
"bytes": [
32,
84,
97,
98,
108,
101
]
},
{
"token": " Definition",
"logprob": -9999.0,
"bytes": [
32,
68,
101,
102,
105,
110,
105,
116,
105,
111,
110
]
},
{
"token": " Core",
"logprob": -9999.0,
"bytes": [
32,
67,
111,
114,
101
]
},
{
"token": " The",
"logprob": -9999.0,
"bytes": [
32,
84,
104,
101
]
},
{
"token": " Type",
"logprob": -9999.0,
"bytes": [
32,
84,
121,
112,
101
]
},
{
"token": " Important",
"logprob": -9999.0,
"bytes": [
32,
73,
109,
112,
111,
114,
116,
97,
110,
116
]
},
{
"token": " First",
"logprob": -9999.0,
"bytes": [
32,
70,
105,
114,
115,
116
]
},
{
"token": " Token",
"logprob": -9999.0,
"bytes": [
32,
84,
111,
107,
101,
110
]
},
{
"token": " (",
"logprob": -9999.0,
"bytes": [
32,
40
]
},
{
"token": " Context",
"logprob": -9999.0,
"bytes": [
32,
67,
111,
110,
116,
101,
120,
116
]
},
{
"token": " Index",
"logprob": -9999.0,
"bytes": [
32,
73,
110,
100,
101,
120
]
},
{
"token": " Design",
"logprob": -9999.0,
"bytes": [
32,
68,
101,
115,
105,
103,
110
]
}
]
},
{
"token": "1",
"logprob": 0.0,
"bytes": [
49
],
"top_logprobs": [
{
"token": "1",
"logprob": 0.0,
"bytes": [
49
]
},
{
"token": "3",
"logprob": -9999.0,
"bytes": [
51
]
},
{
"token": "0",
"logprob": -9999.0,
"bytes": [
48
]
},
{
"token": "①",
"logprob": -9999.0,
"bytes": [
226,
145,
160
]
},
{
"token": "2",
"logprob": -9999.0,
"bytes": [
50
]
},
{
"token": "4",
"logprob": -9999.0,
"bytes": [
52
]
},
{
"token": " Inv",
"logprob": -9999.0,
"bytes": [
32,
73,
110,
118
]
},
{
"token": "۱",
"logprob": -9999.0,
"bytes": [
219,
177
]
},
{
"token": "01",
"logprob": -9999.0,
"bytes": [
48,
49
]
},
{
"token": "5",
"logprob": -9999.0,
"bytes": [
53
]
},
{
"token": "11",
"logprob": -9999.0,
"bytes": [
49,
49
]
},
{
"token": "10",
"logprob": -9999.0,
"bytes": [
49,
48
]
},
{
"token": "Inv",
"logprob": -9999.0,
"bytes": [
73,
110,
118
]
},
{
"token": "12",
"logprob": -9999.0,
"bytes": [
49,
50
]
},
{
"token": "8",
"logprob": -9999.0,
"bytes": [
56
]
},
{
"token": "7",
"logprob": -9999.0,
"bytes": [
55
]
},
{
"token": "Ⅰ",
"logprob": -9999.0,
"bytes": [
226,
133,
160
]
},
{
"token": "100",
"logprob": -9999.0,
"bytes": [
49,
48,
48
]
},
{
"token": "<｜｜end▁of▁sentence｜｜>",
"logprob": -9999.0,
"bytes": [
60,
239,
189,
156,
239,
189,
156,
101,
110,
100,
226,
150,
129,
111,
102,
226,
150,
129,
115,
101,
110,
116,
101,
110,
99,
101,
239,
189,
156,
239,
189,
156,
62
]
},
{
"token": "１",
"logprob": -9999.0,
"bytes": [
239,
188,
145
]
}
]
},
{
"token": ".",
"logprob": 0.0,
"bytes": [
46
],
"top_logprobs": [
{
"token": ".",
"logprob": 0.0,
"bytes": [
46
]
},
{
"token": ".**",
"logprob": -9999.0,
"bytes": [
46,
42,
42
]
},
{
"token": ")",
"logprob": -9999.0,
"bytes": [
41
]
},
{
"token": "\\.",
"logprob": -9999.0,
"bytes": [
92,
46
]
},
{
"token": "️⃣",
"logprob": -9999.0,
"bytes": [
239,
184,
143,
226,
131,
163
]
},
{
"token": "️",
"logprob": -9999.0,
"bytes": [
239,
184,
143
]
},
{
"token": "st",
"logprob": -9999.0,
"bytes": [
115,
116
]
},
{
"token": ".)",
"logprob": -9999.0,
"bytes": [
46,
41
]
},
{
"token": "<｜｜end▁of▁sentence｜｜>",
"logprob": -9999.0,
"bytes": [
60,
239,
189,
156,
239,
189,
156,
101,
110,
100,
226,
150,
129,
111,
102,
226,
150,
129,
115,
101,
110,
116,
101,
110,
99,
101,
239,
189,
156,
239,
189,
156,
62
]
},
{
"token": " **",
"logprob": -9999.0,
"bytes": [
32,
42,
42
]
},
{
"token": ").",
"logprob": -9999.0,
"bytes": [
41,
46
]
},
{
"token": "．",
"logprob": -9999.0,
"bytes": [
239,
188,
142
]
},
{
"token": ".Un",
"logprob": -9999.0,
"bytes": [
46,
85,
110
]
},
{
"token": "..",
"logprob": -9999.0,
"bytes": [
46,
46
]
},
{
"token": ",",
"logprob": -9999.0,
"bytes": [
44
]
},
{
"token": "**",
"logprob": -9999.0,
"bytes": [
42,
42
]
},
{
"token": ":",
"logprob": -9999.0,
"bytes": [
58
]
},
{
"token": "a",
"logprob": -9999.0,
"bytes": [
97
]
},
{
"token": "]",
"logprob": -9999.0,
"bytes": [
93
]
},
{
"token": ".\n\n",
"logprob": -9999.0,
"bytes": [
46,
10,
10
]
}
]
},
{
"token": " **",
"logprob": 0.0,
"bytes": [
32,
42,
42
],
"top_logprobs": [
{
"token": " **",
"logprob": 0.0,
"bytes": [
32,
42,
42
]
},
{
"token": " Un",
"logprob": -9999.0,
"bytes": [
32,
85,
110
]
},
{
"token": " Unique",
"logprob": -9999.0,
"bytes": [
32,
85,
110,
105,
113,
117,
101
]
},
{
"token": " Key",
"logprob": -9999.0,
"bytes": [
32,
75,
101,
121
]
},
{
"token": " Token",
"logprob": -9999.0,
"bytes": [
32,
84,
111,
107,
101,
110
]
},
{
"token": " Comple",
"logprob": -9999.0,
"bytes": [
32,
67,
111,
109,
112,
108,
101
]
},
{
"token": " Prefix",
"logprob": -9999.0,
"bytes": [
32,
80,
114,
101,
102,
105,
120
]
},
{
"token": " No",
"logprob": -9999.0,
"bytes": [
32,
78,
111
]
},
{
"token": " Content",
"logprob": -9999.0,
"bytes": [
32,
67,
111,
110,
116,
101,
110,
116
]
},
{
"token": " The",
"logprob": -9999.0,
"bytes": [
32,
84,
104,
101
]
},
{
"token": " Complete",
"logprob": -9999.0,
"bytes": [
32,
67,
111,
109,
112,
108,
101,
116,
101
]
},
{
"token": " Non",
"logprob": -9999.0,
"bytes": [
32,
78,
111,
110
]
},
{
"token": " Bi",
"logprob": -9999.0,
"bytes": [
32,
66,
105
]
},
{
"token": " Inject",
"logprob": -9999.0,
"bytes": [
32,
73,
110,
106,
101,
99,
116
]
},
{
"token": " Determin",
"logprob": -9999.0,
"bytes": [
32,
68,
101,
116,
101,
114,
109,
105,
110
]
},
{
"token": " ID",
"logprob": -9999.0,
"bytes": [
32,
73,
68
]
},
{
"token": " Membership",
"logprob": -9999.0,
"bytes": [
32,
77,
101,
109,
98,
101,
114,
115,
104,
105,
112
]
},
{
"token": " Ex",
"logprob": -9999.0,
"bytes": [
32,
69,
120
]
},
{
"token": " String",
"logprob": -9999.0,
"bytes": [
32,
83,
116,
114,
105,
110,
103
]
},
{
"token": " Special",
"logprob": -9999.0,
"bytes": [
32,
83,
112,
101,
99,
105,
97,
108
]
}
]
},
{
"token": "Un",
"logprob": 0.0,
"bytes": [
85,
110
],
"top_logprobs": [
{
"token": "Un",
"logprob": 0.0,
"bytes": [
85,
110
]
},
{
"token": "Unique",
"logprob": -9999.0,
"bytes": [
85,
110,
105,
113,
117,
101
]
},
{
"token": "Key",
"logprob": -9999.0,
"bytes": [
75,
101,
121
]
},
{
"token": "Token",
"logprob": -9999.0,
"bytes": [
84,
111,
107,
101,
110
]
},
{
"token": "No",
"logprob": -9999.0,
"bytes": [
78,
111
]
},
{
"token": "Ex",
"logprob": -9999.0,
"bytes": [
69,
120
]
},
{
"token": "Com",
"logprob": -9999.0,
"bytes": [
67,
111,
109
]
},
{
"token": "Prefix",
"logprob": -9999.0,
"bytes": [
80,
114,
101,
102,
105,
120
]
},
{
"token": "Inject",
"logprob": -9999.0,
"bytes": [
73,
110,
106,
101,
99,
116
]
},
{
"token": "Bi",
"logprob": -9999.0,
"bytes": [
66,
105
]
},
{
"token": "Non",
"logprob": -9999.0,
"bytes": [
78,
111,
110
]
},
{
"token": "Complete",
"logprob": -9999.0,
"bytes": [
67,
111,
109,
112,
108,
101,
116,
101
]
},
{
"token": "Identity",
"logprob": -9999.0,
"bytes": [
73,
100,
101,
110,
116,
105,
116,
121
]
},
{
"token": "Content",
"logprob": -9999.0,
"bytes": [
67,
111,
110,
116,
101,
110,
116
]
},
{
"token": "The",
"logprob": -9999.0,
"bytes": [
84,
104,
101
]
},
{
"token": "Cons",
"logprob": -9999.0,
"bytes": [
67,
111,
110,
115
]
},
{
"token": "Special",
"logprob": -9999.0,
"bytes": [
83,
112,
101,
99,
105,
97,
108
]
},
{
"token": "P",
"logprob": -9999.0,
"bytes": [
80
]
},
{
"token": "Cont",
"logprob": -9999.0,
"bytes": [
67,
111,
110,
116
]
},
{
"token": "Lex",
"logprob": -9999.0,
"bytes": [
76,
101,
120
]
}
]
},
{
"token": "iqu",
"logprob": 0.0,
"bytes": [
105,
113,
117
],
"top_logprobs": [
{
"token": "iqu",
"logprob": 0.0,
"bytes": [
105,
113,
117
]
},
{
"token": "icity",
"logprob": -9999.0,
"bytes": [
105,
99,
105,
116,
121
]
},
{
"token": "ambiguous",
"logprob": -9999.0,
"bytes": [
97,
109,
98,
105,
103,
117,
111,
117,
115
]
},
{
"token": "iq",
"logprob": -9999.0,
"bytes": [
105,
113
]
},
{
"token": "ambigu",
"logprob": -9999.0,
"bytes": [
97,
109,
98,
105,
103,
117
]
},
{
"token": "iquity",
"logprob": -9999.0,
"bytes": [
105,
113,
117,
105,
116,
121
]
},
{
"token": "ique",
"logprob": -9999.0,
"bytes": [
105,
113,
117,
101
]
},
{
"token": "icode",
"logprob": -9999.0,
"bytes": [
105,
99,
111,
100,
101
]
},
{
"token": "conditional",
"logprob": -9999.0,
"bytes": [
99,
111,
110,
100,
105,
116,
105,
111,
110,
97,
108
]
},
{
"token": "ified",
"logprob": -9999.0,
"bytes": [
105,
102,
105,
101,
100
]
},
{
"token": "ic",
"logprob": -9999.0,
"bytes": [
105,
99
]
},
{
"token": "i",
"logprob": -9999.0,
"bytes": [
105
]
},
{
"token": "idirectional",
"logprob": -9999.0,
"bytes": [
105,
100,
105,
114,
101,
99,
116,
105,
111,
110,
97,
108
]
},
{
"token": "id",
"logprob": -9999.0,
"bytes": [
105,
100
]
},
{
"token": "rest",
"logprob": -9999.0,
"bytes": [
114,
101,
115,
116
]
},
{
"token": "limited",
"logprob": -9999.0,
"bytes": [
108,
105,
109,
105,
116,
101,
100
]
},
{
"token": "iverse",
"logprob": -9999.0,
"bytes": [
105,
118,
101,
114,
115,
101
]
},
{
"token": "ary",
"logprob": -9999.0,
"bytes": [
97,
114,
121
]
},
{
"token": "iques",
"logprob": -9999.0,
"bytes": [
105,
113,
117,
101,
115
]
},
{
"token": "anim",
"logprob": -9999.0,
"bytes": [
97,
110,
105,
109
]
}
]
},
{
"token": "eness",
"logprob": 0.0,
"bytes": [
101,
110,
101,
115,
115
],
"top_logprobs": [
{
"token": "eness",
"logprob": 0.0,
"bytes": [
101,
110,
101,
115,
115
]
},
{
"token": "ency",
"logprob": -9999.0,
"bytes": [
101,
110,
99,
121
]
},
{
"token": "ess",
"logprob": -9999.0,
"bytes": [
101,
115,
115
]
},
{
"token": "ity",
"logprob": -9999.0,
"bytes": [
105,
116,
121
]
},
{
"token": "iness",
"logprob": -9999.0,
"bytes": [
105,
110,
101,
115,
115
]
},
{
"token": "ely",
"logprob": -9999.0,
"bytes": [
101,
108,
121
]
},
{
"token": "ens",
"logprob": -9999.0,
"bytes": [
101,
110,
115
]
},
{
"token": "enity",
"logprob": -9999.0,
"bytes": [
101,
110,
105,
116,
121
]
},
{
"token": "en",
"logprob": -9999.0,
"bytes": [
101,
110
]
},
{
"token": "ness",
"logprob": -9999.0,
"bytes": [
110,
101,
115,
115
]
},
{
"token": "icity",
"logprob": -9999.0,
"bytes": [
105,
99,
105,
116,
121
]
},
{
"token": "eliness",
"logprob": -9999.0,
"bytes": [
101,
108,
105,
110,
101,
115,
115
]
},
{
"token": "ality",
"logprob": -9999.0,
"bytes": [
97,
108,
105,
116,
121
]
},
{
"token": "enes",
"logprob": -9999.0,
"bytes": [
101,
110,
101,
115
]
},
{
"token": "erness",
"logprob": -9999.0,
"bytes": [
101,
114,
110,
101,
115,
115
]
},
{
"token": "teness",
"logprob": -9999.0,
"bytes": [
116,
101,
110,
101,
115,
115
]
},
{
"token": "acy",
"logprob": -9999.0,
"bytes": [
97,
99,
121
]
},
{
"token": "eless",
"logprob": -9999.0,
"bytes": [
101,
108,
101,
115,
115
]
},
{
"token": "iveness",
"logprob": -9999.0,
"bytes": [
105,
118,
101,
110,
101,
115,
115
]
},
{
"token": "ey",
"logprob": -9999.0,
"bytes": [
101,
121
]
}
]
},
{
"token": " Inv",
"logprob": 0.0,
"bytes": [
32,
73,
110,
118
],
"top_logprobs": [
{
"token": " Inv",
"logprob": 0.0,
"bytes": [
32,
73,
110,
118
]
},
{
"token": " of",
"logprob": -9999.0,
"bytes": [
32,
111,
102
]
},
{
"token": " and",
"logprob": -9999.0,
"bytes": [
32,
97,
110,
100
]
},
{
"token": "/",
"logprob": -9999.0,
"bytes": [
47
]
},
{
"token": " (",
"logprob": -9999.0,
"bytes": [
32,
40
]
},
{
"token": " Const",
"logprob": -9999.0,
"bytes": [
32,
67,
111,
110,
115,
116
]
},
{
"token": " &",
"logprob": -9999.0,
"bytes": [
32,
38
]
},
{
"token": " /",
"logprob": -9999.0,
"bytes": [
32,
47
]
},
{
"token": " invariant",
"logprob": -9999.0,
"bytes": [
32,
105,
110,
118,
97,
114,
105,
97,
110,
116
]
},
{
"token": "**",
"logprob": -9999.0,
"bytes": [
42,
42
]
},
{
"token": " Across",
"logprob": -9999.0,
"bytes": [
32,
65,
99,
114,
111,
115,
115
]
},
{
"token": ":",
"logprob": -9999.0,
"bytes": [
58
]
},
{
"token": " Integrity",
"logprob": -9999.0,
"bytes": [
32,
73,
110,
116,
101,
103,
114,
105,
116,
121
]
},
{
"token": "**\n",
"logprob": -9999.0,
"bytes": [
42,
42,
10
]
},
{
"token": " Comple",
"logprob": -9999.0,
"bytes": [
32,
67,
111,
109,
112,
108,
101
]
},
{
"token": " or",
"logprob": -9999.0,
"bytes": [
32,
111,
114
]
},
{
"token": " by",
"logprob": -9999.0,
"bytes": [
32,
98,
121
]
},
{
"token": "**:",
"logprob": -9999.0,
"bytes": [
42,
42,
58
]
},
{
"token": " Guar",
"logprob": -9999.0,
"bytes": [
32,
71,
117,
97,
114
]
},
{
"token": " Property",
"logprob": -9999.0,
"bytes": [
32,
80,
114,
111,
112,
101,
114,
116,
121
]
}
]
},
{
"token": "ariant",
"logprob": 0.0,
"bytes": [
97,
114,
105,
97,
110,
116
],
"top_logprobs": [
{
"token": "ariant",
"logprob": 0.0,
"bytes": [
97,
114,
105,
97,
110,
116
]
},
{
"token": "ariance",
"logprob": -9999.0,
"bytes": [
97,
114,
105,
97,
110,
99,
101
]
},
{
"token": "ariate",
"logprob": -9999.0,
"bytes": [
97,
114,
105,
97,
116,
101
]
},
{
"token": "ari",
"logprob": -9999.0,
"bytes": [
97,
114,
105
]
},
{
"token": "ariable",
"logprob": -9999.0,
"bytes": [
97,
114,
105,
97,
98,
108,
101
]
},
{
"token": "ariat",
"logprob": -9999.0,
"bytes": [
97,
114,
105,
97,
116
]
},
{
"token": "air",
"logprob": -9999.0,
"bytes": [
97,
105,
114
]
},
{
"token": "ri",
"logprob": -9999.0,
"bytes": [
114,
105
]
},
{
"token": "raint",
"logprob": -9999.0,
"bytes": [
114,
97,
105,
110,
116
]
},
{
"token": "**\n",
"logprob": -9999.0,
"bytes": [
42,
42,
10
]
},
{
"token": "ergent",
"logprob": -9999.0,
"bytes": [
101,
114,
103,
101,
110,
116
]
},
{
"token": "<｜｜end▁of▁sentence｜｜>",
"logprob": -9999.0,
"bytes": [
60,
239,
189,
156,
239,
189,
156,
101,
110,
100,
226,
150,
129,
111,
102,
226,
150,
129,
115,
101,
110,
116,
101,
110,
99,
101,
239,
189,
156,
239,
189,
156,
62
]
},
{
"token": "ocation",
"logprob": -9999.0,
"bytes": [
111,
99,
97,
116,
105,
111,
110
]
},
{
"token": "iol",
"logprob": -9999.0,
"bytes": [
105,
111,
108
]
},
{
"token": "arent",
"logprob": -9999.0,
"bytes": [
97,
114,
101,
110,
116
]
},
{
"token": "ai",
"logprob": -9999.0,
"bytes": [
97,
105
]
},
{
"token": "ag",
"logprob": -9999.0,
"bytes": [
97,
103
]
},
{
"token": "variant",
"logprob": -9999.0,
"bytes": [
118,
97,
114,
105,
97,
110,
116
]
},
{
"token": "ac",
"logprob": -9999.0,
"bytes": [
97,
99
]
},
{
"token": " (",
"logprob": -9999.0,
"bytes": [
32,
40
]
}
]
}
]
},
"finish_reason": "length"
}
],
"usage": {
"prompt_tokens": 17,
"completion_tokens": 24,
"total_tokens": 41,
"prompt_tokens_details": {
"cached_tokens": 0
},
"prompt_cache_hit_tokens": 0,
"prompt_cache_miss_tokens": 17
},
"system_fingerprint": "fp_a18b46594c_prod0820_fp8_kvcache_20260402"
}
Reference in new issue View Git Blame Copy Permalink