1119 lines
58 KiB
C
1119 lines
58 KiB
C
#include "ds4_eval_cases.h"
|
|
|
|
/* Curated hard cases. Source revisions and redistribution notes are recorded in
|
|
* EVAL_DATA.md. Keep this table data-only; grading belongs in ds4_eval.c. */
|
|
|
|
const eval_source eval_hard_sources[] = {
|
|
{"MMLU-Pro", "https://github.com/TIGER-AI-Lab/MMLU-Pro", "Apache-2.0"},
|
|
{"OlympiadBench", "https://huggingface.co/datasets/Hothan/OlympiadBench", "Apache-2.0"},
|
|
{"LiveBench", "https://huggingface.co/datasets/livebench/reasoning", "Apache-2.0"},
|
|
{"NIST Juliet", "https://samate.nist.gov/SARD/test-suites/112", "CC0-1.0"},
|
|
};
|
|
|
|
const size_t eval_hard_source_count =
|
|
sizeof(eval_hard_sources) / sizeof(eval_hard_sources[0]);
|
|
|
|
const eval_case eval_hard_cases[] = {
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "2826",
|
|
.domain = "biology",
|
|
.title = "Biology challenge",
|
|
.question = "A gene C mutates to c with a frequency of 2 \303\227 10^-6 per generation. There are no othe"
|
|
"r forces acting on these alleles and mating is random. How many generations are needed to in"
|
|
"crease the frequency of gene c from 2 percent to 3 percent?",
|
|
.choice[0] = "2,500 generations",
|
|
.choice[1] = "6,200 generations",
|
|
.choice[2] = "15,000 generations",
|
|
.choice[3] = "8,764 generations",
|
|
.choice[4] = "10,000 generations",
|
|
.choice[5] = "20,000 generations",
|
|
.choice[6] = "3,000 generations",
|
|
.choice[7] = "1,000 generations",
|
|
.choice[8] = "7,500 generations",
|
|
.choice[9] = "5,128 generations",
|
|
.answer = "J",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD | EVAL_SUITE_HARD_SMOKE,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "86",
|
|
.domain = "business",
|
|
.title = "Business challenge",
|
|
.question = "The stock of the CCC Corporation is currently valued at $12 and is assumed to possess all th"
|
|
"e properties of geometric Brownian motion. It has an expected annual return of 15%, an annua"
|
|
"l volatility of 20%, and the annual risk-free is 10%. Using a binomial lattice, determine th"
|
|
"e price of a call option on CCC stock maturing in 10 monthes time with a strike price of $14"
|
|
" (Let the distance between nodes on your tree be 1 month in length).",
|
|
.choice[0] = "50.0",
|
|
.choice[1] = "60.0",
|
|
.choice[2] = "55.0",
|
|
.choice[3] = "44.0",
|
|
.choice[4] = "48.0",
|
|
.choice[5] = "53.0",
|
|
.choice[6] = "42.0",
|
|
.choice[7] = "46.0",
|
|
.choice[8] = "51.0",
|
|
.choice[9] = "45.0",
|
|
.answer = "F",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "3546",
|
|
.domain = "chemistry",
|
|
.title = "Chemistry challenge",
|
|
.question = "Suppose that the normalized wavefunction for an electron in a carbon nanotube of length $L=1"
|
|
"0.0 \\mathrm{~nm}$ is: $\\psi=(2 / L)^{1 / 2} \\sin (\\pi x / L)$. Calculate the probability"
|
|
" that the electron is between $x=4.95 \\mathrm{~nm}$ and $5.05 \\mathrm{~nm}$.",
|
|
.choice[0] = "0.020",
|
|
.choice[1] = "0.050",
|
|
.choice[2] = "0.030",
|
|
.choice[3] = "0.045",
|
|
.choice[4] = "0.010",
|
|
.choice[5] = "0.015",
|
|
.choice[6] = "0.035",
|
|
.choice[7] = "0.060",
|
|
.choice[8] = "0.025",
|
|
.choice[9] = "0.040",
|
|
.answer = "A",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "10360",
|
|
.domain = "computer science",
|
|
.title = "Computer Science challenge",
|
|
.question = "Two expressions E and F are said to be unifiable if there are substitutions for the variable"
|
|
"s of E and F that make the expressions lexically identical. In the following three expressio"
|
|
"ns, only w, x, y, and z are variables.\nI. f(w,w)\nII. f(x,1)\nIII. f(y,g(z))\nWhich pairs o"
|
|
"f these expressions is (are) pairs of unifiable expressions?",
|
|
.choice[0] = "(I, III) and (II, III) only",
|
|
.choice[1] = "(II, III) only",
|
|
.choice[2] = "(I, II) and (I, III) only",
|
|
.choice[3] = "(I, II) only",
|
|
.choice[4] = "Only single expressions are unifiable, not pairs",
|
|
.choice[5] = "None of the pairs are unifiable",
|
|
.choice[6] = "(I, II), (I, III), and (II, III)",
|
|
.choice[7] = "All three expressions are unifiable",
|
|
.choice[8] = "(I, II) and (II, III) only",
|
|
.choice[9] = "(I, III) only",
|
|
.answer = "C",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD | EVAL_SUITE_HARD_SMOKE,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "6844",
|
|
.domain = "economics",
|
|
.title = "Economics challenge",
|
|
.question = "The purpose of \"augmenting\" the Dickey-Fuller test regression is to",
|
|
.choice[0] = "Ensure that there is no heteroscedasticity in the test regression residuals.",
|
|
.choice[1] = "Improve the accuracy of the Dickey-Fuller test",
|
|
.choice[2] = "Ensure that the test regression residuals are negatively skewed",
|
|
.choice[3] = "Ensure that the test regression residuals are normally distributed",
|
|
.choice[4] = "Ensure that all of the non-stationarity is taken into account.",
|
|
.choice[5] = "Ensure that the test regression residuals have no outliers",
|
|
.choice[6] = "Ensure that the test regression residuals are positively skewed",
|
|
.choice[7] = "Ensure that there is no autocorrelation in the test regression residuals",
|
|
.choice[8] = "Ensure that the test regression residuals are balanced",
|
|
.choice[9] = "Increase the complexity of the Dickey-Fuller test",
|
|
.answer = "H",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "11292",
|
|
.domain = "engineering",
|
|
.title = "Engineering challenge",
|
|
.question = "Calculate the entropy change for (a) H_2O (s, - 20\302\260C, 1atm) \\rightarrow H_2O (s, 0"
|
|
"\302\260C, 1atm) (b) H_2O (s, - 20\302\260C, 1atm) \\rightarrow H_2O (1, 20\302\260C, 1atm) "
|
|
"Assume C_p (H_2O, 1) = 18 cal deg^-1 mole^-1, C_p (H_2O, s) = 9 cal deg^-1 mole^-1 and\\Delt"
|
|
"aH_fusion= 1440 cal deg^-1 mole^-1, andalso that the heat capacities are independent of temp"
|
|
"erature.",
|
|
.choice[0] = "0.68eu/mole for (a) and 7.22eu for (b)",
|
|
.choice[1] = "1.27eu/mole for (a) and 7.22eu for (b)",
|
|
.choice[2] = "0.68eu/mole for (a) and 5.27eu for (b)",
|
|
.choice[3] = "1.27eu/mole for (a) and 5.27eu for (b)",
|
|
.answer = "A",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "6034",
|
|
.domain = "health",
|
|
.title = "Health challenge",
|
|
.question = "A 23-year-old woman with bone marrow failure is treated with a large dose of rabbit antithym"
|
|
"ocyte globulin. Ten days later, she develops fever, lymphadenopathy, arthralgias, and erythe"
|
|
"ma on her hands and feet. Which of the following is the most likely cause of these symptoms?",
|
|
.choice[0] = "Activation of complement system",
|
|
.choice[1] = "Overproduction of monocytes",
|
|
.choice[2] = "Cytokine secretion by natural killer cells",
|
|
.choice[3] = "Hyperactive B-lymphocyte response",
|
|
.choice[4] = "Allergic reaction to rabbit proteins",
|
|
.choice[5] = "Immune complex deposition in tissues",
|
|
.choice[6] = "Polyclonal T-lymphocyte activation",
|
|
.choice[7] = "Eosinophil degranulation",
|
|
.choice[8] = "Increased production of interferon-gamma",
|
|
.choice[9] = "Acute graft-versus-host disease",
|
|
.answer = "F",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD | EVAL_SUITE_HARD_SMOKE,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "4699",
|
|
.domain = "history",
|
|
.title = "History challenge",
|
|
.question = "This question refers to the following information.\n\"At least one of the [world's] societie"
|
|
"s would have to somehow enormously increase its productivity [in order to achieve global heg"
|
|
"emony]. That quantum jump would have to be made before the various scientific, technological"
|
|
", agricultural, and industrial revolutions on which our post-quantum-leap world rests. It co"
|
|
"uld only be accomplished by exploiting the ecosystems, mineral resources, and human assets o"
|
|
"f whole continents outside the lands of the society making the jump. Western Europe did just"
|
|
" that by means of its brutality and guns and, more important, by geographical and ecological"
|
|
" luck.\"\nCopyright \302\251 2015 Cambridge University Press.\nAlfred Crosby, historian, Eco"
|
|
"logical Imperialism, 2004\nBased on your knowledge of world history, which of the following "
|
|
"contributed LEAST to Western European global hegemony between 1450 and 1750 C.E.?",
|
|
.choice[0] = "The implementation of joint-stock companies",
|
|
.choice[1] = "The exploitation of labor in colonized regions",
|
|
.choice[2] = "The exploration of new sea routes to Asia and the Americas",
|
|
.choice[3] = "The establishment of trading posts along Asian and African coasts",
|
|
.choice[4] = "The exchange of food sources between the Americas and Europe",
|
|
.choice[5] = "The spread of Christianity to non-European regions",
|
|
.choice[6] = "The development and application of steam-powered technologies",
|
|
.choice[7] = "The discovery and colonization of the Americas",
|
|
.choice[8] = "The development of advanced naval technology",
|
|
.choice[9] = "Refinement of gunpowder technologies",
|
|
.answer = "G",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "899",
|
|
.domain = "law",
|
|
.title = "Law challenge",
|
|
.question = "A man and a woman were passengers on a plane. They were seated next to one another in the fi"
|
|
"rst- class section of the plane. Midway through the flight, the woman excused herself and we"
|
|
"nt to the restroom. While she was gone, the man rummaged through the woman's purse and stole"
|
|
" $100 in cash. Unknown to the man, a flight attendant saw him steal the money. As is customa"
|
|
"ry in such situations, the flight attendant radioed ahead, and police officers arrested the "
|
|
"man when the plane landed. The man was subsequently charged with violating an applicable fed"
|
|
"eral larceny statute. During the trial, the prosecuting attorney called the man's wife as a "
|
|
"willing witness against her husband. She proposed to testify that the man confided to her th"
|
|
"at he did steal the woman's money during the flight. The man's attorney objected on the grou"
|
|
"nds of hearsay and privilege. The wife's proposed testimony is",
|
|
.choice[0] = "admissible, because it is privileged, though it is not hearsay.",
|
|
.choice[1] = "admissible, because it is neither hearsay nor privileged.",
|
|
.choice[2] = "admissible, because it is hearsay within a recognized exception and not privileged.",
|
|
.choice[3] = "inadmissible, both because it discloses a privileged communication and because it is hearsay"
|
|
" not within any recognized exception.",
|
|
.choice[4] = "admissible, both because it discloses a privileged communication and because it is hearsay n"
|
|
"ot within any recognized exception.",
|
|
.choice[5] = "inadmissible, because it is hearsay not within any recognized exception, and it discloses a "
|
|
"privileged communication.",
|
|
.choice[6] = "inadmissible, because it discloses a privileged communication, though it is not hearsay.",
|
|
.choice[7] = "admissible, both because it does not disclose a privileged communication and because it is h"
|
|
"earsay within a recognized exception.",
|
|
.choice[8] = "inadmissible, because it is hearsay within a recognized exception, though it is not privileg"
|
|
"ed.",
|
|
.choice[9] = "inadmissible, because it is hearsay not within any recognized exception, though it is not pr"
|
|
"ivileged.",
|
|
.answer = "G",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "7738",
|
|
.domain = "math",
|
|
.title = "Math challenge",
|
|
.question = "Suppose that $f(x)$ is a polynomial that has degree $6$ and $g(x)$ is a polynomial that has "
|
|
"degree $3$. If $h(x)$ is also a polynomial such that $f(g(x)) + g(h(x)) + h(f(x))$ is a poly"
|
|
"nomial of degree $36$, then what is the degree of the polynomial $h$?",
|
|
.choice[0] = "3",
|
|
.choice[1] = "21",
|
|
.choice[2] = "6",
|
|
.choice[3] = "9",
|
|
.choice[4] = "30",
|
|
.choice[5] = "72",
|
|
.choice[6] = "12",
|
|
.choice[7] = "15",
|
|
.choice[8] = "24",
|
|
.choice[9] = "18",
|
|
.answer = "C",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "5078",
|
|
.domain = "other",
|
|
.title = "Other challenge",
|
|
.question = "Are strategies of Arms Control and Deterrence useful in cyber-security?",
|
|
.choice[0] = "No, arms control does not serve any purpose outside the realm of traditional military weapon"
|
|
"s.",
|
|
.choice[1] = "Arms control strategies can be implemented, but deterrence strategies are impossible due to "
|
|
"the anonymous nature of cyber attacks.",
|
|
.choice[2] = "Yes, both strategies are effective in cyber-security as long as there is strong internationa"
|
|
"l cooperation and regulation.",
|
|
.choice[3] = "Effective deterrence requires international cooperation which is achievable in theory, but t"
|
|
"he verification required for traditional arms control strategies is impossible in cyber(ed) "
|
|
"warfare.",
|
|
.choice[4] = "Only arms control is unproblematic, deterrence has failed because international cooperation "
|
|
"cannot be reached and common rules or practices cannot be established.",
|
|
.choice[5] = "Both strategies can only work if there is a clear agreement on what constitutes a cyber weap"
|
|
"on and what constitutes a cyber attack.",
|
|
.choice[6] = "Yes, successful policies of arms control and deterrence have translated well from other secu"
|
|
"rity realms, such as nuclear weapons control strategies.",
|
|
.choice[7] = "Deterrence is ineffective because it encourages an arms race in cyberspace, while arms contr"
|
|
"ol is difficult due to the ease of creating and distributing cyber weapons.",
|
|
.choice[8] = "Both arms control and deterrence strategies are ineffective due to the rapid and unpredictab"
|
|
"le development of cyber warfare technologies.",
|
|
.choice[9] = "No, both strategies are obsolete in the face of cyber warfare's unique challenges such as at"
|
|
"tribution and boundary issues.",
|
|
.answer = "D",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "10775",
|
|
.domain = "philosophy",
|
|
.title = "Philosophy challenge",
|
|
.question = " Construct a complete truth table for the following pairs of propositions. Then, using the t"
|
|
"ruth tables, determine whether the statements are logically equivalent or contradictory. If "
|
|
"neither, determine whether they are consistent or inconsistent. Justify your answers.\n(~M "
|
|
"\342\212\203 ~N) \342\210\250 (O \342\211\241 N) and (~M \302\267 N) \302\267 [(~O \342\210"
|
|
"\250 ~N) \302\267 (O \342\210\250 N)]",
|
|
.choice[0] = "Both logically equivalent and consistent",
|
|
.choice[1] = "Both consistent and contradictory",
|
|
.choice[2] = "Both logically equivalent and contradictory",
|
|
.choice[3] = "Contradictory",
|
|
.choice[4] = "Neither logically equivalent nor contradictory, but inconsistent",
|
|
.choice[5] = "Logically equivalent",
|
|
.choice[6] = "Both inconsistent and contradictory",
|
|
.choice[7] = "Inconsistent",
|
|
.choice[8] = "Neither logically equivalent nor contradictory, but consistent",
|
|
.choice[9] = "Neither logically equivalent nor inconsistent, but contradictory",
|
|
.answer = "D",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "9055",
|
|
.domain = "physics",
|
|
.title = "Physics challenge",
|
|
.question = "Two spaceships approach Earth with equal speeds, as measured by an observer on Earth, but fr"
|
|
"om opposite directions. A meterstick on one spaceship is measured to be 60 cm long by an occ"
|
|
"upant of the other spaceship. What is the speed of each spaceship, as measured by the observ"
|
|
"er on Earth?",
|
|
.choice[0] = "0.4c",
|
|
.choice[1] = "0.1c",
|
|
.choice[2] = "1.0c",
|
|
.choice[3] = "0.9c",
|
|
.choice[4] = "0.2c",
|
|
.choice[5] = "0.5c",
|
|
.choice[6] = "0.6c",
|
|
.choice[7] = "0.3c",
|
|
.choice[8] = "0.8c",
|
|
.choice[9] = "0.7c",
|
|
.answer = "F",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "1995",
|
|
.domain = "psychology",
|
|
.title = "Psychology challenge",
|
|
.question = "An examinee who makes a number of confabulatory responses on the Rorschach has overgeneraliz"
|
|
"ed from a part of the inkblot to the whole. Such responses suggest which of the following?",
|
|
.choice[0] = "an inability to perceive abstract concepts",
|
|
.choice[1] = "high level of stress or anxiety",
|
|
.choice[2] = "a low level of self-esteem",
|
|
.choice[3] = "high intelligence and creativity",
|
|
.choice[4] = "a lack of attention to detail",
|
|
.choice[5] = "a need for immediate gratification",
|
|
.choice[6] = "brain damage, intellectual disability, or emotional disturbance",
|
|
.choice[7] = "a high degree of defensiveness or denial",
|
|
.choice[8] = "a highly imaginative personality",
|
|
.choice[9] = "a high degree of introversion or social withdrawal",
|
|
.answer = "G",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "7754",
|
|
.domain = "math",
|
|
.title = "Math challenge",
|
|
.question = "How many triangles are there whose sides are all integers and whose maximum side length equa"
|
|
"ls 11?",
|
|
.choice[0] = "45",
|
|
.choice[1] = "36",
|
|
.choice[2] = "48",
|
|
.choice[3] = "33",
|
|
.choice[4] = "50",
|
|
.choice[5] = "55",
|
|
.choice[6] = "24",
|
|
.choice[7] = "42",
|
|
.choice[8] = "30",
|
|
.choice[9] = "60",
|
|
.answer = "B",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "3024",
|
|
.domain = "biology",
|
|
.title = "Biology challenge",
|
|
.question = "Two 30\" plants are crossed, resulting in progeny of the following ratio: one 22\", eight 24"
|
|
"\\textquotedblright, twenty-eight 26\", fifty-six 28\", seventy 30\", fifty-six 32\", twenty"
|
|
"-eight 34\", eight 36\", and one 38\". Starting with A and going through the alphabet to rep"
|
|
"resent different alleles, what are the most probable genotypes of the parents?",
|
|
.choice[0] = "AaBbCcDd\303\227AABBCCDD",
|
|
.choice[1] = "AaBbCcEeFf\303\227AaBbCcEeFf",
|
|
.choice[2] = "AaBbCc\303\227AaBbCc",
|
|
.choice[3] = "AaBb\303\227AaBb",
|
|
.choice[4] = "AaBbCcDd\303\227AaBbCcDd",
|
|
.choice[5] = "AaBbCcDdEe\303\227AaBbCcDdEe",
|
|
.choice[6] = "Aa\303\227Aa",
|
|
.choice[7] = "AaBbDd\303\227AaBbDd",
|
|
.choice[8] = "AaBbCc\303\227AaBb",
|
|
.choice[9] = "AABB\303\227AABB",
|
|
.answer = "E",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "671",
|
|
.domain = "business",
|
|
.title = "Business challenge",
|
|
.question = "Frank Bryant is a salesman for the Fast Cool Refrigerator Co. He is compensated for all trav"
|
|
"el expenses, and is allowed a drawing account of $600 per month. He receives a monthly commi"
|
|
"ssion of 2% of his first $30,000 of sales, 2.5% of the next $10,000 of sales, and 3% of all "
|
|
"sales in excess of $40,000. Find the amount due him for a month in which his travel expenses"
|
|
" paid out of pocket are $332.75, his drawings are $600, and his sales total $41,180.",
|
|
.choice[0] = "$1,332.75",
|
|
.choice[1] = "$942.55",
|
|
.choice[2] = "$783.40",
|
|
.choice[3] = "$618.15",
|
|
.choice[4] = "$725.15",
|
|
.choice[5] = "$1,035.40",
|
|
.choice[6] = "$885.40",
|
|
.choice[7] = "$1,118.15",
|
|
.choice[8] = "$1,218.15",
|
|
.choice[9] = "$332.75",
|
|
.answer = "D",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "3696",
|
|
.domain = "chemistry",
|
|
.title = "Chemistry challenge",
|
|
.question = "2 moles of a monoatomic ideal gas at 45\302\260C are compressed adiabatically and reversibly"
|
|
" from 75 liters to 20 liters. Calculate q, W, \\DeltaH and \\DeltaE for the process, excludi"
|
|
"ng vi-brationalcontributions.",
|
|
.choice[0] = "q=0, \316\224E=15.00kJ, \316\224H=22.50kJ",
|
|
.choice[1] = "q=11.21kJ, \316\224E=0, \316\224H=18.68kJ",
|
|
.choice[2] = "q=0, \316\224E=18.68kJ, \316\224H=11.21kJ",
|
|
.choice[3] = "q=18.68kJ, \316\224E=18.68kJ, \316\224H=18.68kJ",
|
|
.choice[4] = "q=0, \316\224E=11.21kJ, \316\224H=18.68kJ",
|
|
.choice[5] = "q=11.21kJ, \316\224E=18.68kJ, \316\224H=0",
|
|
.choice[6] = "q=22.50kJ, \316\224E=15.00kJ, \316\224H=0",
|
|
.choice[7] = "q=0, \316\224E=22.50kJ, \316\224H=15.00kJ",
|
|
.choice[8] = "q=18.68kJ, \316\224E=11.21kJ, \316\224H=0",
|
|
.choice[9] = "q=18.68kJ, \316\224E=0, \316\224H=11.21kJ",
|
|
.answer = "E",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "10567",
|
|
.domain = "computer science",
|
|
.title = "Computer Science challenge",
|
|
.question = "Let F: K \\times R \\to MF:K\303\227R\342\206\222M be a secure PRF. For m \\in Mm\342\210"
|
|
"\210M define E(k,m) = \\big[ r \\gets R,\\ \\text{output } \\big(r,\\ F(k,r) \\oplus m\\big)"
|
|
"\\ \\big]E(k,m)=[r\342\206\220R, output (r, F(k,r)\342\212\225m) ] Is EE symantically secure"
|
|
" under CPA?",
|
|
.choice[0] = "It depends on the size of R",
|
|
.choice[1] = "Yes, but only if R is large enough so r never repeats (w.h.p)",
|
|
.choice[2] = "Yes, but only if M is small enough so m never repeats (w.h.p)",
|
|
.choice[3] = "No, unless F is a weak PRF",
|
|
.choice[4] = "No, unless F is a strong PRF",
|
|
.choice[5] = "Yes, but only if K is large enough so k never repeats (w.h.p)",
|
|
.choice[6] = "It depends on the size of K",
|
|
.choice[7] = "No, there is always a CPA attack on this system",
|
|
.choice[8] = "It depends on what F is used",
|
|
.choice[9] = "Yes, whenever F is a secure PRF",
|
|
.answer = "B",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD | EVAL_SUITE_HARD_SMOKE,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "6955",
|
|
.domain = "economics",
|
|
.title = "Economics challenge",
|
|
.question = "Which of the following is included in U.S. GDP? I. The market value of movies made in Afri"
|
|
"ca by U.S. citizens II. The market value of olive oil made in Italy and sold in the United"
|
|
" States. III. The market value of blue jeans made in the United States and sold in Japan I"
|
|
"V. The market value of wine made in the United States by Canadian citizens",
|
|
.choice[0] = "II and IV only",
|
|
.choice[1] = "I, II and IV only",
|
|
.choice[2] = "I only",
|
|
.choice[3] = "IV only",
|
|
.choice[4] = "I, II and III only",
|
|
.choice[5] = "I and IV only",
|
|
.choice[6] = "III and IV only",
|
|
.choice[7] = "I and III only",
|
|
.choice[8] = "II and III only",
|
|
.choice[9] = "II III and IV only",
|
|
.answer = "G",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "11346",
|
|
.domain = "engineering",
|
|
.title = "Engineering challenge",
|
|
.question = "Calculate the loss of head in a pipe (1/4) in. in diameter and 16 ft. long, when water flows"
|
|
" at half the critical velocity. The critical velocity occurs when Reynolds number is 2500. T"
|
|
"ake viscosity \\mu = 1.01 centipoise = 0.679 \303\227 10^-3 lbm/ft-sec.",
|
|
.choice[0] = "3.12 in",
|
|
.choice[1] = "7.5 in",
|
|
.choice[2] = "0.8 in",
|
|
.choice[3] = "6.3 in",
|
|
.choice[4] = "2.5 in",
|
|
.choice[5] = "5.0 in",
|
|
.choice[6] = "1.75 in",
|
|
.choice[7] = "4.2 in",
|
|
.choice[8] = "4.8 in",
|
|
.choice[9] = "2.0 in",
|
|
.answer = "A",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "6500",
|
|
.domain = "health",
|
|
.title = "Health challenge",
|
|
.question = "A 33-year-old man undergoes a radical thyroidectomy for thyroid cancer. During the operation"
|
|
", moderate hemorrhaging requires ligation of several vessels in the left side of the neck. P"
|
|
"ostoperatively, serum studies show a calcium concentration of 7.5 mg/dL, albumin concentrati"
|
|
"on of 4 g/dL, and parathyroid hormone concentration of 200 pg/mL. Damage to which of the fol"
|
|
"lowing vessels caused the findings in this patient?",
|
|
.choice[0] = "Branch of the costocervical trunk",
|
|
.choice[1] = "Branch of the external carotid artery",
|
|
.choice[2] = "Branch of the thyrocervical trunk",
|
|
.choice[3] = "Tributary of the internal jugular vein",
|
|
.answer = "C",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD | EVAL_SUITE_HARD_SMOKE,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "4674",
|
|
.domain = "history",
|
|
.title = "History challenge",
|
|
.question = "Recent discoveries of Homo naledi fossils in the Dinaledi Chamber of the Rising Star cave sy"
|
|
"stem in South Africa suggest a mixture of post-cranial morphologies or mosaic of hominid tra"
|
|
"its. One of the reasons this new species is controversial is that:",
|
|
.choice[0] = "the Homo naledi fossils were found in the same layer as modern human fossils, suggesting the"
|
|
"y lived at the same time.",
|
|
.choice[1] = "the Homo naledi fossils have traits that do not fit into the current understanding of homini"
|
|
"d evolution.",
|
|
.choice[2] = "paleoanthropologists did not make the discovery and it was found in an inaccessible cave in "
|
|
"South Africa.",
|
|
.choice[3] = "the fossils were found in a region previously thought to be inhabited by a different species"
|
|
" of Homo.",
|
|
.choice[4] = "the Homo naledi fossils show traits of both Homo and Australopithecus, suggesting a potentia"
|
|
"l hybrid species.",
|
|
.choice[5] = "the methods used to extract the fossils from the cave were controversial and potentially dam"
|
|
"aging to the specimens.",
|
|
.choice[6] = "there is, as yet, no firm date that can be applied to the specimens.",
|
|
.choice[7] = "the Homo naledi fossils have been shown to pre-date the earliest known hominid fossils by mi"
|
|
"llions of years.",
|
|
.choice[8] = "there are already two well-known species of Homo and it is unlikely there were more than two"
|
|
" species at one time.",
|
|
.choice[9] = "the researchers who discovered the fossils have been accused of falsifying their findings.",
|
|
.answer = "G",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "1467",
|
|
.domain = "law",
|
|
.title = "Law challenge",
|
|
.question = "A motorcycle collector and his friend had known each other for years. The collector sent a l"
|
|
"etter to the friend that said, \"I am willing to sell you my oldest motorcycle for $4,000.\""
|
|
" The friend sent the collector an email saying, \"I accept your offer.\" The collector owned"
|
|
" three motorcycles: a 1985 model, a 1987 model, and a 1992 model. The friend did not know ab"
|
|
"out the 1985 model, which was the one the collector intended to sell. The collector was unaw"
|
|
"are that the friend intended to buy the 1987 model. When the collector tendered the 1985 mod"
|
|
"el, the friend refused to accept it. Is the friend likely to prevail in a breach of contract"
|
|
" suit against the collector?",
|
|
.choice[0] = "No, because it was the friend's responsibility to know the details about the collector's mot"
|
|
"orcycles before agreeing to purchase.",
|
|
.choice[1] = "No, because by not inquiring further about the meaning of the phrase \"oldest motorcycle,\" "
|
|
"the friend assumed the risk of being mistaken.",
|
|
.choice[2] = "No, the friend won't prevail as he misunderstood the term \"oldest motorcycle\".",
|
|
.choice[3] = "Yes, so long as a reasonable person in the friend's position would have considered the lette"
|
|
"r as referring to the 1987 model.",
|
|
.choice[4] = "No, because neither the friend nor the collector had reason to know of the other's interpret"
|
|
"ation of the words \"oldest motorcycle.\"",
|
|
.choice[5] = "Yes, the friend will prevail as the collector falsely advertised the motorcycle model.",
|
|
.choice[6] = "No, because the friend should have asked for clarification about which model was being sold "
|
|
"before agreeing to the purchase.",
|
|
.choice[7] = "Yes, because the friend could have reasonably assumed that the \"oldest motorcycle\" referre"
|
|
"d to the 1987 model, given his knowledge.",
|
|
.choice[8] = "Yes, because the friend had no reason to know of the collector's meaning and the collector c"
|
|
"ould have avoided the confusion by more careful drafting.",
|
|
.choice[9] = "Yes, because the collector should have clarified which model he was referring to in the lett"
|
|
"er.",
|
|
.answer = "E",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "8771",
|
|
.domain = "math",
|
|
.title = "Math challenge",
|
|
.question = "7.4-11. Some dentists were interested in studying the fusion of embryonic rat palates by a s"
|
|
"tandard transplantation technique. When no treatment is used, the probability of fusion equa"
|
|
"ls approximately 0.89 . The dentists would like to estimate $p$, the probability of fusion, "
|
|
"when vitamin A is lacking.\n(a) How large a sample $n$ of rat embryos is needed for $y / n "
|
|
"\\pm 0.10$ to be a $95 \\%$ confidence interval for $p$ ?",
|
|
.choice[0] = "70",
|
|
.choice[1] = "50",
|
|
.choice[2] = "45",
|
|
.choice[3] = "40",
|
|
.choice[4] = "30",
|
|
.choice[5] = "55",
|
|
.choice[6] = "$38$",
|
|
.choice[7] = "65",
|
|
.choice[8] = "60",
|
|
.choice[9] = "25",
|
|
.answer = "G",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "5848",
|
|
.domain = "other",
|
|
.title = "Other challenge",
|
|
.question = "Lake Trust a simple trust reported the following items of income and expense during the year"
|
|
": Dividend income $2500 Taxable interest income 2000 Capital gains (allocable to corpus) 500"
|
|
"0 Accounting fees (allocable to income) (500) Trustee fees (allocable to income) (750) What "
|
|
"is Lake's distributable net income?",
|
|
.choice[0] = "$5,000",
|
|
.choice[1] = "$6,000",
|
|
.choice[2] = "$9,500",
|
|
.choice[3] = "$2,000",
|
|
.choice[4] = "$4,500",
|
|
.choice[5] = "$10,000",
|
|
.choice[6] = "$8,250",
|
|
.choice[7] = "$7,500",
|
|
.choice[8] = "$1,750",
|
|
.choice[9] = "3250",
|
|
.answer = "J",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "11120",
|
|
.domain = "philosophy",
|
|
.title = "Philosophy challenge",
|
|
.question = " Use indirect truth tables to determine whether each set of propositions is consistent. If t"
|
|
"he set is consistent, choose an option with a consistent valuation. (There may be other cons"
|
|
"istent valuations.)\nD \342\211\241 E\nF \342\210\250 D\n~E \302\267 ~F",
|
|
.choice[0] = "Consistent. Consistent valuation when D and E are true and F is true",
|
|
.choice[1] = "Inconsistent",
|
|
.choice[2] = "Consistent. Consistent valuation when D is true, E and F are false",
|
|
.choice[3] = "Consistent. Consistent valuation when E is true, D and F are false",
|
|
.choice[4] = "Consistent. Consistent valuation when D and E are true and F is false",
|
|
.choice[5] = "Consistent. Consistent valuation when D, E are false and F is true",
|
|
.choice[6] = "Consistent. Consistent valuation when D and F are true and E is false",
|
|
.choice[7] = "Consistent. Consistent valuation when D, E, and F are true",
|
|
.choice[8] = "Consistent. Consistent valuation when F is true, D and E are false",
|
|
.choice[9] = "Consistent. Consistent valuation when D, E and F are false",
|
|
.answer = "B",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "9487",
|
|
.domain = "physics",
|
|
.title = "Physics challenge",
|
|
.question = "A gas sample is known to be a mixture of ethane and butane. A bulb having a $230.0 \\mathrm{"
|
|
"~cm}^3$ capacity is filled with the gas to a pressure of $97.5 \\times 10^3 \\mathrm{~Pa}$ a"
|
|
"t $23.1^{\\circ} \\mathrm{C}$. If the mass of the gas in the bulb is $0.3554 \\mathrm{~g}$, "
|
|
"what is the mole percent of butane in the mixture?\n",
|
|
.choice[0] = "70 %",
|
|
.choice[1] = "27 %",
|
|
.choice[2] = "50 %",
|
|
.choice[3] = "10 %",
|
|
.choice[4] = "60 %",
|
|
.choice[5] = "15 %",
|
|
.choice[6] = "25 %",
|
|
.choice[7] = "32 %",
|
|
.choice[8] = "45 %",
|
|
.choice[9] = "38 %",
|
|
.answer = "H",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD | EVAL_SUITE_HARD_SMOKE,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "2398",
|
|
.domain = "psychology",
|
|
.title = "Psychology challenge",
|
|
.question = "Susan is a first-grade student who is referred to the school psychologist because she will n"
|
|
"ot remain seated at her desk. Her teacher reports that Susan is \"always getting up and walk"
|
|
"ing around the room.\" What observation data is LEAST relevant to the referral concern?",
|
|
.choice[0] = "Susan's academic performance",
|
|
.choice[1] = "The teacher's teaching style",
|
|
.choice[2] = "Number of students in the class",
|
|
.choice[3] = "The school's disciplinary policies",
|
|
.choice[4] = "Classroom environment",
|
|
.choice[5] = "Peer comparison",
|
|
.choice[6] = "Frequency",
|
|
.choice[7] = "Intensity",
|
|
.choice[8] = "Susan's diet",
|
|
.choice[9] = "Duration",
|
|
.answer = "H",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "MMLU-Pro",
|
|
.id = "8863",
|
|
.domain = "math",
|
|
.title = "Math challenge",
|
|
.question = "An investor deposits $1000 in an account paying interest at a rate of 8% compounded monthly,"
|
|
" and also makes additional deposits of \\$25 per month. Find the balance in the account afte"
|
|
"r 3 years.",
|
|
.choice[0] = "$3200.67",
|
|
.choice[1] = "$2050.89",
|
|
.choice[2] = "$2100.45",
|
|
.choice[3] = "$1950.32",
|
|
.choice[4] = "$3000.59",
|
|
.choice[5] = "$2500.00",
|
|
.choice[6] = "2283.63 $",
|
|
.choice[7] = "$2750.48",
|
|
.choice[8] = "$2600.25",
|
|
.choice[9] = "$1800.75",
|
|
.answer = "G",
|
|
.answer_kind = EVAL_ANSWER_CHOICE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "OlympiadBench",
|
|
.id = "1606",
|
|
.domain = "Combinatorics",
|
|
.title = "Olympiad problem 1606",
|
|
.question = "Xenia and Sergey play the following game. Xenia thinks of a positive integer $N$ not exceedi"
|
|
"ng 5000. Then she fixes 20 distinct positive integers $a_{1}, a_{2}, \\ldots, a_{20}$ such t"
|
|
"hat, for each $k=1,2, \\ldots, 20$, the numbers $N$ and $a_{k}$ are congruent modulo $k$. By"
|
|
" a move, Sergey tells Xenia a set $S$ of positive integers not exceeding 20 , and she tells "
|
|
"him back the set $\\left\\{a_{k}: k \\in S\\right\\}$ without spelling out which number corr"
|
|
"esponds to which index. How many moves does Sergey need to determine for sure the number Xen"
|
|
"ia thought of?",
|
|
.answer = "2",
|
|
.answer_kind = EVAL_ANSWER_INTEGER,
|
|
.suites = EVAL_SUITE_HARD | EVAL_SUITE_HARD_SMOKE,
|
|
.max_tokens = 16000,
|
|
},
|
|
{
|
|
.source = "OlympiadBench",
|
|
.id = "1613",
|
|
.domain = "Algebra",
|
|
.title = "Olympiad problem 1613",
|
|
.question = "Determine all positive integers $n$ satisfying the following condition: for every monic poly"
|
|
"nomial $P$ of degree at most $n$ with integer coefficients, there exists a positive integer "
|
|
"$k \\leq n$, and $k+1$ distinct integers $x_{1}, x_{2}, \\ldots, x_{k+1}$ such that\n\n\n\n$"
|
|
"$\n\nP\\left(x_{1}\\right)+P\\left(x_{2}\\right)+\\cdots+P\\left(x_{k}\\right)=P\\left(x_{k+"
|
|
"1}\\right) .\n\n$$\n\n\nNote. A polynomial is monic if the coefficient of the highest power "
|
|
"is one.",
|
|
.answer = "2",
|
|
.answer_kind = EVAL_ANSWER_INTEGER,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 16000,
|
|
},
|
|
{
|
|
.source = "OlympiadBench",
|
|
.id = "1614",
|
|
.domain = "Combinatorics",
|
|
.title = "Olympiad problem 1614",
|
|
.question = "Let $n$ be an integer greater than 1 and let $X$ be an $n$-element set. A non-empty collecti"
|
|
"on of subsets $A_{1}, \\ldots, A_{k}$ of $X$ is tight if the union $A_{1} \\cup \\cdots \\cu"
|
|
"p A_{k}$ is a proper subset of $X$ and no element of $X$ lies in exactly one of the $A_{i}$ "
|
|
"s. Find the largest cardinality of a collection of proper non-empty subsets of $X$, no non-e"
|
|
"mpty subcollection of which is tight.\n\n\n\nNote. A subset $A$ of $X$ is proper if $A \\neq"
|
|
" X$. The sets in a collection are assumed to be distinct. The whole collection is assumed to"
|
|
" be a subcollection.",
|
|
.answer = "2n-2",
|
|
.alias[0] = "2(n-1)",
|
|
.answer_kind = EVAL_ANSWER_EXACT_TEXT,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 16000,
|
|
},
|
|
{
|
|
.source = "OlympiadBench",
|
|
.id = "1618",
|
|
.domain = "Number Theory",
|
|
.title = "Olympiad problem 1618",
|
|
.question = "Determine all prime numbers $p$ and all positive integers $x$ and $y$ satisfying $x^{3}+y^{3"
|
|
"}=$ $p(x y+p)$. Report each solution once as $(x,y,p)$ with $x \\leq y$.",
|
|
.answer = "(1,8,19),(2,7,13),(4,5,7)",
|
|
.alias[0] = "(4,5,7),(2,7,13),(1,8,19)",
|
|
.answer_kind = EVAL_ANSWER_ORDERED_SEQUENCE,
|
|
.suites = EVAL_SUITE_HARD | EVAL_SUITE_HARD_SMOKE,
|
|
.max_tokens = 16000,
|
|
},
|
|
{
|
|
.source = "OlympiadBench",
|
|
.id = "1697",
|
|
.domain = "Combinatorics",
|
|
.title = "Olympiad problem 1697",
|
|
.question = "On a square table of 2011 by 2011 cells we place a finite number of napkins that each cover "
|
|
"a square of 52 by 52 cells. In each cell we write the number of napkins covering it, and we "
|
|
"record the maximal number $k$ of cells that all contain the same nonzero number. Considering"
|
|
" all possible napkin configurations, what is the largest value of $k$ ?",
|
|
.answer = "3986729",
|
|
.answer_kind = EVAL_ANSWER_INTEGER,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 16000,
|
|
},
|
|
{
|
|
.source = "OlympiadBench",
|
|
.id = "1709",
|
|
.domain = "Number Theory",
|
|
.title = "Olympiad problem 1709",
|
|
.question = "For each positive integer $k$, let $t(k)$ be the largest odd divisor of $k$. Determine all p"
|
|
"ositive integers $a$ for which there exists a positive integer $n$ such that all the differe"
|
|
"nces\n\n$$\nt(n+a)-t(n), \\quad t(n+a+1)-t(n+1), \\quad \\ldots, \\quad t(n+2 a-1)-t(n+a-1)"
|
|
"\n$$\n\nare divisible by 4 .",
|
|
.answer = "1,3,5",
|
|
.answer_kind = EVAL_ANSWER_ORDERED_SEQUENCE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 16000,
|
|
},
|
|
{
|
|
.source = "OlympiadBench",
|
|
.id = "1716",
|
|
.domain = "Algebra",
|
|
.title = "Olympiad problem 1716",
|
|
.question = "Let $x_{1}, \\ldots, x_{100}$ be nonnegative real numbers such that $x_{i}+x_{i+1}+x_{i+2} "
|
|
"\\leq 1$ for all $i=1, \\ldots, 100$ (we put $x_{101}=x_{1}, x_{102}=x_{2}$ ). Find the maxi"
|
|
"mal possible value of the sum\n\n$$\nS=\\sum_{i=1}^{100} x_{i} x_{i+2}\n$$",
|
|
.answer = "25/2",
|
|
.alias[0] = "12.5",
|
|
.answer_kind = EVAL_ANSWER_RATIONAL,
|
|
.suites = EVAL_SUITE_HARD | EVAL_SUITE_HARD_SMOKE,
|
|
.max_tokens = 16000,
|
|
},
|
|
{
|
|
.source = "OlympiadBench",
|
|
.id = "1736",
|
|
.domain = "Number Theory",
|
|
.title = "Olympiad problem 1736",
|
|
.question = "Find the least positive integer $n$ for which there exists a set $\\left\\{s_{1}, s_{2}, \\l"
|
|
"dots, s_{n}\\right\\}$ consisting of $n$ distinct positive integers such that\n\n$$\n\\left("
|
|
"1-\\frac{1}{s_{1}}\\right)\\left(1-\\frac{1}{s_{2}}\\right) \\ldots\\left(1-\\frac{1}{s_{n}}"
|
|
"\\right)=\\frac{51}{2010}\n$$",
|
|
.answer = "39",
|
|
.answer_kind = EVAL_ANSWER_INTEGER,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 16000,
|
|
},
|
|
{
|
|
.source = "OlympiadBench",
|
|
.id = "1750",
|
|
.domain = "Combinatorics",
|
|
.title = "Olympiad problem 1750",
|
|
.question = "A diagonal of a regular 2006-gon is called odd if its endpoints divide the boundary into two"
|
|
" parts, each composed of an odd number of sides. Sides are also regarded as odd diagonals.\n"
|
|
"\nSuppose the 2006-gon has been dissected into triangles by 2003 nonintersecting diagonals. "
|
|
"Find the maximum possible number of isosceles triangles with two odd sides.",
|
|
.answer = "1003",
|
|
.answer_kind = EVAL_ANSWER_INTEGER,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 16000,
|
|
},
|
|
{
|
|
.source = "OlympiadBench",
|
|
.id = "1766",
|
|
.domain = "Number Theory",
|
|
.title = "Olympiad problem 1766",
|
|
.question = "Determine all pairs $(x, y)$ of integers satisfying the equation\n\n$$\n1+2^{x}+2^{2 x+1}=y^"
|
|
"{2}\n$$",
|
|
.answer = "(0,2),(0,-2),(4,23),(4,-23)",
|
|
.answer_kind = EVAL_ANSWER_ORDERED_SEQUENCE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 16000,
|
|
},
|
|
{
|
|
.source = "LiveBench",
|
|
.id = "c833574b0ff6e8609bee19b36fabc48501cdcd9a64b99cd0422f54ada1acc53d",
|
|
.domain = "zebra_puzzle",
|
|
.title = "Zebra logic puzzle",
|
|
.question = "There are 4 people standing in a line. From left to right, they are numbered 1 to 4.\nEach p"
|
|
"erson has a set of attributes: Beverage, Job, Movie-Genre, Transport.\nThe attributes have t"
|
|
"he following possible values:\nBeverage: lemonade, fanta, iced-tea, hot-chocolate\nJob: soci"
|
|
"al-worker, software-developer, coach, accountant\nMovie-Genre: martial-arts, horror, documen"
|
|
"tary, satire\nTransport: skateboard, train, ship, helicopter\nEach person has a unique value"
|
|
" for each attribute.\nYou know the following about the people:\nThe person who watches docum"
|
|
"entary is somewhere between the person who watches martial-arts and the person who is a acco"
|
|
"untant\nThe person who travels by skateboard is not anywhere to the left of the person who t"
|
|
"ravels by train\nThe person who watches martial-arts and the person who drinks iced-tea have"
|
|
" the same parity positions\nThe person who watches satire is not anywhere to the right of th"
|
|
"e person who watches horror\nThe person who is a accountant and the person who drinks hot-ch"
|
|
"ocolate have the same parity positions\nThe person who travels by train and the person who d"
|
|
"rinks hot-chocolate have different parity positions\nEither the person who is a coach is the"
|
|
" same as the person who travels by helicopter or the person who is a coach is the same as th"
|
|
"e person who watches documentary, but not both\nThe person who watches satire and the person"
|
|
" who is a accountant have different parity positions\nThe person who watches satire is not a"
|
|
"nywhere to the right of the person who watches documentary\nThe person who travels by skateb"
|
|
"oard is the same as the person who watches horror or the person who drinks fanta is the same"
|
|
" as the person who travels by skateboard or both\nThe person who drinks fanta is somewhere b"
|
|
"etween the person who watches satire and the person who watches martial-arts\nThe person who"
|
|
" is a social-worker and the person who travels by helicopter have the same parity positions"
|
|
"\nThe person who is a accountant is not anywhere to the left of the person who travels by sh"
|
|
"ip\nThe person who drinks iced-tea is not anywhere to the right of the person who drinks fan"
|
|
"ta\nIn the above, 'parity position' means the evenness or oddness of the person's position i"
|
|
"n the line.\nGiven this information, answer the following questions:\nWhat is the movie genr"
|
|
"e of the person who is a software-developer?\nWhat movie genre does the person who drinks ho"
|
|
"t-chocolate watch?\nWhat movie genre does the person in position 2 watch?\nAt what position "
|
|
"is the person who watches horror?",
|
|
.answer = "satire, martial-arts, horror, 2",
|
|
.answer_kind = EVAL_ANSWER_ORDERED_SEQUENCE,
|
|
.suites = EVAL_SUITE_HARD | EVAL_SUITE_HARD_SMOKE,
|
|
.max_tokens = 8192,
|
|
},
|
|
{
|
|
.source = "LiveBench",
|
|
.id = "b7b7e331df4614775210ef5e524ed7f20c5fec6655fb0dd179c94e4d71170862",
|
|
.domain = "zebra_puzzle",
|
|
.title = "Zebra logic puzzle",
|
|
.question = "There are 3 people standing in a line. From left to right, they are numbered 1 to 3.\nEach p"
|
|
"erson has a set of attributes: Beverage, Food, Movie-Genre, Nationality.\nThe attributes hav"
|
|
"e the following possible values:\nBeverage: mirinda, cola, water\nFood: cauliflower, radish,"
|
|
" zucchini\nMovie-Genre: family, spy, crime\nNationality: brazilian, german, canadian\nEach p"
|
|
"erson has a unique value for each attribute.\nYou know the following about the people:\nThe "
|
|
"person who is brazilian is somewhere between the person who is german and the person who dri"
|
|
"nks cola\nThe person who eats cauliflower is not anywhere to the right of the person who dri"
|
|
"nks cola\nThe person who drinks mirinda is somewhere between the person who drinks cola and "
|
|
"the person who is german\nThe person who watches family and the person who eats radish have "
|
|
"the same parity positions\nThe person who watches spy is not anywhere to the right of the pe"
|
|
"rson who watches family\nEither the person who is canadian is the same as the person who wat"
|
|
"ches spy or the person who eats cauliflower is the same as the person who is canadian, but n"
|
|
"ot both\nThe person who eats cauliflower is not anywhere to the right of the person who eats"
|
|
" radish\nIn the above, 'parity position' means the evenness or oddness of the person's posit"
|
|
"ion in the line.\nGiven this information, answer the following questions:\nWhat movie genre "
|
|
"does the person in position 1 watch?\nAt what position is the person who watches crime?\nAt "
|
|
"what position is the person who eats zucchini?\nWhat movie genre does the person in position"
|
|
" 3 watch?",
|
|
.answer = "crime, 1, 2, family",
|
|
.answer_kind = EVAL_ANSWER_ORDERED_SEQUENCE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 8192,
|
|
},
|
|
{
|
|
.source = "LiveBench",
|
|
.id = "0daa7ca38beec4441b9d5c04d0b98912322926f0a3ac28a5097889d4ed83506f",
|
|
.domain = "web_of_lies_v2",
|
|
.title = "Web of lies",
|
|
.question = "In this question, assume each person either always tells the truth or always lies. Tala is a"
|
|
"t the movie theater. The person at the restaurant says the person at the aquarium lies. Ayaa"
|
|
"n is at the aquarium. Ryan is at the botanical garden. The person at the park says the perso"
|
|
"n at the art gallery lies. The person at the museum tells the truth. Zara is at the museum. "
|
|
"Jake is at the art gallery. The person at the art gallery says the person at the theater lie"
|
|
"s. Beatriz is at the park. The person at the movie theater says the person at the train stat"
|
|
"ion lies. Nadia is at the campground. The person at the campground says the person at the ar"
|
|
"t gallery tells the truth. The person at the theater lies. The person at the amusement park "
|
|
"says the person at the aquarium tells the truth. Grace is at the restaurant. The person at t"
|
|
"he aquarium thinks their friend is lying. Nia is at the theater. Kehinde is at the train sta"
|
|
"tion. The person at the theater thinks their friend is lying. The person at the botanical ga"
|
|
"rden says the person at the train station tells the truth. The person at the aquarium says t"
|
|
"he person at the campground tells the truth. The person at the aquarium saw a firetruck. The"
|
|
" person at the train station says the person at the amusement park lies. Mateo is at the amu"
|
|
"sement park. Does the person at the train station tell the truth? Does the person at the amu"
|
|
"sement park tell the truth? Does the person at the aquarium tell the truth?",
|
|
.answer = "no, yes, yes",
|
|
.answer_kind = EVAL_ANSWER_ORDERED_SEQUENCE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 8192,
|
|
},
|
|
{
|
|
.source = "LiveBench",
|
|
.id = "9ee37b9a04ab050936c86b2c5bb7abbaa0bc0e737d59a7bff9ba11e9b4069c1d",
|
|
.domain = "web_of_lies_v2",
|
|
.title = "Web of lies",
|
|
.question = "In this question, assume each person either always tells the truth or always lies. Liam is a"
|
|
"t the movie theater. The person at the beach says the person at the restaurant tells the tru"
|
|
"th. Kehinde is at the aquarium. The person at the amusement park saw a firetruck. The person"
|
|
" at the aquarium tells the truth. Luna is at the library. Jaxon is at the amusement park. Th"
|
|
"e person at the amusement park says the person at the beach tells the truth. Anika is at the"
|
|
" restaurant. The person at the barbershop tells the truth. The person at the observatory say"
|
|
"s the person at the planetarium lies. Hiroshi is at the beach. Isabella is at the planetariu"
|
|
"m. Nia is at the barbershop. The person at the movie theater says the person at the observat"
|
|
"ory lies. The person at the restaurant tells the truth. Max is at the observatory. The perso"
|
|
"n at the library says the person at the amusement park tells the truth. The person at the pl"
|
|
"anetarium says the person at the library tells the truth. Does the person at the movie theat"
|
|
"er tell the truth? Does the person at the observatory tell the truth? Does the person at the"
|
|
" planetarium tell the truth?",
|
|
.answer = "yes, no, yes",
|
|
.answer_kind = EVAL_ANSWER_ORDERED_SEQUENCE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 8192,
|
|
},
|
|
{
|
|
.source = "LiveBench",
|
|
.id = "d7071c9ff5d9779e7ab955366d0ae8db40f785aadfe7ff0b5a7ede98c05c44ea",
|
|
.domain = "web_of_lies_v2",
|
|
.title = "Web of lies",
|
|
.question = "In this question, assume each person either always tells the truth or always lies. Liam is a"
|
|
"t the restaurant. The person at the observatory says the person at the botanical garden lies"
|
|
". Elowen is at the vineyard. The person at the library says the person at the botanical gard"
|
|
"en tells the truth. The person at the train station tells the truth. The person at the botan"
|
|
"ical garden saw a firetruck. The person at the botanical garden says the person at the train"
|
|
" station lies. Jake is at the aquarium. Soren is at the farm. Theo is at the gym. The person"
|
|
" at the train station saw a firetruck. Devika is at the train station. Kehinde is at the lib"
|
|
"rary. The person at the restaurant lies. The person at the farm says the person at the train"
|
|
" station lies. The person at the gym says the person at the train station lies. Hiroshi is a"
|
|
"t the botanical garden. The person at the vineyard lies. The person at the aquarium says the"
|
|
" person at the library tells the truth. Quan is at the observatory. Does the person at the l"
|
|
"ibrary tell the truth? Does the person at the botanical garden tell the truth? Does the pers"
|
|
"on at the train station tell the truth?",
|
|
.answer = "no, no, yes",
|
|
.answer_kind = EVAL_ANSWER_ORDERED_SEQUENCE,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 8192,
|
|
},
|
|
{
|
|
.source = "NIST Juliet",
|
|
.id = "juliet-cwe190-int-multiply-01",
|
|
.domain = "CWE-190",
|
|
.title = "Integer multiplication overflow",
|
|
.question = "The following C function is intended to allocate space for count integers. Identify the line"
|
|
" containing the primary weakness.\n\n 1 int *make_array(size_t count) {\n 2 int *resul"
|
|
"t = NULL;\n 3 if (count == 0)\n 4 return NULL;\n 5 size_t bytes;\n 6 "
|
|
" if (count > SIZE_MAX / sizeof(int))\n 7 return NULL;\n 8 bytes = count * size"
|
|
"of(int);\n 9 result = malloc(bytes);\n10 if (result == NULL)\n11 return N"
|
|
"ULL;\n12 memset(result, 0, count * count * sizeof(int));\n13 return result;\n14 }",
|
|
.answer = "12",
|
|
.answer_kind = EVAL_ANSWER_LINE_SET,
|
|
.suites = EVAL_SUITE_HARD | EVAL_SUITE_HARD_SMOKE,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "NIST Juliet",
|
|
.id = "juliet-cwe121-array-index-01",
|
|
.domain = "CWE-121",
|
|
.title = "Stack array index validation",
|
|
.question = "The following C function should update one element of a fixed-size stack array. Identify the"
|
|
" smallest exact set of line numbers containing the primary weakness.\n\n 1 int update_value"
|
|
"(int index, int value) {\n 2 int data[10] = {0};\n 3 if (index < 10)\n 4 "
|
|
"data[index] = value;\n 5 return data[0];\n 6 }",
|
|
.answer = "3-4",
|
|
.answer_kind = EVAL_ANSWER_LINE_SET,
|
|
.suites = EVAL_SUITE_HARD | EVAL_SUITE_HARD_SMOKE,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "NIST Juliet",
|
|
.id = "juliet-cwe401-realloc-01",
|
|
.domain = "CWE-401",
|
|
.title = "Lost allocation after realloc",
|
|
.question = "The following C function grows an allocated buffer. Identify the line containing the primary"
|
|
" weakness.\n\n 1 char *grow(char *buf, size_t size) {\n 2 if (size > SIZE_MAX / 2)\n 3"
|
|
" return NULL;\n 4 size *= 2;\n 5 buf = realloc(buf, size);\n 6 if (b"
|
|
"uf == NULL)\n 7 return NULL;\n 8 return buf;\n 9 }",
|
|
.answer = "5",
|
|
.answer_kind = EVAL_ANSWER_LINE_SET,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "NIST Juliet",
|
|
.id = "juliet-cwe476-null-after-deref-01",
|
|
.domain = "CWE-476",
|
|
.title = "Null check after dereference",
|
|
.question = "The following C function reads the first byte of a caller-supplied buffer. Identify the line"
|
|
" containing the primary weakness.\n\n 1 int first_byte(const unsigned char *p) {\n 2 i"
|
|
"nt value = p[0];\n 3 if (p == NULL)\n 4 return -1;\n 5 return value;\n 6 "
|
|
" }",
|
|
.answer = "2",
|
|
.answer_kind = EVAL_ANSWER_LINE_SET,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 4096,
|
|
},
|
|
{
|
|
.source = "NIST Juliet",
|
|
.id = "juliet-cwe369-int-divide-01",
|
|
.domain = "CWE-369",
|
|
.title = "Unchecked integer division",
|
|
.question = "The following C function computes a ratio from caller-supplied integers. Identify the line c"
|
|
"ontaining the primary weakness.\n\n 1 int ratio(int total, int parts) {\n 2 if (total "
|
|
"< 0)\n 3 return -1;\n 4 return total / parts;\n 5 }",
|
|
.answer = "4",
|
|
.answer_kind = EVAL_ANSWER_LINE_SET,
|
|
.suites = EVAL_SUITE_HARD,
|
|
.max_tokens = 4096,
|
|
},
|
|
};
|
|
|
|
const size_t eval_hard_case_count =
|
|
sizeof(eval_hard_cases) / sizeof(eval_hard_cases[0]);
|