{ "corpus": "fresh", "provider": "claude", "model": "haiku", "cost_basis": "Claude API list price reported by the claude CLI (haiku, thinking off), grape server rerank + gate, 5 passages", "embed": "BAAI/bge-small-en-v1.5", "meta": { "date": "2026-09-30 16:26", "questions": "questions_fresh.json", "embed_key": "bge-small", "modes": [ "grape", "rag" ], "corpus_files": 12, "corpus_bytes": 632071, "chunks": 778, "settings": { "limit": 8, "snippet": 600, "top_k": 5, "chunk_chars": 1000 }, "thinking": false, "rerank": null, "gate": false, "server_rerank": true, "server_limit": 5, "judge": true, "grape_version": "0.1.0", "bench_commit": null }, "setup_seconds": { "grape": 0.04707497799972771, "grape_update": 0.08384128800025792, "rag": 97.67116754399831 }, "rows": [ { "q": "Who invented the printing press, and around what year?", "type": "fact", "expect": [ "Gutenberg", "1440" ], "grape": { "answer": "According to the excerpts, the German goldsmith Johannes Gutenberg invented the printing press around 1440, utilizing movable type and combining four existing technologies including the screw press, codex book format, and mechanized paper production.", "in": 1249, "out": 52, "cost": 0.001509, "calls": 1, "route": "direct", "coverage": 0.87, "seconds": 3.28, "api_seconds": 1.11, "correct": true }, "rag": { "answer": "Johannes Gutenberg, a German goldsmith, invented the printing press around 1440. This invention revolutionized literary production and communication by integrating existing technologies like movable type, the screw press, and mechanized paper production into a single working system.", "in": 1323, "out": 57, "cost": 0.001608, "calls": 1, "seconds": 3.13, "api_seconds": 1.24, "correct": true } }, { "q": "How many hearts does an octopus have?", "type": "number", "expect": [ [ "three", "3" ] ], "grape": { "answer": "An octopus has three hearts: a systemic or main heart that circulates blood around the body and two branchial or gill hearts that pump blood through the two gills.", "in": 1324, "out": 41, "cost": 0.001529, "calls": 1, "route": "direct", "coverage": 0.84, "seconds": 3.26, "api_seconds": 0.94, "correct": true }, "rag": { "answer": "An octopus has three hearts: a systemic or main heart that circulates blood around the body and two branchial or gill hearts that pump it through the two gills.", "in": 1473, "out": 41, "cost": 0.0016779999999999998, "calls": 1, "seconds": 2.88, "api_seconds": 1.06, "correct": true } }, { "q": "How tall is the Eiffel Tower?", "type": "number", "expect": [ "330" ], "grape": { "answer": "The Eiffel Tower is 330 metres (1,083 ft) tall, about the same height as an 81-storey building, making it the tallest structure in Paris.", "in": 1313, "out": 45, "cost": 0.0015379999999999999, "calls": 1, "route": "direct", "coverage": 1.0, "seconds": 3.43, "api_seconds": 1.0, "correct": true }, "rag": { "answer": "The Eiffel Tower is 330 metres (1,083 ft) tall, about the same height as an 81-storey building, and is the tallest structure in Paris.", "in": 1565, "out": 45, "cost": 0.00179, "calls": 1, "seconds": 3.3, "api_seconds": 1.13, "correct": true } }, { "q": "Who is the Eiffel Tower named after?", "type": "fact", "expect": [ "Gustave" ], "grape": { "answer": "The Eiffel Tower is named after the engineer Gustave Eiffel, whose company designed and built the tower from 1887 to 1889.", "in": 1238, "out": 39, "cost": 0.001433, "calls": 1, "route": "direct", "coverage": 1.0, "seconds": 3.53, "api_seconds": 0.94, "correct": true }, "rag": { "answer": "The Eiffel Tower is named after the engineer Gustave Eiffel, whose company designed and built the tower from 1887 to 1889.", "in": 1477, "out": 39, "cost": 0.001672, "calls": 1, "seconds": 6.29, "api_seconds": 4.55, "correct": true } }, { "q": "Who created the first vaccine?", "type": "fact", "expect": [ "Jenner" ], "grape": { "answer": "Edward Jenner created the first vaccine in 1798, developing the concept of vaccination and creating the first vaccine using cowpox to protect against smallpox.", "in": 1150, "out": 37, "cost": 0.001335, "calls": 1, "route": "direct", "coverage": 1.0, "seconds": 3.24, "api_seconds": 0.94, "correct": true }, "rag": { "answer": "Based on the excerpts, Edward Jenner created the first vaccine in 1798, developing cowpox as a protective inoculation against smallpox, which he described in his work \"Inquiry into the Variolae vaccinae Known as the Cow Pox.\"", "in": 1182, "out": 63, "cost": 0.001497, "calls": 1, "seconds": 2.99, "api_seconds": 1.16, "correct": true } }, { "q": "Where and when were the first modern Olympic Games held?", "type": "fact", "expect": [ "Athens", "1896" ], "grape": { "answer": "The first modern Olympic Games were held in **Athens in 1896**, hosted in the Panathenaic Stadium and bringing together 14 nations and 241 athletes to compete in 43 events.", "in": 1196, "out": 47, "cost": 0.001431, "calls": 1, "route": "direct", "coverage": 1.0, "seconds": 3.37, "api_seconds": 1.06, "correct": true }, "rag": { "answer": "The first modern Olympic Games were held in **Athens in 1896**, following Baron Pierre de Coubertin's founding of the International Olympic Committee (IOC) in 1894.", "in": 1184, "out": 42, "cost": 0.0013939999999999998, "calls": 1, "seconds": 2.92, "api_seconds": 1.08, "correct": true } }, { "q": "Who founded the International Olympic Committee?", "type": "fact", "expect": [ "Coubertin" ], "grape": { "answer": "Baron Pierre de Coubertin founded the International Olympic Committee (IOC) in 1894, inspired by attending the Olympian Games of the Wenlock Olympian Society in 1890 and aiming to promote international peace and friendship through sports.", "in": 1220, "out": 58, "cost": 0.00151, "calls": 1, "route": "direct", "coverage": 1.0, "seconds": 3.2, "api_seconds": 1.05, "correct": true }, "rag": { "answer": "Baron Pierre de Coubertin founded the International Olympic Committee (IOC) in 1894, inspired after attending the Wenlock Olympian Games and aiming to promote international peace and friendship through sports.", "in": 1226, "out": 47, "cost": 0.001461, "calls": 1, "seconds": 2.88, "api_seconds": 1.06, "correct": true } }, { "q": "How big is the Sahara in square kilometres?", "type": "number", "expect": [ [ "9,200,000", "9.2 million", "9 200 000" ] ], "grape": { "answer": "The Sahara covers 9.2 million square kilometres (3,600,000 sq mi), making it the largest hot desert in the world and the third-largest desert overall.", "in": 1335, "out": 43, "cost": 0.0015500000000000002, "calls": 1, "route": "direct", "coverage": 0.71, "seconds": 3.26, "api_seconds": 0.95, "correct": true }, "rag": { "answer": "The Sahara covers 9,200,000 square kilometres (3,600,000 sq mi), making it the largest hot desert in the world and the third-largest desert overall.", "in": 1514, "out": 44, "cost": 0.0017339999999999999, "calls": 1, "seconds": 2.71, "api_seconds": 1.0, "correct": true } }, { "q": "Who popularized the term \"Silk Road\"?", "type": "fact", "expect": [ "Richthofen" ], "grape": { "answer": "Ferdinand von Richthofen, a German scholar, first popularized the term \"Silk Road\" in 1877, deriving it from the German term Seidenstraße based on his seven expeditions to China from 1868 to 1872.", "in": 1236, "out": 61, "cost": 0.0015409999999999998, "calls": 1, "route": "direct", "coverage": 1.0, "seconds": 3.4, "api_seconds": 1.16, "correct": true }, "rag": { "answer": "According to the excerpts, the term \"Silk Road\" was first popularized in 1877 by Ferdinand von Richthofen, a German geographer who made seven expeditions to China from 1868 to 1872. However, the term did not gain widespread acceptance in academia or popularity among the public until the 20th century.", "in": 1410, "out": 78, "cost": 0.0018, "calls": 1, "seconds": 3.09, "api_seconds": 1.26, "correct": true } }, { "q": "According to Chinese legend, who first discovered tea?", "type": "paraphrase", "expect": [ "Shennong" ], "grape": { "answer": "According to Chinese legend, the mythical Shennong first discovered tea in 2737 BC, although evidence suggests that tea drinking may have been introduced from southwest China.", "in": 1351, "out": 39, "cost": 0.001546, "calls": 1, "route": "direct", "coverage": 0.71, "seconds": 3.4, "api_seconds": 1.01, "correct": true }, "rag": { "answer": "According to Chinese legend, the mythical **Shennong** first discovered tea in 2737 BC, although evidence suggests that tea drinking may have been introduced from southwest China.", "in": 1685, "out": 41, "cost": 0.0018900000000000002, "calls": 1, "seconds": 3.12, "api_seconds": 1.29, "correct": true } }, { "q": "In what year was the first direct image of a black hole published?", "type": "number", "expect": [ "2019" ], "grape": { "answer": "The first direct image of a black hole was published in **2019**. On April 10, 2019, the Event Horizon Telescope (EHT) released the first direct image of a black hole and its vicinity, showing the supermassive black hole in Messier 87's galactic center.", "in": 1178, "out": 71, "cost": 0.001533, "calls": 1, "route": "direct", "coverage": 0.89, "seconds": 3.67, "api_seconds": 1.23, "correct": true }, "rag": { "answer": "The first direct image of a black hole was published in **2019**. According to the excerpts, \"On 10 April 2019, the first direct image of a black hole and its vicinity was published, following observations made by the Event Horizon Telescope (EHT) of the supermassive black hole in Messier 87's galactic centre.\"", "in": 1427, "out": 83, "cost": 0.001842, "calls": 1, "seconds": 2.97, "api_seconds": 1.21, "correct": true } }, { "q": "What is the boundary around a black hole beyond which nothing can get out called?", "type": "paraphrase", "expect": [ "event horizon" ], "grape": { "answer": "The boundary around a black hole beyond which nothing can get out is called the **event horizon**. According to the excerpts, it is \"a boundary in spacetime through which matter and light can pass only inward towards the centre of the black hole,\" and \"nothing, not even light, can escape from inside the event horizon.\"", "in": 1139, "out": 71, "cost": 0.001494, "calls": 1, "route": "direct", "coverage": 0.54, "seconds": 3.51, "api_seconds": 1.24, "correct": true }, "rag": { "answer": "The boundary around a black hole beyond which nothing can get out is called the **event horizon**. According to the excerpts, it is defined as \"a boundary in spacetime through which matter and light can pass only inward towards the centre of the black hole,\" and \"nothing, not even light, can escape from inside the event horizon.\"", "in": 1127, "out": 73, "cost": 0.0014919999999999998, "calls": 1, "seconds": 3.15, "api_seconds": 1.4, "correct": true } }, { "q": "How many Nobel Prizes did Marie Curie win, and in which fields?", "type": "number", "expect": [ [ "two", "2" ], "Physics", "Chemistry" ], "grape": { "answer": "Marie Curie won two Nobel Prizes: the 1903 Nobel Prize in Physics (shared with her husband Pierre for their research on radioactivity) and the 1911 Nobel Prize in Chemistry (for the discovery of radium and polonium).", "in": 1157, "out": 57, "cost": 0.0014420000000000001, "calls": 1, "route": "direct", "coverage": 0.91, "seconds": 3.47, "api_seconds": 1.21, "correct": true }, "rag": { "answer": "Marie Curie won two Nobel Prizes: the Nobel Prize in Physics in 1903 (shared with Pierre Curie and Henri Becquerel) for their research on radioactivity, and the Nobel Prize in Chemistry in 1911 for her discovery and isolation of radium and polonium.", "in": 1406, "out": 68, "cost": 0.001746, "calls": 1, "seconds": 2.95, "api_seconds": 1.19, "correct": true } }, { "q": "What illness killed Marie Curie?", "type": "paraphrase", "expect": [ "aplastic" ], "grape": { "answer": "Marie Curie died in 1934 of aplastic anaemia, likely caused by exposure to radiation during her scientific research and radiological work at field hospitals during the First World War.", "in": 1324, "out": 42, "cost": 0.001534, "calls": 1, "route": "direct", "coverage": 0.72, "seconds": 3.48, "api_seconds": 0.97, "correct": true }, "rag": { "answer": "Marie Curie died from aplastic anaemia on 4 July 1934, which is believed to have been contracted from her long-term exposure to radiation that caused damage to her bone marrow.", "in": 1294, "out": 46, "cost": 0.001524, "calls": 1, "seconds": 2.96, "api_seconds": 1.03, "correct": true } }, { "q": "Which ocean does the Amazon empty into?", "type": "paraphrase", "expect": [ "Atlantic" ], "grape": { "answer": "The Amazon River empties into the Atlantic Ocean, releasing up to 300,000 cubic meters of water per second during the rainy season and accounting for about 20% of the Earth's fresh water entering the ocean.", "in": 1644, "out": 60, "cost": 0.001944, "calls": 2, "route": "route-search", "coverage": 0.34, "searched": [ [ "Amazon River ocean Atlantic Pacific" ] ], "seconds": 5.67, "api_seconds": 1.7, "correct": true }, "rag": { "answer": "The Amazon empties into the Atlantic Ocean, releasing up to 300,000 m³/s of water during the rainy season and accounting for about 20% of the Earth's fresh water entering the ocean.", "in": 1610, "out": 48, "cost": 0.00185, "calls": 1, "seconds": 2.9, "api_seconds": 1.11, "correct": true } }, { "q": "What was the last major armed revolt by former samurai?", "type": "fact", "expect": [ "Satsuma" ], "grape": { "answer": "The Satsuma Rebellion of 1877, led by Saigō Takamori, was the last major armed revolt by former samurai. Its defeat confirmed the military and political authority of the central government.", "in": 1256, "out": 50, "cost": 0.001506, "calls": 1, "route": "direct", "coverage": 1.0, "seconds": 3.26, "api_seconds": 1.1, "correct": true }, "rag": { "answer": "The last major armed revolt by former samurai was the **Satsuma Rebellion of 1877**, led by Saigō Takamori, which was the largest of several uprisings in the 1870s and whose defeat confirmed the military and political authority of the central government and marked the end of large-scale armed resistance by disaffected samurai.", "in": 1479, "out": 82, "cost": 0.001889, "calls": 1, "seconds": 3.02, "api_seconds": 1.24, "correct": true } }, { "q": "Besides swordsmanship, which martial skills were a samurai's main training?", "type": "paraphrase", "expect": [ "archery", "horse" ], "grape": { "answer": "According to the excerpts, besides swordsmanship, the primary martial skills in samurai training were **archery and horsemanship**. These three weapons-based disciplines required years of training to master and were skills that only samurai had the right to possess in Japanese history.", "in": 1242, "out": 63, "cost": 0.001557, "calls": 1, "route": "direct", "coverage": 0.72, "seconds": 3.59, "api_seconds": 1.31, "correct": true }, "rag": { "answer": "Based on the excerpts, besides swordsmanship, the samurai's main martial training included **archery and horsemanship**, as stated: \"Swordsmanship, archery, and horsemanship were the primary martial skills.\" Additionally, samurai women were trained in wielding the naginata (polearm) and tantojutsu (knife skills) for household defense.", "in": 1311, "out": 90, "cost": 0.0017610000000000002, "calls": 1, "seconds": 3.41, "api_seconds": 1.51, "correct": true } }, { "q": "Which plant are tea leaves harvested from?", "type": "paraphrase", "expect": [ "Camellia" ], "grape": { "answer": "Tea leaves are harvested from *Camellia sinensis*, an evergreen shrub native to East Asia that originated in the borderlands of southwestern China, northeast India and northern Myanmar. The plant is cultivated in various botanical varieties, with leaves being plucked by hand or harvested using shears, sickles, or machines depending on the grade and scale of production.", "in": 1241, "out": 82, "cost": 0.001651, "calls": 1, "route": "direct", "coverage": 1.0, "seconds": 3.44, "api_seconds": 1.33, "correct": true }, "rag": { "answer": "Tea leaves are harvested from the **Camellia sinensis** plant, an evergreen plant that grows mainly in tropical and subtropical climates. The plant is cultivated and pruned to waist height for ease of harvesting, with tender leaves and new shoots providing the material for tea production.", "in": 1390, "out": 65, "cost": 0.001715, "calls": 1, "seconds": 3.09, "api_seconds": 1.23, "correct": true } }, { "q": "Which came first: Marie Curie's birth or the completion of the Eiffel Tower? Give both years.", "type": "cross", "expect": [ "1867", "1889" ], "grape": { "answer": "Marie Curie's birth came first in 1867, followed by the completion of the Eiffel Tower in 1889.", "in": 1679, "out": 58, "cost": 0.0019690000000000003, "calls": 2, "route": "route-search", "coverage": 0.32, "searched": [ [ "When was Marie Curie born", "When was the Eiffel Tower completed" ] ], "seconds": 6.54, "api_seconds": 1.8, "correct": true }, "rag": { "answer": "Marie Curie's birth came first: she was born in 1867, while the Eiffel Tower's main structural work was completed in 1889.", "in": 1593, "out": 38, "cost": 0.001783, "calls": 1, "seconds": 2.92, "api_seconds": 1.11, "correct": true } }, { "q": "In which years did Gutenberg invent the printing press and the first modern Olympics take place?", "type": "cross", "expect": [ "1440", "1896" ], "grape": { "answer": "Based on the excerpts provided:\n\nJohannes Gutenberg invented the printing press around **1440**, and the first modern Olympics took place in **1896** in Athens, Greece (founded by Baron Pierre de Coubertin's International Olympic Committee in 1894).", "in": 3405, "out": 117, "cost": 0.00399, "calls": 3, "route": "answer-search+again", "coverage": 0.57, "searched": [ [ "In which years did Gutenberg invent the printing press and the first modern Olympics take place?", "Olympic Games first modern Olympics year" ], [ "In which years did Gutenberg invent the printing press and the first modern Olympics take place?", "Olympic Games first modern Olympics year", "first modern Olympic Games year when did start" ] ], "seconds": 10.33, "api_seconds": 3.07, "correct": true }, "rag": { "answer": "Not in the documents.\n\nThe excerpts provide information about when Gutenberg invented the printing press (around 1440), but they do not contain any information about when the first modern Olympics took place.", "in": 1349, "out": 45, "cost": 0.0015739999999999999, "calls": 1, "seconds": 2.96, "api_seconds": 1.14, "correct": false } }, { "q": "ऑक्टोपस के कितने दिल होते हैं?", "type": "multilingual", "expect": [ [ "3", "३", "तीन", "three" ] ], "grape": { "answer": "ऑक्टोपस के **तीन दिल** होते हैं - एक मुख्य (सिस्टमिक) दिल जो पूरे शरीर में रक्त का संचार करता है और दो क्लोमज (गलफड़े) वाले दिल जो दोनों गलफड़ों के माध्यम से रक्त को पंप करते हैं।", "in": 1991, "out": 139, "cost": 0.002686, "calls": 2, "route": "route-search", "coverage": 0.0, "searched": [ [ "Octopus hearts how many" ] ], "seconds": 7.21, "api_seconds": 2.81, "correct": true }, "rag": { "answer": "Not in the documents.", "in": 821, "out": 8, "cost": 0.000861, "calls": 1, "seconds": 2.59, "api_seconds": 0.88, "correct": false } }, { "q": "¿Quién inventó la imprenta?", "type": "multilingual", "expect": [ "Gutenberg" ], "grape": { "answer": "Johannes Gutenberg, el orfebre alemán, inventó la imprenta alrededor de 1440, utilizando tecnologías existentes como la prensa de tornillo, tipos móviles y el formato de códice.", "in": 1520, "out": 67, "cost": 0.0018550000000000003, "calls": 2, "route": "route-search", "coverage": 0.0, "searched": [ [ "Who invented the printing press?" ] ], "seconds": 5.75, "api_seconds": 1.81, "correct": true }, "rag": { "answer": "Not in the documents.", "in": 671, "out": 8, "cost": 0.000711, "calls": 1, "seconds": 2.47, "api_seconds": 0.65, "correct": false } }, { "q": "Quelle est la hauteur de la tour Eiffel ?", "type": "multilingual", "expect": [ "330" ], "grape": { "answer": "La tour Eiffel mesure 330 mètres (1 083 pieds) de hauteur, ce qui en fait la plus haute structure de Paris. Sa hauteur a changé plusieurs fois au fil des années suite à l'installation d'une antenne de télévision en 1957 qui a augmenté sa hauteur de 18,7 m, puis de 5,3 m supplémentaires en 2000.", "in": 1713, "out": 122, "cost": 0.0023230000000000004, "calls": 2, "route": "route-search", "coverage": 0.33, "searched": [ [ "What is the height of the Eiffel Tower?" ] ], "seconds": 6.7, "api_seconds": 2.51, "correct": true }, "rag": { "answer": "Not in the documents.\n\nLes extraits fournis ne contiennent pas la hauteur totale de la tour Eiffel, bien qu'ils mentionnent que la plateforme supérieure du troisième niveau se situe à 276 m (906 ft) au-dessus du sol.", "in": 1331, "out": 72, "cost": 0.001691, "calls": 1, "seconds": 3.13, "api_seconds": 1.33, "correct": false } }, { "q": "What is the capital of Australia?", "type": "out_of_scope", "expect": "NOT FOUND", "grape": { "answer": "Not in the documents.", "in": 0, "out": 0, "cost": 0, "calls": 0, "route": "refused", "coverage": 0.0, "seconds": 0.39, "api_seconds": 0.0, "correct": true }, "rag": { "answer": "Not in the documents.", "in": 1235, "out": 8, "cost": 0.001275, "calls": 1, "seconds": 3.04, "api_seconds": 0.73, "correct": true } }, { "q": "How do I bake chocolate chip cookies?", "type": "out_of_scope", "expect": "NOT FOUND", "grape": { "answer": "Not in the documents.", "in": 0, "out": 0, "cost": 0, "calls": 0, "route": "refused", "coverage": 0.0, "seconds": 0.07, "api_seconds": 0.0, "correct": true }, "rag": { "answer": "Not in the documents.\n\nThe excerpts provided contain information about tea preparation and printing presses, but do not include any information about baking chocolate chip cookies.", "in": 1090, "out": 36, "cost": 0.00127, "calls": 1, "seconds": 3.49, "api_seconds": 1.61, "correct": true } }, { "q": "What was the score of yesterday's football match?", "type": "out_of_scope", "expect": "NOT FOUND", "grape": { "answer": "Not in the documents.", "in": 0, "out": 0, "cost": 0, "calls": 0, "route": "refused", "coverage": 0.0, "seconds": 0.33, "api_seconds": 0.0, "correct": true }, "rag": { "answer": "Not in the documents.", "in": 515, "out": 8, "cost": 0.000555, "calls": 1, "seconds": 2.53, "api_seconds": 0.72, "correct": true } }, { "q": "What was Marie Curie's favourite pizza?", "type": "out_of_scope", "expect": "NOT FOUND", "grape": { "answer": "Not in the documents.", "in": 518, "out": 5, "cost": 0.000543, "calls": 1, "route": "route-none", "coverage": 0.16, "seconds": 3.02, "api_seconds": 0.65, "correct": true }, "rag": { "answer": "Not in the documents.", "in": 1691, "out": 8, "cost": 0.001731, "calls": 1, "seconds": 2.53, "api_seconds": 0.77, "correct": true } }, { "q": "Write a poem about the ocean.", "type": "out_of_scope", "expect": "NOT FOUND", "grape": { "answer": "Not in the documents.\n\nThe excerpts provided discuss poems about landmarks and historical events (the Eiffel Tower, Olympic Games, and samurai death poems) and information about the Amazon River's discharge into the ocean, but they do not contain a poem about the ocean itself that I could share with you.", "in": 1214, "out": 67, "cost": 0.001549, "calls": 1, "route": "direct", "coverage": 0.5, "seconds": 3.81, "api_seconds": 1.53, "correct": true }, "rag": { "answer": "Not in the documents.\n\nThe excerpts provided contain information about octopuses, the Amazon River, and the Sahara, but none of them contain content about writing poetry or general descriptions of the ocean that could be used to create a poem.", "in": 1209, "out": 54, "cost": 0.001479, "calls": 1, "seconds": 3.22, "api_seconds": 1.3, "correct": true } } ] }