{
 "method": "Count the tokens sent to the AI per question on LoCoMo story 1, and save the result.\n\nCompares (a) the whole story-1 conversation, written one message per line as\n\"<speaker> (on <date>): <text>\" (the same format eval_memory.py stores in plain mode), with\n(b) the memory block actually sent for each question in a graded run (\"- <note>\" per line,\nfrom that run's grades file). Counted with Anthropic's count_tokens endpoint for\nclaude-sonnet-5-5; a one-character message is counted too and subtracted, so only the text\nitself is measured.\n\n    ANTHROPIC_API_KEY=... python bench/tokens_story1.py --grades <grades.jsonl> --out <file.json>",
 "model_for_counting": "claude-sonnet-5-5",
 "grades_file": "results/locomo-story1/grades.jsonl",
 "whole_conversation": {
  "messages": 419,
  "tokens": 32264
 },
 "memory_tokens_per_question": {
  "n": 150,
  "mean": 928.0,
  "median": 925.5,
  "min": 437,
  "max": 1075
 },
 "saving_pct": {
  "mean": 97.12,
  "min": 96.67,
  "max": 98.65
 },
 "per_question": [
  {
   "id": "s1q001",
   "memories": 20,
   "tokens": 920,
   "saving_pct_vs_whole_chat": 97.15
  },
  {
   "id": "s1q000",
   "memories": 20,
   "tokens": 1043,
   "saving_pct_vs_whole_chat": 96.77
  },
  {
   "id": "s1q003",
   "memories": 20,
   "tokens": 985,
   "saving_pct_vs_whole_chat": 96.95
  },
  {
   "id": "s1q005",
   "memories": 20,
   "tokens": 803,
   "saving_pct_vs_whole_chat": 97.51
  },
  {
   "id": "s1q006",
   "memories": 20,
   "tokens": 894,
   "saving_pct_vs_whole_chat": 97.23
  },
  {
   "id": "s1q004",
   "memories": 20,
   "tokens": 993,
   "saving_pct_vs_whole_chat": 96.92
  },
  {
   "id": "s1q007",
   "memories": 20,
   "tokens": 923,
   "saving_pct_vs_whole_chat": 97.14
  },
  {
   "id": "s1q002",
   "memories": 20,
   "tokens": 956,
   "saving_pct_vs_whole_chat": 97.04
  },
  {
   "id": "s1q008",
   "memories": 20,
   "tokens": 899,
   "saving_pct_vs_whole_chat": 97.21
  },
  {
   "id": "s1q011",
   "memories": 20,
   "tokens": 963,
   "saving_pct_vs_whole_chat": 97.02
  },
  {
   "id": "s1q010",
   "memories": 20,
   "tokens": 876,
   "saving_pct_vs_whole_chat": 97.28
  },
  {
   "id": "s1q012",
   "memories": 20,
   "tokens": 861,
   "saving_pct_vs_whole_chat": 97.33
  },
  {
   "id": "s1q013",
   "memories": 20,
   "tokens": 937,
   "saving_pct_vs_whole_chat": 97.1
  },
  {
   "id": "s1q014",
   "memories": 20,
   "tokens": 931,
   "saving_pct_vs_whole_chat": 97.11
  },
  {
   "id": "s1q009",
   "memories": 20,
   "tokens": 974,
   "saving_pct_vs_whole_chat": 96.98
  },
  {
   "id": "s1q015",
   "memories": 20,
   "tokens": 918,
   "saving_pct_vs_whole_chat": 97.15
  },
  {
   "id": "s1q020",
   "memories": 20,
   "tokens": 898,
   "saving_pct_vs_whole_chat": 97.22
  },
  {
   "id": "s1q016",
   "memories": 20,
   "tokens": 893,
   "saving_pct_vs_whole_chat": 97.23
  },
  {
   "id": "s1q018",
   "memories": 20,
   "tokens": 956,
   "saving_pct_vs_whole_chat": 97.04
  },
  {
   "id": "s1q017",
   "memories": 20,
   "tokens": 966,
   "saving_pct_vs_whole_chat": 97.01
  },
  {
   "id": "s1q021",
   "memories": 20,
   "tokens": 992,
   "saving_pct_vs_whole_chat": 96.93
  },
  {
   "id": "s1q019",
   "memories": 20,
   "tokens": 919,
   "saving_pct_vs_whole_chat": 97.15
  },
  {
   "id": "s1q022",
   "memories": 20,
   "tokens": 891,
   "saving_pct_vs_whole_chat": 97.24
  },
  {
   "id": "s1q023",
   "memories": 20,
   "tokens": 933,
   "saving_pct_vs_whole_chat": 97.11
  },
  {
   "id": "s1q024",
   "memories": 20,
   "tokens": 911,
   "saving_pct_vs_whole_chat": 97.18
  },
  {
   "id": "s1q025",
   "memories": 20,
   "tokens": 1001,
   "saving_pct_vs_whole_chat": 96.9
  },
  {
   "id": "s1q027",
   "memories": 20,
   "tokens": 920,
   "saving_pct_vs_whole_chat": 97.15
  },
  {
   "id": "s1q026",
   "memories": 20,
   "tokens": 886,
   "saving_pct_vs_whole_chat": 97.25
  },
  {
   "id": "s1q028",
   "memories": 20,
   "tokens": 950,
   "saving_pct_vs_whole_chat": 97.06
  },
  {
   "id": "s1q029",
   "memories": 20,
   "tokens": 843,
   "saving_pct_vs_whole_chat": 97.39
  },
  {
   "id": "s1q031",
   "memories": 20,
   "tokens": 933,
   "saving_pct_vs_whole_chat": 97.11
  },
  {
   "id": "s1q032",
   "memories": 20,
   "tokens": 1002,
   "saving_pct_vs_whole_chat": 96.89
  },
  {
   "id": "s1q033",
   "memories": 20,
   "tokens": 943,
   "saving_pct_vs_whole_chat": 97.08
  },
  {
   "id": "s1q036",
   "memories": 20,
   "tokens": 991,
   "saving_pct_vs_whole_chat": 96.93
  },
  {
   "id": "s1q035",
   "memories": 20,
   "tokens": 914,
   "saving_pct_vs_whole_chat": 97.17
  },
  {
   "id": "s1q034",
   "memories": 20,
   "tokens": 1019,
   "saving_pct_vs_whole_chat": 96.84
  },
  {
   "id": "s1q038",
   "memories": 20,
   "tokens": 896,
   "saving_pct_vs_whole_chat": 97.22
  },
  {
   "id": "s1q039",
   "memories": 20,
   "tokens": 1056,
   "saving_pct_vs_whole_chat": 96.73
  },
  {
   "id": "s1q040",
   "memories": 20,
   "tokens": 863,
   "saving_pct_vs_whole_chat": 97.33
  },
  {
   "id": "s1q042",
   "memories": 20,
   "tokens": 927,
   "saving_pct_vs_whole_chat": 97.13
  },
  {
   "id": "s1q041",
   "memories": 20,
   "tokens": 1075,
   "saving_pct_vs_whole_chat": 96.67
  },
  {
   "id": "s1q045",
   "memories": 20,
   "tokens": 946,
   "saving_pct_vs_whole_chat": 97.07
  },
  {
   "id": "s1q044",
   "memories": 20,
   "tokens": 987,
   "saving_pct_vs_whole_chat": 96.94
  },
  {
   "id": "s1q043",
   "memories": 20,
   "tokens": 1014,
   "saving_pct_vs_whole_chat": 96.86
  },
  {
   "id": "s1q047",
   "memories": 20,
   "tokens": 941,
   "saving_pct_vs_whole_chat": 97.08
  },
  {
   "id": "s1q037",
   "memories": 20,
   "tokens": 903,
   "saving_pct_vs_whole_chat": 97.2
  },
  {
   "id": "s1q048",
   "memories": 20,
   "tokens": 915,
   "saving_pct_vs_whole_chat": 97.16
  },
  {
   "id": "s1q049",
   "memories": 20,
   "tokens": 921,
   "saving_pct_vs_whole_chat": 97.15
  },
  {
   "id": "s1q050",
   "memories": 20,
   "tokens": 954,
   "saving_pct_vs_whole_chat": 97.04
  },
  {
   "id": "s1q052",
   "memories": 20,
   "tokens": 907,
   "saving_pct_vs_whole_chat": 97.19
  },
  {
   "id": "s1q051",
   "memories": 20,
   "tokens": 883,
   "saving_pct_vs_whole_chat": 97.26
  },
  {
   "id": "s1q054",
   "memories": 20,
   "tokens": 900,
   "saving_pct_vs_whole_chat": 97.21
  },
  {
   "id": "s1q055",
   "memories": 20,
   "tokens": 882,
   "saving_pct_vs_whole_chat": 97.27
  },
  {
   "id": "s1q056",
   "memories": 20,
   "tokens": 927,
   "saving_pct_vs_whole_chat": 97.13
  },
  {
   "id": "s1q053",
   "memories": 20,
   "tokens": 903,
   "saving_pct_vs_whole_chat": 97.2
  },
  {
   "id": "s1q058",
   "memories": 20,
   "tokens": 895,
   "saving_pct_vs_whole_chat": 97.23
  },
  {
   "id": "s1q059",
   "memories": 20,
   "tokens": 921,
   "saving_pct_vs_whole_chat": 97.15
  },
  {
   "id": "s1q060",
   "memories": 20,
   "tokens": 890,
   "saving_pct_vs_whole_chat": 97.24
  },
  {
   "id": "s1q061",
   "memories": 20,
   "tokens": 873,
   "saving_pct_vs_whole_chat": 97.29
  },
  {
   "id": "s1q062",
   "memories": 20,
   "tokens": 872,
   "saving_pct_vs_whole_chat": 97.3
  },
  {
   "id": "s1q063",
   "memories": 20,
   "tokens": 1012,
   "saving_pct_vs_whole_chat": 96.86
  },
  {
   "id": "s1q064",
   "memories": 20,
   "tokens": 949,
   "saving_pct_vs_whole_chat": 97.06
  },
  {
   "id": "s1q067",
   "memories": 20,
   "tokens": 931,
   "saving_pct_vs_whole_chat": 97.11
  },
  {
   "id": "s1q065",
   "memories": 20,
   "tokens": 939,
   "saving_pct_vs_whole_chat": 97.09
  },
  {
   "id": "s1q066",
   "memories": 20,
   "tokens": 958,
   "saving_pct_vs_whole_chat": 97.03
  },
  {
   "id": "s1q068",
   "memories": 20,
   "tokens": 859,
   "saving_pct_vs_whole_chat": 97.34
  },
  {
   "id": "s1q069",
   "memories": 20,
   "tokens": 844,
   "saving_pct_vs_whole_chat": 97.38
  },
  {
   "id": "s1q070",
   "memories": 20,
   "tokens": 954,
   "saving_pct_vs_whole_chat": 97.04
  },
  {
   "id": "s1q057",
   "memories": 20,
   "tokens": 924,
   "saving_pct_vs_whole_chat": 97.14
  },
  {
   "id": "s1q071",
   "memories": 20,
   "tokens": 896,
   "saving_pct_vs_whole_chat": 97.22
  },
  {
   "id": "s1q073",
   "memories": 20,
   "tokens": 934,
   "saving_pct_vs_whole_chat": 97.11
  },
  {
   "id": "s1q072",
   "memories": 20,
   "tokens": 904,
   "saving_pct_vs_whole_chat": 97.2
  },
  {
   "id": "s1q074",
   "memories": 20,
   "tokens": 1017,
   "saving_pct_vs_whole_chat": 96.85
  },
  {
   "id": "s1q076",
   "memories": 20,
   "tokens": 947,
   "saving_pct_vs_whole_chat": 97.06
  },
  {
   "id": "s1q075",
   "memories": 20,
   "tokens": 937,
   "saving_pct_vs_whole_chat": 97.1
  },
  {
   "id": "s1q077",
   "memories": 20,
   "tokens": 854,
   "saving_pct_vs_whole_chat": 97.35
  },
  {
   "id": "s1q079",
   "memories": 20,
   "tokens": 883,
   "saving_pct_vs_whole_chat": 97.26
  },
  {
   "id": "s1q078",
   "memories": 20,
   "tokens": 904,
   "saving_pct_vs_whole_chat": 97.2
  },
  {
   "id": "s1q080",
   "memories": 20,
   "tokens": 904,
   "saving_pct_vs_whole_chat": 97.2
  },
  {
   "id": "s1q082",
   "memories": 20,
   "tokens": 1004,
   "saving_pct_vs_whole_chat": 96.89
  },
  {
   "id": "s1q081",
   "memories": 20,
   "tokens": 843,
   "saving_pct_vs_whole_chat": 97.39
  },
  {
   "id": "s1q084",
   "memories": 20,
   "tokens": 895,
   "saving_pct_vs_whole_chat": 97.23
  },
  {
   "id": "s1q083",
   "memories": 20,
   "tokens": 889,
   "saving_pct_vs_whole_chat": 97.24
  },
  {
   "id": "s1q086",
   "memories": 20,
   "tokens": 874,
   "saving_pct_vs_whole_chat": 97.29
  },
  {
   "id": "s1q085",
   "memories": 20,
   "tokens": 956,
   "saving_pct_vs_whole_chat": 97.04
  },
  {
   "id": "s1q087",
   "memories": 20,
   "tokens": 804,
   "saving_pct_vs_whole_chat": 97.51
  },
  {
   "id": "s1q089",
   "memories": 20,
   "tokens": 921,
   "saving_pct_vs_whole_chat": 97.15
  },
  {
   "id": "s1q088",
   "memories": 20,
   "tokens": 908,
   "saving_pct_vs_whole_chat": 97.19
  },
  {
   "id": "s1q090",
   "memories": 20,
   "tokens": 927,
   "saving_pct_vs_whole_chat": 97.13
  },
  {
   "id": "s1q092",
   "memories": 20,
   "tokens": 1017,
   "saving_pct_vs_whole_chat": 96.85
  },
  {
   "id": "s1q091",
   "memories": 20,
   "tokens": 960,
   "saving_pct_vs_whole_chat": 97.02
  },
  {
   "id": "s1q093",
   "memories": 20,
   "tokens": 1022,
   "saving_pct_vs_whole_chat": 96.83
  },
  {
   "id": "s1q095",
   "memories": 20,
   "tokens": 950,
   "saving_pct_vs_whole_chat": 97.06
  },
  {
   "id": "s1q096",
   "memories": 20,
   "tokens": 903,
   "saving_pct_vs_whole_chat": 97.2
  },
  {
   "id": "s1q094",
   "memories": 20,
   "tokens": 896,
   "saving_pct_vs_whole_chat": 97.22
  },
  {
   "id": "s1q097",
   "memories": 20,
   "tokens": 906,
   "saving_pct_vs_whole_chat": 97.19
  },
  {
   "id": "s1q099",
   "memories": 20,
   "tokens": 974,
   "saving_pct_vs_whole_chat": 96.98
  },
  {
   "id": "s1q100",
   "memories": 20,
   "tokens": 1013,
   "saving_pct_vs_whole_chat": 96.86
  },
  {
   "id": "s1q098",
   "memories": 20,
   "tokens": 1063,
   "saving_pct_vs_whole_chat": 96.71
  },
  {
   "id": "s1q101",
   "memories": 20,
   "tokens": 955,
   "saving_pct_vs_whole_chat": 97.04
  },
  {
   "id": "s1q102",
   "memories": 20,
   "tokens": 937,
   "saving_pct_vs_whole_chat": 97.1
  },
  {
   "id": "s1q103",
   "memories": 20,
   "tokens": 925,
   "saving_pct_vs_whole_chat": 97.13
  },
  {
   "id": "s1q104",
   "memories": 20,
   "tokens": 800,
   "saving_pct_vs_whole_chat": 97.52
  },
  {
   "id": "s1q108",
   "memories": 20,
   "tokens": 899,
   "saving_pct_vs_whole_chat": 97.21
  },
  {
   "id": "s1q106",
   "memories": 20,
   "tokens": 870,
   "saving_pct_vs_whole_chat": 97.3
  },
  {
   "id": "s1q107",
   "memories": 20,
   "tokens": 797,
   "saving_pct_vs_whole_chat": 97.53
  },
  {
   "id": "s1q109",
   "memories": 20,
   "tokens": 1000,
   "saving_pct_vs_whole_chat": 96.9
  },
  {
   "id": "s1q110",
   "memories": 20,
   "tokens": 911,
   "saving_pct_vs_whole_chat": 97.18
  },
  {
   "id": "s1q111",
   "memories": 20,
   "tokens": 1031,
   "saving_pct_vs_whole_chat": 96.8
  },
  {
   "id": "s1q105",
   "memories": 20,
   "tokens": 856,
   "saving_pct_vs_whole_chat": 97.35
  },
  {
   "id": "s1q112",
   "memories": 20,
   "tokens": 996,
   "saving_pct_vs_whole_chat": 96.91
  },
  {
   "id": "s1q114",
   "memories": 20,
   "tokens": 949,
   "saving_pct_vs_whole_chat": 97.06
  },
  {
   "id": "s1q115",
   "memories": 20,
   "tokens": 930,
   "saving_pct_vs_whole_chat": 97.12
  },
  {
   "id": "s1q113",
   "memories": 20,
   "tokens": 948,
   "saving_pct_vs_whole_chat": 97.06
  },
  {
   "id": "s1q116",
   "memories": 20,
   "tokens": 866,
   "saving_pct_vs_whole_chat": 97.32
  },
  {
   "id": "s1q118",
   "memories": 20,
   "tokens": 969,
   "saving_pct_vs_whole_chat": 97.0
  },
  {
   "id": "s1q117",
   "memories": 20,
   "tokens": 914,
   "saving_pct_vs_whole_chat": 97.17
  },
  {
   "id": "s1q121",
   "memories": 20,
   "tokens": 898,
   "saving_pct_vs_whole_chat": 97.22
  },
  {
   "id": "s1q119",
   "memories": 20,
   "tokens": 1039,
   "saving_pct_vs_whole_chat": 96.78
  },
  {
   "id": "s1q120",
   "memories": 20,
   "tokens": 996,
   "saving_pct_vs_whole_chat": 96.91
  },
  {
   "id": "s1q122",
   "memories": 20,
   "tokens": 926,
   "saving_pct_vs_whole_chat": 97.13
  },
  {
   "id": "s1q123",
   "memories": 20,
   "tokens": 916,
   "saving_pct_vs_whole_chat": 97.16
  },
  {
   "id": "s1q125",
   "memories": 9,
   "tokens": 437,
   "saving_pct_vs_whole_chat": 98.65
  },
  {
   "id": "s1q124",
   "memories": 20,
   "tokens": 908,
   "saving_pct_vs_whole_chat": 97.19
  },
  {
   "id": "s1q126",
   "memories": 20,
   "tokens": 943,
   "saving_pct_vs_whole_chat": 97.08
  },
  {
   "id": "s1q129",
   "memories": 20,
   "tokens": 997,
   "saving_pct_vs_whole_chat": 96.91
  },
  {
   "id": "s1q130",
   "memories": 20,
   "tokens": 926,
   "saving_pct_vs_whole_chat": 97.13
  },
  {
   "id": "s1q131",
   "memories": 20,
   "tokens": 874,
   "saving_pct_vs_whole_chat": 97.29
  },
  {
   "id": "s1q127",
   "memories": 20,
   "tokens": 917,
   "saving_pct_vs_whole_chat": 97.16
  },
  {
   "id": "s1q128",
   "memories": 20,
   "tokens": 912,
   "saving_pct_vs_whole_chat": 97.17
  },
  {
   "id": "s1q132",
   "memories": 20,
   "tokens": 869,
   "saving_pct_vs_whole_chat": 97.31
  },
  {
   "id": "s1q133",
   "memories": 20,
   "tokens": 878,
   "saving_pct_vs_whole_chat": 97.28
  },
  {
   "id": "s1q134",
   "memories": 20,
   "tokens": 896,
   "saving_pct_vs_whole_chat": 97.22
  },
  {
   "id": "s1q136",
   "memories": 20,
   "tokens": 967,
   "saving_pct_vs_whole_chat": 97.0
  },
  {
   "id": "s1q137",
   "memories": 20,
   "tokens": 945,
   "saving_pct_vs_whole_chat": 97.07
  },
  {
   "id": "s1q135",
   "memories": 20,
   "tokens": 935,
   "saving_pct_vs_whole_chat": 97.1
  },
  {
   "id": "s1q139",
   "memories": 20,
   "tokens": 1000,
   "saving_pct_vs_whole_chat": 96.9
  },
  {
   "id": "s1q140",
   "memories": 20,
   "tokens": 975,
   "saving_pct_vs_whole_chat": 96.98
  },
  {
   "id": "s1q138",
   "memories": 20,
   "tokens": 899,
   "saving_pct_vs_whole_chat": 97.21
  },
  {
   "id": "s1q142",
   "memories": 20,
   "tokens": 884,
   "saving_pct_vs_whole_chat": 97.26
  },
  {
   "id": "s1q141",
   "memories": 20,
   "tokens": 924,
   "saving_pct_vs_whole_chat": 97.14
  },
  {
   "id": "s1q143",
   "memories": 20,
   "tokens": 904,
   "saving_pct_vs_whole_chat": 97.2
  },
  {
   "id": "s1q144",
   "memories": 20,
   "tokens": 926,
   "saving_pct_vs_whole_chat": 97.13
  },
  {
   "id": "s1q146",
   "memories": 20,
   "tokens": 960,
   "saving_pct_vs_whole_chat": 97.02
  },
  {
   "id": "s1q148",
   "memories": 20,
   "tokens": 1020,
   "saving_pct_vs_whole_chat": 96.84
  },
  {
   "id": "s1q151",
   "memories": 20,
   "tokens": 933,
   "saving_pct_vs_whole_chat": 97.11
  },
  {
   "id": "s1q150",
   "memories": 20,
   "tokens": 896,
   "saving_pct_vs_whole_chat": 97.22
  },
  {
   "id": "s1q149",
   "memories": 20,
   "tokens": 956,
   "saving_pct_vs_whole_chat": 97.04
  },
  {
   "id": "s1q145",
   "memories": 20,
   "tokens": 988,
   "saving_pct_vs_whole_chat": 96.94
  },
  {
   "id": "s1q147",
   "memories": 20,
   "tokens": 1045,
   "saving_pct_vs_whole_chat": 96.76
  }
 ]
}