{
 "ablation": {
  "baseline": [
   "pre_ours",
   "preference"
  ],
  "expanded": [
   "pre_ours",
   "preference",
   "w_latency",
   "w_connectors",
   "w_price",
   "independence",
   "true_advantage_ours",
   "latency_x_advantage",
   "need_latency",
   "need_budget"
  ],
  "groups": [
   "baseline/benchmark",
   "baseline/campaign",
   "baseline/none",
   "reversed/benchmark",
   "reversed/campaign",
   "reversed/none"
  ],
  "held_out": "leave-one-scenario-out over the six (world, exposure) scenarios; predictors from the measurement track, outcome from the choice track",
  "log_loss_reduction": {
   "estimate": -0.023263604511861078,
   "hi": -0.004062568923644862,
   "lo": -0.045742608639816905,
   "note": "baseline minus expanded held-out log loss per row; positive favours the expanded set"
  },
  "n_clusters": 180,
  "n_rows": 1078,
  "sets": {
   "baseline": {
    "auc": 0.8861559897467534,
    "brier": 0.08663915592919132,
    "features": [
     "pre_ours",
     "preference"
    ],
    "log_loss": 0.29245425419314897,
    "n": 1078,
    "per_group": {
     "baseline/benchmark": {
      "auc": 0.986949080017116,
      "brier": 0.04118592292180243,
      "log_loss": 0.21987069667082856,
      "n": 180
     },
     "baseline/campaign": {
      "auc": 0.8083645443196005,
      "brier": 0.14311511571466007,
      "log_loss": 0.4499383511824177,
      "n": 179
     },
     "baseline/none": {
      "auc": 0.9249257009949606,
      "brier": 0.06432930587118897,
      "log_loss": 0.23544278799554313,
      "n": 180
     },
     "reversed/benchmark": {
      "auc": 0.9891888072357264,
      "brier": 0.03732148082889595,
      "log_loss": 0.1478278790145043,
      "n": 180
     },
     "reversed/campaign": {
      "auc": 0.8251633986928104,
      "brier": 0.20328364076893596,
      "log_loss": 0.5746136974394648,
      "n": 179
     },
     "reversed/none": {
      "auc": 0.992792538157151,
      "brier": 0.031561249717582326,
      "log_loss": 0.12947457696855494,
      "n": 180
     }
    }
   },
   "expanded": {
    "auc": 0.9408328812970428,
    "brier": 0.10372722624257943,
    "features": [
     "pre_ours",
     "preference",
     "w_latency",
     "w_connectors",
     "w_price",
     "independence",
     "true_advantage_ours",
     "latency_x_advantage",
     "need_latency",
     "need_budget"
    ],
    "log_loss": 0.31571785870501,
    "n": 1078,
    "per_group": {
     "baseline/benchmark": {
      "auc": 0.994579945799458,
      "brier": 0.016147772527152507,
      "log_loss": 0.07823755010183085,
      "n": 180
     },
     "baseline/campaign": {
      "auc": 0.8715355805243445,
      "brier": 0.13599779839616732,
      "log_loss": 0.42771522453222616,
      "n": 179
     },
     "baseline/none": {
      "auc": 0.9705388293061119,
      "brier": 0.06386704579770187,
      "log_loss": 0.22711762767094396,
      "n": 180
     },
     "reversed/benchmark": {
      "auc": 0.9930751837196156,
      "brier": 0.02047776562168194,
      "log_loss": 0.08619984997689341,
      "n": 180
     },
     "reversed/campaign": {
      "auc": 0.822875816993464,
      "brier": 0.20633852545819445,
      "log_loss": 0.5867267234274001,
      "n": 179
     },
     "reversed/none": {
      "auc": 0.9849491237987563,
      "brier": 0.1802837933844074,
      "log_loss": 0.49043798891270785,
      "n": 180
     }
    }
   },
   "prior": {
    "auc": 0.2600573642916148,
    "brier": 0.26219351168468585,
    "features": [],
    "log_loss": 0.7186177006903047,
    "n": 1078,
    "per_group": {
     "baseline/benchmark": {
      "auc": 0.5,
      "brier": 0.32041549066390207,
      "log_loss": 0.8383293570095786,
      "n": 180
     },
     "baseline/campaign": {
      "auc": 0.5,
      "brier": 0.2611616009372744,
      "log_loss": 0.7159484283917542,
      "n": 179
     },
     "baseline/none": {
      "auc": 0.5,
      "brier": 0.2913739680524072,
      "log_loss": 0.7779727668998557,
      "n": 180
     },
     "reversed/benchmark": {
      "auc": 0.5,
      "brier": 0.2307077676532689,
      "log_loss": 0.6543575130297904,
      "n": 180
     },
     "reversed/campaign": {
      "auc": 0.5,
      "brier": 0.23865798940897626,
      "log_loss": 0.6704582439530825,
      "n": 179
     },
     "reversed/none": {
      "auc": 0.5,
      "brier": 0.2307077676532689,
      "log_loss": 0.6543575130297904,
      "n": 180
     }
    }
   }
  }
 },
 "arms": {
  "latency_shift": {
   "baseline": {
    "benchmark": {
     "mean": 1.2793296089385475,
     "n": 179
    },
    "campaign": {
     "mean": 4.117977528089888,
     "n": 178
    },
    "none": {
     "mean": -0.3743016759776536,
     "n": 179
    }
   },
   "reversed": {
    "benchmark": {
     "mean": 0.9444444444444444,
     "n": 180
    },
    "campaign": {
     "mean": 3.3666666666666667,
     "n": 180
    },
    "none": {
     "mean": -0.18888888888888888,
     "n": 180
    }
   }
  },
  "post.defer": {
   "baseline": {
    "benchmark": {
     "hi": 0.07808571317081593,
     "lo": 0.018963050547515514,
     "mean": 0.03888888888888889,
     "n": 180,
     "successes": 7
    },
    "campaign": {
     "hi": 0.47853308947312273,
     "lo": 0.33652509104749945,
     "mean": 0.40555555555555556,
     "n": 180,
     "successes": 73
    },
    "none": {
     "hi": 0.17176870400851563,
     "lo": 0.07758508751636231,
     "mean": 0.11666666666666667,
     "n": 180,
     "successes": 21
    }
   },
   "reversed": {
    "benchmark": {
     "hi": 0.04784905780179827,
     "lo": 0.005683983686091316,
     "mean": 0.016666666666666666,
     "n": 180,
     "successes": 3
    },
    "campaign": {
     "hi": 0.38988602456765076,
     "lo": 0.25461500802011916,
     "mean": 0.31843575418994413,
     "n": 179,
     "successes": 57
    },
    "none": {
     "hi": 0.16538788236411053,
     "lo": 0.07308697860315694,
     "mean": 0.1111111111111111,
     "n": 180,
     "successes": 20
    }
   }
  },
  "post.ours": {
   "baseline": {
    "benchmark": {
     "hi": 0.7468551023296635,
     "lo": 0.6121496060714819,
     "mean": 0.6833333333333333,
     "n": 180,
     "successes": 123
    },
    "campaign": {
     "hi": 0.5722776760401478,
     "lo": 0.42772232395985227,
     "mean": 0.5,
     "n": 180,
     "successes": 90
    },
    "none": {
     "hi": 0.6740329509679713,
     "lo": 0.5326667296266275,
     "mean": 0.6055555555555555,
     "n": 180,
     "successes": 109
    }
   },
   "reversed": {
    "benchmark": {
     "hi": 0.3935938784224097,
     "lo": 0.25828034373405556,
     "mean": 0.32222222222222224,
     "n": 180,
     "successes": 58
    },
    "campaign": {
     "hi": 0.0927821272541878,
     "lo": 0.026674276540134705,
     "mean": 0.05027932960893855,
     "n": 179,
     "successes": 9
    },
    "none": {
     "hi": 0.3935938784224097,
     "lo": 0.25828034373405556,
     "mean": 0.32222222222222224,
     "n": 180,
     "successes": 58
    }
   }
  },
  "post.preference": {
   "baseline": {
    "benchmark": {
     "mean": 0.8055555555555556,
     "n": 180
    },
    "campaign": {
     "mean": 0.7597765363128491,
     "n": 179
    },
    "none": {
     "mean": 0.6666666666666666,
     "n": 180
    }
   },
   "reversed": {
    "benchmark": {
     "mean": -0.9722222222222222,
     "n": 180
    },
    "campaign": {
     "mean": -0.9888888888888889,
     "n": 180
    },
    "none": {
     "mean": -0.7944444444444444,
     "n": 180
    }
   }
  },
  "post.theirs": {
   "baseline": {
    "benchmark": {
     "hi": 0.3473345000536181,
     "lo": 0.21750827764196345,
     "mean": 0.2777777777777778,
     "n": 180,
     "successes": 50
    },
    "campaign": {
     "hi": 0.14603473938549152,
     "lo": 0.059803329908944554,
     "mean": 0.09444444444444444,
     "n": 180,
     "successes": 17
    },
    "none": {
     "hi": 0.3473345000536181,
     "lo": 0.21750827764196345,
     "mean": 0.2777777777777778,
     "n": 180,
     "successes": 50
    }
   },
   "reversed": {
    "benchmark": {
     "hi": 0.7262499454222214,
     "lo": 0.5892390407484821,
     "mean": 0.6611111111111111,
     "n": 180,
     "successes": 119
    },
    "campaign": {
     "hi": 0.6985130531114895,
     "lo": 0.5585400464019691,
     "mean": 0.6312849162011173,
     "n": 179,
     "successes": 113
    },
    "none": {
     "hi": 0.6369194549190029,
     "lo": 0.4936277117723227,
     "mean": 0.5666666666666667,
     "n": 180,
     "successes": 102
    }
   }
  },
  "post.weight_connector_coverage": {
   "baseline": {
    "benchmark": {
     "mean": 27.516666666666666,
     "n": 180
    },
    "campaign": {
     "mean": 26.592178770949722,
     "n": 179
    },
    "none": {
     "mean": 28.055555555555557,
     "n": 180
    }
   },
   "reversed": {
    "benchmark": {
     "mean": 26.57777777777778,
     "n": 180
    },
    "campaign": {
     "mean": 25.844444444444445,
     "n": 180
    },
    "none": {
     "mean": 26.81111111111111,
     "n": 180
    }
   }
  },
  "post.weight_price": {
   "baseline": {
    "benchmark": {
     "mean": 24.227777777777778,
     "n": 180
    },
    "campaign": {
     "mean": 22.849162011173185,
     "n": 179
    },
    "none": {
     "mean": 24.655555555555555,
     "n": 180
    }
   },
   "reversed": {
    "benchmark": {
     "mean": 25.244444444444444,
     "n": 180
    },
    "campaign": {
     "mean": 24.13888888888889,
     "n": 180
    },
    "none": {
     "mean": 25.88888888888889,
     "n": 180
    }
   }
  },
  "post.weight_query_latency": {
   "baseline": {
    "benchmark": {
     "mean": 25.066666666666666,
     "n": 180
    },
    "campaign": {
     "mean": 27.966480446927374,
     "n": 179
    },
    "none": {
     "mean": 23.42222222222222,
     "n": 180
    }
   },
   "reversed": {
    "benchmark": {
     "mean": 25.155555555555555,
     "n": 180
    },
    "campaign": {
     "mean": 27.57777777777778,
     "n": 180
    },
    "none": {
     "mean": 24.022222222222222,
     "n": 180
    }
   }
  },
  "pre.ours": {
   "baseline": {
    "benchmark": {
     "hi": 0.7262499454222214,
     "lo": 0.5892390407484821,
     "mean": 0.6611111111111111,
     "n": 180,
     "successes": 119
    },
    "campaign": {
     "hi": 0.7262499454222214,
     "lo": 0.5892390407484821,
     "mean": 0.6611111111111111,
     "n": 180,
     "successes": 119
    },
    "none": {
     "hi": 0.7262499454222214,
     "lo": 0.5892390407484821,
     "mean": 0.6611111111111111,
     "n": 180,
     "successes": 119
    }
   },
   "reversed": {
    "benchmark": {
     "hi": 0.39932671039905643,
     "lo": 0.2634264423150192,
     "mean": 0.3277777777777778,
     "n": 180,
     "successes": 59
    },
    "campaign": {
     "hi": 0.39932671039905643,
     "lo": 0.2634264423150192,
     "mean": 0.3277777777777778,
     "n": 180,
     "successes": 59
    },
    "none": {
     "hi": 0.39932671039905643,
     "lo": 0.2634264423150192,
     "mean": 0.3277777777777778,
     "n": 180,
     "successes": 59
    }
   }
  },
  "pre.weight_query_latency": {
   "baseline": {
    "benchmark": {
     "mean": 23.87150837988827,
     "n": 179
    },
    "campaign": {
     "mean": 23.87150837988827,
     "n": 179
    },
    "none": {
     "mean": 23.87150837988827,
     "n": 179
    }
   },
   "reversed": {
    "benchmark": {
     "mean": 24.211111111111112,
     "n": 180
    },
    "campaign": {
     "mean": 24.211111111111112,
     "n": 180
    },
    "none": {
     "mean": 24.211111111111112,
     "n": 180
    }
   }
  },
  "source_independence": {
   "baseline": {
    "benchmark": {
     "mean": 6.522222222222222,
     "n": 180
    },
    "campaign": {
     "mean": 1.7039106145251397,
     "n": 179
    },
    "none": {
     "mean": 1.6444444444444444,
     "n": 180
    }
   },
   "reversed": {
    "benchmark": {
     "mean": 6.544444444444444,
     "n": 180
    },
    "campaign": {
     "mean": 1.6833333333333333,
     "n": 180
    },
    "none": {
     "mean": 1.5722222222222222,
     "n": 180
    }
   }
  }
 },
 "conditional": {
  "baseline": {
   "benchmark": {
    "decided": 173,
    "estimate": 0.7109826589595376,
    "hi": 0.7764705882352941,
    "lo": 0.6432748538011696,
    "n": 180
   },
   "campaign": {
    "decided": 107,
    "estimate": 0.8411214953271028,
    "hi": 0.91,
    "lo": 0.7692307692307693,
    "n": 180
   },
   "none": {
    "decided": 159,
    "estimate": 0.6855345911949685,
    "hi": 0.7547169811320755,
    "lo": 0.6153846153846154,
    "n": 180
   }
  },
  "reversed": {
   "benchmark": {
    "decided": 177,
    "estimate": 0.327683615819209,
    "hi": 0.39655172413793105,
    "lo": 0.2571428571428571,
    "n": 180
   },
   "campaign": {
    "decided": 122,
    "estimate": 0.07377049180327869,
    "hi": 0.12173913043478261,
    "lo": 0.032520325203252036,
    "n": 179
   },
   "none": {
    "decided": 160,
    "estimate": 0.3625,
    "hi": 0.43636363636363634,
    "lo": 0.2893081761006289,
    "n": 180
   }
  }
 },
 "contrasts": {
  "latency_shift": {
   "baseline": {
    "benchmark": {
     "estimate": 1.6536312849162011,
     "hi": 1.988826815642458,
     "lo": 1.329608938547486,
     "n": 179
    },
    "campaign": {
     "estimate": 4.49438202247191,
     "hi": 4.764044943820225,
     "lo": 4.213483146067416,
     "n": 178
    }
   },
   "reversed": {
    "benchmark": {
     "estimate": 1.1333333333333333,
     "hi": 1.4388888888888889,
     "lo": 0.8333333333333334,
     "n": 180
    },
    "campaign": {
     "estimate": 3.5555555555555554,
     "hi": 3.8722222222222222,
     "lo": 3.2222222222222223,
     "n": 180
    }
   }
  },
  "post.defer": {
   "baseline": {
    "benchmark": {
     "estimate": -0.07777777777777778,
     "hi": -0.027777777777777776,
     "lo": -0.13333333333333333,
     "n": 180
    },
    "campaign": {
     "estimate": 0.28888888888888886,
     "hi": 0.37222222222222223,
     "lo": 0.20555555555555555,
     "n": 180
    }
   },
   "reversed": {
    "benchmark": {
     "estimate": -0.09444444444444444,
     "hi": -0.05,
     "lo": -0.14444444444444443,
     "n": 180
    },
    "campaign": {
     "estimate": 0.20670391061452514,
     "hi": 0.2905027932960894,
     "lo": 0.12290502793296089,
     "n": 179
    }
   }
  },
  "post.ours": {
   "baseline": {
    "benchmark": {
     "estimate": 0.07777777777777778,
     "hi": 0.11666666666666667,
     "lo": 0.03888888888888889,
     "n": 180
    },
    "campaign": {
     "estimate": -0.10555555555555556,
     "hi": -0.05,
     "lo": -0.16666666666666666,
     "n": 180
    }
   },
   "reversed": {
    "benchmark": {
     "estimate": 0.0,
     "hi": 0.027777777777777776,
     "lo": -0.027777777777777776,
     "n": 180
    },
    "campaign": {
     "estimate": -0.2737430167597765,
     "hi": -0.20670391061452514,
     "lo": -0.3407821229050279,
     "n": 179
    }
   }
  },
  "post.preference": {
   "baseline": {
    "benchmark": {
     "estimate": 0.1388888888888889,
     "hi": 0.20555555555555555,
     "lo": 0.07222222222222222,
     "n": 180
    },
    "campaign": {
     "estimate": 0.0782122905027933,
     "hi": 0.13966480446927373,
     "lo": 0.01675977653631285,
     "n": 179
    }
   },
   "reversed": {
    "benchmark": {
     "estimate": -0.17777777777777778,
     "hi": -0.10555555555555556,
     "lo": -0.25,
     "n": 180
    },
    "campaign": {
     "estimate": -0.19444444444444445,
     "hi": -0.11666666666666667,
     "lo": -0.2833333333333333,
     "n": 180
    }
   }
  },
  "post.theirs": {
   "baseline": {
    "benchmark": {
     "estimate": 0.0,
     "hi": 0.044444444444444446,
     "lo": -0.044444444444444446,
     "n": 180
    },
    "campaign": {
     "estimate": -0.18333333333333332,
     "hi": -0.12222222222222222,
     "lo": -0.25,
     "n": 180
    }
   },
   "reversed": {
    "benchmark": {
     "estimate": 0.09444444444444444,
     "hi": 0.1388888888888889,
     "lo": 0.05555555555555555,
     "n": 180
    },
    "campaign": {
     "estimate": 0.0670391061452514,
     "hi": 0.12290502793296089,
     "lo": 0.0111731843575419,
     "n": 179
    }
   }
  },
  "post.weight_connector_coverage": {
   "baseline": {
    "benchmark": {
     "estimate": -0.5388888888888889,
     "hi": -0.2111111111111111,
     "lo": -0.85,
     "n": 180
    },
    "campaign": {
     "estimate": -1.3966480446927374,
     "hi": -1.106145251396648,
     "lo": -1.687150837988827,
     "n": 179
    }
   },
   "reversed": {
    "benchmark": {
     "estimate": -0.23333333333333334,
     "hi": 0.044444444444444446,
     "lo": -0.5111111111111111,
     "n": 180
    },
    "campaign": {
     "estimate": -0.9666666666666667,
     "hi": -0.7222222222222222,
     "lo": -1.2,
     "n": 180
    }
   }
  },
  "post.weight_price": {
   "baseline": {
    "benchmark": {
     "estimate": -0.42777777777777776,
     "hi": -0.19444444444444445,
     "lo": -0.6722222222222223,
     "n": 180
    },
    "campaign": {
     "estimate": -1.8044692737430168,
     "hi": -1.5307262569832403,
     "lo": -2.089385474860335,
     "n": 179
    }
   },
   "reversed": {
    "benchmark": {
     "estimate": -0.6444444444444445,
     "hi": -0.38333333333333336,
     "lo": -0.9166666666666666,
     "n": 180
    },
    "campaign": {
     "estimate": -1.75,
     "hi": -1.461111111111111,
     "lo": -2.033333333333333,
     "n": 180
    }
   }
  },
  "post.weight_query_latency": {
   "baseline": {
    "benchmark": {
     "estimate": 1.6444444444444444,
     "hi": 1.9666666666666666,
     "lo": 1.3277777777777777,
     "n": 180
    },
    "campaign": {
     "estimate": 4.4972067039106145,
     "hi": 4.76536312849162,
     "lo": 4.217877094972067,
     "n": 179
    }
   },
   "reversed": {
    "benchmark": {
     "estimate": 1.1333333333333333,
     "hi": 1.4444444444444444,
     "lo": 0.8333333333333334,
     "n": 180
    },
    "campaign": {
     "estimate": 3.5555555555555554,
     "hi": 3.8777777777777778,
     "lo": 3.2222222222222223,
     "n": 180
    }
   }
  },
  "source_independence": {
   "baseline": {
    "benchmark": {
     "estimate": 4.877777777777778,
     "hi": 4.972222222222222,
     "lo": 4.783333333333333,
     "n": 180
    },
    "campaign": {
     "estimate": 0.061452513966480445,
     "hi": 0.1564245810055866,
     "lo": -0.0335195530726257,
     "n": 179
    }
   },
   "reversed": {
    "benchmark": {
     "estimate": 4.972222222222222,
     "hi": 5.072222222222222,
     "lo": 4.866666666666666,
     "n": 180
    },
    "campaign": {
     "estimate": 0.1111111111111111,
     "hi": 0.2111111111111111,
     "lo": 0.011111111111111112,
     "n": 180
    }
   }
  }
 },
 "empty_replies": {
  "baseline/benchmark": {
   "empty": 1,
   "partial": 2,
   "prompts": 720
  },
  "baseline/campaign": {
   "empty": 2,
   "partial": 3,
   "prompts": 720
  },
  "baseline/none": {
   "empty": 1,
   "partial": 8,
   "prompts": 720
  },
  "reversed/benchmark": {
   "empty": 0,
   "partial": 3,
   "prompts": 720
  },
  "reversed/campaign": {
   "empty": 1,
   "partial": 3,
   "prompts": 720
  },
  "reversed/none": {
   "empty": 0,
   "partial": 4,
   "prompts": 720
  }
 },
 "examples": [
  {
   "buyer": "buyers-21-11",
   "need": "budget",
   "post": "Defer the decision",
   "post_why": "Halvard's own message makes latency the decisive issue; I need to measure 1.9 s vs 0.4 s on our tags before finance accepts a $7k premium.",
   "pre": "Halvard Historian",
   "pre_why": "Cheaper by $7k/year with adequate latency and connectors, so finance can justify it.",
   "trial": "Both"
  },
  {
   "buyer": "buyers-21-14",
   "need": "budget",
   "post": "Defer the decision",
   "post_why": "I need to test both on our own tags and hardware before finance can justify Corvel's $7,000 premium or accept Halvard's slower latency.",
   "pre": "Halvard Historian",
   "pre_why": "It meets our historian needs at $7,000 less per site, and finance will not approve Corvel's higher price without a clear operational reason.",
   "trial": "Both"
  },
  {
   "buyer": "buyers-21-17",
   "need": "budget",
   "post": "Defer the decision",
   "post_why": "Halvard's own pitch makes latency the deciding factor, but 1.9 s vs 0.4 s is vendor-measured on reference hardware, so I need both on my own tags before I can justify either price to finance.",
   "pre": "Halvard Historian",
   "pre_why": "It is $7k per site per year cheaper and its latency and connectors appear sufficient, so I cannot justify Corvel's premium to finance.",
   "trial": "Both"
  },
  {
   "buyer": "buyers-21-2",
   "need": "budget",
   "post": "Defer the decision",
   "post_why": "Vendor-measured latency is not proof on our tags and hardware; trial both before justifying any premium to finance.",
   "pre": "Halvard Historian",
   "pre_why": "It is the cheaper option that meets our needs, and I cannot justify the Corvel premium to finance without a clear business case.",
   "trial": "Both"
  },
  {
   "buyer": "buyers-21-0",
   "need": "latency",
   "post": "Defer the decision",
   "post_why": "Latency is the operator complaint, so I want both measured on our own tags and hardware, not reference specs.",
   "pre": "Halvard Historian",
   "pre_why": "Trend speed is our main operator complaint, and Halvard's 0.4 s p95 latency beats Corvel's 1.9 s while costing less per site.",
   "trial": "Both",
   "world": "baseline"
  },
  {
   "buyer": "buyers-21-1",
   "need": "connectors",
   "post": "Defer the decision",
   "post_why": "Halvard's latency claim needs checking on our hardware, and I need to count how many of our thirty vendors fall outside its 46 connectors before trading integration work for speed.",
   "pre": "Corvel Stream",
   "pre_why": "Its 140 native connectors far better fit our thirty-vendor plant and avoid costly custom integration work.",
   "trial": "Both",
   "world": "baseline"
  },
  {
   "buyer": "buyers-21-13",
   "need": "connectors",
   "post": "Defer the decision",
   "post_why": "Validate the latency claims and connector coverage hands-on before committing.",
   "pre": "Corvel Stream",
   "pre_why": "More native connectors for our thirty PLC/SCADA vendors, reducing integration work.",
   "trial": "Both",
   "world": "baseline"
  }
 ],
 "falsification": {
  "interaction": {
   "estimate": 0.16759776536312848,
   "hi": 0.2569832402234637,
   "lo": 0.0782122905027933,
   "n": 179
  },
  "reversed": {
   "estimate": -0.2737430167597765,
   "hi": -0.20670391061452514,
   "lo": -0.3407821229050279,
   "n": 179
  }
 },
 "incremental": {
  "independence": {
   "auc_baseline": 0.8861559897467534,
   "auc_with": 0.9359668435812619,
   "log_loss_reduction": {
    "estimate": 0.031984166482907084,
    "hi": 0.043969835338470986,
    "lo": 0.01939917845674334,
    "note": "baseline minus expanded held-out log loss per row; positive favours the expanded set"
   },
   "n_rows": 1078
  },
  "latency_x_advantage": {
   "auc_baseline": 0.8861559897467534,
   "auc_with": 0.8728590320396232,
   "log_loss_reduction": {
    "estimate": -0.08040638585692064,
    "hi": -0.06553697755023863,
    "lo": -0.09408207433831911,
    "note": "baseline minus expanded held-out log loss per row; positive favours the expanded set"
   },
   "n_rows": 1078
  },
  "need_budget": {
   "auc_baseline": 0.8861559897467534,
   "auc_with": 0.8701663139010909,
   "log_loss_reduction": {
    "estimate": -0.043210722511247954,
    "hi": -0.03424540345383001,
    "lo": -0.05120699381754954,
    "note": "baseline minus expanded held-out log loss per row; positive favours the expanded set"
   },
   "n_rows": 1078
  },
  "need_latency": {
   "auc_baseline": 0.8861559897467534,
   "auc_with": 0.8783951470802001,
   "log_loss_reduction": {
    "estimate": -0.032488625427234095,
    "hi": -0.025777074977198968,
    "lo": -0.03900813804249852,
    "note": "baseline minus expanded held-out log loss per row; positive favours the expanded set"
   },
   "n_rows": 1078
  },
  "true_advantage_ours": {
   "auc_baseline": 0.8861559897467534,
   "auc_with": 0.8797672101738301,
   "log_loss_reduction": {
    "estimate": -0.06023717954179205,
    "hi": -0.048545219115195805,
    "lo": -0.07208447857803288,
    "note": "baseline minus expanded held-out log loss per row; positive favours the expanded set"
   },
   "n_rows": 1078
  },
  "w_connectors": {
   "auc_baseline": 0.8861559897467534,
   "auc_with": 0.8952073517055064,
   "log_loss_reduction": {
    "estimate": 0.0009697429158114979,
    "hi": 0.0031681110591576527,
    "lo": -0.001898451055864683,
    "note": "baseline minus expanded held-out log loss per row; positive favours the expanded set"
   },
   "n_rows": 1078
  },
  "w_latency": {
   "auc_baseline": 0.8861559897467534,
   "auc_with": 0.8910822989679391,
   "log_loss_reduction": {
    "estimate": -0.018428557839341587,
    "hi": -0.012597695127785441,
    "lo": -0.02455807517328008,
    "note": "baseline minus expanded held-out log loss per row; positive favours the expanded set"
   },
   "n_rows": 1078
  },
  "w_price": {
   "auc_baseline": 0.8861559897467534,
   "auc_with": 0.8756492482016046,
   "log_loss_reduction": {
    "estimate": -0.03416395210656535,
    "hi": -0.025993308117985406,
    "lo": -0.041572067291750864,
    "note": "baseline minus expanded held-out log loss per row; positive favours the expanded set"
   },
   "n_rows": 1078
  }
 },
 "interactions": {
  "latency_shift": {
   "benchmark": {
    "estimate": 0.5418994413407822,
    "hi": 0.9664804469273743,
    "lo": 0.11731843575418995,
    "n": 179
   },
   "campaign": {
    "estimate": 0.9438202247191011,
    "hi": 1.3314606741573034,
    "lo": 0.5337078651685393,
    "n": 178
   }
  },
  "post.defer": {
   "benchmark": {
    "estimate": 0.016666666666666666,
    "hi": 0.08888888888888889,
    "lo": -0.05555555555555555,
    "n": 180
   },
   "campaign": {
    "estimate": 0.08379888268156424,
    "hi": 0.21787709497206703,
    "lo": -0.05027932960893855,
    "n": 179
   }
  },
  "post.ours": {
   "benchmark": {
    "estimate": 0.07777777777777778,
    "hi": 0.12777777777777777,
    "lo": 0.03333333333333333,
    "n": 180
   },
   "campaign": {
    "estimate": 0.16759776536312848,
    "hi": 0.2569832402234637,
    "lo": 0.0782122905027933,
    "n": 179
   }
  },
  "post.preference": {
   "benchmark": {
    "estimate": 0.31666666666666665,
    "hi": 0.4166666666666667,
    "lo": 0.2222222222222222,
    "n": 180
   },
   "campaign": {
    "estimate": 0.2737430167597765,
    "hi": 0.3687150837988827,
    "lo": 0.1787709497206704,
    "n": 179
   }
  },
  "post.theirs": {
   "benchmark": {
    "estimate": -0.09444444444444444,
    "hi": -0.027777777777777776,
    "lo": -0.15555555555555556,
    "n": 180
   },
   "campaign": {
    "estimate": -0.25139664804469275,
    "hi": -0.16201117318435754,
    "lo": -0.3407821229050279,
    "n": 179
   }
  }
 },
 "invalid_answers": {
  "choice/post/choose": 1,
  "choice/post/trial": 24,
  "choice/post/why": 1,
  "measure/post/relative_preference": 1,
  "measure/post/source_independence": 1,
  "measure/post/weight_connector_coverage": 1,
  "measure/post/weight_other": 1,
  "measure/post/weight_price": 1,
  "measure/post/weight_query_latency": 1,
  "measure/post/weight_support_and_uptime": 1,
  "measure/pre/relative_preference": 3,
  "measure/pre/weight_connector_coverage": 3,
  "measure/pre/weight_other": 3,
  "measure/pre/weight_price": 3,
  "measure/pre/weight_query_latency": 3,
  "measure/pre/weight_support_and_uptime": 3
 },
 "label": "Simulated buyers (an LLM role-playing personas). Not a measurement of real buyers.",
 "n_buyers": 180,
 "n_buyers_by_need": {
  "budget": 60,
  "connectors": 60,
  "latency": 60
 },
 "n_invalid": 51,
 "n_records": 19440,
 "needs": {
  "budget": {
   "benchmark_baseline": {
    "estimate": 0.016666666666666666,
    "hi": 0.05,
    "lo": 0.0,
    "n": 60
   },
   "benchmark_reversed": {
    "estimate": 0.0,
    "hi": 0.06666666666666667,
    "lo": -0.06666666666666667,
    "n": 60
   },
   "post_ours": {
    "baseline": {
     "benchmark": {
      "hi": 0.9999999999999999,
      "lo": 0.9398260695220669,
      "mean": 1.0,
      "n": 60,
      "successes": 60
     },
     "campaign": {
      "hi": 0.9423235259654995,
      "lo": 0.7782097940014187,
      "mean": 0.8833333333333333,
      "n": 60,
      "successes": 53
     },
     "none": {
      "hi": 0.9970519161391262,
      "lo": 0.9114466177322051,
      "mean": 0.9833333333333333,
      "n": 60,
      "successes": 59
     }
    },
    "reversed": {
     "benchmark": {
      "hi": 0.9828508088208587,
      "lo": 0.8629926537490016,
      "mean": 0.95,
      "n": 60,
      "successes": 57
     },
     "campaign": {
      "hi": 0.26114825891159615,
      "lo": 0.08097349242295689,
      "mean": 0.15,
      "n": 60,
      "successes": 9
     },
     "none": {
      "hi": 0.9828508088208587,
      "lo": 0.8629926537490016,
      "mean": 0.95,
      "n": 60,
      "successes": 57
     }
    }
   },
   "pre_ours": {
    "baseline": {
     "hi": 0.9970519161391262,
     "lo": 0.9114466177322051,
     "mean": 0.9833333333333333,
     "n": 60,
     "successes": 59
    },
    "reversed": {
     "hi": 0.9970519161391262,
     "lo": 0.9114466177322051,
     "mean": 0.9833333333333333,
     "n": 60,
     "successes": 59
    }
   },
   "primary": {
    "estimate": -0.1,
    "hi": -0.016666666666666666,
    "lo": -0.18333333333333332,
    "n": 60
   },
   "reversed": {
    "estimate": -0.8,
    "hi": -0.7,
    "lo": -0.9,
    "n": 60
   }
  },
  "connectors": {
   "benchmark_baseline": {
    "estimate": 0.05,
    "hi": 0.1,
    "lo": 0.0,
    "n": 60
   },
   "benchmark_reversed": {
    "estimate": 0.016666666666666666,
    "hi": 0.05,
    "lo": 0.0,
    "n": 60
   },
   "post_ours": {
    "baseline": {
     "benchmark": {
      "hi": 0.13700734625099836,
      "lo": 0.01714919117914121,
      "mean": 0.05,
      "n": 60,
      "successes": 3
     },
     "campaign": {
      "hi": 0.06017393047793289,
      "lo": -6.938893903907228e-18,
      "mean": 0.0,
      "n": 60,
      "successes": 0
     },
     "none": {
      "hi": 0.06017393047793289,
      "lo": -6.938893903907228e-18,
      "mean": 0.0,
      "n": 60,
      "successes": 0
     }
    },
    "reversed": {
     "benchmark": {
      "hi": 0.08855338226779477,
      "lo": 0.002948083860873682,
      "mean": 0.016666666666666666,
      "n": 60,
      "successes": 1
     },
     "campaign": {
      "hi": 0.06017393047793289,
      "lo": -6.938893903907228e-18,
      "mean": 0.0,
      "n": 60,
      "successes": 0
     },
     "none": {
      "hi": 0.06017393047793289,
      "lo": -6.938893903907228e-18,
      "mean": 0.0,
      "n": 60,
      "successes": 0
     }
    }
   },
   "pre_ours": {
    "baseline": {
     "hi": 0.06017393047793289,
     "lo": -6.938893903907228e-18,
     "mean": 0.0,
     "n": 60,
     "successes": 0
    },
    "reversed": {
     "hi": 0.06017393047793289,
     "lo": -6.938893903907228e-18,
     "mean": 0.0,
     "n": 60,
     "successes": 0
    }
   },
   "primary": {
    "estimate": 0.0,
    "hi": 0.0,
    "lo": 0.0,
    "n": 60
   },
   "reversed": {
    "estimate": 0.0,
    "hi": 0.0,
    "lo": 0.0,
    "n": 60
   }
  },
  "latency": {
   "benchmark_baseline": {
    "estimate": 0.16666666666666666,
    "hi": 0.26666666666666666,
    "lo": 0.08333333333333333,
    "n": 60
   },
   "benchmark_reversed": {
    "estimate": -0.016666666666666666,
    "hi": 0.0,
    "lo": -0.05,
    "n": 60
   },
   "post_ours": {
    "baseline": {
     "benchmark": {
      "hi": 0.9999999999999999,
      "lo": 0.9398260695220669,
      "mean": 1.0,
      "n": 60,
      "successes": 60
     },
     "campaign": {
      "hi": 0.7291191041511176,
      "lo": 0.4901736454040315,
      "mean": 0.6166666666666667,
      "n": 60,
      "successes": 37
     },
     "none": {
      "hi": 0.9068692279867897,
      "lo": 0.7196814850279217,
      "mean": 0.8333333333333334,
      "n": 60,
      "successes": 50
     }
    },
    "reversed": {
     "benchmark": {
      "hi": 0.06017393047793289,
      "lo": -6.938893903907228e-18,
      "mean": 0.0,
      "n": 60,
      "successes": 0
     },
     "campaign": {
      "hi": 0.061131479784092065,
      "lo": 0.0,
      "mean": 0.0,
      "n": 59,
      "successes": 0
     },
     "none": {
      "hi": 0.08855338226779477,
      "lo": 0.002948083860873682,
      "mean": 0.016666666666666666,
      "n": 60,
      "successes": 1
     }
    }
   },
   "pre_ours": {
    "baseline": {
     "hi": 0.9999999999999999,
     "lo": 0.9398260695220669,
     "mean": 1.0,
     "n": 60,
     "successes": 60
    },
    "reversed": {
     "hi": 0.06017393047793289,
     "lo": -6.938893903907228e-18,
     "mean": 0.0,
     "n": 60,
     "successes": 0
    }
   },
   "primary": {
    "estimate": -0.21666666666666667,
    "hi": -0.06666666666666667,
    "lo": -0.36666666666666664,
    "n": 60
   },
   "reversed": {
    "estimate": -0.01694915254237288,
    "hi": 0.0,
    "lo": -0.05084745762711865,
    "n": 59
   }
  }
 },
 "primary": {
  "contrast": "campaign - none, baseline world",
  "estimate": -0.10555555555555556,
  "hi": -0.05,
  "lo": -0.16666666666666666,
  "n": 180,
  "outcome": "post.ours",
  "per_seed": {
   "21": {
    "estimate": -0.1,
    "hi": 0.016666666666666666,
    "lo": -0.21666666666666667,
    "n": 60
   },
   "22": {
    "estimate": -0.13333333333333333,
    "hi": -0.05,
    "lo": -0.23333333333333334,
    "n": 60
   },
   "23": {
    "estimate": -0.08333333333333333,
    "hi": 0.016666666666666666,
    "lo": -0.18333333333333332,
    "n": 60
   }
  }
 },
 "run": {
  "arms": [
   "none",
   "benchmark",
   "campaign"
  ],
  "buyers_per_seed": 60,
  "engine": "0.3.0",
  "jobs": 360,
  "journal": {
   "calls": 2880,
   "completion_tokens": 733570,
   "entries": 2880,
   "hits": 0,
   "prompt_tokens": 2392305
  },
  "model": "deepseek-flash",
  "offline": false,
  "phase": "final",
  "prompt_version": "measure-v1",
  "reasoning_effort": "low",
  "records": 19440,
  "seconds": 507.2,
  "seeds": [
   21,
   22,
   23
  ],
  "spec_digest": "a820e54fb6d94889",
  "study": "comparative-positioning",
  "temperature": 0.7,
  "worlds": [
   "baseline",
   "reversed"
  ]
 },
 "study": "comparative-positioning",
 "switching": {
  "baseline": {
   "benchmark": {
    "Corvel Stream -> Corvel Stream": 50,
    "Corvel Stream -> Defer the decision": 7,
    "Corvel Stream -> Halvard Historian": 3,
    "Defer the decision -> Halvard Historian": 1,
    "Halvard Historian -> Halvard Historian": 119
   },
   "campaign": {
    "Corvel Stream -> Corvel Stream": 17,
    "Corvel Stream -> Defer the decision": 43,
    "Defer the decision -> Halvard Historian": 1,
    "Halvard Historian -> Defer the decision": 30,
    "Halvard Historian -> Halvard Historian": 89
   },
   "none": {
    "Corvel Stream -> Corvel Stream": 50,
    "Corvel Stream -> Defer the decision": 10,
    "Defer the decision -> Halvard Historian": 1,
    "Halvard Historian -> Defer the decision": 11,
    "Halvard Historian -> Halvard Historian": 108
   }
  },
  "reversed": {
   "benchmark": {
    "Corvel Stream -> Corvel Stream": 118,
    "Corvel Stream -> Defer the decision": 2,
    "Corvel Stream -> Halvard Historian": 1,
    "Halvard Historian -> Corvel Stream": 1,
    "Halvard Historian -> Defer the decision": 1,
    "Halvard Historian -> Halvard Historian": 57
   },
   "campaign": {
    "Corvel Stream -> Corvel Stream": 110,
    "Corvel Stream -> Defer the decision": 10,
    "Corvel Stream -> None": 1,
    "Halvard Historian -> Corvel Stream": 3,
    "Halvard Historian -> Defer the decision": 47,
    "Halvard Historian -> Halvard Historian": 9
   },
   "none": {
    "Corvel Stream -> Corvel Stream": 102,
    "Corvel Stream -> Defer the decision": 18,
    "Corvel Stream -> Halvard Historian": 1,
    "Halvard Historian -> Defer the decision": 2,
    "Halvard Historian -> Halvard Historian": 57
   }
  }
 },
 "synthetic": true,
 "trial": {
  "baseline": {
   "benchmark": {
    "Both": 24,
    "Corvel Stream": 30,
    "Halvard Historian": 123,
    "Neither": 1,
    "missing": 2
   },
   "campaign": {
    "Both": 123,
    "Corvel Stream": 3,
    "Halvard Historian": 51,
    "missing": 3
   },
   "none": {
    "Both": 48,
    "Corvel Stream": 32,
    "Halvard Historian": 87,
    "Neither": 5,
    "missing": 8
   }
  },
  "reversed": {
   "benchmark": {
    "Both": 15,
    "Corvel Stream": 100,
    "Halvard Historian": 55,
    "Neither": 7,
    "missing": 3
   },
   "campaign": {
    "Both": 118,
    "Corvel Stream": 48,
    "Halvard Historian": 8,
    "Neither": 2,
    "missing": 4
   },
   "none": {
    "Both": 52,
    "Corvel Stream": 57,
    "Halvard Historian": 58,
    "Neither": 9,
    "missing": 4
   }
  }
 },
 "worlds": {
  "baseline": {
   "description": "Halvard is faster on the emphasised feature (query latency); Corvel has more connectors and costs more.",
   "latency": {
    "Corvel Stream": 1.9,
    "Halvard Historian": 0.4
   }
  },
  "reversed": {
   "description": "Counterfactual: Corvel is faster on the emphasised feature; everything else is unchanged.",
   "latency": {
    "Corvel Stream": 0.4,
    "Halvard Historian": 1.9
   }
  }
 }
}