Tutorial 2 — Computing All 26 Measures¶
A detailed walkthrough of each block of the 26-measure array, plus how to call individual measures directly.
Setup¶
In [1]:
Copied!
import numpy as np
from metasignal import stdpy
MEASURE_NAMES = [
"meta-d'", "AUC2", "Gamma", "Phi", "DeltaConf",
"M-Ratio", "AUC2-Ratio", "Gamma-Ratio", "Phi-Ratio", "DeltaConf-Ratio",
"M-Diff", "AUC2-Diff", "Gamma-Diff", "Phi-Diff", "DeltaConf-Diff",
"metaNoise", "metaUncertainty", "d'", "c", "mean_conf",
"logL", "AIC", "BIC", "AICc", "k", "n",
]
N_MEAS = 26
rng = np.random.default_rng(0)
n_trials, n_ratings = 400, 4
stim = rng.choice([0, 1], n_trials)
resp = np.where(rng.random(n_trials) < 0.78, stim, 1 - stim)
correct = stim == resp
conf = np.where(
correct,
rng.integers(3, n_ratings + 1, n_trials),
rng.integers(1, 3, n_trials),
)
print("Data ready:", n_trials, "trials,", n_ratings, "ratings")
import numpy as np
from metasignal import stdpy
MEASURE_NAMES = [
"meta-d'", "AUC2", "Gamma", "Phi", "DeltaConf",
"M-Ratio", "AUC2-Ratio", "Gamma-Ratio", "Phi-Ratio", "DeltaConf-Ratio",
"M-Diff", "AUC2-Diff", "Gamma-Diff", "Phi-Diff", "DeltaConf-Diff",
"metaNoise", "metaUncertainty", "d'", "c", "mean_conf",
"logL", "AIC", "BIC", "AICc", "k", "n",
]
N_MEAS = 26
rng = np.random.default_rng(0)
n_trials, n_ratings = 400, 4
stim = rng.choice([0, 1], n_trials)
resp = np.where(rng.random(n_trials) < 0.78, stim, 1 - stim)
correct = stim == resp
conf = np.where(
correct,
rng.integers(3, n_ratings + 1, n_trials),
rng.integers(1, 3, n_trials),
)
print("Data ready:", n_trials, "trials,", n_ratings, "ratings")
Data ready: 400 trials, 4 ratings
Block 1 — Metacognitive sensitivity (indices 0–4)¶
These five measures ask: how well does confidence track accuracy?
In [2]:
Copied!
meas = stdpy.compute_all_measures(stim, resp, conf, n_ratings=n_ratings)
labels = ["meta-d'", "AUC2", "Gamma", "Phi", "DeltaConf"]
for i, name in enumerate(labels):
print(f" {name:<12} = {meas[i]:.4f}")
meas = stdpy.compute_all_measures(stim, resp, conf, n_ratings=n_ratings)
labels = ["meta-d'", "AUC2", "Gamma", "Phi", "DeltaConf"]
for i, name in enumerate(labels):
print(f" {name:<12} = {meas[i]:.4f}")
meta-d' = 7.6993 AUC2 = 1.0000 Gamma = 1.0000 Phi = 0.8541 DeltaConf = 1.9822
Block 2 & 3 — Efficiency ratios and differences (indices 5–14)¶
Normalise observed metacognition by the expected performance of an ideal observer with the same d'. Removes spurious dependence on task difficulty.
In [3]:
Copied!
nr_s1, nr_s2 = stdpy.trials_to_counts(stim, resp, conf, n_ratings=n_ratings)
expected = stdpy.sdt_expect_conf(nr_s1, nr_s2)
nr_s1_exp = np.array(expected["nR_S1_exp"])
nr_s2_exp = np.array(expected["nR_S2_exp"])
auc2_obs = stdpy.compute_type2_auc(nr_s1, nr_s2)
auc2_exp = stdpy.compute_type2_auc(nr_s1_exp, nr_s2_exp)
print(f"AUC2 observed = {auc2_obs:.4f}")
print(f"AUC2 ideal = {auc2_exp:.4f}")
print(f"AUC2-Ratio = {meas[6]:.4f} (obs / ideal)")
print(f"AUC2-Diff = {meas[11]:.4f} (obs − ideal)")
nr_s1, nr_s2 = stdpy.trials_to_counts(stim, resp, conf, n_ratings=n_ratings)
expected = stdpy.sdt_expect_conf(nr_s1, nr_s2)
nr_s1_exp = np.array(expected["nR_S1_exp"])
nr_s2_exp = np.array(expected["nR_S2_exp"])
auc2_obs = stdpy.compute_type2_auc(nr_s1, nr_s2)
auc2_exp = stdpy.compute_type2_auc(nr_s1_exp, nr_s2_exp)
print(f"AUC2 observed = {auc2_obs:.4f}")
print(f"AUC2 ideal = {auc2_exp:.4f}")
print(f"AUC2-Ratio = {meas[6]:.4f} (obs / ideal)")
print(f"AUC2-Diff = {meas[11]:.4f} (obs − ideal)")
AUC2 observed = 1.0000 AUC2 ideal = 0.7045 AUC2-Ratio = 1.4194 (obs / ideal) AUC2-Diff = 0.2955 (obs − ideal)
Individual measure functions¶
In [4]:
Copied!
gamma = stdpy.compute_gamma(nr_s1, nr_s2)
phi = stdpy.compute_phi(nr_s1, nr_s2)
dc = stdpy.compute_delta_conf(nr_s1, nr_s2)
print(f"Gamma = {gamma:.4f}")
print(f"Phi = {phi:.4f}")
print(f"DeltaConf = {dc['delta_conf']:.4f}")
result = stdpy.fit_meta_d_mle(nr_s1, nr_s2)
print(f"meta_da = {result['meta_da']:.4f}")
print(f"M_ratio = {result['M_ratio']:.4f}")
gamma = stdpy.compute_gamma(nr_s1, nr_s2)
phi = stdpy.compute_phi(nr_s1, nr_s2)
dc = stdpy.compute_delta_conf(nr_s1, nr_s2)
print(f"Gamma = {gamma:.4f}")
print(f"Phi = {phi:.4f}")
print(f"DeltaConf = {dc['delta_conf']:.4f}")
result = stdpy.fit_meta_d_mle(nr_s1, nr_s2)
print(f"meta_da = {result['meta_da']:.4f}")
print(f"M_ratio = {result['M_ratio']:.4f}")
Gamma = 1.0000 Phi = 0.8541 DeltaConf = 1.9822
meta_da = 10.0000 M_ratio = 6.5174
Block 4 — Meta-noise and meta-uncertainty (indices 15–16)¶
In [5]:
Copied!
noise_res = stdpy.compute_meta_noise(stim, resp, conf, n_ratings=n_ratings)
uncert = stdpy.compute_meta_uncertainty(stim, resp, conf, n_ratings=n_ratings)
print(f"metaNoise = {noise_res['meta_noise']:.4f}")
print(f"metaUncertainty = {uncert:.4f}")
noise_res = stdpy.compute_meta_noise(stim, resp, conf, n_ratings=n_ratings)
uncert = stdpy.compute_meta_uncertainty(stim, resp, conf, n_ratings=n_ratings)
print(f"metaNoise = {noise_res['meta_noise']:.4f}")
print(f"metaUncertainty = {uncert:.4f}")
metaNoise = 0.0967 metaUncertainty = 0.0100
Full 26-measure summary¶
In [6]:
Copied!
print(f"{'Index':<6} {'Measure':<20} {'Value':>10}")
print("-" * 40)
for i, (name, val) in enumerate(zip(MEASURE_NAMES, meas)):
vstr = f"{val:10.4f}" if not np.isnan(val) else " NaN"
print(f"[{i:2d}] {name:<20} {vstr}")
print(f"{'Index':<6} {'Measure':<20} {'Value':>10}")
print("-" * 40)
for i, (name, val) in enumerate(zip(MEASURE_NAMES, meas)):
vstr = f"{val:10.4f}" if not np.isnan(val) else " NaN"
print(f"[{i:2d}] {name:<20} {vstr}")
Index Measure Value ---------------------------------------- [ 0] meta-d' 7.6993 [ 1] AUC2 1.0000 [ 2] Gamma 1.0000 [ 3] Phi 0.8541 [ 4] DeltaConf 1.9822 [ 5] M-Ratio 5.0487 [ 6] AUC2-Ratio 1.4194 [ 7] Gamma-Ratio 1.6469 [ 8] Phi-Ratio 2.7385 [ 9] DeltaConf-Ratio 2.8927 [10] M-Diff 6.1743 [11] AUC2-Diff 0.2955 [12] Gamma-Diff 0.3928 [13] Phi-Diff 0.5422 [14] DeltaConf-Diff 1.2970 [15] metaNoise 0.0967 [16] metaUncertainty 0.0100 [17] d' 1.5343 [18] c -0.0616 [19] mean_conf 3.0575 [20] logL -289.2670 [21] AIC 592.5340 [22] BIC 620.5092 [23] AICc 592.8183 [24] k 7.0000 [25] n 402.0000