| """CPU-only scoring and uncertainty from curated published records.""" | |
| import re | |
| import numpy as np | |
| LANGUAGES=('english','hindi','hinglish') | |
| def parse_choice(text): | |
| match=re.fullmatch(r'\s*(?:Answer\s*:\s*)?\(?([A-D])\)?[.。]?\s*',text,flags=re.I) | |
| return match.group(1).upper() if match else None | |
| def validate_pairs(rows): | |
| expected={(t,s) for t in range(10) for s in range(20,40)} | |
| matrices={} | |
| if len(rows)!=600: raise ValueError('Exactly600 matched language-episode rows required') | |
| for lang in LANGUAGES: | |
| subset=[r for r in rows if r['condition']==lang] | |
| keys=[(r['task_id'],r['initial_state_index']) for r in subset] | |
| if len(keys)!=200 or len(set(keys))!=200 or set(keys)!=expected: raise ValueError('Duplicate or missing task/state pair') | |
| if any(type(r['success']) is not bool for r in subset): raise ValueError('Success must be boolean, not a missing/error value') | |
| lookup={k:r['success'] for k,r in zip(keys,subset)} | |
| matrices[lang]=np.array([[lookup[t,s] for s in range(20,40)] for t in range(10)],dtype=int) | |
| return matrices | |
| def paired_difference(left,right,repetitions=10000): | |
| left,right=np.asarray(left,dtype=int),np.asarray(right,dtype=int) | |
| if left.shape!=right.shape or left.ndim!=2: raise ValueError('Paired task/state matrices required') | |
| delta=left-right | |
| draws=np.random.default_rng(7).integers(0,left.shape[1],size=(repetitions,*left.shape)) | |
| estimates=delta[np.arange(left.shape[0])[None,:,None],draws].mean(axis=(1,2)) | |
| return dict(difference=float(delta.mean()),ci95=np.quantile(estimates,[.025,.975]).tolist(), | |
| both_success=int(((left==1)&(right==1)).sum()),left_only=int(((left==1)&(right==0)).sum()), | |
| right_only=int(((left==0)&(right==1)).sum()),both_failure=int(((left==0)&(right==0)).sum())) | |
| def wilson(successes,total): | |
| z=1.959963984540054; p=successes/total; denominator=1+z*z/total | |
| center=(p+z*z/(2*total))/denominator | |
| half=z*np.sqrt(p*(1-p)/total+z*z/(4*total*total))/denominator | |
| return max(0,center-half),min(1,center+half) | |
| def paired_items(rows): | |
| joined={} | |
| for row in rows: | |
| entry=joined.setdefault(row['id'],{}) | |
| if row['language'] in entry: raise ValueError('Duplicate item/language') | |
| entry[row['language']]=row | |
| for item,entry in joined.items(): | |
| if set(entry)!=set(LANGUAGES): raise ValueError('Missing language') | |
| if len({r['answer'] for r in entry.values()})!=1: raise ValueError('Answer-key mismatch') | |
| if len({r['image'] for r in entry.values()})!=1: raise ValueError('Image identity mismatch') | |
| return dict(sorted(joined.items())) | |
| def contingency(left,right): | |
| if len(left)!=len(right): raise ValueError('Unequal paired denominator') | |
| return dict(both_correct=sum(bool(a) and bool(b) for a,b in zip(left,right)), | |
| left_only=sum(bool(a) and not b for a,b in zip(left,right)), | |
| right_only=sum(not a and bool(b) for a,b in zip(left,right)), | |
| both_incorrect=sum(not a and not b for a,b in zip(left,right)),total=len(left)) | |
Xet Storage Details
- Size:
- 3.09 kB
- Xet hash:
- bb1d47b8be8982bc3a50f95ff2fb22734e982c48ce3ada6404077fce765c16d9
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.