"""Score frozen mechanical criteria; summary meaning needs separate review."""
from pathlib import Path
import csv, json, statistics

ROOT=Path(__file__).resolve().parents[2]
OUT=ROOT/'outputs/qwen-local-test'
fixtures=json.loads((OUT/'fixtures.json').read_text(encoding='utf-8'))

def unique_object(pairs):
    result={}
    for key,value in pairs:
        if key in result:raise ValueError('duplicate JSON key')
        result[key]=value
    return result

def assess(task,output):
    if task['id']=='extraction':
        try:
            value=json.loads(output,object_pairs_hook=unique_object)
            assert isinstance(value,dict) and set(value)==set(task['expected'])
            assert value==task['expected']
            assert all(isinstance(value[k],str) for k in ('event','date','time','city'))
            assert all(type(value[k]) in (int,float) for k in ('capacity','ticket_gbp'))
            assert value['contact_email'] is None
            return {'automatic':'pass','detail':'Exact keys, values and required types; valid JSON without duplicate keys.'}
        except (ValueError,AssertionError):
            return {'automatic':'fail','detail':'Does not satisfy exact JSON extraction criteria.'}
    if task['id']=='arithmetic':
        passed=output.strip()==task['expected']
        return {'automatic':'pass' if passed else 'fail','detail':'Trimmed response must equal 176.'}
    lines=output.strip().splitlines()
    structure=len(lines)==3 and all(x.startswith('- ') for x in lines)
    counts=[len(x[2:].split()) for x in lines] if structure else []
    return {'automatic':'semantic-review-required' if structure and all(n<=25 for n in counts) else 'fail','bullet_word_counts':counts,'detail':'Meaning, complete fact coverage and no added facts require independent semantic review.'}

rows=[];records=[]
for task in fixtures['tasks']:
    for run in range(1,4):
        data=json.loads((OUT/f"{task['id']}-{run}.json").read_text(encoding='utf-8'))
        assert data['request']['prompt']==task['prompt'] and data['request']['options']==fixtures['options']
        assert data['request']['think'] is False
        score=assess(task,data['output'])
        metrics=data.get('metrics') or {}
        record={'task':task['id'],'run':run,'answer_check':score,'transport_error':data['error'],'completion_reason':metrics.get('done_reason'),'reported_done':metrics.get('done'),'output_tokens':metrics.get('eval_count'),'output_limit_respected':isinstance(metrics.get('eval_count'),int) and metrics['eval_count']<=256,'memory_samples':data['memory_sample_count']}
        records.append(record)
        rows.append({'task':task['id'],'run':run,'model_state':'unloaded' if run==1 else 'resident','first_content_seconds':data['first_content_seconds'],'client_seconds':data['client_seconds'],'load_seconds':metrics.get('load_duration',0)/1e9,'prompt_tokens':metrics.get('prompt_eval_count'),'output_tokens':metrics.get('eval_count'),'generation_tokens_per_second':data.get('generation_tokens_per_second'),'sampled_peak_working_set_sum_gib':data['sampled_peak_rss_sum_bytes']/1024**3,'completion_reason':metrics.get('done_reason'),'automatic_answer_check':score['automatic']})
with (OUT/'request-metrics.csv').open('w',newline='',encoding='utf-8') as f:
    writer=csv.DictWriter(f,fieldnames=list(rows[0]));writer.writeheader();writer.writerows(rows)
(OUT/'mechanical-review.json').write_text(json.dumps(records,indent=2),encoding='utf-8')
groups=[]
for task in fixtures['tasks']:
    subset=[r for r in rows if r['task']==task['id']]
    groups.append({'task':task['id'],'unloaded_seconds':subset[0]['client_seconds'],'resident_seconds_range':[min(r['client_seconds'] for r in subset[1:]),max(r['client_seconds'] for r in subset[1:])],'generation_tps_range':[min(r['generation_tokens_per_second'] or 0 for r in subset),max(r['generation_tokens_per_second'] or 0 for r in subset)],'peak_sampled_working_set_sum_gib':max(r['sampled_peak_working_set_sum_gib'] for r in subset)})
(OUT/'timing-summary.json').write_text(json.dumps(groups,indent=2),encoding='utf-8')
print(json.dumps({'mechanical_checks':records,'timing_summary':groups},indent=2))
