"""개찰 데이터 6개월 관측 — outreach/out/raw 의 일별 jsonl.gz(공사=3, 용역=5)를 한 번 훑어 관측치를 낸다.
사용: python3 observe.py [raw_dir] [--out observations.json]   (17.5M 행, 순수 파이썬 약 5분)
"""
import gzip, json, glob, os, sys, argparse, collections, statistics, datetime, re
ap = argparse.ArgumentParser(); ap.add_argument('raw', nargs='?', default=os.path.expanduser('~/Projects/miriboa/outreach/out/raw')); ap.add_argument('--out', default='observations.json')
a = ap.parse_args()
DIV = {'3': '공사', '5': '용역'}
files = sorted(glob.glob(os.path.join(a.raw, '*.jsonl.gz')))
empty_days = []; day_rows = collections.Counter()
notices = {}                      # id -> dict
bid_rates = collections.defaultdict(list)          # (div, decision) -> [bidprcRt]
dq = collections.Counter(); rows = 0; lag = collections.Counter(); null = collections.Counter(); fields = ['presmptPrce', 'bssAmt', 'sucsfLwstlmtRt', 'bidprcAmt', 'fnlSucsfRt', 'fnlSucsfCorpBizrno', 'opengDate']
def f(x):
    try: return float(str(x).replace(',', ''))
    except Exception: return None
for fp in files:
    m = re.match(r'(\d)_(\d{8})_', os.path.basename(fp)); div = DIV.get(m.group(1), m.group(1)); day = m.group(2)
    n = 0
    with gzip.open(fp, 'rt', encoding='utf8') as fh:
        for line in fh:
            try: d = json.loads(line)
            except Exception: continue
            n += 1; rows += 1
            for k in fields:
                if not d.get(k): null[k] += 1
            nid = d.get('bidNtceNo')
            if not nid: continue
            no = notices.get(nid)
            if no is None:
                no = notices[nid] = {'div': div, 'month': (d.get('opengDate') or day)[:7].replace('-', ''), 'method': d.get('cntrctCnclsMthdNm') or '?', 'decision': d.get('bidwinrDcsnMthdNm') or '?', 'agency': d.get('ntceInsttNm') or '', 'ord': d.get('bidNtceOrd') or '000', 'kind': d.get('ntceKindNm') or '', 'presmpt': f(d.get('presmptPrce')), 'lowlimit': f(d.get('sucsfLwstlmtRt')), 'bidders': 0, 'win_rate': None, 'win_amt': None, 'status': d.get('opengRsltDivNm') or ''}
            no['bidders'] += 1
            if d.get('sucsfYn') == 'Y' and no['win_rate'] is None:
                no['win_rate'] = f(d.get('fnlSucsfRt') or d.get('bidprcRt')); no['win_amt'] = f(d.get('fnlSucsfAmt') or d.get('bidprcAmt'))
            r = f(d.get('bidprcRt'))
            if r is not None and 0 < r < 200: bid_rates[(div, no['decision'])].append(r)
            reason = (d.get('dqlfctnRsn') or '').strip()
            if reason and reason != '정상': dq[reason] += 1
            od, bd = d.get('opengDate'), d.get('dataBssDate')
            if od and bd:
                try: lag[(datetime.date.fromisoformat(bd) - datetime.date.fromisoformat(od)).days] += 1
                except Exception: pass
    day_rows[(div, day)] = n
    if n == 0: empty_days.append((div, day))
    print(f'  {os.path.basename(fp)} rows={n}', file=sys.stderr) if n == 0 else None
print('rows', rows, 'notices', len(notices), 'files', len(files))
N = list(notices.values())
def pct(v, p): v = sorted(v); return v[min(len(v)-1, int(p*len(v)))] if v else None
out = {'rows': rows, 'notices': len(notices), 'files': len(files), 'empty_days': empty_days}
# 1 월별
by_month = collections.defaultdict(lambda: collections.Counter())
for no in N: by_month[no['month']][no['div']] += 1
out['by_month'] = {m: dict(c) for m, c in sorted(by_month.items())}
# 2 계약방법 × 낙찰자결정
mm = collections.Counter((no['div'], no['method'], no['decision']) for no in N)
out['method_decision'] = [(k, v) for k, v in mm.most_common(20)]
# 3 응찰자 수
bd = collections.defaultdict(list)
for no in N: bd[(no['div'], no['decision'])].append(no['bidders'])
out['bidders'] = {f'{k[0]}|{k[1]}': {'n': len(v), 'median': statistics.median(v), 'p90': pct(v, 0.9), 'max': max(v), 'single': sum(1 for x in v if x == 1)/len(v)} for k, v in bd.items() if len(v) >= 200}
allb = [no['bidders'] for no in N]; out['bidders_all'] = {'median': statistics.median(allb), 'p90': pct(allb, .9), 'p99': pct(allb, .99), 'max': max(allb), 'single_share': sum(1 for x in allb if x == 1)/len(allb)}
# 4 낙찰률 분포 (div×decision, 0.5%p 구간)
wr = collections.defaultdict(list)
for no in N:
    if no['win_rate'] is not None and 0 < no['win_rate'] < 150: wr[(no['div'], no['decision'])].append(no['win_rate'])
out['win_rate'] = {}
for k, v in wr.items():
    if len(v) < 200: continue
    h = collections.Counter(round(x * 2) / 2 for x in v)
    out['win_rate'][f'{k[0]}|{k[1]}'] = {'n': len(v), 'median': statistics.median(v), 'p10': pct(v, .1), 'p90': pct(v, .9), 'top_bins': [(b, c/len(v)) for b, c in h.most_common(5)]}
# 5 응찰률 밀집: 낙찰하한율 대비 (응찰률 - 하한율) 0.1%p 히스토그램, 적격심사만
gap = collections.defaultdict(lambda: collections.Counter()); gapn = collections.Counter()
# 다시 훑지 않고 공고별 하한율은 있으니 응찰률 목록을 공고별로 못 붙였다 — 응찰률 자체의 0.1%p 히스토그램으로 대신
for k, v in bid_rates.items():
    if len(v) < 5000: continue
    h = collections.Counter(round(x, 1) for x in v)
    out.setdefault('bid_rate_hist', {})[f'{k[0]}|{k[1]}'] = {'n': len(v), 'median': statistics.median(v), 'top_bins': [(b, c/len(v)) for b, c in h.most_common(8)]}
# 6 부적격 사유
out['dq_reasons'] = [(r, c) for r, c in dq.most_common(12)]; out['dq_total'] = sum(dq.values())
# 7 재공고·추정가격
out['renotice_share'] = sum(1 for no in N if no['ord'] != '000' or '재공고' in no['kind'])/len(N)
pr = collections.defaultdict(list)
for no in N:
    if no['presmpt']: pr[no['div']].append(no['presmpt'])
out['presmpt'] = {k: {'n': len(v), 'median': statistics.median(v), 'p90': pct(v, .9), 'total_bn': sum(v)/1e9} for k, v in pr.items()}
# 8 신선도
tot = sum(lag.values()); out['lag_days'] = {'median': None, 'dist': sorted(((k, c/tot) for k, c in lag.items()), key=lambda x: -x[1])[:6]}
# 9 결측
out['null_rate'] = {k: v/rows for k, v in null.items()}
# 10 기관·낙찰자 집중
ag = collections.Counter(no['agency'] for no in N); out['agencies'] = {'n': len(ag), 'top': ag.most_common(8)}
json.dump(out, open(a.out, 'w'), ensure_ascii=False, indent=1, default=str)
print('wrote', a.out)