"""2차 관측 — (1) 파일별 수집 기준일(dataBssDate)로 수집 시점 파악, (2) 낙찰하한율 분포, (3) 응찰률−하한율 간격(0.1%p) 히스토그램:
응찰가가 하한선 바로 위에 얼마나 몰리는지 = '복권형 입찰' 의 실측. 사용: python3 observe2.py [raw_dir] [--out observations2.json]
"""
import gzip, json, glob, os, re, argparse, collections, statistics
ap = argparse.ArgumentParser(); ap.add_argument('raw', nargs='?', default=os.path.expanduser('~/Projects/miriboa/outreach/out/raw')); ap.add_argument('--out', default='observations2.json'); a = ap.parse_args()
DIV = {'3': '공사', '5': '용역'}
def f(x):
    try: return float(str(x).replace(',', ''))
    except Exception: return None
bss = {}; low = collections.defaultdict(collections.Counter); gap = collections.defaultdict(collections.Counter); gapn = collections.Counter(); seen = set(); within1 = collections.Counter(); below = collections.Counter(); tot = collections.Counter()
for fp in sorted(glob.glob(os.path.join(a.raw, '*.jsonl.gz'))):
    m = re.match(r'(\d)_(\d{8})_', os.path.basename(fp)); div = DIV.get(m.group(1)); day = m.group(2)
    with gzip.open(fp, 'rt', encoding='utf8') as fh:
        for line in fh:
            try: d = json.loads(line)
            except Exception: continue
            if (div, day) not in bss: bss[(div, day)] = d.get('dataBssDate')
            dec = d.get('bidwinrDcsnMthdNm') or '?'; nid = d.get('bidNtceNo'); ll = f(d.get('sucsfLwstlmtRt'))
            if nid and nid not in seen and ll is not None: seen.add(nid); low[(div, dec)][round(ll, 3)] += 1
            r = f(d.get('bidprcRt'))
            if ll and r and dec in ('적격심사제', '소액수의견적', '제한적최저가(낙찰하한율)'):
                g = r - ll; k = (div, dec); tot[k] += 1
                if g < 0: below[k] += 1
                elif g <= 1.0: within1[k] += 1
                if -2 <= g <= 3: gap[k][round(g, 1)] += 1
out = {'collection_basis_by_month': {}, 'lowlimit': {}, 'gap': {}}
bm = collections.defaultdict(collections.Counter)
for (div, day), b in bss.items(): bm[f'{div}|{day[:6]}'][b] += 1
out['collection_basis_by_month'] = {k: dict(v) for k, v in sorted(bm.items())}
for k, c in low.items():
    if sum(c.values()) >= 200: out['lowlimit'][f'{k[0]}|{k[1]}'] = {'n': sum(c.values()), 'top': [(v, n/sum(c.values())) for v, n in c.most_common(5)]}
for k in tot:
    out['gap'][f'{k[0]}|{k[1]}'] = {'bids': tot[k], 'below_floor': below[k]/tot[k], 'within_1pt_above': within1[k]/tot[k], 'hist_0.1pt': sorted(((g, n/tot[k]) for g, n in gap[k].items()), key=lambda x: x[0])}
json.dump(out, open(a.out, 'w'), ensure_ascii=False, indent=1); print('wrote', a.out)