import csv,datetime,collections,re,json
rows=list(csv.DictReader(open('input/crawler_logs.csv',encoding='utf-8')))
L={r['url']:r for r in csv.DictReader(open('input/chome-ledger.csv',encoding='utf-8-sig'))}
J=lambda s:datetime.datetime.fromisoformat(s.replace('+00','+00:00'))+datetime.timedelta(hours=9)
ALIAS={'other:AhrefsBot':'ahrefsbot','other:SemrushBot':'semrushbot','other:ShapBot':'shapbot','other:AdsBot-Google-Mobile':'other:AdsBot-Google'}
def name(r):
    b=r['bot_name']
    if b=='other':
        ua=r['user_agent']
        m=re.search(r'compatible;\s*([A-Za-z0-9_\-\.]+)',ua) or re.search(r'([A-Za-z][A-Za-z0-9_\-]*bot[A-Za-z0-9_\-]*)',ua,re.I) or re.search(r'^([A-Za-z0-9_\-]+)/',ua)
        b='other:'+(m.group(1) if m else '?')
    return ALIAS.get(b,b)
ischome=lambda p:p.startswith('/chome/tokyo/') and p.count('/')>=5
isward=lambda p:p.startswith('/chome/tokyo/') and p.count('/')==4
for r in rows: r['n']=name(r); r['t']=J(r['accessed_at'])
