import csv,datetime,collections,json,math
import matplotlib; matplotlib.use('Agg')
import matplotlib.pyplot as plt
from matplotlib import font_manager as fm
from matplotlib.patches import Polygon
from matplotlib.collections import PatchCollection
for f in ['/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc','/usr/share/fonts/opentype/noto/NotoSansCJK-Medium.ttc']:
    fm.fontManager.addfont(f)
plt.rcParams.update({'font.family':'Noto Sans CJK JP','font.size':10,'axes.edgecolor':'#c9c8c3','axes.labelcolor':'#52514e',
    'xtick.color':'#52514e','ytick.color':'#52514e','text.color':'#0b0b0b','figure.facecolor':'#fcfcfb','axes.facecolor':'#fcfcfb',
    'savefig.facecolor':'#fcfcfb','axes.spines.top':False,'axes.spines.right':False})
BLUE,ORANGE,AQUA,YELLOW='#2a78d6','#eb6834','#1baf7a','#eda100'; MUTED='#8b8a85'; GRID='#e6e5e0'
exec(open('analyze.py').read())
days=[datetime.date(2026,9,13)+datetime.timedelta(d) for d in range(31)]

# ---------- Fig1: small multiples
panels=[('Googlebot（Google 検索）',['googlebot'],BLUE),('Bingbot（Microsoft 検索）',['bingbot'],BLUE),('GoogleOther（Google）',['googleother'],BLUE),
        ('ClaudeBot（Anthropic）',['claudebot'],ORANGE),('meta-externalagent（Meta）',['meta-externalagent'],ORANGE),
        ('その他の AI（ShapBot・Amazonbot・GPTBot ほか）',['shapbot','amazonbot','gptbot','other:ExaSearchBot'],ORANGE),
        ('SEO ツール（MJ12bot・Semrush・serpstat）',['other:MJ12bot','semrushbot','other:serpstatbot'],AQUA)]
fig,axes=plt.subplots(len(panels),1,figsize=(9,10.5),sharex=True)
x=list(range(len(days)))
events=[(1,'練馬区の50件を公開'),(20,'374件を追加'),(26,'361件を追加')]
for ax,(title,bots,col) in zip(axes,panels):
    v=[len({r['path'] for r in rows if r['n'] in bots and ischome(r['path']) and r['t'].date()==d}) for d in days]
    ax.bar(x,v,width=0.7,color=col,linewidth=0)
    m=max(v) or 1
    ax.set_ylim(0,m*1.25); ax.set_yticks([0,m]); ax.tick_params(axis='y',labelsize=8,length=0)
    ax.text(0.0,1.02,title,transform=ax.transAxes,fontsize=9.5,ha='left',va='bottom')
    ax.text(1.0,1.02,f'合計 {len({r["path"] for r in rows if r["n"] in bots and ischome(r["path"])})}ページ',transform=ax.transAxes,fontsize=8.5,ha='right',va='bottom',color='#52514e')
    for e,_ in events: ax.axvline(e-0.5,color=MUTED,lw=0.8,ls=(0,(3,3)),zorder=0)
    ax.yaxis.grid(False); ax.spines['left'].set_visible(False)
for e,lab in events:
    axes[0].text(e-0.3,axes[0].get_ylim()[1]*1.32,lab,fontsize=8,color='#52514e',ha='left',va='bottom',clip_on=False)
axes[-1].set_xticks([i for i,d in enumerate(days) if d.day in (14,21,28) or (d.month==10 and d.day in (5,12))])
axes[-1].set_xticklabels([f'{d.month}/{d.day}' for d in days if d.day in (14,21,28) or (d.month==10 and d.day in (5,12))])
fig.suptitle('クローラごとに、1日に取得した丁目ページ数（重複を除く）',x=0.02,ha='left',fontsize=13,y=0.995)
fig.text(0.02,0.962,'2026年9月13日〜10月13日・日本時間。縦軸の目盛りはパネルごとに異なる（上端の数字が最大値）',fontsize=8.5,color='#52514e',ha='left')
fig.text(0.02,0.005,'herahin.net のアクセスログより作成',fontsize=8,color=MUTED,ha='left')
plt.tight_layout(rect=(0,0.015,1,0.925),h_pad=1.6)
fig.savefig('fig/fig1_daily_by_crawler.png',dpi=200)

# ---------- Fig2: cumulative coverage
tz=datetime.timezone(datetime.timedelta(hours=9))
for r in rows: r['t']=r['t'].replace(tzinfo=tz)
pub={'B2':datetime.datetime(2026,10,3,23,15,tzinfo=tz),'B5':datetime.datetime(2026,10,9,10,20,tzinfo=tz)}
pub['B3']=pub['B4']=pub['B2']
def hours(bot,bs):
    f={}
    for r in rows:
        if r['n']==bot and ischome(r['path']) and r['path'] in L and L[r['path']]['batch'] in bs:
            h=(r['t']-pub[L[r['path']]['batch']]).total_seconds()/3600
            f[r['path']]=min(f.get(r['path'],1e9),h)
    return sorted(f.values())
fig,ax=plt.subplots(figsize=(9,5))
H=96
lab_pos={}
series=[('ClaudeBot','claudebot',('B5',),361,ORANGE,'-','ClaudeBot・10/9 公開分'),
        ('ClaudeBot','claudebot',('B2','B3','B4'),374,ORANGE,(0,(4,3)),'ClaudeBot・10/3 公開分'),
        ('Googlebot','googlebot',('B5',),361,BLUE,'-','Googlebot・10/9 公開分'),
        ('Googlebot','googlebot',('B2','B3','B4'),374,BLUE,(0,(4,3)),'Googlebot・10/3 公開分')]
for _,bot,bs,tot,col,ls,lab in series:
    hs=[h for h in hours(bot,bs) if h<=H]
    xs=[0]+[h for h in hs for _ in (0,1)]+[H]; ys=[0]
    c=0
    for h in hs: ys+= [c/tot*100]; c+=1; ys+=[c/tot*100]
    ys+=[c/tot*100]
    ax.plot(xs,ys,color=col,lw=2,ls=ls,solid_capstyle='round')
    yl=c/tot*100
    lab_pos[lab]=(hs,yl)
ax.set_xlim(0,H); ax.set_ylim(-2,105)
ax.set_xticks(range(0,H+1,24)); ax.set_xticklabels([f'{h}時間' if h else '公開' for h in range(0,H+1,24)])
ax.set_yticks([0,25,50,75,100]); ax.set_yticklabels(['0%','25%','50%','75%','100%'])
ax.yaxis.grid(True,color=GRID,lw=0.8); ax.set_axisbelow(True); ax.spines['left'].set_visible(False); ax.tick_params(length=0)
ax.text(3,93,'ClaudeBot・10/9 公開分\n公開1.6時間後に361件を全件取得',fontsize=8.5,va='top',color='#0b0b0b')
ax.text(24,80,'ClaudeBot・10/3 公開分\n公開22時間後に374件を全件取得',fontsize=8.5,va='top',color='#0b0b0b')
ax.text(H-1,20,f'Googlebot・10/9 公開分　4日で{lab_pos["Googlebot・10/9 公開分"][1]:.0f}%',fontsize=8.5,ha='right',va='bottom',color='#0b0b0b')
ax.text(56,2.0,f'Googlebot・10/3 公開分（破線）\n4日で{lab_pos["Googlebot・10/3 公開分"][1]:.0f}%',fontsize=8.5,ha='left',va='bottom',color='#0b0b0b')
fig.suptitle('新しく公開したページのうち、取得済みになった割合',x=0.02,ha='left',fontsize=13)
fig.text(0.02,0.905,'公開からの経過時間ごと（最初の4日間）。実線が 10/9 に公開した361件、破線が 10/3 に公開した374件。どちらも手動リクエストはしていない',fontsize=8.5,color='#52514e',ha='left')
fig.text(0.02,0.01,'herahin.net のアクセスログより作成',fontsize=8,color=MUTED,ha='left')
plt.tight_layout(rect=(0,0.03,1,0.89))
fig.savefig('fig/fig2_coverage_after_publish.png',dpi=200)

# ---------- Fig3: map
rings=json.load(open('input/rings.json'))
first={}
for r in rows:
    if r['n']=='googlebot' and ischome(r['path']):
        first[r['path']]=min(first.get(r['path'],r['t']),r['t'])
bins=[('10/3〜10/8',datetime.date(2026,10,8),'#3987e5'),('10/9〜10/13',datetime.date(2026,10,13),'#86b6ef')]
lat0=math.radians(35.7)
fig,ax=plt.subplots(figsize=(9,8.8))
def poly(ring,**kw): return Polygon([(x*math.cos(lat0),y) for x,y in ring],closed=True,**kw)
for u,ring in rings.items():
    t=first.get(u); col='#e6e5e0'
    if t:
        for _,lim,c in bins:
            if t.date()<=lim: col=c; break
    ax.add_patch(poly(ring,facecolor=col,edgecolor='#fcfcfb',lw=0.4))
for u,ring in rings.items():
    if L.get(u,{}).get('batch')=='B1':
        ax.add_patch(poly(ring,facecolor=YELLOW,edgecolor='#fcfcfb',lw=0.4))
# ward boundaries: union of chome polygons per ward (small buffer closes slivers between chome)
from shapely.geometry import Polygon as SP
from shapely.ops import unary_union
wp=collections.defaultdict(list)
for u,ring in rings.items():
    g=SP([(x*math.cos(lat0),y) for x,y in ring]).buffer(0)
    wp[u.split('/')[3]].append(g.buffer(0.0004))
for w,gs in wp.items():
    g=unary_union(gs).buffer(-0.0004)
    for part in getattr(g,'geoms',[g]):
        x,y=part.exterior.xy
        ax.plot(x,y,color='#2b2a27',lw=1.0,solid_joinstyle='round',zorder=5)
# ward labels
wc=collections.defaultdict(list)
for u,ring in rings.items():
    w=u.split('/')[3]; xs=[p[0] for p in ring]; ys=[p[1] for p in ring]
    wc[w].append((sum(xs)/len(xs),sum(ys)/len(ys)))
WN={'nerima':'練馬区','nakano':'中野区','suginami':'杉並区','itabashi':'板橋区','toshima':'豊島区','shinjuku':'新宿区'}
for w,pts in wc.items():
    x=sum(p[0] for p in pts)/len(pts); y=sum(p[1] for p in pts)/len(pts)
    ax.text(x*math.cos(lat0),y,WN[w],fontsize=10,ha='center',va='center',color='#0b0b0b',
            bbox=dict(boxstyle='round,pad=0.25',fc='#fcfcfb',ec='none',alpha=0.85),zorder=6)
ax.autoscale_view(); ax.set_aspect('equal'); ax.axis('off')
from matplotlib.patches import Patch
h=[Patch(facecolor=c,label=f'{l} に初めて取得') for l,_,c in bins]+[Patch(facecolor='#e6e5e0',label='未取得（10/13 時点）'),
   Patch(facecolor=YELLOW,label='手動でインデックス登録をリクエストした50件（すべて取得済み）')]
from matplotlib.lines import Line2D
h.append(Line2D([0],[0],color='#2b2a27',lw=0.9,label='区の境界'))
ax.legend(handles=h,loc='upper left',bbox_to_anchor=(0,0),ncol=2,frameon=False,fontsize=8.5)
fig.suptitle('Googlebot が丁目ページを初めて取得した時期',x=0.02,ha='left',fontsize=13)
fig.text(0.02,0.93,'公開中の785丁目（練馬・中野・杉並・板橋・豊島・新宿）。2026年10月13日時点',fontsize=8.5,color='#52514e',ha='left')
fig.text(0.02,0.01,'herahin.net のアクセスログより作成。境界はすみか診断のデータ（東京都オープンデータ）',fontsize=8,color=MUTED,ha='left')
plt.tight_layout(rect=(0,0.08,1,0.92))
fig.savefig('fig/fig3_googlebot_spread_map.png',dpi=200)

# ---------- Fig3b: map colored by days from publish to Googlebot's first fetch
rings=json.load(open('input/rings.json'))
first={}
for r in rows:
    if r['n']=='googlebot' and ischome(r['path']):
        first[r['path']]=min(first.get(r['path'],r['t']),r['t'])
PUBN={'B2':datetime.datetime(2026,10,3,23,15),'B5':datetime.datetime(2026,10,9,10,20)}; PUBN['B3']=PUBN['B4']=PUBN['B2']
bins=[('2日以内',2,'#174f99'),('2〜3日',3,'#2a78d6'),('3〜5日',5,'#7aaeea'),('5日より後',99,'#c4daf5')]
lat0=math.radians(35.7)
fig,ax=plt.subplots(figsize=(9,8.8))
def poly(ring,**kw): return Polygon([(x*math.cos(lat0),y) for x,y in ring],closed=True,**kw)
for u,ring in rings.items():
    t=first.get(u); col='#e6e5e0'
    if t and L.get(u,{}).get('batch') in PUBN:
        dd=(t.replace(tzinfo=None)-PUBN[L[u]['batch']]).total_seconds()/86400
        for _,lim,c in bins:
            if dd<=lim: col=c; break
    ax.add_patch(poly(ring,facecolor=col,edgecolor='#fcfcfb',lw=0.4))
for u,ring in rings.items():
    if L.get(u,{}).get('batch')=='B1':
        ax.add_patch(poly(ring,facecolor=YELLOW,edgecolor='#fcfcfb',lw=0.4))
# ward boundaries: union of chome polygons per ward (small buffer closes slivers between chome)
from shapely.geometry import Polygon as SP
from shapely.ops import unary_union
wp=collections.defaultdict(list)
for u,ring in rings.items():
    g=SP([(x*math.cos(lat0),y) for x,y in ring]).buffer(0)
    wp[u.split('/')[3]].append(g.buffer(0.0004))
for w,gs in wp.items():
    g=unary_union(gs).buffer(-0.0004)
    for part in getattr(g,'geoms',[g]):
        x,y=part.exterior.xy
        ax.plot(x,y,color='#2b2a27',lw=1.0,solid_joinstyle='round',zorder=5)
# ward labels
wc=collections.defaultdict(list)
for u,ring in rings.items():
    w=u.split('/')[3]; xs=[p[0] for p in ring]; ys=[p[1] for p in ring]
    wc[w].append((sum(xs)/len(xs),sum(ys)/len(ys)))
WN={'nerima':'練馬区','nakano':'中野区','suginami':'杉並区','itabashi':'板橋区','toshima':'豊島区','shinjuku':'新宿区'}
for w,pts in wc.items():
    x=sum(p[0] for p in pts)/len(pts); y=sum(p[1] for p in pts)/len(pts)
    ax.text(x*math.cos(lat0),y,WN[w],fontsize=10,ha='center',va='center',color='#0b0b0b',
            bbox=dict(boxstyle='round,pad=0.25',fc='#fcfcfb',ec='none',alpha=0.85),zorder=6)
ax.autoscale_view(); ax.set_aspect('equal'); ax.axis('off')
from matplotlib.patches import Patch
h=[Patch(facecolor=c,label=f'公開から{l}に初めて取得') for l,_,c in bins]+[Patch(facecolor='#e6e5e0',label='未取得（10/13 時点）'),
   Patch(facecolor=YELLOW,label='手動でインデックス登録をリクエストした50件（すべて取得済み）')]
from matplotlib.lines import Line2D
h.append(Line2D([0],[0],color='#2b2a27',lw=0.9,label='区の境界'))
ax.legend(handles=h,loc='upper left',bbox_to_anchor=(0,0),ncol=2,frameon=False,fontsize=8.5)
fig.suptitle('公開から Googlebot が初めて取得するまでの日数',x=0.02,ha='left',fontsize=13)
fig.text(0.02,0.93,'公開中の785丁目。10/3 公開（練馬の残り・中野・杉並）は10日間、10/9 公開（板橋・豊島・新宿）は4.6日間の観察。2026年10月13日時点',fontsize=8.5,color='#52514e',ha='left')
fig.text(0.02,0.01,'herahin.net のアクセスログより作成。境界はすみか診断のデータ（東京都オープンデータ）',fontsize=8,color=MUTED,ha='left')
plt.tight_layout(rect=(0,0.08,1,0.92))
fig.savefig('fig/fig3b_googlebot_days_map.png',dpi=200)

# ---------- Fig3c: map colored by time from Googlebot's first sitemap read after publish to its first fetch
rings=json.load(open('input/rings.json'))
first={}
for r in rows:
    if r['n']=='googlebot' and ischome(r['path']):
        first[r['path']]=min(first.get(r['path'],r['t']),r['t'])
PUBN={'B2':datetime.datetime(2026,10,3,23,15),'B5':datetime.datetime(2026,10,9,10,20)}; PUBN['B3']=PUBN['B4']=PUBN['B2']
SMR=sorted(r['t'].replace(tzinfo=None) for r in rows if r['n']=='googlebot' and r['path'].startswith('/sitemap'))
SM={k:min(t for t in SMR if t>=v) for k,v in PUBN.items()}  # 公開後に Googlebot が最初に sitemap を読んだ時刻
bins=[('24時間以内',1,'#174f99'),('24〜48時間',2,'#2a78d6'),('2〜4日',4,'#7aaeea'),('4日より後',99,'#c4daf5')]
lat0=math.radians(35.7)
fig,ax=plt.subplots(figsize=(9,8.8))
def poly(ring,**kw): return Polygon([(x*math.cos(lat0),y) for x,y in ring],closed=True,**kw)
for u,ring in rings.items():
    t=first.get(u); col='#e6e5e0'
    if t and L.get(u,{}).get('batch') in PUBN:
        dd=(t.replace(tzinfo=None)-SM[L[u]['batch']]).total_seconds()/86400
        for _,lim,c in bins:
            if dd<=lim: col=c; break
    ax.add_patch(poly(ring,facecolor=col,edgecolor='#fcfcfb',lw=0.4))
for u,ring in rings.items():
    if L.get(u,{}).get('batch')=='B1':
        ax.add_patch(poly(ring,facecolor=YELLOW,edgecolor='#fcfcfb',lw=0.4))
# ward boundaries: union of chome polygons per ward (small buffer closes slivers between chome)
from shapely.geometry import Polygon as SP
from shapely.ops import unary_union
wp=collections.defaultdict(list)
for u,ring in rings.items():
    g=SP([(x*math.cos(lat0),y) for x,y in ring]).buffer(0)
    wp[u.split('/')[3]].append(g.buffer(0.0004))
for w,gs in wp.items():
    g=unary_union(gs).buffer(-0.0004)
    for part in getattr(g,'geoms',[g]):
        x,y=part.exterior.xy
        ax.plot(x,y,color='#2b2a27',lw=1.0,solid_joinstyle='round',zorder=5)
# highlight: first natural index (10/6)
HK='/chome/tokyo/nerima/hikawadai-3/'
if HK in rings:
    ring=rings[HK]; ax.add_patch(poly(ring,facecolor='none',edgecolor='#d6336c',lw=2.2,zorder=8))
    cx=sum(p[0] for p in ring)/len(ring)*math.cos(lat0); cy=sum(p[1] for p in ring)/len(ring)
    ax.annotate('氷川台三丁目\n10/6 初めて手動リクエストなしで\nインデックス登録',xy=(cx,cy),xytext=(-120,175),textcoords='offset points',fontsize=8.5,color='#0b0b0b',
        ha='right',va='bottom',multialignment='left',zorder=9,bbox=dict(boxstyle='round,pad=0.3',fc='#fcfcfb',ec='#d6336c',lw=1),
        arrowprops=dict(arrowstyle='-',color='#d6336c',lw=1.2))
# ward labels
wc=collections.defaultdict(list)
for u,ring in rings.items():
    w=u.split('/')[3]; xs=[p[0] for p in ring]; ys=[p[1] for p in ring]
    wc[w].append((sum(xs)/len(xs),sum(ys)/len(ys)))
WN={'nerima':'練馬区','nakano':'中野区','suginami':'杉並区','itabashi':'板橋区','toshima':'豊島区','shinjuku':'新宿区'}
for w,pts in wc.items():
    x=sum(p[0] for p in pts)/len(pts); y=sum(p[1] for p in pts)/len(pts)
    ax.text(x*math.cos(lat0),y,WN[w],fontsize=10,ha='center',va='center',color='#0b0b0b',
            bbox=dict(boxstyle='round,pad=0.25',fc='#fcfcfb',ec='none',alpha=0.85),zorder=6)
ax.autoscale_view(); ax.set_aspect('equal'); ax.axis('off')
from matplotlib.patches import Patch
h=[Patch(facecolor=c,label=f'sitemap を読んでから{l}に取得') for l,_,c in bins]+[Patch(facecolor='#e6e5e0',label='未取得（10/13 時点）'),
   Patch(facecolor=YELLOW,label='手動でインデックス登録をリクエストした50件（すべて取得済み）')]
from matplotlib.lines import Line2D
h.append(Line2D([0],[0],color='#2b2a27',lw=0.9,label='区の境界'))
ax.legend(handles=h,loc='upper left',bbox_to_anchor=(0,0),ncol=2,frameon=False,fontsize=8.5)
fig.suptitle('Googlebot が sitemap を読んでから、丁目ページを初めて取得するまで',x=0.02,ha='left',fontsize=13)
fig.text(0.02,0.945,'起点は公開後に初めて sitemap を読んだ時刻（10/3 公開分は 10/5 07:04、10/9 公開分は 10/10 15:45）。\n10/3 公開分は8.7日間、10/9 公開分は3.3日間の観察。2026年10月13日時点',fontsize=8.5,color='#52514e',ha='left',va='top')
fig.text(0.02,0.01,'herahin.net のアクセスログより作成。境界はすみか診断のデータ（東京都オープンデータ）',fontsize=8,color=MUTED,ha='left')
plt.tight_layout(rect=(0,0.08,1,0.9))
fig.savefig('fig/fig3c_googlebot_after_sitemap_map.png',dpi=200)
print('ok')
