import csv,datetime,collections,json,math,random,statistics as st
import matplotlib; matplotlib.use('Agg')
import matplotlib.pyplot as plt
from matplotlib import font_manager as fm
for f in ['/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc','/usr/share/fonts/opentype/noto/NotoSansCJK-Medium.ttc']: fm.fontManager.addfont(f)
plt.rcParams.update({'font.family':'Noto Sans CJK JP','font.size':10,'axes.edgecolor':'#c9c8c3','axes.labelcolor':'#52514e',
 'xtick.color':'#52514e','ytick.color':'#52514e','text.color':'#0b0b0b','figure.facecolor':'#fcfcfb','axes.facecolor':'#fcfcfb',
 'savefig.facecolor':'#fcfcfb','axes.spines.top':False,'axes.spines.right':False})
BLUE,ORANGE,AQUA,YELLOW='#2a78d6','#eb6834','#1baf7a','#eda100'; MUTED='#8b8a85'; GRID='#e6e5e0'; INK2='#52514e'
exec(open('analyze.py').read())
tz=datetime.timezone(datetime.timedelta(hours=9))
for r in rows: r['t']=r['t'].replace(tzinfo=tz)
PUB={'B1':datetime.datetime(2026,9,14,9,0,tzinfo=tz),'B2':datetime.datetime(2026,10,3,23,15,tzinfo=tz),'B5':datetime.datetime(2026,10,9,10,20,tzinfo=tz)}
PUB['B3']=PUB['B4']=PUB['B2']
END=datetime.datetime(2026,10,14,0,0,tzinfo=tz)
MANUAL=set(u for u,x in L.items() if x['batch']=='B1')
BOTS=[('Googlebot','googlebot',BLUE),('Bingbot','bingbot',BLUE),('GoogleOther','googleother',BLUE),('ShapBot','shapbot',ORANGE),
      ('ClaudeBot','claudebot',ORANGE),('meta-externalagent','meta-externalagent',ORANGE),('Amazonbot','amazonbot',ORANGE),
      ('MJ12bot（SEO）','other:MJ12bot',AQUA),('SemrushBot（SEO）','semrushbot',AQUA),('serpstatbot（SEO）','other:serpstatbot',AQUA)]
def firsts(bot):
    f={}
    for r in rows:
        if r['n']==bot and ischome(r['path']) and r['path'] in L:
            if r['path'] not in f or r['t']<f[r['path']]: f[r['path']]=r['t']
    return f

# ---------- Fig4: days to first fetch per bot (manual vs not)
random.seed(1)
PANELS=[('手動リクエストあり：練馬区の50件（9/14 公開）',('B1',),34.5,[0,7,14,21,28]),('手動リクエストなし：735件（10/3・10/9 公開）',('B2','B3','B4','B5'),12.2,[0,2,4,6,8,10])]
fig,axs=plt.subplots(1,2,figsize=(10.5,6.4),sharey=True,gridspec_kw={'width_ratios':[1.15,1]})
for pi,(pl,bs,xmax,ticks) in enumerate(PANELS):
    ax=axs[pi]; tot=sum(1 for x in L.values() if x['batch'] in bs)
    for i,(lab,bot,col) in enumerate(BOTS):
        f=firsts(bot); ds=[]
        for u,t in f.items():
            if L[u]['batch'] not in bs: continue
            dd=(t-PUB[L[u]['batch']]).total_seconds()/86400; ds.append(dd)
            c=YELLOW if (pi==0 and bot in('googlebot','googleother')) else col
            ax.scatter(dd,i+random.uniform(-0.27,0.27),s=9,color=c,alpha=0.8,linewidths=0,zorder=3)
        if ds:
            m=st.median(ds); ax.plot([m,m],[i-0.38,i+0.38],color='#0b0b0b',lw=1.6,zorder=4)
        ax.text(xmax*0.995,i,f'{len(ds)}/{tot}',ha='right',va='center',fontsize=7.5,color=INK2 if ds else MUTED)
    ax.set_xlim(-xmax*0.02,xmax); ax.set_title(pl,fontsize=9.5,loc='left',color='#0b0b0b')
    ax.xaxis.grid(True,color=GRID,lw=0.8); ax.set_axisbelow(True); ax.tick_params(length=0); ax.spines['left'].set_visible(False)
    ax.set_xticks(ticks); ax.set_xticklabels([f'{v}日' for v in ticks],fontsize=8.5)
axs[0].text(10.5,6.75,'Google の2つは\n手動リクエストの影響を\n受けているので黄色',fontsize=7.5,color='#0b0b0b',va='center')
axs[0].set_yticks(range(len(BOTS))); axs[0].set_yticklabels([b[0] for b in BOTS]); axs[0].set_ylim(len(BOTS)-0.4,-0.7)
axs[1].text(10.2,9.6,'',fontsize=1)
fig.supxlabel('公開から初めて取得されるまでの日数（左右で横軸の幅が違う）',fontsize=9.5,color=INK2,y=0.045)
fig.suptitle('丁目ページが公開されてから、各クローラに初めて取得されるまで',x=0.02,ha='left',fontsize=13)
fig.text(0.02,0.925,'1つの点が1ページ。黒い縦線は中央値、右端の数字は取得したページ数／公開したページ数。10/13 までの集計なので、\n右側は 10/3 公開分が10日、10/9 公開分が4.6日までしか見ていない。左側の公開時刻は 9/14 9:00 とした',fontsize=8.2,color=INK2,ha='left',va='top')
fig.text(0.02,0.01,'herahin.net のアクセスログより作成',fontsize=8,color=MUTED,ha='left')
plt.tight_layout(rect=(0,0.03,1,0.87))
fig.savefig('fig/fig4_days_to_first_fetch.png',dpi=200)

# ---------- Fig5: coverage manual vs not
groups=[('手動リクエストあり（練馬区の50件）',('B1',),YELLOW),('手動リクエストなし（735件）',('B2','B3','B4','B5'),BLUE)]
bots5=BOTS
fig,ax=plt.subplots(figsize=(9,6.0))
H=0.36
for gi,(gl,bs,col) in enumerate(groups):
    tot=sum(1 for x in L.values() if x['batch'] in bs)
    for i,(lab,bot,_) in enumerate(bots5):
        f=firsts(bot); n=sum(1 for u in f if L[u]['batch'] in bs)
        v=n/tot*100; y=i+(gi-0.5)*(H+0.03)
        ax.barh(y,v,height=H,color=col,linewidth=0)
        ax.text(v+1,y,f'{v:.0f}%',va='center',fontsize=7.5,color=INK2)
ax.set_yticks(range(len(bots5))); ax.set_yticklabels([b[0] for b in bots5]); ax.invert_yaxis()
ax.set_xlim(0,112); ax.set_xticks([0,25,50,75,100]); ax.set_xticklabels(['0%','25%','50%','75%','100%'])
ax.xaxis.grid(True,color=GRID,lw=0.8); ax.set_axisbelow(True); ax.tick_params(length=0); ax.spines['left'].set_visible(False)
from matplotlib.patches import Patch
ax.legend(handles=[Patch(color=c,label=l) for l,_,c in groups],loc='lower right',frameon=False,fontsize=8.5)
fig.suptitle('各クローラが取得した丁目ページの割合',x=0.02,ha='left',fontsize=13)
fig.text(0.02,0.925,'10/13 時点。手動リクエストした50件は9月から公開していたので、来た時期の早いクローラにも取られている',fontsize=8.2,color=INK2,ha='left')
fig.text(0.02,0.01,'herahin.net のアクセスログより作成',fontsize=8,color=MUTED,ha='left')
plt.tight_layout(rect=(0,0.02,1,0.9))
fig.savefig('fig/fig5_coverage_by_batch.png',dpi=200)

# ---------- Fig7: Googlebot by ward
NB=set(json.load(open('input/b1_neighbors.json')))
B1U=[u for u,x in L.items() if x['batch']=='B1']
WARDS=[('練馬区：手動リクエストした50件',B1U,YELLOW),
       ('練馬区：その隣の丁目',[u for u,x in L.items() if x['batch']=='B2' and u in NB],BLUE),
       ('練馬区：それ以外',[u for u,x in L.items() if x['batch']=='B2' and u not in NB],BLUE),
       ('中野区',[u for u,x in L.items() if x['batch']=='B3'],BLUE),('杉並区',[u for u,x in L.items() if x['batch']=='B4'],BLUE),
       ('板橋区',[u for u,x in L.items() if x['batch']=='B5' and x['city']=='板橋区'],BLUE),
       ('豊島区',[u for u,x in L.items() if x['batch']=='B5' and x['city']=='豊島区'],BLUE),
       ('新宿区',[u for u,x in L.items() if x['batch']=='B5' and x['city']=='新宿区'],BLUE)]
gf=firsts('googlebot')
fig,axs=plt.subplots(1,2,figsize=(9.5,5.6),sharey=True)
for i,(wl,us,col) in enumerate(WARDS):
    got=[(gf[u]-PUB[L[u]['batch']]).total_seconds()/86400 for u in us if u in gf]
    p=len(got)/len(us)*100
    axs[0].barh(i,p,height=0.6,color=col); axs[0].text(p+1.5,i,f'{p:.0f}%（{len(got)}/{len(us)}）',va='center',fontsize=8.5,color=INK2)
    if i==0:
        axs[1].text(0.2,i,'手動リクエストしたので省略',va='center',fontsize=8,color=MUTED); continue
    for dd in got: axs[1].scatter(dd,i+random.uniform(-0.22,0.22),s=9,color=col,alpha=0.75,linewidths=0,zorder=3)
    if got:
        m=st.median(got); axs[1].plot([m,m],[i-0.34,i+0.34],color='#0b0b0b',lw=1.6,zorder=4)
        axs[1].text(10.4,i,f'中央値 {m:.1f}日',ha='right',va='center',fontsize=7.8,color='#0b0b0b')
axs[0].set_yticks(range(len(WARDS))); axs[0].set_yticklabels([w[0] for w in WARDS]); axs[0].set_ylim(len(WARDS)-0.45,-0.55)
axs[0].set_xlim(0,125); axs[0].set_xticks([0,25,50,75,100]); axs[0].set_xticklabels(['0%','25%','50%','75%','100%'])
axs[0].set_title('取得された丁目の割合（10/13 時点）',fontsize=9.5,loc='left')
axs[1].set_xlim(0,10.5); axs[1].set_xticks([0,2,4,6,8]); axs[1].set_xticklabels([f'{v}日' for v in [0,2,4,6,8]])
axs[1].set_title('公開から初めて取得されるまでの日数',fontsize=9.5,loc='left')
for ax in axs:
    ax.xaxis.grid(True,color=GRID,lw=0.8); ax.set_axisbelow(True); ax.tick_params(length=0); ax.spines['left'].set_visible(False)
    for yy in (0.5,4.5): ax.axhline(yy,color='#c9c8c3',lw=0.8)
axs[0].text(122,2.5,'10/3\n公開',ha='right',va='center',fontsize=7.5,color=MUTED)
axs[0].text(122,6,'10/9\n公開',ha='right',va='center',fontsize=7.5,color=MUTED)
fig.suptitle('Googlebot が丁目ページを取りに来た割合と早さ（区ごと）',x=0.02,ha='left',fontsize=13)
fig.text(0.02,0.925,'「その隣の丁目」は、手動リクエストした50件のどれかと境界が接している丁目。\n10/9 公開の3区は、公開から4.6日しかたっていない',fontsize=8.2,color=INK2,ha='left',va='top')
fig.text(0.02,0.01,'herahin.net のアクセスログより作成',fontsize=8,color=MUTED,ha='left')
plt.tight_layout(rect=(0,0.03,1,0.86))
fig.savefig('fig/fig7_googlebot_by_ward.png',dpi=200)
print('ok')

# ---------- Fig8: heatmap Googlebot daily pages by area
import matplotlib.colors as mcolors
def area(u):
    x=L[u]
    return '練馬区：手動リクエストした50件' if x['batch']=='B1' else x['city']
AREAS=['練馬区：手動リクエストした50件','練馬区','中野区','杉並区','板橋区','豊島区','新宿区']
SIZE=collections.Counter(area(u) for u in L)
g=[r for r in rows if r['n']=='googlebot' and ischome(r['path']) and r['path'] in L]
DAYS=[datetime.date(2026,10,3)+datetime.timedelta(d) for d in range(11)]
M=[[0]*len(DAYS) for _ in AREAS]
for j,d in enumerate(DAYS):
    seen=set()
    for r in g:
        if r['t'].date()==d and r['path'] not in seen:
            seen.add(r['path']); M[AREAS.index(area(r['path']))][j]+=1
cmap=mcolors.LinearSegmentedColormap.from_list('b',['#f1f0ec','#cfe0f6','#86b6ef','#2a78d6','#184f95'])
fig,ax=plt.subplots(figsize=(9.5,4.8))
for i,a in enumerate(AREAS):
    for j,_ in enumerate(DAYS):
        v=M[i][j]; p=v/SIZE[a]
        ax.add_patch(plt.Rectangle((j,i),0.94,0.9,color=cmap(min(p/0.2,1.0)) if v else '#f1f0ec',lw=0))
        if v: ax.text(j+0.47,i+0.45,str(v),ha='center',va='center',fontsize=8.5,color='#fcfcfb' if p>0.1 else '#0b0b0b')
    if i>0: ax.text(-0.15,i+0.45,'',fontsize=1)
ax.set_xlim(0,len(DAYS)); ax.set_ylim(len(AREAS),-0.1)
ax.set_xticks([j+0.47 for j in range(len(DAYS))]); ax.set_xticklabels([f'{d.month}/{d.day}' for d in DAYS],fontsize=8.5)
ax.set_yticks([i+0.45 for i in range(len(AREAS))]); ax.set_yticklabels([f'{a}（{SIZE[a]}）' for a in AREAS],fontsize=9)
for s in ax.spines.values(): s.set_visible(False)
ax.tick_params(length=0)
for x,lab in [(0,'10/3 23:15 練馬・中野・杉並を公開'),(6,'10/9 10:20 板橋・豊島・新宿を公開')]:
    ax.annotate('',xy=(x+0.05,-0.12),xytext=(x+0.05,-0.6),arrowprops=dict(arrowstyle='-',color=MUTED,lw=0.8),annotation_clip=False)
    ax.text(x+0.15,-0.42,lab,fontsize=7.8,color=INK2,va='center',clip_on=False)
ax.axhline(1,color='#fcfcfb',lw=3); ax.axhline(4,color='#c9c8c3',lw=0.8)
fig.suptitle('Googlebot が1日に取得した丁目ページ数（地域ごと）',x=0.02,ha='left',fontsize=13)
fig.text(0.02,0.925,'数字は重複を除いたページ数。色はその地域の丁目数に対する割合（濃いほど高い）。括弧内は地域の丁目数。\n手動リクエストした50件の 10/3〜10/7 は、リクエストに応じた取得を含む',fontsize=8.2,va='top',color=INK2,ha='left')
fig.text(0.02,0.01,'herahin.net のアクセスログより作成',fontsize=8,color=MUTED,ha='left')
plt.tight_layout(rect=(0,0.03,1,0.82))
fig.savefig('fig/fig8_googlebot_daily_by_area.png',dpi=200)
print('fig8')
