PythonでAmazonをWebスクレイピングする方法

pandas を pdとして インポートする
npとしてnumpy をインポート
importmatplotlib.pyplotasplt
importseabornassns
matplotlib インライン
インポートre
importtime
fromdatetimeimportdatetime
importmatplotlib.datesasmdates
importmatplotlib.tickerasticker
fromurllib.requestimporturlopen
frombs4importBeautifulSoup
インポートリクエスト

no_pages = 2
 
def get_data(pageNo):  
    headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0", "Accept-Encoding":"gzip, deflate", "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "DNT":"1","Connection":"close", "Upgrade-Insecure-Requests":"1"}
 
    r = requests.get('https://www.amazon.in/gp/bestsellers/books/ref=zg_bs_pg_'+str(pageNo)+'?ie=UTF8&pg='+str(pageNo), headers=headers)#, proxies=proxies)
    content = r.content
    soup = BeautifulSoup(content)
    #print(soup)
 
    alls = []
    for d in soup.findAll('div', attrs={'class':'a-section a-spacing-none aok-relative'}):
        #print(d)
        name = d.find('span', attrs={'class':'zg-text-center-align'})
        n = name.find_all('img', alt=True)
        #print(n[0]['alt'])
        author = d.find('a', attrs={'class':'a-size-small a-link-child'})
        rating = d.find('span', attrs={'class':'a-icon-alt'})
        users_rated = d.find('a', attrs={'class':'a-size-small a-link-normal'})
        price = d.find('span', attrs={'class':'p13n-sc-price'})
 
        all1=[]
 
        if name is not None:
            #print(n[0]['alt'])
            all1.append(n[0]['alt'])
        else:
            all1.append("unknown-product")
 
        if author is not None:
            #print(author.text)
            all1.append(author.text)
        elif author is None:
            author = d.find('span', attrs={'class':'a-size-small a-color-base'})
            if author is not None:
                all1.append(author.text)
            else:    
                all1.append('0')
 
        if rating is not None:
            #print(rating.text)
            all1.append(rating.text)
        else:
            all1.append('-1')
 
        if users_rated is not None:
            #print(price.text)
            all1.append(users_rated.text)
        else:
            all1.append('0')     
 
        if price is not None:
            #print(price.text)
            all1.append(price.text)
        else:
            all1.append('0')
        alls.append(all1)    
    return alls

for i in range(1,no_pages+1)：
    results.append(get_data(i))
flatten = lambda l:[item for sublist in l for item in sublist].
df = pd.DataFrame(flatten(results),columns=['Book Name','Author','Rating','Customers_Rated','Price'])
df.to_csv('amazon_products.csv', index=False, encoding='utf-8')

df =pd.read_csv("amazon_products.csv")
df.shape

df.head(61)

df['Rating']= df['Rating'].apply(lambda x: x.split()[0])
df['Rating']= pd.to_numeric(df['Rating'])
df["Price"]= df["Price"].str.replace('₹','')
df["Price"] = df["Price"].str.replace(',','')
df['Price']= df['Price'].apply(lambda x: x.split('.')[0])
df['Price']= df['Price'].astype(int)
df["Customers_Rated"]= df["Customers_Rated"].str.replace(',','')
df['Customers_Rated']= pd.to_numeric(df['Customers_Rated'], errors='ignore')
df.head()

df.replace(str(0), np.nan,inplace=True)
df.replace(0, np.nan,inplace=True)
count_nan = len(df) - df.count()
count_nan

df =df.dropna()

data=data.sort_values(['評価'],axis=0,ascending=False)[:15].
データ

data= df[df['Customers_Rated'] > 1000].
data=data.sort_values(['評価'],axis=0,ascending=False)[:15].
データ

p = figure(x_range=data.iloc[:,0],plot_width=800,plot_height=600,title="顧客評価1000人以上の人気書籍",toolbar_location=None,tools="")
 
p.vbar(x=data.iloc[:,0],top=data.iloc[:,2],width=0.9)
 
p.xgrid.grid_line_color = None
p.y_range.start = 0
p.xaxis.major_label_orientation = math.pi/2
show(p)

from bokeh.transformimport factor_cmap
from bokeh.modelsimport レジェンド
from bokeh.palettesimport Dark2_5 as palette（パレットとしてDark2_5をインポート
インポート itertools
from bokeh.palettesimport d3
#colorsは プロットで使用できる色のリストです。
colors = itertools.cycle(palette)
 
パレット = d3['Category20'][20］
index_cmap =factor_cmap('Author', palette=palette、
                         factors=data["Author"])
p=figure(plot_width=700,plot_height=700, title = "トップ著者：評価 vs. 評価された顧客")
p.scatter('Rating','Customers_Rated',source=data,fill_alpha=0.6, fill_color=index_cmap,size=20,legend='Author')
p.xaxis.axis_label='RATING'
p.yaxis.axis_label='CUSTOMERS RATED'
p.legend.location='top_left'
 
表示(p)