from selenium import webdriver
import time
driver = webdriver.Chrome()
driver.get('https://www.csdn.net/nav/python')
#下拉若干次
for i in range(10):
        driver.execute_script('window.scrollTo(0,document.body.scrollHeight)')
	time.sleep(1)
#定位所有链接
blog_url = driver.find_elements_by_css_selector('div.title > h2 > a') #注意:这里保存的是所有element对象
for i in range(len(blog_url)):
	url = blog_url[i].get_attribute('href')
	driver.get(url)
	#------------相关信息爬取(省略)----------
	driver.back() #返回目录页

for i in range(len(blog_url)):
	blog_refind_url = driver.find_elements_by_css_selector('div.title > h2 > a')  #重新定位
	url = blog_refind_url[i].get_attribute('href')
	driver.get(url)
	#------------相关信息爬取(省略)----------
	driver.back() #返回目录页

from selenium import webdriver
import os
os.chdir('C:/Users/dell/Desktop')
import time
import pandas as pd
def scroll_down(driver,num):
    for i in range(num):
        driver.execute_script('window.scrollTo(0,document.body.scrollHeight)')
        time.sleep(1)
def save_data(df):
    data=pd.DataFrame(df,columns=['blog_name','code_time','blog_num',
                                'view_num','fans_num','likes_num',
                                'comments_num','collections_num'])
    data.to_csv('csdn_user.csv',index=False,encoding='gb18030')
def crawler_csdn(parts_list):
    opt = webdriver.ChromeOptions()
    opt.add_experimental_option('excludeSwitches',['enable-automation'])
    opt.add_argument('--headless')
    opt.add_argument('--disable-gpu')
    opt.add_argument('blink-settings=imagesEnabled=false')
    driver = webdriver.Chrome(options=opt)
    df = []
    for part in parts_list:
        count=0
        url_des='https://www.csdn.net/nav/'+part
        driver.get(url_des)
        scroll_down(driver,30)
        time.sleep(2)
        print('开始爬取{}部分'.format(part))
        blog_list=[]
        blog_url = driver.find_elements_by_css_selector('div.title > h2 > a')
        for url in blog_url:
            blog_list.append(url.get_attribute('href'))
        print('共{}个博主'.format(len(blog_list)))
        for i in range(len(blog_list)):
            try:
                driver.get(blog_list[i])
                blog_name = driver.find_element_by_css_selector('div.profile-intro-name-boxTop > a >span.name').text
                code_time = driver.find_element_by_css_selector('span.personal-home-page.personal-home-years').text
                blog_num = driver.find_element_by_css_selector(
                    'div.data-info.d-flex.item-tiling>dl.text-center>a>dt>span.count').text
                inf_list = driver.find_elements_by_css_selector('div.data-info.d-flex.item-tiling>dl.text-center>dt>span.count')
                df.append([blog_name, code_time, blog_num,
                           inf_list[0].text, inf_list[2].text, inf_list[3].text,
                           inf_list[4].text, inf_list[5].text])
                count += 1
                print('第{}个博主信息爬取完成'.format(count))
            except:
                print('相关信息不全')
        print('{}部分爬取完成'.format(part))
    return df
if __name__ =='__main__':
    start = time.time()
    parts_list=['Python','Java','web','arch','db']
    df = crawler_csdn(parts_list)
    save_data(df)
    end = time.time()
    spend_time = int((end-start)/60)
    print('共花费{}分钟'.format(spend_time))

{"startUrl":"https://blog.csdn.net/nav/python","selectors":[{"parentSelectors":
["_root"],"type":"SelectorElementScroll","multiple":true,"id":"container","selector":"ul.feedlist_mod li.clearfix:nth-of-type(-n+300)","delay":"2000"},{"parentSelectors":
["container"],"type":"SelectorLink","multiple":false,"id":"inf","selector":"h2 a","delay":""},{"parentSelectors":
["inf"],"type":"SelectorText","multiple":false,"id":"name","selector":"div.profile-intro-name-boxTop span.name","regex":"","delay":""},{"parentSelectors":
["inf"],"type":"SelectorText","multiple":false,"id":"blog_num","selector":"dl.text-center:nth-of-type(1) a span.count","regex":"","delay":""},{"parentSelectors":
["inf"],"type":"SelectorText","multiple":false,"id":"code_time","selector":"span.personal-home-page.personal-home-years","regex":"","delay":""},{"parentSelectors":
["inf"],"type":"SelectorText","multiple":false,"id":"views_num","selector":"div.data-info:nth-of-type(2) dl.text-center > dt span.count","regex":"","delay":""},{"parentSelectors":
["inf"],"type":"SelectorText","multiple":false,"id":"fans","selector":"dl#fanBox.text-center span.count","regex":"","delay":""},{"parentSelectors":
["inf"],"type":"SelectorText","multiple":false,"id":"agreement","selector":"dl.text-center:nth-of-type(3) > dt span.count","regex":"","delay":""},{"parentSelectors":
["inf"],"type":"SelectorText","multiple":false,"id":"comment","selector":"div.data-info:nth-of-type(4) dl.text-center:nth-of-type(4) span.count","regex":"","delay":""},{"parentSelectors":["inf"],"type":"SelectorText","multiple":false,"id":"collection_num","selector":"dl.text-center:nth-of-type(5) span.count","regex":"","delay":""}],"_id":"csdn"}