一个python selenium的实用实例，比demo重，但也不算太复杂。
trick总结如下：

最新chromedriver的地址，https://googlechromelabs.github.io/chrome-for-testing，这很重要，不然就要处理chrome自动更新之类的烦人问题。很多下载来源都有点过时。
用options配置webdriver的地址，以及socks5代理。
driver.page_source可以打印当前html，可以辅助判断执行进度。
XPath (XML Path Language) 是一门在 XML 文档中通过元素和属性进行导航的语言，属于w3c标准。HTML是标准的XML，所以HTML也可以使用XPath。XPATH选择器的发挥比较稳定。
python有个traceback模块，print(traceback.format_exc())可以原样raise exception的同时，忽略exception。

爬虫项目，遇到了element not interactable，可能是尝试点击的时候元素还未可见。
解决办法就是在 until.elementLocated 之后再添加一个 until.elementIsVisible 的判断。
我这的原因是，网页没有做页面适配，如果窗口太小，按钮是看不见的。所以治本的解决方案是把窗口最大化，上面的判断只能起到问题诊断的作用而非治疗作用。
把sleep都换成until，优雅一点。

# 如果前端页面改了的话，需要修改相应xpath，页面改动大的话需要修改代码import selenium.common.exceptions
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.wait import WebDriverWait
# 根据网页标题、网址以及元素是否存在/可见/可点击等条件来决定我们是否需要继续等待
from selenium.webdriver.support import expected_conditions as EC
import configparser
import csvclass Spider:def __init__(self):self.config = configparser.ConfigParser()self.config.read('config.ini')options = Options()options.binary_location = self.config['envs']['chrome_driver_location']options.add_argument('--proxy-server=' + self.config['envs']['proxy'])options.add_argument('--start-maximized')self.driver = webdriver.Chrome(options)def request(self, url):self.driver.get(url)def test_page(self):print(self.driver.page_source)def login(self):wait = WebDriverWait(self.driver, 10)username = self.config['envs']['username']password = self.config['envs']['password']xpath_username = self.config['xpath']['username_input']xpath_password = self.config['xpath']['password_input']xpath_login = self.config['xpath']['login_button']username_input = wait.until(EC.presence_of_element_located((By.XPATH, xpath_username)))password_input = wait.until(EC.presence_of_element_located((By.XPATH, xpath_password)))username_input.send_keys(username)password_input.send_keys(password)signin_button = self.driver.find_element(By.XPATH, xpath_login)signin_button.click()def close(self):self.driver.quit()def export_running_job(self):login = Falsewait = WebDriverWait(self.driver, 10)xpath_table = self.config['xpath']['table_body']xpath_template = self.config['xpath']['detail_button']xpath_groupid = self.config['xpath']['groupid_input']xpath_topiclist = self.config['xpath']['topiclist_pre']xpath_next = self.config['frontend']['next_button']xpath_close = self.config['xpath']['close_button']for key in self.config['urls']:url_login = self.config['login'][key]url_flink = self.config['urls'][key]if not login:self.request(url_login)self.login()login = Trueself.request(url_flink)self.driver.maximize_window()# print(xpath_template)rows = wait.until(EC.presence_of_all_elements_located((By.XPATH, xpath_table)))groupid_ls, topic_ls = [], []while True:segments = xpath_template.split('/')for i in range(1, len(rows) + 1):segments[-3] = 'tr[' + str(i) + ']'xpath_item = '/'.join(segments)# print(xpath_item)detail_button = wait.until(EC.visibility_of_element_located((By.XPATH, xpath_item)))detail_button.click()groupid = wait.until(EC.presence_of_element_located((By.XPATH, xpath_groupid)))self.driver.execute_script("arguments[0].removeAttribute('disabled')", groupid)groupid_ls.append(groupid.get_attribute("value"))topiclist = wait.until(EC.presence_of_element_located((By.XPATH, xpath_topiclist)))topic_ls.append(topiclist.get_attribute("innerHTML"))close_button = wait.until(EC.visibility_of_element_located((By.XPATH, xpath_close)))close_button.click()if len(rows) == int(self.config['frontend']['table_maxsize']):next_button = self.driver.find_element(By.XPATH, xpath_next)next_button.click()else:breakwith open(key+'.csv', 'wt') as f:cw = csv.writer(f, lineterminator='\n')for i in range(len(groupid_ls)):cw.writerow([groupid_ls[i], topic_ls[i]])self.close()if __name__ == '__main__':spider = Spider()spider.export_running_job()