使用正則提取數(shù)據(jù)，請求庫requests,看代碼，在存入數(shù)據(jù)庫時，報錯ERROR 1054 (42S22): Unknown column ‘title' in ‘field list'。原來是我寫sql 有問題，sql = “insert into poem(title,author,content,create_time) values({},{},{},{})”.format(title, author,content,crate_time)
應(yīng)該寫成sql = “insert into poem(title,author,content,create_time) values('{}','{}','{}','{}')”.format(title, author,content,crate_time)。
把插入的值放入引號中。
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
|
import datetimeimport reimport pymysqlimport requestsurl = "https://www.gushiwen.org/"headers = { 'User-Agent': "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50"}class Spiderpoem(object): conn = pymysql.Connect(host="localhost", port=3306, user="root", password='mysql', database='poem_data', charset="utf8") cs1 = conn.cursor() def get_requests(self, url, headers=None): """發(fā)送請求""" resp = requests.get(url, headers=headers) if resp.status_code == 200: # print(resp.request.headers) return resp.text return None def get_parse(self, response): """解析網(wǎng)頁""" re_data = { "title": r'<div\sclass="sons">.*?<b>(.*?)</b>.*?</div>', "author": r'<p>.*?class="source">.*?<a.*?>(.*?)</a>.*?<a.*?>(.*?)</a>.*?</p>', "content": r'<div\sclass="contson".*?>(.*?)</div>' } titles = self.reg_con(re_data["title"], response) authors = self.reg_con(re_data["author"], response) poems_list = self.reg_con(re_data["content"], response) contents = list() for item in poems_list: ite = re.sub(r'<.*?>|\s', "", item) contents.append(ite.strip()) for value in zip(titles, authors, contents): title, author, content = value author = "".join([author[0], '.', author[1]]) poem = { "title": title, "author": author, "content": content } yield poem def reg_con(self, params, response): """正則匹配""" if not response: return "請求錯誤" param = re.compile(params, re.DOTALL) # re.DOTALL 匹配換行等價于re.S result = re.findall(param, response) return result @classmethod def save_data(cls, poem): title = poem.get("title") author = poem.get("author") content = poem.get("content") crate_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") sql = "insert into poem(title,author,content,create_time) values('{}','{}','{}','{}')".format(title, author, content, crate_time) count = cls.cs1.execute(sql) print(count) cls.conn.commit() def main(self): resp = self.get_requests(url, headers) for it in self.get_parse(resp): self.save_data(it) self.cs1.close() self.conn.close()if __name__ == '__main__': Spiderpoem().main() |
總結(jié)
以上所述是小編給大家介紹的python 爬取古詩文存入mysql數(shù)據(jù)庫的方法，希望對大家有所幫助，如果大家有任何疑問請給我留言，小編會及時回復(fù)大家的。在此也非常感謝大家對服務(wù)器之家網(wǎng)站的支持！
如果你覺得本文對你有幫助，歡迎轉(zhuǎn)載，煩請注明出處，謝謝！
原文鏈接：https://blog.csdn.net/weixin_44224529/article/details/103841355
