实践|Spider_BQG(爬书)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
| import requests
from tqdm import tqdm
from bs4 import BeautifulSoup
def get_content(target):
req = requests.get(url=target)
req.encoding = 'utf-8'
html = req.text
bf = BeautifulSoup(html, 'lxml')
texts = bf.find('div', id='content')
content = texts.text.strip().split('\xa0' * 4)
return content
class spider(object):
def __init__(self, book_name, target):
self.__book_name = book_name
self.__target = target
self.__server = 'https://www.xsbiquge.com'
def do_project(self):
req = requests.get(url=self.__target)
req.encoding = 'utf-8'
html = req.text
chapter_bs = BeautifulSoup(html, 'lxml')
chapters = chapter_bs.find('div', id='list')
chapters = chapters.find_all('a')
for chapter in tqdm(chapters):
chapter_name = chapter.string
url = self.__server + chapter.get('href')
content = get_content(url)
with open(self.__book_name, 'a', encoding='utf-8') as f:
f.write(chapter_name)
f.write('\n')
f.write('\n'.join(content))
f.write('\n')
def main():
spider1 = spider("book_name.txt", "https://www.xsbiquge.com/88_88080/")
spider1.do_project()
print("运行结束")
if __name__ == '__main__':
main()
|