v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

scrap href all page from gouter

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-03-02 23:28:04 +01:00
parent 991590f808
commit e42ffd98ae
1 changed files with 31 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
web_scrap.py
View File

@ -5,25 +5,45 @@
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
import requests import requests
page = requests.get("https://www.clarissariviere.com") URL = "www.clarissariviere.com"
page = requests.get("https://{0}".format(URL))
page_url = []
if page.status_code == 200: if page.status_code == 200:
soup = BeautifulSoup(page.text, 'html.parser') soup = BeautifulSoup(page.text, 'html.parser')
ul = soup.find_all("ul", id="listsmooth") ul = soup.find_all("ul", id="listsmooth")
for anchor in ul[0].find_all("a"): for anchor in ul[0].find_all("a"):
href = anchor.get('href', '/') href = anchor.get('href', '/')
if href != "#" and href != "http://www.clarissariviere.com/": if href != "#":
print(href) page_url.append(href)
for i in page_url:
for i in range(1,100): page = requests.get(i)
paging = i * 10
page = requests.get("https://www.clarissariviere.com/archives/p{0}-10.html".format(i))
soup = BeautifulSoup(page.text, 'html.parser')
if page.status_code == 200: if page.status_code == 200:
h2 = soup.find_all("h2") print("page : {0}".format(i))
for title in h2: soup = BeautifulSoup(page.text, 'html.parser')
print(title.find_all("a")[0].get("href", "/")) class_div = pagingfirstline = soup.find_all("div", class_="pagingfirstline")
if len(class_div) > 0:
pagingfirstline = class_div[0].find_all("a")
if len(pagingfirstline) > 1:
lastpage = pagingfirstline[len(pagingfirstline)-1].get("href", "/")
element_lastpage = lastpage.split("/")[len(lastpage.split("/"))-1]
number_page = element_lastpage.split("-")[0].split("p")[1]
number_lastpage = int(number_page) / 10
for j in range(1,int(number_lastpage)):
paging = j * 10
categorie = i.split("/")
url_paging = "https://{0}/archives/p{1}-10.html".format(URL, paging)
if len(categorie) != 4:
url_paging = "https://{0}/archives/{1}/p{2}-10.html".format(URL, categorie[4], paging)
print(url_paging)
page = requests.get(url_paging)
if page.status_code == 200:
soup = BeautifulSoup(page.text, 'html.parser')
h2 = soup.find_all("h2")
for title in h2:
print(title.find_all("a")[0].get("href", "/"))