v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

add array unique of webpage

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-03-03 20:03:48 +01:00
parent e42ffd98ae
commit d21af4f60a
1 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
web_scrap.py
View File

@ -18,6 +18,7 @@ if page.status_code == 200:
if href != "#": if href != "#":
page_url.append(href) page_url.append(href)
webpage = []
for i in page_url: for i in page_url:
page = requests.get(i) page = requests.get(i)
if page.status_code == 200: if page.status_code == 200:
@ -43,7 +44,11 @@ for i in page_url:
soup = BeautifulSoup(page.text, 'html.parser') soup = BeautifulSoup(page.text, 'html.parser')
h2 = soup.find_all("h2") h2 = soup.find_all("h2")
for title in h2: for title in h2:
print(title.find_all("a")[0].get("href", "/")) href = title.find_all("a")[0].get("href", "/")
if href not in webpage:
webpage.append(href)
print(webpage)