v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

create function for every task

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-03-05 20:12:58 +01:00
parent 4de811c607
commit a3aceccba7
1 changed files with 62 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
web_scrap.py
View File

@ -19,11 +19,12 @@ def mkdir_path(path_dir):
if not os.path.exists(repath):
os.mkdir(repath)
mkdir_path(BACKUP_DIR)
page = requests.get("https://{0}".format(URL))
def getUrlPage(url):
print(url)
page = requests.get(url)
page_url = []
if page.status_code == 200:
soup = BeautifulSoup(page.text, 'html.parser')
@ -38,6 +39,8 @@ for i in page_url:
page = requests.get(i)
if page.status_code == 200:
print("page : {0}".format(i))
if i not in webpage:
webpage.append(i)
soup = BeautifulSoup(page.text, 'html.parser')
class_div = pagingfirstline = soup.find_all("div", class_="pagingfirstline")
if len(class_div) > 0:
@ -50,10 +53,12 @@ for i in page_url:
for j in range(1,int(number_lastpage)):
paging = j * 10
categorie = urlparse(i).path.split("/")
url_paging = "https://{0}/archives/p{1}-10.html".format(URL, paging)
url_paging = "{0}/archives/p{1}-10.html".format(url, paging)
if len(categorie) > 2:
url_paging = "https://{0}/archives/{1}/p{2}-10.html".format(URL, categorie[2], paging)
url_paging = "{0}/archives/{1}/p{2}-10.html".format(url, categorie[2], paging)
print(url_paging)
if url_paging not in webpage:
webpage.append(url_paging)
page = requests.get(url_paging)
if page.status_code == 200:
soup = BeautifulSoup(page.text, 'html.parser')
@ -61,11 +66,24 @@ for i in page_url:
for title in h2:
href = title.find_all("a")[0].get("href", "/")
if href not in webpage:
webpage.append(href)
o = urlparse(href)
o = o._replace(scheme="https").geturl()
webpage.append(o)
return webpage
def downloadPage(url):
o = urlparse(url)
o = o._replace(scheme="https")
o = o._replace(fragment="")
webpage = getUrlPage(o.geturl().replace(":///", "://"))
for i in webpage:
o = urlparse(i)
path_web = o.path.split("/")
path_web.pop(len(path_web)-1)
dir_page_web = "/".join(path_web)
mkdir_path("{0}/{1}".format(BACKUP_DIR, dir_page_web))
if __name__ == '__main__':
downloadPage(URL)