v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

create function for every task

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-03-05 20:12:58 +01:00
parent 4de811c607
commit a3aceccba7
1 changed files with 62 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
web_scrap.py
View File

@ -19,13 +19,14 @@ def mkdir_path(path_dir):
if not os.path.exists(repath): if not os.path.exists(repath):
os.mkdir(repath) os.mkdir(repath)
mkdir_path(BACKUP_DIR)
page = requests.get("https://{0}".format(URL))
page_url = [] def getUrlPage(url):
if page.status_code == 200: print(url)
page = requests.get(url)
page_url = []
if page.status_code == 200:
soup = BeautifulSoup(page.text, 'html.parser') soup = BeautifulSoup(page.text, 'html.parser')
ul = soup.find_all("ul", id="listsmooth") ul = soup.find_all("ul", id="listsmooth")
for anchor in ul[0].find_all("a"): for anchor in ul[0].find_all("a"):
@ -33,11 +34,13 @@ if page.status_code == 200:
if href != "#": if href != "#":
page_url.append(href) page_url.append(href)
webpage = [] webpage = []
for i in page_url: for i in page_url:
page = requests.get(i) page = requests.get(i)
if page.status_code == 200: if page.status_code == 200:
print("page : {0}".format(i)) print("page : {0}".format(i))
if i not in webpage:
webpage.append(i)
soup = BeautifulSoup(page.text, 'html.parser') soup = BeautifulSoup(page.text, 'html.parser')
class_div = pagingfirstline = soup.find_all("div", class_="pagingfirstline") class_div = pagingfirstline = soup.find_all("div", class_="pagingfirstline")
if len(class_div) > 0: if len(class_div) > 0:
@ -50,10 +53,12 @@ for i in page_url:
for j in range(1,int(number_lastpage)): for j in range(1,int(number_lastpage)):
paging = j * 10 paging = j * 10
categorie = urlparse(i).path.split("/") categorie = urlparse(i).path.split("/")
url_paging = "https://{0}/archives/p{1}-10.html".format(URL, paging) url_paging = "{0}/archives/p{1}-10.html".format(url, paging)
if len(categorie) > 2: if len(categorie) > 2:
url_paging = "https://{0}/archives/{1}/p{2}-10.html".format(URL, categorie[2], paging) url_paging = "{0}/archives/{1}/p{2}-10.html".format(url, categorie[2], paging)
print(url_paging) print(url_paging)
if url_paging not in webpage:
webpage.append(url_paging)
page = requests.get(url_paging) page = requests.get(url_paging)
if page.status_code == 200: if page.status_code == 200:
soup = BeautifulSoup(page.text, 'html.parser') soup = BeautifulSoup(page.text, 'html.parser')
@ -61,11 +66,24 @@ for i in page_url:
for title in h2: for title in h2:
href = title.find_all("a")[0].get("href", "/") href = title.find_all("a")[0].get("href", "/")
if href not in webpage: if href not in webpage:
webpage.append(href) o = urlparse(href)
o = o._replace(scheme="https").geturl()
webpage.append(o)
return webpage
for i in webpage:
def downloadPage(url):
o = urlparse(url)
o = o._replace(scheme="https")
o = o._replace(fragment="")
webpage = getUrlPage(o.geturl().replace(":///", "://"))
for i in webpage:
o = urlparse(i) o = urlparse(i)
path_web = o.path.split("/") path_web = o.path.split("/")
path_web.pop(len(path_web)-1) path_web.pop(len(path_web)-1)
dir_page_web = "/".join(path_web) dir_page_web = "/".join(path_web)
mkdir_path("{0}/{1}".format(BACKUP_DIR, dir_page_web)) mkdir_path("{0}/{1}".format(BACKUP_DIR, dir_page_web))
if __name__ == '__main__':
downloadPage(URL)