v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

re-organisation fonction et renommage

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-03-07 21:54:18 +01:00
parent e809e376e5
commit 559f5f1e83
1 changed files with 11 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
web_scrap.py
View File

@ -3,7 +3,7 @@ from bs4 import BeautifulSoup
from urllib.parse import urlparse
import requests, os, argparse, logging
def mkdir_path(path_dir, logger):
def mkdirPath(path_dir, logger):
if not os.path.exists(path_dir):
makedir = []
pathh = path_dir.split("/")
@ -64,22 +64,20 @@ def getUrlPage(url, logger):
return webpage
def downloadPage(url, backup_dir, logger):
o = urlparse(url)
o = o._replace(scheme="https")
webpage = getUrlPage(o.geturl().replace(":///", "://"), logger)
def downloadPageHTML(webpage, backup_dir, logger):
for i in range(0, len(webpage)):
o = urlparse(webpage[i])
path_web = o.path.split("/")
filePageWeb = path_web[len(path_web)-1]
path_web.pop(len(path_web)-1)
dir_page_web = "/".join(path_web)
mkdir_path("{0}/{1}".format(backup_dir, dir_page_web), logger)
mkdirPath("{0}/{1}".format(backup_dir, dir_page_web), logger)
r = requests.get(webpage[i])
if r.status_code == 200:
fileDownload = "{0}/index.html".format(backup_dir)
if len(dir_page_web) > 0 and len(filePageWeb) > 0:
fileDownload = "{0}/{1}/{2}".format(backup_dir, dir_page_web, filePageWeb)
fileDownload = "{0}{1}/{2}".format(backup_dir, dir_page_web, filePageWeb)
logger.info("{0}/{1} : {2}".format(i, len(webpage), fileDownload))
open(fileDownload, "wb").write(r.content)
@ -95,7 +93,7 @@ if __name__ == '__main__':
logger = logging.getLogger('web_scrap')
ch = logging.StreamHandler()
if args.debug is not None:
if args.debug is True:
logger.setLevel(logging.DEBUG)
ch.setLevel(logging.DEBUG)
else:
@ -106,4 +104,8 @@ if __name__ == '__main__':
ch.setFormatter(formatter)
logger.addHandler(ch)
downloadPage(args.url, args.dir, logger)
o = urlparse(args.url)
o = o._replace(scheme="https")
webpage = getUrlPage(o.geturl().replace(":///", "://"), logger)
downloadPageHTML(webpage, args.dir, logger)