v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

add exception for connection and createdir

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-03-09 20:27:49 +01:00
parent 2b3729a7bc
commit bf5a5b7eb3
1 changed files with 36 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
web_scrap.py
View File

@ -12,11 +12,19 @@ def mkdirPath(path_dir, logger):
repath = "/".join(makedir) repath = "/".join(makedir)
if not os.path.exists(repath): if not os.path.exists(repath):
logger.debug("Dossier crée : {0}".format(repath)) logger.debug("Dossier crée : {0}".format(repath))
os.mkdir(repath) try:
os.mkdir(repath)
except Exception as err:
logger.error("Directory error : {0}".format(err))
exit(1)
def getScriptCss(url, js, css, logger): def getScriptCss(url, js, css, logger):
page = requests.get(url) try:
page = requests.get(url)
except Exception as err:
logger.error("Connection error : {0}".format(err))
exit(1)
page_url = [] page_url = []
if page.status_code == 200: if page.status_code == 200:
soup = BeautifulSoup(page.text, 'html.parser') soup = BeautifulSoup(page.text, 'html.parser')
@ -51,7 +59,11 @@ def getScriptCss(url, js, css, logger):
def getImg(webpage, logger): def getImg(webpage, logger):
page_img = [] page_img = []
for i in webpage: for i in webpage:
page = requests.get(i) try:
page = requests.get(i)
except Exception as err:
logger.error("Connection error : {0}".format(err))
exit(1)
if page.status_code == 200: if page.status_code == 200:
soup = BeautifulSoup(page.text, 'html.parser') soup = BeautifulSoup(page.text, 'html.parser')
img = soup.find_all("img") img = soup.find_all("img")
@ -67,7 +79,11 @@ def getImg(webpage, logger):
return page_img return page_img
def getUrlPage(url, logger): def getUrlPage(url, logger):
page = requests.get(url) try:
page = requests.get(url)
except Exception as err:
logger.error("Connection error : {0}".format(err))
exit(1)
page_url = [] page_url = []
if page.status_code == 200: if page.status_code == 200:
soup = BeautifulSoup(page.text, 'html.parser') soup = BeautifulSoup(page.text, 'html.parser')
@ -79,7 +95,11 @@ def getUrlPage(url, logger):
webpage = [] webpage = []
for i in page_url: for i in page_url:
page = requests.get(i) try:
page = requests.get(i)
except Exception as err:
logger.error("Connection error : {0}".format(err))
exit(1)
if page.status_code == 200: if page.status_code == 200:
logger.info("page : {0}".format(i)) logger.info("page : {0}".format(i))
if i not in webpage: if i not in webpage:
@ -124,7 +144,11 @@ def downloadPage(webpage, backup_dir, logger):
path_web.pop(len(path_web)-1) path_web.pop(len(path_web)-1)
dir_page_web = "/".join(path_web) dir_page_web = "/".join(path_web)
mkdirPath("{0}/{1}/{2}".format(backup_dir, o.netloc, dir_page_web), logger) mkdirPath("{0}/{1}/{2}".format(backup_dir, o.netloc, dir_page_web), logger)
r = requests.get(webpage[i]) try:
r = requests.get(webpage[i])
except Exception as err:
logger.error("Connection error : {0}".format(err))
exit(1)
if r.status_code == 200: if r.status_code == 200:
fileDownload = "{0}/{1}/index.html".format(backup_dir, o.netloc) fileDownload = "{0}/{1}/index.html".format(backup_dir, o.netloc)
if len(dir_page_web) > 0 and len(filePageWeb) > 0: if len(dir_page_web) > 0 and len(filePageWeb) > 0:
@ -159,10 +183,12 @@ if __name__ == '__main__':
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter) ch.setFormatter(formatter)
logger.addHandler(ch) logger.addHandler(ch)
try:
o = urlparse(args.url) o = urlparse(args.url)
o = o._replace(scheme="https") o = o._replace(scheme="https")
url = o.geturl().replace(":///", "://") url = o.geturl().replace(":///", "://")
except Exception as err:
logger.error("Error : {0}".format(err))
if args.js is False: if args.js is False:
script = getScriptCss(url, True, False, logger) script = getScriptCss(url, True, False, logger)
downloadPage(script, "{0}/{1}/{2}".format(args.dir, o.path, "dists/js"), logger) downloadPage(script, "{0}/{1}/{2}".format(args.dir, o.path, "dists/js"), logger)