v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

logger web_scrap

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-03-06 22:54:32 +01:00
parent c7dc2d626f
commit 3ccebbac36
1 changed files with 26 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
web_scrap.py
View File

@ -1,9 +1,9 @@
#!/usr/bin/python3
from bs4 import BeautifulSoup
from urllib.parse import urlparse
import requests, os, argparse
import requests, os, argparse, logging
def mkdir_path(path_dir):
def mkdir_path(path_dir, logger):
if not os.path.exists(path_dir):
makedir = []
pathh = path_dir.split("/")
@ -11,10 +11,11 @@ def mkdir_path(path_dir):
makedir.append(i)
repath = "/".join(makedir)
if not os.path.exists(repath):
logger.debug("Dossier crée : {0}".format(repath))
os.mkdir(repath)
def getUrlPage(url):
def getUrlPage(url, logger):
page = requests.get(url)
page_url = []
if page.status_code == 200:
@ -29,7 +30,7 @@ def getUrlPage(url):
for i in page_url:
page = requests.get(i)
if page.status_code == 200:
print("page : {0}".format(i))
logger.info("page : {0}".format(i))
if i not in webpage:
webpage.append(i)
soup = BeautifulSoup(page.text, 'html.parser')
@ -47,7 +48,7 @@ def getUrlPage(url):
url_paging = "{0}/archives/p{1}-10.html".format(url, paging)
if len(categorie) > 2:
url_paging = "{0}/archives/{1}/p{2}-10.html".format(url, categorie[2], paging)
print(url_paging)
logger.info(url_paging)
if url_paging not in webpage:
webpage.append(url_paging)
page = requests.get(url_paging)
@ -63,23 +64,23 @@ def getUrlPage(url):
return webpage
def downloadPage(url, backup_dir):
def downloadPage(url, backup_dir, logger):
o = urlparse(url)
o = o._replace(scheme="https")
webpage = getUrlPage(o.geturl().replace(":///", "://"))
webpage = getUrlPage(o.geturl().replace(":///", "://"), logger)
for i in range(0, len(webpage)):
o = urlparse(webpage[i])
path_web = o.path.split("/")
filePageWeb = path_web[len(path_web)-1]
path_web.pop(len(path_web)-1)
dir_page_web = "/".join(path_web)
mkdir_path("{0}/{1}".format(backup_dir, dir_page_web))
mkdir_path("{0}/{1}".format(backup_dir, dir_page_web), logger)
r = requests.get(webpage[i])
if r.status_code == 200:
fileDownload = "{0}/index.html".format(backup_dir)
if len(dir_page_web) > 0 and len(filePageWeb) > 0:
fileDownload = "{0}/{1}/{2}".format(backup_dir, dir_page_web, filePageWeb)
print("{0}/{1} : {2}".format(i, len(webpage), fileDownload))
logger.info("{0}/{1} : {2}".format(i, len(webpage), fileDownload))
open(fileDownload, "wb").write(r.content)
@ -89,6 +90,20 @@ if __name__ == '__main__':
parser.add_argument("--dir",
default="backup",
help="backup file path")
parser.add_argument("--verbosity", help="Verbosity", action="store_false")
parser.add_argument("--debug", help="Verbosity", action="store_true")
args = parser.parse_args()
downloadPage(args.url, args.dir)
logger = logging.getLogger('web_scrap')
ch = logging.StreamHandler()
if args.debug is not None:
logger.setLevel(logging.DEBUG)
ch.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
ch.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)
downloadPage(args.url, args.dir, logger)