diff --git a/web_scrap.py b/web_scrap.py index d08dfbd..949a884 100644 --- a/web_scrap.py +++ b/web_scrap.py @@ -3,7 +3,7 @@ from bs4 import BeautifulSoup from urllib.parse import urlparse import requests, os, argparse, logging -def mkdir_path(path_dir, logger): +def mkdirPath(path_dir, logger): if not os.path.exists(path_dir): makedir = [] pathh = path_dir.split("/") @@ -64,22 +64,20 @@ def getUrlPage(url, logger): return webpage -def downloadPage(url, backup_dir, logger): - o = urlparse(url) - o = o._replace(scheme="https") - webpage = getUrlPage(o.geturl().replace(":///", "://"), logger) +def downloadPageHTML(webpage, backup_dir, logger): + for i in range(0, len(webpage)): o = urlparse(webpage[i]) path_web = o.path.split("/") filePageWeb = path_web[len(path_web)-1] path_web.pop(len(path_web)-1) dir_page_web = "/".join(path_web) - mkdir_path("{0}/{1}".format(backup_dir, dir_page_web), logger) + mkdirPath("{0}/{1}".format(backup_dir, dir_page_web), logger) r = requests.get(webpage[i]) if r.status_code == 200: fileDownload = "{0}/index.html".format(backup_dir) if len(dir_page_web) > 0 and len(filePageWeb) > 0: - fileDownload = "{0}/{1}/{2}".format(backup_dir, dir_page_web, filePageWeb) + fileDownload = "{0}{1}/{2}".format(backup_dir, dir_page_web, filePageWeb) logger.info("{0}/{1} : {2}".format(i, len(webpage), fileDownload)) open(fileDownload, "wb").write(r.content) @@ -95,7 +93,7 @@ if __name__ == '__main__': logger = logging.getLogger('web_scrap') ch = logging.StreamHandler() - if args.debug is not None: + if args.debug is True: logger.setLevel(logging.DEBUG) ch.setLevel(logging.DEBUG) else: @@ -106,4 +104,8 @@ if __name__ == '__main__': ch.setFormatter(formatter) logger.addHandler(ch) - downloadPage(args.url, args.dir, logger) \ No newline at end of file + o = urlparse(args.url) + o = o._replace(scheme="https") + webpage = getUrlPage(o.geturl().replace(":///", "://"), logger) + + downloadPageHTML(webpage, args.dir, logger) \ No newline at end of file