add exception for connection and createdir

web_scrap.py

@@ -12,11 +12,19 @@ def mkdirPath(path_dir, logger):
             repath = "/".join(makedir)
             if not os.path.exists(repath):
                 logger.debug("Dossier crée : {0}".format(repath))
+                try:
                     os.mkdir(repath)
+                except Exception as err:
+                    logger.error("Directory error : {0}".format(err))
+                    exit(1)
 
 
 def getScriptCss(url, js, css, logger):
+    try:
         page = requests.get(url)
+    except Exception as err:
+        logger.error("Connection error : {0}".format(err))
+        exit(1)
     page_url = []
     if page.status_code == 200:
         soup = BeautifulSoup(page.text, 'html.parser')
@@ -51,7 +59,11 @@ def getScriptCss(url, js, css, logger):
 def getImg(webpage, logger):
     page_img = []
     for i in webpage:
+        try:
             page = requests.get(i)
+        except Exception as err:
+            logger.error("Connection error : {0}".format(err))
+            exit(1)
         if page.status_code == 200:
             soup = BeautifulSoup(page.text, 'html.parser')
             img = soup.find_all("img")
@@ -67,7 +79,11 @@ def getImg(webpage, logger):
     return page_img
 
 def getUrlPage(url, logger):
+    try:
         page = requests.get(url)
+    except Exception as err:
+            logger.error("Connection error : {0}".format(err))
+            exit(1)
     page_url = []
     if page.status_code == 200:
         soup = BeautifulSoup(page.text, 'html.parser')
@@ -79,7 +95,11 @@ def getUrlPage(url, logger):
 
     webpage = []
     for i in page_url:
+        try:
             page = requests.get(i)
+        except Exception as err:
+            logger.error("Connection error : {0}".format(err))
+            exit(1)
         if page.status_code == 200:
             logger.info("page : {0}".format(i))
             if i not in webpage:
@@ -124,7 +144,11 @@ def downloadPage(webpage, backup_dir, logger):
         path_web.pop(len(path_web)-1)
         dir_page_web = "/".join(path_web)
         mkdirPath("{0}/{1}/{2}".format(backup_dir, o.netloc, dir_page_web), logger)
+        try:
             r = requests.get(webpage[i])
+        except Exception as err:
+            logger.error("Connection error : {0}".format(err))
+            exit(1)
         if r.status_code == 200:
             fileDownload = "{0}/{1}/index.html".format(backup_dir, o.netloc)
             if len(dir_page_web) > 0 and len(filePageWeb) > 0:
@@ -159,10 +183,12 @@ if __name__ == '__main__':
     formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
     ch.setFormatter(formatter)
     logger.addHandler(ch)
-
+    try:
         o = urlparse(args.url)
         o = o._replace(scheme="https")
         url = o.geturl().replace(":///", "://")
+    except Exception as err:
+        logger.error("Error : {0}".format(err))
     if args.js is False:
         script = getScriptCss(url, True, False, logger)
         downloadPage(script, "{0}/{1}/{2}".format(args.dir, o.path, "dists/js"), logger)