From bf5a5b7eb306aea6cc270a54f24b0cfcea1d6398 Mon Sep 17 00:00:00 2001
From: Valentin CZERYBA <valcze80@gmail.com>
Date: Thu, 9 Mar 2023 20:27:49 +0100
Subject: [PATCH] add exception for connection and createdir

---
 web_scrap.py | 46 ++++++++++++++++++++++++++++++++++++----------
 1 file changed, 36 insertions(+), 10 deletions(-)

diff --git a/web_scrap.py b/web_scrap.py
index d11f7f3..26fb8f4 100644
--- a/web_scrap.py
+++ b/web_scrap.py
@@ -12,11 +12,19 @@ def mkdirPath(path_dir, logger):
             repath = "/".join(makedir)
             if not os.path.exists(repath):
                 logger.debug("Dossier crée : {0}".format(repath))
-                os.mkdir(repath)
+                try:
+                    os.mkdir(repath)
+                except Exception as err:
+                    logger.error("Directory error : {0}".format(err))
+                    exit(1)
 
 
 def getScriptCss(url, js, css, logger):
-    page = requests.get(url)
+    try:
+        page = requests.get(url)
+    except Exception as err:
+        logger.error("Connection error : {0}".format(err))
+        exit(1)
     page_url = []
     if page.status_code == 200:
         soup = BeautifulSoup(page.text, 'html.parser')
@@ -51,7 +59,11 @@ def getScriptCss(url, js, css, logger):
 def getImg(webpage, logger):
     page_img = []
     for i in webpage:
-        page = requests.get(i)
+        try:
+            page = requests.get(i)
+        except Exception as err:
+            logger.error("Connection error : {0}".format(err))
+            exit(1)
         if page.status_code == 200:
             soup = BeautifulSoup(page.text, 'html.parser')
             img = soup.find_all("img")
@@ -67,7 +79,11 @@ def getImg(webpage, logger):
     return page_img
 
 def getUrlPage(url, logger):
-    page = requests.get(url)
+    try:
+        page = requests.get(url)
+    except Exception as err:
+            logger.error("Connection error : {0}".format(err))
+            exit(1)
     page_url = []
     if page.status_code == 200:
         soup = BeautifulSoup(page.text, 'html.parser')
@@ -79,7 +95,11 @@ def getUrlPage(url, logger):
 
     webpage = []
     for i in page_url:
-        page = requests.get(i)
+        try:
+            page = requests.get(i)
+        except Exception as err:
+            logger.error("Connection error : {0}".format(err))
+            exit(1)
         if page.status_code == 200:
             logger.info("page : {0}".format(i))
             if i not in webpage:
@@ -124,7 +144,11 @@ def downloadPage(webpage, backup_dir, logger):
         path_web.pop(len(path_web)-1)
         dir_page_web = "/".join(path_web)
         mkdirPath("{0}/{1}/{2}".format(backup_dir, o.netloc, dir_page_web), logger)
-        r = requests.get(webpage[i])
+        try:
+            r = requests.get(webpage[i])
+        except Exception as err:
+            logger.error("Connection error : {0}".format(err))
+            exit(1)
         if r.status_code == 200:
             fileDownload = "{0}/{1}/index.html".format(backup_dir, o.netloc)
             if len(dir_page_web) > 0 and len(filePageWeb) > 0:
@@ -159,10 +183,12 @@ if __name__ == '__main__':
     formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
     ch.setFormatter(formatter)
     logger.addHandler(ch)
-
-    o = urlparse(args.url)
-    o = o._replace(scheme="https")
-    url = o.geturl().replace(":///", "://")
+    try:
+        o = urlparse(args.url)
+        o = o._replace(scheme="https")
+        url = o.geturl().replace(":///", "://")
+    except Exception as err:
+        logger.error("Error : {0}".format(err))
     if args.js is False:
         script = getScriptCss(url, True, False, logger)
         downloadPage(script, "{0}/{1}/{2}".format(args.dir, o.path, "dists/js"), logger)