web_scrap.py

@@ -48,24 +48,6 @@ def getScriptCss(url, js, css, logger):
                 
     return page_url
 
-def getImg(webpage, logger):
-    page_img = []
-    for i in webpage:
-        page = requests.get(i)
-        if page.status_code == 200:
-            soup = BeautifulSoup(page.text, 'html.parser')
-            img = soup.find_all("img")
-            logger.info("image from page: {0} : ".format(i))
-            for anchor in img:
-                src = anchor.get("src", "/")
-                if src != "/":
-                    if src not in page_img:
-                        logger.info("image: {0} : ".format(src))
-                        page_img.append(src)
-
-
-    return page_img
-
 def getUrlPage(url, logger):
     page = requests.get(url)
     page_url = []
@@ -123,12 +105,12 @@ def downloadPage(webpage, backup_dir, logger):
         filePageWeb = path_web[len(path_web)-1]
         path_web.pop(len(path_web)-1)
         dir_page_web = "/".join(path_web)
-        mkdirPath("{0}/{1}/{2}".format(backup_dir, o.netloc, dir_page_web), logger)
+        mkdirPath("{0}/{1}".format(backup_dir, dir_page_web), logger)
         r = requests.get(webpage[i])
         if r.status_code == 200:
-            fileDownload = "{0}/{1}/index.html".format(backup_dir, o.netloc)
+            fileDownload = "{0}/index.html".format(backup_dir)
             if len(dir_page_web) > 0 and len(filePageWeb) > 0:
-                fileDownload = "{0}/{1}{2}/{3}".format(backup_dir, o.netloc, dir_page_web, filePageWeb)
+                fileDownload = "{0}{1}/{2}".format(backup_dir, dir_page_web, filePageWeb)
             logger.info("{0}/{1} : {2}".format(i+1, len(webpage), fileDownload))
             open(fileDownload, "wb").write(r.content)
 
@@ -140,11 +122,6 @@ if __name__ == '__main__':
                       default="backup",
                       help="backup file path")
     parser.add_argument("--debug", help="Verbosity",  action="store_true")
-    parser.add_argument("--no-css", help="No CSS", dest="css",  action="store_true")
-    parser.add_argument("--no-js", help="No JS", dest="js",  action="store_true")
-    parser.add_argument("--no-img", help="No img", dest="img",  action="store_true")
-    parser.add_argument("--no-html", help="No HTML", dest="html", action="store_true")
-
     args = parser.parse_args()
     logger = logging.getLogger('web_scrap')
     ch = logging.StreamHandler()
@@ -163,21 +140,10 @@ if __name__ == '__main__':
     o = urlparse(args.url)
     o = o._replace(scheme="https")
     url = o.geturl().replace(":///", "://")
-    if args.js is False:
+    script = getScriptCss(url, True, False, logger)
-        script = getScriptCss(url, True, False, logger)
+    downloadPage(script, "{0}/{1}".format(args.dir, "dists/js"), logger)
-        downloadPage(script, "{0}/{1}".format(args.dir, "dists/js"), logger)
+    css = getScriptCss(url, False, True, logger)
-    
+    downloadPage(css, "{0}/{1}".format(args.dir, "dists/css"), logger)
-    if args.css is False:
+    webpage = getUrlPage(url, logger)
-        css = getScriptCss(url, False, True, logger)
-        downloadPage(css, "{0}/{1}".format(args.dir, "dists/css"), logger)
-    
-    if args.html is False or args.img is False:
-        webpage = getUrlPage(url, logger)
-    if args.html is False:
-        downloadPage(webpage, args.dir, logger)
-    
-    if args.img is False:
-        page_src = getImg(webpage, logger)
-        downloadPage(page_src, "{0}/{1}".format(args.dir, "img"), logger)
 
-    #
+    downloadPage(script, args.dir, logger)