web_scrap.py

@@ -15,39 +15,6 @@ def mkdirPath(path_dir, logger):
                 os.mkdir(repath)
 
 
-def getScriptCss(url, js, css, logger):
-    page = requests.get(url)
-    page_url = []
-    if page.status_code == 200:
-        soup = BeautifulSoup(page.text, 'html.parser')
-        if js is True:
-            script = soup.find_all("script")
-            for anchor in script:
-                src = anchor.get("src", "/")
-                if src != "/":
-                    u = urlparse(url)
-                    o = urlparse(src)
-                    if o.netloc == "":
-                        o = o._replace(netloc=u.netloc)
-                        o = o._replace(scheme=u.scheme)
-                    page_url.append(o.geturl())
-        if css is True:
-            link = soup.find_all("link")
-            for anchor in link:
-                rel = anchor.get("rel")
-                if rel[0] == "stylesheet":
-                    href = anchor.get("href", "/")
-                    if href != "/":
-                        u = urlparse(url)
-                        o = urlparse(href)
-                        if o.netloc == "":
-                            o = o._replace(netloc=u.netloc)
-                            o = o._replace(scheme=u.scheme)
-                        page_url.append(o.geturl())
-                
-                
-    return page_url
-
 def getUrlPage(url, logger):
     page = requests.get(url)
     page_url = []
@@ -97,7 +64,7 @@ def getUrlPage(url, logger):
     return webpage
 
 
-def downloadPage(webpage, backup_dir, logger):
+def downloadPageHTML(webpage, backup_dir, logger):
    
     for i in range(0, len(webpage)):
         o = urlparse(webpage[i])
@@ -111,7 +78,7 @@ def downloadPage(webpage, backup_dir, logger):
             fileDownload = "{0}/index.html".format(backup_dir)
             if len(dir_page_web) > 0 and len(filePageWeb) > 0:
                 fileDownload = "{0}{1}/{2}".format(backup_dir, dir_page_web, filePageWeb)
-            logger.info("{0}/{1} : {2}".format(i+1, len(webpage), fileDownload))
+            logger.info("{0}/{1} : {2}".format(i, len(webpage), fileDownload))
             open(fileDownload, "wb").write(r.content)
 
 
@@ -139,11 +106,6 @@ if __name__ == '__main__':
 
     o = urlparse(args.url)
     o = o._replace(scheme="https")
-    url = o.geturl().replace(":///", "://")
+    webpage = getUrlPage(o.geturl().replace(":///", "://"), logger)
-    script = getScriptCss(url, True, False, logger)
-    downloadPage(script, "{0}/{1}".format(args.dir, "dists/js"), logger)
-    css = getScriptCss(url, False, True, logger)
-    downloadPage(css, "{0}/{1}".format(args.dir, "dists/css"), logger)
-    #webpage = getUrlPage(url, logger)
 
-    #downloadPage(script, "{0}/{1}".format(args.dir, "dists/js"), logger)
+    downloadPageHTML(webpage, args.dir, logger)