download css

web_scrap.py

@@ -15,6 +15,39 @@ def mkdirPath(path_dir, logger):
                 os.mkdir(repath)
 
 
+def getScriptCss(url, js, css, logger):
+    page = requests.get(url)
+    page_url = []
+    if page.status_code == 200:
+        soup = BeautifulSoup(page.text, 'html.parser')
+        if js is True:
+            script = soup.find_all("script")
+            for anchor in script:
+                src = anchor.get("src", "/")
+                if src != "/":
+                    u = urlparse(url)
+                    o = urlparse(src)
+                    if o.netloc == "":
+                        o = o._replace(netloc=u.netloc)
+                        o = o._replace(scheme=u.scheme)
+                    page_url.append(o.geturl())
+        if css is True:
+            link = soup.find_all("link")
+            for anchor in link:
+                rel = anchor.get("rel")
+                if rel[0] == "stylesheet":
+                    href = anchor.get("href", "/")
+                    if href != "/":
+                        u = urlparse(url)
+                        o = urlparse(href)
+                        if o.netloc == "":
+                            o = o._replace(netloc=u.netloc)
+                            o = o._replace(scheme=u.scheme)
+                        page_url.append(o.geturl())
+                
+                
+    return page_url
+
 def getUrlPage(url, logger):
     page = requests.get(url)
     page_url = []
@@ -64,7 +97,7 @@ def getUrlPage(url, logger):
     return webpage
 
 
-def downloadPageHTML(webpage, backup_dir, logger):
+def downloadPage(webpage, backup_dir, logger):
    
     for i in range(0, len(webpage)):
         o = urlparse(webpage[i])
@@ -78,7 +111,7 @@ def downloadPageHTML(webpage, backup_dir, logger):
             fileDownload = "{0}/index.html".format(backup_dir)
             if len(dir_page_web) > 0 and len(filePageWeb) > 0:
                 fileDownload = "{0}{1}/{2}".format(backup_dir, dir_page_web, filePageWeb)
-            logger.info("{0}/{1} : {2}".format(i, len(webpage), fileDownload))
+            logger.info("{0}/{1} : {2}".format(i+1, len(webpage), fileDownload))
             open(fileDownload, "wb").write(r.content)
 
 
@@ -106,6 +139,11 @@ if __name__ == '__main__':
 
     o = urlparse(args.url)
     o = o._replace(scheme="https")
-    webpage = getUrlPage(o.geturl().replace(":///", "://"), logger)
+    url = o.geturl().replace(":///", "://")
+    script = getScriptCss(url, True, False, logger)
+    downloadPage(script, "{0}/{1}".format(args.dir, "dists/js"), logger)
+    css = getScriptCss(url, False, True, logger)
+    downloadPage(css, "{0}/{1}".format(args.dir, "dists/css"), logger)
+    #webpage = getUrlPage(url, logger)
 
-    downloadPageHTML(webpage, args.dir, logger)
+    #downloadPage(script, "{0}/{1}".format(args.dir, "dists/js"), logger)