download script js
This commit is contained in:
parent
559f5f1e83
commit
1a67ab7dbf
28
web_scrap.py
28
web_scrap.py
@ -15,6 +15,25 @@ def mkdirPath(path_dir, logger):
|
||||
os.mkdir(repath)
|
||||
|
||||
|
||||
def getScriptCss(url, js, css, logger):
|
||||
page = requests.get(url)
|
||||
page_url = []
|
||||
if page.status_code == 200:
|
||||
soup = BeautifulSoup(page.text, 'html.parser')
|
||||
if js is True:
|
||||
script = soup.find_all("script")
|
||||
for anchor in script:
|
||||
src = anchor.get("src", "/")
|
||||
if src != "/":
|
||||
u = urlparse(url)
|
||||
o = urlparse(src)
|
||||
if o.netloc == "":
|
||||
o = o._replace(netloc=u.netloc)
|
||||
o = o._replace(scheme=u.scheme)
|
||||
page_url.append(o.geturl())
|
||||
|
||||
return page_url
|
||||
|
||||
def getUrlPage(url, logger):
|
||||
page = requests.get(url)
|
||||
page_url = []
|
||||
@ -64,7 +83,7 @@ def getUrlPage(url, logger):
|
||||
return webpage
|
||||
|
||||
|
||||
def downloadPageHTML(webpage, backup_dir, logger):
|
||||
def downloadPage(webpage, backup_dir, logger):
|
||||
|
||||
for i in range(0, len(webpage)):
|
||||
o = urlparse(webpage[i])
|
||||
@ -106,6 +125,9 @@ if __name__ == '__main__':
|
||||
|
||||
o = urlparse(args.url)
|
||||
o = o._replace(scheme="https")
|
||||
webpage = getUrlPage(o.geturl().replace(":///", "://"), logger)
|
||||
url = o.geturl().replace(":///", "://")
|
||||
script = getScriptCss(url, True, False, logger)
|
||||
logger.info(script)
|
||||
#webpage = getUrlPage(url, logger)
|
||||
|
||||
downloadPageHTML(webpage, args.dir, logger)
|
||||
downloadPage(script, "{0}/{1}".format(args.dir, "dists/js"), logger)
|
Loading…
x
Reference in New Issue
Block a user