v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

add argument

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-03-08 22:01:11 +01:00
parent 896cfa0d52
commit 21d24d638d
1 changed files with 40 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
web_scrap.py
View File

@ -48,6 +48,24 @@ def getScriptCss(url, js, css, logger):
return page_url return page_url
def getImg(webpage, logger):
page_img = []
for i in webpage:
page = requests.get(i)
if page.status_code == 200:
soup = BeautifulSoup(page.text, 'html.parser')
img = soup.find_all("img")
logger.info("image from page: {0} : ".format(i))
for anchor in img:
src = anchor.get("src", "/")
if src != "/":
if src not in page_img:
logger.info("image: {0} : ".format(src))
page_img.append(src)
return page_img
def getUrlPage(url, logger): def getUrlPage(url, logger):
page = requests.get(url) page = requests.get(url)
page_url = [] page_url = []
@ -122,6 +140,11 @@ if __name__ == '__main__':
default="backup", default="backup",
help="backup file path") help="backup file path")
parser.add_argument("--debug", help="Verbosity", action="store_true") parser.add_argument("--debug", help="Verbosity", action="store_true")
parser.add_argument("--no-css", help="No CSS", dest="css", action="store_true")
parser.add_argument("--no-js", help="No JS", dest="js", action="store_true")
parser.add_argument("--no-img", help="No img", dest="img", action="store_true")
parser.add_argument("--no-html", help="No HTML", dest="html", action="store_true")
args = parser.parse_args() args = parser.parse_args()
logger = logging.getLogger('web_scrap') logger = logging.getLogger('web_scrap')
ch = logging.StreamHandler() ch = logging.StreamHandler()
@ -140,10 +163,21 @@ if __name__ == '__main__':
o = urlparse(args.url) o = urlparse(args.url)
o = o._replace(scheme="https") o = o._replace(scheme="https")
url = o.geturl().replace(":///", "://") url = o.geturl().replace(":///", "://")
script = getScriptCss(url, True, False, logger) if args.js is False:
downloadPage(script, "{0}/{1}".format(args.dir, "dists/js"), logger) script = getScriptCss(url, True, False, logger)
css = getScriptCss(url, False, True, logger) downloadPage(script, "{0}/{1}".format(args.dir, "dists/js"), logger)
downloadPage(css, "{0}/{1}".format(args.dir, "dists/css"), logger)
webpage = getUrlPage(url, logger)
downloadPage(script, args.dir, logger) if args.css is False:
css = getScriptCss(url, False, True, logger)
downloadPage(css, "{0}/{1}".format(args.dir, "dists/css"), logger)
if args.html is False or args.img is False:
webpage = getUrlPage(url, logger)
if args.html is False:
downloadPage(webpage, args.dir, logger)
if args.img is False:
page_src = getImg(webpage, logger)
downloadPage(page_src, "{0}/{1}".format(args.dir, "img"), logger)
#