2023-02-28 21:42:21 +01:00
|
|
|
#!/usr/bin/python3
|
|
|
|
from bs4 import BeautifulSoup
|
2023-03-04 16:12:42 +01:00
|
|
|
from urllib.parse import urlparse
|
2023-03-06 22:54:32 +01:00
|
|
|
import requests, os, argparse, logging
|
2023-03-04 18:45:32 +01:00
|
|
|
|
2023-03-07 21:54:18 +01:00
|
|
|
def mkdirPath(path_dir, logger):
|
2023-03-04 18:35:06 +01:00
|
|
|
if not os.path.exists(path_dir):
|
|
|
|
makedir = []
|
|
|
|
pathh = path_dir.split("/")
|
|
|
|
for i in pathh:
|
|
|
|
makedir.append(i)
|
|
|
|
repath = "/".join(makedir)
|
|
|
|
if not os.path.exists(repath):
|
2023-03-06 22:54:32 +01:00
|
|
|
logger.debug("Dossier crée : {0}".format(repath))
|
2023-03-09 20:27:49 +01:00
|
|
|
try:
|
|
|
|
os.mkdir(repath)
|
|
|
|
except Exception as err:
|
|
|
|
logger.error("Directory error : {0}".format(err))
|
|
|
|
exit(1)
|
2023-03-04 18:35:06 +01:00
|
|
|
|
2023-02-28 21:42:21 +01:00
|
|
|
|
2023-03-07 22:42:05 +01:00
|
|
|
def getScriptCss(url, js, css, logger):
|
2023-03-09 20:27:49 +01:00
|
|
|
try:
|
|
|
|
page = requests.get(url)
|
|
|
|
except Exception as err:
|
|
|
|
logger.error("Connection error : {0}".format(err))
|
|
|
|
exit(1)
|
2023-03-07 22:42:05 +01:00
|
|
|
page_url = []
|
|
|
|
if page.status_code == 200:
|
|
|
|
soup = BeautifulSoup(page.text, 'html.parser')
|
|
|
|
if js is True:
|
|
|
|
script = soup.find_all("script")
|
|
|
|
for anchor in script:
|
|
|
|
src = anchor.get("src", "/")
|
|
|
|
if src != "/":
|
2023-03-09 20:35:47 +01:00
|
|
|
try:
|
|
|
|
u = urlparse(url)
|
|
|
|
o = urlparse(src)
|
|
|
|
except Exception as err:
|
|
|
|
logger.error("parsing error : {0}".format(err))
|
|
|
|
exit(1)
|
2023-03-07 22:42:05 +01:00
|
|
|
if o.netloc == "":
|
|
|
|
o = o._replace(netloc=u.netloc)
|
|
|
|
o = o._replace(scheme=u.scheme)
|
|
|
|
page_url.append(o.geturl())
|
2023-03-07 22:50:40 +01:00
|
|
|
if css is True:
|
|
|
|
link = soup.find_all("link")
|
|
|
|
for anchor in link:
|
|
|
|
rel = anchor.get("rel")
|
|
|
|
if rel[0] == "stylesheet":
|
|
|
|
href = anchor.get("href", "/")
|
|
|
|
if href != "/":
|
2023-03-09 20:35:47 +01:00
|
|
|
try:
|
|
|
|
u = urlparse(url)
|
|
|
|
o = urlparse(href)
|
|
|
|
except Exception as err:
|
|
|
|
logger.error("parsing error : {0}".format(err))
|
|
|
|
exit(1)
|
2023-03-07 22:50:40 +01:00
|
|
|
if o.netloc == "":
|
|
|
|
o = o._replace(netloc=u.netloc)
|
|
|
|
o = o._replace(scheme=u.scheme)
|
|
|
|
page_url.append(o.geturl())
|
|
|
|
|
2023-03-07 22:42:05 +01:00
|
|
|
|
|
|
|
return page_url
|
|
|
|
|
2023-03-08 22:01:11 +01:00
|
|
|
def getImg(webpage, logger):
|
|
|
|
page_img = []
|
|
|
|
for i in webpage:
|
2023-03-09 20:27:49 +01:00
|
|
|
try:
|
|
|
|
page = requests.get(i)
|
|
|
|
except Exception as err:
|
|
|
|
logger.error("Connection error : {0}".format(err))
|
|
|
|
exit(1)
|
2023-03-08 22:01:11 +01:00
|
|
|
if page.status_code == 200:
|
|
|
|
soup = BeautifulSoup(page.text, 'html.parser')
|
|
|
|
img = soup.find_all("img")
|
|
|
|
logger.info("image from page: {0} : ".format(i))
|
|
|
|
for anchor in img:
|
|
|
|
src = anchor.get("src", "/")
|
|
|
|
if src != "/":
|
|
|
|
if src not in page_img:
|
|
|
|
logger.info("image: {0} : ".format(src))
|
|
|
|
page_img.append(src)
|
|
|
|
|
|
|
|
|
|
|
|
return page_img
|
|
|
|
|
2023-03-06 22:54:32 +01:00
|
|
|
def getUrlPage(url, logger):
|
2023-03-09 20:27:49 +01:00
|
|
|
try:
|
|
|
|
page = requests.get(url)
|
|
|
|
except Exception as err:
|
|
|
|
logger.error("Connection error : {0}".format(err))
|
|
|
|
exit(1)
|
2023-03-05 20:12:58 +01:00
|
|
|
page_url = []
|
2023-02-28 22:24:16 +01:00
|
|
|
if page.status_code == 200:
|
2023-03-02 23:28:04 +01:00
|
|
|
soup = BeautifulSoup(page.text, 'html.parser')
|
2023-03-05 20:12:58 +01:00
|
|
|
ul = soup.find_all("ul", id="listsmooth")
|
|
|
|
for anchor in ul[0].find_all("a"):
|
|
|
|
href = anchor.get('href', '/')
|
|
|
|
if href != "#":
|
|
|
|
page_url.append(href)
|
|
|
|
|
|
|
|
webpage = []
|
|
|
|
for i in page_url:
|
2023-03-09 20:27:49 +01:00
|
|
|
try:
|
|
|
|
page = requests.get(i)
|
|
|
|
except Exception as err:
|
|
|
|
logger.error("Connection error : {0}".format(err))
|
|
|
|
exit(1)
|
2023-03-05 20:12:58 +01:00
|
|
|
if page.status_code == 200:
|
2023-03-06 22:54:32 +01:00
|
|
|
logger.info("page : {0}".format(i))
|
2023-03-05 20:12:58 +01:00
|
|
|
if i not in webpage:
|
|
|
|
webpage.append(i)
|
|
|
|
soup = BeautifulSoup(page.text, 'html.parser')
|
|
|
|
class_div = pagingfirstline = soup.find_all("div", class_="pagingfirstline")
|
|
|
|
if len(class_div) > 0:
|
|
|
|
pagingfirstline = class_div[0].find_all("a")
|
|
|
|
if len(pagingfirstline) > 1:
|
|
|
|
lastpage = pagingfirstline[len(pagingfirstline)-1].get("href", "/")
|
|
|
|
element_lastpage = lastpage.split("/")[len(lastpage.split("/"))-1]
|
|
|
|
number_page = element_lastpage.split("-")[0].split("p")[1]
|
|
|
|
number_lastpage = int(number_page) / 10
|
|
|
|
for j in range(1,int(number_lastpage)):
|
|
|
|
paging = j * 10
|
|
|
|
categorie = urlparse(i).path.split("/")
|
|
|
|
url_paging = "{0}/archives/p{1}-10.html".format(url, paging)
|
|
|
|
if len(categorie) > 2:
|
|
|
|
url_paging = "{0}/archives/{1}/p{2}-10.html".format(url, categorie[2], paging)
|
2023-03-06 22:54:32 +01:00
|
|
|
logger.info(url_paging)
|
2023-03-05 20:12:58 +01:00
|
|
|
if url_paging not in webpage:
|
|
|
|
webpage.append(url_paging)
|
|
|
|
page = requests.get(url_paging)
|
|
|
|
if page.status_code == 200:
|
|
|
|
soup = BeautifulSoup(page.text, 'html.parser')
|
|
|
|
h2 = soup.find_all("h2")
|
|
|
|
for title in h2:
|
|
|
|
href = title.find_all("a")[0].get("href", "/")
|
|
|
|
if href not in webpage:
|
2023-03-09 20:35:47 +01:00
|
|
|
try:
|
|
|
|
o = urlparse(href)
|
|
|
|
o = o._replace(scheme="https").geturl()
|
|
|
|
except Exception as err:
|
|
|
|
logger.error("parsing error : {0}".format(err))
|
|
|
|
exit(1)
|
2023-03-05 20:12:58 +01:00
|
|
|
webpage.append(o)
|
2023-03-05 21:44:30 +01:00
|
|
|
return webpage
|
2023-03-05 20:12:58 +01:00
|
|
|
|
|
|
|
|
2023-03-07 22:42:05 +01:00
|
|
|
def downloadPage(webpage, backup_dir, logger):
|
2023-03-07 21:54:18 +01:00
|
|
|
|
2023-03-05 21:44:30 +01:00
|
|
|
for i in range(0, len(webpage)):
|
2023-03-09 20:35:47 +01:00
|
|
|
try:
|
|
|
|
o = urlparse(webpage[i])
|
|
|
|
except Exception as err:
|
|
|
|
logger.error("parsing error : {0}".format(err))
|
|
|
|
exit(1)
|
2023-03-05 20:12:58 +01:00
|
|
|
path_web = o.path.split("/")
|
2023-03-05 21:44:30 +01:00
|
|
|
filePageWeb = path_web[len(path_web)-1]
|
2023-03-05 20:12:58 +01:00
|
|
|
path_web.pop(len(path_web)-1)
|
|
|
|
dir_page_web = "/".join(path_web)
|
2023-03-08 22:05:25 +01:00
|
|
|
mkdirPath("{0}/{1}/{2}".format(backup_dir, o.netloc, dir_page_web), logger)
|
2023-03-09 20:27:49 +01:00
|
|
|
try:
|
|
|
|
r = requests.get(webpage[i])
|
|
|
|
except Exception as err:
|
|
|
|
logger.error("Connection error : {0}".format(err))
|
|
|
|
exit(1)
|
2023-03-05 21:44:30 +01:00
|
|
|
if r.status_code == 200:
|
2023-03-08 22:05:25 +01:00
|
|
|
fileDownload = "{0}/{1}/index.html".format(backup_dir, o.netloc)
|
2023-03-05 21:44:30 +01:00
|
|
|
if len(dir_page_web) > 0 and len(filePageWeb) > 0:
|
2023-03-08 22:05:25 +01:00
|
|
|
fileDownload = "{0}/{1}{2}/{3}".format(backup_dir, o.netloc, dir_page_web, filePageWeb)
|
2023-03-07 22:50:40 +01:00
|
|
|
logger.info("{0}/{1} : {2}".format(i+1, len(webpage), fileDownload))
|
2023-03-09 20:35:47 +01:00
|
|
|
try:
|
|
|
|
open(fileDownload, "wb").write(r.content)
|
|
|
|
except Exception as err:
|
|
|
|
logger.error("file error : {0}".format(err))
|
|
|
|
exit(1)
|
2023-03-05 20:12:58 +01:00
|
|
|
|
2023-03-03 20:03:48 +01:00
|
|
|
|
2023-03-05 20:12:58 +01:00
|
|
|
if __name__ == '__main__':
|
2023-03-05 21:44:30 +01:00
|
|
|
parser = argparse.ArgumentParser()
|
|
|
|
parser.add_argument("--url", help="canblog URL to be scraping", required=True)
|
|
|
|
parser.add_argument("--dir",
|
|
|
|
default="backup",
|
|
|
|
help="backup file path")
|
2023-03-06 22:54:32 +01:00
|
|
|
parser.add_argument("--debug", help="Verbosity", action="store_true")
|
2023-03-08 22:01:11 +01:00
|
|
|
parser.add_argument("--no-css", help="No CSS", dest="css", action="store_true")
|
|
|
|
parser.add_argument("--no-js", help="No JS", dest="js", action="store_true")
|
|
|
|
parser.add_argument("--no-img", help="No img", dest="img", action="store_true")
|
|
|
|
parser.add_argument("--no-html", help="No HTML", dest="html", action="store_true")
|
|
|
|
|
2023-03-05 21:44:30 +01:00
|
|
|
args = parser.parse_args()
|
2023-03-06 22:54:32 +01:00
|
|
|
logger = logging.getLogger('web_scrap')
|
|
|
|
ch = logging.StreamHandler()
|
|
|
|
|
2023-03-07 21:54:18 +01:00
|
|
|
if args.debug is True:
|
2023-03-06 22:54:32 +01:00
|
|
|
logger.setLevel(logging.DEBUG)
|
|
|
|
ch.setLevel(logging.DEBUG)
|
|
|
|
else:
|
|
|
|
logger.setLevel(logging.INFO)
|
|
|
|
ch.setLevel(logging.INFO)
|
|
|
|
|
|
|
|
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
|
|
|
ch.setFormatter(formatter)
|
|
|
|
logger.addHandler(ch)
|
2023-03-09 20:27:49 +01:00
|
|
|
try:
|
|
|
|
o = urlparse(args.url)
|
|
|
|
o = o._replace(scheme="https")
|
|
|
|
url = o.geturl().replace(":///", "://")
|
|
|
|
except Exception as err:
|
2023-03-09 20:35:47 +01:00
|
|
|
logger.error("parsing error : {0}".format(err))
|
2023-03-08 22:01:11 +01:00
|
|
|
if args.js is False:
|
|
|
|
script = getScriptCss(url, True, False, logger)
|
2023-03-08 22:41:35 +01:00
|
|
|
downloadPage(script, "{0}/{1}/{2}".format(args.dir, o.path, "dists/js"), logger)
|
2023-03-08 22:01:11 +01:00
|
|
|
|
|
|
|
if args.css is False:
|
|
|
|
css = getScriptCss(url, False, True, logger)
|
2023-03-08 22:41:35 +01:00
|
|
|
downloadPage(css, "{0}/{1}/{2}".format(args.dir, o.path, "dists/css"), logger)
|
2023-03-08 22:01:11 +01:00
|
|
|
|
|
|
|
if args.html is False or args.img is False:
|
|
|
|
webpage = getUrlPage(url, logger)
|
|
|
|
if args.html is False:
|
|
|
|
downloadPage(webpage, args.dir, logger)
|
|
|
|
|
|
|
|
if args.img is False:
|
|
|
|
page_src = getImg(webpage, logger)
|
2023-03-08 22:41:35 +01:00
|
|
|
downloadPage(page_src, "{0}/{1}/{2}".format(args.dir, o.path, "img"), logger)
|