v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

add debug thread

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-04-26 23:03:43 +02:00
parent 08ff16527d
commit a0b816fe18
2 changed files with 16 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
import_export_canalblog.py
View File

@ -168,15 +168,14 @@ if __name__ == '__main__':
exportWp.downloadCss() exportWp.downloadCss()
if args.html is False or args.img is False: if args.html is False or args.img is False:
ex = futures.ThreadPoolExecutor(max_workers=int(args.parallel)) try:
#args = [int(args.parallel), exportWp, args.html, args.img] with futures.ThreadPoolExecutor(max_workers=int(args.parallel)) as ex:
#f = ex.map(download, args) wait_for = [
#print("wait !!!") ex.submit(download, i, int(args.parallel), url, logger, args.parser, args.directory, args.html, args.img)
wait_for = [ for i in range(0, int(args.parallel))
ex.submit(download, i, int(args.parallel), url, logger, args.parser, args.directory, args.html, args.img) ]
for i in range(0, int(args.parallel)) except Exception as err:
] logger.error("Threading error : {0}".format(err))
#for i in range(0, int(args.parallel)):
# threading.Thread(name="Thread-{0}".format(i + 1), target=download, args=(i, 3, exportWp,args.html, args.img)).start()
exit(0) exit(0)

14
lib/WPExport.py
View File

@ -150,7 +150,7 @@ class WPExport:
makedir.append(i) makedir.append(i)
repath = "/".join(makedir) repath = "/".join(makedir)
if not os.path.exists(repath): if not os.path.exists(repath):
self._logger.debug("Dossier crée : {0}".format(repath)) self._logger.debug("{0} : Dossier crée : {1}".format(self._name, repath))
try: try:
if len(repath) > 0: if len(repath) > 0:
os.mkdir(repath) os.mkdir(repath)
@ -216,21 +216,21 @@ class WPExport:
try: try:
page = self._request.get(i) page = self._request.get(i)
except Exception as err: except Exception as err:
self._logger.error("Connection error : {0}".format(err)) self._logger.error("{0} : Connection error : {1}".format(self._name, err))
exit(1) exit(1)
if page.status_code == 200: if page.status_code == 200:
soup = BeautifulSoup(page.text, self._parser) soup = BeautifulSoup(page.text, self._parser)
img = soup.find_all("img") img = soup.find_all("img")
self._logger.info("image from page: {0} : ".format(i)) self._logger.info("{0} : image from page: {1} : ".format(self._name,i))
for anchor in img: for anchor in img:
src = anchor.get("src", "/") src = anchor.get("src", "/")
if src != "/": if src != "/":
if src not in page_img: if src not in page_img:
self._logger.info("image: {0} : ".format(src)) self._logger.info("{0} : image: {1} : ".format(self._name, src))
page_img.append(src) page_img.append(src)
else: else:
self._logger.error("Image did not get due status code : {0}".format(page.status_code)) self._logger.error("{0} : Image did not get due status code : {1}".format(self._name, page.status_code))
self._logger.debug(page.content) self._logger.debug("{0} : {1}".format(self._name, page.content))
return page_img return page_img
@ -258,7 +258,7 @@ class WPExport:
fileDownload = "{0}/{1}/index.html".format(backup_dir, o.netloc) fileDownload = "{0}/{1}/index.html".format(backup_dir, o.netloc)
if len(dir_page_web) > 0 and len(filePageWeb) > 0: if len(dir_page_web) > 0 and len(filePageWeb) > 0:
fileDownload = "{0}/{1}{2}/{3}".format(backup_dir, o.netloc, dir_page_web, filePageWeb) fileDownload = "{0}/{1}{2}/{3}".format(backup_dir, o.netloc, dir_page_web, filePageWeb)
self._logger.info("{0}/{1} : {2}".format(i+1, len(webpage), fileDownload)) self._logger.info("{0} : {1}/{2} : {3}".format(self._name, i+1, len(webpage), fileDownload))
try: try:
open(fileDownload, "wb").write(r.content) open(fileDownload, "wb").write(r.content)
except Exception as err: except Exception as err: