v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

add message error and debug for export

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-04-20 20:53:50 +02:00
parent 34d6cc39d2
commit 4e6ae92217
1 changed files with 20 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
lib/WPExport.py
View File

@ -69,6 +69,10 @@ class WPExport:
href = anchor.get('href', '/') href = anchor.get('href', '/')
if href != "#": if href != "#":
page_url.append(href) page_url.append(href)
else:
self._logger.error("Url did not get due status code : {0}".format(page.status_code))
self._logger.debug(page.content)
webpage = [] webpage = []
for i in page_url: for i in page_url:
@ -113,6 +117,10 @@ class WPExport:
self._logger.error("parsing error : {0}".format(err)) self._logger.error("parsing error : {0}".format(err))
exit(1) exit(1)
webpage.append(o) webpage.append(o)
else:
self._logger.error("web didn't get due status code : {0}".format(page.status_code))
self._logger.debug(page.content)
return webpage return webpage
@ -179,6 +187,10 @@ class WPExport:
o = o._replace(netloc=u.netloc) o = o._replace(netloc=u.netloc)
o = o._replace(scheme=u.scheme) o = o._replace(scheme=u.scheme)
page_url.append(o.geturl()) page_url.append(o.geturl())
else:
self._logger.error("JS or CSS did not get due status code : {0}".format(page.status_code))
self._logger.debug(page.content)
return page_url return page_url
# Get image # Get image
@ -201,6 +213,10 @@ class WPExport:
if src not in page_img: if src not in page_img:
self._logger.info("image: {0} : ".format(src)) self._logger.info("image: {0} : ".format(src))
page_img.append(src) page_img.append(src)
else:
self._logger.error("Image did not get due status code : {0}".format(page.status_code))
self._logger.debug(page.content)
return page_img return page_img
@ -233,3 +249,6 @@ class WPExport:
except Exception as err: except Exception as err:
self._logger.error("file error : {0}".format(err)) self._logger.error("file error : {0}".format(err))
exit(1) exit(1)
else:
self._logger.error("Not download due status code : {0}".format(r.status_code))
self._logger.debug(r.content)