v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

fix webpage

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-09-05 22:06:14 +02:00
parent 3059f785c2
commit eaec1ba9d4
2 changed files with 17 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
import_export_canalblog.py
View File

@ -55,7 +55,7 @@ def remove(index, number, args, basic, logger, ssl_wordpress):
def download(name_thread, max_thread, url, logger, parser, directory, html, img, ssl_canalblog, revert, tmp): def download(name_thread, max_thread, url, logger, parser, directory, html, img, ssl_canalblog, revert, tmp):
exportWp = WPExport(name="Thread-{0}".format(int(name_thread) + 1), url=url, logger=logger, parser=parser, directory=directory, ssl_canalblog=ssl_canalblog) exportWp = WPExport(name="Thread-{0}".format(int(name_thread) + 1), url=url, logger=logger, parser=parser, directory=directory, ssl_canalblog=ssl_canalblog)
if not revert: if revert is False:
exportWp.getUrlPage(name_thread, max_thread) exportWp.getUrlPage(name_thread, max_thread)
for i in ["article", "page"]: for i in ["article", "page"]:
for j in ["publications", "principal"]: for j in ["publications", "principal"]:

20
lib/WPExport.py
View File

@ -58,14 +58,26 @@ class WPExport:
# Download HTML # Download HTML
def downloadHTML(self, first, second): def downloadHTML(self, first, second):
self._downloadPage(webpage[first][second], self._dir) try:
with open("{0}/{1}.json".format(self._tmp, self._name)) as file:
webpage = json.loads(file.read())
self._downloadPage(webpage[first][second], self._dir)
except Exception as ex:
self._logger.error("{0} : Read file json from tmp : {1}".format(self._name, ex))
# Download Image # Download Image
def downloadImg(self, first, second): def downloadImg(self, first, second):
page_src = self._getImg(webpage[first][second]) try:
o = urlparse(self._url) with open("{0}/{1}.json".format(self._tmp, self._name)) as file:
self._downloadPage(page_src, "{0}/{1}/{2}".format(self._dir, o.path, "img")) webpage = json.loads(file.read())
page_src = self._getImg(webpage[first][second])
o = urlparse(self._url)
self._downloadPage(page_src, "{0}/{1}/{2}".format(self._dir, o.path, "img"))
except Exception as ex:
self._logger.error("{0} : Read file json from tmp : {1}".format(self._name, ex))
# Get URL # Get URL