v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

add open file tmp

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-06-20 21:38:39 +02:00
parent 35ff22d463
commit 0eab1d885b
2 changed files with 39 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
lib/WPExport.py
View File

@ -161,7 +161,14 @@ class WPExport:
except Exception as err: except Exception as err:
self._logger.error("{0} : Exception error : {1}".format(self._name, err)) self._logger.error("{0} : Exception error : {1}".format(self._name, err))
exit(1) exit(1)
return webpage try:
string_webpage = json.dumps(webpage)
open("{0}/{1}.json".format(self._tmp, self._name), "wt").write(string_webpage)
except Exception as ex:
self._logger.error("{0} : Error for writing webpage : {1}".format(self._name, ex))

15
lib/WPImport.py
View File

@ -40,7 +40,11 @@ class WPimport:
self._wordpress = wordpress self._wordpress = wordpress
def fromUrl(self, first, second): def fromUrl(self, first, second):
for i in range(0, len(webpage[first][second])): try:
content_file = open("{0}/{1}.json".format(self._name, self._tmp))
webpage_content = json.loads(content_file)
webpage = webpage_content[first][second]
for i in range(0, len(webpage)):
try: try:
r = self._request.get(webpage[i]) r = self._request.get(webpage[i])
if r.status_code == 200: if r.status_code == 200:
@ -51,16 +55,25 @@ class WPimport:
self._addOrUpdatePost(soup) self._addOrUpdatePost(soup)
else: else:
self._addOrUpdateFeaturedMedia(soup) self._addOrUpdateFeaturedMedia(soup)
del webpage_content[first][second][i]
open("{0}/{1}.json".format(self._tmp, self._name), "wt").write(webpage_content)
else: else:
self._logger.error("{0} : Connection error for get url {1} with status code : {2}".format(self._name, webpage[i], r.status_code)) self._logger.error("{0} : Connection error for get url {1} with status code : {2}".format(self._name, webpage[i], r.status_code))
self._logger.debug("{0} : {1}".format(self._name, r.content)) self._logger.debug("{0} : {1}".format(self._name, r.content))
except ConnectionError as err: except ConnectionError as err:
self._logger.error("{0} : Connection error for get url {1} : {2}".format(self._name, webpage[i], err)) self._logger.error("{0} : Connection error for get url {1} : {2}".format(self._name, webpage[i], err))
exit(1) exit(1)
except IOError as err:
self._logger.error("{0} : Connection error for IO url {1} : {2}".format(self._name, webpage[i], err))
exit(1)
except Exception as err: except Exception as err:
self._logger.error("{0} : Exception error for get url {1} : {2}".format(self._name, webpage[i], err)) self._logger.error("{0} : Exception error for get url {1} : {2}".format(self._name, webpage[i], err))
except Exception as ex:
self._logger.error("{0} : Read file json from tmp : {1}".format(self._name, ex))
def fromDirectory(self, directory="", number_thread=1, max_thread=1): def fromDirectory(self, directory="", number_thread=1, max_thread=1):
self._directory = directory self._directory = directory
directory = "{0}/archives".format(directory) directory = "{0}/archives".format(directory)