v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

fix thread fromDirectory and fromUrl

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-05-01 21:18:57 +02:00
parent 939e744d1d
commit 73c0998ae0
2 changed files with 4 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
import_export_canalblog.py
View File

@ -149,7 +149,6 @@ if __name__ == '__main__':
for i in wordpress: for i in wordpress:
importWp.setUrl(i) importWp.setUrl(i)
importWp.fromFile(files=args.file.split(",")) importWp.fromFile(files=args.file.split(","))
exit(0)
if len(args.directory) > 0: if len(args.directory) > 0:
try: try:
with futures.ThreadPoolExecutor(max_workers=int(args.parallel)) as ex: with futures.ThreadPoolExecutor(max_workers=int(args.parallel)) as ex:
@ -159,7 +158,6 @@ if __name__ == '__main__':
] ]
except Exception as err: except Exception as err:
logger.error("Threading error : {0}".format(err)) logger.error("Threading error : {0}".format(err))
exit(0)
if len(args.canalblog) > 0: if len(args.canalblog) > 0:
try: try:
with futures.ThreadPoolExecutor(max_workers=int(args.parallel)) as ex: with futures.ThreadPoolExecutor(max_workers=int(args.parallel)) as ex:
@ -169,7 +167,7 @@ if __name__ == '__main__':
] ]
except Exception as err: except Exception as err:
logger.error("Threading error : {0}".format(err)) logger.error("Threading error : {0}".format(err))
exit(0)
if args.command == "export": if args.command == "export":

13
lib/WPImport.py
View File

@ -59,17 +59,10 @@ class WPimport:
def fromFile(self, files=[], number_thread=1, max_thread=1): def fromFile(self, files=[], number_thread=1, max_thread=1):
divFiles = int(len(files) / max_thread)
currentRangeFiles = int(divFiles * (number_thread+1))
firstRange = int(currentRangeFiles - divFiles)
self._logger.debug("{0} : index : {1}".format(self._name,number_thread))
self._logger.debug("{0} : first range : {1}".format(self._name,firstRange)) for i in range(0, len(files)):
self._logger.debug("{0} : last range : {1}".format(self._name,currentRangeFiles))
for i in range(firstRange, currentRangeFiles):
if os.path.exists(files[i]): if os.path.exists(files[i]):
self._logger.info("{0} : File is being processed : {1}".format(self._name, files[i])) self._logger.info("{0} : ({1}/{2}) File is being processed : {3}".format(self._name, i+1, len(files), files[i]))
with open(files[i], 'r') as f: with open(files[i], 'r') as f:
content = f.read() content = f.read()
self._logger.debug("{0} : Size of article : {1}".format(self._name, len(content))) self._logger.debug("{0} : Size of article : {1}".format(self._name, len(content)))
@ -408,7 +401,7 @@ class WPimport:
self._logger.error("{0} : Connection error with status code for get image : {1}".format(self._name, page_img.status_code)) self._logger.error("{0} : Connection error with status code for get image : {1}".format(self._name, page_img.status_code))
self._logger.debug("{0} : {1}".format(self._name, page_img.content)) self._logger.debug("{0} : {1}".format(self._name, page_img.content))
self._debug("{0} : Number of image : {1}".format(self._name, len(list_img))) self._logger.debug("{0} : Number of image : {1}".format(self._name, len(list_img)))
comment_post = self._getComment(comment) comment_post = self._getComment(comment)
a = itemfooter[0].find_all("a", {"rel": True}) a = itemfooter[0].find_all("a", {"rel": True})