v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

fix thread fromDirectory and fromUrl

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-05-01 21:18:57 +02:00
parent 939e744d1d
commit 73c0998ae0
2 changed files with 4 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
import_export_canalblog.py
View File

@ -149,7 +149,6 @@ if __name__ == '__main__':
for i in wordpress:
importWp.setUrl(i)
importWp.fromFile(files=args.file.split(","))
exit(0)
if len(args.directory) > 0:
try:
with futures.ThreadPoolExecutor(max_workers=int(args.parallel)) as ex:
@ -159,7 +158,6 @@ if __name__ == '__main__':
]
except Exception as err:
logger.error("Threading error : {0}".format(err))
exit(0)
if len(args.canalblog) > 0:
try:
with futures.ThreadPoolExecutor(max_workers=int(args.parallel)) as ex:
@ -169,7 +167,7 @@ if __name__ == '__main__':
]
except Exception as err:
logger.error("Threading error : {0}".format(err))
exit(0)
if args.command == "export":

13
lib/WPImport.py
View File

@ -59,17 +59,10 @@ class WPimport:
def fromFile(self, files=[], number_thread=1, max_thread=1):
divFiles = int(len(files) / max_thread)
currentRangeFiles = int(divFiles * (number_thread+1))
firstRange = int(currentRangeFiles - divFiles)
self._logger.debug("{0} : index : {1}".format(self._name,number_thread))
self._logger.debug("{0} : first range : {1}".format(self._name,firstRange))
self._logger.debug("{0} : last range : {1}".format(self._name,currentRangeFiles))
for i in range(firstRange, currentRangeFiles):
for i in range(0, len(files)):
if os.path.exists(files[i]):
self._logger.info("{0} : File is being processed : {1}".format(self._name, files[i]))
self._logger.info("{0} : ({1}/{2}) File is being processed : {3}".format(self._name, i+1, len(files), files[i]))
with open(files[i], 'r') as f:
content = f.read()
self._logger.debug("{0} : Size of article : {1}".format(self._name, len(content)))
@ -408,7 +401,7 @@ class WPimport:
self._logger.error("{0} : Connection error with status code for get image : {1}".format(self._name, page_img.status_code))
self._logger.debug("{0} : {1}".format(self._name, page_img.content))
self._debug("{0} : Number of image : {1}".format(self._name, len(list_img)))
self._logger.debug("{0} : Number of image : {1}".format(self._name, len(list_img)))
comment_post = self._getComment(comment)
a = itemfooter[0].find_all("a", {"rel": True})