v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

Merge pull request 'add wait' (#16) from wait_remove into master

Browse Source 
Reviewed-on: #16
This commit is contained in:
v4l3n71n 2023-05-29 21:36:38 +00:00
commit 4bd6f5c038
1 changed files with 7 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
import_export_canalblog.py
View File

@ -3,7 +3,7 @@ from requests.auth import HTTPBasicAuth
from getpass import getpass
from urllib.parse import urlparse
from concurrent import futures
from concurrent.futures import as_completed, wait
from concurrent.futures import as_completed, wait, ALL_COMPLETED
import argparse, logging, threading
from lib.WPImport import WPimport
@ -221,9 +221,10 @@ if __name__ == '__main__':
importWp.setUrl(i)
importWp.fromFile(files=args.file.split(","))
if len(args.directory) > 0:
remove(args, basic, logger, ssl_wordpress)
try:
with futures.ThreadPoolExecutor(max_workers=int(args.parallel)) as ex:
wait_for = [ ex.submit(remove, i, args.parallel, args, basic, logger, ssl_wordpress) for i in range(0, int(args.parallel)) ]
wait(wait_for, return_when=ALL_COMPLETED)
wait_for = [
ex.submit(importDirectory, i, int(args.parallel), args.directory, logger, args.parser, args.wordpress, basic, args.serial, ssl_wordpress)
for i in range(0, int(args.parallel))
@ -231,13 +232,16 @@ if __name__ == '__main__':
except Exception as err:
logger.error("Threading error : {0}".format(err))
if len(args.canalblog) > 0:
remove(args, basic, logger, ssl_wordpress)
try:
with futures.ThreadPoolExecutor(max_workers=int(args.parallel)) as ex:
wait_for = [ ex.submit(remove, i, args.parallel, args, basic, logger, ssl_wordpress) for i in range(0, int(args.parallel)) ]
wait(wait_for, return_when=ALL_COMPLETED)
wait_for = [
ex.submit(importUrl, i, int(args.parallel), args.canalblog, logger, args.parser, args.wordpress, basic, args.serial, ssl_wordpress, ssl_canalblog)
for i in range(0, int(args.parallel))
]
except Exception as err:
logger.error("Threading error : {0}".format(err))
exit(0)