v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

fix thread in parallelism

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-04-25 16:15:45 +02:00
parent 0acd5067cb
commit 08ff16527d
2 changed files with 6 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
import_export_canalblog.py
View File

@ -9,10 +9,10 @@ from lib.WPImport import WPimport
from lib.WPExport import WPExport from lib.WPExport import WPExport
def download(name_thread, max_thread, exportWp, html, img): def download(name_thread, max_thread, url, logger, parser, directory, html, img):
#def download(args): #def download(args):
exportWp = WPExport(name="Thread-{0}".format(int(name_thread) + 1), url=url, logger=logger, parser=parser, directory=directory)
webpage = exportWp.getUrlPage(name_thread, max_thread) webpage = exportWp.getUrlPage(name_thread, max_thread)
if html is False: if html is False:
@ -151,7 +151,7 @@ if __name__ == '__main__':
if args.command == "export": if args.command == "export":
canalblog = args.url.split(",") canalblog = args.url.split(",")
exportWp = WPExport("", logger, args.parser, args.directory) exportWp = WPExport(logger=logger, parser=args.parser, directory=args.directory)
for canal in canalblog: for canal in canalblog:
try: try:
o = urlparse(canal) o = urlparse(canal)
@ -173,7 +173,7 @@ if __name__ == '__main__':
#f = ex.map(download, args) #f = ex.map(download, args)
#print("wait !!!") #print("wait !!!")
wait_for = [ wait_for = [
ex.submit(download, i, int(args.parallel), exportWp, args.html, args.img) ex.submit(download, i, int(args.parallel), url, logger, args.parser, args.directory, args.html, args.img)
for i in range(0, int(args.parallel)) for i in range(0, int(args.parallel))
] ]

5
lib/WPExport.py
View File

@ -6,12 +6,12 @@ from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry from requests.packages.urllib3.util.retry import Retry
class WPExport: class WPExport:
def __init__(self, url, logger, parser, directory): def __init__(self, name = "Thread-0", url = "", logger = None, parser = "html.parser", directory = "backup"):
self._url = url self._url = url
self._logger = logger self._logger = logger
self._parser = parser self._parser = parser
self._dir = directory self._dir = directory
self._name = "Thread-0" self._name = name
self._request = requests.Session() self._request = requests.Session()
@ -62,7 +62,6 @@ class WPExport:
# Get URL # Get URL
def getUrlPage(self, index_thread, max_thread): def getUrlPage(self, index_thread, max_thread):
self._name = "Thread-{0}".format(index_thread + 1)
try: try:
page = self._request.get(self._url) page = self._request.get(self._url)
except Exception as err: except Exception as err: