v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

thread 50%

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-04-25 00:34:25 +02:00
parent aaac2385a3
commit 0acd5067cb
2 changed files with 19 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
import_export_canalblog.py
View File

@ -2,14 +2,18 @@
from requests.auth import HTTPBasicAuth from requests.auth import HTTPBasicAuth
from getpass import getpass from getpass import getpass
from urllib.parse import urlparse from urllib.parse import urlparse
from concurrent import futures
import argparse, logging, threading import argparse, logging, threading
import multiprocessing
from lib.WPImport import WPimport from lib.WPImport import WPimport
from lib.WPExport import WPExport from lib.WPExport import WPExport
def download(name_thread, max_thread, exportWp, html, img): def download(name_thread, max_thread, exportWp, html, img):
exportWp.setName(name_thread) #def download(args):
webpage = exportWp.getUrlPage(name_thread, max_thread) webpage = exportWp.getUrlPage(name_thread, max_thread)
if html is False: if html is False:
exportWp.downloadHTML(webpage) exportWp.downloadHTML(webpage)
@ -164,12 +168,15 @@ if __name__ == '__main__':
exportWp.downloadCss() exportWp.downloadCss()
if args.html is False or args.img is False: if args.html is False or args.img is False:
threads = [] ex = futures.ThreadPoolExecutor(max_workers=int(args.parallel))
for i in range(0, int(args.parallel)): #args = [int(args.parallel), exportWp, args.html, args.img]
t1 = multiprocessing.Process(name="Process-{0}".format(i + 1), target=download, args=(i, 3, exportWp,args.html, args.img)) #f = ex.map(download, args)
threads.append(t1) #print("wait !!!")
wait_for = [
ex.submit(download, i, int(args.parallel), exportWp, args.html, args.img)
for i in range(0, int(args.parallel))
]
for thread in threads: #for i in range(0, int(args.parallel)):
thread.start() # threading.Thread(name="Thread-{0}".format(i + 1), target=download, args=(i, 3, exportWp,args.html, args.img)).start()
thread.join()
exit(0) exit(0)

5
lib/WPExport.py
View File

@ -11,7 +11,7 @@ class WPExport:
self._logger = logger self._logger = logger
self._parser = parser self._parser = parser
self._dir = directory self._dir = directory
self._name = "Process-0" self._name = "Thread-0"
self._request = requests.Session() self._request = requests.Session()
@ -26,7 +26,7 @@ class WPExport:
# Set name # Set name
def setName(self, name): def setName(self, name):
self._name = "Process-{}".format(int(name) + 1) self._name = "Thread-{0}".format(int(name) + 1)
# Set URL # Set URL
@ -62,6 +62,7 @@ class WPExport:
# Get URL # Get URL
def getUrlPage(self, index_thread, max_thread): def getUrlPage(self, index_thread, max_thread):
self._name = "Thread-{0}".format(index_thread + 1)
try: try:
page = self._request.get(self._url) page = self._request.get(self._url)
except Exception as err: except Exception as err: