v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

thread #9

Merged
v4l3n71n merged 15 commits from thread into master 2023-05-01 20:05:04 +00:00
Conversation 0 Commits 15 Files Changed 4 +19 -11
2 changed files with 19 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 0acd5067cb - Show all commits

25
import_export_canalblog.py
View File

@ -2,14 +2,18 @@
from requests.auth import HTTPBasicAuth from requests.auth import HTTPBasicAuth
from getpass import getpass from getpass import getpass
from urllib.parse import urlparse from urllib.parse import urlparse
from concurrent import futures
import argparse, logging, threading import argparse, logging, threading
import multiprocessing
from lib.WPImport import WPimport from lib.WPImport import WPimport
from lib.WPExport import WPExport from lib.WPExport import WPExport
def download(name_thread, max_thread, exportWp, html, img): def download(name_thread, max_thread, exportWp, html, img):
exportWp.setName(name_thread) #def download(args):
webpage = exportWp.getUrlPage(name_thread, max_thread) webpage = exportWp.getUrlPage(name_thread, max_thread)
if html is False: if html is False:
exportWp.downloadHTML(webpage) exportWp.downloadHTML(webpage)
@ -164,12 +168,15 @@ if __name__ == '__main__':
exportWp.downloadCss() exportWp.downloadCss()
if args.html is False or args.img is False: if args.html is False or args.img is False:
threads = [] ex = futures.ThreadPoolExecutor(max_workers=int(args.parallel))
for i in range(0, int(args.parallel)): #args = [int(args.parallel), exportWp, args.html, args.img]
t1 = multiprocessing.Process(name="Process-{0}".format(i + 1), target=download, args=(i, 3, exportWp,args.html, args.img)) #f = ex.map(download, args)
threads.append(t1) #print("wait !!!")
wait_for = [
ex.submit(download, i, int(args.parallel), exportWp, args.html, args.img)
for i in range(0, int(args.parallel))
]
for thread in threads: #for i in range(0, int(args.parallel)):
thread.start() # threading.Thread(name="Thread-{0}".format(i + 1), target=download, args=(i, 3, exportWp,args.html, args.img)).start()
thread.join()
exit(0) exit(0)

5
lib/WPExport.py
View File

@ -11,7 +11,7 @@ class WPExport:
self._logger = logger self._logger = logger
self._parser = parser self._parser = parser
self._dir = directory self._dir = directory
self._name = "Process-0" self._name = "Thread-0"
self._request = requests.Session() self._request = requests.Session()
@ -26,7 +26,7 @@ class WPExport:
# Set name # Set name
def setName(self, name): def setName(self, name):
self._name = "Process-{}".format(int(name) + 1) self._name = "Thread-{0}".format(int(name) + 1)
# Set URL # Set URL
@ -62,6 +62,7 @@ class WPExport:
# Get URL # Get URL
def getUrlPage(self, index_thread, max_thread): def getUrlPage(self, index_thread, max_thread):
self._name = "Thread-{0}".format(index_thread + 1)
try: try:
page = self._request.get(self._url) page = self._request.get(self._url)
except Exception as err: except Exception as err: