fix previos commit

.gitignore

@@ -1,4 +1,5 @@
 backup/
 backup1/
+backup2/
 web_scrap.log
 __pycache__/

import_export_canalblog.py

@@ -3,13 +3,14 @@ from requests.auth import HTTPBasicAuth
 from getpass import getpass
 from urllib.parse import urlparse
 import argparse, logging, threading
-
+import multiprocessing
 from lib.WPImport import WPimport
 from lib.WPExport import WPExport
 
 
-def download(exportWp, html, img):
+def download(name_thread, max_thread, exportWp, html, img):
-    webpage = exportWp.getUrlPage()
+    exportWp.setName(name_thread)
+    webpage = exportWp.getUrlPage(name_thread, max_thread)
     if html is False:
         exportWp.downloadHTML(webpage)
     
@@ -25,6 +26,7 @@ if __name__ == '__main__':
     parser.add_argument("--logfile", help="Log file",  default="")
     parser.add_argument("--quiet", help="No console output", action="store_true")
     parser.add_argument("--parser", help="Parser content", default="html.parser")
+    parser.add_argument("--parallel", help="Define number thread (default : 1)", default=1)
 
     subparsers = parser.add_subparsers(dest="command")
 
@@ -162,10 +164,12 @@ if __name__ == '__main__':
                 exportWp.downloadCss()
     
             if args.html is False or args.img is False:
-                webpage = exportWp.getUrlPage()
+                threads = []
-            if args.html is False:
+                for i in range(0, int(args.parallel)):
-                exportWp.downloadHTML(webpage)
+                    t1 = multiprocessing.Process(name="Process-{0}".format(i + 1), target=download, args=(i, 3, exportWp,args.html, args.img))
-    
+                    threads.append(t1)
-            if args.img is False:
+
-                exportWp.downloadImg(webpage)
+                for thread in threads:
+                    thread.start()
+                    thread.join()
         exit(0)

lib/WPExport.py

@@ -11,6 +11,7 @@ class WPExport:
         self._logger = logger
         self._parser = parser
         self._dir = directory
+        self._name = "Process-0"
 
         self._request = requests.Session()
 
@@ -22,6 +23,11 @@ class WPExport:
 
     # Public method
 
+    # Set name
+
+    def setName(self, name):
+        self._name = "Process-{}".format(int(name) + 1)
+
     # Set URL
 
     def setUrl(self, url):
@@ -55,11 +61,11 @@ class WPExport:
 
 
      # Get URL
-    def getUrlPage(self):
+    def getUrlPage(self, index_thread, max_thread):
         try:
             page = self._request.get(self._url)
         except Exception as err:
-            self._logger.error("Connection error : {0}".format(err))
+            self._logger.error("{0} : Connection error : {1}".format(self._name, err))
             exit(1)
         page_url = []
         if page.status_code == 200:
@@ -70,8 +76,8 @@ class WPExport:
                 if href != "#":
                     page_url.append(href)
         else:
-            self._logger.error("Url did not get due status code : {0}".format(page.status_code))
+            self._logger.error("{0} : URL did not get due status code : {1}".format(self._name, page.status_code))
-            self._logger.debug(page.content)
+            self._logger.debug("{0} : {1}".format(self._name, page.content))
 
 
         webpage = []
@@ -79,10 +85,10 @@ class WPExport:
             try:
                 page = self._request.get(i)
             except Exception as err:
-                self._logger.error("Connection error : {0}".format(err))
+                self._logger.error("{0} : Connection error : {1}".format(self._name, err))
                 exit(1)
             if page.status_code == 200:
-                self._logger.info("page : {0}".format(i))
+                self._logger.info("{0} : page : {1}".format(self._name, i))
                 if i not in webpage:
                     webpage.append(i)
                 soup = BeautifulSoup(page.text, self._parser)
@@ -94,13 +100,22 @@ class WPExport:
                         element_lastpage = lastpage.split("/")[len(lastpage.split("/"))-1]
                         number_page = element_lastpage.split("-")[0].split("p")[1]
                         number_lastpage = int(number_page) / 10
-                        for j in range(1,int(number_lastpage)):
+
+                        setPageDivided = int(number_lastpage) / max_thread
+                        setPagePart = setPageDivided * (index_thread + 1)
+                        firstPagePart = (setPagePart - setPageDivided) + 1
+
+                        self._logger.debug("{0} : Total page : {1}".format(self._name,int(number_lastpage)))
+                        self._logger.debug("{0} : First range : {1}".format(self._name, int(firstPagePart)))
+                        self._logger.debug("{0} : Last range : {1}".format(self._name, int(setPagePart)))
+
+                        for j in range(int(firstPagePart),int(setPagePart)):
                             paging = j * 10
                             categorie = urlparse(i).path.split("/")
                             url_paging = "{0}/archives/p{1}-10.html".format(self._url, paging)
                             if len(categorie) > 2:
                                 url_paging = "{0}/archives/{1}/p{2}-10.html".format(self._url, categorie[2], paging)
-                            self._logger.info(url_paging)
+                            self._logger.info("{0} : {1}".format(self._name, url_paging))
                             if url_paging not in webpage:
                                 webpage.append(url_paging)
                             page = self._request.get(url_paging)
@@ -118,7 +133,7 @@ class WPExport:
                                             exit(1)
                                         webpage.append(o)
             else:
-                self._logger.error("web didn't get due status code : {0}".format(page.status_code))
+                self._logger.error("{0} : web didn't get due status code : {1}".format(self._name, page.status_code))
                 self._logger.debug(page.content)
         
         return webpage