add remove multithread

import_export_canalblog.py

@@ -10,8 +10,8 @@ from lib.WPImport import WPimport
 from lib.WPExport import WPExport
 from lib.WPRemove import WPRemove
 
-def remove(args, basic, logger, ssl_wordpress):
-    removeWp = WPRemove(basic=basic, wordpress="", logger=logger, ssl_wordpress=ssl_wordpress)
+def remove(index, number, args, basic, logger, ssl_wordpress):
+    removeWp = WPRemove(basic=basic, wordpress="", logger=logger, ssl_wordpress=ssl_wordpress, index_name=index, number_thread=number)
     if args.remove == True:
         for i in args.wordpress.split(","):
             removeWp.setUrl(i)
@@ -280,5 +280,9 @@ if __name__ == '__main__':
 
 
     if args.command == "remove":
-        remove(args, basic, logger, ssl_wordpress) 
+        try:
+            with futures.ThreadPoolExecutor(max_workers=int(args.parallel)) as ex:
+                wait_for = [ ex.submit(remove, i, args.parallel, args, basic, logger, ssl_wordpress) for i in range(0, int(args.parallel)) ]
+        except Exception as err:
+            logger.error("Thread error for remove : {0}".format(err))
         exit(0)

lib/WPRemove.py

@@ -69,30 +69,60 @@ class WPRemove:
     # Private method
     
     def _removeAll(self, composant):
-        params = {"per_page":100}
-        try:
-            self._logger.info("{0} : List {2} to remove for url : {1}".format(self._name, self._wordpress, composant))
+        count = self._getCount(composant)
+        self._logger.debug("{0} : Count for {1} : {2}".format(self._name, composant, count))
+        if count > 0:
+            self._logger.debug("{0} : Number thread for {1} : {2}".format(self._name, composant, self._number_thread))
+
+            page = count / int(self._number_thread)
+            self._logger.debug("{0} : Page for {1} : {2}".format(self._name, composant, page))
+
+            if page > int(page):
+                page = int(page) + 1
+            if page > 100:
+                page = 100
+            params = {"per_page":page, "page":self._index_thread}
+            self._logger.info("{0} : Params for {1} : {2}".format(self._name, composant, params))
+
+            try:
+                self._logger.info("{0} : List {2} to remove for url : {1}".format(self._name, self._wordpress, composant))
             
-            r = self._request.get("{2}://{0}/wp-json/wp/v2/{1}".format(self._wordpress, composant, self._protocol), auth=self._basic, params=params, headers=self._headers_json)
+                r = self._request.get("{2}://{0}/wp-json/wp/v2/{1}".format(self._wordpress, composant, self._protocol), auth=self._basic, params=params, headers=self._headers_json)
         
-            if r.status_code == 200:
-                result = r.json()
-                if len(result) > 0:
-                    for i in result:
-                        self._logger.info("{0} : Remove {2} for url {1} : {3}".format(self._name, self._wordpress, composant, i["title"]["rendered"]))
-                        params = {"force":1}
-                        try:
-                            r = self._request.delete("{3}://{0}/wp-json/wp/v2/{1}/{2}".format(self._wordpress, composant, i["id"], self._protocol), auth=self._basic, headers=self._headers_json , params=params)
-                            if r.status_code == 200:
-                                self._logger.info("{0} : Post removed for URL {1} {2} : {3}".format(self._name, self._wordpress, composant, i["title"]["rendered"]))
-                            else:
-                                self._logger.error("{0} : Connection error for post {1} {2} {3} with status code {4}".format(self._name, self._wordpress, composant, i["title"]["rendered"], r.status_code))
-                        except Exception as err:
-                            self._logger.error("{0} : Connection error for {1} remove : {2}".format(self._name, composant, err))
-                            exit(1)
-                    self._removeAll(composant)
-            else:
-                self._logger.error("{0} : Error for list to remove {1} due status code {2}".format(self._name, composant, r.status_code))
-                self._logger.debug("{0} : Content error for {1} : {2}".format(self._name, composant, r.content))
-        except Exception as err:
-            self._logger.error("{0} : Connection error for list {1} to remove : {2}".format(self._name, composant, err))
+                if r.status_code == 200:
+                    result = r.json()
+                    if len(result) > 0:
+                        for i in result:
+                            is_delete = True
+                            self._logger.info(i["slug"])
+                            if i["slug"] == "non-classe":
+                                is_delete = False
+                            if is_delete is True:
+                                if composant == "tags" or composant == "categories":
+                                    title = i["name"]
+                                else:
+                                    title = i["title"]["rendered"]
+                                self._logger.info("{0} : Remove {2} for url {1} : {3}".format(self._name, self._wordpress, composant, title))
+                                params = {"force":1}
+                                try:
+                                    r = self._request.delete("{3}://{0}/wp-json/wp/v2/{1}/{2}".format(self._wordpress, composant, i["id"], self._protocol), auth=self._basic, headers=self._headers_json , params=params)
+                                    if r.status_code == 200:
+                                        self._logger.info("{0} : Post removed for URL {1} {2} : {3}".format(self._name, self._wordpress, composant, title))
+                                    else:
+                                        self._logger.error("{0} : Connection error for post {1} {2} {3} with status code {4}".format(self._name, self._wordpress, composant, title, r.status_code))
+                                except ConnectionError as err:
+                                    self._logger.error("{0} : Connection error for {1} remove : {2}".format(self._name, composant, err))
+                                    exit(1)
+                                except Exception as err:
+                                    self._logger.error("{0} : Exception error for {1} remove : {2}".format(self._name, composant, err))
+                                self._removeAll(composant)
+                if r.status_code == 400:
+                    self._logger.error("{0} : No content for {1} to remove : {2}".format(self._name, composant, r.status_code))
+                else:
+                    self._logger.error("{0} : Error for list to remove {1} due status code {2}".format(self._name, composant, r.status_code))
+                    self._logger.debug("{0} : Content error for {1} : {2}".format(self._name, composant, r.content))
+            except ConnectionError as err:
+                self._logger.error("{0} : Connection error for list {1} to remove : {2}".format(self._name, composant, err))
+                exit(1)
+            except Exception as err:
+                self._logger.error("{0} : Exception error for list {1} to remove : {2}".format(self._name, composant, err))