From aff69bfcbc5d2e89ae366e000ed575fe7785b495 Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Sat, 27 May 2023 00:06:11 +0200 Subject: [PATCH] add multithread for remove --- import_export_canalblog.py | 10 ++++-- lib/WPRemove.py | 71 ++++++++++++++++++++++++-------------- 2 files changed, 53 insertions(+), 28 deletions(-) diff --git a/import_export_canalblog.py b/import_export_canalblog.py index 0b17753..4fb77b3 100644 --- a/import_export_canalblog.py +++ b/import_export_canalblog.py @@ -10,8 +10,8 @@ from lib.WPImport import WPimport from lib.WPExport import WPExport from lib.WPRemove import WPRemove -def remove(args, basic, logger, ssl_wordpress): - removeWp = WPRemove(basic=basic, wordpress="", logger=logger, ssl_wordpress=ssl_wordpress) +def remove(index, number, args, basic, logger, ssl_wordpress): + removeWp = WPRemove(basic=basic, wordpress="", logger=logger, ssl_wordpress=ssl_wordpress, index_name=index, number_thread=number) if args.remove == True: for i in args.wordpress.split(","): removeWp.setUrl(i) @@ -280,5 +280,9 @@ if __name__ == '__main__': if args.command == "remove": - remove(args, basic, logger, ssl_wordpress) + try: + with futures.ThreadPoolExecutor(max_workers=int(args.parallel)) as ex: + wait_for = [ ex.submit(remove, i, args.parallel, args, basic, logger, ssl_wordpress) for i in range(0, int(args.parallel)) ] + except Exception as err: + logger.error("Thread error for remove : {0}".format(err)) exit(0) \ No newline at end of file diff --git a/lib/WPRemove.py b/lib/WPRemove.py index b8a9f6f..a876011 100644 --- a/lib/WPRemove.py +++ b/lib/WPRemove.py @@ -69,30 +69,51 @@ class WPRemove: # Private method def _removeAll(self, composant): - params = {"per_page":100} - try: - self._logger.info("{0} : List {2} to remove for url : {1}".format(self._name, self._wordpress, composant)) + count = self._getCount(composant) + self._logger.debug("{0} : Count for {1} : {2}".format(self._name, composant, count)) + if count > 0: + self._logger.debug("{0} : Number thread for {1} : {2}".format(self._name, composant, self._number_thread)) + + page = count / int(self._number_thread) + self._logger.debug("{0} : Page for {1} : {2}".format(self._name, composant, page)) + + if page > int(page): + page = int(page) + 1 + if page > 100: + page = 100 + params = {"per_page":page, "page":self._index_thread} + self._logger.info("{0} : Params for {1} : {2}".format(self._name, composant, params)) + + try: + self._logger.info("{0} : List {2} to remove for url : {1}".format(self._name, self._wordpress, composant)) - r = self._request.get("{2}://{0}/wp-json/wp/v2/{1}".format(self._wordpress, composant, self._protocol), auth=self._basic, params=params, headers=self._headers_json) + r = self._request.get("{2}://{0}/wp-json/wp/v2/{1}".format(self._wordpress, composant, self._protocol), auth=self._basic, params=params, headers=self._headers_json) - if r.status_code == 200: - result = r.json() - if len(result) > 0: - for i in result: - self._logger.info("{0} : Remove {2} for url {1} : {3}".format(self._name, self._wordpress, composant, i["title"]["rendered"])) - params = {"force":1} - try: - r = self._request.delete("{3}://{0}/wp-json/wp/v2/{1}/{2}".format(self._wordpress, composant, i["id"], self._protocol), auth=self._basic, headers=self._headers_json , params=params) - if r.status_code == 200: - self._logger.info("{0} : Post removed for URL {1} {2} : {3}".format(self._name, self._wordpress, composant, i["title"]["rendered"])) - else: - self._logger.error("{0} : Connection error for post {1} {2} {3} with status code {4}".format(self._name, self._wordpress, composant, i["title"]["rendered"], r.status_code)) - except Exception as err: - self._logger.error("{0} : Connection error for {1} remove : {2}".format(self._name, composant, err)) - exit(1) - self._removeAll(composant) - else: - self._logger.error("{0} : Error for list to remove {1} due status code {2}".format(self._name, composant, r.status_code)) - self._logger.debug("{0} : Content error for {1} : {2}".format(self._name, composant, r.content)) - except Exception as err: - self._logger.error("{0} : Connection error for list {1} to remove : {2}".format(self._name, composant, err)) + if r.status_code == 200: + result = r.json() + if len(result) > 0: + for i in result: + self._logger.info("{0} : Remove {2} for url {1} : {3}".format(self._name, self._wordpress, composant, i["title"]["rendered"])) + params = {"force":1} + try: + r = self._request.delete("{3}://{0}/wp-json/wp/v2/{1}/{2}".format(self._wordpress, composant, i["id"], self._protocol), auth=self._basic, headers=self._headers_json , params=params) + if r.status_code == 200: + self._logger.info("{0} : Post removed for URL {1} {2} : {3}".format(self._name, self._wordpress, composant, i["title"]["rendered"])) + else: + self._logger.error("{0} : Connection error for post {1} {2} {3} with status code {4}".format(self._name, self._wordpress, composant, i["title"]["rendered"], r.status_code)) + except ConnectionError as err: + self._logger.error("{0} : Connection error for {1} remove : {2}".format(self._name, composant, err)) + exit(1) + except Exception as err: + self._logger.error("{0} : Exception error for {1} remove : {2}".format(self._name, composant, err)) + self._removeAll(composant) + if r.status_code == 400: + self._logger.error("{0} : No content for {1} to remove : {2}".format(self._name, composant, r.status_code)) + else: + self._logger.error("{0} : Error for list to remove {1} due status code {2}".format(self._name, composant, r.status_code)) + self._logger.debug("{0} : Content error for {1} : {2}".format(self._name, composant, r.content)) + except ConnectionError as err: + self._logger.error("{0} : Connection error for list {1} to remove : {2}".format(self._name, composant, err)) + exit(1) + except Exception as err: + self._logger.error("{0} : Exception error for list {1} to remove : {2}".format(self._name, composant, err))