From 69504687efd515f3f633316965709fc729e4bfd9 Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Fri, 26 May 2023 16:38:19 +0200 Subject: [PATCH 1/5] add count --- lib/WPRemove.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lib/WPRemove.py b/lib/WPRemove.py index 8d38ca7..1e5799e 100644 --- a/lib/WPRemove.py +++ b/lib/WPRemove.py @@ -32,6 +32,24 @@ class WPRemove: # Public method + def _getCount(self, composant): + count = 0 + try: + params = {"per_page":1} + self._logger.info("{0} : Get count {2} to remove for url : {1}".format(self._name, self._wordpress, composant)) + r = self._request.get("{2}://{0}/wp-json/wp/v2/{1}".format(self._wordpress, composant, self._protocol), params=params, auth=self._basic, headers=self._headers_json) + if r.status_code == 200: + count = int(r.headers["X-WP-Total"]) + else: + self._logger.error("{0} : Error for list to remove {1} due status code {2}".format(self._name, composant, r.status_code)) + self._logger.debug("{0} : Content error for {1} : {2}".format(self._name, composant, r.content)) + except ConnectionError as err: + self._logger.error("{0} : Connection error for list {1} to remove : {2}".format(self._name, composant, err)) + exit(1) + except Exception as err: + self._logger.error("{0} : Exception error for list {1} to remove : {2}".format(self._name, composant, err)) + return count + def setUrl(self, wordpress): self._wordpress = wordpress From e21721cac115f379409aa6311fd727613d612b86 Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Fri, 26 May 2023 17:44:28 +0200 Subject: [PATCH 2/5] move exception --- lib/WPRemove.py | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/lib/WPRemove.py b/lib/WPRemove.py index 1e5799e..919970e 100644 --- a/lib/WPRemove.py +++ b/lib/WPRemove.py @@ -73,24 +73,25 @@ class WPRemove: self._logger.info("{0} : List {2} to remove for url : {1}".format(self._name, self._wordpress, composant)) r = self._request.get("{2}://{0}/wp-json/wp/v2/{1}".format(self._wordpress, composant, self._protocol), auth=self._basic, params=params, headers=self._headers_json) + + if r.status_code == 200: + result = r.json() + if len(result) > 0: + for i in result: + self._logger.info("{0} : Remove {2} for url {1} : {3}".format(self._name, self._wordpress, composant, i["title"]["rendered"])) + params = {"force":1} + try: + r = self._request.delete("{3}://{0}/wp-json/wp/v2/{1}/{2}".format(self._wordpress, composant, i["id"], self._protocol), auth=self._basic, headers=self._headers_json , params=params) + if r.status_code == 200: + self._logger.info("{0} : Post removed for URL {1} {2} : {3}".format(self._name, self._wordpress, composant, i["title"]["rendered"])) + else: + self._logger.error("{0} : Connection error for post {1} {2} {3} with status code {4}".format(self._name, self._wordpress, composant, i["title"]["rendered"], r.status_code)) + except Exception as err: + self._logger.error("{0} : Connection error for {1} remove : {2}".format(self._name, composant, err)) + exit(1) + self._removeAll(composant) + else: + self._logger.error("{0} : Error for list to remove {1} due status code {2}".format(self._name, composant, r.status_code)) + self._logger.debug("{0} : Content error for {1} : {2}".format(self._name, composant, r.content)) except Exception as err: self._logger.error("{0} : Connection error for list {1} to remove : {2}".format(self._name, composant, err)) - if r.status_code == 200: - result = r.json() - if len(result) > 0: - for i in result: - self._logger.info("{0} : Remove {2} for url {1} : {3}".format(self._name, self._wordpress, composant, i["title"]["rendered"])) - params = {"force":1} - try: - r = self._request.delete("{3}://{0}/wp-json/wp/v2/{1}/{2}".format(self._wordpress, composant, i["id"], self._protocol), auth=self._basic, headers=self._headers_json , params=params) - if r.status_code == 200: - self._logger.info("{0} : Post removed for URL {1} {2} : {3}".format(self._name, self._wordpress, composant, i["title"]["rendered"])) - else: - self._logger.error("{0} : Connection error for post {1} {2} {3} with status code {4}".format(self._name, self._wordpress, composant, i["title"]["rendered"], r.status_code)) - except Exception as err: - self._logger.error("{0} : Connection error for {1} remove : {2}".format(self._name, composant, err)) - exit(1) - self._removeAll(composant) - else: - self._logger.error("{0} : Error for list to remove {1} due status code {2}".format(self._name, composant, r.status_code)) - self._logger.debug("{0} : Content error for {1} : {2}".format(self._name, composant, r.content)) From fd426f150d708c675cc9ea461fa1fae6208288ef Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Fri, 26 May 2023 17:50:57 +0200 Subject: [PATCH 3/5] add variable --- lib/WPRemove.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/WPRemove.py b/lib/WPRemove.py index 919970e..b8a9f6f 100644 --- a/lib/WPRemove.py +++ b/lib/WPRemove.py @@ -8,14 +8,15 @@ from requests.packages.urllib3.util.retry import Retry class WPRemove: # Constructor - def __init__(self, name="Thread-0", basic=None, wordpress="", logger=None, ssl_wordpress=True): - self._name = name + def __init__(self, index_name=1, number_thread=1, basic=None, wordpress="", logger=None, ssl_wordpress=True): self._basic = basic self._wordpress = wordpress self._logger = logger self._headers_json = {'Content-Type': 'application/json', 'Accept':'application/json'} - + self._name = "Thread-{0}".format(index_name) + self._index_thread = index_name self._protocol = "https" + self._number_thread = number_thread if ssl_wordpress is False: self._protocol = "http" self._request = requests.Session() From aff69bfcbc5d2e89ae366e000ed575fe7785b495 Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Sat, 27 May 2023 00:06:11 +0200 Subject: [PATCH 4/5] add multithread for remove --- import_export_canalblog.py | 10 ++++-- lib/WPRemove.py | 71 ++++++++++++++++++++++++-------------- 2 files changed, 53 insertions(+), 28 deletions(-) diff --git a/import_export_canalblog.py b/import_export_canalblog.py index 0b17753..4fb77b3 100644 --- a/import_export_canalblog.py +++ b/import_export_canalblog.py @@ -10,8 +10,8 @@ from lib.WPImport import WPimport from lib.WPExport import WPExport from lib.WPRemove import WPRemove -def remove(args, basic, logger, ssl_wordpress): - removeWp = WPRemove(basic=basic, wordpress="", logger=logger, ssl_wordpress=ssl_wordpress) +def remove(index, number, args, basic, logger, ssl_wordpress): + removeWp = WPRemove(basic=basic, wordpress="", logger=logger, ssl_wordpress=ssl_wordpress, index_name=index, number_thread=number) if args.remove == True: for i in args.wordpress.split(","): removeWp.setUrl(i) @@ -280,5 +280,9 @@ if __name__ == '__main__': if args.command == "remove": - remove(args, basic, logger, ssl_wordpress) + try: + with futures.ThreadPoolExecutor(max_workers=int(args.parallel)) as ex: + wait_for = [ ex.submit(remove, i, args.parallel, args, basic, logger, ssl_wordpress) for i in range(0, int(args.parallel)) ] + except Exception as err: + logger.error("Thread error for remove : {0}".format(err)) exit(0) \ No newline at end of file diff --git a/lib/WPRemove.py b/lib/WPRemove.py index b8a9f6f..a876011 100644 --- a/lib/WPRemove.py +++ b/lib/WPRemove.py @@ -69,30 +69,51 @@ class WPRemove: # Private method def _removeAll(self, composant): - params = {"per_page":100} - try: - self._logger.info("{0} : List {2} to remove for url : {1}".format(self._name, self._wordpress, composant)) + count = self._getCount(composant) + self._logger.debug("{0} : Count for {1} : {2}".format(self._name, composant, count)) + if count > 0: + self._logger.debug("{0} : Number thread for {1} : {2}".format(self._name, composant, self._number_thread)) + + page = count / int(self._number_thread) + self._logger.debug("{0} : Page for {1} : {2}".format(self._name, composant, page)) + + if page > int(page): + page = int(page) + 1 + if page > 100: + page = 100 + params = {"per_page":page, "page":self._index_thread} + self._logger.info("{0} : Params for {1} : {2}".format(self._name, composant, params)) + + try: + self._logger.info("{0} : List {2} to remove for url : {1}".format(self._name, self._wordpress, composant)) - r = self._request.get("{2}://{0}/wp-json/wp/v2/{1}".format(self._wordpress, composant, self._protocol), auth=self._basic, params=params, headers=self._headers_json) + r = self._request.get("{2}://{0}/wp-json/wp/v2/{1}".format(self._wordpress, composant, self._protocol), auth=self._basic, params=params, headers=self._headers_json) - if r.status_code == 200: - result = r.json() - if len(result) > 0: - for i in result: - self._logger.info("{0} : Remove {2} for url {1} : {3}".format(self._name, self._wordpress, composant, i["title"]["rendered"])) - params = {"force":1} - try: - r = self._request.delete("{3}://{0}/wp-json/wp/v2/{1}/{2}".format(self._wordpress, composant, i["id"], self._protocol), auth=self._basic, headers=self._headers_json , params=params) - if r.status_code == 200: - self._logger.info("{0} : Post removed for URL {1} {2} : {3}".format(self._name, self._wordpress, composant, i["title"]["rendered"])) - else: - self._logger.error("{0} : Connection error for post {1} {2} {3} with status code {4}".format(self._name, self._wordpress, composant, i["title"]["rendered"], r.status_code)) - except Exception as err: - self._logger.error("{0} : Connection error for {1} remove : {2}".format(self._name, composant, err)) - exit(1) - self._removeAll(composant) - else: - self._logger.error("{0} : Error for list to remove {1} due status code {2}".format(self._name, composant, r.status_code)) - self._logger.debug("{0} : Content error for {1} : {2}".format(self._name, composant, r.content)) - except Exception as err: - self._logger.error("{0} : Connection error for list {1} to remove : {2}".format(self._name, composant, err)) + if r.status_code == 200: + result = r.json() + if len(result) > 0: + for i in result: + self._logger.info("{0} : Remove {2} for url {1} : {3}".format(self._name, self._wordpress, composant, i["title"]["rendered"])) + params = {"force":1} + try: + r = self._request.delete("{3}://{0}/wp-json/wp/v2/{1}/{2}".format(self._wordpress, composant, i["id"], self._protocol), auth=self._basic, headers=self._headers_json , params=params) + if r.status_code == 200: + self._logger.info("{0} : Post removed for URL {1} {2} : {3}".format(self._name, self._wordpress, composant, i["title"]["rendered"])) + else: + self._logger.error("{0} : Connection error for post {1} {2} {3} with status code {4}".format(self._name, self._wordpress, composant, i["title"]["rendered"], r.status_code)) + except ConnectionError as err: + self._logger.error("{0} : Connection error for {1} remove : {2}".format(self._name, composant, err)) + exit(1) + except Exception as err: + self._logger.error("{0} : Exception error for {1} remove : {2}".format(self._name, composant, err)) + self._removeAll(composant) + if r.status_code == 400: + self._logger.error("{0} : No content for {1} to remove : {2}".format(self._name, composant, r.status_code)) + else: + self._logger.error("{0} : Error for list to remove {1} due status code {2}".format(self._name, composant, r.status_code)) + self._logger.debug("{0} : Content error for {1} : {2}".format(self._name, composant, r.content)) + except ConnectionError as err: + self._logger.error("{0} : Connection error for list {1} to remove : {2}".format(self._name, composant, err)) + exit(1) + except Exception as err: + self._logger.error("{0} : Exception error for list {1} to remove : {2}".format(self._name, composant, err)) From c76b20e64ae168d669c8d9dfde5222db82d2ad72 Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Sat, 27 May 2023 00:16:41 +0200 Subject: [PATCH 5/5] add remove multithread --- lib/WPRemove.py | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/lib/WPRemove.py b/lib/WPRemove.py index a876011..8b631ad 100644 --- a/lib/WPRemove.py +++ b/lib/WPRemove.py @@ -93,20 +93,29 @@ class WPRemove: result = r.json() if len(result) > 0: for i in result: - self._logger.info("{0} : Remove {2} for url {1} : {3}".format(self._name, self._wordpress, composant, i["title"]["rendered"])) - params = {"force":1} - try: - r = self._request.delete("{3}://{0}/wp-json/wp/v2/{1}/{2}".format(self._wordpress, composant, i["id"], self._protocol), auth=self._basic, headers=self._headers_json , params=params) - if r.status_code == 200: - self._logger.info("{0} : Post removed for URL {1} {2} : {3}".format(self._name, self._wordpress, composant, i["title"]["rendered"])) + is_delete = True + self._logger.info(i["slug"]) + if i["slug"] == "non-classe": + is_delete = False + if is_delete is True: + if composant == "tags" or composant == "categories": + title = i["name"] else: - self._logger.error("{0} : Connection error for post {1} {2} {3} with status code {4}".format(self._name, self._wordpress, composant, i["title"]["rendered"], r.status_code)) - except ConnectionError as err: - self._logger.error("{0} : Connection error for {1} remove : {2}".format(self._name, composant, err)) - exit(1) - except Exception as err: - self._logger.error("{0} : Exception error for {1} remove : {2}".format(self._name, composant, err)) - self._removeAll(composant) + title = i["title"]["rendered"] + self._logger.info("{0} : Remove {2} for url {1} : {3}".format(self._name, self._wordpress, composant, title)) + params = {"force":1} + try: + r = self._request.delete("{3}://{0}/wp-json/wp/v2/{1}/{2}".format(self._wordpress, composant, i["id"], self._protocol), auth=self._basic, headers=self._headers_json , params=params) + if r.status_code == 200: + self._logger.info("{0} : Post removed for URL {1} {2} : {3}".format(self._name, self._wordpress, composant, title)) + else: + self._logger.error("{0} : Connection error for post {1} {2} {3} with status code {4}".format(self._name, self._wordpress, composant, title, r.status_code)) + except ConnectionError as err: + self._logger.error("{0} : Connection error for {1} remove : {2}".format(self._name, composant, err)) + exit(1) + except Exception as err: + self._logger.error("{0} : Exception error for {1} remove : {2}".format(self._name, composant, err)) + self._removeAll(composant) if r.status_code == 400: self._logger.error("{0} : No content for {1} to remove : {2}".format(self._name, composant, r.status_code)) else: