From b54785c4559ac82c389a767f1aa666f61150be13 Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Fri, 14 Apr 2023 23:10:07 +0200 Subject: [PATCH 01/16] add parent comment WIP --- lib/WPImport.py | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/lib/WPImport.py b/lib/WPImport.py index 39b1228..48bc916 100644 --- a/lib/WPImport.py +++ b/lib/WPImport.py @@ -223,6 +223,25 @@ class WPimport: if page.status_code == 201: self._logger.info("Commentaire ajoute pour {0}".format(title)) + + def _getComment(self, comment): + comment_post = [] + for i in comment: + comment_div = find_all("div", class_="comment_item")[0] + comment_item = comment_div.text.split("\n") + footer = comment_div.find_all("div", class_="itemfooter") + comment_author = footer[0].text.split(",")[0].replace("Posté par ", "") + comment_date = footer[0].find_all("abbr")[0].get("title") + comment_content = "

" + for j in range(0, len(comment_item)-2): + if len(comment_item[j]) > 0: + comment_content = comment_content + comment_item[j] + "
" + comment_content = comment_content + "

" + parent = -1 + parent_class = "level-1" + + comment_post.append({"author": comment_author, "date": comment_date, "content": comment_content}) + return comment_post ## Add or Update post def _addOrUpdatePost(self, soup): @@ -250,7 +269,7 @@ class WPimport: articleacreator = soup.find_all("span", class_="articlecreator") dateheader = soup.find_all("div", class_="dateheader") itemfooter = soup.find_all("div", class_="itemfooter") - comment = soup.find_all("div", class_="comment_item") + comment = soup.find_all("li", class_="comment") img_a = articlebody[0].find_all("a", {"target": "_blank"}) list_img = [] for i in img_a: @@ -284,18 +303,8 @@ class WPimport: new_img["new_src"]=media["rendered"] list_img.append(new_img) - comment_post = [] - for i in comment: - comment_item = i.text.split("\n") - footer = i.find_all("div", class_="itemfooter") - comment_author = footer[0].text.split(",")[0].replace("Posté par ", "") - comment_date = footer[0].find_all("abbr")[0].get("title") - comment_content = "

" - for j in range(0, len(comment_item)-2): - if len(comment_item[j]) > 0: - comment_content = comment_content + comment_item[j] + "
" - comment_content = comment_content + "

" - comment_post.append({"author": comment_author, "date": comment_date, "content": comment_content}) + comment_post = self._getComment(comment) + a = itemfooter[0].find_all("a", {"rel": True}) for i in a: rel = i.get("rel") From 0e15e88f3174bd650d9b2ca83aaab0d6590f891e Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Sun, 16 Apr 2023 19:16:23 +0200 Subject: [PATCH 02/16] Get level comment 50% --- lib/WPImport.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/lib/WPImport.py b/lib/WPImport.py index 48bc916..a39e42e 100644 --- a/lib/WPImport.py +++ b/lib/WPImport.py @@ -223,7 +223,24 @@ class WPimport: if page.status_code == 201: self._logger.info("Commentaire ajoute pour {0}".format(title)) + ## Check class name + + def _hasClassName(self, tag, className): + for i in tag["class"]: + if i == className: + return True + return False + ## Get class name + + def _getClassName(self, tag, className): + for i in tag["class"]: + if re.match(className, i): + return i + return "" + + ## Get all comments + def _getComment(self, comment): comment_post = [] for i in comment: @@ -239,8 +256,21 @@ class WPimport: comment_content = comment_content + "

" parent = -1 parent_class = "level-1" + if self._hasClassName(i, parent_class) is False: + block = False + className = self._getClassName(tag, "level-").split("-") + level = 1 + if len(className) > 0: + level = int(className[1]) + for j in range(i-1, 0): + if block is False: + levelName = "level-{0}".format(level - 1) + if self._hasClassName(comment[j], levelName): + parent = j + parent_class = levelName + block = True - comment_post.append({"author": comment_author, "date": comment_date, "content": comment_content}) + comment_post.append({"author": comment_author, "date": comment_date, "content": comment_content, "parent_id":parent, "parent_class": parent_class}) return comment_post ## Add or Update post From bd8ac241c1253c60c587da2c426b39a6a291cf9a Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Sun, 16 Apr 2023 19:32:00 +0200 Subject: [PATCH 03/16] debug level comment --- lib/WPImport.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/WPImport.py b/lib/WPImport.py index a39e42e..5cd0ac7 100644 --- a/lib/WPImport.py +++ b/lib/WPImport.py @@ -226,6 +226,7 @@ class WPimport: ## Check class name def _hasClassName(self, tag, className): + print(tag["class"]) for i in tag["class"]: if i == className: return True @@ -243,8 +244,8 @@ class WPimport: def _getComment(self, comment): comment_post = [] - for i in comment: - comment_div = find_all("div", class_="comment_item")[0] + for i in range(0, len(comment)): + comment_div = comment[i].find("div", class_="comment_item") comment_item = comment_div.text.split("\n") footer = comment_div.find_all("div", class_="itemfooter") comment_author = footer[0].text.split(",")[0].replace("Posté par ", "") @@ -256,15 +257,17 @@ class WPimport: comment_content = comment_content + "

" parent = -1 parent_class = "level-1" - if self._hasClassName(i, parent_class) is False: + print(self._hasClassName(comment[i], parent_class)) + if self._hasClassName(comment[i], parent_class) is False: block = False - className = self._getClassName(tag, "level-").split("-") + className = self._getClassName(comment[i], "level-").split("-") level = 1 if len(className) > 0: level = int(className[1]) for j in range(i-1, 0): if block is False: levelName = "level-{0}".format(level - 1) + print(levelName) if self._hasClassName(comment[j], levelName): parent = j parent_class = levelName @@ -334,6 +337,7 @@ class WPimport: list_img.append(new_img) comment_post = self._getComment(comment) + exit(0) a = itemfooter[0].find_all("a", {"rel": True}) for i in a: From 581b6941a688d56dacf0d407317428ecd01cfb2c Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Sun, 16 Apr 2023 21:06:04 +0200 Subject: [PATCH 04/16] parent id 75% --- lib/WPImport.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/lib/WPImport.py b/lib/WPImport.py index 5cd0ac7..4760aa5 100644 --- a/lib/WPImport.py +++ b/lib/WPImport.py @@ -226,7 +226,6 @@ class WPimport: ## Check class name def _hasClassName(self, tag, className): - print(tag["class"]) for i in tag["class"]: if i == className: return True @@ -256,24 +255,20 @@ class WPimport: comment_content = comment_content + comment_item[j] + "
" comment_content = comment_content + "

" parent = -1 - parent_class = "level-1" - print(self._hasClassName(comment[i], parent_class)) - if self._hasClassName(comment[i], parent_class) is False: + if self._hasClassName(comment[i], "level-1") is False: block = False className = self._getClassName(comment[i], "level-").split("-") level = 1 if len(className) > 0: level = int(className[1]) - for j in range(i-1, 0): + for j in range(i-1, 0, -1): if block is False: levelName = "level-{0}".format(level - 1) - print(levelName) - if self._hasClassName(comment[j], levelName): + if self._hasClassName(comment[j], levelName) is True: parent = j - parent_class = levelName block = True - comment_post.append({"author": comment_author, "date": comment_date, "content": comment_content, "parent_id":parent, "parent_class": parent_class}) + comment_post.append({"author": comment_author, "date": comment_date, "content": comment_content, "parent_id":parent}) return comment_post ## Add or Update post From 4cf301b2169d9757d501018ebe9395933fd32137 Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Sun, 16 Apr 2023 21:25:32 +0200 Subject: [PATCH 05/16] parent comment 90% --- lib/WPImport.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/WPImport.py b/lib/WPImport.py index 4760aa5..47db168 100644 --- a/lib/WPImport.py +++ b/lib/WPImport.py @@ -215,6 +215,18 @@ class WPimport: if page.status_code == 200: self._logger.info("Commentaire mise à jour pour {0}".format(title)) else: + if i["parent_id"] != -1: + parent_id = int(i["parent_id"]) + try: + page = self._request.get("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, params=params) + except Exception as err: + self._logger.error("Connection error : {0}".format(err)) + exit(1) + if page.status_code == 200: + result = page.json() + for j in result: + if comment[parent_id]["author"] == j["author_name"] and comment[parent_id]["date"] == j["date"]: + data["parent"]=j["id"] try: page = self._request.post("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, data=data) except Exception as err: From ae7cb1e4e0b3890ea2af0b0eebd8a0e99219577c Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Sun, 16 Apr 2023 21:26:48 +0200 Subject: [PATCH 06/16] remove exit useless --- lib/WPImport.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/WPImport.py b/lib/WPImport.py index 47db168..ae2e972 100644 --- a/lib/WPImport.py +++ b/lib/WPImport.py @@ -344,7 +344,6 @@ class WPimport: list_img.append(new_img) comment_post = self._getComment(comment) - exit(0) a = itemfooter[0].find_all("a", {"rel": True}) for i in a: From a5e7cb89f79da97b8804300cfb47753620725662 Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Mon, 17 Apr 2023 23:44:09 +0200 Subject: [PATCH 07/16] add error status code --- lib/WPImport.py | 46 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/lib/WPImport.py b/lib/WPImport.py index ae2e972..dd5de7d 100644 --- a/lib/WPImport.py +++ b/lib/WPImport.py @@ -41,6 +41,8 @@ class WPimport: self._addOrUpdatePost(soup) else: self._addOrUpdateFeaturedMedia(soup) + else: + self._logger.error("Connection error with status code : {0}".format(r.status_code)) def fromDirectory(self, directory): @@ -131,9 +133,17 @@ class WPimport: exit(1) if r.status_code == 200: self._logger.info("Ajout media featured : {0}".format(r.json()["title"]["raw"])) + else: + self._logger.error("Connection error with status code : {0}".format(r.status_code)) else: self._logger.info("Aucun media trouvé pour {0}".format(h2)) - + else: + self._logger.error("Connection error with status code : {0}".format(page.status_code)) + else: + self._logger.error("Connection error with status code : {0}".format(page.status_code)) + else: + self._logger.error("Connection error with status code : {0}".format(page.status_code)) + ## Association image to post def _linkImgPost(self, title, list_img, post_id): @@ -146,6 +156,8 @@ class WPimport: exit(1) if r.status_code == 200: self._logger.info("Association d'une image à l'article {0}".format(title)) + else: + self._logger.error("Connection error with status code : {0}".format(r.status_code)) ## Add or update img @@ -170,6 +182,8 @@ class WPimport: exit(1) if r.status_code == 200: self._logger.info("Image supprimé {0}".format(img_name)) + else: + self._logger.error("Image not removed due status code : {0}".format(r.status_code)) data = page.content img_type = "image/png" if img_name.split(".")[1] == "jpg" or img_name.split(".")[1] == "jpeg": @@ -185,6 +199,10 @@ class WPimport: res = r.json() media["id"] = res["id"] media["rendered"] = res["guid"]["rendered"] + else: + self._logger.error("Image not added due status code : {0}".format(r.status_code)) + else: + self._logger.error("Connection error with status code : {0}".format(r.status_code)) return media ## Add or update comment @@ -214,6 +232,8 @@ class WPimport: exit(1) if page.status_code == 200: self._logger.info("Commentaire mise à jour pour {0}".format(title)) + else: + self._logger.error("Comment not updated due status code : {0}".format(page.status_code)) else: if i["parent_id"] != -1: parent_id = int(i["parent_id"]) @@ -227,13 +247,21 @@ class WPimport: for j in result: if comment[parent_id]["author"] == j["author_name"] and comment[parent_id]["date"] == j["date"]: data["parent"]=j["id"] + else: + self._logger.error("Connection error with status code : {0}".format(page.status_code)) + self._logger.info(data) try: page = self._request.post("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, data=data) except Exception as err: self._logger.error("Connection error : {0}".format(err)) exit(1) + self._logger.info(page.status_code) if page.status_code == 201: self._logger.info("Commentaire ajoute pour {0}".format(title)) + else: + self._logger.error("Comment not added due status code : {0}".format(page.status_code)) + else: + self._logger.error("Connection error with status code : {0}".format(page.status_code)) ## Check class name @@ -302,6 +330,8 @@ class WPimport: elements[i] = page.json() element[i] = [] listelement[i] = [] + else: + self._logger.error("Connection error with status code : {0}".format(page.status_code)) articletitle = soup.find_all("h2", class_="articletitle") articlebody = soup.find_all("div", class_="articlebody") @@ -332,6 +362,8 @@ class WPimport: except Exception as err: self._logger.error("Connection error : {0}".format(err)) exit(1) + if page_img.status_code not in [200, 404]: + self._logger.error("Connection error with status code : {0}".format(page_img.status_code)) if page_img.status_code == 200: media=self._addOrUpdateMedia(href_img, page_img) new_img["id"]=media["id"] @@ -371,6 +403,8 @@ class WPimport: if page.status_code == 201: result = page.json() listelement[i].append(result["id"]) + else: + self._logger.error("{0} not added due status code : {1}".format(i, page.status_code)) title = articletitle[0].text author = articleacreator[0].text.lower() @@ -398,6 +432,8 @@ class WPimport: if page.status_code == 200: result = page.json() data["author"] = result[0]["id"] + else: + self._logger.error("Connection error with status code : {0}".format(page.status_code)) params = {"search":title} try: @@ -424,6 +460,10 @@ class WPimport: self._logger.info("Article mis à jour : {0}".format(result["title"]["raw"])) self._addOrUpdateComment(result["id"], comment_post, result["title"]["raw"]) self._linkImgPost(result["title"]["raw"], list_img, result["id"]) + else: + self._logger.error("Post not updated due status code : {0}".format(page.status_code)) + else: + self._logger.error("Connection error with status code : {0}".format(page.status_code)) if page_exist == False: try: @@ -435,4 +475,6 @@ class WPimport: result = page.json() self._logger.info("Article ajoute : {0}".format(result["title"]["raw"])) self._addOrUpdateComment(result["id"], comment_post, result["title"]["raw"]) - self._linkImgPost(result["title"]["raw"], list_img, result["id"]) \ No newline at end of file + self._linkImgPost(result["title"]["raw"], list_img, result["id"]) + else: + self._logger.error("Post not added due status code : {0}".format(r.status_code)) \ No newline at end of file From d64aed6240632d915401fd3c9247d9af95fea258 Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Tue, 18 Apr 2023 00:00:32 +0200 Subject: [PATCH 08/16] update error message + add debug --- lib/WPImport.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/lib/WPImport.py b/lib/WPImport.py index dd5de7d..3dd4a75 100644 --- a/lib/WPImport.py +++ b/lib/WPImport.py @@ -13,6 +13,7 @@ class WPimport: self._wordpress = wordpress self._logger = logger self._parser = parser + self._headers_json = {'Content-Type': 'application/json', 'Accept':'application/json'} self._request = requests.Session() @@ -124,10 +125,9 @@ class WPimport: res = page.json() if len(res) > 0: id_media = res[0]["id"] - headers = {'Content-Type': 'application/json', 'Accept':'application/json'} data = {"featured_media": id_media} try: - r = self._request.post("http://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, result[0]["id"]), auth=self._basic, headers=headers, data=json.dumps(data)) + r = self._request.post("http://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, result[0]["id"]), auth=self._basic, headers=self._headers_json, data=json.dumps(data)) except Exception as err: self._logger.error("Connection error : {0}".format(err)) exit(1) @@ -181,7 +181,7 @@ class WPimport: self._logger.error("Connection error : {0}".format(err)) exit(1) if r.status_code == 200: - self._logger.info("Image supprimé {0}".format(img_name)) + self._logger.info("Image removed {0}".format(img_name)) else: self._logger.error("Image not removed due status code : {0}".format(r.status_code)) data = page.content @@ -195,7 +195,7 @@ class WPimport: self._logger.error("Connection error : {0}".format(err)) exit(1) if r.status_code == 201: - self._logger.info("Ajout d'image {0}".format(img_name)) + self._logger.info("Image added {0}".format(img_name)) res = r.json() media["id"] = res["id"] media["rendered"] = res["guid"]["rendered"] @@ -231,9 +231,9 @@ class WPimport: self._logger.error("Connection error : {0}".format(err)) exit(1) if page.status_code == 200: - self._logger.info("Commentaire mise à jour pour {0}".format(title)) + self._logger.info("Comment updated for {0}".format(title)) else: - self._logger.error("Comment not updated due status code : {0}".format(page.status_code)) + self._logger.error("Comment not updated for {0} due status code : {1}".format(title, page.status_code)) else: if i["parent_id"] != -1: parent_id = int(i["parent_id"]) @@ -257,9 +257,9 @@ class WPimport: exit(1) self._logger.info(page.status_code) if page.status_code == 201: - self._logger.info("Commentaire ajoute pour {0}".format(title)) + self._logger.info("Comment added for {0}".format(title)) else: - self._logger.error("Comment not added due status code : {0}".format(page.status_code)) + self._logger.error("Comment not added for {0} due status code : {1}".format(title, page.status_code)) else: self._logger.error("Connection error with status code : {0}".format(page.status_code)) @@ -362,8 +362,7 @@ class WPimport: except Exception as err: self._logger.error("Connection error : {0}".format(err)) exit(1) - if page_img.status_code not in [200, 404]: - self._logger.error("Connection error with status code : {0}".format(page_img.status_code)) + if page_img.status_code == 200: media=self._addOrUpdateMedia(href_img, page_img) new_img["id"]=media["id"] @@ -374,6 +373,8 @@ class WPimport: new_img["id"]=media["id"] new_img["new_src"]=media["rendered"] list_img.append(new_img) + if page_img.status_code not in [200, 404]: + self._logger.error("Connection error with status code : {0}".format(page_img.status_code)) comment_post = self._getComment(comment) @@ -395,8 +396,9 @@ class WPimport: listelement[i].append(k["id"]) if element_exist is False: data = {"name": j} + self._logger.debug("URL : {0} with data : {1} and with headers : {2}".format("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i), data, self._headers_json)) try: - page = self._request.post("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i), auth=self._basic, data=data) + page = self._request.post("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i), auth=self._basic, headers=self._headers_json, data=data) except Exception as err: self._logger.error("Connection error : {0}".format(err)) exit(1) From edb9442b1cc7c8455ee9f8c6b39a7853853b0f39 Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Tue, 18 Apr 2023 21:50:36 +0200 Subject: [PATCH 09/16] add search tags and categories before create tags and categories --- lib/WPImport.py | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/lib/WPImport.py b/lib/WPImport.py index 3dd4a75..8f853b1 100644 --- a/lib/WPImport.py +++ b/lib/WPImport.py @@ -14,7 +14,7 @@ class WPimport: self._logger = logger self._parser = parser self._headers_json = {'Content-Type': 'application/json', 'Accept':'application/json'} - + self._request = requests.Session() retries = Retry(total=10, @@ -321,17 +321,8 @@ class WPimport: listelement = {} for i in liste: - try: - page = self._request.get("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress,i)) - except Exception as err: - self._logger.error("Connection error : {0}".format(err)) - exit(1) - if page.status_code == 200: - elements[i] = page.json() - element[i] = [] - listelement[i] = [] - else: - self._logger.error("Connection error with status code : {0}".format(page.status_code)) + element[i] = [] + listelement[i] = [] articletitle = soup.find_all("h2", class_="articletitle") articlebody = soup.find_all("div", class_="articlebody") @@ -390,13 +381,25 @@ class WPimport: for i in liste: for j in element[i]: element_exist = False - for k in elements[i]: - if k["name"] == j: - element_exist = True - listelement[i].append(k["id"]) + try: + params = {"params":j} + page = self._request.get("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i), auth=self._basic, params=params) + except Exception as err: + self._logger.error("Connection error : {0}".format(err)) + exit(1) + if page.status_code == 200: + element_exist = True + result = page.json() + listelement[i].append(result[0]["id"]) + + else: + self._logger.error("{0} not finded due status code : {1}".format(i, page.status_code)) + if element_exist is False: data = {"name": j} - self._logger.debug("URL : {0} with data : {1} and with headers : {2}".format("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i), data, self._headers_json)) + self._logger.debug("URL : {0} ".format("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i))) + self._logger.debug("data : {0}".format(data)) + self._logger.debug("headers : {0}".format(self._headers_form)) try: page = self._request.post("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i), auth=self._basic, headers=self._headers_json, data=data) except Exception as err: @@ -407,7 +410,7 @@ class WPimport: listelement[i].append(result["id"]) else: self._logger.error("{0} not added due status code : {1}".format(i, page.status_code)) - + self._logger.debug(listelement) title = articletitle[0].text author = articleacreator[0].text.lower() body = articlebody[0].find_all("p") From 84cc204007e6ac1222a8eaac15f24b996b7df7dd Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Tue, 18 Apr 2023 22:01:44 +0200 Subject: [PATCH 10/16] comment update/add in fixing --- lib/WPImport.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/lib/WPImport.py b/lib/WPImport.py index 8f853b1..15d8f89 100644 --- a/lib/WPImport.py +++ b/lib/WPImport.py @@ -14,7 +14,7 @@ class WPimport: self._logger = logger self._parser = parser self._headers_json = {'Content-Type': 'application/json', 'Accept':'application/json'} - + self._request = requests.Session() retries = Retry(total=10, @@ -223,8 +223,22 @@ class WPimport: if i["author"] == j["author_name"] and i["date"] == j["date"]: comment_exist = True id_comment = j["id"] - data = {"post": post, "content": i["content"], "date": i["date"], "author_name": i["author"]} + data = {"post": post, "content": i["content"], "date": i["date"], "author_name": i["author"], "status": "approved"} if comment_exist is True: + if i["parent_id"] != -1: + parent_id = int(i["parent_id"]) + try: + page = self._request.get("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, params=params) + except Exception as err: + self._logger.error("Connection error : {0}".format(err)) + exit(1) + if page.status_code == 200: + result = page.json() + for j in result: + if comment[parent_id]["author"] == j["author_name"] and comment[parent_id]["date"] == j["date"]: + data["parent"]=j["id"] + else: + self._logger.error("Connection error with status code : {0}".format(page.status_code)) try: page = page = self._request.post("http://{0}/wp-json/wp/v2/comments/{1}".format(self._wordpress, id_comment), auth=self._basic, data=data) except Exception as err: @@ -260,6 +274,7 @@ class WPimport: self._logger.info("Comment added for {0}".format(title)) else: self._logger.error("Comment not added for {0} due status code : {1}".format(title, page.status_code)) + self._logger.debug(page.content) else: self._logger.error("Connection error with status code : {0}".format(page.status_code)) From 1c252c9a14c60ee5d9c6ec9e64ebb458452e0b5e Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Wed, 19 Apr 2023 22:21:15 +0200 Subject: [PATCH 11/16] replace post by delete --- lib/WPImport.py | 62 ++++++++++++++++++------------------------------- 1 file changed, 23 insertions(+), 39 deletions(-) diff --git a/lib/WPImport.py b/lib/WPImport.py index 15d8f89..ff7ac92 100644 --- a/lib/WPImport.py +++ b/lib/WPImport.py @@ -225,56 +225,40 @@ class WPimport: id_comment = j["id"] data = {"post": post, "content": i["content"], "date": i["date"], "author_name": i["author"], "status": "approved"} if comment_exist is True: - if i["parent_id"] != -1: - parent_id = int(i["parent_id"]) - try: - page = self._request.get("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, params=params) - except Exception as err: - self._logger.error("Connection error : {0}".format(err)) - exit(1) - if page.status_code == 200: - result = page.json() - for j in result: - if comment[parent_id]["author"] == j["author_name"] and comment[parent_id]["date"] == j["date"]: - data["parent"]=j["id"] - else: - self._logger.error("Connection error with status code : {0}".format(page.status_code)) try: - page = page = self._request.post("http://{0}/wp-json/wp/v2/comments/{1}".format(self._wordpress, id_comment), auth=self._basic, data=data) + page = page = self._request.delete("http://{0}/wp-json/wp/v2/comments/{1}".format(self._wordpress, id_comment), auth=self._basic) except Exception as err: self._logger.error("Connection error : {0}".format(err)) exit(1) if page.status_code == 200: - self._logger.info("Comment updated for {0}".format(title)) + self._logger.info("Comment deleted for {0}".format(title)) else: - self._logger.error("Comment not updated for {0} due status code : {1}".format(title, page.status_code)) - else: - if i["parent_id"] != -1: - parent_id = int(i["parent_id"]) - try: - page = self._request.get("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, params=params) - except Exception as err: - self._logger.error("Connection error : {0}".format(err)) - exit(1) - if page.status_code == 200: - result = page.json() - for j in result: - if comment[parent_id]["author"] == j["author_name"] and comment[parent_id]["date"] == j["date"]: - data["parent"]=j["id"] - else: - self._logger.error("Connection error with status code : {0}".format(page.status_code)) - self._logger.info(data) + self._logger.error("Comment not deleted for {0} due status code : {1}".format(title, page.status_code)) + + if i["parent_id"] != -1: + parent_id = int(i["parent_id"]) try: - page = self._request.post("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, data=data) + page = self._request.get("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, params=params) except Exception as err: self._logger.error("Connection error : {0}".format(err)) exit(1) - self._logger.info(page.status_code) - if page.status_code == 201: - self._logger.info("Comment added for {0}".format(title)) + if page.status_code == 200: + result = page.json() + for j in result: + if comment[parent_id]["author"] == j["author_name"] and comment[parent_id]["date"] == j["date"]: + data["parent"]=j["id"] else: - self._logger.error("Comment not added for {0} due status code : {1}".format(title, page.status_code)) - self._logger.debug(page.content) + self._logger.error("Connection error with status code : {0}".format(page.status_code)) + try: + page = self._request.post("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, data=data) + except Exception as err: + self._logger.error("Connection error : {0}".format(err)) + exit(1) + if page.status_code == 201: + self._logger.info("Comment added for {0}".format(title)) + else: + self._logger.error("Comment not added for {0} due status code : {1}".format(title, page.status_code)) + self._logger.debug(page.content) else: self._logger.error("Connection error with status code : {0}".format(page.status_code)) From f8d103ff61e205f0d1fbdf84e162d5661479549d Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Wed, 19 Apr 2023 23:16:39 +0200 Subject: [PATCH 12/16] fix add comment --- lib/WPImport.py | 79 ++++++++++++++++++++++++------------------------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/lib/WPImport.py b/lib/WPImport.py index ff7ac92..6ce7c0e 100644 --- a/lib/WPImport.py +++ b/lib/WPImport.py @@ -209,24 +209,25 @@ class WPimport: def _addOrUpdateComment(self, post, comment, title): params = {"post": post} - block = True - try: - page = self._request.get("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, params=params) - except Exception as err: - self._logger.error("Connection error : {0}".format(err)) - exit(1) - if page.status_code == 200: - result = page.json() - for i in comment: - comment_exist = False + for i in comment: + id_comment = [] + comment_exist = False + try: + params = {"post": post, "author_name":i["author"], "date":i["date"]} + page = self._request.get("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, params=params) + except Exception as err: + self._logger.error("Connection error : {0}".format(err)) + exit(1) + if page.status_code == 200: + result = page.json() for j in result: - if i["author"] == j["author_name"] and i["date"] == j["date"]: - comment_exist = True - id_comment = j["id"] - data = {"post": post, "content": i["content"], "date": i["date"], "author_name": i["author"], "status": "approved"} - if comment_exist is True: + comment_exist = True + id_comment.append(j["id"]) + data = {"post": post, "content": i["content"], "date": i["date"], "author_name": i["author"], "status": "approved"} + if comment_exist is True: + for j in id_comment: try: - page = page = self._request.delete("http://{0}/wp-json/wp/v2/comments/{1}".format(self._wordpress, id_comment), auth=self._basic) + page = page = self._request.delete("http://{0}/wp-json/wp/v2/comments/{1}".format(self._wordpress, j), auth=self._basic) except Exception as err: self._logger.error("Connection error : {0}".format(err)) exit(1) @@ -235,32 +236,31 @@ class WPimport: else: self._logger.error("Comment not deleted for {0} due status code : {1}".format(title, page.status_code)) - if i["parent_id"] != -1: - parent_id = int(i["parent_id"]) - try: - page = self._request.get("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, params=params) - except Exception as err: - self._logger.error("Connection error : {0}".format(err)) - exit(1) - if page.status_code == 200: - result = page.json() - for j in result: - if comment[parent_id]["author"] == j["author_name"] and comment[parent_id]["date"] == j["date"]: - data["parent"]=j["id"] - else: - self._logger.error("Connection error with status code : {0}".format(page.status_code)) + if i["parent_id"] != -1: + parent_id = int(i["parent_id"]) + params = {"post": post, "author_name":comment[parent_id]["author"], "date":comment[parent_id]["date"]} try: - page = self._request.post("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, data=data) + page = self._request.get("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, params=params) except Exception as err: self._logger.error("Connection error : {0}".format(err)) exit(1) - if page.status_code == 201: - self._logger.info("Comment added for {0}".format(title)) + if page.status_code == 200: + result = page.json() + if len(result) > 0: + data["parent"]=result[0]["id"] else: - self._logger.error("Comment not added for {0} due status code : {1}".format(title, page.status_code)) - self._logger.debug(page.content) - else: - self._logger.error("Connection error with status code : {0}".format(page.status_code)) + self._logger.error("Connection error for parent comment with status code : {0}".format(page.status_code)) + try: + page = self._request.post("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, data=data) + except Exception as err: + self._logger.error("Connection error : {0}".format(err)) + exit(1) + if page.status_code == 201: + self._logger.info("Comment added for {0}".format(title)) + else: + self._logger.error("Comment not added for {0} due status code : {1}".format(title, page.status_code)) + self._logger.debug(page.content) + self._logger.debug("Data : {0}".format(data)) ## Check class name @@ -392,7 +392,7 @@ class WPimport: listelement[i].append(result[0]["id"]) else: - self._logger.error("{0} not finded due status code : {1}".format(i, page.status_code)) + self._logger.error("{0} not found due status code : {1}".format(i, page.status_code)) if element_exist is False: data = {"name": j} @@ -409,7 +409,6 @@ class WPimport: listelement[i].append(result["id"]) else: self._logger.error("{0} not added due status code : {1}".format(i, page.status_code)) - self._logger.debug(listelement) title = articletitle[0].text author = articleacreator[0].text.lower() body = articlebody[0].find_all("p") @@ -467,7 +466,7 @@ class WPimport: else: self._logger.error("Post not updated due status code : {0}".format(page.status_code)) else: - self._logger.error("Connection error with status code : {0}".format(page.status_code)) + self._logger.error("Connection for update post error with status code : {0}".format(page.status_code)) if page_exist == False: try: From ca39826a115e273be547107079f601407b057cf7 Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Wed, 19 Apr 2023 23:53:11 +0200 Subject: [PATCH 13/16] fix comment parent 75% --- lib/WPImport.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/WPImport.py b/lib/WPImport.py index 6ce7c0e..6dc6bb2 100644 --- a/lib/WPImport.py +++ b/lib/WPImport.py @@ -208,7 +208,6 @@ class WPimport: ## Add or update comment def _addOrUpdateComment(self, post, comment, title): - params = {"post": post} for i in comment: id_comment = [] comment_exist = False @@ -223,19 +222,21 @@ class WPimport: for j in result: comment_exist = True id_comment.append(j["id"]) - data = {"post": post, "content": i["content"], "date": i["date"], "author_name": i["author"], "status": "approved"} if comment_exist is True: for j in id_comment: try: - page = page = self._request.delete("http://{0}/wp-json/wp/v2/comments/{1}".format(self._wordpress, j), auth=self._basic) + params = {"force":1} + page = self._request.delete("http://{0}/wp-json/wp/v2/comments/{1}".format(self._wordpress, j), params=params, auth=self._basic) except Exception as err: self._logger.error("Connection error : {0}".format(err)) exit(1) if page.status_code == 200: self._logger.info("Comment deleted for {0}".format(title)) + self._logger.debug("Comment deleted : {0}".format(i)) else: self._logger.error("Comment not deleted for {0} due status code : {1}".format(title, page.status_code)) - + data = {"post": post, "content": i["content"], "date": i["date"], "author_name": i["author"], "status": "approved"} + if i["parent_id"] != -1: parent_id = int(i["parent_id"]) params = {"post": post, "author_name":comment[parent_id]["author"], "date":comment[parent_id]["date"]} @@ -257,6 +258,7 @@ class WPimport: exit(1) if page.status_code == 201: self._logger.info("Comment added for {0}".format(title)) + self._logger.debug("Data : {0}".format(data)) else: self._logger.error("Comment not added for {0} due status code : {1}".format(title, page.status_code)) self._logger.debug(page.content) From c44ffc5a86d7a0f48888cce2fad5cabcab28299f Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Thu, 20 Apr 2023 00:08:56 +0200 Subject: [PATCH 14/16] double comment --- lib/WPImport.py | 91 ++++++++++++++++++++++++------------------------- 1 file changed, 45 insertions(+), 46 deletions(-) diff --git a/lib/WPImport.py b/lib/WPImport.py index 6dc6bb2..79b3545 100644 --- a/lib/WPImport.py +++ b/lib/WPImport.py @@ -208,61 +208,60 @@ class WPimport: ## Add or update comment def _addOrUpdateComment(self, post, comment, title): - for i in comment: - id_comment = [] - comment_exist = False - try: - params = {"post": post, "author_name":i["author"], "date":i["date"]} - page = self._request.get("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, params=params) - except Exception as err: - self._logger.error("Connection error : {0}".format(err)) - exit(1) - if page.status_code == 200: - result = page.json() - for j in result: - comment_exist = True - id_comment.append(j["id"]) - if comment_exist is True: - for j in id_comment: - try: - params = {"force":1} - page = self._request.delete("http://{0}/wp-json/wp/v2/comments/{1}".format(self._wordpress, j), params=params, auth=self._basic) - except Exception as err: - self._logger.error("Connection error : {0}".format(err)) - exit(1) - if page.status_code == 200: - self._logger.info("Comment deleted for {0}".format(title)) - self._logger.debug("Comment deleted : {0}".format(i)) - else: - self._logger.error("Comment not deleted for {0} due status code : {1}".format(title, page.status_code)) - data = {"post": post, "content": i["content"], "date": i["date"], "author_name": i["author"], "status": "approved"} + for i in comment: - if i["parent_id"] != -1: - parent_id = int(i["parent_id"]) - params = {"post": post, "author_name":comment[parent_id]["author"], "date":comment[parent_id]["date"]} try: + params = {"post": post, "author_name":i["author"], "date":i["date"]} page = self._request.get("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, params=params) except Exception as err: self._logger.error("Connection error : {0}".format(err)) exit(1) if page.status_code == 200: result = page.json() - if len(result) > 0: - data["parent"]=result[0]["id"] + for j in result: + try: + params = {"force":1} + page = self._request.delete("http://{0}/wp-json/wp/v2/comments/{1}".format(self._wordpress, j["id"]), params=params, auth=self._basic) + except Exception as err: + self._logger.error("Connection error : {0}".format(err)) + exit(1) + if page.status_code == 200: + self._logger.info("Comment deleted for {0}".format(title)) + self._logger.debug("Comment deleted : {0}".format(j)) + else: + self._logger.error("Comment not deleted for {0} due status code : {1}".format(title, page.status_code)) else: - self._logger.error("Connection error for parent comment with status code : {0}".format(page.status_code)) - try: - page = self._request.post("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, data=data) - except Exception as err: - self._logger.error("Connection error : {0}".format(err)) - exit(1) - if page.status_code == 201: - self._logger.info("Comment added for {0}".format(title)) - self._logger.debug("Data : {0}".format(data)) - else: - self._logger.error("Comment not added for {0} due status code : {1}".format(title, page.status_code)) - self._logger.debug(page.content) - self._logger.debug("Data : {0}".format(data)) + self._logger.error("Comment not listed for {0} due status code : {1}".format(title, page.status_code)) + + for i in comment: + data = {"post": post, "content": i["content"], "date": i["date"], "author_name": i["author"], "status": "approved"} + + if i["parent_id"] != -1: + parent_id = int(i["parent_id"]) + params = {"post": post, "author_name":comment[parent_id]["author"], "date":comment[parent_id]["date"]} + try: + page = self._request.get("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, params=params) + except Exception as err: + self._logger.error("Connection error : {0}".format(err)) + exit(1) + if page.status_code == 200: + result = page.json() + if len(result) > 0: + data["parent"]=result[0]["id"] + else: + self._logger.error("Connection error for parent comment with status code : {0}".format(page.status_code)) + try: + page = self._request.post("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, data=data) + except Exception as err: + self._logger.error("Connection error : {0}".format(err)) + exit(1) + if page.status_code == 201: + self._logger.info("Comment added for {0}".format(title)) + self._logger.debug("Data : {0}".format(data)) + else: + self._logger.error("Comment not added for {0} due status code : {1}".format(title, page.status_code)) + self._logger.debug(page.content) + self._logger.debug("Data : {0}".format(data)) ## Check class name From 34d6cc39d22ce90dc77347bdf5b37060bafe088d Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Thu, 20 Apr 2023 20:48:37 +0200 Subject: [PATCH 15/16] add debug message for error request --- lib/WPImport.py | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/lib/WPImport.py b/lib/WPImport.py index 79b3545..d925f67 100644 --- a/lib/WPImport.py +++ b/lib/WPImport.py @@ -44,6 +44,7 @@ class WPimport: self._addOrUpdateFeaturedMedia(soup) else: self._logger.error("Connection error with status code : {0}".format(r.status_code)) + self._logger.debug(r.content) def fromDirectory(self, directory): @@ -135,14 +136,22 @@ class WPimport: self._logger.info("Ajout media featured : {0}".format(r.json()["title"]["raw"])) else: self._logger.error("Connection error with status code : {0}".format(r.status_code)) + self._logger.debug(r.content) + else: self._logger.info("Aucun media trouvé pour {0}".format(h2)) else: self._logger.error("Connection error with status code : {0}".format(page.status_code)) + self._logger.debug(page.content) + else: self._logger.error("Connection error with status code : {0}".format(page.status_code)) + self._logger.debug(page.content) + else: self._logger.error("Connection error with status code : {0}".format(page.status_code)) + self._logger.debug(page.content) + ## Association image to post @@ -158,6 +167,8 @@ class WPimport: self._logger.info("Association d'une image à l'article {0}".format(title)) else: self._logger.error("Connection error with status code : {0}".format(r.status_code)) + self._logger.debug(r.content) + ## Add or update img @@ -184,6 +195,8 @@ class WPimport: self._logger.info("Image removed {0}".format(img_name)) else: self._logger.error("Image not removed due status code : {0}".format(r.status_code)) + self._logger.debug(r.content) + data = page.content img_type = "image/png" if img_name.split(".")[1] == "jpg" or img_name.split(".")[1] == "jpeg": @@ -201,8 +214,12 @@ class WPimport: media["rendered"] = res["guid"]["rendered"] else: self._logger.error("Image not added due status code : {0}".format(r.status_code)) + self._logger.debug(r.content) + else: self._logger.error("Connection error with status code : {0}".format(r.status_code)) + self._logger.debug(r.content) + return media ## Add or update comment @@ -230,9 +247,12 @@ class WPimport: self._logger.debug("Comment deleted : {0}".format(j)) else: self._logger.error("Comment not deleted for {0} due status code : {1}".format(title, page.status_code)) + self._logger.debug(page.content) + else: self._logger.error("Comment not listed for {0} due status code : {1}".format(title, page.status_code)) - + self._logger.debug(page.content) + for i in comment: data = {"post": post, "content": i["content"], "date": i["date"], "author_name": i["author"], "status": "approved"} @@ -250,6 +270,8 @@ class WPimport: data["parent"]=result[0]["id"] else: self._logger.error("Connection error for parent comment with status code : {0}".format(page.status_code)) + self._logger.debug(page.content) + try: page = self._request.post("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, data=data) except Exception as err: @@ -261,7 +283,6 @@ class WPimport: else: self._logger.error("Comment not added for {0} due status code : {1}".format(title, page.status_code)) self._logger.debug(page.content) - self._logger.debug("Data : {0}".format(data)) ## Check class name @@ -366,6 +387,8 @@ class WPimport: list_img.append(new_img) if page_img.status_code not in [200, 404]: self._logger.error("Connection error with status code : {0}".format(page_img.status_code)) + self._logger.debug(page_img.content) + comment_post = self._getComment(comment) @@ -394,6 +417,8 @@ class WPimport: else: self._logger.error("{0} not found due status code : {1}".format(i, page.status_code)) + self._logger.debug(page.content) + if element_exist is False: data = {"name": j} @@ -410,6 +435,8 @@ class WPimport: listelement[i].append(result["id"]) else: self._logger.error("{0} not added due status code : {1}".format(i, page.status_code)) + self._logger.debug(page.content) + title = articletitle[0].text author = articleacreator[0].text.lower() body = articlebody[0].find_all("p") @@ -438,6 +465,8 @@ class WPimport: data["author"] = result[0]["id"] else: self._logger.error("Connection error with status code : {0}".format(page.status_code)) + self._logger.debug(page.content) + params = {"search":title} try: @@ -466,8 +495,12 @@ class WPimport: self._linkImgPost(result["title"]["raw"], list_img, result["id"]) else: self._logger.error("Post not updated due status code : {0}".format(page.status_code)) + self._logger.debug(page.content) + else: self._logger.error("Connection for update post error with status code : {0}".format(page.status_code)) + self._logger.debug(page.content) + if page_exist == False: try: @@ -481,4 +514,5 @@ class WPimport: self._addOrUpdateComment(result["id"], comment_post, result["title"]["raw"]) self._linkImgPost(result["title"]["raw"], list_img, result["id"]) else: - self._logger.error("Post not added due status code : {0}".format(r.status_code)) \ No newline at end of file + self._logger.error("Post not added due status code : {0}".format(r.status_code)) + self._logger.debug(r.content) From 4e6ae92217edde9ccbcd8baeb7a66eb3e54df4c3 Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Thu, 20 Apr 2023 20:53:50 +0200 Subject: [PATCH 16/16] add message error and debug for export --- lib/WPExport.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/lib/WPExport.py b/lib/WPExport.py index fffc32a..da4f809 100644 --- a/lib/WPExport.py +++ b/lib/WPExport.py @@ -69,6 +69,10 @@ class WPExport: href = anchor.get('href', '/') if href != "#": page_url.append(href) + else: + self._logger.error("Url did not get due status code : {0}".format(page.status_code)) + self._logger.debug(page.content) + webpage = [] for i in page_url: @@ -113,6 +117,10 @@ class WPExport: self._logger.error("parsing error : {0}".format(err)) exit(1) webpage.append(o) + else: + self._logger.error("web didn't get due status code : {0}".format(page.status_code)) + self._logger.debug(page.content) + return webpage @@ -179,6 +187,10 @@ class WPExport: o = o._replace(netloc=u.netloc) o = o._replace(scheme=u.scheme) page_url.append(o.geturl()) + else: + self._logger.error("JS or CSS did not get due status code : {0}".format(page.status_code)) + self._logger.debug(page.content) + return page_url # Get image @@ -201,6 +213,10 @@ class WPExport: if src not in page_img: self._logger.info("image: {0} : ".format(src)) page_img.append(src) + else: + self._logger.error("Image did not get due status code : {0}".format(page.status_code)) + self._logger.debug(page.content) + return page_img @@ -232,4 +248,7 @@ class WPExport: open(fileDownload, "wb").write(r.content) except Exception as err: self._logger.error("file error : {0}".format(err)) - exit(1) \ No newline at end of file + exit(1) + else: + self._logger.error("Not download due status code : {0}".format(r.status_code)) + self._logger.debug(r.content)