From 1311ef2ff27a88b757f5d96f05e0486eeb197c89 Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Thu, 13 Apr 2023 21:54:35 +0200 Subject: [PATCH] add retry --- lib/WPExport.py | 21 +++++++++++++++------ lib/WPImport.py | 49 +++++++++++++++++++++++++++++-------------------- 2 files changed, 44 insertions(+), 26 deletions(-) diff --git a/lib/WPExport.py b/lib/WPExport.py index c5b6be9..9d58600 100644 --- a/lib/WPExport.py +++ b/lib/WPExport.py @@ -2,6 +2,8 @@ from bs4 import BeautifulSoup from urllib.parse import urlparse import requests, os, argparse, logging +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry class WPExport: def __init__(self, url, logger, parser, directory): @@ -10,6 +12,13 @@ class WPExport: self._parser = parser self._dir = directory + self._request = requests.Session() + + retries = Retry(total=5, + status_forcelist=[429, 500, 502, 503, 504]) + + self._request.mount('http://', HTTPAdapter(max_retries=retries)) + # Public method @@ -48,7 +57,7 @@ class WPExport: # Get URL def getUrlPage(self): try: - page = requests.get(self._url) + page = self._request.get(self._url) except Exception as err: self._logger.error("Connection error : {0}".format(err)) exit(1) @@ -64,7 +73,7 @@ class WPExport: webpage = [] for i in page_url: try: - page = requests.get(i) + page = self._request.get(i) except Exception as err: self._logger.error("Connection error : {0}".format(err)) exit(1) @@ -90,7 +99,7 @@ class WPExport: self._logger.info(url_paging) if url_paging not in webpage: webpage.append(url_paging) - page = requests.get(url_paging) + page = self._request.get(url_paging) if page.status_code == 200: soup = BeautifulSoup(page.text, self._parser) h2 = soup.find_all("h2") @@ -131,7 +140,7 @@ class WPExport: # Get Css and JS def _getScriptCss(self, js, css): try: - page = requests.get(self._url) + page = self._request.get(self._url) except Exception as err: self._logger.error("Connection error : {0}".format(err)) exit(1) @@ -178,7 +187,7 @@ class WPExport: page_img = [] for i in webpage: try: - page = requests.get(i) + page = self._request.get(i) except Exception as err: self._logger.error("Connection error : {0}".format(err)) exit(1) @@ -210,7 +219,7 @@ class WPExport: dir_page_web = "/".join(path_web) self._mkdirPath("{0}/{1}/{2}".format(backup_dir, o.netloc, dir_page_web)) try: - r = requests.get(webpage[i]) + r = self._request.get(webpage[i]) except Exception as err: self._logger.error("Connection error : {0}".format(err)) exit(1) diff --git a/lib/WPImport.py b/lib/WPImport.py index 60e8cbc..2ecdfc6 100644 --- a/lib/WPImport.py +++ b/lib/WPImport.py @@ -3,6 +3,8 @@ from bs4 import BeautifulSoup from urllib.parse import urlparse import requests, os, logging, re, json +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry class WPimport: # Constructor @@ -12,6 +14,13 @@ class WPimport: self._logger = logger self._parser = parser + self._request = requests.Session() + + retries = Retry(total=5, + status_forcelist=[429, 500, 502, 503, 504]) + + self._request.mount('http://', HTTPAdapter(max_retries=retries)) + # Public method def setUrl(self, wordpress): @@ -19,7 +28,7 @@ class WPimport: def fromUrl(self, webpage): for i in range(0, len(webpage)): - r = requests.get(webpage[i]) + r = self._request.get(webpage[i]) if r.status_code == 200: self._logger.info("({0}/{1} : Page en cours d'import : {2}".format(i+1, len(webpage), webpage[i])) soup = BeautifulSoup(r.content, self._parser) @@ -79,7 +88,7 @@ class WPimport: for i in item_div: h2 = i.find_all("h2")[0].text params = {"search":h2, "type":"post"} - page = requests.get("http://{0}/wp-json/wp/v2/search".format(self._wordpress), auth=self._basic, params=params) + page = self._request.get("http://{0}/wp-json/wp/v2/search".format(self._wordpress), auth=self._basic, params=params) if page.status_code == 200: result = page.json() if len(result) > 0: @@ -87,19 +96,19 @@ class WPimport: img = i.find_all("img") if len(img) > 0: img_src = img[0].get("src") - page = requests.get(img_src) + page = self._request.get(img_src) if page.status_code == 200: name_img = img_src.replace("_q", "") name_img = name_img.split("/")[len(name_img.split("/"))-1] params = {"search": name_img} - page = requests.get("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, params=params) + page = self._request.get("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, params=params) if page.status_code == 200: res = page.json() if len(res) > 0: id_media = res[0]["id"] headers = {'Content-Type': 'application/json', 'Accept':'application/json'} data = {"featured_media": id_media} - r = requests.post("http://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, result[0]["id"]), auth=self._basic, headers=headers, data=json.dumps(data)) + r = self._request.post("http://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, result[0]["id"]), auth=self._basic, headers=headers, data=json.dumps(data)) if r.status_code == 200: self._logger.info("Ajout media featured : {0}".format(r.json()["title"]["raw"])) else: @@ -110,7 +119,7 @@ class WPimport: def _linkImgPost(self, title, list_img, post_id): for i in list_img: data = {"post": post_id} - r = requests.post("http://{0}/wp-json/wp/v2/media/{1}".format(self._wordpress, i["id"]), auth=self._basic, data=data) + r = self._request.post("http://{0}/wp-json/wp/v2/media/{1}".format(self._wordpress, i["id"]), auth=self._basic, data=data) if r.status_code == 200: self._logger.info("Association d'une image à l'article {0}".format(title)) @@ -121,12 +130,12 @@ class WPimport: split_fileimg = href_img.split("/") img_name = split_fileimg[len(split_fileimg)-1] params = { "search": img_name} - r = requests.get("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, params=params) + r = self._request.get("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, params=params) if r.status_code == 200: res = r.json() if len(res) > 0: params = {"force":1} - r = requests.delete("http://{0}/wp-json/wp/v2/media/{1}".format(self._wordpress, res[0]["id"]), auth=self._basic, params=params) + r = self._request.delete("http://{0}/wp-json/wp/v2/media/{1}".format(self._wordpress, res[0]["id"]), auth=self._basic, params=params) if r.status_code == 200: self._logger.info("Image supprimé {0}".format(img_name)) data = page.content @@ -134,7 +143,7 @@ class WPimport: if img_name.split(".")[1] == "jpg" or img_name.split(".")[1] == "jpeg": img_type = "image/jpg" headers={ 'Content-Type': img_type,'Content-Disposition' : 'attachment; filename={0}'.format(img_name)} - r = requests.post("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, headers=headers, data=data) + r = self._request.post("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, headers=headers, data=data) if r.status_code == 201: self._logger.info("Ajout d'image {0}".format(img_name)) res = r.json() @@ -147,7 +156,7 @@ class WPimport: def _addOrUpdateComment(self, post, comment, title): params = {"post": post} block = True - page = requests.get("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, params=params) + page = self._request.get("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, params=params) if page.status_code == 200: result = page.json() for i in comment: @@ -158,11 +167,11 @@ class WPimport: id_comment = j["id"] data = {"post": post, "content": i["content"], "date": i["date"], "author_name": i["author"]} if comment_exist is True: - page = page = requests.post("http://{0}/wp-json/wp/v2/comments/{1}".format(self._wordpress, id_comment), auth=self._basic, data=data) + page = page = self._request.post("http://{0}/wp-json/wp/v2/comments/{1}".format(self._wordpress, id_comment), auth=self._basic, data=data) if page.status_code == 200: self._logger.info("Commentaire mise à jour pour {0}".format(title)) else: - page = requests.post("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, data=data) + page = self._request.post("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, data=data) if page.status_code == 201: self._logger.info("Commentaire ajoute pour {0}".format(title)) @@ -177,7 +186,7 @@ class WPimport: listelement = {} for i in liste: - page = requests.get("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress,i)) + page = self._request.get("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress,i)) if page.status_code == 200: elements[i] = page.json() element[i] = [] @@ -200,10 +209,10 @@ class WPimport: href_img = img[0].get("src") new_img["old_src"]=href_img new_img["old_href"]=href_a - page_img = requests.get(href_img) + page_img = self._request.get(href_img) if page_img.status_code == 404: href_img = href_a - page_img = requests.get(href_a) + page_img = self._request.get(href_a) if page_img.status_code == 200: media=self._addOrUpdateMedia(href_img, page_img) new_img["id"]=media["id"] @@ -245,7 +254,7 @@ class WPimport: listelement[i].append(k["id"]) if element_exist is False: data = {"name": j} - page = requests.post("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i), auth=self._basic, data=data) + page = self._request.post("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i), auth=self._basic, data=data) if page.status_code == 201: result = page.json() listelement[i].append(result["id"]) @@ -268,13 +277,13 @@ class WPimport: time = dateheader[0].text.split(" ") data = {"title":title, "content":bodyhtml, "status":"publish", "date": "{0}-{1}-{2}T{3}:00".format(time[2],month[time[1]],time[0], hour), "tags": listelement["tags"], "categories": listelement["categories"]} params = {"search":author} - page = requests.get("http://{0}/wp-json/wp/v2/users".format(self._wordpress), auth=self._basic, params=params) + page = self._request.get("http://{0}/wp-json/wp/v2/users".format(self._wordpress), auth=self._basic, params=params) if page.status_code == 200: result = page.json() data["author"] = result[0]["id"] params = {"search":title} - page = requests.get("http://{0}/wp-json/wp/v2/posts".format(self._wordpress), auth=self._basic, params=params) + page = self._request.get("http://{0}/wp-json/wp/v2/posts".format(self._wordpress), auth=self._basic, params=params) page_exist = True headers = {'Content-Type': 'application/json', 'Accept':'application/json'} if page.status_code == 200: @@ -284,7 +293,7 @@ class WPimport: else: self._logger.info("La page {0} existe deja et mis à jour".format(title)) post_id = result[0]["id"] - page = requests.post("http://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, post_id), auth=self._basic, headers=headers, data=json.dumps(data)) + page = self._request.post("http://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, post_id), auth=self._basic, headers=headers, data=json.dumps(data)) if page.status_code == 200: result = page.json() self._logger.info("Article mis à jour : {0}".format(result["title"]["raw"])) @@ -292,7 +301,7 @@ class WPimport: self._linkImgPost(result["title"]["raw"], list_img, result["id"]) if page_exist == False: - page = requests.post("http://{0}/wp-json/wp/v2/posts".format(self._wordpress), auth=self._basic, headers=headers, data=json.dumps(data)) + page = self._request.post("http://{0}/wp-json/wp/v2/posts".format(self._wordpress), auth=self._basic, headers=headers, data=json.dumps(data)) if page.status_code == 201: result = page.json() self._logger.info("Article ajoute : {0}".format(result["title"]["raw"]))