v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

add retry

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-04-13 21:54:35 +02:00
parent f5e82fe4c4
commit 1311ef2ff2
2 changed files with 44 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
lib/WPExport.py
View File

@ -2,6 +2,8 @@
from bs4 import BeautifulSoup
from urllib.parse import urlparse
import requests, os, argparse, logging
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
class WPExport:
def __init__(self, url, logger, parser, directory):
@ -10,6 +12,13 @@ class WPExport:
self._parser = parser
self._dir = directory
self._request = requests.Session()
retries = Retry(total=5,
status_forcelist=[429, 500, 502, 503, 504])
self._request.mount('http://', HTTPAdapter(max_retries=retries))
# Public method
@ -48,7 +57,7 @@ class WPExport:
# Get URL
def getUrlPage(self):
try:
page = requests.get(self._url)
page = self._request.get(self._url)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
@ -64,7 +73,7 @@ class WPExport:
webpage = []
for i in page_url:
try:
page = requests.get(i)
page = self._request.get(i)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
@ -90,7 +99,7 @@ class WPExport:
self._logger.info(url_paging)
if url_paging not in webpage:
webpage.append(url_paging)
page = requests.get(url_paging)
page = self._request.get(url_paging)
if page.status_code == 200:
soup = BeautifulSoup(page.text, self._parser)
h2 = soup.find_all("h2")
@ -131,7 +140,7 @@ class WPExport:
# Get Css and JS
def _getScriptCss(self, js, css):
try:
page = requests.get(self._url)
page = self._request.get(self._url)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
@ -178,7 +187,7 @@ class WPExport:
page_img = []
for i in webpage:
try:
page = requests.get(i)
page = self._request.get(i)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
@ -210,7 +219,7 @@ class WPExport:
dir_page_web = "/".join(path_web)
self._mkdirPath("{0}/{1}/{2}".format(backup_dir, o.netloc, dir_page_web))
try:
r = requests.get(webpage[i])
r = self._request.get(webpage[i])
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)

49
lib/WPImport.py
View File

@ -3,6 +3,8 @@
from bs4 import BeautifulSoup
from urllib.parse import urlparse
import requests, os, logging, re, json
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
class WPimport:
# Constructor
@ -12,6 +14,13 @@ class WPimport:
self._logger = logger
self._parser = parser
self._request = requests.Session()
retries = Retry(total=5,
status_forcelist=[429, 500, 502, 503, 504])
self._request.mount('http://', HTTPAdapter(max_retries=retries))
# Public method
def setUrl(self, wordpress):
@ -19,7 +28,7 @@ class WPimport:
def fromUrl(self, webpage):
for i in range(0, len(webpage)):
r = requests.get(webpage[i])
r = self._request.get(webpage[i])
if r.status_code == 200:
self._logger.info("({0}/{1} : Page en cours d'import : {2}".format(i+1, len(webpage), webpage[i]))
soup = BeautifulSoup(r.content, self._parser)
@ -79,7 +88,7 @@ class WPimport:
for i in item_div:
h2 = i.find_all("h2")[0].text
params = {"search":h2, "type":"post"}
page = requests.get("http://{0}/wp-json/wp/v2/search".format(self._wordpress), auth=self._basic, params=params)
page = self._request.get("http://{0}/wp-json/wp/v2/search".format(self._wordpress), auth=self._basic, params=params)
if page.status_code == 200:
result = page.json()
if len(result) > 0:
@ -87,19 +96,19 @@ class WPimport:
img = i.find_all("img")
if len(img) > 0:
img_src = img[0].get("src")
page = requests.get(img_src)
page = self._request.get(img_src)
if page.status_code == 200:
name_img = img_src.replace("_q", "")
name_img = name_img.split("/")[len(name_img.split("/"))-1]
params = {"search": name_img}
page = requests.get("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, params=params)
page = self._request.get("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, params=params)
if page.status_code == 200:
res = page.json()
if len(res) > 0:
id_media = res[0]["id"]
headers = {'Content-Type': 'application/json', 'Accept':'application/json'}
data = {"featured_media": id_media}
r = requests.post("http://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, result[0]["id"]), auth=self._basic, headers=headers, data=json.dumps(data))
r = self._request.post("http://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, result[0]["id"]), auth=self._basic, headers=headers, data=json.dumps(data))
if r.status_code == 200:
self._logger.info("Ajout media featured : {0}".format(r.json()["title"]["raw"]))
else:
@ -110,7 +119,7 @@ class WPimport:
def _linkImgPost(self, title, list_img, post_id):
for i in list_img:
data = {"post": post_id}
r = requests.post("http://{0}/wp-json/wp/v2/media/{1}".format(self._wordpress, i["id"]), auth=self._basic, data=data)
r = self._request.post("http://{0}/wp-json/wp/v2/media/{1}".format(self._wordpress, i["id"]), auth=self._basic, data=data)
if r.status_code == 200:
self._logger.info("Association d'une image à l'article {0}".format(title))
@ -121,12 +130,12 @@ class WPimport:
split_fileimg = href_img.split("/")
img_name = split_fileimg[len(split_fileimg)-1]
params = { "search": img_name}
r = requests.get("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, params=params)
r = self._request.get("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, params=params)
if r.status_code == 200:
res = r.json()
if len(res) > 0:
params = {"force":1}
r = requests.delete("http://{0}/wp-json/wp/v2/media/{1}".format(self._wordpress, res[0]["id"]), auth=self._basic, params=params)
r = self._request.delete("http://{0}/wp-json/wp/v2/media/{1}".format(self._wordpress, res[0]["id"]), auth=self._basic, params=params)
if r.status_code == 200:
self._logger.info("Image supprimé {0}".format(img_name))
data = page.content
@ -134,7 +143,7 @@ class WPimport:
if img_name.split(".")[1] == "jpg" or img_name.split(".")[1] == "jpeg":
img_type = "image/jpg"
headers={ 'Content-Type': img_type,'Content-Disposition' : 'attachment; filename={0}'.format(img_name)}
r = requests.post("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, headers=headers, data=data)
r = self._request.post("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, headers=headers, data=data)
if r.status_code == 201:
self._logger.info("Ajout d'image {0}".format(img_name))
res = r.json()
@ -147,7 +156,7 @@ class WPimport:
def _addOrUpdateComment(self, post, comment, title):
params = {"post": post}
block = True
page = requests.get("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, params=params)
page = self._request.get("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, params=params)
if page.status_code == 200:
result = page.json()
for i in comment:
@ -158,11 +167,11 @@ class WPimport:
id_comment = j["id"]
data = {"post": post, "content": i["content"], "date": i["date"], "author_name": i["author"]}
if comment_exist is True:
page = page = requests.post("http://{0}/wp-json/wp/v2/comments/{1}".format(self._wordpress, id_comment), auth=self._basic, data=data)
page = page = self._request.post("http://{0}/wp-json/wp/v2/comments/{1}".format(self._wordpress, id_comment), auth=self._basic, data=data)
if page.status_code == 200:
self._logger.info("Commentaire mise à jour pour {0}".format(title))
else:
page = requests.post("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, data=data)
page = self._request.post("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, data=data)
if page.status_code == 201:
self._logger.info("Commentaire ajoute pour {0}".format(title))
@ -177,7 +186,7 @@ class WPimport:
listelement = {}
for i in liste:
page = requests.get("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress,i))
page = self._request.get("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress,i))
if page.status_code == 200:
elements[i] = page.json()
element[i] = []
@ -200,10 +209,10 @@ class WPimport:
href_img = img[0].get("src")
new_img["old_src"]=href_img
new_img["old_href"]=href_a
page_img = requests.get(href_img)
page_img = self._request.get(href_img)
if page_img.status_code == 404:
href_img = href_a
page_img = requests.get(href_a)
page_img = self._request.get(href_a)
if page_img.status_code == 200:
media=self._addOrUpdateMedia(href_img, page_img)
new_img["id"]=media["id"]
@ -245,7 +254,7 @@ class WPimport:
listelement[i].append(k["id"])
if element_exist is False:
data = {"name": j}
page = requests.post("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i), auth=self._basic, data=data)
page = self._request.post("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i), auth=self._basic, data=data)
if page.status_code == 201:
result = page.json()
listelement[i].append(result["id"])
@ -268,13 +277,13 @@ class WPimport:
time = dateheader[0].text.split(" ")
data = {"title":title, "content":bodyhtml, "status":"publish", "date": "{0}-{1}-{2}T{3}:00".format(time[2],month[time[1]],time[0], hour), "tags": listelement["tags"], "categories": listelement["categories"]}
params = {"search":author}
page = requests.get("http://{0}/wp-json/wp/v2/users".format(self._wordpress), auth=self._basic, params=params)
page = self._request.get("http://{0}/wp-json/wp/v2/users".format(self._wordpress), auth=self._basic, params=params)
if page.status_code == 200:
result = page.json()
data["author"] = result[0]["id"]
params = {"search":title}
page = requests.get("http://{0}/wp-json/wp/v2/posts".format(self._wordpress), auth=self._basic, params=params)
page = self._request.get("http://{0}/wp-json/wp/v2/posts".format(self._wordpress), auth=self._basic, params=params)
page_exist = True
headers = {'Content-Type': 'application/json', 'Accept':'application/json'}
if page.status_code == 200:
@ -284,7 +293,7 @@ class WPimport:
else:
self._logger.info("La page {0} existe deja et mis à jour".format(title))
post_id = result[0]["id"]
page = requests.post("http://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, post_id), auth=self._basic, headers=headers, data=json.dumps(data))
page = self._request.post("http://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, post_id), auth=self._basic, headers=headers, data=json.dumps(data))
if page.status_code == 200:
result = page.json()
self._logger.info("Article mis à jour : {0}".format(result["title"]["raw"]))
@ -292,7 +301,7 @@ class WPimport:
self._linkImgPost(result["title"]["raw"], list_img, result["id"])
if page_exist == False:
page = requests.post("http://{0}/wp-json/wp/v2/posts".format(self._wordpress), auth=self._basic, headers=headers, data=json.dumps(data))
page = self._request.post("http://{0}/wp-json/wp/v2/posts".format(self._wordpress), auth=self._basic, headers=headers, data=json.dumps(data))
if page.status_code == 201:
result = page.json()
self._logger.info("Article ajoute : {0}".format(result["title"]["raw"]))