v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

add try/except for request

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-04-13 22:14:30 +02:00
parent 225c7ecabb
commit 74e7f1d74b
1 changed files with 100 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
lib/WPImport.py
View File

@ -28,7 +28,11 @@ class WPimport:
def fromUrl(self, webpage): def fromUrl(self, webpage):
for i in range(0, len(webpage)): for i in range(0, len(webpage)):
r = self._request.get(webpage[i]) try:
r = self._request.get(webpage[i])
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if r.status_code == 200: if r.status_code == 200:
self._logger.info("({0}/{1} : Page en cours d'import : {2}".format(i+1, len(webpage), webpage[i])) self._logger.info("({0}/{1} : Page en cours d'import : {2}".format(i+1, len(webpage), webpage[i]))
soup = BeautifulSoup(r.content, self._parser) soup = BeautifulSoup(r.content, self._parser)
@ -88,7 +92,11 @@ class WPimport:
for i in item_div: for i in item_div:
h2 = i.find_all("h2")[0].text h2 = i.find_all("h2")[0].text
params = {"search":h2, "type":"post"} params = {"search":h2, "type":"post"}
page = self._request.get("http://{0}/wp-json/wp/v2/search".format(self._wordpress), auth=self._basic, params=params) try:
page = self._request.get("http://{0}/wp-json/wp/v2/search".format(self._wordpress), auth=self._basic, params=params)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if page.status_code == 200: if page.status_code == 200:
result = page.json() result = page.json()
if len(result) > 0: if len(result) > 0:
@ -96,19 +104,31 @@ class WPimport:
img = i.find_all("img") img = i.find_all("img")
if len(img) > 0: if len(img) > 0:
img_src = img[0].get("src") img_src = img[0].get("src")
page = self._request.get(img_src) try:
page = self._request.get(img_src)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if page.status_code == 200: if page.status_code == 200:
name_img = img_src.replace("_q", "") name_img = img_src.replace("_q", "")
name_img = name_img.split("/")[len(name_img.split("/"))-1] name_img = name_img.split("/")[len(name_img.split("/"))-1]
params = {"search": name_img} params = {"search": name_img}
page = self._request.get("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, params=params) try:
page = self._request.get("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, params=params)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if page.status_code == 200: if page.status_code == 200:
res = page.json() res = page.json()
if len(res) > 0: if len(res) > 0:
id_media = res[0]["id"] id_media = res[0]["id"]
headers = {'Content-Type': 'application/json', 'Accept':'application/json'} headers = {'Content-Type': 'application/json', 'Accept':'application/json'}
data = {"featured_media": id_media} data = {"featured_media": id_media}
r = self._request.post("http://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, result[0]["id"]), auth=self._basic, headers=headers, data=json.dumps(data)) try:
r = self._request.post("http://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, result[0]["id"]), auth=self._basic, headers=headers, data=json.dumps(data))
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if r.status_code == 200: if r.status_code == 200:
self._logger.info("Ajout media featured : {0}".format(r.json()["title"]["raw"])) self._logger.info("Ajout media featured : {0}".format(r.json()["title"]["raw"]))
else: else:
@ -119,7 +139,11 @@ class WPimport:
def _linkImgPost(self, title, list_img, post_id): def _linkImgPost(self, title, list_img, post_id):
for i in list_img: for i in list_img:
data = {"post": post_id} data = {"post": post_id}
r = self._request.post("http://{0}/wp-json/wp/v2/media/{1}".format(self._wordpress, i["id"]), auth=self._basic, data=data) try:
r = self._request.post("http://{0}/wp-json/wp/v2/media/{1}".format(self._wordpress, i["id"]), auth=self._basic, data=data)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if r.status_code == 200: if r.status_code == 200:
self._logger.info("Association d'une image à l'article {0}".format(title)) self._logger.info("Association d'une image à l'article {0}".format(title))
@ -130,12 +154,20 @@ class WPimport:
split_fileimg = href_img.split("/") split_fileimg = href_img.split("/")
img_name = split_fileimg[len(split_fileimg)-1] img_name = split_fileimg[len(split_fileimg)-1]
params = { "search": img_name} params = { "search": img_name}
r = self._request.get("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, params=params) try:
r = self._request.get("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, params=params)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if r.status_code == 200: if r.status_code == 200:
res = r.json() res = r.json()
if len(res) > 0: if len(res) > 0:
params = {"force":1} params = {"force":1}
r = self._request.delete("http://{0}/wp-json/wp/v2/media/{1}".format(self._wordpress, res[0]["id"]), auth=self._basic, params=params) try:
r = self._request.delete("http://{0}/wp-json/wp/v2/media/{1}".format(self._wordpress, res[0]["id"]), auth=self._basic, params=params)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if r.status_code == 200: if r.status_code == 200:
self._logger.info("Image supprimé {0}".format(img_name)) self._logger.info("Image supprimé {0}".format(img_name))
data = page.content data = page.content
@ -143,7 +175,11 @@ class WPimport:
if img_name.split(".")[1] == "jpg" or img_name.split(".")[1] == "jpeg": if img_name.split(".")[1] == "jpg" or img_name.split(".")[1] == "jpeg":
img_type = "image/jpg" img_type = "image/jpg"
headers={ 'Content-Type': img_type,'Content-Disposition' : 'attachment; filename={0}'.format(img_name)} headers={ 'Content-Type': img_type,'Content-Disposition' : 'attachment; filename={0}'.format(img_name)}
r = self._request.post("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, headers=headers, data=data) try:
r = self._request.post("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, headers=headers, data=data)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if r.status_code == 201: if r.status_code == 201:
self._logger.info("Ajout d'image {0}".format(img_name)) self._logger.info("Ajout d'image {0}".format(img_name))
res = r.json() res = r.json()
@ -156,7 +192,11 @@ class WPimport:
def _addOrUpdateComment(self, post, comment, title): def _addOrUpdateComment(self, post, comment, title):
params = {"post": post} params = {"post": post}
block = True block = True
page = self._request.get("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, params=params) try:
page = self._request.get("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, params=params)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if page.status_code == 200: if page.status_code == 200:
result = page.json() result = page.json()
for i in comment: for i in comment:
@ -167,11 +207,19 @@ class WPimport:
id_comment = j["id"] id_comment = j["id"]
data = {"post": post, "content": i["content"], "date": i["date"], "author_name": i["author"]} data = {"post": post, "content": i["content"], "date": i["date"], "author_name": i["author"]}
if comment_exist is True: if comment_exist is True:
page = page = self._request.post("http://{0}/wp-json/wp/v2/comments/{1}".format(self._wordpress, id_comment), auth=self._basic, data=data) try:
page = page = self._request.post("http://{0}/wp-json/wp/v2/comments/{1}".format(self._wordpress, id_comment), auth=self._basic, data=data)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if page.status_code == 200: if page.status_code == 200:
self._logger.info("Commentaire mise à jour pour {0}".format(title)) self._logger.info("Commentaire mise à jour pour {0}".format(title))
else: else:
page = self._request.post("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, data=data) try:
page = self._request.post("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, data=data)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if page.status_code == 201: if page.status_code == 201:
self._logger.info("Commentaire ajoute pour {0}".format(title)) self._logger.info("Commentaire ajoute pour {0}".format(title))
@ -186,7 +234,11 @@ class WPimport:
listelement = {} listelement = {}
for i in liste: for i in liste:
page = self._request.get("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress,i)) try:
page = self._request.get("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress,i))
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if page.status_code == 200: if page.status_code == 200:
elements[i] = page.json() elements[i] = page.json()
element[i] = [] element[i] = []
@ -209,10 +261,18 @@ class WPimport:
href_img = img[0].get("src") href_img = img[0].get("src")
new_img["old_src"]=href_img new_img["old_src"]=href_img
new_img["old_href"]=href_a new_img["old_href"]=href_a
page_img = self._request.get(href_img) try:
page_img = self._request.get(href_img)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if page_img.status_code == 404: if page_img.status_code == 404:
href_img = href_a href_img = href_a
page_img = self._request.get(href_a) try:
page_img = self._request.get(href_a)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if page_img.status_code == 200: if page_img.status_code == 200:
media=self._addOrUpdateMedia(href_img, page_img) media=self._addOrUpdateMedia(href_img, page_img)
new_img["id"]=media["id"] new_img["id"]=media["id"]
@ -254,7 +314,11 @@ class WPimport:
listelement[i].append(k["id"]) listelement[i].append(k["id"])
if element_exist is False: if element_exist is False:
data = {"name": j} data = {"name": j}
page = self._request.post("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i), auth=self._basic, data=data) try:
page = self._request.post("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i), auth=self._basic, data=data)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if page.status_code == 201: if page.status_code == 201:
result = page.json() result = page.json()
listelement[i].append(result["id"]) listelement[i].append(result["id"])
@ -277,13 +341,21 @@ class WPimport:
time = dateheader[0].text.split(" ") time = dateheader[0].text.split(" ")
data = {"title":title, "content":bodyhtml, "status":"publish", "date": "{0}-{1}-{2}T{3}:00".format(time[2],month[time[1]],time[0], hour), "tags": listelement["tags"], "categories": listelement["categories"]} data = {"title":title, "content":bodyhtml, "status":"publish", "date": "{0}-{1}-{2}T{3}:00".format(time[2],month[time[1]],time[0], hour), "tags": listelement["tags"], "categories": listelement["categories"]}
params = {"search":author} params = {"search":author}
page = self._request.get("http://{0}/wp-json/wp/v2/users".format(self._wordpress), auth=self._basic, params=params) try:
page = self._request.get("http://{0}/wp-json/wp/v2/users".format(self._wordpress), auth=self._basic, params=params)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if page.status_code == 200: if page.status_code == 200:
result = page.json() result = page.json()
data["author"] = result[0]["id"] data["author"] = result[0]["id"]
params = {"search":title} params = {"search":title}
page = self._request.get("http://{0}/wp-json/wp/v2/posts".format(self._wordpress), auth=self._basic, params=params) try:
page = self._request.get("http://{0}/wp-json/wp/v2/posts".format(self._wordpress), auth=self._basic, params=params)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
page_exist = True page_exist = True
headers = {'Content-Type': 'application/json', 'Accept':'application/json'} headers = {'Content-Type': 'application/json', 'Accept':'application/json'}
if page.status_code == 200: if page.status_code == 200:
@ -293,7 +365,11 @@ class WPimport:
else: else:
self._logger.info("La page {0} existe deja et mis à jour".format(title)) self._logger.info("La page {0} existe deja et mis à jour".format(title))
post_id = result[0]["id"] post_id = result[0]["id"]
page = self._request.post("http://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, post_id), auth=self._basic, headers=headers, data=json.dumps(data)) try:
page = self._request.post("http://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, post_id), auth=self._basic, headers=headers, data=json.dumps(data))
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if page.status_code == 200: if page.status_code == 200:
result = page.json() result = page.json()
self._logger.info("Article mis à jour : {0}".format(result["title"]["raw"])) self._logger.info("Article mis à jour : {0}".format(result["title"]["raw"]))
@ -301,7 +377,11 @@ class WPimport:
self._linkImgPost(result["title"]["raw"], list_img, result["id"]) self._linkImgPost(result["title"]["raw"], list_img, result["id"])
if page_exist == False: if page_exist == False:
page = self._request.post("http://{0}/wp-json/wp/v2/posts".format(self._wordpress), auth=self._basic, headers=headers, data=json.dumps(data)) try:
page = self._request.post("http://{0}/wp-json/wp/v2/posts".format(self._wordpress), auth=self._basic, headers=headers, data=json.dumps(data))
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if page.status_code == 201: if page.status_code == 201:
result = page.json() result = page.json()
self._logger.info("Article ajoute : {0}".format(result["title"]["raw"])) self._logger.info("Article ajoute : {0}".format(result["title"]["raw"]))