v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

add name thread in message logger

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-04-28 23:14:57 +02:00
parent bf4c2480f8
commit 5a4bdbb420
1 changed files with 77 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

169
lib/WPImport.py
View File

@ -33,10 +33,10 @@ class WPimport:
try: try:
r = self._request.get(webpage[i]) r = self._request.get(webpage[i])
except Exception as err: except Exception as err:
self._logger.error("Connection error : {0}".format(err)) self._logger.error("{0} : Connection error for get url {1} : {2}".format(self._name, webpage[i], err))
exit(1) exit(1)
if r.status_code == 200: if r.status_code == 200:
self._logger.info("({0}/{1} : Page en cours d'import : {2}".format(i+1, len(webpage), webpage[i])) self._logger.info("{0} : ({1}/{2} : Page is importing : {3}".format(self._name, i+1, len(webpage), webpage[i]))
soup = BeautifulSoup(r.content, self._parser) soup = BeautifulSoup(r.content, self._parser)
articlebody = soup.find_all("div", class_="articlebody") articlebody = soup.find_all("div", class_="articlebody")
if len(articlebody) > 0: if len(articlebody) > 0:
@ -44,8 +44,8 @@ class WPimport:
else: else:
self._addOrUpdateFeaturedMedia(soup) self._addOrUpdateFeaturedMedia(soup)
else: else:
self._logger.error("Connection error with status code : {0}".format(r.status_code)) self._logger.error("{0} : Connection error for get url {1} with status code : {2}".format(self._name, webpage[i], r.status_code))
self._logger.debug(r.content) self._logger.debug("{0} : {1}".format(self._name, r.content))
def fromDirectory(self, directory): def fromDirectory(self, directory):
@ -58,7 +58,7 @@ class WPimport:
def fromFile(self, files): def fromFile(self, files):
for file in files: for file in files:
if os.path.exists(file): if os.path.exists(file):
self._logger.info("Fichier en cours de traitement : {0}".format(file)) self._logger.info("{0} : File is being processed : {1}".format(self._name, file))
with open(file, 'r') as f: with open(file, 'r') as f:
content = f.read() content = f.read()
soup = BeautifulSoup(content, self._parser) soup = BeautifulSoup(content, self._parser)
@ -70,21 +70,6 @@ class WPimport:
# Private method # Private method
## From files in split by thread
def fromFile(self, files):
for i in range(0, len(files)):
if os.path.exists(file):
self._logger.info("Fichier en cours de traitement : {0}".format(files[i]))
with open(file, 'r') as f:
content = f.read()
soup = BeautifulSoup(content, self._parser)
articlebody = soup.find_all("div", class_="articlebody")
if len(articlebody) > 0:
self._addOrUpdatePost(soup)
else:
self._addOrUpdateFeaturedMedia(soup)
## Get all files ## Get all files
def _getFiles(self, item): def _getFiles(self, item):
@ -115,7 +100,7 @@ class WPimport:
try: try:
page = self._request.get("http://{0}/wp-json/wp/v2/search".format(self._wordpress), auth=self._basic, params=params) page = self._request.get("http://{0}/wp-json/wp/v2/search".format(self._wordpress), auth=self._basic, params=params)
except Exception as err: except Exception as err:
self._logger.error("Connection error : {0}".format(err)) self._logger.error("{0} : Connection error : {1}".format(self._name, err))
exit(1) exit(1)
if page.status_code == 200: if page.status_code == 200:
result = page.json() result = page.json()
@ -127,7 +112,7 @@ class WPimport:
try: try:
page = self._request.get(img_src) page = self._request.get(img_src)
except Exception as err: except Exception as err:
self._logger.error("Connection error : {0}".format(err)) self._logger.error("{0} : Connection error for get featured media : {1}".format(self._name, err))
exit(1) exit(1)
if page.status_code == 200: if page.status_code == 200:
name_img = img_src.replace("_q", "") name_img = img_src.replace("_q", "")
@ -136,7 +121,7 @@ class WPimport:
try: try:
page = self._request.get("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, params=params) page = self._request.get("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, params=params)
except Exception as err: except Exception as err:
self._logger.error("Connection error : {0}".format(err)) self._logger.error("{0} : Connection error search featured media : {1}".format(self._name, err))
exit(1) exit(1)
if page.status_code == 200: if page.status_code == 200:
res = page.json() res = page.json()
@ -146,27 +131,27 @@ class WPimport:
try: try:
r = self._request.post("http://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, result[0]["id"]), auth=self._basic, headers=self._headers_json, data=json.dumps(data)) r = self._request.post("http://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, result[0]["id"]), auth=self._basic, headers=self._headers_json, data=json.dumps(data))
except Exception as err: except Exception as err:
self._logger.error("Connection error : {0}".format(err)) self._logger.error("{0} : Connection error for post media featured : {1}".format(self._name, err))
exit(1) exit(1)
if r.status_code == 200: if r.status_code == 200:
self._logger.info("Ajout media featured : {0}".format(r.json()["title"]["raw"])) self._logger.info("{0} : Add media featured : {1}".format(self._name, r.json()["title"]["raw"]))
else: else:
self._logger.error("Connection error with status code : {0}".format(r.status_code)) self._logger.error("{0} : Connection error with status code for featured media : {1}".format(self._name, r.status_code))
self._logger.debug(r.content) self._logger.debug("{0} : {1}".format(self._name, r.content))
else: else:
self._logger.info("Aucun media trouvé pour {0}".format(h2)) self._logger.info("{0} : No media found for {1}".format(self._name, h2))
else: else:
self._logger.error("Connection error with status code : {0}".format(page.status_code)) self._logger.error("{0} : Connection error with status code for search featured media: {1}".format(self._name, page.status_code))
self._logger.debug(page.content) self._logger.debug("{0} : {1}".format(self._name, page.content))
else: else:
self._logger.error("Connection error with status code : {0}".format(page.status_code)) self._logger.error("{0} : Connection error for get featured media with status code : {1}".format(self._name, page.status_code))
self._logger.debug(page.content) self._logger.debug("{0} : {1}".format(self._name, page.content))
else: else:
self._logger.error("Connection error with status code : {0}".format(page.status_code)) self._logger.error("{0} : Connection error with status code for featured media : {1}".format(self._name, page.status_code))
self._logger.debug(page.content) self._logger.debug("{0} : {1}".format(self._name, page.content))
## Association image to post ## Association image to post
@ -177,13 +162,13 @@ class WPimport:
try: try:
r = self._request.post("http://{0}/wp-json/wp/v2/media/{1}".format(self._wordpress, i["id"]), auth=self._basic, data=data) r = self._request.post("http://{0}/wp-json/wp/v2/media/{1}".format(self._wordpress, i["id"]), auth=self._basic, data=data)
except Exception as err: except Exception as err:
self._logger.error("Connection error : {0}".format(err)) self._logger.error("{0} : Connection error for link image to post : {1}".format(self._name, err))
exit(1) exit(1)
if r.status_code == 200: if r.status_code == 200:
self._logger.info("Association d'une image à l'article {0}".format(title)) self._logger.info("{0} : Link image to post {1}".format(self._name, title))
else: else:
self._logger.error("Connection error with status code : {0}".format(r.status_code)) self._logger.error("{0} Connection error with status code for link image to post : {1}".format(self._name, r.status_code))
self._logger.debug(r.content) self._logger.debug("{0} : {1}".format(self._name, r.content))
## Add or update img ## Add or update img
@ -196,7 +181,7 @@ class WPimport:
try: try:
r = self._request.get("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, params=params) r = self._request.get("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, params=params)
except Exception as err: except Exception as err:
self._logger.error("Connection error : {0}".format(err)) self._logger.error("{0} : Connection error for search media : {1}".format(self._name, err))
exit(1) exit(1)
if r.status_code == 200: if r.status_code == 200:
res = r.json() res = r.json()
@ -205,13 +190,13 @@ class WPimport:
try: try:
r = self._request.delete("http://{0}/wp-json/wp/v2/media/{1}".format(self._wordpress, res[0]["id"]), auth=self._basic, params=params) r = self._request.delete("http://{0}/wp-json/wp/v2/media/{1}".format(self._wordpress, res[0]["id"]), auth=self._basic, params=params)
except Exception as err: except Exception as err:
self._logger.error("Connection error : {0}".format(err)) self._logger.error("{0} Connection error for delete image : {1}".format(self._name, err))
exit(1) exit(1)
if r.status_code == 200: if r.status_code == 200:
self._logger.info("Image removed {0}".format(img_name)) self._logger.info("{0} : Image removed {1}".format(self._name, img_name))
else: else:
self._logger.error("Image not removed due status code : {0}".format(r.status_code)) self._logger.error("{0} : Image not removed due status code : {1}".format(self._name, r.status_code))
self._logger.debug(r.content) self._logger.debug("{0} : {1}".format(self._name, r.content))
data = page.content data = page.content
img_type = "image/png" img_type = "image/png"
@ -221,20 +206,20 @@ class WPimport:
try: try:
r = self._request.post("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, headers=headers, data=data) r = self._request.post("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, headers=headers, data=data)
except Exception as err: except Exception as err:
self._logger.error("Connection error : {0}".format(err)) self._logger.error("{0} : Connection error for add image : {1}".format(self._name, err))
exit(1) exit(1)
if r.status_code == 201: if r.status_code == 201:
self._logger.info("Image added {0}".format(img_name)) self._logger.info("{0} : Image added {1}".format(self._name, img_name))
res = r.json() res = r.json()
media["id"] = res["id"] media["id"] = res["id"]
media["rendered"] = res["guid"]["rendered"] media["rendered"] = res["guid"]["rendered"]
else: else:
self._logger.error("Image not added due status code : {0}".format(r.status_code)) self._logger.error("{0} : Image not added due status code : {1}".format(self._name, r.status_code))
self._logger.debug(r.content) self._logger.debug(r.content)
else: else:
self._logger.error("Connection error with status code : {0}".format(r.status_code)) self._logger.error("{0} : Connection error for search image with status code : {1}".format(self._name, r.status_code))
self._logger.debug(r.content) self._logger.debug("{0} : {1}".format(self._name, r.content))
return media return media
@ -247,7 +232,7 @@ class WPimport:
params = {"post": post, "author_name":i["author"], "date":i["date"]} params = {"post": post, "author_name":i["author"], "date":i["date"]}
page = self._request.get("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, params=params) page = self._request.get("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, params=params)
except Exception as err: except Exception as err:
self._logger.error("Connection error : {0}".format(err)) self._logger.error("{0} : Connection error for search comment : {1}".format(self._name, err))
exit(1) exit(1)
if page.status_code == 200: if page.status_code == 200:
result = page.json() result = page.json()
@ -256,18 +241,18 @@ class WPimport:
params = {"force":1} params = {"force":1}
page = self._request.delete("http://{0}/wp-json/wp/v2/comments/{1}".format(self._wordpress, j["id"]), params=params, auth=self._basic) page = self._request.delete("http://{0}/wp-json/wp/v2/comments/{1}".format(self._wordpress, j["id"]), params=params, auth=self._basic)
except Exception as err: except Exception as err:
self._logger.error("Connection error : {0}".format(err)) self._logger.error("{0} : Connection error for delete comment : {1}".format(self._name, err))
exit(1) exit(1)
if page.status_code == 200: if page.status_code == 200:
self._logger.info("Comment deleted for {0}".format(title)) self._logger.info("{0} : Comment deleted for {1}".format(self._name, title))
self._logger.debug("Comment deleted : {0}".format(j)) self._logger.debug("{0} : Comment deleted : {1}".format(self._name, j))
else: else:
self._logger.error("Comment not deleted for {0} due status code : {1}".format(title, page.status_code)) self._logger.error("{0} : Comment not deleted for {1} due status code : {2}".format(self._name, title, page.status_code))
self._logger.debug(page.content) self._logger.debug("{0} : {1}".format(self._name, page.content))
else: else:
self._logger.error("Comment not listed for {0} due status code : {1}".format(title, page.status_code)) self._logger.error("{0} : Comment not listed for {1} due status code : {2}".format(self._name, title, page.status_code))
self._logger.debug(page.content) self._logger.debug("{0} : {1}".format(self._name, page.content))
for i in comment: for i in comment:
data = {"post": post, "content": i["content"], "date": i["date"], "author_name": i["author"], "status": "approved"} data = {"post": post, "content": i["content"], "date": i["date"], "author_name": i["author"], "status": "approved"}
@ -278,27 +263,27 @@ class WPimport:
try: try:
page = self._request.get("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, params=params) page = self._request.get("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, params=params)
except Exception as err: except Exception as err:
self._logger.error("Connection error : {0}".format(err)) self._logger.error("{0} : Connection error for parent comment : {1}".format(self._name, err))
exit(1) exit(1)
if page.status_code == 200: if page.status_code == 200:
result = page.json() result = page.json()
if len(result) > 0: if len(result) > 0:
data["parent"]=result[0]["id"] data["parent"]=result[0]["id"]
else: else:
self._logger.error("Connection error for parent comment with status code : {0}".format(page.status_code)) self._logger.error("{0} : Connection error for parent comment with status code : {1}".format(self._name, page.status_code))
self._logger.debug(page.content) self._logger.debug("{0} : {1}".format(self._name, page.content))
try: try:
page = self._request.post("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, data=data) page = self._request.post("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, data=data)
except Exception as err: except Exception as err:
self._logger.error("Connection error : {0}".format(err)) self._logger.error("{0} : Connection error for add comment : {1}".format(self._name, err))
exit(1) exit(1)
if page.status_code == 201: if page.status_code == 201:
self._logger.info("Comment added for {0}".format(title)) self._logger.info("{0} : Comment added for {1}".format(self._name, title))
self._logger.debug("Data : {0}".format(data)) self._logger.debug("{0} : Data : {1}".format(self._name, data))
else: else:
self._logger.error("Comment not added for {0} due status code : {1}".format(title, page.status_code)) self._logger.error("{0} : Comment not added for {1} due status code : {2}".format(self._name, title, page.status_code))
self._logger.debug(page.content) self._logger.debug("{0} : {1}".format(self._name, page.content))
## Check class name ## Check class name
@ -381,14 +366,14 @@ class WPimport:
try: try:
page_img = self._request.get(href_img) page_img = self._request.get(href_img)
except Exception as err: except Exception as err:
self._logger.error("Connection error : {0}".format(err)) self._logger.error("{0} : Connection error for get image : {1}".format(self._name, err))
exit(1) exit(1)
if page_img.status_code == 404: if page_img.status_code == 404:
href_img = href_a href_img = href_a
try: try:
page_img = self._request.get(href_a) page_img = self._request.get(href_a)
except Exception as err: except Exception as err:
self._logger.error("Connection error : {0}".format(err)) self._logger.error("{0} : Connection error for get image : {1}".format(self._name, err))
exit(1) exit(1)
if page_img.status_code == 200: if page_img.status_code == 200:
@ -402,8 +387,8 @@ class WPimport:
new_img["new_src"]=media["rendered"] new_img["new_src"]=media["rendered"]
list_img.append(new_img) list_img.append(new_img)
if page_img.status_code not in [200, 404]: if page_img.status_code not in [200, 404]:
self._logger.error("Connection error with status code : {0}".format(page_img.status_code)) self._logger.error("{0} : Connection error with status code for get image : {1}".format(self._name, page_img.status_code))
self._logger.debug(page_img.content) self._logger.debug("{0} : {1}".format(self._name, page_img.content))
comment_post = self._getComment(comment) comment_post = self._getComment(comment)
@ -424,7 +409,7 @@ class WPimport:
params = {"params":j} params = {"params":j}
page = self._request.get("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i), auth=self._basic, params=params) page = self._request.get("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i), auth=self._basic, params=params)
except Exception as err: except Exception as err:
self._logger.error("Connection error : {0}".format(err)) self._logger.error("{0} : Connection error for {1} : {2}".format(self._name, i, err))
exit(1) exit(1)
if page.status_code == 200: if page.status_code == 200:
element_exist = True element_exist = True
@ -432,26 +417,26 @@ class WPimport:
listelement[i].append(result[0]["id"]) listelement[i].append(result[0]["id"])
else: else:
self._logger.error("{0} not found due status code : {1}".format(i, page.status_code)) self._logger.error("{0} : {1} not found due status code : {2}".format(self._name, i, page.status_code))
self._logger.debug(page.content) self._logger.debug("{0} : {1}".format(self._name, page.content))
if element_exist is False: if element_exist is False:
data = {"name": j} data = {"name": j}
self._logger.debug("URL : {0} ".format("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i))) self._logger.debug("{0} : URL : {1} ".format("http://{1}/wp-json/wp/v2/{2}".format(self._name, self._wordpress, i)))
self._logger.debug("data : {0}".format(data)) self._logger.debug("{0} : data : {1}".format(self._name, data))
self._logger.debug("headers : {0}".format(self._headers_form)) self._logger.debug("{0} : headers : {1}".format(self._name, self._headers_form))
try: try:
page = self._request.post("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i), auth=self._basic, headers=self._headers_json, data=data) page = self._request.post("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i), auth=self._basic, headers=self._headers_json, data=data)
except Exception as err: except Exception as err:
self._logger.error("Connection error : {0}".format(err)) self._logger.error("{0} : Connection error for post {1} : {2}".format(self._name, i, err))
exit(1) exit(1)
if page.status_code == 201: if page.status_code == 201:
result = page.json() result = page.json()
listelement[i].append(result["id"]) listelement[i].append(result["id"])
else: else:
self._logger.error("{0} not added due status code : {1}".format(i, page.status_code)) self._logger.error("{0} : {1} not added due status code : {2}".format(self._name, i, page.status_code))
self._logger.debug(page.content) self._logger.debug("{0} : {1}".format(self._name, page.content))
title = articletitle[0].text title = articletitle[0].text
author = articleacreator[0].text.lower() author = articleacreator[0].text.lower()
@ -474,21 +459,21 @@ class WPimport:
try: try:
page = self._request.get("http://{0}/wp-json/wp/v2/users".format(self._wordpress), auth=self._basic, params=params) page = self._request.get("http://{0}/wp-json/wp/v2/users".format(self._wordpress), auth=self._basic, params=params)
except Exception as err: except Exception as err:
self._logger.error("Connection error : {0}".format(err)) self._logger.error("{0} : Connection error for get author : {1}".format(self._name, err))
exit(1) exit(1)
if page.status_code == 200: if page.status_code == 200:
result = page.json() result = page.json()
data["author"] = result[0]["id"] data["author"] = result[0]["id"]
else: else:
self._logger.error("Connection error with status code : {0}".format(page.status_code)) self._logger.error("{0} : Connection error with status code for get author : {1}".format(self._name, page.status_code))
self._logger.debug(page.content) self._logger.debug("{0} : {1}".format(page.content))
params = {"search":title} params = {"search":title}
try: try:
page = self._request.get("http://{0}/wp-json/wp/v2/posts".format(self._wordpress), auth=self._basic, params=params) page = self._request.get("http://{0}/wp-json/wp/v2/posts".format(self._wordpress), auth=self._basic, params=params)
except Exception as err: except Exception as err:
self._logger.error("Connection error : {0}".format(err)) self._logger.error("{0} : Connection error for search post : {1}".format(self._name, err))
exit(1) exit(1)
page_exist = True page_exist = True
headers = {'Content-Type': 'application/json', 'Accept':'application/json'} headers = {'Content-Type': 'application/json', 'Accept':'application/json'}
@ -497,38 +482,38 @@ class WPimport:
if len(result) == 0: if len(result) == 0:
page_exist = False page_exist = False
else: else:
self._logger.info("La page {0} existe deja et mis à jour".format(title)) self._logger.info("{0} : Page {1} already exist and going to update".format(self._name, title))
post_id = result[0]["id"] post_id = result[0]["id"]
try: try:
page = self._request.post("http://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, post_id), auth=self._basic, headers=headers, data=json.dumps(data)) page = self._request.post("http://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, post_id), auth=self._basic, headers=headers, data=json.dumps(data))
except Exception as err: except Exception as err:
self._logger.error("Connection error : {0}".format(err)) self._logger.error("{0} : Connection error for update post : {1}".format(self._name, err))
exit(1) exit(1)
if page.status_code == 200: if page.status_code == 200:
result = page.json() result = page.json()
self._logger.info("Article mis à jour : {0}".format(result["title"]["raw"])) self._logger.info("{0} : Post updated : {1}".format(self._name, result["title"]["raw"]))
self._addOrUpdateComment(result["id"], comment_post, result["title"]["raw"]) self._addOrUpdateComment(result["id"], comment_post, result["title"]["raw"])
self._linkImgPost(result["title"]["raw"], list_img, result["id"]) self._linkImgPost(result["title"]["raw"], list_img, result["id"])
else: else:
self._logger.error("Post not updated due status code : {0}".format(page.status_code)) self._logger.error("{0} : Post not updated due status code : {1}".format(self._name, page.status_code))
self._logger.debug(page.content) self._logger.debug("{0} : {1}".format(self._name, page.content))
else: else:
self._logger.error("Connection for update post error with status code : {0}".format(page.status_code)) self._logger.error("{0} : Connection for update post error with status code : {1}".format(self._name, page.status_code))
self._logger.debug(page.content) self._logger.debug("{0} : {1}".format(self._name, page.content))
if page_exist == False: if page_exist == False:
try: try:
page = self._request.post("http://{0}/wp-json/wp/v2/posts".format(self._wordpress), auth=self._basic, headers=headers, data=json.dumps(data)) page = self._request.post("http://{0}/wp-json/wp/v2/posts".format(self._wordpress), auth=self._basic, headers=headers, data=json.dumps(data))
except Exception as err: except Exception as err:
self._logger.error("Connection error : {0}".format(err)) self._logger.error("{0} : Connection error for create post : {1}".format(self._name, err))
exit(1) exit(1)
if page.status_code == 201: if page.status_code == 201:
result = page.json() result = page.json()
self._logger.info("Article ajoute : {0}".format(result["title"]["raw"])) self._logger.info("{0} : Post added : {1}".format(self._name, result["title"]["raw"]))
self._addOrUpdateComment(result["id"], comment_post, result["title"]["raw"]) self._addOrUpdateComment(result["id"], comment_post, result["title"]["raw"])
self._linkImgPost(result["title"]["raw"], list_img, result["id"]) self._linkImgPost(result["title"]["raw"], list_img, result["id"])
else: else:
self._logger.error("Post not added due status code : {0}".format(r.status_code)) self._logger.error("{0} : Post not added due status code : {1}".format(self._name, r.status_code))
self._logger.debug(r.content) self._logger.debug("{0} : {1}".format(self._name, r.content))