v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

Merge pull request 'parent-comment' (#8) from parent-comment into master

Browse Source 
Reviewed-on: #8
This commit is contained in:
v4l3n71n 2023-04-20 19:30:45 +00:00
commit 5a5658d955
2 changed files with 207 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
lib/WPExport.py
View File

@ -69,6 +69,10 @@ class WPExport:
href = anchor.get('href', '/')
if href != "#":
page_url.append(href)
else:
self._logger.error("Url did not get due status code : {0}".format(page.status_code))
self._logger.debug(page.content)
webpage = []
for i in page_url:
@ -113,6 +117,10 @@ class WPExport:
self._logger.error("parsing error : {0}".format(err))
exit(1)
webpage.append(o)
else:
self._logger.error("web didn't get due status code : {0}".format(page.status_code))
self._logger.debug(page.content)
return webpage
@ -179,6 +187,10 @@ class WPExport:
o = o._replace(netloc=u.netloc)
o = o._replace(scheme=u.scheme)
page_url.append(o.geturl())
else:
self._logger.error("JS or CSS did not get due status code : {0}".format(page.status_code))
self._logger.debug(page.content)
return page_url
# Get image
@ -201,6 +213,10 @@ class WPExport:
if src not in page_img:
self._logger.info("image: {0} : ".format(src))
page_img.append(src)
else:
self._logger.error("Image did not get due status code : {0}".format(page.status_code))
self._logger.debug(page.content)
return page_img
@ -233,3 +249,6 @@ class WPExport:
except Exception as err:
self._logger.error("file error : {0}".format(err))
exit(1)
else:
self._logger.error("Not download due status code : {0}".format(r.status_code))
self._logger.debug(r.content)

221
lib/WPImport.py
View File

@ -13,6 +13,7 @@ class WPimport:
self._wordpress = wordpress
self._logger = logger
self._parser = parser
self._headers_json = {'Content-Type': 'application/json', 'Accept':'application/json'}
self._request = requests.Session()
@ -41,6 +42,9 @@ class WPimport:
self._addOrUpdatePost(soup)
else:
self._addOrUpdateFeaturedMedia(soup)
else:
self._logger.error("Connection error with status code : {0}".format(r.status_code))
self._logger.debug(r.content)
def fromDirectory(self, directory):
@ -122,17 +126,32 @@ class WPimport:
res = page.json()
if len(res) > 0:
id_media = res[0]["id"]
headers = {'Content-Type': 'application/json', 'Accept':'application/json'}
data = {"featured_media": id_media}
try:
r = self._request.post("http://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, result[0]["id"]), auth=self._basic, headers=headers, data=json.dumps(data))
r = self._request.post("http://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, result[0]["id"]), auth=self._basic, headers=self._headers_json, data=json.dumps(data))
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if r.status_code == 200:
self._logger.info("Ajout media featured : {0}".format(r.json()["title"]["raw"]))
else:
self._logger.error("Connection error with status code : {0}".format(r.status_code))
self._logger.debug(r.content)
else:
self._logger.info("Aucun media trouvé pour {0}".format(h2))
else:
self._logger.error("Connection error with status code : {0}".format(page.status_code))
self._logger.debug(page.content)
else:
self._logger.error("Connection error with status code : {0}".format(page.status_code))
self._logger.debug(page.content)
else:
self._logger.error("Connection error with status code : {0}".format(page.status_code))
self._logger.debug(page.content)
## Association image to post
@ -146,6 +165,10 @@ class WPimport:
exit(1)
if r.status_code == 200:
self._logger.info("Association d'une image à l'article {0}".format(title))
else:
self._logger.error("Connection error with status code : {0}".format(r.status_code))
self._logger.debug(r.content)
## Add or update img
@ -169,7 +192,11 @@ class WPimport:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if r.status_code == 200:
self._logger.info("Image supprimé {0}".format(img_name))
self._logger.info("Image removed {0}".format(img_name))
else:
self._logger.error("Image not removed due status code : {0}".format(r.status_code))
self._logger.debug(r.content)
data = page.content
img_type = "image/png"
if img_name.split(".")[1] == "jpg" or img_name.split(".")[1] == "jpeg":
@ -181,17 +208,57 @@ class WPimport:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if r.status_code == 201:
self._logger.info("Ajout d'image {0}".format(img_name))
self._logger.info("Image added {0}".format(img_name))
res = r.json()
media["id"] = res["id"]
media["rendered"] = res["guid"]["rendered"]
else:
self._logger.error("Image not added due status code : {0}".format(r.status_code))
self._logger.debug(r.content)
else:
self._logger.error("Connection error with status code : {0}".format(r.status_code))
self._logger.debug(r.content)
return media
## Add or update comment
def _addOrUpdateComment(self, post, comment, title):
params = {"post": post}
block = True
for i in comment:
try:
params = {"post": post, "author_name":i["author"], "date":i["date"]}
page = self._request.get("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, params=params)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if page.status_code == 200:
result = page.json()
for j in result:
try:
params = {"force":1}
page = self._request.delete("http://{0}/wp-json/wp/v2/comments/{1}".format(self._wordpress, j["id"]), params=params, auth=self._basic)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if page.status_code == 200:
self._logger.info("Comment deleted for {0}".format(title))
self._logger.debug("Comment deleted : {0}".format(j))
else:
self._logger.error("Comment not deleted for {0} due status code : {1}".format(title, page.status_code))
self._logger.debug(page.content)
else:
self._logger.error("Comment not listed for {0} due status code : {1}".format(title, page.status_code))
self._logger.debug(page.content)
for i in comment:
data = {"post": post, "content": i["content"], "date": i["date"], "author_name": i["author"], "status": "approved"}
if i["parent_id"] != -1:
parent_id = int(i["parent_id"])
params = {"post": post, "author_name":comment[parent_id]["author"], "date":comment[parent_id]["date"]}
try:
page = self._request.get("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, params=params)
except Exception as err:
@ -199,30 +266,71 @@ class WPimport:
exit(1)
if page.status_code == 200:
result = page.json()
for i in comment:
comment_exist = False
for j in result:
if i["author"] == j["author_name"] and i["date"] == j["date"]:
comment_exist = True
id_comment = j["id"]
data = {"post": post, "content": i["content"], "date": i["date"], "author_name": i["author"]}
if comment_exist is True:
try:
page = page = self._request.post("http://{0}/wp-json/wp/v2/comments/{1}".format(self._wordpress, id_comment), auth=self._basic, data=data)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if page.status_code == 200:
self._logger.info("Commentaire mise à jour pour {0}".format(title))
if len(result) > 0:
data["parent"]=result[0]["id"]
else:
self._logger.error("Connection error for parent comment with status code : {0}".format(page.status_code))
self._logger.debug(page.content)
try:
page = self._request.post("http://{0}/wp-json/wp/v2/comments".format(self._wordpress), auth=self._basic, data=data)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if page.status_code == 201:
self._logger.info("Commentaire ajoute pour {0}".format(title))
self._logger.info("Comment added for {0}".format(title))
self._logger.debug("Data : {0}".format(data))
else:
self._logger.error("Comment not added for {0} due status code : {1}".format(title, page.status_code))
self._logger.debug(page.content)
## Check class name
def _hasClassName(self, tag, className):
for i in tag["class"]:
if i == className:
return True
return False
## Get class name
def _getClassName(self, tag, className):
for i in tag["class"]:
if re.match(className, i):
return i
return ""
## Get all comments
def _getComment(self, comment):
comment_post = []
for i in range(0, len(comment)):
comment_div = comment[i].find("div", class_="comment_item")
comment_item = comment_div.text.split("\n")
footer = comment_div.find_all("div", class_="itemfooter")
comment_author = footer[0].text.split(",")[0].replace("Posté par ", "")
comment_date = footer[0].find_all("abbr")[0].get("title")
comment_content = "<p>"
for j in range(0, len(comment_item)-2):
if len(comment_item[j]) > 0:
comment_content = comment_content + comment_item[j] + "<br />"
comment_content = comment_content + "</p>"
parent = -1
if self._hasClassName(comment[i], "level-1") is False:
block = False
className = self._getClassName(comment[i], "level-").split("-")
level = 1
if len(className) > 0:
level = int(className[1])
for j in range(i-1, 0, -1):
if block is False:
levelName = "level-{0}".format(level - 1)
if self._hasClassName(comment[j], levelName) is True:
parent = j
block = True
comment_post.append({"author": comment_author, "date": comment_date, "content": comment_content, "parent_id":parent})
return comment_post
## Add or Update post
def _addOrUpdatePost(self, soup):
@ -234,13 +342,6 @@ class WPimport:
listelement = {}
for i in liste:
try:
page = self._request.get("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress,i))
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if page.status_code == 200:
elements[i] = page.json()
element[i] = []
listelement[i] = []
@ -250,7 +351,7 @@ class WPimport:
articleacreator = soup.find_all("span", class_="articlecreator")
dateheader = soup.find_all("div", class_="dateheader")
itemfooter = soup.find_all("div", class_="itemfooter")
comment = soup.find_all("div", class_="comment_item")
comment = soup.find_all("li", class_="comment")
img_a = articlebody[0].find_all("a", {"target": "_blank"})
list_img = []
for i in img_a:
@ -273,6 +374,7 @@ class WPimport:
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if page_img.status_code == 200:
media=self._addOrUpdateMedia(href_img, page_img)
new_img["id"]=media["id"]
@ -283,19 +385,13 @@ class WPimport:
new_img["id"]=media["id"]
new_img["new_src"]=media["rendered"]
list_img.append(new_img)
if page_img.status_code not in [200, 404]:
self._logger.error("Connection error with status code : {0}".format(page_img.status_code))
self._logger.debug(page_img.content)
comment_post = self._getComment(comment)
comment_post = []
for i in comment:
comment_item = i.text.split("\n")
footer = i.find_all("div", class_="itemfooter")
comment_author = footer[0].text.split(",")[0].replace("Posté par ", "")
comment_date = footer[0].find_all("abbr")[0].get("title")
comment_content = "<p>"
for j in range(0, len(comment_item)-2):
if len(comment_item[j]) > 0:
comment_content = comment_content + comment_item[j] + "<br />"
comment_content = comment_content + "</p>"
comment_post.append({"author": comment_author, "date": comment_date, "content": comment_content})
a = itemfooter[0].find_all("a", {"rel": True})
for i in a:
rel = i.get("rel")
@ -308,20 +404,38 @@ class WPimport:
for i in liste:
for j in element[i]:
element_exist = False
for k in elements[i]:
if k["name"] == j:
try:
params = {"params":j}
page = self._request.get("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i), auth=self._basic, params=params)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if page.status_code == 200:
element_exist = True
listelement[i].append(k["id"])
result = page.json()
listelement[i].append(result[0]["id"])
else:
self._logger.error("{0} not found due status code : {1}".format(i, page.status_code))
self._logger.debug(page.content)
if element_exist is False:
data = {"name": j}
self._logger.debug("URL : {0} ".format("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i)))
self._logger.debug("data : {0}".format(data))
self._logger.debug("headers : {0}".format(self._headers_form))
try:
page = self._request.post("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i), auth=self._basic, data=data)
page = self._request.post("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i), auth=self._basic, headers=self._headers_json, data=data)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if page.status_code == 201:
result = page.json()
listelement[i].append(result["id"])
else:
self._logger.error("{0} not added due status code : {1}".format(i, page.status_code))
self._logger.debug(page.content)
title = articletitle[0].text
author = articleacreator[0].text.lower()
@ -349,6 +463,10 @@ class WPimport:
if page.status_code == 200:
result = page.json()
data["author"] = result[0]["id"]
else:
self._logger.error("Connection error with status code : {0}".format(page.status_code))
self._logger.debug(page.content)
params = {"search":title}
try:
@ -375,6 +493,14 @@ class WPimport:
self._logger.info("Article mis à jour : {0}".format(result["title"]["raw"]))
self._addOrUpdateComment(result["id"], comment_post, result["title"]["raw"])
self._linkImgPost(result["title"]["raw"], list_img, result["id"])
else:
self._logger.error("Post not updated due status code : {0}".format(page.status_code))
self._logger.debug(page.content)
else:
self._logger.error("Connection for update post error with status code : {0}".format(page.status_code))
self._logger.debug(page.content)
if page_exist == False:
try:
@ -387,3 +513,6 @@ class WPimport:
self._logger.info("Article ajoute : {0}".format(result["title"]["raw"]))
self._addOrUpdateComment(result["id"], comment_post, result["title"]["raw"])
self._linkImgPost(result["title"]["raw"], list_img, result["id"])
else:
self._logger.error("Post not added due status code : {0}".format(r.status_code))
self._logger.debug(r.content)