v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

add post fine

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-07-17 23:20:47 +02:00
parent aa8ac9cfcb
commit d5ddf1601b
1 changed files with 57 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
lib/WPMenu.py
View File

@ -83,48 +83,56 @@ class WPMenu:
return title_rendered return title_rendered
def _getIdfromTitlePost(self, content): def _getIdfromTitlePost(self, content):
idMenu = 0 idMenu = {"id":0, "type":""}
soup = BeautifulSoup(content, self._parser) soup = BeautifulSoup(content, self._parser)
articletitle = soup.find("h2", class_="articletitle") articletitle = soup.find("h2", class_="articletitle").get_text()
for index in range(0,10): exist = False
params = {"search":articletitle, "per_page":"100", "page":index} for index in range(1,10):
if exist is False:
params = {"search":articletitle, "per_page":100, "page":index}
try: try:
page = self._request_wordpress.get("{2}://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i, self._protocol_wordpress), auth=self._basic, params=params) self._logger.debug("{0} : Get Url for post : {1} {2}".format(self._name, "{1}://{0}/wp-json/wp/v2/posts".format(self._wordpress, self._protocol_wordpress), params))
page = self._request_wordpress.get("{1}://{0}/wp-json/wp/v2/posts".format(self._wordpress, self._protocol_wordpress), auth=self._basic, params=params)
if page.status_code == 200: if page.status_code == 200:
result = page.json() result = page.json()
self._logger.info("{0} : Get content post : {1}".format(self._name, len(result))) self._logger.info("{0} : Get content post : {1}".format(self._name, len(result)))
if len(result) > 0:
for i in result: for i in result:
title_rendered = i["name"] title_rendered = i["title"]["rendered"]
if len(articletitle) != len(title_rendered): if len(articletitle) != len(title_rendered):
title_rendered = self._replaceCaracter(title_rendered) title_rendered = self._replaceCaracter(title_rendered)
self._logger.debug("{0} : comparaison debug {1} {2}".format(self._name, articletitle, title_rendered))
if articletitle == title_rendered: if articletitle == title_rendered:
idMenu = i["id"] self._logger.debug("{0} : get post id : {1}".format(self._name, i))
idMenu = {"id":i["id"], "type":"post"}
exist = True
else:
self._logger.debug("{0} : {2} {1}".format(self._name, result, len(result)))
break
elif page.status_code == 400: elif page.status_code == 400:
self._logger.debug("{0} : {2} {1}".format(self._name, page.content, page.status_code))
break break
else: else:
self._logger.error("{0} : Post didn't get due status code : {1}".format(self._name, page.status_code)) self._logger.error("{0} : Post didn't get due status code : {1}".format(self._name, page.status_code))
self._logger.debug("{0} : {1}".format(self._name, page.content)) self._logger.debug("{0} : {1}".format(self._name, page.content))
except ConnectionError as err: except ConnectionError as err:
self._logger.error("{0} : Connection error for get url {1} : {2}".format(self._name, href, err)) self._logger.error("{0} : Connection error for get content : {1}".format(self._name, err))
exit(1) exit(1)
except Exception as err: except Exception as err:
self._logger.error("{0} : Exception error for get url {1} : {2}".format(self._name, href, err)) self._logger.error("{0} : Exception error for get content : {1} ".format(self._name, err))
return idMenu return idMenu
def _getIdFromPost(self, href): def _getIdFromPost(self, href):
idMenu = 0 idMenu = {"id":0, "type":""}
o = urlparse(href) o = urlparse(href)
if len(o.netloc) > 0: if len(o.netloc) > 0:
try: try:
page = self._request_canalblog.get(href) page = self._request_canalblog.get(href)
if page.status_code == 200: if page.status_code == 200:
self._logger.info("{0} : Get content : {1}".format(self._name, href)) self._logger.info("{0} : Get content : {1}".format(self._name, href))
self._getIdfromTitlePost(page.content) idMenu = self._getIdfromTitlePost(page.content)
else: else:
self._logger.error("{0} : {2} didn't get due status code : {1}".format(self._name, page.status_code, i)) self._logger.error("{0} : {2} didn't get due status code : {1}".format(self._name, page.status_code, href))
self._logger.debug("{0} : {1}".format(self._name, page.content)) self._logger.debug("{0} : {1}".format(self._name, page.content))
except ConnectionError as err: except ConnectionError as err:
self._logger.error("{0} : Connection error for get url {1} : {2}".format(self._name, href, err)) self._logger.error("{0} : Connection error for get url {1} : {2}".format(self._name, href, err))
@ -135,7 +143,7 @@ class WPMenu:
if os.path.exists("{0}/..{1}".format(self._directory, o.path)): if os.path.exists("{0}/..{1}".format(self._directory, o.path)):
try: try:
content = open("{0}/..{1}".format(self._directory, o.path), "r") content = open("{0}/..{1}".format(self._directory, o.path), "r")
self._getIdfromTitlePost(content) idMenu = self._getIdfromTitlePost(content)
except Exception as err: except Exception as err:
self._logger.error("{0} : Exception error for get file content {1} : {2}".format(self._name, href, err)) self._logger.error("{0} : Exception error for get file content {1} : {2}".format(self._name, href, err))
return idMenu return idMenu
@ -143,7 +151,7 @@ class WPMenu:
def _getIdFromReverse(self, title, href): def _getIdFromReverse(self, title, href):
self._logger.info("{0} : get title {1} from href {2}".format(self._name, title, href)) self._logger.info("{0} : get title {1} from href {2}".format(self._name, title, href))
idMenu = 0 idMenu = {"id":0, "type":""}
if href != "#": if href != "#":
title = href[::-1] title = href[::-1]
link = title.split("/")[0] link = title.split("/")[0]
@ -159,11 +167,11 @@ class WPMenu:
return idMenu return idMenu
def _getId(self, title): def _getId(self, title):
idMenu = 0 idMenu = {"id": 0, "type":""}
exist = False exist = False
if exist is False:
for i in ["categories", "tags"]: for i in ["categories", "tags"]:
for index in range(1,10): for index in range(1,10):
if exist is False:
try: try:
params = {"search":title, "per_page":"100", "page":index} params = {"search":title, "per_page":"100", "page":index}
self._logger.info("{0} Get menu {1} {2} {3}".format(self._name, "{2}://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i, self._protocol_wordpress), index, title)) self._logger.info("{0} Get menu {1} {2} {3}".format(self._name, "{2}://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i, self._protocol_wordpress), index, title))
@ -175,8 +183,10 @@ class WPMenu:
self._logger.info("{0} info : {1} {2} {3}".format(self._name, j["name"], j["slug"], title)) self._logger.info("{0} info : {1} {2} {3}".format(self._name, j["name"], j["slug"], title))
if j["name"] == title or j["slug"] == title: if j["name"] == title or j["slug"] == title:
self._logger.info("{0} : comparaison ok : {1} {2}".format(self._name, j["id"], i)) self._logger.info("{0} : comparaison ok : {1} {2}".format(self._name, j["id"], i))
idMenu = j["id"] idMenu = {"id": j["id"], "type": i}
exist = True exist = True
else:
break
elif page.status_code == 400: elif page.status_code == 400:
break break
else: else:
@ -223,27 +233,27 @@ class WPMenu:
for j in range(0, len(menu)): for j in range(0, len(menu)):
self._logger.info("{0} : Children for : {1}".format(self._name, menu[j]["title"])) self._logger.info("{0} : Children for : {1}".format(self._name, menu[j]["title"]))
if menu[j]["title"] == children[i]["parent"]: if menu[j]["title"] == children[i]["parent"]:
menu[j]["children"].append({"id":"", "title":children[i]["title"], "parent": children[i]["parent"], "href":children[i]["href"]}) menu[j]["children"].append({"id":"", "type":"", "title":children[i]["title"], "parent": children[i]["parent"], "href":children[i]["href"]})
for i in range(0, len(menu)): for i in range(0, len(menu)):
self._logger.info("{0} : Menu {1} {2}".format(self._name, menu[i]["title"], len(menu[i]["children"]))) self._logger.info("{0} : Menu {1} {2}".format(self._name, menu[i]["title"], len(menu[i]["children"])))
if menu[i]["title"] != "home": if menu[i]["title"] != "home":
for j in range(0, len(menu[i]["children"])): for j in range(0, len(menu[i]["children"])):
idMenu = self._getId(menu[i]["children"][j]["title"]) idMenu = self._getId(menu[i]["children"][j]["title"])
if idMenu == 0: if idMenu["id"] == 0:
self._logger.debug("{0} : content children {1}".format(self._name, menu[i]["children"][j])) self._logger.debug("{0} : content children {1}".format(self._name, menu[i]["children"][j]))
idMenu = self._getIdFromReverse(menu[i]["children"][j]["title"], menu[i]["children"][j]["href"]) idMenu = self._getIdFromReverse(menu[i]["children"][j]["title"], menu[i]["children"][j]["href"])
if idMenu != 0: if idMenu["id"] != 0:
menu[i]["children"][j] = {"id":idMenu, "title": menu[i]["children"][j]["title"], "parent": menu[i]["children"][j]["parent"]} menu[i]["children"][j] = {"id":idMenu["id"], "type": idMenu["type"], "title": menu[i]["children"][j]["title"], "parent": menu[i]["children"][j]["parent"]}
idMenu = self._getId(menu[i]["title"]) idMenu = self._getId(menu[i]["title"])
self._logger.debug("{0} : content parent {1}".format(self._name, menu[i])) self._logger.debug("{0} : content parent {1}".format(self._name, menu[i]))
if idMenu == 0: if idMenu["id"] == 0:
idMenu = self._getIdFromReverse(menu[i]["title"], menu[i]["href"]) idMenu = self._getIdFromReverse(menu[i]["title"], menu[i]["href"])
if idMenu != 0: if idMenu["id"] != 0:
menu[i] = {"id":idMenu, "title":menu[i]["title"], "children": menu[i]["children"]} menu[i] = {"id":idMenu["id"], "type": idMenu["type"], "title":menu[i]["title"], "children": menu[i]["children"]}
for i in menu: for i in menu:
self._logger.info("{0} Menu parent {1} with id : {2}".format(self._name, i["title"], i["id"])) self._logger.info("{0} Menu parent {1} with id {2} and type {3}".format(self._name, i["title"], i["id"], i["type"]))
for j in i["children"]: for j in i["children"]:
self._logger.info("{0} Menu children {1} with id : {2}".format(self._name, j["title"], j["id"])) self._logger.info("{0} Menu children {1} with id {2} and type {3}".format(self._name, j["title"], j["id"], j["type"]))