v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

add link

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-07-18 22:09:20 +02:00
parent d5ddf1601b
commit c6894648d1
1 changed files with 15 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
lib/WPMenu.py
View File

@ -83,7 +83,7 @@ class WPMenu:
return title_rendered return title_rendered
def _getIdfromTitlePost(self, content): def _getIdfromTitlePost(self, content):
idMenu = {"id":0, "type":""} idMenu = {"id":0, "type":"", "link":""}
soup = BeautifulSoup(content, self._parser) soup = BeautifulSoup(content, self._parser)
articletitle = soup.find("h2", class_="articletitle").get_text() articletitle = soup.find("h2", class_="articletitle").get_text()
exist = False exist = False
@ -104,7 +104,7 @@ class WPMenu:
self._logger.debug("{0} : comparaison debug {1} {2}".format(self._name, articletitle, title_rendered)) self._logger.debug("{0} : comparaison debug {1} {2}".format(self._name, articletitle, title_rendered))
if articletitle == title_rendered: if articletitle == title_rendered:
self._logger.debug("{0} : get post id : {1}".format(self._name, i)) self._logger.debug("{0} : get post id : {1}".format(self._name, i))
idMenu = {"id":i["id"], "type":"post"} idMenu = {"id":i["id"], "type":"post", "link": i["link"]}
exist = True exist = True
else: else:
self._logger.debug("{0} : {2} {1}".format(self._name, result, len(result))) self._logger.debug("{0} : {2} {1}".format(self._name, result, len(result)))
@ -123,7 +123,7 @@ class WPMenu:
return idMenu return idMenu
def _getIdFromPost(self, href): def _getIdFromPost(self, href):
idMenu = {"id":0, "type":""} idMenu = {"id":0, "type":"", "link":""}
o = urlparse(href) o = urlparse(href)
if len(o.netloc) > 0: if len(o.netloc) > 0:
try: try:
@ -151,7 +151,7 @@ class WPMenu:
def _getIdFromReverse(self, title, href): def _getIdFromReverse(self, title, href):
self._logger.info("{0} : get title {1} from href {2}".format(self._name, title, href)) self._logger.info("{0} : get title {1} from href {2}".format(self._name, title, href))
idMenu = {"id":0, "type":""} idMenu = {"id":0, "type":"", "link":""}
if href != "#": if href != "#":
title = href[::-1] title = href[::-1]
link = title.split("/")[0] link = title.split("/")[0]
@ -167,7 +167,7 @@ class WPMenu:
return idMenu return idMenu
def _getId(self, title): def _getId(self, title):
idMenu = {"id": 0, "type":""} idMenu = {"id": 0, "type":"", "link":""}
exist = False exist = False
if exist is False: if exist is False:
for i in ["categories", "tags"]: for i in ["categories", "tags"]:
@ -183,7 +183,7 @@ class WPMenu:
self._logger.info("{0} info : {1} {2} {3}".format(self._name, j["name"], j["slug"], title)) self._logger.info("{0} info : {1} {2} {3}".format(self._name, j["name"], j["slug"], title))
if j["name"] == title or j["slug"] == title: if j["name"] == title or j["slug"] == title:
self._logger.info("{0} : comparaison ok : {1} {2}".format(self._name, j["id"], i)) self._logger.info("{0} : comparaison ok : {1} {2}".format(self._name, j["id"], i))
idMenu = {"id": j["id"], "type": i} idMenu = {"id": j["id"], "type": i, "link": j["link"]}
exist = True exist = True
else: else:
break break
@ -211,14 +211,14 @@ class WPMenu:
href = anchor.find("a").get("href") href = anchor.find("a").get("href")
if href == "{0}://{1}/".format(self._protocol_canalblog, self._canalblog): if href == "{0}://{1}/".format(self._protocol_canalblog, self._canalblog):
parent = "home" parent = "home"
itemMenu = {"id":"", "type":"", "title": parent, "href":href, "children":list()} itemMenu = {"id":"", "type":"", "title": parent, "link":"", "href":href, "children":list()}
if href == "#": if href == "#":
li = anchor.find("ul").find_all("li") li = anchor.find("ul").find_all("li")
for child in li: for child in li:
a = child.find("a") a = child.find("a")
href = a.get("href") href = a.get("href")
self._logger.info("{0} Parent {1} : Child {2}".format(self._name, parent, a.get_text())) self._logger.info("{0} Parent {1} : Child {2}".format(self._name, parent, a.get_text()))
children.append({"title": a.get_text(), "parent": parent, "href":href}) children.append({"title": a.get_text(), "parent": parent, "href":href, "link":""})
menu.append(itemMenu) menu.append(itemMenu)
@ -233,7 +233,7 @@ class WPMenu:
for j in range(0, len(menu)): for j in range(0, len(menu)):
self._logger.info("{0} : Children for : {1}".format(self._name, menu[j]["title"])) self._logger.info("{0} : Children for : {1}".format(self._name, menu[j]["title"]))
if menu[j]["title"] == children[i]["parent"]: if menu[j]["title"] == children[i]["parent"]:
menu[j]["children"].append({"id":"", "type":"", "title":children[i]["title"], "parent": children[i]["parent"], "href":children[i]["href"]}) menu[j]["children"].append({"id":"", "type":"", "title":children[i]["title"], "parent": children[i]["parent"], "link":"", "href":children[i]["href"]})
for i in range(0, len(menu)): for i in range(0, len(menu)):
self._logger.info("{0} : Menu {1} {2}".format(self._name, menu[i]["title"], len(menu[i]["children"]))) self._logger.info("{0} : Menu {1} {2}".format(self._name, menu[i]["title"], len(menu[i]["children"])))
@ -244,16 +244,18 @@ class WPMenu:
self._logger.debug("{0} : content children {1}".format(self._name, menu[i]["children"][j])) self._logger.debug("{0} : content children {1}".format(self._name, menu[i]["children"][j]))
idMenu = self._getIdFromReverse(menu[i]["children"][j]["title"], menu[i]["children"][j]["href"]) idMenu = self._getIdFromReverse(menu[i]["children"][j]["title"], menu[i]["children"][j]["href"])
if idMenu["id"] != 0: if idMenu["id"] != 0:
menu[i]["children"][j] = {"id":idMenu["id"], "type": idMenu["type"], "title": menu[i]["children"][j]["title"], "parent": menu[i]["children"][j]["parent"]} menu[i]["children"][j] = {"id":idMenu["id"], "type": idMenu["type"], "link": idMenu["link"], "title": menu[i]["children"][j]["title"], "parent": menu[i]["children"][j]["parent"]}
idMenu = self._getId(menu[i]["title"]) idMenu = self._getId(menu[i]["title"])
self._logger.debug("{0} : content parent {1}".format(self._name, menu[i])) self._logger.debug("{0} : content parent {1}".format(self._name, menu[i]))
self._logger.debug("{0} : content idMenu {1}".format(self._name, idMenu))
if idMenu["id"] == 0: if idMenu["id"] == 0:
idMenu = self._getIdFromReverse(menu[i]["title"], menu[i]["href"]) idMenu = self._getIdFromReverse(menu[i]["title"], menu[i]["href"])
if idMenu["id"] != 0: if idMenu["id"] != 0:
menu[i] = {"id":idMenu["id"], "type": idMenu["type"], "title":menu[i]["title"], "children": menu[i]["children"]} menu[i] = {"id":idMenu["id"], "type": idMenu["type"], "title":menu[i]["title"], "link":idMenu["link"], "children": menu[i]["children"]}
for i in menu: for i in menu:
self._logger.info("{0} Menu parent {1} with id {2} and type {3}".format(self._name, i["title"], i["id"], i["type"])) self._logger.info("{0} Menu parent {1} with id {2} and type {3} and link {4}".format(self._name, i["title"], i["id"], i["type"], i["link"]))
for j in i["children"]: for j in i["children"]:
self._logger.info("{0} Menu children {1} with id {2} and type {3}".format(self._name, j["title"], j["id"], j["type"])) self._logger.info("{0} Menu children {1} with id {2} and type {3} and link {4}".format(self._name, j["title"], j["id"], j["type"], j["link"]))