v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

working menu

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-07-06 00:42:47 +02:00
parent 95f5203727
commit c6ccf98b1b
1 changed files with 38 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
lib/WPMenu.py
View File

@ -79,26 +79,42 @@ class WPMenu:
soup = BeautifulSoup(content, self._parser) soup = BeautifulSoup(content, self._parser)
ul = soup.find_all("ul", id="listsmooth") ul = soup.find_all("ul", id="listsmooth")
menu = list() menu = list()
parents = list()
for anchor in ul[0].find_all("li"): for anchor in ul[0].find_all("li"):
li = anchor.find_all("li") parent = anchor.find("a").get_text()
for child in li: href = anchor.find("a").get("href")
a = child.find("a") if href == "#":
self._logger.info("{0} {1} : {2}".format(self._name, anchor.find("a").get_text(), a.get_text())) li = anchor.find_all("li")
for i in ["categories", "tags"]: for child in li:
try: a = child.find("a")
params = {"search":a.get_text(), "per_page":"100"} self._logger.info("{0} {1} : {2}".format(self._name, anchor.find("a").get_text(), a.get_text()))
page = self._request_wordpress.get("{2}://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i, self._protocol_wordpress), auth=self._basic, params=params) for i in ["categories", "tags"]:
if page.status_code == 200: try:
result = page.json() params = {"search":a.get_text(), "per_page":"100"}
if len(result) > 0: page = self._request_wordpress.get("{2}://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i, self._protocol_wordpress), auth=self._basic, params=params)
menu.append({"id":result[0]["id"], "type":i, "title": a.get_text(), "parent":anchor.find("a").get_text()}) if page.status_code == 200:
else: result = page.json()
self._logger.error("{0} : {2} didn't get due status code : {1}".format(self._name, page.status_code, i)) if len(result) > 0:
self._logger.debug("{0} : {1}".format(self._name, page.content)) menu.append({"id":result[0]["id"], "type":i, "title": a.get_text(), "parent":parent})
except ConnectionError as err: else:
self._logger.error("{0} : Connection error for get url {1} : {2}".format(self._name, "{2}://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i, self._protocol_wordpress), err)) self._logger.error("{0} : {2} didn't get due status code : {1}".format(self._name, page.status_code, i))
exit(1) self._logger.debug("{0} : {1}".format(self._name, page.content))
except Exception as err: except ConnectionError as err:
self._logger.error("{0} : Exception error for get url {1} : {2}".format(self._name, "{2}://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i, self._protocol_wordpress), err)) self._logger.error("{0} : Connection error for get url {1} : {2}".format(self._name, "{2}://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i, self._protocol_wordpress), err))
for i in menu: exit(1)
self._logger.info("{0} : Menu : {1}".format(self._name, i)) except Exception as err:
self._logger.error("{0} : Exception error for get url {1} : {2}".format(self._name, "{2}://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i, self._protocol_wordpress), err))
else:
self._logger.info("{0} : Parent title : {1}".format(self._name, parent))
parents.append(parent)
for i in range(0, len(menu)-1):
self._logger.info("{0} : Menu : {1} {2}".format(self._name, menu[i], len(parents)))
for j in range(0, len(parents)-1):
self._logger.info("{0} : Comparaison : {1} {2}".format(self._name, menu[i]["title"], parents[j]))
if menu[i]["title"] == parents[j]:
self._logger.info("{0} del : {1}".format(self._name, parents[j]))
del parents[j]
for i in parents:
self._logger.info("{0} : Parent : {1}".format(self._name, i))