v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

add search tags and categories before create tags and categories

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-04-18 21:50:36 +02:00
parent d64aed6240
commit edb9442b1c
1 changed files with 21 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
lib/WPImport.py
View File

@ -14,7 +14,7 @@ class WPimport:
self._logger = logger
self._parser = parser
self._headers_json = {'Content-Type': 'application/json', 'Accept':'application/json'}
self._request = requests.Session()
retries = Retry(total=10,
@ -321,17 +321,8 @@ class WPimport:
listelement = {}
for i in liste:
try:
page = self._request.get("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress,i))
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if page.status_code == 200:
elements[i] = page.json()
element[i] = []
listelement[i] = []
else:
self._logger.error("Connection error with status code : {0}".format(page.status_code))
element[i] = []
listelement[i] = []
articletitle = soup.find_all("h2", class_="articletitle")
articlebody = soup.find_all("div", class_="articlebody")
@ -390,13 +381,25 @@ class WPimport:
for i in liste:
for j in element[i]:
element_exist = False
for k in elements[i]:
if k["name"] == j:
element_exist = True
listelement[i].append(k["id"])
try:
params = {"params":j}
page = self._request.get("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i), auth=self._basic, params=params)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
if page.status_code == 200:
element_exist = True
result = page.json()
listelement[i].append(result[0]["id"])
else:
self._logger.error("{0} not finded due status code : {1}".format(i, page.status_code))
if element_exist is False:
data = {"name": j}
self._logger.debug("URL : {0} with data : {1} and with headers : {2}".format("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i), data, self._headers_json))
self._logger.debug("URL : {0} ".format("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i)))
self._logger.debug("data : {0}".format(data))
self._logger.debug("headers : {0}".format(self._headers_form))
try:
page = self._request.post("http://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i), auth=self._basic, headers=self._headers_json, data=data)
except Exception as err:
@ -407,7 +410,7 @@ class WPimport:
listelement[i].append(result["id"])
else:
self._logger.error("{0} not added due status code : {1}".format(i, page.status_code))
self._logger.debug(listelement)
title = articletitle[0].text
author = articleacreator[0].text.lower()
body = articlebody[0].find_all("p")