v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

fix import 50%

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-05-02 16:59:31 +02:00
parent 3161a06459
commit 4789fe80aa
1 changed files with 77 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
lib/WPImport.py
View File

@ -18,7 +18,7 @@ class WPimport:
self._request = requests.Session()
retries = Retry(total=10,
retries = Retry(connect=10, read=10, redirect=5,
status_forcelist=[429, 500, 502, 503, 504], backoff_factor=2)
self._request.mount('http://', HTTPAdapter(max_retries=retries))
@ -26,7 +26,7 @@ class WPimport:
# Destructor
def __del__(self):
self._logger.info("{0} : Import finished for {1}".format(self._name, self._wordpress))
print("{0} : Import finished for {1}".format(self._name, self._wordpress))
# Public method
@ -192,9 +192,16 @@ class WPimport:
## Add or update img
def _addOrUpdateMedia(self, href_img, page):
media_authorized = ["png", "jpg", "jpeg", "svg"]
media = {"id":"", "rendered":""}
split_fileimg = href_img.split("/")
img_name = split_fileimg[len(split_fileimg)-1]
img_type_file = img_name.split(".")[len(img_name.split("."))-1]
is_img = True
if img_type_file not in media_authorized:
self._logger.error("{0} : Element {1} is not image".format(self._name,img_name))
is_img = False
if is_img is True:
self._logger.debug("{0} : Search for image {1} with URL {2}".format(self._name, img_name, "http://{0}/wp-json/wp/v2/media".format(self._wordpress)))
params = { "search": img_name}
try:
@ -205,6 +212,7 @@ class WPimport:
self._logger.debug("{0} : Search for image {1} and his status code {2}".format(self._name, img_name, r.status_code))
if r.status_code == 200:
res = r.json()
self._logger.debug("{0} : Number of image in search : {1}".format(self._name, len(res)))
if len(res) > 0:
params = {"force":1}
try:
@ -215,13 +223,11 @@ class WPimport:
if r.status_code == 200:
self._logger.info("{0} : Image removed {1}".format(self._name, img_name))
else:
self._logger.error("{0} : Image not removed due status code : {1}".format(self._name, r.status_code))
self._logger.error("{0} : Image {1} not removed due status code : {2}".format(self._name, img_name, r.status_code))
self._logger.debug("{0} : {1}".format(self._name, r.content))
data = page.content
img_type = "image/png"
if img_name.split(".")[1] == "jpg" or img_name.split(".")[1] == "jpeg":
img_type = "image/jpg"
img_type = "image/{0}".format(img_type_file)
headers={ 'Content-Type': img_type,'Content-Disposition' : 'attachment; filename={0}'.format(img_name)}
try:
r = self._request.post("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, headers=headers, data=data)
@ -234,8 +240,8 @@ class WPimport:
media["id"] = res["id"]
media["rendered"] = res["guid"]["rendered"]
else:
self._logger.error("{0} : Image not added due status code : {1}".format(self._name, r.status_code))
self._logger.debug(r.content)
self._logger.error("{0} : Image {1}.{2} not added due status code : {3}".format(self._name, img_name, img_type, r.status_code))
self._logger.debug("{0} : {1}".format(self._name, r.content))
else:
self._logger.error("{0} : Connection error for search image with status code : {1}".format(self._name, r.status_code))
@ -498,35 +504,31 @@ class WPimport:
except Exception as err:
self._logger.error("{0} : Connection error for search post : {1}".format(self._name, err))
exit(1)
page_exist = True
headers = {'Content-Type': 'application/json', 'Accept':'application/json'}
if page.status_code == 200:
result = page.json()
if len(result) == 0:
page_exist = False
else:
self._logger.info("{0} : Page {1} already exist and going to update".format(self._name, title))
post_id = result[0]["id"]
if len(result) > 0:
for i in result:
self._logger.info("{0} : Page {1} already exist and going to delete".format(self._name, title))
post_id = i["id"]
try:
page = self._request.post("http://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, post_id), auth=self._basic, headers=headers, data=json.dumps(data))
params = {"force":1}
page = self._request.delete("http://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, post_id), auth=self._basic, headers=headers, params=params)
except Exception as err:
self._logger.error("{0} : Connection error for update post : {1}".format(self._name, err))
self._logger.error("{0} : Connection error for delete post : {1}".format(self._name, err))
exit(1)
if page.status_code == 200:
result = page.json()
self._logger.info("{0} : Post updated : {1}".format(self._name, result["title"]["raw"]))
self._addOrUpdateComment(result["id"], comment_post, result["title"]["raw"])
self._linkImgPost(result["title"]["raw"], list_img, result["id"])
else:
self._logger.error("{0} : Post not updated due status code : {1}".format(self._name, page.status_code))
self._logger.debug("{0} : {1}".format(self._name, page.content))
self._logger.info("{0} : Post deleted : {1}".format(self._name, title))
else:
self._logger.error("{0} : Connection for update post error with status code : {1}".format(self._name, page.status_code))
self._logger.error("{0} : Post not deleted due status code : {1}".format(self._name, page.status_code))
self._logger.debug("{0} : {1}".format(self._name, page.content))
else:
self._logger.error("{0} : Connection for delete post error with status code : {1}".format(self._name, page.status_code))
self._logger.debug("{0} : {1}".format(self._name, page.content))
if page_exist == False:
try:
page = self._request.post("http://{0}/wp-json/wp/v2/posts".format(self._wordpress), auth=self._basic, headers=headers, data=json.dumps(data))
except Exception as err: