v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

fix import 50%

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-05-02 16:59:31 +02:00
parent 3161a06459
commit 4789fe80aa
1 changed files with 77 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

152
lib/WPImport.py
View File

@ -18,7 +18,7 @@ class WPimport:
self._request = requests.Session() self._request = requests.Session()
retries = Retry(total=10, retries = Retry(connect=10, read=10, redirect=5,
status_forcelist=[429, 500, 502, 503, 504], backoff_factor=2) status_forcelist=[429, 500, 502, 503, 504], backoff_factor=2)
self._request.mount('http://', HTTPAdapter(max_retries=retries)) self._request.mount('http://', HTTPAdapter(max_retries=retries))
@ -26,7 +26,7 @@ class WPimport:
# Destructor # Destructor
def __del__(self): def __del__(self):
self._logger.info("{0} : Import finished for {1}".format(self._name, self._wordpress)) print("{0} : Import finished for {1}".format(self._name, self._wordpress))
# Public method # Public method
@ -192,54 +192,60 @@ class WPimport:
## Add or update img ## Add or update img
def _addOrUpdateMedia(self, href_img, page): def _addOrUpdateMedia(self, href_img, page):
media_authorized = ["png", "jpg", "jpeg", "svg"]
media = {"id":"", "rendered":""} media = {"id":"", "rendered":""}
split_fileimg = href_img.split("/") split_fileimg = href_img.split("/")
img_name = split_fileimg[len(split_fileimg)-1] img_name = split_fileimg[len(split_fileimg)-1]
self._logger.debug("{0} : Search for image {1} with URL {2}".format(self._name, img_name, "http://{0}/wp-json/wp/v2/media".format(self._wordpress))) img_type_file = img_name.split(".")[len(img_name.split("."))-1]
params = { "search": img_name} is_img = True
try: if img_type_file not in media_authorized:
r = self._request.get("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, params=params) self._logger.error("{0} : Element {1} is not image".format(self._name,img_name))
except Exception as err: is_img = False
self._logger.error("{0} : Connection error for search media : {1}".format(self._name, err)) if is_img is True:
exit(1) self._logger.debug("{0} : Search for image {1} with URL {2}".format(self._name, img_name, "http://{0}/wp-json/wp/v2/media".format(self._wordpress)))
self._logger.debug("{0} : Search for image {1} and his status code {2}".format(self._name, img_name, r.status_code)) params = { "search": img_name}
if r.status_code == 200: try:
res = r.json() r = self._request.get("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, params=params)
if len(res) > 0: except Exception as err:
params = {"force":1} self._logger.error("{0} : Connection error for search media : {1}".format(self._name, err))
exit(1)
self._logger.debug("{0} : Search for image {1} and his status code {2}".format(self._name, img_name, r.status_code))
if r.status_code == 200:
res = r.json()
self._logger.debug("{0} : Number of image in search : {1}".format(self._name, len(res)))
if len(res) > 0:
params = {"force":1}
try:
r = self._request.delete("http://{0}/wp-json/wp/v2/media/{1}".format(self._wordpress, res[0]["id"]), auth=self._basic, params=params)
except Exception as err:
self._logger.error("{0} Connection error for delete image : {1}".format(self._name, err))
exit(1)
if r.status_code == 200:
self._logger.info("{0} : Image removed {1}".format(self._name, img_name))
else:
self._logger.error("{0} : Image {1} not removed due status code : {2}".format(self._name, img_name, r.status_code))
self._logger.debug("{0} : {1}".format(self._name, r.content))
data = page.content
img_type = "image/{0}".format(img_type_file)
headers={ 'Content-Type': img_type,'Content-Disposition' : 'attachment; filename={0}'.format(img_name)}
try: try:
r = self._request.delete("http://{0}/wp-json/wp/v2/media/{1}".format(self._wordpress, res[0]["id"]), auth=self._basic, params=params) r = self._request.post("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, headers=headers, data=data)
except Exception as err: except Exception as err:
self._logger.error("{0} Connection error for delete image : {1}".format(self._name, err)) self._logger.error("{0} : Connection error for add image : {1}".format(self._name, err))
exit(1) exit(1)
if r.status_code == 200: if r.status_code == 201:
self._logger.info("{0} : Image removed {1}".format(self._name, img_name)) self._logger.info("{0} : Image added {1}".format(self._name, img_name))
res = r.json()
media["id"] = res["id"]
media["rendered"] = res["guid"]["rendered"]
else: else:
self._logger.error("{0} : Image not removed due status code : {1}".format(self._name, r.status_code)) self._logger.error("{0} : Image {1}.{2} not added due status code : {3}".format(self._name, img_name, img_type, r.status_code))
self._logger.debug("{0} : {1}".format(self._name, r.content)) self._logger.debug("{0} : {1}".format(self._name, r.content))
data = page.content
img_type = "image/png"
if img_name.split(".")[1] == "jpg" or img_name.split(".")[1] == "jpeg":
img_type = "image/jpg"
headers={ 'Content-Type': img_type,'Content-Disposition' : 'attachment; filename={0}'.format(img_name)}
try:
r = self._request.post("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, headers=headers, data=data)
except Exception as err:
self._logger.error("{0} : Connection error for add image : {1}".format(self._name, err))
exit(1)
if r.status_code == 201:
self._logger.info("{0} : Image added {1}".format(self._name, img_name))
res = r.json()
media["id"] = res["id"]
media["rendered"] = res["guid"]["rendered"]
else: else:
self._logger.error("{0} : Image not added due status code : {1}".format(self._name, r.status_code)) self._logger.error("{0} : Connection error for search image with status code : {1}".format(self._name, r.status_code))
self._logger.debug(r.content) self._logger.debug("{0} : {1}".format(self._name, r.content))
else:
self._logger.error("{0} : Connection error for search image with status code : {1}".format(self._name, r.status_code))
self._logger.debug("{0} : {1}".format(self._name, r.content))
return media return media
@ -498,45 +504,41 @@ class WPimport:
except Exception as err: except Exception as err:
self._logger.error("{0} : Connection error for search post : {1}".format(self._name, err)) self._logger.error("{0} : Connection error for search post : {1}".format(self._name, err))
exit(1) exit(1)
page_exist = True
headers = {'Content-Type': 'application/json', 'Accept':'application/json'} headers = {'Content-Type': 'application/json', 'Accept':'application/json'}
if page.status_code == 200: if page.status_code == 200:
result = page.json() result = page.json()
if len(result) == 0: if len(result) > 0:
page_exist = False for i in result:
else: self._logger.info("{0} : Page {1} already exist and going to delete".format(self._name, title))
self._logger.info("{0} : Page {1} already exist and going to update".format(self._name, title)) post_id = i["id"]
post_id = result[0]["id"] try:
try: params = {"force":1}
page = self._request.post("http://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, post_id), auth=self._basic, headers=headers, data=json.dumps(data)) page = self._request.delete("http://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, post_id), auth=self._basic, headers=headers, params=params)
except Exception as err: except Exception as err:
self._logger.error("{0} : Connection error for update post : {1}".format(self._name, err)) self._logger.error("{0} : Connection error for delete post : {1}".format(self._name, err))
exit(1) exit(1)
if page.status_code == 200: if page.status_code == 200:
result = page.json() result = page.json()
self._logger.info("{0} : Post updated : {1}".format(self._name, result["title"]["raw"])) self._logger.info("{0} : Post deleted : {1}".format(self._name, title))
self._addOrUpdateComment(result["id"], comment_post, result["title"]["raw"])
self._linkImgPost(result["title"]["raw"], list_img, result["id"]) else:
else: self._logger.error("{0} : Post not deleted due status code : {1}".format(self._name, page.status_code))
self._logger.error("{0} : Post not updated due status code : {1}".format(self._name, page.status_code)) self._logger.debug("{0} : {1}".format(self._name, page.content))
self._logger.debug("{0} : {1}".format(self._name, page.content))
else: else:
self._logger.error("{0} : Connection for update post error with status code : {1}".format(self._name, page.status_code)) self._logger.error("{0} : Connection for delete post error with status code : {1}".format(self._name, page.status_code))
self._logger.debug("{0} : {1}".format(self._name, page.content)) self._logger.debug("{0} : {1}".format(self._name, page.content))
if page_exist == False: try:
try: page = self._request.post("http://{0}/wp-json/wp/v2/posts".format(self._wordpress), auth=self._basic, headers=headers, data=json.dumps(data))
page = self._request.post("http://{0}/wp-json/wp/v2/posts".format(self._wordpress), auth=self._basic, headers=headers, data=json.dumps(data)) except Exception as err:
except Exception as err: self._logger.error("{0} : Connection error for create post : {1}".format(self._name, err))
self._logger.error("{0} : Connection error for create post : {1}".format(self._name, err)) exit(1)
exit(1) if page.status_code == 201:
if page.status_code == 201: result = page.json()
result = page.json() self._logger.info("{0} : Post added : {1}".format(self._name, result["title"]["raw"]))
self._logger.info("{0} : Post added : {1}".format(self._name, result["title"]["raw"])) self._addOrUpdateComment(result["id"], comment_post, result["title"]["raw"])
self._addOrUpdateComment(result["id"], comment_post, result["title"]["raw"]) self._linkImgPost(result["title"]["raw"], list_img, result["id"])
self._linkImgPost(result["title"]["raw"], list_img, result["id"]) else:
else: self._logger.error("{0} : Post not added due status code : {1}".format(self._name, r.status_code))
self._logger.error("{0} : Post not added due status code : {1}".format(self._name, r.status_code)) self._logger.debug("{0} : {1}".format(self._name, r.content))
self._logger.debug("{0} : {1}".format(self._name, r.content))