v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

Merge pull request 'fix-export' (#26) from fix-export into master

Browse Source 
Reviewed-on: #26
This commit is contained in:
v4l3n71n 2023-09-15 21:43:35 +00:00
commit 9ab484da8d
1 changed files with 35 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
lib/WPImport.py
View File

@ -141,7 +141,7 @@ class WPimport:
def _getInfoAlbum(self, link): def _getInfoAlbum(self, link):
if self._ssl_canalblog: if self._ssl_canalblog:
link = link.replace("http", "https") link = link.replace("http", "https").replace("httpss", "https")
self._logger.info("{0} : Info album : {1}".format(self._name, link)) self._logger.info("{0} : Info album : {1}".format(self._name, link))
link_o = urlparse(link) link_o = urlparse(link)
if len(link_o.netloc) > 0: if len(link_o.netloc) > 0:
@ -162,14 +162,17 @@ class WPimport:
if os.path.exists("{0}/..{1}".format(self._directory, link_o)): if os.path.exists("{0}/..{1}".format(self._directory, link_o)):
page_img = open("{0}/..{1}".format(self._directory, link_o), "r") page_img = open("{0}/..{1}".format(self._directory, link_o), "r")
soup = BeautifulSoup(page_img, self._parser) soup = BeautifulSoup(page_img, self._parser)
paragraph = soup.find("div", class_="albumbody").find("p") paragraphs = soup.find("div", class_="albumbody").find_all("p")
self._logger.info("{0} get paragraph : {1}".format(self._name, paragraph)) for paragraph in paragraphs:
split_paragraph = str(paragraph).split("<br>") self._logger.info("{0} get paragraph : {1}".format(self._name, paragraph))
self._logger.info("{0} length paragraph splitted : {1}".format(self._name, len(split_paragraph))) split_paragraph = str(paragraph).split("<br>")
if len(split_paragraph) == 1: self._logger.info("{0} length paragraph splitted : {1}".format(self._name, len(split_paragraph)))
split_paragraph = str(paragraph).split("<br/>") if len(split_paragraph) == 1:
self._logger.info("{0} get paragraph splitted : {1}".format(self._name, split_paragraph)) split_paragraph = str(paragraph).split("<br/>")
author = split_paragraph[1].split(":")[1].replace(" ", "").lower() self._logger.info("{0} get paragraph splitted : {1}".format(self._name, split_paragraph))
if len(split_paragraph) > 1:
if len(split_paragraph[1].split(":")) > 1:
author = split_paragraph[1].split(":")[1].replace(" ", "").lower()
return author return author
@ -302,7 +305,7 @@ class WPimport:
exit(1) exit(1)
except Exception as err: except Exception as err:
self._logger.error("{0} : Exception error for update page : {1}".format(self._name, err)) self._logger.error("{0} : Exception error for update page : {1}".format(self._name, err))
if page.status_code == 400: elif page.status_code == 400:
self._logger.error("{0} : Connection for update post unauthorized : {1}".format(self._name, page.status_code)) self._logger.error("{0} : Connection for update post unauthorized : {1}".format(self._name, page.status_code))
self._logger.debug("{0} : {1}".format(self._name, page.content)) self._logger.debug("{0} : {1}".format(self._name, page.content))
break break
@ -458,6 +461,21 @@ class WPimport:
self._logger.error("{0} : Exception error for post media featured : {1}".format(self._name, err)) self._logger.error("{0} : Exception error for post media featured : {1}".format(self._name, err))
else: else:
self._logger.info("{0} : No media found for {1}".format(self._name, h2)) self._logger.info("{0} : No media found for {1}".format(self._name, h2))
try:
page = self._request.get(img_src.replace("_q", ""))
if page.status_code == 200:
self._logger.info("{0} : Get image for media featured : {1}".format(self._name, img_src.replace("_q", "")))
media = self._addOrUpdateMedia(img_src.replace("_q", ""), page)
if media["id"] != "":
self._addOrUpdateFeaturedMedia(soup)
else:
self._logger.error("{0} : Connection error with status code for get image for featured media : {1}".format(self._name, page.status_code))
self._logger.debug("{0} : {1}".format(self._name, page.content))
except ConnectionError as err:
self._logger.error("{0} : Connection error for get to image for featured media : {1}".format(self._name, err))
exit(1)
except Exception as err:
self._logger.error("{0} : Exception error for get to image media featured : {1}".format(self._name, err))
else: else:
self._logger.error("{0} : Connection error with status code for search featured media: {1}".format(self._name, page.status_code)) self._logger.error("{0} : Connection error with status code for search featured media: {1}".format(self._name, page.status_code))
self._logger.debug("{0} : {1}".format(self._name, page.content)) self._logger.debug("{0} : {1}".format(self._name, page.content))
@ -817,7 +835,7 @@ class WPimport:
listelement[i].append(k["id"]) listelement[i].append(k["id"])
else: else:
break break
if page.status_code == 400: elif page.status_code == 400:
self._logger.error("{0} : {1} not found due status code : {2}".format(self._name, i, page.status_code)) self._logger.error("{0} : {1} not found due status code : {2}".format(self._name, i, page.status_code))
self._logger.debug("{0} : {1}".format(self._name, page.content)) self._logger.debug("{0} : {1}".format(self._name, page.content))
break break
@ -867,12 +885,15 @@ class WPimport:
bodyhtml = bodyhtml.replace(i["old_href"], o.path) bodyhtml = bodyhtml.replace(i["old_href"], o.path)
bodyhtml = bodyhtml.replace(i["old_src"], o.path) bodyhtml = bodyhtml.replace(i["old_src"], o.path)
hour = articledate[0].text hour = articledate[0].text
time = dateheader[0].text.split(" ") time = ""
if len(dateheader) > 0:
time = dateheader[0].text.split(" ")
self._logger.debug("{0} : Title post : |{1}|".format(self._name, title)) self._logger.debug("{0} : Title post : |{1}|".format(self._name, title))
title = self._removeSpace(title) title = self._removeSpace(title)
self._logger.debug("{0} : Rendered Title post : |{1}|".format(self._name, title)) self._logger.debug("{0} : Rendered Title post : |{1}|".format(self._name, title))
data = {"title":title, "content":bodyhtml, "status":"publish", "tags": listelement["tags"], "categories": listelement["categories"]}
data = {"title":title, "content":bodyhtml, "status":"publish", "date": "{0}-{1}-{2}T{3}:00".format(time[2],month[time[1]],time[0], hour), "tags": listelement["tags"], "categories": listelement["categories"]} if len(dateheader) > 0:
data = {"title":title, "content":bodyhtml, "status":"publish", "date": "{0}-{1}-{2}T{3}:00".format(time[2],month[time[1]],time[0], hour), "tags": listelement["tags"], "categories": listelement["categories"]}
self._logger.debug("{0} : Data for post : |{1}| : {2}" .format(self._name, title, data)) self._logger.debug("{0} : Data for post : |{1}| : {2}" .format(self._name, title, data))
data["author"] = self._getAuthor(author) data["author"] = self._getAuthor(author)