v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

Compare commits

base: v4l3n71n:41e192f903381c805b1ebd097cea994c11f64034
Branches Tags
v4l3n71n:master
v4l3n71n:4.0.4 v4l3n71n:4.0.3 v4l3n71n:4.0.2 v4l3n71n:4.0.1 v4l3n71n:4.0.0 v4l3n71n:3.1.0 v4l3n71n:3.0.0 v4l3n71n:2.0.1 v4l3n71n:2.0.0 v4l3n71n:1.0.0
...
compare: v4l3n71n:4.0.3
Branches Tags
v4l3n71n:master
v4l3n71n:4.0.4 v4l3n71n:4.0.3 v4l3n71n:4.0.2 v4l3n71n:4.0.1 v4l3n71n:4.0.0 v4l3n71n:3.1.0 v4l3n71n:3.0.0 v4l3n71n:2.0.1 v4l3n71n:2.0.0 v4l3n71n:1.0.0

10 Commits
41e192f903 ... 4.0.3

Author SHA1 Message Date
v4l3n71n
9ab484da8d Merge pull request 'fix-export' (#26) from fix-export into master 
Reviewed-on: #26
 2023-09-15 21:43:35 +00:00
Valentin CZERYBA
c275f10fb0 test ok for featured media 2023-09-15 23:38:35 +02:00
Valentin CZERYBA
fa1854052b fix if 2023-09-15 23:14:24 +02:00
Valentin CZERYBA
0e5afe60a7 add image featured media 2023-09-15 23:13:08 +02:00
Valentin CZERYBA
d2cfb949f8 add image for featured image 2023-09-15 23:11:25 +02:00
Valentin CZERYBA
bce1643135 fix replace 2023-09-14 21:27:09 +02:00
Valentin CZERYBA
0c94783852 fix album author 2023-09-14 21:23:27 +02:00
Valentin CZERYBA
6f78a48e57 fix index 2023-09-11 23:20:48 +02:00
Valentin CZERYBA
dbab60c7dc fix import album 2023-09-11 23:20:22 +02:00
v4l3n71n
dce2c2dfa5 Merge pull request 'fix webpage' (#25) from fix-export into master 
Reviewed-on: #25
 2023-09-05 20:10:07 +00:00
1 changed files with 27 additions and 9 deletions
Expand all files Collapse all files

36
lib/WPImport.py
View File

@@ -141,7 +141,7 @@ class WPimport:
def _getInfoAlbum(self, link):
if self._ssl_canalblog:
link = link.replace("http", "https")
link = link.replace("http", "https").replace("httpss", "https")
self._logger.info("{0} : Info album : {1}".format(self._name, link))
link_o = urlparse(link)
if len(link_o.netloc) > 0:
@@ -162,14 +162,17 @@ class WPimport:
if os.path.exists("{0}/..{1}".format(self._directory, link_o)):
page_img = open("{0}/..{1}".format(self._directory, link_o), "r")
soup = BeautifulSoup(page_img, self._parser)
paragraph = soup.find("div", class_="albumbody").find("p")
self._logger.info("{0} get paragraph : {1}".format(self._name, paragraph))
split_paragraph = str(paragraph).split("<br>")
self._logger.info("{0} length paragraph splitted : {1}".format(self._name, len(split_paragraph)))
if len(split_paragraph) == 1:
split_paragraph = str(paragraph).split("<br/>")
self._logger.info("{0} get paragraph splitted : {1}".format(self._name, split_paragraph))
author = split_paragraph[1].split(":")[1].replace(" ", "").lower()
paragraphs = soup.find("div", class_="albumbody").find_all("p")
for paragraph in paragraphs:
self._logger.info("{0} get paragraph : {1}".format(self._name, paragraph))
split_paragraph = str(paragraph).split("<br>")
self._logger.info("{0} length paragraph splitted : {1}".format(self._name, len(split_paragraph)))
if len(split_paragraph) == 1:
split_paragraph = str(paragraph).split("<br/>")
self._logger.info("{0} get paragraph splitted : {1}".format(self._name, split_paragraph))
if len(split_paragraph) > 1:
if len(split_paragraph[1].split(":")) > 1:
author = split_paragraph[1].split(":")[1].replace(" ", "").lower()
return author
@@ -458,6 +461,21 @@ class WPimport:
self._logger.error("{0} : Exception error for post media featured : {1}".format(self._name, err))
else:
self._logger.info("{0} : No media found for {1}".format(self._name, h2))
try:
page = self._request.get(img_src.replace("_q", ""))
if page.status_code == 200:
self._logger.info("{0} : Get image for media featured : {1}".format(self._name, img_src.replace("_q", "")))
media = self._addOrUpdateMedia(img_src.replace("_q", ""), page)
if media["id"] != "":
self._addOrUpdateFeaturedMedia(soup)
else:
self._logger.error("{0} : Connection error with status code for get image for featured media : {1}".format(self._name, page.status_code))
self._logger.debug("{0} : {1}".format(self._name, page.content))
except ConnectionError as err:
self._logger.error("{0} : Connection error for get to image for featured media : {1}".format(self._name, err))
exit(1)
except Exception as err:
self._logger.error("{0} : Exception error for get to image media featured : {1}".format(self._name, err))
else:
self._logger.error("{0} : Connection error with status code for search featured media: {1}".format(self._name, page.status_code))
self._logger.debug("{0} : {1}".format(self._name, page.content))