v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

add album test wip

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-08-17 00:18:45 +02:00
parent 5959ab5b2e
commit 418bea3778
2 changed files with 40 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
import_export_canalblog.py
View File

@ -263,8 +263,9 @@ if __name__ == '__main__':
for i in wordpress: for i in wordpress:
importWp.setUrl(i) importWp.setUrl(i)
importWp.fromFile(files=args.file.split(",")) importWp.fromFile(files=args.file.split(","))
exit(0)
menuWp = WPMenu(name="Thread-1", basic=basic, wordpress=args.wordpress, logger=logger, parser=args.parser, ssl_canalblog=ssl_canalblog, ssl_wordpress=ssl_wordpress) menuWp = WPMenu(name="Thread-1", basic=basic, wordpress=args.wordpress, logger=logger, parser=args.parser, ssl_canalblog=ssl_canalblog, ssl_wordpress=ssl_wordpress)
menuWp.fromFile("{0}/index.html".format(args.file.split(",")[0])) menuWp.fromFile("{0}".format(args.file.split(",")[0]))
if len(args.directory) > 0: if len(args.directory) > 0:
try: try:
with futures.ThreadPoolExecutor(max_workers=int(args.parallel)) as ex: with futures.ThreadPoolExecutor(max_workers=int(args.parallel)) as ex:

100
lib/WPImport.py
View File

@ -120,73 +120,49 @@ class WPimport:
albumtitle = albumbody.find("h2").get_text() albumtitle = albumbody.find("h2").get_text()
self._logger.debug("{0} : Title of the album : {1}".format(self._name, albumtitle)) self._logger.debug("{0} : Title of the album : {1}".format(self._name, albumtitle))
albumdesc = albumbody.find("div", class_="albumdesc").find("p") albumdesc = albumbody.find("div", class_="albumdesc").find("p")
img_a = albumbody.find_all("a", {"target": "_blank"}) img_a = albumbody.find_all("img")
list_img = []
if self._no_image is False: if self._no_image is False:
self._logger.debug("{0} : Number of image's tag : {1}".format(self._name, len(img_a)))
for i in img_a: for i in img_a:
new_img = {} new_img = {}
img = i.find_all("img") href_img = i.get("src")
self._logger.debug("{0} : Number of image's tag : {1}".format(self._name, len(img))) href_img_o = urlparse(href_img)
if len(img) > 0: new_img["old_src"]=href_img
href_a = i.get("href") try:
href_img = img[0].get("src") if len(href_img_o.netloc) > 0:
href_a_o = urlparse(href_a) img_ok = False
href_img_o = urlparse(href_img) page_img = self._request.get(href_img)
new_img["old_src"]=href_img if page_img.status_code == 200:
new_img["old_href"]=href_a img_ok = True
try: else:
if len(href_img_o.netloc) > 0: if os.path.exists("{0}/..{1}".format(self._directory, href_img)):
img_ok = False page_img = open("{0}/..{1}".format(self._directory, href_img), "r")
page_img = self._request.get(href_img) img_ok = True
self._logger.debug("{0} : Status code for image {1} : {2}".format(self._name, href_img, page_img.status_code))
if page_img.status_code == 404: if img_ok is True:
href_img = href_a media=self._addOrUpdateMedia(href_img, page_img)
try: new_img["id"]=media["id"]
page_img = self._request.get(href_a) new_img["new_src"]=media["rendered"]
if page_img.status_code == 200: list_img.append(new_img)
img_ok = True except ConnectionError as err:
except ConnectionError as err: self._logger.error("{0} : Connection error for get image : {1}".format(self._name, err))
self._logger.error("{0} : Connection error for get image : {1}".format(self._name, err)) exit(1)
exit(1) except Exception as err:
except Exception as err: self._logger.error("{0} : Exception error for get image : {1}".format(self._name, err))
self._logger.error("{0} : Exception error for get image : {1}".format(self._name, err)) exit(1)
exit(1) content_html = ""
if len(list_img) > 0:
else: content_html = "<!-- wp:paragraph --><p>{0}</p><!-- /wp:paragraph -->\n\n".format(albumdesc)
if os.path.exists("{0}/..{1}".format(self._directory, href_img)):
page_img = open("{0}/..{1}".format(self._directory, href_img), "r")
img_ok = True
else:
if os.path.exists("{0}/..{1}".format(self._directory, href_a)):
page_img = open("{0}/..{1}".format(self._directory, href_a), "r")
img_ok = True
self._logger.debug("{0} : Status code for image {1} : {2}".format(self._name, href_img, page_img.status_code))
if img_ok is True:
media=self._addOrUpdateMedia(href_img, page_img)
new_img["id"]=media["id"]
new_img["new_src"]=media["rendered"]
list_img.append(new_img)
if href_img != href_a:
media=self._addOrUpdateMedia(href_a, page_img)
new_img["id"]=media["id"]
new_img["new_src"]=media["rendered"]
list_img.append(new_img)
if page_img.status_code not in [200, 404]:
self._logger.error("{0} : Connection error with status code for get image : {1}".format(self._name, page_img.status_code))
self._logger.debug("{0} : {1}".format(self._name, page_img.content))
except ConnectionError as err:
self._logger.error("{0} : Connection error for get image : {1}".format(self._name, err))
exit(1)
except Exception as err:
self._logger.error("{0} : Exception error for get image : {1}".format(self._name, err))
exit(1)
content_html = "<!-- wp:paragraph --><p>{0}</p><!-- /wp:paragraph -->\n\n".format(albumdesc)
content_html = content_html + "<figure class=\"wp-block-gallery has-nested-images columns-default is-cropped\"><!-- wp:image {'id':{0},\"sizeSlug\":\"large\",\"linkDestination\":\"none\"} --><figure class=\"wp-block-image size-large'><img src=\"{1}\" alt=\"\" class=\"wp-image-{0}\"/></figure><!-- /wp:image --> \n\n".format(list_img[0]["id"],list_img[1]["new_src"]) content_html = content_html + "<figure class=\"wp-block-gallery has-nested-images columns-default is-cropped\"><!-- wp:image {'id':{0},\"sizeSlug\":\"large\",\"linkDestination\":\"none\"} --><figure class=\"wp-block-image size-large'><img src=\"{1}\" alt=\"\" class=\"wp-image-{0}\"/></figure><!-- /wp:image --> \n\n".format(list_img[0]["id"],list_img[1]["new_src"])
for i in range(1, len(list_img)): for i in range(1, len(list_img)):
content_html = content_html + "<!-- wp:image {'id':{0},\"sizeSlug\":\"large\",\"linkDestination\":\"none\"} --><figure class=\"wp-block-image size-large\"><img src='{0}' alt=\"\" class='wp-image-{1}'/></figure><!-- /wp:image -->\n\n".format(list_img[i]["id"],list_img[i]["new_src"]) content_html = content_html + "<!-- wp:image {'id':{0},\"sizeSlug\":\"large\",\"linkDestination\":\"none\"} --><figure class=\"wp-block-image size-large\"><img src='{0}' alt=\"\" class='wp-image-{1}'/></figure><!-- /wp:image -->\n\n".format(list_img[i]["id"],list_img[i]["new_src"])
content_html = content_html + "<!-- /wp:gallery -->" content_html = content_html + "<!-- /wp:gallery -->"
self._logger.info("{0} : content html : {1}".format(self._name, content_html))