v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

add debug message + fix error directory list

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-05-01 15:45:34 +02:00
parent ab3720fbbc
commit 0029898e6e
4 changed files with 32 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
import_export_canalblog.py
View File

@ -36,6 +36,7 @@ def importUrl(name_thread, max_thread, canalblog, logger, parser, wordpress, bas
exit(1)
exportWp = WPExport(name="Thread-{0}".format(int(name_thread) + 1), url=url, logger=logger, parser=parser)
webpage = exportWp.getUrlPage(name_thread, max_thread)
for j in wordpress:
importWp = WPimport(name=name, basic=basic, wordpress=j, logger=logger, parser=parser)
importWp.fromUrl(webpage)
@ -59,7 +60,8 @@ def importUrl(name_thread, max_thread, canalblog, logger, parser, wordpress, bas
def importDirectory(name_thread, max_thread, directory, logger, parser, wordpress, basic, serial):
name = "Thread-{0}".format(int(name_thread) + 1)
directory = args.directory.split(",")
directory = directory.split(",")
wordpress = wordpress.split(",")
if serial is False:
for i in wordpress:
importWp = WPimport(name=name, basic=basic, wordpress=i, logger=logger, parser=parser)

2
lib/WPExport.py
View File

@ -103,7 +103,7 @@ class WPExport:
setPageDivided = int(number_lastpage) / max_thread
setPagePart = setPageDivided * (index_thread + 1)
firstPagePart = (setPagePart - setPageDivided) + 1
firstPagePart = (setPagePart - setPageDivided)
self._logger.debug("{0} : Total page : {1}".format(self._name,int(number_lastpage)))
self._logger.debug("{0} : First range : {1}".format(self._name, int(firstPagePart)))

12
lib/WPImport.py
View File

@ -36,7 +36,7 @@ class WPimport:
self._logger.error("{0} : Connection error for get url {1} : {2}".format(self._name, webpage[i], err))
exit(1)
if r.status_code == 200:
self._logger.info("{0} : ({1}/{2} : Page is importing : {3}".format(self._name, i+1, len(webpage), webpage[i]))
self._logger.info("{0} : ({1}/{2}) : Page is importing : {3}".format(self._name, i+1, len(webpage), webpage[i]))
soup = BeautifulSoup(r.content, self._parser)
articlebody = soup.find_all("div", class_="articlebody")
if len(articlebody) > 0:
@ -72,6 +72,7 @@ class WPimport:
self._logger.info("{0} : File is being processed : {1}".format(self._name, files[i]))
with open(files[i], 'r') as f:
content = f.read()
self._logger.debug("{0} : Size of article : {1}".format(self._name, len(content)))
soup = BeautifulSoup(content, self._parser)
articlebody = soup.find_all("div", class_="articlebody")
self._logger.debug("{0} : Number of article : {1}".format(self._name, len(articlebody)))
@ -189,12 +190,14 @@ class WPimport:
media = {"id":"", "rendered":""}
split_fileimg = href_img.split("/")
img_name = split_fileimg[len(split_fileimg)-1]
self._logger.debug("{0} : Search for image {1} with URL {2}".format(self._name, img_name, "http://{0}/wp-json/wp/v2/media".format(self._wordpress)))
params = { "search": img_name}
try:
r = self._request.get("http://{0}/wp-json/wp/v2/media".format(self._wordpress), auth=self._basic, params=params)
except Exception as err:
self._logger.error("{0} : Connection error for search media : {1}".format(self._name, err))
exit(1)
self._logger.debug("{0} : Search for image {1} and his status code {2}".format(self._name, img_name, r.status_code))
if r.status_code == 200:
res = r.json()
if len(res) > 0:
@ -359,6 +362,7 @@ class WPimport:
listelement[i] = []
articletitle = soup.find_all("h2", class_="articletitle")
self._logger.debug("{0} : Title of the article : {1}".format(self._name, articletitle))
articlebody = soup.find_all("div", class_="articlebody")
articledate = soup.find_all("span", class_="articledate")
articleacreator = soup.find_all("span", class_="articlecreator")
@ -366,10 +370,12 @@ class WPimport:
itemfooter = soup.find_all("div", class_="itemfooter")
comment = soup.find_all("li", class_="comment")
img_a = articlebody[0].find_all("a", {"target": "_blank"})
self._logger.debug("{0} : Number of image's link : {1}".format(self._name, len(img_a)))
list_img = []
for i in img_a:
new_img = {}
img = i.find_all("img")
self._logger.debug("{0} : Number of image's tag : {1}".format(self._name, len(img)))
if len(img) > 0:
href_a = i.get("href")
href_img = img[0].get("src")
@ -387,7 +393,7 @@ class WPimport:
except Exception as err:
self._logger.error("{0} : Connection error for get image : {1}".format(self._name, err))
exit(1)
self._logger.debug("{0} : Status code for image {1} : {2}".format(self._name, href_img, page_img.status_code))
if page_img.status_code == 200:
media=self._addOrUpdateMedia(href_img, page_img)
new_img["id"]=media["id"]
@ -402,7 +408,7 @@ class WPimport:
self._logger.error("{0} : Connection error with status code for get image : {1}".format(self._name, page_img.status_code))
self._logger.debug("{0} : {1}".format(self._name, page_img.content))
self._debug("{0} : Number of image : {1}".format(self._name, len(list_img)))
comment_post = self._getComment(comment)
a = itemfooter[0].find_all("a", {"rel": True})

19
wp-navigation Normal file
View File

@ -0,0 +1,19 @@
"<!-- wp:page-list /-->\n\n
<!-- wp:navigation-submenu {"label":"Toto","type":"page","id":2,"url":"http://localhost:8080/page-d-exemple/","kind":"post-type"} -->\n
<!-- wp:navigation-link {"label":"Publications","type":"category","id":18,"url":"http://localhost:8080/category/publications/","kind":"taxonomy"} /-->\n
<!-- /wp:navigation-submenu -->"
URL de requête: http://localhost:8080/wp-json/wp/v2/navigation/161?_locale=user
"<!-- wp:page-list /-->\n\n
<!-- wp:navigation-submenu {\"label\":\"Toto\",\"type\":\"page\",\"id\":2,\"url\":\"http://localhost:8080/page-d-exemple/\",\"kind\":\"post-type\"} -->\n
<!-- wp:navigation-link {\"label\":\"Publications\",\"type\":\"category\",\"id\":18,\"url\":\"http://localhost:8080/category/publications/\",\"kind\":\"taxonomy\"} /-->\n\n
<!-- wp:navigation-link {\"label\":\"Osez 20 histoires érotiques dans un train\",\"type\":\"post\",\"id\":155,\"url\":\"http://localhost:8080/osez-20-histoires-erotiques-dans-un-train/\",\"kind\":\"post-type\"} /-->\n
<!-- /wp:navigation-submenu -->"
{"id":161,"content":"<!-- wp:page-list /-->\n\n<!-- wp:navigation-submenu {\"label\":\"Toto\",\"type\":\"page\",\"id\":2,\"url\":\"http://localhost:8080/page-d-exemple/\",\"kind\":\"post-type\"} -->\n<!-- wp:navigation-link {\"label\":\"Publications\",\"type\":\"category\",\"id\":18,\"url\":\"http://localhost:8080/category/publications/\",\"kind\":\"taxonomy\"} /-->\n\n<!-- wp:navigation-link {\"label\":\"Osez 20 histoires érotiques dans une masion close\",\"type\":\"post\",\"id\":155,\"url\":\"http://localhost:8080/osez-20-histoires-erotiques-dans-un-train/\",\"kind\":\"post-type\"} /-->\n<!-- /wp:navigation-submenu -->"}
curl -vvv -u v4l3n71n -X POST -d '{\"content\":\"< wp:page-list /-->\n\n< wp:navigation-submenu {\"label\":\"Toto\",\"type\":\"page\",\"id\":2,\"url\":\"http://localhost:8080/page-d-exemple/\",\"kind\":\"post-type\"} -->\n<!-- wp:navigation-link {\"label\":\"Publications\",\"type\":\"category\",\"id\":18,\"url\":\"http://localhost:8080/category/publications/\",\"kind\":\"taxonomy\"} /-->\n\n<!-- wp:navigation-link {\"label\":\"Osez 20 histoires érotiques dans un train\",\"type\":\"post\",\"id\":155,\"url\":\"http://localhost:8080/osez-20-histoires-erotiques-dans-un-train/\",\"kind\":\"post-type\"} /-->\n<!-- /wp:navigation-submenu -->"}' http://localhost:8080/wp-json/wp/v2/navigation
curl http://localhost:8080/wp-json/wp/v2/navigation |jq .