v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

change parameter for webpage

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-06-20 00:17:38 +02:00
parent 7dace5bdb7
commit 35ff22d463
3 changed files with 16 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
import_export_canalblog.py
View File

@ -43,15 +43,15 @@ def remove(index, number, args, basic, logger, ssl_wordpress):
def download(name_thread, max_thread, url, logger, parser, directory, html, img, ssl_canalblog, revert, tmp):
exportWp = WPExport(name="Thread-{0}".format(int(name_thread) + 1), url=url, logger=logger, parser=parser, directory=directory, ssl_canalblog=ssl_canalblog)
if not revert:
webpage = exportWp.getUrlPage(name_thread, max_thread)
exportWp.getUrlPage(name_thread, max_thread)
for i in ["article", "page"]:
for j in ["publications", "principal"]:
if html is False:
exportWp.downloadHTML(webpage[j][i])
exportWp.downloadHTML(j, i)
if img is False:
exportWp.downloadImg(webpage[j][i])
exportWp.downloadImg(j, i)
del exportWp
@ -71,15 +71,15 @@ def importUrl(name_thread, max_thread, canalblog, logger, parser, wordpress, bas
except Exception as err:
logger.error("{0} : parsing error : {1}".format(name, err))
exit(1)
exportWp = WPExport(name="Thread-{0}".format(int(name_thread) + 1), url=url, logger=logger, parser=parser, ssl_canalblog=ssl_canalblog)
exportWp = WPExport(name="Thread-{0}".format(int(name_thread) + 1), url=url, logger=logger, parser=parser, ssl_canalblog=ssl_canalblog, tmp=tmp)
if not revert:
webpage = exportWp.getUrlPage(name_thread, max_thread)
exportWp.getUrlPage(name_thread, max_thread)
del exportWp
for j in wordpress:
importWp = WPimport(name=name, basic=basic, wordpress=j, logger=logger, parser=parser, ssl_wordpress=ssl_wordpress, no_create=create, no_update=update, no_image=image)
importWp = WPimport(name=name, basic=basic, wordpress=j, logger=logger, parser=parser, ssl_wordpress=ssl_wordpress, no_create=create, no_update=update, no_image=image, tmp=tmp)
for k in ["article", "page"]:
for l in ["publications", "principal"]:
importWp.fromUrl(webpage[l][k])
importWp.fromUrl(l, k)
del importWp
else:
@ -96,9 +96,9 @@ def importUrl(name_thread, max_thread, canalblog, logger, parser, wordpress, bas
exit(1)
exportWp = WPExport(name=name, url=url, logger=logger, parser=parser, ssl_canalblog=ssl_canalblog)
if not revert:
webpage = exportWp.getUrlPage(name_thread, max_thread)
exportWp.getUrlPage(name_thread, max_thread)
del exportWp
importWp = WPimport(name=name, basic=basic, wordpress=wordpress[i], logger=logger, parser=parser, ssl_wordpress=ssl_wordpress, no_create=create, no_update=update, no_image=image)
importWp = WPimport(name=name, basic=basic, wordpress=wordpress[i], logger=logger, parser=parser, ssl_wordpress=ssl_wordpress, no_create=create, no_update=update, no_image=image, tmp=tmp)
for k in ["article", "page"]:
for l in ["publications", "principal"]:

8
lib/WPExport.py
View File

@ -57,13 +57,13 @@ class WPExport:
# Download HTML
def downloadHTML(self, webpage):
self._downloadPage(webpage, self._dir)
def downloadHTML(self, first, second):
self._downloadPage(webpage[first][second], self._dir)
# Download Image
def downloadImg(self, webpage):
page_src = self._getImg(webpage)
def downloadImg(self, first, second):
page_src = self._getImg(webpage[first][second])
o = urlparse(self._url)
self._downloadPage(page_src, "{0}/{1}/{2}".format(self._dir, o.path, "img"))

4
lib/WPImport.py
View File

@ -39,8 +39,8 @@ class WPimport:
def setUrl(self, wordpress):
self._wordpress = wordpress
def fromUrl(self, webpage):
for i in range(0, len(webpage)):
def fromUrl(self, first, second):
for i in range(0, len(webpage[first][second])):
try:
r = self._request.get(webpage[i])
if r.status_code == 200: