v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

fix WPExport

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-04-09 23:49:10 +02:00
parent 19c62f38d4
commit cd50e45493
2 changed files with 10 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
WPExport.py
View File

@ -25,7 +25,7 @@ class WPExport:
def downloadCss(self): def downloadCss(self):
css = self._getScriptCss(False, True) css = self._getScriptCss(False, True)
o = urlparse(self._url) o = urlparse(self._url)
self._downloadPage(script, "{0}/{1}/{2}".format(self._dir, o.path, "dists/css")) self._downloadPage(css, "{0}/{1}/{2}".format(self._dir, o.path, "dists/css"))
# Download HTML # Download HTML
@ -79,9 +79,9 @@ class WPExport:
for j in range(1,int(number_lastpage)): for j in range(1,int(number_lastpage)):
paging = j * 10 paging = j * 10
categorie = urlparse(i).path.split("/") categorie = urlparse(i).path.split("/")
url_paging = "{0}/archives/p{1}-10.html".format(url, paging) url_paging = "{0}/archives/p{1}-10.html".format(self._url, paging)
if len(categorie) > 2: if len(categorie) > 2:
url_paging = "{0}/archives/{1}/p{2}-10.html".format(url, categorie[2], paging) url_paging = "{0}/archives/{1}/p{2}-10.html".format(self._url, categorie[2], paging)
self._logger.info(url_paging) self._logger.info(url_paging)
if url_paging not in webpage: if url_paging not in webpage:
webpage.append(url_paging) webpage.append(url_paging)
@ -105,7 +105,7 @@ class WPExport:
# Private method # Private method
# #
# Create path # Create path
def _mkdirPath(self, path_dir, logger): def _mkdirPath(self, path_dir):
if not os.path.exists(path_dir): if not os.path.exists(path_dir):
makedir = [] makedir = []
pathh = path_dir.split("/") pathh = path_dir.split("/")
@ -126,7 +126,7 @@ class WPExport:
# Get Css and JS # Get Css and JS
def _getScriptCss(self, js, css): def _getScriptCss(self, js, css):
try: try:
page = requests.get(url) page = requests.get(self._url)
except Exception as err: except Exception as err:
self._logger.error("Connection error : {0}".format(err)) self._logger.error("Connection error : {0}".format(err))
exit(1) exit(1)
@ -139,7 +139,7 @@ class WPExport:
src = anchor.get("src", "/") src = anchor.get("src", "/")
if src != "/": if src != "/":
try: try:
u = urlparse(url) u = urlparse(self._url)
o = urlparse(src) o = urlparse(src)
except Exception as err: except Exception as err:
self._logger.error("parsing error : {0}".format(err)) self._logger.error("parsing error : {0}".format(err))
@ -156,7 +156,7 @@ class WPExport:
href = anchor.get("href", "/") href = anchor.get("href", "/")
if href != "/": if href != "/":
try: try:
u = urlparse(url) u = urlparse(self._url)
o = urlparse(href) o = urlparse(href)
except Exception as err: except Exception as err:
self._logger.error("parsing error : {0}".format(err)) self._logger.error("parsing error : {0}".format(err))

5
import_export_canalblog.py
View File

@ -1,8 +1,9 @@
#!/usr/bin/python3 #!/usr/bin/python3
from requests.auth import HTTPBasicAuth from requests.auth import HTTPBasicAuth
from getpass import getpass from getpass import getpass
from urllib.parse import urlparse
import argparse, logging import argparse, logging
import WPImport import WPImport, WPExport
if __name__ == '__main__': if __name__ == '__main__':
@ -82,7 +83,7 @@ if __name__ == '__main__':
except Exception as err: except Exception as err:
logger.error("parsing error : {0}".format(err)) logger.error("parsing error : {0}".format(err))
exit(1) exit(1)
exportWp = WPExport.WPExport(url, logger, args.parser, args.dir) exportWp = WPExport.WPExport(url, logger, args.parser, args.directory)
if args.js is False: if args.js is False:
exportWp.downloadJs() exportWp.downloadJs()