v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

fix WPExport

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-04-09 23:49:10 +02:00
parent 19c62f38d4
commit cd50e45493
2 changed files with 10 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
WPExport.py
View File

@ -25,7 +25,7 @@ class WPExport:
def downloadCss(self):
css = self._getScriptCss(False, True)
o = urlparse(self._url)
self._downloadPage(script, "{0}/{1}/{2}".format(self._dir, o.path, "dists/css"))
self._downloadPage(css, "{0}/{1}/{2}".format(self._dir, o.path, "dists/css"))
# Download HTML
@ -79,9 +79,9 @@ class WPExport:
for j in range(1,int(number_lastpage)):
paging = j * 10
categorie = urlparse(i).path.split("/")
url_paging = "{0}/archives/p{1}-10.html".format(url, paging)
url_paging = "{0}/archives/p{1}-10.html".format(self._url, paging)
if len(categorie) > 2:
url_paging = "{0}/archives/{1}/p{2}-10.html".format(url, categorie[2], paging)
url_paging = "{0}/archives/{1}/p{2}-10.html".format(self._url, categorie[2], paging)
self._logger.info(url_paging)
if url_paging not in webpage:
webpage.append(url_paging)
@ -105,7 +105,7 @@ class WPExport:
# Private method
#
# Create path
def _mkdirPath(self, path_dir, logger):
def _mkdirPath(self, path_dir):
if not os.path.exists(path_dir):
makedir = []
pathh = path_dir.split("/")
@ -126,7 +126,7 @@ class WPExport:
# Get Css and JS
def _getScriptCss(self, js, css):
try:
page = requests.get(url)
page = requests.get(self._url)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
exit(1)
@ -139,7 +139,7 @@ class WPExport:
src = anchor.get("src", "/")
if src != "/":
try:
u = urlparse(url)
u = urlparse(self._url)
o = urlparse(src)
except Exception as err:
self._logger.error("parsing error : {0}".format(err))
@ -156,7 +156,7 @@ class WPExport:
href = anchor.get("href", "/")
if href != "/":
try:
u = urlparse(url)
u = urlparse(self._url)
o = urlparse(href)
except Exception as err:
self._logger.error("parsing error : {0}".format(err))

5
import_export_canalblog.py
View File

@ -1,8 +1,9 @@
#!/usr/bin/python3
from requests.auth import HTTPBasicAuth
from getpass import getpass
from urllib.parse import urlparse
import argparse, logging
import WPImport
import WPImport, WPExport
if __name__ == '__main__':
@ -82,7 +83,7 @@ if __name__ == '__main__':
except Exception as err:
logger.error("parsing error : {0}".format(err))
exit(1)
exportWp = WPExport.WPExport(url, logger, args.parser, args.dir)
exportWp = WPExport.WPExport(url, logger, args.parser, args.directory)
if args.js is False:
exportWp.downloadJs()