v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

insert #6

Merged
v4l3n71n merged 60 commits from insert into master 2023-04-11 21:29:54 +00:00
Conversation 0 Commits 60 Files Changed 6 +10 -9
2 changed files with 10 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit cd50e45493 - Show all commits

14
WPExport.py
View File

@ -25,7 +25,7 @@ class WPExport:
def downloadCss(self): def downloadCss(self):
css = self._getScriptCss(False, True) css = self._getScriptCss(False, True)
o = urlparse(self._url) o = urlparse(self._url)
self._downloadPage(script, "{0}/{1}/{2}".format(self._dir, o.path, "dists/css")) self._downloadPage(css, "{0}/{1}/{2}".format(self._dir, o.path, "dists/css"))
# Download HTML # Download HTML
@ -79,9 +79,9 @@ class WPExport:
for j in range(1,int(number_lastpage)): for j in range(1,int(number_lastpage)):
paging = j * 10 paging = j * 10
categorie = urlparse(i).path.split("/") categorie = urlparse(i).path.split("/")
url_paging = "{0}/archives/p{1}-10.html".format(url, paging) url_paging = "{0}/archives/p{1}-10.html".format(self._url, paging)
if len(categorie) > 2: if len(categorie) > 2:
url_paging = "{0}/archives/{1}/p{2}-10.html".format(url, categorie[2], paging) url_paging = "{0}/archives/{1}/p{2}-10.html".format(self._url, categorie[2], paging)
self._logger.info(url_paging) self._logger.info(url_paging)
if url_paging not in webpage: if url_paging not in webpage:
webpage.append(url_paging) webpage.append(url_paging)
@ -105,7 +105,7 @@ class WPExport:
# Private method # Private method
# #
# Create path # Create path
def _mkdirPath(self, path_dir, logger): def _mkdirPath(self, path_dir):
if not os.path.exists(path_dir): if not os.path.exists(path_dir):
makedir = [] makedir = []
pathh = path_dir.split("/") pathh = path_dir.split("/")
@ -126,7 +126,7 @@ class WPExport:
# Get Css and JS # Get Css and JS
def _getScriptCss(self, js, css): def _getScriptCss(self, js, css):
try: try:
page = requests.get(url) page = requests.get(self._url)
except Exception as err: except Exception as err:
self._logger.error("Connection error : {0}".format(err)) self._logger.error("Connection error : {0}".format(err))
exit(1) exit(1)
@ -139,7 +139,7 @@ class WPExport:
src = anchor.get("src", "/") src = anchor.get("src", "/")
if src != "/": if src != "/":
try: try:
u = urlparse(url) u = urlparse(self._url)
o = urlparse(src) o = urlparse(src)
except Exception as err: except Exception as err:
self._logger.error("parsing error : {0}".format(err)) self._logger.error("parsing error : {0}".format(err))
@ -156,7 +156,7 @@ class WPExport:
href = anchor.get("href", "/") href = anchor.get("href", "/")
if href != "/": if href != "/":
try: try:
u = urlparse(url) u = urlparse(self._url)
o = urlparse(href) o = urlparse(href)
except Exception as err: except Exception as err:
self._logger.error("parsing error : {0}".format(err)) self._logger.error("parsing error : {0}".format(err))

5
import_export_canalblog.py
View File

@ -1,8 +1,9 @@
#!/usr/bin/python3 #!/usr/bin/python3
from requests.auth import HTTPBasicAuth from requests.auth import HTTPBasicAuth
from getpass import getpass from getpass import getpass
from urllib.parse import urlparse
import argparse, logging import argparse, logging
import WPImport import WPImport, WPExport
if __name__ == '__main__': if __name__ == '__main__':
@ -82,7 +83,7 @@ if __name__ == '__main__':
except Exception as err: except Exception as err:
logger.error("parsing error : {0}".format(err)) logger.error("parsing error : {0}".format(err))
exit(1) exit(1)
exportWp = WPExport.WPExport(url, logger, args.parser, args.dir) exportWp = WPExport.WPExport(url, logger, args.parser, args.directory)
if args.js is False: if args.js is False:
exportWp.downloadJs() exportWp.downloadJs()