v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

add public method from URL

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-06-30 23:52:56 +02:00
parent d10867a983
commit 4b6b06aade
1 changed files with 29 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
lib/WPMenu.py
View File

@ -8,17 +8,21 @@ from requests.packages.urllib3.util.retry import Retry
class WPMenu: class WPMenu:
# Constructor # Constructor
def __init__(self, name="Thread-0", basic=None, wordpress="", logger=None, parser="html.parser", ssl_wordpress=True): def __init__(self, name="Thread-0", basic=None, canalblog="", wordpress="", logger=None, parser="html.parser", ssl_canalblog=True, ssl_wordpress=True):
self._name = name self._name = name
self._basic = basic self._basic = basic
self._canalblog = canalblog
self._wordpress = wordpress self._wordpress = wordpress
self._logger = logger self._logger = logger
self._parser = parser self._parser = parser
self._headers_json = {'Content-Type': 'application/json; charset=utf-8', 'Accept':'application/json'} self._headers_json = {'Content-Type': 'application/json; charset=utf-8', 'Accept':'application/json'}
self._protocol = "https" self._protocol_wordpress, self._protocol_canalblog = "https"
self._directory = "backup" self._directory = "backup"
if ssl_wordpress is False: if ssl_wordpress is False:
self._protocol = "http" self._protocol_wordpress = "http"
if ssl_canalblog is False:
self._protocol_canalblog = "http"
self._request = requests.Session() self._request = requests.Session()
retries = Retry(connect=10, read=10, redirect=5, retries = Retry(connect=10, read=10, redirect=5,
@ -29,3 +33,24 @@ class WPMenu:
# Destructor # Destructor
def __del__(self): def __del__(self):
print("{0} : Import finished for {1}".format(self._name, self._wordpress)) print("{0} : Import finished for {1}".format(self._name, self._wordpress))
# Public method
## Get from URL
def fromUrl(self):
try:
o = urlparse(self._canalblog)
o = o._replace(scheme=self._protocol_canalblog)
i = o.geturl().replace(":///", "://")
page = self._request.get(i)
if page.status_code == 200:
soup = BeautifulSoup(page.text, self._parser)
else:
self._logger.error("{0} : index didn't get due status code : {1}".format(self._name, page.status_code))
self._logger.debug("{0} : {1}".format(self._name, page.content))
except ConnectionError as err:
self._logger.error("{0} : Connection error for get url {1} : {2}".format(self._name, webpage[i], err))
exit(1)
except Exception as err:
self._logger.error("{0} : Exception error for get url {1} : {2}".format(self._name, webpage[i], err))