v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

test WIP

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-07-01 22:16:35 +02:00
parent f0b4ba5e27
commit 9bbf769b40
2 changed files with 15 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
import_export_canalblog.py
View File

@ -247,7 +247,7 @@ if __name__ == '__main__':
os.makedirs(args.tmp, exist_ok=True) os.makedirs(args.tmp, exist_ok=True)
if args.command == "import" or args.command == "remove" or args.command = "menu": if args.command == "import" or args.command == "remove" or args.command == "menu":
password = args.password password = args.password
if len(args.password) == 0: if len(args.password) == 0:
password = getpass() password = getpass()

23
lib/WPMenu.py
View File

@ -16,19 +16,23 @@ class WPMenu:
self._logger = logger self._logger = logger
self._parser = parser self._parser = parser
self._headers_json = {'Content-Type': 'application/json; charset=utf-8', 'Accept':'application/json'} self._headers_json = {'Content-Type': 'application/json; charset=utf-8', 'Accept':'application/json'}
self._protocol_wordpress, self._protocol_canalblog = "https" self._protocol_wordpress = "https"
self._protocol_canalblog = "https"
self._directory = "backup" self._directory = "backup"
if ssl_wordpress is False: if ssl_wordpress is False:
self._protocol_wordpress = "http" self._protocol_wordpress = "http"
if ssl_canalblog is False: if ssl_canalblog is False:
self._protocol_canalblog = "http" self._protocol_canalblog = "http"
self._request = requests.Session() self._request_canalblog = requests.Session()
self._request_wordpress = requests.Session()
retries = Retry(connect=10, read=10, redirect=5, retries = Retry(connect=10, read=10, redirect=5,
status_forcelist=[429, 500, 502, 503, 504], backoff_factor=2) status_forcelist=[429, 500, 502, 503, 504], backoff_factor=2)
self._request.mount('{0}://'.format(self._protocol), HTTPAdapter(max_retries=retries)) self._request_canalblog.mount('{0}://'.format(self._protocol_canalblog), HTTPAdapter(max_retries=retries))
self._request_wordpress.mount('{0}://'.format(self._protocol_wordpress), HTTPAdapter(max_retries=retries))
# Destructor # Destructor
def __del__(self): def __del__(self):
@ -56,7 +60,7 @@ class WPMenu:
o = urlparse(canalblog) o = urlparse(canalblog)
o = o._replace(scheme=self._protocol_canalblog) o = o._replace(scheme=self._protocol_canalblog)
i = o.geturl().replace(":///", "://") i = o.geturl().replace(":///", "://")
page = self._request.get(i) page = self._request_canalblog.get(i)
if page.status_code == 200: if page.status_code == 200:
self._logger.info("{0} : Page web is being processed : {1}".format(self._name, i)) self._logger.info("{0} : Page web is being processed : {1}".format(self._name, i))
self._menu(page.content) self._menu(page.content)
@ -64,17 +68,18 @@ class WPMenu:
self._logger.error("{0} : index didn't get due status code : {1}".format(self._name, page.status_code)) self._logger.error("{0} : index didn't get due status code : {1}".format(self._name, page.status_code))
self._logger.debug("{0} : {1}".format(self._name, page.content)) self._logger.debug("{0} : {1}".format(self._name, page.content))
except ConnectionError as err: except ConnectionError as err:
self._logger.error("{0} : Connection error for get url {1} : {2}".format(self._name, webpage[i], err)) self._logger.error("{0} : Connection error for get url {1} : {2}".format(self._name, canalblog, err))
exit(1) exit(1)
except Exception as err: except Exception as err:
self._logger.error("{0} : Exception error for get url {1} : {2}".format(self._name, webpage[i], err)) self._logger.error("{0} : Exception error for get url {1} : {2}".format(self._name, canalblog, err))
def _menu(self, content): def _menu(self, content):
soup = BeautifulSoup(page.text, self._parser) soup = BeautifulSoup(content, self._parser)
ul = soup.find_all("ul", id="listsmooth") ul = soup.find_all("ul", id="listsmooth")
for anchor in ul[0].find_all("li"): for anchor in ul[0].find_all("li"):
li = anchor.find_all("li") li = anchor.find_all("li")
for content in li: for child in li:
a = content.find("a") a = child.find("a")
self._logger.info("{0} {1} : {2}".format(self._name, child, a))