v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

Add parallelism

Browse Source
This commit is contained in:
Valentin CZERYBA
2023-04-24 23:15:29 +02:00
parent a39e2200bd
commit 88f258ffba
12 changed files with 172 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
lib/WPExport.py
View File

@@ -11,6 +11,7 @@ class WPExport:
self._logger = logger
self._parser = parser
self._dir = directory
self._name = "Process-0"
self._request = requests.Session()
@@ -22,6 +23,11 @@ class WPExport:
# Public method
# Set name
def setName(self, name):
self._name = "Process-{}".format(int(name) + 1)
# Set URL
def setUrl(self, url):
@@ -55,11 +61,11 @@ class WPExport:
# Get URL
def getUrlPage(self):
def getUrlPage(self, index_thread, max_thread):
try:
page = self._request.get(self._url)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
self._logger.error("{0} : Connection error : {1}".format(self._name, err))
exit(1)
page_url = []
if page.status_code == 200:
@@ -70,8 +76,8 @@ class WPExport:
if href != "#":
page_url.append(href)
else:
self._logger.error("Url did not get due status code : {0}".format(page.status_code))
self._logger.debug(page.content)
self._logger.error("{0} : URL did not get due status code : {1}".format(self._name, page.status_code))
self._logger.debug("{0} : {1}".format(self._name, page.content))
webpage = []
@@ -79,10 +85,10 @@ class WPExport:
try:
page = self._request.get(i)
except Exception as err:
self._logger.error("Connection error : {0}".format(err))
self._logger.error("{0} : Connection error : {1}".format(self._name, err))
exit(1)
if page.status_code == 200:
self._logger.info("page : {0}".format(i))
self._logger.info("{0} : page : {1}".format(self._name, i))
if i not in webpage:
webpage.append(i)
soup = BeautifulSoup(page.text, self._parser)
@@ -94,13 +100,22 @@ class WPExport:
element_lastpage = lastpage.split("/")[len(lastpage.split("/"))-1]
number_page = element_lastpage.split("-")[0].split("p")[1]
number_lastpage = int(number_page) / 10
for j in range(1,int(number_lastpage)):
setPageDivided = int(number_lastpage) / max_thread
setPagePart = setPageDivided * (index_thread + 1)
firstPagePart = (setPagePart - setPageDivided) + 1
self._logger.debug("{0} : Total page : {1}".format(self._name,int(number_lastpage)))
self._logger.debug("{0} : First range : {1}".format(self._name, int(firstPagePart)))
self._logger.debug("{0} : Last range : {1}".format(self._name, int(setPagePart)))
for j in range(int(firstPagePart),int(setPagePart)):
paging = j * 10
categorie = urlparse(i).path.split("/")
url_paging = "{0}/archives/p{1}-10.html".format(self._url, paging)
if len(categorie) > 2:
url_paging = "{0}/archives/{1}/p{2}-10.html".format(self._url, categorie[2], paging)
self._logger.info(url_paging)
self._logger.info("{0} : {1}".format(self._name, url_paging))
if url_paging not in webpage:
webpage.append(url_paging)
page = self._request.get(url_paging)
@@ -118,7 +133,7 @@ class WPExport:
exit(1)
webpage.append(o)
else:
self._logger.error("web didn't get due status code : {0}".format(page.status_code))
self._logger.error("{0} : web didn't get due status code : {1}".format(self._name, page.status_code))
self._logger.debug(page.content)
return webpage