v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

WPchange wip

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-06-06 00:22:16 +02:00
parent 3e76892676
commit c631909cb6
1 changed files with 15 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
lib/WPChange.py
View File

@ -4,10 +4,11 @@ import requests, os, logging, re, json
class WPRemove: class WPRemove:
# Constructor # Constructor
def __init__(self, index_name=1, number_thread=1, logger=None): def __init__(self, index_name=1, number_thread=1, logger=None, parser="html.parser"):
self._name = "Thread-{0}".format(index_name) self._name = "Thread-{0}".format(index_name)
self._logger = logger self._logger = logger
self._number_thread = number_thread self._number_thread = number_thread
self._parser = parser
# Destructor # Destructor
def __del__(self): def __del__(self):
@ -17,7 +18,7 @@ class WPRemove:
# Public method # Public method
## from file ## from file
def fromFile(self, files=[], number_thread=1, max_thread=1): def fromFile(self, files=[], number_thread=1, max_thread=1):
divFiles = int(len(files) / max_thread) divFiles = int(len(files) / max_thread)
currentRangeFiles = int(divFiles * (number_thread+1)) currentRangeFiles = int(divFiles * (number_thread+1))
@ -30,17 +31,8 @@ class WPRemove:
for i in range(firstRange, currentRangeFiles): for i in range(firstRange, currentRangeFiles):
if os.path.exists(files[i]): if os.path.exists(files[i]):
self._logger.info("{0} : ({1}/{2}) File is being processed : {3}".format(self._name, i+1, currentRangeFiles + 1, files[i])) self._logger.info("{0} : ({1}/{2}) File is being processed : {3}".format(self._name, i+1, currentRangeFiles + 1, files[i]))
with open(files[i], 'r') as f: self._change(files[i])
content = f.read()
self._logger.debug("{0} : Size of article : {1}".format(self._name, len(content)))
soup = BeautifulSoup(content, self._parser)
articlebody = soup.find_all("div", class_="articlebody")
self._logger.debug("{0} : Number of article : {1}".format(self._name, len(articlebody)))
if len(articlebody) > 0:
self._addOrUpdatePost(soup)
else:
self._addOrUpdateFeaturedMedia(soup)
## From directory ## From directory
@ -76,3 +68,13 @@ class WPRemove:
sub.append("{0}/{1}".format(item, i)) sub.append("{0}/{1}".format(item, i))
subdirectory = self._getDirectories(sub, "{0}/{1}".format(item, i)) subdirectory = self._getDirectories(sub, "{0}/{1}".format(item, i))
return subdirectory return subdirectory
## Change path img file
def _change(self, file):
with open(file, 'r') as f:
content = f.read()
soup = BeautifulSoup(content, self._parser)
img = soup.find_all("img")