v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

change WIP test

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-06-11 20:24:22 +02:00
parent 16368c13bb
commit 9e7e1b27fd
1 changed files with 25 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
lib/WPChange.py
View File

@ -21,14 +21,16 @@ class WPChange:
def fromFile(self, files=[], number_thread=1, max_thread=1): def fromFile(self, files=[], number_thread=1, max_thread=1):
divFiles = int(len(files) / max_thread) divFiles = int(len(files) / max_thread)
currentRangeFiles = int(divFiles * (number_thread+1)) currentRangeFiles = int(divFiles * (number_thread))
firstRange = int(currentRangeFiles - divFiles) firstRange = int(currentRangeFiles - divFiles)
self._logger.debug("{0} : index : {1}".format(self._name,number_thread)) self._logger.debug("{0} : index : {1}".format(self._name,number_thread))
self._logger.debug("{0} : first range : {1}".format(self._name,firstRange)) self._logger.debug("{0} : first range : {1}".format(self._name,firstRange))
self._logger.debug("{0} : last range : {1}".format(self._name,currentRangeFiles)) self._logger.debug("{0} : last range : {1}".format(self._name,currentRangeFiles))
for i in range(firstRange, currentRangeFiles): for i in range(firstRange, currentRangeFiles):
if os.path.exists(files[i]): if os.path.exists(files[i]):
self._logger.info("{0} : ({1}/{2}) File is being processed : {3}".format(self._name, i+1, currentRangeFiles + 1, files[i])) self._logger.info("{0} : ({1}/{2}) File is being processed : {3}".format(self._name, i+1, currentRangeFiles + 1, files[i]))
self._change(files[i]) self._change(files[i])
@ -72,12 +74,27 @@ class WPChange:
## Change path img file ## Change path img file
def _change(self, file): def _change(self, file):
with open(file, 'r') as f: try:
content = f.read() with open(file, 'r') as f:
soup = BeautifulSoup(content, self._parser) content = f.read()
img = soup.find_all("img") soup = BeautifulSoup(content, self._parser)
for i in img: img = soup.find_all("img")
src = i.get("src") for i in img:
self._logger.info(src) src = i.get("src")
o = urlparse(src)
if len(o.netloc) > 0:
self._logger.info("{0} : Change source {1} /dists/img/{2}/{3}".format(self._name, src, o.netloc, o.path))
content = content.replace(src, "/dists/img/{0}/{1}".format(o.netloc, o.path))
try:
with open(file, "w") as f:
self._logger.info("{0} : File write : {1}".format(self._name, file))
f.write(content)
except Exception as ex:
self._logger.error("{0} : Error for write file {1} : {2}".format(self._name, file, ex))
except Exception as ex:
self._logger.error("{0} : Error for read file {1} : {2}".format(self._name, file, ex))