v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

add thread for directory import

Browse Source
This commit is contained in:
Valentin CZERYBA 2023-04-28 23:37:13 +02:00
parent 5a4bdbb420
commit 7a1286c4e2
2 changed files with 33 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
import_export_canalblog.py
View File

@ -57,9 +57,22 @@ def importUrl(name_thread, max_thread, canalblog, logger, parser, wordpress, bas
importWp.fromUrl(webpage)
def importDirectory(name_thread, max_thread, directory, logger, parser, wordpress, basic):
def importDirectory(name_thread, max_thread, directory, logger, parser, wordpress, basic, serial):
name = "Thread-{0}".format(int(name_thread) + 1)
importWp = WPimport(name=name, basic=basic, wordpress=wordpress, logger=logger, parser=parser)
directory = args.directory.split(",")
if serial is False:
for i in wordpress:
importWp = WPimport(name=name, basic=basic, wordpress=i, logger=logger, parser=parser)
for j in directory:
importWp.fromDirectory(j)
else:
if len(directory) != len(wordpress):
logger.error("{0} : Error : Number directory is differant than wordpress".format(name))
exit(1)
for i in range(0, len(wordpress)-1):
importWp = WPimport(name=name, basic=basic, wordpress=wordpress[i], logger=logger, parser=parser)
importWp.fromDirectory(directory[i])
@ -133,22 +146,17 @@ if __name__ == '__main__':
if len(args.file) > 0:
for i in wordpress:
importWp.setUrl(i)
importWp.fromFile(args.file.split(","))
importWp.fromFile(files=args.file.split(","))
exit(0)
if len(args.directory) > 0:
directory = args.directory.split(",")
if args.serial is False:
for i in wordpress:
importWp.setUrl(i)
for j in directory:
importWp.fromDirectory(j)
else:
if len(directory) != len(wordpress):
logger.error("ERREUR : Le nombre de dossier n'est pas equivalent au nombre d'URL wordpress")
exit(1)
for i in range(0, len(wordpress)-1):
importWp.setUrl(wordpress[i])
importWp.fromDirectory(directory[i])
try:
with futures.ThreadPoolExecutor(max_workers=int(args.parallel)) as ex:
wait_for = [
ex.submit(importDi, i, int(args.parallel), args.directory, logger, args.parser, args.wordpress, basic, args.serial)
for i in range(0, int(args.parallel))
]
except Exception as err:
logger.error("Threading error : {0}".format(err))
exit(0)
if len(args.canalblog) > 0:
try:

15
lib/WPImport.py
View File

@ -48,18 +48,21 @@ class WPimport:
self._logger.debug("{0} : {1}".format(self._name, r.content))
def fromDirectory(self, directory):
def fromDirectory(self, directory="", number_thread=1, max_thread=1):
directory = "{0}/archives".format(directory)
directories = self._getDirectories([], "{0}".format(directory))
files = self._getFiles(directories)
self.fromFile(files)
def fromFile(self, files):
for file in files:
if os.path.exists(file):
self._logger.info("{0} : File is being processed : {1}".format(self._name, file))
with open(file, 'r') as f:
def fromFile(self, files=[], number_thread=1, max_thread=1):
divFiles = int(len(files) / max_thread)
currentRangeFiles = int(divFiles * number_thread)
firstRange = int(currentRangeFiles - divFiles)
for i in range(firstRange, currentRangeFiles):
if os.path.exists(files[i]):
self._logger.info("{0} : File is being processed : {1}".format(self._name, files[i]))
with open(files[i], 'r') as f:
content = f.read()
soup = BeautifulSoup(content, self._parser)
articlebody = soup.find_all("div", class_="articlebody")