diff --git a/import_export_canalblog.py b/import_export_canalblog.py index 0066654..12e98fc 100644 --- a/import_export_canalblog.py +++ b/import_export_canalblog.py @@ -57,9 +57,22 @@ def importUrl(name_thread, max_thread, canalblog, logger, parser, wordpress, bas importWp.fromUrl(webpage) -def importDirectory(name_thread, max_thread, directory, logger, parser, wordpress, basic): +def importDirectory(name_thread, max_thread, directory, logger, parser, wordpress, basic, serial): name = "Thread-{0}".format(int(name_thread) + 1) - importWp = WPimport(name=name, basic=basic, wordpress=wordpress, logger=logger, parser=parser) + directory = args.directory.split(",") + if serial is False: + for i in wordpress: + importWp = WPimport(name=name, basic=basic, wordpress=i, logger=logger, parser=parser) + for j in directory: + importWp.fromDirectory(j) + + else: + if len(directory) != len(wordpress): + logger.error("{0} : Error : Number directory is differant than wordpress".format(name)) + exit(1) + for i in range(0, len(wordpress)-1): + importWp = WPimport(name=name, basic=basic, wordpress=wordpress[i], logger=logger, parser=parser) + importWp.fromDirectory(directory[i]) @@ -133,22 +146,17 @@ if __name__ == '__main__': if len(args.file) > 0: for i in wordpress: importWp.setUrl(i) - importWp.fromFile(args.file.split(",")) + importWp.fromFile(files=args.file.split(",")) exit(0) if len(args.directory) > 0: - directory = args.directory.split(",") - if args.serial is False: - for i in wordpress: - importWp.setUrl(i) - for j in directory: - importWp.fromDirectory(j) - else: - if len(directory) != len(wordpress): - logger.error("ERREUR : Le nombre de dossier n'est pas equivalent au nombre d'URL wordpress") - exit(1) - for i in range(0, len(wordpress)-1): - importWp.setUrl(wordpress[i]) - importWp.fromDirectory(directory[i]) + try: + with futures.ThreadPoolExecutor(max_workers=int(args.parallel)) as ex: + wait_for = [ + ex.submit(importDi, i, int(args.parallel), args.directory, logger, args.parser, args.wordpress, basic, args.serial) + for i in range(0, int(args.parallel)) + ] + except Exception as err: + logger.error("Threading error : {0}".format(err)) exit(0) if len(args.canalblog) > 0: try: diff --git a/lib/WPImport.py b/lib/WPImport.py index 80a1d7c..42c3b5c 100644 --- a/lib/WPImport.py +++ b/lib/WPImport.py @@ -48,18 +48,21 @@ class WPimport: self._logger.debug("{0} : {1}".format(self._name, r.content)) - def fromDirectory(self, directory): + def fromDirectory(self, directory="", number_thread=1, max_thread=1): directory = "{0}/archives".format(directory) directories = self._getDirectories([], "{0}".format(directory)) files = self._getFiles(directories) self.fromFile(files) - def fromFile(self, files): - for file in files: - if os.path.exists(file): - self._logger.info("{0} : File is being processed : {1}".format(self._name, file)) - with open(file, 'r') as f: + def fromFile(self, files=[], number_thread=1, max_thread=1): + divFiles = int(len(files) / max_thread) + currentRangeFiles = int(divFiles * number_thread) + firstRange = int(currentRangeFiles - divFiles) + for i in range(firstRange, currentRangeFiles): + if os.path.exists(files[i]): + self._logger.info("{0} : File is being processed : {1}".format(self._name, files[i])) + with open(files[i], 'r') as f: content = f.read() soup = BeautifulSoup(content, self._parser) articlebody = soup.find_all("div", class_="articlebody")