From 193b0e6ef72e36ef0908189917be529e86b774e6 Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Tue, 27 Jun 2023 14:37:45 +0200 Subject: [PATCH 1/4] add tmp files wip --- lib/WPImport.py | 60 +++++++++++++++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/lib/WPImport.py b/lib/WPImport.py index 432d50d..bad58cb 100644 --- a/lib/WPImport.py +++ b/lib/WPImport.py @@ -74,22 +74,45 @@ class WPimport: self._logger.error("{0} : Read file json from tmp : {1}".format(self._name, ex)) - def fromDirectory(self, directory="", number_thread=1, max_thread=1): + def fromDirectory(self, directory="", number_thread=1, max_thread=1, revert=False): self._directory = directory directory = "{0}/archives".format(directory) directories = self._getDirectories([], "{0}".format(directory)) if len(directories) > 0: files = self._getFiles(directories) - self.fromFile(files=files, number_thread=number_thread, max_thread=max_thread) + if args.revert is False: + self._tmpFiles(files=files, number_thread=number_thread, max_thread=max_thread) + self.fromFile() else: self._logger.error("{0} : No files for {1}".format(self._name, directory)) - + + def fromFile(self, files=[]): + try: + with open("{0}/{1}.json".format(self._tmp, self._name)) as file: + files = json.loads(file.read()) + self._logger.debug("{0} : size of webpage : {1}".format(self._name, len(webpage))) + for i in range(0, len(files)): + if os.path.exists(files[i]): + self._logger.info("{0} : ({1}/{2}) File is being processed : {3}".format(self._name, i+1, currentRangeFiles + 1, files[i])) + with open(files[i], 'r') as f: + content = f.read() + self._logger.debug("{0} : Size of article : {1}".format(self._name, len(content))) + soup = BeautifulSoup(content, self._parser) + articlebody = soup.find_all("div", class_="articlebody") + self._logger.debug("{0} : Number of article : {1}".format(self._name, len(articlebody))) + if len(articlebody) > 0: + self._addOrUpdatePost(soup) + else: + self._addOrUpdateFeaturedMedia(soup) + except Exception as ex: + self._logger.error("{0} : Read file json from tmp : {1}".format(self._name, ex)) + + + # Private method - - - def fromFile(self, files=[], number_thread=1, max_thread=1): + def _tmpFiles(self, number_thread=1, max_thread=1): divFiles = int(len(files) / max_thread) currentRangeFiles = int(divFiles * (number_thread+1)) firstRange = int(currentRangeFiles - divFiles) @@ -97,23 +120,16 @@ class WPimport: self._logger.debug("{0} : first range : {1}".format(self._name,firstRange)) self._logger.debug("{0} : last range : {1}".format(self._name,currentRangeFiles)) - + webpage = [] for i in range(firstRange, currentRangeFiles): - if os.path.exists(files[i]): - self._logger.info("{0} : ({1}/{2}) File is being processed : {3}".format(self._name, i+1, currentRangeFiles + 1, files[i])) - with open(files[i], 'r') as f: - content = f.read() - self._logger.debug("{0} : Size of article : {1}".format(self._name, len(content))) - soup = BeautifulSoup(content, self._parser) - articlebody = soup.find_all("div", class_="articlebody") - self._logger.debug("{0} : Number of article : {1}".format(self._name, len(articlebody))) - if len(articlebody) > 0: - self._addOrUpdatePost(soup) - else: - self._addOrUpdateFeaturedMedia(soup) - - - # Private method + webpage.append(files[i]) + + try: + string_webpage = json.dumps(webpage) + open("{0}/{1}.json".format(self._tmp, self._name), "wt").write(string_webpage) + except Exception as ex: + self._logger.error("{0} : Error for writing webpage : {1}".format(self._name, ex)) + ## replace caracter From 55d62cebfb5805edc5e8d3aa952305965735ceda Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Tue, 27 Jun 2023 14:48:48 +0200 Subject: [PATCH 2/4] separate files method --- lib/WPImport.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/lib/WPImport.py b/lib/WPImport.py index bad58cb..dde5f05 100644 --- a/lib/WPImport.py +++ b/lib/WPImport.py @@ -82,12 +82,31 @@ class WPimport: files = self._getFiles(directories) if args.revert is False: self._tmpFiles(files=files, number_thread=number_thread, max_thread=max_thread) - self.fromFile() + self._fromFileTmp() else: self._logger.error("{0} : No files for {1}".format(self._name, directory)) def fromFile(self, files=[]): + for i in range(0, len(files)): + if os.path.exists(files[i]): + self._logger.info("{0} : ({1}/{2}) File is being processed : {3}".format(self._name, i+1, currentRangeFiles + 1, files[i])) + with open(files[i], 'r') as f: + content = f.read() + self._logger.debug("{0} : Size of article : {1}".format(self._name, len(content))) + soup = BeautifulSoup(content, self._parser) + articlebody = soup.find_all("div", class_="articlebody") + self._logger.debug("{0} : Number of article : {1}".format(self._name, len(articlebody))) + if len(articlebody) > 0: + self._addOrUpdatePost(soup) + else: + self._addOrUpdateFeaturedMedia(soup) + + + + # Private method + + def _fromFileTmp(self): try: with open("{0}/{1}.json".format(self._tmp, self._name)) as file: files = json.loads(file.read()) @@ -107,9 +126,6 @@ class WPimport: self._addOrUpdateFeaturedMedia(soup) except Exception as ex: self._logger.error("{0} : Read file json from tmp : {1}".format(self._name, ex)) - - - # Private method def _tmpFiles(self, number_thread=1, max_thread=1): From 9f87f38347f8376204f22d80b12b7e1b69e0abfc Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Wed, 28 Jun 2023 23:03:27 +0200 Subject: [PATCH 3/4] fix file tmp for directory --- import_export_canalblog.py | 31 ++++++++++++++++++------------- lib/WPImport.py | 10 +++++----- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/import_export_canalblog.py b/import_export_canalblog.py index 86ffde7..3b14d59 100644 --- a/import_export_canalblog.py +++ b/import_export_canalblog.py @@ -11,6 +11,18 @@ from lib.WPExport import WPExport from lib.WPRemove import WPRemove from lib.WPChange import WPChange +def errorRevert(logger, revert, tmp): + if revert is True: + files_tmp = glob.glob("{0}/*.json".format(tmp)) + if len(files_tmp) == 0: + logger.error("Error revert, because files not found") + exit(1) + if len(files_tmp) != int(args.parallel): + for file_r in files_tmp: + os.remove(file_r) + logger.error("Error revert, because number files tmp is incompatible with parallel number") + exit(1) + def change(index, number, args, logger): changeWp = WPChange(logger=logger, index_name=index, number_thread=number) changeWp.fromDirectory(args.directory) @@ -106,7 +118,7 @@ def importUrl(name_thread, max_thread, canalblog, logger, parser, wordpress, bas del importWp -def importDirectory(name_thread, max_thread, directory, logger, parser, wordpress, basic, serial, ssl_wordpress, create, update, image): +def importDirectory(name_thread, max_thread, directory, logger, parser, wordpress, basic, serial, ssl_wordpress, create, update, image, revert): name = "Thread-{0}".format(int(name_thread) + 1) directory = directory.split(",") wordpress = wordpress.split(",") @@ -114,7 +126,7 @@ def importDirectory(name_thread, max_thread, directory, logger, parser, wordpres for i in wordpress: importWp = WPimport(name=name, basic=basic, wordpress=i, logger=logger, parser=parser, ssl_wordpress=ssl_wordpress, no_create=create, no_update=update, no_image=image) for j in directory: - importWp.fromDirectory(j, name_thread, max_thread) + importWp.fromDirectory(j, name_thread, max_thread, revert) del importWp else: @@ -123,7 +135,7 @@ def importDirectory(name_thread, max_thread, directory, logger, parser, wordpres exit(1) for i in range(0, len(wordpress)-1): importWp = WPimport(name=name, basic=basic, wordpress=wordpress[i], logger=logger, parser=parser, ssl_wordpress=ssl_wordpress, no_create=create, no_update=update, no_image=image) - importWp.fromDirectory(directory[i]) + importWp.fromDirectory(directory[i], name_thread, max_thread, revert) del importWp @@ -249,8 +261,9 @@ if __name__ == '__main__': with futures.ThreadPoolExecutor(max_workers=int(args.parallel)) as ex: wait_for = [ ex.submit(remove, i, args.parallel, args, basic, logger, ssl_wordpress) for i in range(0, int(args.parallel)) ] wait(wait_for, return_when=ALL_COMPLETED) + errorRevert(logger, args.revert, args.tmp) wait_for = [ - ex.submit(importDirectory, i, int(args.parallel), args.directory, logger, args.parser, args.wordpress, basic, args.serial, ssl_wordpress, args.create, args.update, args.image) + ex.submit(importDirectory, i, int(args.parallel), args.directory, logger, args.parser, args.wordpress, basic, args.serial, ssl_wordpress, args.create, args.update, args.image, args.revert) for i in range(0, int(args.parallel)) ] except Exception as err: @@ -260,15 +273,7 @@ if __name__ == '__main__': with futures.ThreadPoolExecutor(max_workers=int(args.parallel)) as ex: wait_for = [ ex.submit(remove, i, args.parallel, args, basic, logger, ssl_wordpress) for i in range(0, int(args.parallel)) ] wait(wait_for, return_when=ALL_COMPLETED) - if args.revert is True: - files_tmp = glob.glob("{0}/*.json".format(args.tmp)) - if len(files_tmp) == 0: - logger.error("Error revert, because files not found") - exit(1) - if len(files_tmp) != int(args.parallel): - for file_r in files_tmp: - os.remove(file_r) - + errorRevert(logger, args.revert, args.tmp) wait_for = [ ex.submit(importUrl, i, int(args.parallel), args.canalblog, logger, args.parser, args.wordpress, basic, args.serial, ssl_wordpress, ssl_canalblog, args.create, args.update, args.image, args.revert, args.tmp) for i in range(0, int(args.parallel)) diff --git a/lib/WPImport.py b/lib/WPImport.py index dde5f05..9f0767a 100644 --- a/lib/WPImport.py +++ b/lib/WPImport.py @@ -80,7 +80,7 @@ class WPimport: directories = self._getDirectories([], "{0}".format(directory)) if len(directories) > 0: files = self._getFiles(directories) - if args.revert is False: + if revert is False: self._tmpFiles(files=files, number_thread=number_thread, max_thread=max_thread) self._fromFileTmp() else: @@ -90,7 +90,7 @@ class WPimport: def fromFile(self, files=[]): for i in range(0, len(files)): if os.path.exists(files[i]): - self._logger.info("{0} : ({1}/{2}) File is being processed : {3}".format(self._name, i+1, currentRangeFiles + 1, files[i])) + self._logger.info("{0} : ({1}/{2}) File is being processed : {3}".format(self._name, i+1, len(files), files[i])) with open(files[i], 'r') as f: content = f.read() self._logger.debug("{0} : Size of article : {1}".format(self._name, len(content))) @@ -110,10 +110,10 @@ class WPimport: try: with open("{0}/{1}.json".format(self._tmp, self._name)) as file: files = json.loads(file.read()) - self._logger.debug("{0} : size of webpage : {1}".format(self._name, len(webpage))) + self._logger.debug("{0} : size of webpage : {1}".format(self._name, len(files))) for i in range(0, len(files)): if os.path.exists(files[i]): - self._logger.info("{0} : ({1}/{2}) File is being processed : {3}".format(self._name, i+1, currentRangeFiles + 1, files[i])) + self._logger.info("{0} : ({1}/{2}) File is being processed : {3}".format(self._name, i+1, len(files), files[i])) with open(files[i], 'r') as f: content = f.read() self._logger.debug("{0} : Size of article : {1}".format(self._name, len(content))) @@ -128,7 +128,7 @@ class WPimport: self._logger.error("{0} : Read file json from tmp : {1}".format(self._name, ex)) - def _tmpFiles(self, number_thread=1, max_thread=1): + def _tmpFiles(self, files=[], number_thread=1, max_thread=1): divFiles = int(len(files) / max_thread) currentRangeFiles = int(divFiles * (number_thread+1)) firstRange = int(currentRangeFiles - divFiles) From 699cecad4ff178c381f09a16cd01675ce0038ecd Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Wed, 28 Jun 2023 23:28:24 +0200 Subject: [PATCH 4/4] change with tmp files --- import_export_canalblog.py | 9 ++++--- lib/WPChange.py | 53 +++++++++++++++++++++++++++++++++++--- 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/import_export_canalblog.py b/import_export_canalblog.py index 3b14d59..a404d16 100644 --- a/import_export_canalblog.py +++ b/import_export_canalblog.py @@ -23,9 +23,9 @@ def errorRevert(logger, revert, tmp): logger.error("Error revert, because number files tmp is incompatible with parallel number") exit(1) -def change(index, number, args, logger): - changeWp = WPChange(logger=logger, index_name=index, number_thread=number) - changeWp.fromDirectory(args.directory) +def change(index, number, args, logger, tmp, revert): + changeWp = WPChange(logger=logger, index_name=index, number_thread=number, tmp=tmp) + changeWp.fromDirectory(args.directory, revert) del changeWp @@ -333,7 +333,8 @@ if __name__ == '__main__': if len(args.directory) > 0: try: with futures.ThreadPoolExecutor(max_workers=int(args.parallel)) as ex: - wait_for = [ ex.submit(change, i, args.parallel, args, logger) for i in range(0, int(args.parallel)) ] + errorRevert(logger, args.revert, args.tmp) + wait_for = [ ex.submit(change, i, args.parallel, args, logger, args.tmp, args.revert) for i in range(0, int(args.parallel)) ] except Exception as err: logger.error("Thread error for remove : {0}".format(err)) if len(args.file) > 0: diff --git a/lib/WPChange.py b/lib/WPChange.py index 8d2b626..26ac449 100644 --- a/lib/WPChange.py +++ b/lib/WPChange.py @@ -4,11 +4,13 @@ import requests, os, logging, re, json class WPChange: # Constructor - def __init__(self, index_name=1, number_thread=1, logger=None, parser="html.parser"): + def __init__(self, index_name=1, number_thread=1, logger=None, parser="html.parser", tmp="/tmp/import_export_canablog"): self._name = "Thread-{0}".format(index_name) self._logger = logger self._number_thread = number_thread self._parser = parser + self._tmp = tmp + self._index_name = index_name # Destructor def __del__(self): @@ -37,19 +39,62 @@ class WPChange: ## From directory - - def fromDirectory(self, directory="", number_thread=1, max_thread=1): + + + def fromDirectory(self, directory="", revert=False): + self._directory = directory directory = "{0}/archives".format(directory) directories = self._getDirectories([], "{0}".format(directory)) if len(directories) > 0: files = self._getFiles(directories) - self.fromFile(files, number_thread, max_thread) + if revert is False: + self._tmpFiles(files=files, number_thread=self._index_name, max_thread=self._number_thread) + self._fromFileTmp() else: self._logger.error("{0} : No files for {1}".format(self._name, directory)) + + + def fromFile(self, files=[]): + for i in range(0, len(files)): + if os.path.exists(files[i]): + self._logger.info("{0} : ({1}/{2}) File is being processed : {3}".format(self._name, i+1, len(files), files[i])) + self._change(files[i]) # Private method + def _fromFileTmp(self): + try: + with open("{0}/{1}.json".format(self._tmp, self._name)) as file: + files = json.loads(file.read()) + self._logger.debug("{0} : size of webpage : {1}".format(self._name, len(files))) + for i in range(0, len(files)): + if os.path.exists(files[i]): + self._logger.info("{0} : ({1}/{2}) File is being processed : {3}".format(self._name, i+1, len(files), files[i])) + self._change(files[i]) + except Exception as ex: + self._logger.error("{0} : Read file json from tmp : {1}".format(self._name, ex)) + + + def _tmpFiles(self, files=[], number_thread=1, max_thread=1): + print() + divFiles = int(len(files) / int(max_thread)) + currentRangeFiles = int(divFiles * (int(number_thread)+1)) + firstRange = int(currentRangeFiles - divFiles) + self._logger.debug("{0} : index : {1}".format(self._name,number_thread)) + + self._logger.debug("{0} : first range : {1}".format(self._name,firstRange)) + self._logger.debug("{0} : last range : {1}".format(self._name,currentRangeFiles)) + webpage = [] + for i in range(firstRange, currentRangeFiles): + webpage.append(files[i]) + + try: + string_webpage = json.dumps(webpage) + open("{0}/{1}.json".format(self._tmp, self._name), "wt").write(string_webpage) + except Exception as ex: + self._logger.error("{0} : Error for writing webpage : {1}".format(self._name, ex)) + ## Get all files def _getFiles(self, item):