From 7dace5bdb7d10250c438c6f8926a6139345634e3 Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Mon, 19 Jun 2023 23:58:59 +0200 Subject: [PATCH 1/8] add file tmp --- import_export_canalblog.py | 25 ++++++++++++++++--------- lib/WPExport.py | 4 ++-- lib/WPImport.py | 3 ++- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/import_export_canalblog.py b/import_export_canalblog.py index 418df05..a5d60ed 100644 --- a/import_export_canalblog.py +++ b/import_export_canalblog.py @@ -5,7 +5,7 @@ from urllib.parse import urlparse from concurrent import futures from concurrent.futures import as_completed, wait, ALL_COMPLETED -import argparse, logging, threading +import argparse, logging, threading, os, glob from lib.WPImport import WPimport from lib.WPExport import WPExport from lib.WPRemove import WPRemove @@ -40,9 +40,10 @@ def remove(index, number, args, basic, logger, ssl_wordpress): del removeWp -def download(name_thread, max_thread, url, logger, parser, directory, html, img, ssl_canalblog): +def download(name_thread, max_thread, url, logger, parser, directory, html, img, ssl_canalblog, revert, tmp): exportWp = WPExport(name="Thread-{0}".format(int(name_thread) + 1), url=url, logger=logger, parser=parser, directory=directory, ssl_canalblog=ssl_canalblog) - webpage = exportWp.getUrlPage(name_thread, max_thread) + if not revert: + webpage = exportWp.getUrlPage(name_thread, max_thread) for i in ["article", "page"]: for j in ["publications", "principal"]: if html is False: @@ -54,7 +55,7 @@ def download(name_thread, max_thread, url, logger, parser, directory, html, img, -def importUrl(name_thread, max_thread, canalblog, logger, parser, wordpress, basic, serial, ssl_wordpress, ssl_canalblog, create, update, image): +def importUrl(name_thread, max_thread, canalblog, logger, parser, wordpress, basic, serial, ssl_wordpress, ssl_canalblog, create, update, image, revert, tmp): canalblog = canalblog.split(",") wordpress = wordpress.split(",") name = "Thread-{0}".format(int(name_thread) + 1) @@ -71,7 +72,8 @@ def importUrl(name_thread, max_thread, canalblog, logger, parser, wordpress, bas logger.error("{0} : parsing error : {1}".format(name, err)) exit(1) exportWp = WPExport(name="Thread-{0}".format(int(name_thread) + 1), url=url, logger=logger, parser=parser, ssl_canalblog=ssl_canalblog) - webpage = exportWp.getUrlPage(name_thread, max_thread) + if not revert: + webpage = exportWp.getUrlPage(name_thread, max_thread) del exportWp for j in wordpress: importWp = WPimport(name=name, basic=basic, wordpress=j, logger=logger, parser=parser, ssl_wordpress=ssl_wordpress, no_create=create, no_update=update, no_image=image) @@ -93,7 +95,8 @@ def importUrl(name_thread, max_thread, canalblog, logger, parser, wordpress, bas logger.error("parsing error : {0}".format(err)) exit(1) exportWp = WPExport(name=name, url=url, logger=logger, parser=parser, ssl_canalblog=ssl_canalblog) - webpage = exportWp.getUrlPage(name_thread, max_thread) + if not revert: + webpage = exportWp.getUrlPage(name_thread, max_thread) del exportWp importWp = WPimport(name=name, basic=basic, wordpress=wordpress[i], logger=logger, parser=parser, ssl_wordpress=ssl_wordpress, no_create=create, no_update=update, no_image=image) @@ -127,6 +130,7 @@ def importDirectory(name_thread, max_thread, directory, logger, parser, wordpres if __name__ == '__main__': + TMP = "/tmp/import_export_canablog" parser = argparse.ArgumentParser() parser.add_argument("--debug", help="Verbosity", action="store_true") parser.add_argument("--logfile", help="Log file", default="") @@ -134,6 +138,7 @@ if __name__ == '__main__': parser.add_argument("--parser", help="Parser content", default="html.parser") parser.add_argument("--parallel", help="Define number thread (default : 1)", default=1) parser.add_argument("--no-ssl", help="No ssl for canalblog and/or wordpress (example wordpress,canalblog)", dest="ssl", default="") + parser.add_argument("--revert", help="Restart a work from stopping work", action="store_true") subparsers = parser.add_subparsers(dest="command") @@ -188,7 +193,9 @@ if __name__ == '__main__': help="File") - + if not os.path.exists(TMP): + os.mkdir(TMP) + args = parser.parse_args() logger = logging.getLogger('import export canalblog') @@ -256,7 +263,7 @@ if __name__ == '__main__': wait_for = [ ex.submit(remove, i, args.parallel, args, basic, logger, ssl_wordpress) for i in range(0, int(args.parallel)) ] wait(wait_for, return_when=ALL_COMPLETED) wait_for = [ - ex.submit(importUrl, i, int(args.parallel), args.canalblog, logger, args.parser, args.wordpress, basic, args.serial, ssl_wordpress, ssl_canalblog, args.create, args.update, args.image) + ex.submit(importUrl, i, int(args.parallel), args.canalblog, logger, args.parser, args.wordpress, basic, args.serial, ssl_wordpress, ssl_canalblog, args.create, args.update, args.image, args.revert, TMP) for i in range(0, int(args.parallel)) ] @@ -294,7 +301,7 @@ if __name__ == '__main__': try: with futures.ThreadPoolExecutor(max_workers=int(args.parallel)) as ex: wait_for = [ - ex.submit(download, i, int(args.parallel), url, logger, args.parser, args.directory, args.html, args.img, ssl_canalblog) + ex.submit(download, i, int(args.parallel), url, logger, args.parser, args.directory, args.html, args.img, ssl_canalblog, args.revert, TMP) for i in range(0, int(args.parallel)) ] except Exception as err: diff --git a/lib/WPExport.py b/lib/WPExport.py index f0775e7..5577ddb 100644 --- a/lib/WPExport.py +++ b/lib/WPExport.py @@ -6,7 +6,7 @@ from requests.adapters import HTTPAdapter from requests.packages.urllib3.util.retry import Retry class WPExport: - def __init__(self, name = "Thread-0", url = "", logger = None, parser = "html.parser", directory = "backup", ssl_canalblog=True): + def __init__(self, name = "Thread-0", url = "", logger = None, parser = "html.parser", directory = "backup", ssl_canalblog=True, tmp="/tmp/import_export_canablog"): self._url = url self._logger = logger self._parser = parser @@ -22,7 +22,7 @@ class WPExport: status_forcelist=[429, 500, 502, 503, 504], backoff_factor=2) self._request.mount('{0}://'.format(self._protocol), HTTPAdapter(max_retries=retries)) - + self._tmp = tmp # Destructor def __del__(self): diff --git a/lib/WPImport.py b/lib/WPImport.py index 42d6596..bc82824 100644 --- a/lib/WPImport.py +++ b/lib/WPImport.py @@ -8,7 +8,7 @@ from requests.packages.urllib3.util.retry import Retry class WPimport: # Constructor - def __init__(self, name="Thread-0", basic=None, wordpress="", logger=None, parser="html.parser", ssl_wordpress=True, no_create=False, no_update=False, no_image=False): + def __init__(self, name="Thread-0", basic=None, wordpress="", logger=None, parser="html.parser", ssl_wordpress=True, no_create=False, no_update=False, no_image=False, tmp="/tmp/import_export_canablog"): self._name = name self._basic = basic self._wordpress = wordpress @@ -28,6 +28,7 @@ class WPimport: self._no_create = no_create self._no_update = no_update self._no_image = no_image + self._tmp = tmp # Destructor def __del__(self): From 35ff22d463226a9ecb4663807fe1f55ff740ffaf Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Tue, 20 Jun 2023 00:17:38 +0200 Subject: [PATCH 2/8] change parameter for webpage --- import_export_canalblog.py | 20 ++++++++++---------- lib/WPExport.py | 8 ++++---- lib/WPImport.py | 4 ++-- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/import_export_canalblog.py b/import_export_canalblog.py index a5d60ed..fdca77d 100644 --- a/import_export_canalblog.py +++ b/import_export_canalblog.py @@ -43,15 +43,15 @@ def remove(index, number, args, basic, logger, ssl_wordpress): def download(name_thread, max_thread, url, logger, parser, directory, html, img, ssl_canalblog, revert, tmp): exportWp = WPExport(name="Thread-{0}".format(int(name_thread) + 1), url=url, logger=logger, parser=parser, directory=directory, ssl_canalblog=ssl_canalblog) if not revert: - webpage = exportWp.getUrlPage(name_thread, max_thread) + exportWp.getUrlPage(name_thread, max_thread) for i in ["article", "page"]: for j in ["publications", "principal"]: if html is False: - exportWp.downloadHTML(webpage[j][i]) + exportWp.downloadHTML(j, i) if img is False: - exportWp.downloadImg(webpage[j][i]) - + exportWp.downloadImg(j, i) + del exportWp @@ -71,15 +71,15 @@ def importUrl(name_thread, max_thread, canalblog, logger, parser, wordpress, bas except Exception as err: logger.error("{0} : parsing error : {1}".format(name, err)) exit(1) - exportWp = WPExport(name="Thread-{0}".format(int(name_thread) + 1), url=url, logger=logger, parser=parser, ssl_canalblog=ssl_canalblog) + exportWp = WPExport(name="Thread-{0}".format(int(name_thread) + 1), url=url, logger=logger, parser=parser, ssl_canalblog=ssl_canalblog, tmp=tmp) if not revert: - webpage = exportWp.getUrlPage(name_thread, max_thread) + exportWp.getUrlPage(name_thread, max_thread) del exportWp for j in wordpress: - importWp = WPimport(name=name, basic=basic, wordpress=j, logger=logger, parser=parser, ssl_wordpress=ssl_wordpress, no_create=create, no_update=update, no_image=image) + importWp = WPimport(name=name, basic=basic, wordpress=j, logger=logger, parser=parser, ssl_wordpress=ssl_wordpress, no_create=create, no_update=update, no_image=image, tmp=tmp) for k in ["article", "page"]: for l in ["publications", "principal"]: - importWp.fromUrl(webpage[l][k]) + importWp.fromUrl(l, k) del importWp else: @@ -96,9 +96,9 @@ def importUrl(name_thread, max_thread, canalblog, logger, parser, wordpress, bas exit(1) exportWp = WPExport(name=name, url=url, logger=logger, parser=parser, ssl_canalblog=ssl_canalblog) if not revert: - webpage = exportWp.getUrlPage(name_thread, max_thread) + exportWp.getUrlPage(name_thread, max_thread) del exportWp - importWp = WPimport(name=name, basic=basic, wordpress=wordpress[i], logger=logger, parser=parser, ssl_wordpress=ssl_wordpress, no_create=create, no_update=update, no_image=image) + importWp = WPimport(name=name, basic=basic, wordpress=wordpress[i], logger=logger, parser=parser, ssl_wordpress=ssl_wordpress, no_create=create, no_update=update, no_image=image, tmp=tmp) for k in ["article", "page"]: for l in ["publications", "principal"]: diff --git a/lib/WPExport.py b/lib/WPExport.py index 5577ddb..814f1f9 100644 --- a/lib/WPExport.py +++ b/lib/WPExport.py @@ -57,13 +57,13 @@ class WPExport: # Download HTML - def downloadHTML(self, webpage): - self._downloadPage(webpage, self._dir) + def downloadHTML(self, first, second): + self._downloadPage(webpage[first][second], self._dir) # Download Image - def downloadImg(self, webpage): - page_src = self._getImg(webpage) + def downloadImg(self, first, second): + page_src = self._getImg(webpage[first][second]) o = urlparse(self._url) self._downloadPage(page_src, "{0}/{1}/{2}".format(self._dir, o.path, "img")) diff --git a/lib/WPImport.py b/lib/WPImport.py index bc82824..671ba83 100644 --- a/lib/WPImport.py +++ b/lib/WPImport.py @@ -39,8 +39,8 @@ class WPimport: def setUrl(self, wordpress): self._wordpress = wordpress - def fromUrl(self, webpage): - for i in range(0, len(webpage)): + def fromUrl(self, first, second): + for i in range(0, len(webpage[first][second])): try: r = self._request.get(webpage[i]) if r.status_code == 200: From 0eab1d885bf1abc3b4be827ed17f2f548ad6909f Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Tue, 20 Jun 2023 21:38:39 +0200 Subject: [PATCH 3/8] add open file tmp --- lib/WPExport.py | 9 ++++++++- lib/WPImport.py | 49 +++++++++++++++++++++++++++++++------------------ 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/lib/WPExport.py b/lib/WPExport.py index 814f1f9..df44100 100644 --- a/lib/WPExport.py +++ b/lib/WPExport.py @@ -161,7 +161,14 @@ class WPExport: except Exception as err: self._logger.error("{0} : Exception error : {1}".format(self._name, err)) exit(1) - return webpage + try: + string_webpage = json.dumps(webpage) + open("{0}/{1}.json".format(self._tmp, self._name), "wt").write(string_webpage) + except Exception as ex: + self._logger.error("{0} : Error for writing webpage : {1}".format(self._name, ex)) + + + diff --git a/lib/WPImport.py b/lib/WPImport.py index 671ba83..c600990 100644 --- a/lib/WPImport.py +++ b/lib/WPImport.py @@ -40,25 +40,38 @@ class WPimport: self._wordpress = wordpress def fromUrl(self, first, second): - for i in range(0, len(webpage[first][second])): - try: - r = self._request.get(webpage[i]) - if r.status_code == 200: - self._logger.info("{0} : ({1}/{2}) : Page is importing : {3}".format(self._name, i+1, len(webpage), webpage[i])) - soup = BeautifulSoup(r.content, self._parser) - articlebody = soup.find_all("div", class_="articlebody") - if len(articlebody) > 0: - self._addOrUpdatePost(soup) + try: + content_file = open("{0}/{1}.json".format(self._name, self._tmp)) + webpage_content = json.loads(content_file) + webpage = webpage_content[first][second] + for i in range(0, len(webpage)): + try: + r = self._request.get(webpage[i]) + if r.status_code == 200: + self._logger.info("{0} : ({1}/{2}) : Page is importing : {3}".format(self._name, i+1, len(webpage), webpage[i])) + soup = BeautifulSoup(r.content, self._parser) + articlebody = soup.find_all("div", class_="articlebody") + if len(articlebody) > 0: + self._addOrUpdatePost(soup) + else: + self._addOrUpdateFeaturedMedia(soup) + del webpage_content[first][second][i] + open("{0}/{1}.json".format(self._tmp, self._name), "wt").write(webpage_content) else: - self._addOrUpdateFeaturedMedia(soup) - else: - self._logger.error("{0} : Connection error for get url {1} with status code : {2}".format(self._name, webpage[i], r.status_code)) - self._logger.debug("{0} : {1}".format(self._name, r.content)) - except ConnectionError as err: - self._logger.error("{0} : Connection error for get url {1} : {2}".format(self._name, webpage[i], err)) - exit(1) - except Exception as err: - self._logger.error("{0} : Exception error for get url {1} : {2}".format(self._name, webpage[i], err)) + self._logger.error("{0} : Connection error for get url {1} with status code : {2}".format(self._name, webpage[i], r.status_code)) + self._logger.debug("{0} : {1}".format(self._name, r.content)) + except ConnectionError as err: + self._logger.error("{0} : Connection error for get url {1} : {2}".format(self._name, webpage[i], err)) + exit(1) + except IOError as err: + self._logger.error("{0} : Connection error for IO url {1} : {2}".format(self._name, webpage[i], err)) + exit(1) + except Exception as err: + self._logger.error("{0} : Exception error for get url {1} : {2}".format(self._name, webpage[i], err)) + + + except Exception as ex: + self._logger.error("{0} : Read file json from tmp : {1}".format(self._name, ex)) def fromDirectory(self, directory="", number_thread=1, max_thread=1): From d1b6e8048acabf5fe8797807d4d78ef8ce09b6c1 Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Sun, 25 Jun 2023 21:16:05 +0200 Subject: [PATCH 4/8] add revert files json --- import_export_canalblog.py | 8 ++++---- lib/WPImport.py | 26 ++++++++++++++++++-------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/import_export_canalblog.py b/import_export_canalblog.py index fdca77d..9594a8b 100644 --- a/import_export_canalblog.py +++ b/import_export_canalblog.py @@ -107,13 +107,13 @@ def importUrl(name_thread, max_thread, canalblog, logger, parser, wordpress, bas del importWp -def importDirectory(name_thread, max_thread, directory, logger, parser, wordpress, basic, serial, ssl_wordpress, create, update, image): +def importDirectory(name_thread, max_thread, directory, logger, parser, wordpress, basic, serial, ssl_wordpress, create, update, image, tmp, revert): name = "Thread-{0}".format(int(name_thread) + 1) directory = directory.split(",") wordpress = wordpress.split(",") if serial is False: for i in wordpress: - importWp = WPimport(name=name, basic=basic, wordpress=i, logger=logger, parser=parser, ssl_wordpress=ssl_wordpress, no_create=create, no_update=update, no_image=image) + importWp = WPimport(name=name, basic=basic, wordpress=i, logger=logger, parser=parser, ssl_wordpress=ssl_wordpress, no_create=create, no_update=update, no_image=image, tmp=tmp, revert=revert) for j in directory: importWp.fromDirectory(j, name_thread, max_thread) del importWp @@ -123,7 +123,7 @@ def importDirectory(name_thread, max_thread, directory, logger, parser, wordpres logger.error("{0} : Error : Number directory is differant than wordpress".format(name)) exit(1) for i in range(0, len(wordpress)-1): - importWp = WPimport(name=name, basic=basic, wordpress=wordpress[i], logger=logger, parser=parser, ssl_wordpress=ssl_wordpress, no_create=create, no_update=update, no_image=image) + importWp = WPimport(name=name, basic=basic, wordpress=wordpress[i], logger=logger, parser=parser, ssl_wordpress=ssl_wordpress, no_create=create, no_update=update, no_image=image, tmp=tmp, revert=revert) importWp.fromDirectory(directory[i]) del importWp @@ -252,7 +252,7 @@ if __name__ == '__main__': wait_for = [ ex.submit(remove, i, args.parallel, args, basic, logger, ssl_wordpress) for i in range(0, int(args.parallel)) ] wait(wait_for, return_when=ALL_COMPLETED) wait_for = [ - ex.submit(importDirectory, i, int(args.parallel), args.directory, logger, args.parser, args.wordpress, basic, args.serial, ssl_wordpress, args.create, args.update, args.image) + ex.submit(importDirectory, i, int(args.parallel), args.directory, logger, args.parser, args.wordpress, basic, args.serial, ssl_wordpress, args.create, args.update, args.image, TMP, revert) for i in range(0, int(args.parallel)) ] except Exception as err: diff --git a/lib/WPImport.py b/lib/WPImport.py index c600990..315b0c9 100644 --- a/lib/WPImport.py +++ b/lib/WPImport.py @@ -8,7 +8,7 @@ from requests.packages.urllib3.util.retry import Retry class WPimport: # Constructor - def __init__(self, name="Thread-0", basic=None, wordpress="", logger=None, parser="html.parser", ssl_wordpress=True, no_create=False, no_update=False, no_image=False, tmp="/tmp/import_export_canablog"): + def __init__(self, name="Thread-0", basic=None, wordpress="", logger=None, parser="html.parser", ssl_wordpress=True, no_create=False, no_update=False, no_image=False, tmp="/tmp/import_export_canablog", revert=False): self._name = name self._basic = basic self._wordpress = wordpress @@ -29,6 +29,7 @@ class WPimport: self._no_update = no_update self._no_image = no_image self._tmp = tmp + self._revert = revert # Destructor def __del__(self): @@ -75,14 +76,23 @@ class WPimport: def fromDirectory(self, directory="", number_thread=1, max_thread=1): - self._directory = directory - directory = "{0}/archives".format(directory) - directories = self._getDirectories([], "{0}".format(directory)) - if len(directories) > 0: - files = self._getFiles(directories) - self.fromFile(files, number_thread, max_thread) + if self._revert: + self._directory = directory + directory = "{0}/archives".format(directory) + directories = self._getDirectories([], "{0}".format(directory)) + if len(directories) > 0: + files = self._getFiles(directories) + self.fromFile(files, number_thread, max_thread) + else: + self._logger.error("{0} : No files for {1}".format(self._name, directory)) else: - self._logger.error("{0} : No files for {1}".format(self._name, directory)) + try: + files = open("{0}/{1}.json".format(self._name, self._tmp)) + self.fromFile(files, number_thread, max_thread) + except Exception as ex: + self._logger.error("{0} : Read file json from tmp : {1}".format(self._name, ex)) + + def fromFile(self, files=[], number_thread=1, max_thread=1): From 9149a6c5cb51d360078166469a659e3568e74936 Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Mon, 26 Jun 2023 22:44:42 +0200 Subject: [PATCH 5/8] rollback webpage --- import_export_canalblog.py | 6 +++--- lib/WPImport.py | 26 ++++++++++---------------- 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/import_export_canalblog.py b/import_export_canalblog.py index 9594a8b..22e67d8 100644 --- a/import_export_canalblog.py +++ b/import_export_canalblog.py @@ -107,13 +107,13 @@ def importUrl(name_thread, max_thread, canalblog, logger, parser, wordpress, bas del importWp -def importDirectory(name_thread, max_thread, directory, logger, parser, wordpress, basic, serial, ssl_wordpress, create, update, image, tmp, revert): +def importDirectory(name_thread, max_thread, directory, logger, parser, wordpress, basic, serial, ssl_wordpress, create, update, image): name = "Thread-{0}".format(int(name_thread) + 1) directory = directory.split(",") wordpress = wordpress.split(",") if serial is False: for i in wordpress: - importWp = WPimport(name=name, basic=basic, wordpress=i, logger=logger, parser=parser, ssl_wordpress=ssl_wordpress, no_create=create, no_update=update, no_image=image, tmp=tmp, revert=revert) + importWp = WPimport(name=name, basic=basic, wordpress=i, logger=logger, parser=parser, ssl_wordpress=ssl_wordpress, no_create=create, no_update=update, no_image=image) for j in directory: importWp.fromDirectory(j, name_thread, max_thread) del importWp @@ -123,7 +123,7 @@ def importDirectory(name_thread, max_thread, directory, logger, parser, wordpres logger.error("{0} : Error : Number directory is differant than wordpress".format(name)) exit(1) for i in range(0, len(wordpress)-1): - importWp = WPimport(name=name, basic=basic, wordpress=wordpress[i], logger=logger, parser=parser, ssl_wordpress=ssl_wordpress, no_create=create, no_update=update, no_image=image, tmp=tmp, revert=revert) + importWp = WPimport(name=name, basic=basic, wordpress=wordpress[i], logger=logger, parser=parser, ssl_wordpress=ssl_wordpress, no_create=create, no_update=update, no_image=image) importWp.fromDirectory(directory[i]) del importWp diff --git a/lib/WPImport.py b/lib/WPImport.py index 315b0c9..65453da 100644 --- a/lib/WPImport.py +++ b/lib/WPImport.py @@ -8,7 +8,7 @@ from requests.packages.urllib3.util.retry import Retry class WPimport: # Constructor - def __init__(self, name="Thread-0", basic=None, wordpress="", logger=None, parser="html.parser", ssl_wordpress=True, no_create=False, no_update=False, no_image=False, tmp="/tmp/import_export_canablog", revert=False): + def __init__(self, name="Thread-0", basic=None, wordpress="", logger=None, parser="html.parser", ssl_wordpress=True, no_create=False, no_update=False, no_image=False, tmp="/tmp/import_export_canablog"): self._name = name self._basic = basic self._wordpress = wordpress @@ -29,7 +29,6 @@ class WPimport: self._no_update = no_update self._no_image = no_image self._tmp = tmp - self._revert = revert # Destructor def __del__(self): @@ -76,21 +75,16 @@ class WPimport: def fromDirectory(self, directory="", number_thread=1, max_thread=1): - if self._revert: - self._directory = directory - directory = "{0}/archives".format(directory) - directories = self._getDirectories([], "{0}".format(directory)) - if len(directories) > 0: - files = self._getFiles(directories) - self.fromFile(files, number_thread, max_thread) - else: - self._logger.error("{0} : No files for {1}".format(self._name, directory)) + self._directory = directory + directory = "{0}/archives".format(directory) + directories = self._getDirectories([], "{0}".format(directory)) + if len(directories) > 0: + files = self._getFiles(directories) + self.fromFile(files=files, number_thread=number_thread, max_thread=max_thread) else: - try: - files = open("{0}/{1}.json".format(self._name, self._tmp)) - self.fromFile(files, number_thread, max_thread) - except Exception as ex: - self._logger.error("{0} : Read file json from tmp : {1}".format(self._name, ex)) + self._logger.error("{0} : No files for {1}".format(self._name, directory)) + + From 8e0abc40bd065fc6ed7d52fdfe0f001fc001ca37 Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Mon, 26 Jun 2023 23:09:54 +0200 Subject: [PATCH 6/8] check files tmp --- import_export_canalblog.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/import_export_canalblog.py b/import_export_canalblog.py index 22e67d8..a0552ee 100644 --- a/import_export_canalblog.py +++ b/import_export_canalblog.py @@ -54,7 +54,6 @@ def download(name_thread, max_thread, url, logger, parser, directory, html, img, del exportWp - def importUrl(name_thread, max_thread, canalblog, logger, parser, wordpress, basic, serial, ssl_wordpress, ssl_canalblog, create, update, image, revert, tmp): canalblog = canalblog.split(",") wordpress = wordpress.split(",") @@ -120,7 +119,7 @@ def importDirectory(name_thread, max_thread, directory, logger, parser, wordpres else: if len(directory) != len(wordpress): - logger.error("{0} : Error : Number directory is differant than wordpress".format(name)) + logger.error("{0} : Error : Number directory is different than wordpress".format(name)) exit(1) for i in range(0, len(wordpress)-1): importWp = WPimport(name=name, basic=basic, wordpress=wordpress[i], logger=logger, parser=parser, ssl_wordpress=ssl_wordpress, no_create=create, no_update=update, no_image=image) @@ -252,7 +251,7 @@ if __name__ == '__main__': wait_for = [ ex.submit(remove, i, args.parallel, args, basic, logger, ssl_wordpress) for i in range(0, int(args.parallel)) ] wait(wait_for, return_when=ALL_COMPLETED) wait_for = [ - ex.submit(importDirectory, i, int(args.parallel), args.directory, logger, args.parser, args.wordpress, basic, args.serial, ssl_wordpress, args.create, args.update, args.image, TMP, revert) + ex.submit(importDirectory, i, int(args.parallel), args.directory, logger, args.parser, args.wordpress, basic, args.serial, ssl_wordpress, args.create, args.update, args.image) for i in range(0, int(args.parallel)) ] except Exception as err: @@ -262,6 +261,13 @@ if __name__ == '__main__': with futures.ThreadPoolExecutor(max_workers=int(args.parallel)) as ex: wait_for = [ ex.submit(remove, i, args.parallel, args, basic, logger, ssl_wordpress) for i in range(0, int(args.parallel)) ] wait(wait_for, return_when=ALL_COMPLETED) + if args.revert is True: + files_tmp = glob.glob("{0}/*.json".format(tmp)) + if len(files_tmp) > 0: + if len(files_tmp) != args.parallel: + for file_r in files_tmp: + os.remove(file_r) + wait_for = [ ex.submit(importUrl, i, int(args.parallel), args.canalblog, logger, args.parser, args.wordpress, basic, args.serial, ssl_wordpress, ssl_canalblog, args.create, args.update, args.image, args.revert, TMP) for i in range(0, int(args.parallel)) From a67ff868f3e64b2692365acbe7d7068e42e8e53b Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Mon, 26 Jun 2023 23:52:03 +0200 Subject: [PATCH 7/8] fix json read file --- lib/WPExport.py | 2 +- lib/WPImport.py | 56 ++++++++++++++++++++++++------------------------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/lib/WPExport.py b/lib/WPExport.py index df44100..53dac23 100644 --- a/lib/WPExport.py +++ b/lib/WPExport.py @@ -1,7 +1,7 @@ #!/usr/bin/python3 from bs4 import BeautifulSoup from urllib.parse import urlparse -import requests, os, argparse, logging +import requests, os, argparse, logging, json from requests.adapters import HTTPAdapter from requests.packages.urllib3.util.retry import Retry diff --git a/lib/WPImport.py b/lib/WPImport.py index 65453da..432d50d 100644 --- a/lib/WPImport.py +++ b/lib/WPImport.py @@ -41,35 +41,35 @@ class WPimport: def fromUrl(self, first, second): try: - content_file = open("{0}/{1}.json".format(self._name, self._tmp)) - webpage_content = json.loads(content_file) - webpage = webpage_content[first][second] - for i in range(0, len(webpage)): - try: - r = self._request.get(webpage[i]) - if r.status_code == 200: - self._logger.info("{0} : ({1}/{2}) : Page is importing : {3}".format(self._name, i+1, len(webpage), webpage[i])) - soup = BeautifulSoup(r.content, self._parser) - articlebody = soup.find_all("div", class_="articlebody") - if len(articlebody) > 0: - self._addOrUpdatePost(soup) + with open("{0}/{1}.json".format(self._tmp, self._name)) as file: + webpage_content = json.loads(file.read()) + self._logger.debug("{0} : size of webpage : {1}".format(self._name, len(webpage_content))) + webpage = webpage_content[first][second] + for i in range(0, len(webpage)): + try: + r = self._request.get(webpage[i]) + if r.status_code == 200: + self._logger.info("{0} : ({1}/{2}) : Page is importing : {3}".format(self._name, i+1, len(webpage), webpage[i])) + soup = BeautifulSoup(r.content, self._parser) + articlebody = soup.find_all("div", class_="articlebody") + if len(articlebody) > 0: + self._addOrUpdatePost(soup) + else: + self._addOrUpdateFeaturedMedia(soup) + del webpage_content[first][second][i] + webpage_content = json.dumps(webpage_content) + open("{0}/{1}.json".format(self._tmp, self._name), "wt").write(webpage_content) else: - self._addOrUpdateFeaturedMedia(soup) - del webpage_content[first][second][i] - open("{0}/{1}.json".format(self._tmp, self._name), "wt").write(webpage_content) - else: - self._logger.error("{0} : Connection error for get url {1} with status code : {2}".format(self._name, webpage[i], r.status_code)) - self._logger.debug("{0} : {1}".format(self._name, r.content)) - except ConnectionError as err: - self._logger.error("{0} : Connection error for get url {1} : {2}".format(self._name, webpage[i], err)) - exit(1) - except IOError as err: - self._logger.error("{0} : Connection error for IO url {1} : {2}".format(self._name, webpage[i], err)) - exit(1) - except Exception as err: - self._logger.error("{0} : Exception error for get url {1} : {2}".format(self._name, webpage[i], err)) - - + self._logger.error("{0} : Connection error for get url {1} with status code : {2}".format(self._name, webpage[i], r.status_code)) + self._logger.debug("{0} : {1}".format(self._name, r.content)) + except ConnectionError as err: + self._logger.error("{0} : Connection error for get url {1} : {2}".format(self._name, webpage[i], err)) + exit(1) + except IOError as err: + self._logger.error("{0} : Connection error for IO url {1} : {2}".format(self._name, webpage[i], err)) + exit(1) + except Exception as err: + self._logger.error("{0} : Exception error for get url {1} : {2}".format(self._name, webpage[i], err)) except Exception as ex: self._logger.error("{0} : Read file json from tmp : {1}".format(self._name, ex)) From 781d8959c426316cef38f8d3b8e018f7e6217713 Mon Sep 17 00:00:00 2001 From: Valentin CZERYBA Date: Tue, 27 Jun 2023 00:25:23 +0200 Subject: [PATCH 8/8] fix tmp directory parameter --- import_export_canalblog.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/import_export_canalblog.py b/import_export_canalblog.py index a0552ee..86ffde7 100644 --- a/import_export_canalblog.py +++ b/import_export_canalblog.py @@ -129,7 +129,6 @@ def importDirectory(name_thread, max_thread, directory, logger, parser, wordpres if __name__ == '__main__': - TMP = "/tmp/import_export_canablog" parser = argparse.ArgumentParser() parser.add_argument("--debug", help="Verbosity", action="store_true") parser.add_argument("--logfile", help="Log file", default="") @@ -138,6 +137,7 @@ if __name__ == '__main__': parser.add_argument("--parallel", help="Define number thread (default : 1)", default=1) parser.add_argument("--no-ssl", help="No ssl for canalblog and/or wordpress (example wordpress,canalblog)", dest="ssl", default="") parser.add_argument("--revert", help="Restart a work from stopping work", action="store_true") + parser.add_argument("--tmp", help="directory tmp", default="/tmp/import_export_canablog") subparsers = parser.add_subparsers(dest="command") @@ -191,9 +191,6 @@ if __name__ == '__main__': default="", help="File") - - if not os.path.exists(TMP): - os.mkdir(TMP) args = parser.parse_args() @@ -229,6 +226,8 @@ if __name__ == '__main__': fileHandler.setFormatter(formatter) logger.addHandler(fileHandler) + os.makedirs(args.tmp, exist_ok=True) + if args.command == "import" or args.command == "remove": password = args.password if len(args.password) == 0: @@ -262,14 +261,16 @@ if __name__ == '__main__': wait_for = [ ex.submit(remove, i, args.parallel, args, basic, logger, ssl_wordpress) for i in range(0, int(args.parallel)) ] wait(wait_for, return_when=ALL_COMPLETED) if args.revert is True: - files_tmp = glob.glob("{0}/*.json".format(tmp)) - if len(files_tmp) > 0: - if len(files_tmp) != args.parallel: - for file_r in files_tmp: - os.remove(file_r) + files_tmp = glob.glob("{0}/*.json".format(args.tmp)) + if len(files_tmp) == 0: + logger.error("Error revert, because files not found") + exit(1) + if len(files_tmp) != int(args.parallel): + for file_r in files_tmp: + os.remove(file_r) wait_for = [ - ex.submit(importUrl, i, int(args.parallel), args.canalblog, logger, args.parser, args.wordpress, basic, args.serial, ssl_wordpress, ssl_canalblog, args.create, args.update, args.image, args.revert, TMP) + ex.submit(importUrl, i, int(args.parallel), args.canalblog, logger, args.parser, args.wordpress, basic, args.serial, ssl_wordpress, ssl_canalblog, args.create, args.update, args.image, args.revert, args.tmp) for i in range(0, int(args.parallel)) ] @@ -307,7 +308,7 @@ if __name__ == '__main__': try: with futures.ThreadPoolExecutor(max_workers=int(args.parallel)) as ex: wait_for = [ - ex.submit(download, i, int(args.parallel), url, logger, args.parser, args.directory, args.html, args.img, ssl_canalblog, args.revert, TMP) + ex.submit(download, i, int(args.parallel), url, logger, args.parser, args.directory, args.html, args.img, ssl_canalblog, args.revert, args.tmp) for i in range(0, int(args.parallel)) ] except Exception as err: