diff --git a/lib/WPExport.py b/lib/WPExport.py index cbdc1bc..f0775e7 100644 --- a/lib/WPExport.py +++ b/lib/WPExport.py @@ -72,20 +72,23 @@ class WPExport: def getUrlPage(self, index_thread, max_thread): try: page = self._request.get(self._url) - except Exception as err: + + page_url = [] + if page.status_code == 200: + soup = BeautifulSoup(page.text, self._parser) + ul = soup.find_all("ul", id="listsmooth") + for anchor in ul[0].find_all("a"): + href = anchor.get('href', '/') + if href != "#": + page_url.append(href) + else: + self._logger.error("{0} : URL did not get due status code : {1}".format(self._name, page.status_code)) + self._logger.debug("{0} : {1}".format(self._name, page.content)) + except ConnectionError as err: self._logger.error("{0} : Connection error : {1}".format(self._name, err)) exit(1) - page_url = [] - if page.status_code == 200: - soup = BeautifulSoup(page.text, self._parser) - ul = soup.find_all("ul", id="listsmooth") - for anchor in ul[0].find_all("a"): - href = anchor.get('href', '/') - if href != "#": - page_url.append(href) - else: - self._logger.error("{0} : URL did not get due status code : {1}".format(self._name, page.status_code)) - self._logger.debug("{0} : {1}".format(self._name, page.content)) + except Exception as err: + self._logger.error("{0} : Exception error : {1}".format(self._name, err)) webpage = {"principal": {"page":[], "article":[]}, "publications": {"page":[], "article":[]}} for i in page_url: @@ -97,63 +100,67 @@ class WPExport: section = "principal" try: page = self._request.get(i) - except Exception as err: + + if page.status_code == 200: + self._logger.info("{0} : page : {1}".format(self._name, i)) + if i not in webpage[section]["page"]: + webpage[section]["page"].append(i) + soup = BeautifulSoup(page.text, self._parser) + class_div = soup.find_all("div", class_="pagingfirstline") + if len(class_div) > 0: + pagingfirstline = class_div[0].find_all("a") + if len(pagingfirstline) > 1: + lastpage = pagingfirstline[len(pagingfirstline)-1].get("href", "/") + self._logger.debug("{0} : Last page {1}".format(self._name, lastpage)) + + element_lastpage = lastpage.split("/")[len(lastpage.split("/"))-1] + number_page = element_lastpage.split("-")[0].split("p")[1] + number_lastpage = int(number_page) / 10 + + setPageDivided = int(number_lastpage) / max_thread + if setPageDivided > int(setPageDivided): + setPageDivided = setPageDivided + 1 + setPagePart = setPageDivided * (index_thread + 1) + firstPagePart = (setPagePart - setPageDivided) + + self._logger.debug("{0} : Total page : {1}".format(self._name,int(number_lastpage))) + self._logger.debug("{0} : First range : {1}".format(self._name, int(firstPagePart))) + self._logger.debug("{0} : Last range : {1}".format(self._name, int(setPagePart))) + + for j in range(int(firstPagePart),int(setPagePart)+1): + paging = j * 10 + categorie = urlparse(i).path.split("/") + url_paging = "{0}/archives/p{1}-10.html".format(self._url, paging) + if len(categorie) > 2: + url_paging = "{0}/archives/{1}/p{2}-10.html".format(self._url, categorie[2], paging) + self._logger.info("{0} : {1}".format(self._name, url_paging)) + if url_paging not in webpage[section]["page"]: + webpage[section]["page"].append(url_paging) + page = self._request.get(url_paging) + if page.status_code == 200: + soup = BeautifulSoup(page.text, self._parser) + h2 = soup.find_all("h2") + self._logger.debug("{0} : {1} H2 : {2}".format(self._name, url_paging, h2)) + for title in h2: + self._logger.debug("{0} : {1} a : {2}".format(self._name, url_paging, title.find_all("a"))) + href = title.find_all("a")[0].get("href", "/") + if href not in webpage[section]["article"]: + try: + o = urlparse(href) + o = o._replace(scheme="https").geturl() + webpage[section]["article"].append(o) + except Exception as err: + self._logger.error("parsing error : {0}".format(err)) + exit(1) + else: + self._logger.error("{0} : web didn't get due status code : {1}".format(self._name, page.status_code)) + self._logger.debug("{0} : {1}".format(self._name, page.content)) + except ConnectionError as err: self._logger.error("{0} : Connection error : {1}".format(self._name, err)) exit(1) - if page.status_code == 200: - self._logger.info("{0} : page : {1}".format(self._name, i)) - if i not in webpage[section]["page"]: - webpage[section]["page"].append(i) - soup = BeautifulSoup(page.text, self._parser) - class_div = soup.find_all("div", class_="pagingfirstline") - if len(class_div) > 0: - pagingfirstline = class_div[0].find_all("a") - if len(pagingfirstline) > 1: - lastpage = pagingfirstline[len(pagingfirstline)-1].get("href", "/") - self._logger.debug("{0} : Last page {1}".format(self._name, lastpage)) - - element_lastpage = lastpage.split("/")[len(lastpage.split("/"))-1] - number_page = element_lastpage.split("-")[0].split("p")[1] - number_lastpage = int(number_page) / 10 - - setPageDivided = int(number_lastpage) / max_thread - if setPageDivided > int(setPageDivided): - setPageDivided = setPageDivided + 1 - setPagePart = setPageDivided * (index_thread + 1) - firstPagePart = (setPagePart - setPageDivided) - - self._logger.debug("{0} : Total page : {1}".format(self._name,int(number_lastpage))) - self._logger.debug("{0} : First range : {1}".format(self._name, int(firstPagePart))) - self._logger.debug("{0} : Last range : {1}".format(self._name, int(setPagePart))) - - for j in range(int(firstPagePart),int(setPagePart)+1): - paging = j * 10 - categorie = urlparse(i).path.split("/") - url_paging = "{0}/archives/p{1}-10.html".format(self._url, paging) - if len(categorie) > 2: - url_paging = "{0}/archives/{1}/p{2}-10.html".format(self._url, categorie[2], paging) - self._logger.info("{0} : {1}".format(self._name, url_paging)) - if url_paging not in webpage[section]["page"]: - webpage[section]["page"].append(url_paging) - page = self._request.get(url_paging) - if page.status_code == 200: - soup = BeautifulSoup(page.text, self._parser) - h2 = soup.find_all("h2") - self._logger.debug("{0} : {1} H2 : {2}".format(self._name, url_paging, h2)) - for title in h2: - self._logger.debug("{0} : {1} a : {2}".format(self._name, url_paging, title.find_all("a"))) - href = title.find_all("a")[0].get("href", "/") - if href not in webpage[section]["article"]: - try: - o = urlparse(href) - o = o._replace(scheme="https").geturl() - except Exception as err: - self._logger.error("parsing error : {0}".format(err)) - exit(1) - webpage[section]["article"].append(o) - else: - self._logger.error("{0} : web didn't get due status code : {1}".format(self._name, page.status_code)) - self._logger.debug("{0} : {1}".format(self._name, page.content)) + except Exception as err: + self._logger.error("{0} : Exception error : {1}".format(self._name, err)) + exit(1) return webpage @@ -183,47 +190,52 @@ class WPExport: def _getScriptCss(self, js, css): try: page = self._request.get(self._url) - except Exception as err: - self._logger.error("Connection error : {0}".format(err)) - exit(1) - page_url = [] - if page.status_code == 200: - soup = BeautifulSoup(page.text, self._parser) - if js is True: - script = soup.find_all("script") - for anchor in script: - src = anchor.get("src", "/") - if src != "/": - try: - u = urlparse(self._url) - o = urlparse(src) - except Exception as err: - self._logger.error("parsing error : {0}".format(err)) - exit(1) - if o.netloc == "": - o = o._replace(netloc=u.netloc) - o = o._replace(scheme=u.scheme) - page_url.append(o.geturl()) - if css is True: - link = soup.find_all("link") - for anchor in link: - rel = anchor.get("rel") - if rel[0] == "stylesheet": - href = anchor.get("href", "/") - if href != "/": + + page_url = [] + if page.status_code == 200: + soup = BeautifulSoup(page.text, self._parser) + if js is True: + script = soup.find_all("script") + for anchor in script: + src = anchor.get("src", "/") + if src != "/": try: u = urlparse(self._url) - o = urlparse(href) + o = urlparse(src) + if o.netloc == "": + o = o._replace(netloc=u.netloc) + o = o._replace(scheme=u.scheme) + page_url.append(o.geturl()) except Exception as err: self._logger.error("parsing error : {0}".format(err)) exit(1) - if o.netloc == "": - o = o._replace(netloc=u.netloc) - o = o._replace(scheme=u.scheme) - page_url.append(o.geturl()) - else: - self._logger.error("JS or CSS did not get due status code : {0}".format(page.status_code)) - self._logger.debug(page.content) + + if css is True: + link = soup.find_all("link") + for anchor in link: + rel = anchor.get("rel") + if rel[0] == "stylesheet": + href = anchor.get("href", "/") + if href != "/": + try: + u = urlparse(self._url) + o = urlparse(href) + if o.netloc == "": + o = o._replace(netloc=u.netloc) + o = o._replace(scheme=u.scheme) + page_url.append(o.geturl()) + except Exception as err: + self._logger.error("parsing error : {0}".format(err)) + exit(1) + + else: + self._logger.error("JS or CSS did not get due status code : {0}".format(page.status_code)) + self._logger.debug(page.content) + except ConnectionError as err: + self._logger.error("Connection error : {0}".format(err)) + exit(1) + except Exception as err: + self._logger.error("Exception error : {0}".format(err)) return page_url @@ -234,22 +246,25 @@ class WPExport: for i in webpage: try: page = self._request.get(i) - except Exception as err: + + if page.status_code == 200: + soup = BeautifulSoup(page.text, self._parser) + img = soup.find_all("img") + self._logger.info("{0} : image from page: {1} : ".format(self._name,i)) + for anchor in img: + src = anchor.get("src", "/") + if src != "/": + if src not in page_img: + self._logger.info("{0} : image: {1} : ".format(self._name, src)) + page_img.append(src) + else: + self._logger.error("{0} : Image did not get due status code : {1}".format(self._name, page.status_code)) + self._logger.debug("{0} : {1}".format(self._name, page.content)) + except ConnectionError as err: self._logger.error("{0} : Connection error : {1}".format(self._name, err)) exit(1) - if page.status_code == 200: - soup = BeautifulSoup(page.text, self._parser) - img = soup.find_all("img") - self._logger.info("{0} : image from page: {1} : ".format(self._name,i)) - for anchor in img: - src = anchor.get("src", "/") - if src != "/": - if src not in page_img: - self._logger.info("{0} : image: {1} : ".format(self._name, src)) - page_img.append(src) - else: - self._logger.error("{0} : Image did not get due status code : {1}".format(self._name, page.status_code)) - self._logger.debug("{0} : {1}".format(self._name, page.content)) + except Exception as err: + self._logger.error("{0} : Exception error : {1}".format(self._name, err)) return page_img @@ -260,31 +275,33 @@ class WPExport: for i in range(0, len(webpage)): try: o = urlparse(webpage[i]) + + path_web = o.path.split("/") + filePageWeb = path_web[len(path_web)-1] + path_web.pop(len(path_web)-1) + dir_page_web = "/".join(path_web) + self._mkdirPath("{0}/{1}/{2}".format(backup_dir, o.netloc, dir_page_web)) + try: + r = self._request.get(webpage[i]) + + if r.status_code == 200: + fileDownload = "{0}/{1}/index.html".format(backup_dir, o.netloc) + if len(dir_page_web) > 0 and len(filePageWeb) > 0: + fileDownload = "{0}/{1}{2}/{3}".format(backup_dir, o.netloc, dir_page_web, filePageWeb) + self._logger.info("{0} : {1}/{2} : {3}".format(self._name, i+1, len(webpage), fileDownload)) + try: + open(fileDownload, "wb").write(r.content) + except Exception as err: + self._logger.error("file error : {0}".format(err)) + exit(1) + else: + self._logger.error("Not download due status code : {0}".format(r.status_code)) + self._logger.debug(r.content) + except ConnectionError as err: + self._logger.error("{0} : Connection error : {1}".format(self._name, err)) + exit(1) + except Exception as err: + self._logger.error("{0} Exception error : {1}".format(self._name, err)) except Exception as err: self._logger.error("parsing error : {0}".format(err)) exit(1) - path_web = o.path.split("/") - filePageWeb = path_web[len(path_web)-1] - path_web.pop(len(path_web)-1) - dir_page_web = "/".join(path_web) - self._mkdirPath("{0}/{1}/{2}".format(backup_dir, o.netloc, dir_page_web)) - try: - r = self._request.get(webpage[i]) - except ConnectionError as err: - self._logger.error("{0} : Connection error : {1}".format(self._name, err)) - exit(1) - except Exception as err: - self._logger.error("{0} Exception error : {1}".format(self._name, err)) - if r.status_code == 200: - fileDownload = "{0}/{1}/index.html".format(backup_dir, o.netloc) - if len(dir_page_web) > 0 and len(filePageWeb) > 0: - fileDownload = "{0}/{1}{2}/{3}".format(backup_dir, o.netloc, dir_page_web, filePageWeb) - self._logger.info("{0} : {1}/{2} : {3}".format(self._name, i+1, len(webpage), fileDownload)) - try: - open(fileDownload, "wb").write(r.content) - except Exception as err: - self._logger.error("file error : {0}".format(err)) - exit(1) - else: - self._logger.error("Not download due status code : {0}".format(r.status_code)) - self._logger.debug(r.content) diff --git a/lib/WPImport.py b/lib/WPImport.py index 3f343cc..2640c16 100644 --- a/lib/WPImport.py +++ b/lib/WPImport.py @@ -39,20 +39,22 @@ class WPimport: for i in range(0, len(webpage)): try: r = self._request.get(webpage[i]) - except Exception as err: + if r.status_code == 200: + self._logger.info("{0} : ({1}/{2}) : Page is importing : {3}".format(self._name, i+1, len(webpage), webpage[i])) + soup = BeautifulSoup(r.content, self._parser) + articlebody = soup.find_all("div", class_="articlebody") + if len(articlebody) > 0: + self._addOrUpdatePost(soup) + else: + self._addOrUpdateFeaturedMedia(soup) + else: + self._logger.error("{0} : Connection error for get url {1} with status code : {2}".format(self._name, webpage[i], r.status_code)) + self._logger.debug("{0} : {1}".format(self._name, r.content)) + except ConnectionError as err: self._logger.error("{0} : Connection error for get url {1} : {2}".format(self._name, webpage[i], err)) exit(1) - if r.status_code == 200: - self._logger.info("{0} : ({1}/{2}) : Page is importing : {3}".format(self._name, i+1, len(webpage), webpage[i])) - soup = BeautifulSoup(r.content, self._parser) - articlebody = soup.find_all("div", class_="articlebody") - if len(articlebody) > 0: - self._addOrUpdatePost(soup) - else: - self._addOrUpdateFeaturedMedia(soup) - else: - self._logger.error("{0} : Connection error for get url {1} with status code : {2}".format(self._name, webpage[i], r.status_code)) - self._logger.debug("{0} : {1}".format(self._name, r.content)) + except Exception as err: + self._logger.error("{0} : Exception error for get url {1} : {2}".format(self._name, webpage[i], err)) def fromDirectory(self, directory="", number_thread=1, max_thread=1): @@ -137,60 +139,65 @@ class WPimport: params = {"search":h2, "type":"post"} try: page = self._request.get("{1}://{0}/wp-json/wp/v2/search".format(self._wordpress, self._protocol), auth=self._basic, params=params) - except Exception as err: + if page.status_code == 200: + result = page.json() + if len(result) > 0: + if h2 == result[0]["title"]: + img = i.find_all("img") + if len(img) > 0: + img_src = img[0].get("src") + try: + page = self._request.get(img_src) + if page.status_code == 200: + name_img = img_src.replace("_q", "") + name_img = name_img.split("/")[len(name_img.split("/"))-1] + params = {"search": name_img} + try: + page = self._request.get("{1}://{0}/wp-json/wp/v2/media".format(self._wordpress, self._protocol), auth=self._basic, params=params) + + if page.status_code == 200: + res = page.json() + if len(res) > 0: + id_media = res[0]["id"] + data = {"featured_media": id_media} + try: + r = self._request.post("{2}://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, result[0]["id"], self._protocol), auth=self._basic, headers=self._headers_json, data=json.dumps(data)) + if r.status_code == 200: + self._logger.info("{0} : Add media featured : {1}".format(self._name, r.json()["title"]["raw"])) + else: + self._logger.error("{0} : Connection error with status code for featured media : {1}".format(self._name, r.status_code)) + self._logger.debug("{0} : {1}".format(self._name, r.content)) + except ConnectionError as err: + self._logger.error("{0} : Connection error for post media featured : {1}".format(self._name, err)) + exit(1) + except Exception as err: + self._logger.error("{0} : Exception error for post media featured : {1}".format(self._name, err)) + else: + self._logger.info("{0} : No media found for {1}".format(self._name, h2)) + else: + self._logger.error("{0} : Connection error with status code for search featured media: {1}".format(self._name, page.status_code)) + self._logger.debug("{0} : {1}".format(self._name, page.content)) + except ConnectionError as err: + self._logger.error("{0} : Connection error search featured media : {1}".format(self._name, err)) + exit(1) + except Exception as err: + self._logger.error("{0} : Exception error search featured media : {1}".format(self._name, err)) + else: + self._logger.error("{0} : Connection error for get featured media with status code : {1}".format(self._name, page.status_code)) + self._logger.debug("{0} : {1}".format(self._name, page.content)) + except ConnectionError as err: + self._logger.error("{0} : Connection error for get featured media : {1}".format(self._name, err)) + exit(1) + except Exception as err: + self._logger.error("{0} : Exception error for get featured media : {1}".format(self._name, err)) + else: + self._logger.error("{0} : Connection error with status code for featured media : {1}".format(self._name, page.status_code)) + self._logger.debug("{0} : {1}".format(self._name, page.content)) + except ConnectionError as err: self._logger.error("{0} : Connection error : {1}".format(self._name, err)) exit(1) - if page.status_code == 200: - result = page.json() - if len(result) > 0: - if h2 == result[0]["title"]: - img = i.find_all("img") - if len(img) > 0: - img_src = img[0].get("src") - try: - page = self._request.get(img_src) - except Exception as err: - self._logger.error("{0} : Connection error for get featured media : {1}".format(self._name, err)) - exit(1) - if page.status_code == 200: - name_img = img_src.replace("_q", "") - name_img = name_img.split("/")[len(name_img.split("/"))-1] - params = {"search": name_img} - try: - page = self._request.get("{1}://{0}/wp-json/wp/v2/media".format(self._wordpress, self._protocol), auth=self._basic, params=params) - except Exception as err: - self._logger.error("{0} : Connection error search featured media : {1}".format(self._name, err)) - exit(1) - if page.status_code == 200: - res = page.json() - if len(res) > 0: - id_media = res[0]["id"] - data = {"featured_media": id_media} - try: - r = self._request.post("{2}://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, result[0]["id"], self._protocol), auth=self._basic, headers=self._headers_json, data=json.dumps(data)) - except Exception as err: - self._logger.error("{0} : Connection error for post media featured : {1}".format(self._name, err)) - exit(1) - if r.status_code == 200: - self._logger.info("{0} : Add media featured : {1}".format(self._name, r.json()["title"]["raw"])) - else: - self._logger.error("{0} : Connection error with status code for featured media : {1}".format(self._name, r.status_code)) - self._logger.debug("{0} : {1}".format(self._name, r.content)) - - else: - self._logger.info("{0} : No media found for {1}".format(self._name, h2)) - else: - self._logger.error("{0} : Connection error with status code for search featured media: {1}".format(self._name, page.status_code)) - self._logger.debug("{0} : {1}".format(self._name, page.content)) - - else: - self._logger.error("{0} : Connection error for get featured media with status code : {1}".format(self._name, page.status_code)) - self._logger.debug("{0} : {1}".format(self._name, page.content)) - - else: - self._logger.error("{0} : Connection error with status code for featured media : {1}".format(self._name, page.status_code)) - self._logger.debug("{0} : {1}".format(self._name, page.content)) - + except Exception as err: + self._logger.error("{0} : Connection error : {1}".format(self._name, err)) ## Association image to post @@ -199,14 +206,16 @@ class WPimport: data = {"post": post_id} try: r = self._request.post("{2}://{0}/wp-json/wp/v2/media/{1}".format(self._wordpress, i["id"], self._protocol), auth=self._basic, data=data) - except Exception as err: + if r.status_code == 200: + self._logger.info("{0} : Link image to post {1}".format(self._name, title)) + else: + self._logger.error("{0} Connection error with status code for link image to post : {1}".format(self._name, r.status_code)) + self._logger.debug("{0} : {1}".format(self._name, r.content)) + except ConnectionError as err: self._logger.error("{0} : Connection error for link image to post : {1}".format(self._name, err)) exit(1) - if r.status_code == 200: - self._logger.info("{0} : Link image to post {1}".format(self._name, title)) - else: - self._logger.error("{0} Connection error with status code for link image to post : {1}".format(self._name, r.status_code)) - self._logger.debug("{0} : {1}".format(self._name, r.content)) + except Exception as err: + self._logger.error("{0} : Exception error for link image to post : {1}".format(self._name, err)) ## Add or update img @@ -226,49 +235,57 @@ class WPimport: params = { "search": img_name} try: r = self._request.get("{1}://{0}/wp-json/wp/v2/media".format(self._wordpress, self._protocol), auth=self._basic, params=params) - except Exception as err: + + self._logger.debug("{0} : Search for image {1} and his status code {2}".format(self._name, img_name, r.status_code)) + if r.status_code == 200: + res = r.json() + self._logger.debug("{0} : Number of image in search : {1}".format(self._name, len(res))) + if len(res) > 0: + params = {"force":1} + try: + r = self._request.delete("{2}://{0}/wp-json/wp/v2/media/{1}".format(self._wordpress, res[0]["id"], self._protocol), auth=self._basic, params=params) + if r.status_code == 200: + self._logger.info("{0} : Image removed {1}".format(self._name, img_name)) + else: + self._logger.error("{0} : Image {1} not removed due status code : {2}".format(self._name, img_name, r.status_code)) + self._logger.debug("{0} : {1}".format(self._name, r.content)) + except ConnectionError as err: + self._logger.error("{0} Connection error for delete image : {1}".format(self._name, err)) + exit(1) + except Exception as err: + self._logger.error("{0} Exception error for delete image : {1}".format(self._name, err)) + + data = page.content + img_type = "image/{0}".format(img_type_file) + if img_type_file == "jpg": + img_type = "image/jpeg" + headers={ 'Content-Type': img_type,'Content-Disposition' : 'attachment; filename={0}'.format(img_name)} + try: + r = self._request.post("{1}://{0}/wp-json/wp/v2/media".format(self._wordpress, self._protocol), auth=self._basic, headers=headers, data=data) + + if r.status_code == 201: + self._logger.info("{0} : Image added {1}".format(self._name, img_name)) + res = r.json() + media["id"] = res["id"] + media["rendered"] = res["guid"]["rendered"] + else: + self._logger.error("{0} : Image {1}.{2} not added due status code : {3}".format(self._name, img_name, img_type, r.status_code)) + self._logger.debug("{0} : {1}".format(self._name, r.content)) + except ConnectionError as err: + self._logger.error("{0} : Connection error for add image : {1}".format(self._name, err)) + exit(1) + except Exception as err: + self._logger.error("{0} : Exception error for add image : {1}".format(self._name, err)) + exit(1) + + else: + self._logger.error("{0} : Connection error for search image with status code : {1}".format(self._name, r.status_code)) + self._logger.debug("{0} : {1}".format(self._name, r.content)) + except ConnectionError as err: self._logger.error("{0} : Connection error for search media : {1}".format(self._name, err)) exit(1) - self._logger.debug("{0} : Search for image {1} and his status code {2}".format(self._name, img_name, r.status_code)) - if r.status_code == 200: - res = r.json() - self._logger.debug("{0} : Number of image in search : {1}".format(self._name, len(res))) - if len(res) > 0: - params = {"force":1} - try: - r = self._request.delete("{2}://{0}/wp-json/wp/v2/media/{1}".format(self._wordpress, res[0]["id"], self._protocol), auth=self._basic, params=params) - except Exception as err: - self._logger.error("{0} Connection error for delete image : {1}".format(self._name, err)) - exit(1) - if r.status_code == 200: - self._logger.info("{0} : Image removed {1}".format(self._name, img_name)) - else: - self._logger.error("{0} : Image {1} not removed due status code : {2}".format(self._name, img_name, r.status_code)) - self._logger.debug("{0} : {1}".format(self._name, r.content)) - - data = page.content - img_type = "image/{0}".format(img_type_file) - if img_type_file == "jpg": - img_type = "image/jpeg" - headers={ 'Content-Type': img_type,'Content-Disposition' : 'attachment; filename={0}'.format(img_name)} - try: - r = self._request.post("{1}://{0}/wp-json/wp/v2/media".format(self._wordpress, self._protocol), auth=self._basic, headers=headers, data=data) - except Exception as err: - self._logger.error("{0} : Connection error for add image : {1}".format(self._name, err)) - exit(1) - if r.status_code == 201: - self._logger.info("{0} : Image added {1}".format(self._name, img_name)) - res = r.json() - media["id"] = res["id"] - media["rendered"] = res["guid"]["rendered"] - else: - self._logger.error("{0} : Image {1}.{2} not added due status code : {3}".format(self._name, img_name, img_type, r.status_code)) - self._logger.debug("{0} : {1}".format(self._name, r.content)) - - else: - self._logger.error("{0} : Connection error for search image with status code : {1}".format(self._name, r.status_code)) - self._logger.debug("{0} : {1}".format(self._name, r.content)) - + except Exception as err: + self._logger.error("{0} : Exception error for search media : {1}".format(self._name, err)) return media ## Add or update comment @@ -279,28 +296,34 @@ class WPimport: try: params = {"post": post, "author_name":i["author"], "date":i["date"]} page = self._request.get("{1}://{0}/wp-json/wp/v2/comments".format(self._wordpress, self._protocol), auth=self._basic, params=params) - except Exception as err: + + if page.status_code == 200: + result = page.json() + for j in result: + try: + params = {"force":1} + page = self._request.delete("{2}://{0}/wp-json/wp/v2/comments/{1}".format(self._wordpress, j["id"], self._protocol), params=params, auth=self._basic) + + if page.status_code == 200: + self._logger.info("{0} : Comment deleted for {1}".format(self._name, title)) + self._logger.debug("{0} : Comment deleted : {1}".format(self._name, j)) + else: + self._logger.error("{0} : Comment not deleted for {1} due status code : {2}".format(self._name, title, page.status_code)) + self._logger.debug("{0} : {1}".format(self._name, page.content)) + except ConnectionError as err: + self._logger.error("{0} : Connection error for delete comment : {1}".format(self._name, err)) + exit(1) + except Exception as err: + self._logger.error("{0} : Exception error for delete comment : {1}".format(self._name, err)) + + else: + self._logger.error("{0} : Comment not listed for {1} due status code : {2}".format(self._name, title, page.status_code)) + self._logger.debug("{0} : {1}".format(self._name, page.content)) + except ConnectionError as err: self._logger.error("{0} : Connection error for search comment : {1}".format(self._name, err)) exit(1) - if page.status_code == 200: - result = page.json() - for j in result: - try: - params = {"force":1} - page = self._request.delete("{2}://{0}/wp-json/wp/v2/comments/{1}".format(self._wordpress, j["id"], self._protocol), params=params, auth=self._basic) - except Exception as err: - self._logger.error("{0} : Connection error for delete comment : {1}".format(self._name, err)) - exit(1) - if page.status_code == 200: - self._logger.info("{0} : Comment deleted for {1}".format(self._name, title)) - self._logger.debug("{0} : Comment deleted : {1}".format(self._name, j)) - else: - self._logger.error("{0} : Comment not deleted for {1} due status code : {2}".format(self._name, title, page.status_code)) - self._logger.debug("{0} : {1}".format(self._name, page.content)) - - else: - self._logger.error("{0} : Comment not listed for {1} due status code : {2}".format(self._name, title, page.status_code)) - self._logger.debug("{0} : {1}".format(self._name, page.content)) + except Exception as err: + self._logger.error("{0} : Exception error for search comment : {1}".format(self._name, err)) for i in comment: data = {"post": post, "content": i["content"], "date": i["date"], "author_name": i["author"], "status": "approved"} @@ -310,28 +333,35 @@ class WPimport: params = {"post": post, "author_name":comment[parent_id]["author"], "date":comment[parent_id]["date"]} try: page = self._request.get("{1}://{0}/wp-json/wp/v2/comments".format(self._wordpress, self._protocol), auth=self._basic, params=params) - except Exception as err: + + if page.status_code == 200: + result = page.json() + if len(result) > 0: + data["parent"]=result[0]["id"] + else: + self._logger.error("{0} : Connection error for parent comment with status code : {1}".format(self._name, page.status_code)) + self._logger.debug("{0} : {1}".format(self._name, page.content)) + except ConnectionError as err: self._logger.error("{0} : Connection error for parent comment : {1}".format(self._name, err)) exit(1) - if page.status_code == 200: - result = page.json() - if len(result) > 0: - data["parent"]=result[0]["id"] - else: - self._logger.error("{0} : Connection error for parent comment with status code : {1}".format(self._name, page.status_code)) - self._logger.debug("{0} : {1}".format(self._name, page.content)) + except Exception as err: + self._logger.error("{0} : Exception error for parent comment : {1}".format(self._name, err)) try: page = self._request.post("{1}://{0}/wp-json/wp/v2/comments".format(self._wordpress, self._protocol), auth=self._basic, data=data) - except Exception as err: + + if page.status_code == 201: + self._logger.info("{0} : Comment added for {1}".format(self._name, title)) + self._logger.debug("{0} : Data : {1}".format(self._name, data)) + else: + self._logger.error("{0} : Comment not added for {1} due status code : {2}".format(self._name, title, page.status_code)) + self._logger.debug("{0} : {1}".format(self._name, page.content)) + except ConnectionError as err: self._logger.error("{0} : Connection error for add comment : {1}".format(self._name, err)) exit(1) - if page.status_code == 201: - self._logger.info("{0} : Comment added for {1}".format(self._name, title)) - self._logger.debug("{0} : Data : {1}".format(self._name, data)) - else: - self._logger.error("{0} : Comment not added for {1} due status code : {2}".format(self._name, title, page.status_code)) - self._logger.debug("{0} : {1}".format(self._name, page.content)) + except Exception as err: + self._logger.error("{0} : Exception error for add comment : {1}".format(self._name, err)) + exit(1) ## Check class name @@ -416,30 +446,37 @@ class WPimport: new_img["old_href"]=href_a try: page_img = self._request.get(href_img) - except Exception as err: - self._logger.error("{0} : Connection error for get image : {1}".format(self._name, err)) - exit(1) - if page_img.status_code == 404: - href_img = href_a - try: - page_img = self._request.get(href_a) - except Exception as err: - self._logger.error("{0} : Connection error for get image : {1}".format(self._name, err)) - exit(1) - self._logger.debug("{0} : Status code for image {1} : {2}".format(self._name, href_img, page_img.status_code)) - if page_img.status_code == 200: - media=self._addOrUpdateMedia(href_img, page_img) - new_img["id"]=media["id"] - new_img["new_src"]=media["rendered"] - list_img.append(new_img) - if href_img != href_a: - media=self._addOrUpdateMedia(href_a, page_img) + + if page_img.status_code == 404: + href_img = href_a + try: + page_img = self._request.get(href_a) + except ConnectionError as err: + self._logger.error("{0} : Connection error for get image : {1}".format(self._name, err)) + exit(1) + except Exception as err: + self._logger.error("{0} : Exception error for get image : {1}".format(self._name, err)) + exit(1) + self._logger.debug("{0} : Status code for image {1} : {2}".format(self._name, href_img, page_img.status_code)) + if page_img.status_code == 200: + media=self._addOrUpdateMedia(href_img, page_img) new_img["id"]=media["id"] new_img["new_src"]=media["rendered"] list_img.append(new_img) - if page_img.status_code not in [200, 404]: - self._logger.error("{0} : Connection error with status code for get image : {1}".format(self._name, page_img.status_code)) - self._logger.debug("{0} : {1}".format(self._name, page_img.content)) + if href_img != href_a: + media=self._addOrUpdateMedia(href_a, page_img) + new_img["id"]=media["id"] + new_img["new_src"]=media["rendered"] + list_img.append(new_img) + if page_img.status_code not in [200, 404]: + self._logger.error("{0} : Connection error with status code for get image : {1}".format(self._name, page_img.status_code)) + self._logger.debug("{0} : {1}".format(self._name, page_img.content)) + except ConnectionError as err: + self._logger.error("{0} : Connection error for get image : {1}".format(self._name, err)) + exit(1) + except Exception as err: + self._logger.error("{0} : Exception error for get image : {1}".format(self._name, err)) + exit(1) self._logger.debug("{0} : Number of image : {1}".format(self._name, len(list_img))) comment_post = self._getComment(comment) @@ -462,36 +499,36 @@ class WPimport: try: params = {"search":title_element, "per_page":"100", "page":index} page = self._request.get("{2}://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i, self._protocol), auth=self._basic, params=params) + + if page.status_code == 200: + result = page.json() + self._logger.debug("{0} : content {3} {2} : {1}".format(self._name, result, title_element, i)) + if len(result) > 0: + for k in result: + title_rendered = k["name"] + self._logger.debug("{0} : content {2} : {1}".format(self._name, title_rendered, i)) + self._logger.debug("{0} : size of content {3} : {2} - {1}".format(self._name, len(title_rendered), len(title_element), i)) + if len(title_element) != len(title_rendered): + title_rendered = self._replaceCaracter(title_rendered) + + if title_element == title_rendered: + self._logger.info("{0} : {1} found : {2}".format(self._name, i, title_rendered)) + element_exist = True + listelement[i].append(k["id"]) + else: + break + if page.status_code == 400: + self._logger.error("{0} : {1} not found due status code : {2}".format(self._name, i, page.status_code)) + self._logger.debug("{0} : {1}".format(self._name, page.content)) + break + else: + self._logger.error("{0} : {1} not found due status code : {2}".format(self._name, i, page.status_code)) + self._logger.debug("{0} : {1}".format(self._name, page.content)) except ConnectionError as err: self._logger.error("{0} : Connection error for {1} : {2}".format(self._name, i, err)) exit(1) except Exception as err: self._logger.error("{0} : Exception error for {1} : {2}".format(self._name, i, err)) - if page.status_code == 200: - result = page.json() - self._logger.debug("{0} : content {3} {2} : {1}".format(self._name, result, title_element, i)) - if len(result) > 0: - for k in result: - title_rendered = k["name"] - self._logger.debug("{0} : content {2} : {1}".format(self._name, title_rendered, i)) - self._logger.debug("{0} : size of content {3} : {2} - {1}".format(self._name, len(title_rendered), len(title_element), i)) - if len(title_element) != len(title_rendered): - title_rendered = self._replaceCaracter(title_rendered) - - if title_element == title_rendered: - self._logger.info("{0} : {1} found : {2}".format(self._name, i, title_rendered)) - element_exist = True - listelement[i].append(k["id"]) - else: - break - if page.status_code == 400: - self._logger.error("{0} : {1} not found due status code : {2}".format(self._name, i, page.status_code)) - self._logger.debug("{0} : {1}".format(self._name, page.content)) - break - else: - self._logger.error("{0} : {1} not found due status code : {2}".format(self._name, i, page.status_code)) - self._logger.debug("{0} : {1}".format(self._name, page.content)) - self._logger.debug("{0} : Element {3} {2} is {1}".format(self._name, element_exist, title_element, i)) if element_exist == False: data = {"name": title_element} @@ -499,18 +536,19 @@ class WPimport: self._logger.debug("{0} : Data : {1}".format(self._name, data)) try: page = self._request.post("{2}://{0}/wp-json/wp/v2/{1}".format(self._wordpress, i, self._protocol), auth=self._basic, headers=self._headers_json, data=json.dumps(data)) + + if page.status_code == 201: + self._logger.info("{0} : {1} created : {2}".format(self._name, i, j)) + result = page.json() + listelement[i].append(result["id"]) + else: + self._logger.error("{0} : {1} not added due status code : {2}".format(self._name, i, page.status_code)) + self._logger.debug("{0} : {1}".format(self._name, page.content)) except ConnectionError as err: self._logger.error("{0} : Connection error for post {1} : {2}".format(self._name, i, err)) exit(1) except Exception as err: self._logger.error("{0} : Exception error for post {1} : {2}".format(self._name, i, err)) - if page.status_code == 201: - self._logger.info("{0} : {1} created : {2}".format(self._name, i, j)) - result = page.json() - listelement[i].append(result["id"]) - else: - self._logger.error("{0} : {1} not added due status code : {2}".format(self._name, i, page.status_code)) - self._logger.debug("{0} : {1}".format(self._name, page.content)) title = articletitle[0].text author = articleacreator[0].text.lower() @@ -541,21 +579,19 @@ class WPimport: page = self._request.get("{1}://{0}/wp-json/wp/v2/users".format(self._wordpress, self._protocol), auth=self._basic, headers=self._headers_json, params=params) self._logger.debug("{0} : End Search author : {1}".format(self._name, author)) self._logger.debug("{0} : Debug requests : {1}".format(self._name, page.content)) - - + if page.status_code == 200: + self._logger.info("{0} : Get author id : {1}".format(self._name, result)) + result = page.json() + for a in result: + data["author"] = a["id"] + else: + self._logger.error("{0} : Connection error with status code for get author : {1}".format(self._name, page.status_code)) + self._logger.debug("{0} : {1}".format(page.content)) except ConnectionError as err: self._logger.error("{0} : Connection error for get author : {1}".format(self._name, err)) exit(1) except Exception as err: self._logger.error("{0} : Exception error for get author : {1}".format(self._name, err)) - if page.status_code == 200: - self._logger.info("{0} : Get author id : {1}".format(self._name, result)) - result = page.json() - for a in result: - data["author"] = a["id"] - else: - self._logger.error("{0} : Connection error with status code for get author : {1}".format(self._name, page.status_code)) - self._logger.debug("{0} : {1}".format(page.content)) page_is_exist = False for index in range(1,10): @@ -563,87 +599,89 @@ class WPimport: try: self._logger.info("{0} : Search post with index {2} : {1}".format(self._name, title, index)) page = self._request.get("{1}://{0}/wp-json/wp/v2/posts".format(self._wordpress, self._protocol), auth=self._basic, params=params, headers=self._headers_json) + if page.status_code == 200: + self._logger.debug("{0} : Encoding : {1}".format(self._name, page.encoding)) + page.encoding = "utf-8" + result = page.json() + if len(result) == 0: + break + self._logger.info("{0} : Number result posts : {1}".format(self._name, len(result))) + count = 0 + for i in result: + title_rendered = i["title"]["rendered"] + self._logger.info("{0} : Search title posts for |{2}| : |{1}|".format(self._name, title_rendered, title)) + if len(title_rendered) != len(title): + title_rendered = self._replaceCaracter(title_rendered) + self._logger.debug("{0} : Search title posts for |{2}| : |{1}|".format(self._name, title_rendered, title)) + self._logger.debug("{0} : SIze of title : {1} - {2}".format(self._name, len(title), len(title_rendered))) + if title_rendered == title: + page_is_exist = True + post_id = i["id"] + count = count + 1 + if count > 1: + self._logger.info("{0} : Page {1} is double and going to delete".format(self._name, title)) + try: + params = {"force":1} + page = self._request.delete("{2}://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, post_id, self._protocol), auth=self._basic, headers=self._headers_json, params=params) + if page.status_code == 200: + self._logger.info("{0} : Post deleted : {1}".format(self._name, title)) + else: + self._logger.error("{0} : Post not updated due status code : {1}".format(self._name, page.status_code)) + self._logger.debug("{0} : {1}".format(self._name, page.content)) + except ConnectionError as err: + self._logger.error("{0} : Connection error for deleted post : {1}".format(self._name, err)) + exit(1) + except Exception as err: + self._logger.error("{0} : Exception error for deleted post : {1}".format(self._name, err)) + + else: + self._logger.debug("{0} : Data for post to update : {1}".format(self._name, i)) + self._logger.info("{0} : Page {1} already exist and going to update".format(self._name, title)) + + try: + page = self._request.post("{2}://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, post_id, self._protocol), auth=self._basic, headers=self._headers_json, data=json.dumps(data)) + + if page.status_code == 200: + result = page.json() + self._logger.info("{0} : Post updated : {1}".format(self._name, title)) + self._addOrUpdateComment(result["id"], comment_post, result["title"]["raw"]) + self._linkImgPost(result["title"]["raw"], list_img, result["id"]) + else: + self._logger.error("{0} : Post not updated due status code : {1}".format(self._name, page.status_code)) + self._logger.debug("{0} : {1}".format(self._name, page.content)) + except ConnectionError as err: + self._logger.error("{0} : Connection error for update post : {1}".format(self._name, err)) + exit(1) + except Exception as err: + self._logger.error("{0} : Exception error for update post : {1}".format(self._name, err)) + if page.status_code == 400: + self._logger.error("{0} : Connection for update post unauthorized : {1}".format(self._name, page.status_code)) + self._logger.debug("{0} : {1}".format(self._name, page.content)) + break + else: + self._logger.error("{0} : Connection for update post error with status code : {1}".format(self._name, page.status_code)) + self._logger.debug("{0} : {1}".format(self._name, page.content)) except ConnectionError as err: self._logger.error("{0} : Connection error for search post : {1}".format(self._name, err)) exit(1) except Exception as err: self._logger.error("{0} : Exception error for search post : {1}".format(self._name, err)) - if page.status_code == 200: - self._logger.debug("{0} : Encoding : {1}".format(self._name, page.encoding)) - page.encoding = "utf-8" - result = page.json() - if len(result) == 0: - break - self._logger.info("{0} : Number result posts : {1}".format(self._name, len(result))) - count = 0 - for i in result: - title_rendered = i["title"]["rendered"] - self._logger.info("{0} : Search title posts for |{2}| : |{1}|".format(self._name, title_rendered, title)) - if len(title_rendered) != len(title): - title_rendered = self._replaceCaracter(title_rendered) - self._logger.debug("{0} : Search title posts for |{2}| : |{1}|".format(self._name, title_rendered, title)) - self._logger.debug("{0} : SIze of title : {1} - {2}".format(self._name, len(title), len(title_rendered))) - if title_rendered == title: - page_is_exist = True - post_id = i["id"] - count = count + 1 - if count > 1: - self._logger.info("{0} : Page {1} is double and going to delete".format(self._name, title)) - try: - params = {"force":1} - page = self._request.delete("{2}://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, post_id, self._protocol), auth=self._basic, headers=self._headers_json, params=params) - except ConnectionError as err: - self._logger.error("{0} : Connection error for deleted post : {1}".format(self._name, err)) - exit(1) - except Exception as err: - self._logger.error("{0} : Exception error for deleted post : {1}".format(self._name, err)) - if page.status_code == 200: - self._logger.info("{0} : Post deleted : {1}".format(self._name, title)) - else: - self._logger.error("{0} : Post not updated due status code : {1}".format(self._name, page.status_code)) - self._logger.debug("{0} : {1}".format(self._name, page.content)) - - else: - self._logger.debug("{0} : Data for post to update : {1}".format(self._name, i)) - self._logger.info("{0} : Page {1} already exist and going to update".format(self._name, title)) - - try: - page = self._request.post("{2}://{0}/wp-json/wp/v2/posts/{1}".format(self._wordpress, post_id, self._protocol), auth=self._basic, headers=self._headers_json, data=json.dumps(data)) - except ConnectionError as err: - self._logger.error("{0} : Connection error for update post : {1}".format(self._name, err)) - exit(1) - except Exception as err: - self._logger.error("{0} : Exception error for update post : {1}".format(self._name, err)) - if page.status_code == 200: - result = page.json() - self._logger.info("{0} : Post updated : {1}".format(self._name, title)) - self._addOrUpdateComment(result["id"], comment_post, result["title"]["raw"]) - self._linkImgPost(result["title"]["raw"], list_img, result["id"]) - else: - self._logger.error("{0} : Post not updated due status code : {1}".format(self._name, page.status_code)) - self._logger.debug("{0} : {1}".format(self._name, page.content)) - if page.status_code == 400: - self._logger.error("{0} : Connection for update post unauthorized : {1}".format(self._name, page.status_code)) - self._logger.debug("{0} : {1}".format(self._name, page.content)) - break - else: - self._logger.error("{0} : Connection for update post error with status code : {1}".format(self._name, page.status_code)) - self._logger.debug("{0} : {1}".format(self._name, page.content)) if page_is_exist is False: try: self._logger.info("{0} : Creating posts : {1}".format(self._name, data["title"])) page = self._request.post("{1}://{0}/wp-json/wp/v2/posts".format(self._wordpress, self._protocol), auth=self._basic, headers=self._headers_json, data=json.dumps(data)) + + if page.status_code == 201: + result = page.json() + self._logger.info("{0} : Post added : {1}".format(self._name, result["title"]["raw"])) + self._addOrUpdateComment(result["id"], comment_post, result["title"]["raw"]) + self._linkImgPost(result["title"]["raw"], list_img, result["id"]) + else: + self._logger.error("{0} : Post not added due status code : {1}".format(self._name, r.status_code)) + self._logger.debug("{0} : {1}".format(self._name, r.content)) except ConnectionError as err: self._logger.error("{0} : Connection error for create post : {1}".format(self._name, err)) exit(1) except Exception as err: self._logger.error("{0} : Exception error for create post : {1}".format(self._name, err)) - if page.status_code == 201: - result = page.json() - self._logger.info("{0} : Post added : {1}".format(self._name, result["title"]["raw"])) - self._addOrUpdateComment(result["id"], comment_post, result["title"]["raw"]) - self._linkImgPost(result["title"]["raw"], list_img, result["id"]) - else: - self._logger.error("{0} : Post not added due status code : {1}".format(self._name, r.status_code)) - self._logger.debug("{0} : {1}".format(self._name, r.content))