v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

Merge pull request 'remove-thread' (#14) from remove-thread into master

Browse Source 
Reviewed-on: #14
This commit is contained in:
v4l3n71n 2023-05-26 22:18:19 +00:00
commit 42cfb30583
2 changed files with 85 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
import_export_canalblog.py
View File

@ -10,8 +10,8 @@ from lib.WPImport import WPimport
from lib.WPExport import WPExport from lib.WPExport import WPExport
from lib.WPRemove import WPRemove from lib.WPRemove import WPRemove
def remove(args, basic, logger, ssl_wordpress): def remove(index, number, args, basic, logger, ssl_wordpress):
removeWp = WPRemove(basic=basic, wordpress="", logger=logger, ssl_wordpress=ssl_wordpress) removeWp = WPRemove(basic=basic, wordpress="", logger=logger, ssl_wordpress=ssl_wordpress, index_name=index, number_thread=number)
if args.remove == True: if args.remove == True:
for i in args.wordpress.split(","): for i in args.wordpress.split(","):
removeWp.setUrl(i) removeWp.setUrl(i)
@ -280,5 +280,9 @@ if __name__ == '__main__':
if args.command == "remove": if args.command == "remove":
remove(args, basic, logger, ssl_wordpress) try:
with futures.ThreadPoolExecutor(max_workers=int(args.parallel)) as ex:
wait_for = [ ex.submit(remove, i, args.parallel, args, basic, logger, ssl_wordpress) for i in range(0, int(args.parallel)) ]
except Exception as err:
logger.error("Thread error for remove : {0}".format(err))
exit(0) exit(0)

106
lib/WPRemove.py
View File

@ -8,14 +8,15 @@ from requests.packages.urllib3.util.retry import Retry
class WPRemove: class WPRemove:
# Constructor # Constructor
def __init__(self, name="Thread-0", basic=None, wordpress="", logger=None, ssl_wordpress=True): def __init__(self, index_name=1, number_thread=1, basic=None, wordpress="", logger=None, ssl_wordpress=True):
self._name = name
self._basic = basic self._basic = basic
self._wordpress = wordpress self._wordpress = wordpress
self._logger = logger self._logger = logger
self._headers_json = {'Content-Type': 'application/json', 'Accept':'application/json'} self._headers_json = {'Content-Type': 'application/json', 'Accept':'application/json'}
self._name = "Thread-{0}".format(index_name)
self._index_thread = index_name
self._protocol = "https" self._protocol = "https"
self._number_thread = number_thread
if ssl_wordpress is False: if ssl_wordpress is False:
self._protocol = "http" self._protocol = "http"
self._request = requests.Session() self._request = requests.Session()
@ -32,6 +33,24 @@ class WPRemove:
# Public method # Public method
def _getCount(self, composant):
count = 0
try:
params = {"per_page":1}
self._logger.info("{0} : Get count {2} to remove for url : {1}".format(self._name, self._wordpress, composant))
r = self._request.get("{2}://{0}/wp-json/wp/v2/{1}".format(self._wordpress, composant, self._protocol), params=params, auth=self._basic, headers=self._headers_json)
if r.status_code == 200:
count = int(r.headers["X-WP-Total"])
else:
self._logger.error("{0} : Error for list to remove {1} due status code {2}".format(self._name, composant, r.status_code))
self._logger.debug("{0} : Content error for {1} : {2}".format(self._name, composant, r.content))
except ConnectionError as err:
self._logger.error("{0} : Connection error for list {1} to remove : {2}".format(self._name, composant, err))
exit(1)
except Exception as err:
self._logger.error("{0} : Exception error for list {1} to remove : {2}".format(self._name, composant, err))
return count
def setUrl(self, wordpress): def setUrl(self, wordpress):
self._wordpress = wordpress self._wordpress = wordpress
@ -50,29 +69,60 @@ class WPRemove:
# Private method # Private method
def _removeAll(self, composant): def _removeAll(self, composant):
params = {"per_page":100} count = self._getCount(composant)
try: self._logger.debug("{0} : Count for {1} : {2}".format(self._name, composant, count))
self._logger.info("{0} : List {2} to remove for url : {1}".format(self._name, self._wordpress, composant)) if count > 0:
self._logger.debug("{0} : Number thread for {1} : {2}".format(self._name, composant, self._number_thread))
r = self._request.get("{2}://{0}/wp-json/wp/v2/{1}".format(self._wordpress, composant, self._protocol), auth=self._basic, params=params, headers=self._headers_json) page = count / int(self._number_thread)
except Exception as err: self._logger.debug("{0} : Page for {1} : {2}".format(self._name, composant, page))
self._logger.error("{0} : Connection error for list {1} to remove : {2}".format(self._name, composant, err))
if r.status_code == 200: if page > int(page):
result = r.json() page = int(page) + 1
if len(result) > 0: if page > 100:
for i in result: page = 100
self._logger.info("{0} : Remove {2} for url {1} : {3}".format(self._name, self._wordpress, composant, i["title"]["rendered"])) params = {"per_page":page, "page":self._index_thread}
params = {"force":1} self._logger.info("{0} : Params for {1} : {2}".format(self._name, composant, params))
try:
r = self._request.delete("{3}://{0}/wp-json/wp/v2/{1}/{2}".format(self._wordpress, composant, i["id"], self._protocol), auth=self._basic, headers=self._headers_json , params=params) try:
if r.status_code == 200: self._logger.info("{0} : List {2} to remove for url : {1}".format(self._name, self._wordpress, composant))
self._logger.info("{0} : Post removed for URL {1} {2} : {3}".format(self._name, self._wordpress, composant, i["title"]["rendered"]))
else: r = self._request.get("{2}://{0}/wp-json/wp/v2/{1}".format(self._wordpress, composant, self._protocol), auth=self._basic, params=params, headers=self._headers_json)
self._logger.error("{0} : Connection error for post {1} {2} {3} with status code {4}".format(self._name, self._wordpress, composant, i["title"]["rendered"], r.status_code))
except Exception as err: if r.status_code == 200:
self._logger.error("{0} : Connection error for {1} remove : {2}".format(self._name, composant, err)) result = r.json()
exit(1) if len(result) > 0:
self._removeAll(composant) for i in result:
else: is_delete = True
self._logger.error("{0} : Error for list to remove {1} due status code {2}".format(self._name, composant, r.status_code)) self._logger.info(i["slug"])
self._logger.debug("{0} : Content error for {1} : {2}".format(self._name, composant, r.content)) if i["slug"] == "non-classe":
is_delete = False
if is_delete is True:
if composant == "tags" or composant == "categories":
title = i["name"]
else:
title = i["title"]["rendered"]
self._logger.info("{0} : Remove {2} for url {1} : {3}".format(self._name, self._wordpress, composant, title))
params = {"force":1}
try:
r = self._request.delete("{3}://{0}/wp-json/wp/v2/{1}/{2}".format(self._wordpress, composant, i["id"], self._protocol), auth=self._basic, headers=self._headers_json , params=params)
if r.status_code == 200:
self._logger.info("{0} : Post removed for URL {1} {2} : {3}".format(self._name, self._wordpress, composant, title))
else:
self._logger.error("{0} : Connection error for post {1} {2} {3} with status code {4}".format(self._name, self._wordpress, composant, title, r.status_code))
except ConnectionError as err:
self._logger.error("{0} : Connection error for {1} remove : {2}".format(self._name, composant, err))
exit(1)
except Exception as err:
self._logger.error("{0} : Exception error for {1} remove : {2}".format(self._name, composant, err))
self._removeAll(composant)
if r.status_code == 400:
self._logger.error("{0} : No content for {1} to remove : {2}".format(self._name, composant, r.status_code))
else:
self._logger.error("{0} : Error for list to remove {1} due status code {2}".format(self._name, composant, r.status_code))
self._logger.debug("{0} : Content error for {1} : {2}".format(self._name, composant, r.content))
except ConnectionError as err:
self._logger.error("{0} : Connection error for list {1} to remove : {2}".format(self._name, composant, err))
exit(1)
except Exception as err:
self._logger.error("{0} : Exception error for list {1} to remove : {2}".format(self._name, composant, err))