v4l3n71n/web_scrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases 10 Wiki Activity

Compare commits

base: v4l3n71n:703cc8922a814d5bd10c6902731ec56882da8d35
Branches Tags
v4l3n71n:master
v4l3n71n:4.0.4
v4l3n71n:4.0.3
v4l3n71n:4.0.2
v4l3n71n:4.0.1
v4l3n71n:4.0.0
v4l3n71n:3.1.0
v4l3n71n:3.0.0
v4l3n71n:2.0.1
v4l3n71n:2.0.0
v4l3n71n:1.0.0
..
compare: v4l3n71n:f01a69a1e754c827746e36e44892d99b6e665a2e
Branches Tags
v4l3n71n:master
v4l3n71n:4.0.4
v4l3n71n:4.0.3
v4l3n71n:4.0.2
v4l3n71n:4.0.1
v4l3n71n:4.0.0
v4l3n71n:3.1.0
v4l3n71n:3.0.0
v4l3n71n:2.0.1
v4l3n71n:2.0.0
v4l3n71n:1.0.0

No commits in common. "703cc8922a814d5bd10c6902731ec56882da8d35" and "f01a69a1e754c827746e36e44892d99b6e665a2e" have entirely different histories.
703cc8922a ... f01a69a1e7

1 changed files with 12 additions and 29 deletions
Show Stats Expand all files Collapse all files

41
lib/WPImport.py
View File

@ -16,7 +16,6 @@ class WPimport:
self._parser = parser
self._headers_json = {'Content-Type': 'application/json; charset=utf-8', 'Accept':'application/json'}
self._protocol = "https"
self._directory = "backup"
if ssl_wordpress is False:
self._protocol = "http"
self._request = requests.Session()
@ -61,7 +60,6 @@ class WPimport:
def fromDirectory(self, directory="", number_thread=1, max_thread=1):
self._directory = directory
directory = "{0}/archives".format(directory)
directories = self._getDirectories([], "{0}".format(directory))
if len(directories) > 0:
@ -447,38 +445,23 @@ class WPimport:
if len(img) > 0:
href_a = i.get("href")
href_img = img[0].get("src")
href_a_o = urlparse(href_a)
href_img_o = urlparse(href_img)
new_img["old_src"]=href_img
new_img["old_href"]=href_a
try:
if len(href_img_o.netloc) > 0:
img_ok = False
page_img = self._request.get(href_img)
page_img = self._request.get(href_img)
if page_img.status_code == 404:
href_img = href_a
try:
page_img = self._request.get(href_a)
if page_img.status_code == 200:
img_ok = True
except ConnectionError as err:
self._logger.error("{0} : Connection error for get image : {1}".format(self._name, err))
exit(1)
except Exception as err:
self._logger.error("{0} : Exception error for get image : {1}".format(self._name, err))
exit(1)
else:
if os.path.exists("{0}/..{1}".format(self._directory, href_img)):
page_img = open("{0}/..{1}".format(self._directory, href_img), "r")
img_ok = True
else:
if os.path.exists("{0}/..{1}".format(self._directory, href_a)):
page_img = open("{0}/..{1}".format(self._directory, href_a), "r")
img_ok = True
if page_img.status_code == 404:
href_img = href_a
try:
page_img = self._request.get(href_a)
except ConnectionError as err:
self._logger.error("{0} : Connection error for get image : {1}".format(self._name, err))
exit(1)
except Exception as err:
self._logger.error("{0} : Exception error for get image : {1}".format(self._name, err))
exit(1)
self._logger.debug("{0} : Status code for image {1} : {2}".format(self._name, href_img, page_img.status_code))
if img_ok is True:
if page_img.status_code == 200:
media=self._addOrUpdateMedia(href_img, page_img)
new_img["id"]=media["id"]
new_img["new_src"]=media["rendered"]