check image exist in media

.gitignore

@@ -0,0 +1,2 @@
+backup/
+web_scrap.log

insert_wordpress.py

@@ -0,0 +1,188 @@
+#!/usr/bin/python3
+from bs4 import BeautifulSoup
+from urllib.parse import urlparse
+from requests.auth import HTTPBasicAuth
+from getpass import getpass
+import requests, os, argparse, logging, re
+
+class WPimport:
+    
+    def __init__(self, basic, wordpress):
+        self.basic = basic
+        self.wordpress = wordpress
+
+    def fromFile(self, file):
+        with open(file, 'r') as f:
+            contents = f.read()
+            self.insertWordpress(contents)
+
+    def insertWordpress(self, content):
+        tags = []
+        month = {"janvier":"01", "février": "02", "mars": "03", "avril":"04", "mai": "05", "juin": "06", "juillet": "07", "août": "08", "septembre": "09", "octobre": "10", "novembre": "11", "décembre": "12"}
+        liste = ["categories", "tags"]
+        elements = {}
+        element = {}
+        listelement = {}
+
+        for i in liste:
+            page = requests.get("http://{0}/wp-json/wp/v2/{1}".format(self.wordpress,i))
+            if page.status_code == 200:
+                elements[i] = page.json()
+                element[i] = []
+                listelement[i] = []
+
+        soup = BeautifulSoup(content, 'html.parser')
+        articletitle = soup.find_all("h2", class_="articletitle")
+        articlebody = soup.find_all("div", class_="articlebody")
+        articledate = soup.find_all("span", class_="articledate")
+        articleacreator = soup.find_all("span", class_="articlecreator")
+        dateheader = soup.find_all("div", class_="dateheader")
+        itemfooter = soup.find_all("div", class_="itemfooter")
+        comment = soup.find_all("div", class_="comment_item")
+        img_a = articlebody[0].find_all("a", {"target": "_blank"})
+        list_img = []
+        for i in img_a:
+            new_img = {}
+            img = i.find_all("img")
+            if len(img) > 0:
+                href_a = i.get("href")
+                href_img = img[0].get("src")
+                page_img = requests.get(href_img)
+                img_break = False
+                if page_img.status_code == 404:
+                    href_img = href_a
+                    img_break = True
+                page = requests.get(href_img)
+                if page.status_code == 200:
+            
+                    split_fileimg = href_img.split("/")
+                    img_name = split_fileimg[len(split_fileimg)-1]
+                    params = { "search": img_name}
+                    r = requests.get("http://{0}/wp-json/wp/v2/media".format(self.wordpress), auth=self.basic, params=params)
+                    if r.status_code == 200:
+                        res = r.json()
+                        if len(res) == 0:
+                            data = page.content
+                            img_type = "image/png"
+                            if img_name.split(".")[1]:
+                                img_type = "image/jpg"
+                            headers={ 'Content-Type': img_type,'Content-Disposition' : 'attachment; filename={0}'.format(img_name)}                   
+                            r = requests.post("http://{0}/wp-json/wp/v2/media".format(self.wordpress), auth=self.basic, headers=headers, data=data)
+                            if r.status_code == 201:
+                                res = r.json()
+
+                        new_img["old_src"]=href_img
+                        new_img["old_href"]=href_a
+                        new_img["id"]=res["id"]
+                        new_img["new_src"]=res["guid"]["rendered"]
+                        new_img["break"]=img_break
+                        list_img.append(new_img)
+                
+                            
+        comment_post = []
+        for i in comment:
+            comment_item = i.text.split("\n")
+            footer = i.find_all("div", class_="itemfooter")
+            comment_author = footer[0].text.split(",")[0].replace("Posté par ", "")
+            comment_date = footer[0].find_all("abbr")[0].get("title")
+            comment_content = "<p>"
+            for j in range(0, len(comment_item)-2):
+                if len(comment_item[j]) > 0:
+                    comment_content = comment_content + comment_item[j] + "<br />"
+            comment_content = comment_content + "</p>"
+            comment_post.append({"author": comment_author, "date": comment_date, "content": comment_content})
+        a = itemfooter[0].find_all("a", {"rel": True})
+        for i in a:
+            rel = i.get("rel")
+            if rel[0] == 'tag':
+                href = i.get("href")
+                if re.search(r'/tag/', href):
+                    element["tags"].append(i.text)
+                if re.search(r'/archives/', href):
+                    element["categories"].append(i.text)
+        for i in liste:
+            for j in element[i]:
+                element_exist = False
+                for k in elements[i]:
+                    if k["name"] == j:
+                        element_exist = True
+                        array = listelement[i].append(k["id"])
+                if element_exist is False:
+                    data = {"name": j}
+                    page = requests.post("http://{0}/wp-json/wp/v2/{1}".format(self.wordpress, i), auth=self.basic, data=data)
+                    if page.status_code == 201:
+                        result = page.json()
+                        listelement[i].append(result["id"])
+
+        title = articletitle[0].text
+        author = articleacreator[0].text.lower()        
+        body = articlebody[0].find_all("p")
+        bodyhtml = "<p>"
+        for i in body:
+            if len(i.text) == 1:
+                bodyhtml = bodyhtml + "<br />"
+            else:
+                bodyhtml = bodyhtml + str(i).replace("<p>", "").replace("</p>", "").replace("<br>", "<br />") + "<br />"
+        bodyhtml = bodyhtml + "</p>"
+        for i in list_img:
+            o = urlparse(i["new_src"])
+            if i == True:
+                print(i["old_href"])
+                bodyhtml = bodyhtml.replace(i["old_href"], o.path)
+            else:
+                print(i["old_src"])
+                bodyhtml = bodyhtml.replace(i["old_src"], o.path)
+        print(bodyhtml)
+        exit(0)
+        hour = articledate[0].text
+        time = dateheader[0].text.split(" ")
+
+        data = {"title":title, "content":bodyhtml, "status":"publish", "date": "{0}-{1}-{2}T{3}:00".format(time[2],month[time[1]],time[0], hour), "tags": listelement["tags"], "categories": listelement["categories"]}
+        params = {"search":author}
+        page = requests.get("http://{0}/wp-json/wp/v2/users".format(self.wordpress), auth=self.basic, params=params)
+        if page.status_code == 200:
+            result = page.json()
+            data["author"] = result[0]["id"]
+
+        params = {"search":title}
+        page = requests.get("http://{0}/wp-json/wp/v2/posts".format(self.wordpress), auth=self.basic, params=params)
+        page_exist = True
+        if page.status_code:
+            result = page.json()
+            if len(result) == 0:
+                page_exist = False
+            else:
+                print("La page {0} existe deja".format(title))
+
+        if page_exist == False:
+            page = requests.post("http://{0}/wp-json/wp/v2/posts".format(self.wordpress), auth=self.basic, data=data)
+            if page.status_code == 201:
+                result = page.json()
+                print("Article ajoute : {0}".format(result["title"]["raw"]))
+                print(comment_post)
+                for i in comment_post:
+                    data = {"post": result["id"], "content": i["content"], "date": i["date"], "author_name": i["author"]}
+                    page = requests.post("http://{0}/wp-json/wp/v2/comments".format(self.wordpress), auth=self.basic, data=data)
+                    print(page.status_code)
+                    if page.status_code == 201:
+                        print("Commentaire ajoute pour {0}".format(result["title"]["raw"]))
+
+
+
+    
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--user", help="wordpress user", required=True)
+    parser.add_argument("--file", help="HTML file", required=True)
+    parser.add_argument("--wordpress", help="URL Wordpress", required=True)
+    args = parser.parse_args()
+    password = getpass()
+    if len(password) == 0:
+        print("No password error !!! ")
+        exit(1)
+
+    basic = HTTPBasicAuth(args.user, password)
+    importWp = WPimport(basic, args.wordpress)
+    importWp.fromFile(args.file)