web_scrap/insert_wordpress.py

#!/usr/bin/python3
from bs4 import BeautifulSoup
from urllib.parse import urlparse
from requests.auth import HTTPBasicAuth
from getpass import getpass
import requests, os, argparse, logging, re


if __name__ == '__main__':
    tags = []
    month = {"janvier":"01", "février": "02", "mars": "03", "avril":"04", "mai": "05", "juin": "06", "juillet": "07", "août": "08", "septembre": "09", "octobre": "10", "novembre": "11", "décembre": "12"}
    parser = argparse.ArgumentParser()
    parser.add_argument("--user", help="wordpress user", required=True)
    parser.add_argument("--file", help="HTML file", required=True)
    args = parser.parse_args()
    password = getpass()
    if len(password) == 0:
        print("No password error !!! ")
        exit(1)
    basic = HTTPBasicAuth(args.user, password)

    liste = ["categories", "tags"]
    elements = {}
    element = {}
    listelement = {}

    for i in liste:
        page = requests.get("http://localhost:8080/wp-json/wp/v2/{0}".format(i))
        if page.status_code == 200:
            elements[i] = page.json()
            element[i] = []
            listelement[i] = []

    
    with open(args.file, 'r') as f:
        contents = f.read()

        soup = BeautifulSoup(contents, 'html.parser')
        articletitle = soup.find_all("h2", class_="articletitle")
        articlebody = soup.find_all("div", class_="articlebody")
        articledate = soup.find_all("span", class_="articledate")
        dateheader = soup.find_all("div", class_="dateheader")
        itemfooter = soup.find_all("div", class_="itemfooter")
        a = itemfooter[0].find_all("a", {"rel": True})
        for i in a:
            rel = i.get("rel")
            if rel[0] == 'tag':
                href = i.get("href")
                if re.search(r'/tag/', href):
                    element["tags"].append(i.text)
                if re.search(r'/archives/', href):
                    element["categories"].append(i.text)
        for i in liste:
            for j in element[i]:
                element_exist = False
                for k in elements[i]:
                    if k["name"] == j:
                        element_exist = True
                        array = listelement[i].append(k["id"])
                if element_exist is False:
                    data = {"name": j}
                    page = requests.post("http://localhost:8080/wp-json/wp/v2/{0}".format(i), auth=basic, data=data)
                    if page.status_code == 201:
                        result = page.json()
                        listelement[i].append(result["id"])


      

        title = articletitle[0].text
        body = articlebody[0].find_all("p")
        bodyhtml = "<p>"
        for i in body:
            if len(i.text) == 1:
                bodyhtml = bodyhtml + "<br />"
            else:
                bodyhtml = bodyhtml + str(i).replace("<p>", "").replace("</p>", "").replace("<br>", "<br />") + "<br />"
        bodyhtml = bodyhtml + "</p>"
        print(bodyhtml)
        hour = articledate[0].text
        time = dateheader[0].text.split(" ")
        data = {"title":title, "content":bodyhtml, "status":"publish", "date": "{0}-{1}-{2}T{3}:00".format(time[2],month[time[1]],time[0], hour), "tags": listelement["tags"], "categories": listelement["categories"]}
    params = {"search":title}
    page = requests.get("http://localhost:8080/wp-json/wp/v2/posts", auth=basic, params=params)
    page_exist = True
    if page.status_code:
        result = page.json()
        if len(result) == 0:
            page_exist = False

    if page_exist == False:
        page = requests.post("http://localhost:8080/wp-json/wp/v2/posts", auth=basic, data=data)
        if page.status_code == 201:
            print(page.content)