web_scrap/insert_wordpress.py

79 lines
3.1 KiB
Python

#!/usr/bin/python3
from bs4 import BeautifulSoup
from urllib.parse import urlparse
from requests.auth import HTTPBasicAuth
from getpass import getpass
import requests, os, argparse, logging, re
if __name__ == '__main__':
tags = []
month = {"janvier":"01", "février": "02", "mars": "03", "avril":"04", "mai": "05", "juin": "06", "juillet": "07", "août": "08", "septembre": "09", "octobre": "10", "novembre": "11", "décembre": "12"}
parser = argparse.ArgumentParser()
parser.add_argument("--user", help="wordpress user", required=True)
parser.add_argument("--file", help="HTML file", required=True)
args = parser.parse_args()
password = getpass()
if len(password) == 0:
print("No password error !!! ")
exit(1)
basic = HTTPBasicAuth(args.user, password)
liste = ["categories", "tags"]
elements = {}
element = {}
listelement = {}
for i in liste:
page = requests.get("http://localhost:8080/wp-json/wp/v2/{0}".format(i))
if page.status_code == 200:
elements[i] = page.json()
element[i] = []
listelement[i] = []
with open(args.file, 'r') as f:
contents = f.read()
soup = BeautifulSoup(contents, 'html.parser')
articletitle = soup.find_all("h2", class_="articletitle")
articlebody = soup.find_all("div", class_="articlebody")
articledate = soup.find_all("span", class_="articledate")
dateheader = soup.find_all("div", class_="dateheader")
itemfooter = soup.find_all("div", class_="itemfooter")
a = itemfooter[0].find_all("a", {"rel": True})
for i in a:
rel = i.get("rel")
if rel[0] == 'tag':
href = i.get("href")
if re.search(r'/tag/', href):
element["tags"].append(i.text)
if re.search(r'/archives/', href):
element["categories"].append(i.text)
for i in liste:
for j in element[i]:
element_exist = False
for k in elements[i]:
if k["name"] == j:
element_exist = True
array = listelement[i].append(k["id"])
if element_exist is False:
data = {"name": j}
page = requests.post("http://localhost:8080/wp-json/wp/v2/{0}".format(i), auth=basic, data=data)
if page.status_code == 201:
result = page.json()
listelement[i].append(result["id"])
title = articletitle[0].text
body = articlebody[0]
hour = articledate[0].text
time = dateheader[0].text.split(" ")
data = {"title":title, "content":body, "status":"publish", "date": "{0}-{1}-{2}T{3}:00".format(time[2],month[time[1]],time[0], hour), "tags": listelement["tags"], "categories": listelement["categories"]}
page = requests.post("http://localhost:8080/wp-json/wp/v2/posts", auth=basic, data=data)
print(page.status_code)
if page.status_code == 201:
print(page.content)