web_scrap/lib/WPExport.py

#!/usr/bin/python3
from bs4 import BeautifulSoup
from urllib.parse import urlparse
import requests, os, argparse, logging
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry

class WPExport:
    def __init__(self, name = "Thread-0", url = "", logger = None, parser = "html.parser", directory = "backup"):
        self._url = url
        self._logger = logger
        self._parser = parser
        self._dir = directory
        self._name = name

        
        self._request = requests.Session()

        retries = Retry(total=10,
                status_forcelist=[429, 500, 502, 503, 504], backoff_factor=2)

        self._request.mount('http://', HTTPAdapter(max_retries=retries))


    # Destructor 
    def __del__(self):
        self._logger.info("{0} : Export finished for {1}".format(self._name, self._url))

    # Public method


    # Set name

    def setName(self, name):
        self._name = "Thread-{0}".format(int(name) + 1)

    # Set URL

    def setUrl(self, url):
        self._url = url

    # Download JS

    def downloadJs(self):
        script = self._getScriptCss(True, False)
        o = urlparse(self._url)
        self._downloadPage(script, "{0}/{1}/{2}".format(self._dir, o.path, "dists/js"))

    # Download CSS

    def downloadCss(self):
        css = self._getScriptCss(False, True)
        o = urlparse(self._url)
        self._downloadPage(css, "{0}/{1}/{2}".format(self._dir, o.path, "dists/css"))

    # Download HTML

    def downloadHTML(self, webpage):
        self._downloadPage(webpage, self._dir)

    # Download Image
    
    def downloadImg(self, webpage):
        page_src = self._getImg(webpage)
        o = urlparse(self._url)
        self._downloadPage(page_src, "{0}/{1}/{2}".format(self._dir, o.path, "img"))


     # Get URL
    def getUrlPage(self, index_thread, max_thread):
        try:
            page = self._request.get(self._url)
        except Exception as err:
            self._logger.error("{0} : Connection error : {1}".format(self._name, err))
            exit(1)
        page_url = []
        if page.status_code == 200:
            soup = BeautifulSoup(page.text, self._parser)
            ul = soup.find_all("ul", id="listsmooth")
            for anchor in ul[0].find_all("a"):
                href = anchor.get('href', '/')
                if href != "#":
                    page_url.append(href)
        else:
            self._logger.error("{0} : URL did not get due status code : {1}".format(self._name, page.status_code))
            self._logger.debug("{0} : {1}".format(self._name, page.content))

        webpage = {"principal": {"page":[], "article":[]}, "publications": {"page":[], "article":[]}}
        for i in page_url:
            section = "publications"
            o = urlparse(i)
            o = o._replace(scheme="https")
            i = o.geturl().replace(":///", "://")
            if i == "{0}/".format(self._url):
                section = "principal"
            try:
                page = self._request.get(i)
            except Exception as err:
                self._logger.error("{0} : Connection error : {1}".format(self._name, err))
                exit(1)
            if page.status_code == 200:
                self._logger.info("{0} : page : {1}".format(self._name, i))
                if i not in webpage[section]["page"]:
                    webpage[section]["page"].append(i)
                soup = BeautifulSoup(page.text, self._parser)
                class_div = pagingfirstline = soup.find_all("div", class_="pagingfirstline")
                if len(class_div) > 0:
                    pagingfirstline = class_div[0].find_all("a")
                    if len(pagingfirstline) > 1:
                        lastpage = pagingfirstline[len(pagingfirstline)-1].get("href", "/")
                        element_lastpage = lastpage.split("/")[len(lastpage.split("/"))-1]
                        number_page = element_lastpage.split("-")[0].split("p")[1]
                        number_lastpage = int(number_page) / 10

                        setPageDivided = int(number_lastpage) / max_thread
                        setPagePart = setPageDivided * (index_thread + 1)
                        firstPagePart = (setPagePart - setPageDivided)

                        self._logger.debug("{0} : Total page : {1}".format(self._name,int(number_lastpage)))
                        self._logger.debug("{0} : First range : {1}".format(self._name, int(firstPagePart)))
                        self._logger.debug("{0} : Last range : {1}".format(self._name, int(setPagePart)))

                        for j in range(int(firstPagePart),int(setPagePart)):
                            paging = j * 10
                            categorie = urlparse(i).path.split("/")
                            url_paging = "{0}/archives/p{1}-10.html".format(self._url, paging)
                            if len(categorie) > 2:
                                url_paging = "{0}/archives/{1}/p{2}-10.html".format(self._url, categorie[2], paging)
                            self._logger.info("{0} : {1}".format(self._name, url_paging))
                            if url_paging not in webpage[section]["page"]:
                                webpage[section]["page"].append(url_paging)
                            page = self._request.get(url_paging)
                            if page.status_code == 200:
                                soup = BeautifulSoup(page.text, self._parser)
                                h2 = soup.find_all("h2")
                                for title in h2:
                                    href = title.find_all("a")[0].get("href", "/")
                                    if href not in webpage[section]["article"]:
                                        try:
                                            o = urlparse(href)
                                            o = o._replace(scheme="https").geturl()
                                        except Exception as err:
                                            self._logger.error("parsing error : {0}".format(err))
                                            exit(1)
                                        webpage[section]["article"].append(o)
            else:
                self._logger.error("{0} : web didn't get due status code : {1}".format(self._name, page.status_code))
                self._logger.debug("{0} : {1}".format(self._name, page.content))
        return webpage
        


    # Private method
    # 
    # Create path        
    def _mkdirPath(self, path_dir):
        if not os.path.exists(path_dir):
            makedir = []
            pathh = path_dir.split("/")
            for i in pathh:
                makedir.append(i)
                repath = "/".join(makedir)
                if not os.path.exists(repath):
                    self._logger.debug("{0} : Dossier crée : {1}".format(self._name, repath))
                    try:
                        if len(repath) > 0:
                            os.mkdir(repath)
                    except Exception as err:
                        self._logger.error("Directory error : {0}".format(err))
                        self._logger.debug("Directory error : {0} {1} {2} {3} {4}".format(err, path_dir, repath, pathh, makedir))
                        exit(1)


    # Get Css and JS
    def _getScriptCss(self, js, css):
        try:
            page = self._request.get(self._url)
        except Exception as err:
            self._logger.error("Connection error : {0}".format(err))
            exit(1)
        page_url = []
        if page.status_code == 200:
            soup = BeautifulSoup(page.text, self._parser)
            if js is True:
                script = soup.find_all("script")
                for anchor in script:
                    src = anchor.get("src", "/")
                    if src != "/":
                        try:
                            u = urlparse(self._url)
                            o = urlparse(src)
                        except Exception as err:
                            self._logger.error("parsing error : {0}".format(err))
                            exit(1)
                        if o.netloc == "":
                            o = o._replace(netloc=u.netloc)
                            o = o._replace(scheme=u.scheme)
                        page_url.append(o.geturl())
            if css is True:
                link = soup.find_all("link")
                for anchor in link:
                    rel = anchor.get("rel")
                    if rel[0] == "stylesheet":
                        href = anchor.get("href", "/")
                        if href != "/":
                            try:
                                u = urlparse(self._url)
                                o = urlparse(href)
                            except Exception as err:
                                self._logger.error("parsing error : {0}".format(err))
                                exit(1)
                            if o.netloc == "":
                                o = o._replace(netloc=u.netloc)
                                o = o._replace(scheme=u.scheme)
                            page_url.append(o.geturl())        
        else:
            self._logger.error("JS or CSS did not get due status code : {0}".format(page.status_code))
            self._logger.debug(page.content)

        return page_url

    # Get image

    def _getImg(self, webpage):
        page_img = []
        for i in webpage:
            try:
                page = self._request.get(i)
            except Exception as err:
                self._logger.error("{0} : Connection error : {1}".format(self._name, err))
                exit(1)
            if page.status_code == 200:
                soup = BeautifulSoup(page.text, self._parser)
                img = soup.find_all("img")
                self._logger.info("{0} : image from page: {1} : ".format(self._name,i))
                for anchor in img:
                    src = anchor.get("src", "/")
                    if src != "/":
                        if src not in page_img:
                            self._logger.info("{0} : image: {1} : ".format(self._name, src))
                            page_img.append(src)
            else:
                self._logger.error("{0} : Image did not get due status code : {1}".format(self._name, page.status_code))
                self._logger.debug("{0} : {1}".format(self._name, page.content))

        return page_img


    # Download page
    def _downloadPage(self, webpage, backup_dir):
   
        for i in range(0, len(webpage)):
            try:
                o = urlparse(webpage[i])
            except Exception as err:
                self._logger.error("parsing error : {0}".format(err))
                exit(1)
            path_web = o.path.split("/")
            filePageWeb = path_web[len(path_web)-1]
            path_web.pop(len(path_web)-1)
            dir_page_web = "/".join(path_web)
            self._mkdirPath("{0}/{1}/{2}".format(backup_dir, o.netloc, dir_page_web))
            try:
                r = self._request.get(webpage[i])
            except Exception as err:
                self._logger.error("Connection error : {0}".format(err))
                exit(1)
            if r.status_code == 200:
                fileDownload = "{0}/{1}/index.html".format(backup_dir, o.netloc)
                if len(dir_page_web) > 0 and len(filePageWeb) > 0:
                    fileDownload = "{0}/{1}{2}/{3}".format(backup_dir, o.netloc, dir_page_web, filePageWeb)
                self._logger.info("{0} : {1}/{2} : {3}".format(self._name, i+1, len(webpage), fileDownload))
                try:
                    open(fileDownload, "wb").write(r.content)
                except Exception as err:
                    self._logger.error("file error : {0}".format(err))
                    exit(1)
            else:
                self._logger.error("Not download due status code : {0}".format(r.status_code))
                self._logger.debug(r.content)