web_scrap/import_export_canalblog.py

#!/usr/bin/python3
from requests.auth import HTTPBasicAuth
from getpass import getpass
from urllib.parse import urlparse
from concurrent import futures
from concurrent.futures import as_completed, wait, ALL_COMPLETED

import argparse, logging, threading, os, glob
from lib.WPImport import WPimport
from lib.WPExport import WPExport
from lib.WPRemove import WPRemove
from lib.WPChange import WPChange

def change(index, number, args, logger):
    changeWp = WPChange(logger=logger, index_name=index, number_thread=number)
    changeWp.fromDirectory(args.directory)

    del changeWp

def remove(index, number, args, basic, logger, ssl_wordpress):
    removeWp = WPRemove(basic=basic, wordpress="", logger=logger, ssl_wordpress=ssl_wordpress, index_name=index, number_thread=number)
    if args.remove == True:
        for i in args.wordpress.split(","):
            removeWp.setUrl(i)
            removeWp.cleanPosts()
            removeWp.cleanTags()
            removeWp.cleanCategories()
            removeWp.cleanMedia()
    else:
        for i in args.wordpress.split(","):
            removeWp.setUrl(i)
            if args.posts == True:
                removeWp.cleanPosts()
            if args.categories == True:
                removeWp.cleanCategories()
            if args.tags == True:
                removeWp.cleanTags()
            if args.media == True:
                removeWp.cleanMedia()
    del removeWp


def download(name_thread, max_thread, url, logger, parser, directory, html, img, ssl_canalblog, revert, tmp):
    exportWp = WPExport(name="Thread-{0}".format(int(name_thread) + 1), url=url, logger=logger, parser=parser, directory=directory, ssl_canalblog=ssl_canalblog)    
    if not revert:
        exportWp.getUrlPage(name_thread, max_thread)
    for i in ["article", "page"]:
        for j in ["publications", "principal"]:
            if html is False:
                exportWp.downloadHTML(j, i)

            if img is False:
                exportWp.downloadImg(j, i)
    del exportWp



def importUrl(name_thread, max_thread, canalblog, logger, parser, wordpress, basic, serial, ssl_wordpress, ssl_canalblog, create, update, image, revert, tmp):
    canalblog = canalblog.split(",")
    wordpress = wordpress.split(",")
    name = "Thread-{0}".format(int(name_thread) + 1)
    protocol = "https"
    if ssl_canalblog is False:
        protocol = "http"
    if serial is False:
        for canal in canalblog:
            try:
                o = urlparse(canal)
                o = o._replace(scheme=protocol)
                url = o.geturl().replace(":///", "://")
            except Exception as err:
                logger.error("{0} : parsing error : {1}".format(name, err))
                exit(1)
            exportWp = WPExport(name="Thread-{0}".format(int(name_thread) + 1), url=url, logger=logger, parser=parser, ssl_canalblog=ssl_canalblog, tmp=tmp)
            if not revert:
                exportWp.getUrlPage(name_thread, max_thread)
            del exportWp
            for j in wordpress:
                importWp = WPimport(name=name, basic=basic, wordpress=j, logger=logger, parser=parser, ssl_wordpress=ssl_wordpress, no_create=create, no_update=update, no_image=image, tmp=tmp)
                for k in ["article", "page"]:
                    for l in ["publications", "principal"]:
                        importWp.fromUrl(l, k)

                del importWp
    else:
        if len(canalblog) != len(wordpress):
            logger.error("{0} : ERREUR : Le nombre de dossier n'est pas equivalent au nombre d'URL wordpress".format(name))
            exit(1)
        for i in range(0, len(canalblog)-1):
            try:
                o = urlparse(canalblog[i])
                o = o._replace(scheme=protocol)
                url = o.geturl().replace(":///", "://")
            except Exception as err:
                logger.error("parsing error : {0}".format(err))
                exit(1)
            exportWp = WPExport(name=name, url=url, logger=logger, parser=parser, ssl_canalblog=ssl_canalblog)
            if not revert:
                exportWp.getUrlPage(name_thread, max_thread)
            del exportWp
            importWp = WPimport(name=name, basic=basic, wordpress=wordpress[i], logger=logger, parser=parser, ssl_wordpress=ssl_wordpress, no_create=create, no_update=update, no_image=image, tmp=tmp)

            for k in ["article", "page"]:
                for l in ["publications", "principal"]:
                    importWp.fromUrl(webpage[l][k])

            del importWp
    

def importDirectory(name_thread, max_thread, directory, logger, parser, wordpress, basic, serial, ssl_wordpress, create, update, image):
    name = "Thread-{0}".format(int(name_thread) + 1)
    directory = directory.split(",")
    wordpress = wordpress.split(",")
    if serial is False:
        for i in wordpress:
            importWp = WPimport(name=name, basic=basic, wordpress=i, logger=logger, parser=parser, ssl_wordpress=ssl_wordpress, no_create=create, no_update=update, no_image=image)
            for j in directory:
                importWp.fromDirectory(j, name_thread, max_thread)
                del importWp

    else:
        if len(directory) != len(wordpress):
            logger.error("{0} : Error : Number directory is differant than wordpress".format(name))
            exit(1)
        for i in range(0, len(wordpress)-1):
            importWp = WPimport(name=name, basic=basic, wordpress=wordpress[i], logger=logger, parser=parser, ssl_wordpress=ssl_wordpress, no_create=create, no_update=update, no_image=image)
            importWp.fromDirectory(directory[i])
            del importWp



if __name__ == '__main__':
    TMP = "/tmp/import_export_canablog"
    parser = argparse.ArgumentParser()
    parser.add_argument("--debug", help="Verbosity",  action="store_true")
    parser.add_argument("--logfile", help="Log file",  default="")
    parser.add_argument("--quiet", help="No console output", action="store_true")
    parser.add_argument("--parser", help="Parser content", default="html.parser")
    parser.add_argument("--parallel", help="Define number thread (default : 1)", default=1)
    parser.add_argument("--no-ssl", help="No ssl for canalblog and/or wordpress (example wordpress,canalblog)", dest="ssl", default="")
    parser.add_argument("--revert", help="Restart a work from stopping work", action="store_true")

    subparsers = parser.add_subparsers(dest="command")

    import_parser = subparsers.add_parser("import")
    import_parser.add_argument("--user", help="wordpress user", required=True)
    import_parser.add_argument("--password", help="password wordpress's user", default="")
    import_parser.add_argument("--file", help="HTML file", default="")
    import_parser.add_argument("--directory", help="HTML directory", default="")
    import_parser.add_argument("--canalblog", help="URL Canalblog", default="")
    import_parser.add_argument("--wordpress", help="URL Wordpress", required=True)
    import_parser.add_argument("--serial", help="Serial execution", action="store_true")
    import_parser.add_argument("--remove-all", dest="remove", help="Remove all", action="store_true")
    import_parser.add_argument("--remove-posts", help="Remove all posts", dest="posts", action="store_true")
    import_parser.add_argument("--remove-categories", help="Remove all categories", dest="categories", action="store_true")
    import_parser.add_argument("--remove-tags", help="Remove all tags", dest="tags", action="store_true")
    import_parser.add_argument("--remove-media", help="Remove all media", dest="media", action="store_true")
    import_parser.add_argument("--no-create", help="No create post", dest="create", default="store_false", action="store_true")
    import_parser.add_argument("--no-update", help="No update post", dest="update", default="store_false", action="store_true")
    import_parser.add_argument("--no-image", help="No image add or update", dest="image", default="store_false", action="store_true")



    remove_parser = subparsers.add_parser("remove")
    remove_parser.add_argument("--user", help="wordpress user", required=True)
    remove_parser.add_argument("--password", help="password wordpress's user", default="")
    remove_parser.add_argument("--wordpress", help="URL Wordpress", required=True)
    remove_parser.add_argument("--all", dest="remove", help="Remove all (posts, media, tags, categories)", action="store_true")
    remove_parser.add_argument("--posts", help="Remove all posts", action="store_true")
    remove_parser.add_argument("--categories", help="Remove all categories", action="store_true")
    remove_parser.add_argument("--tags", help="Remove all tags", action="store_true")
    remove_parser.add_argument("--media", help="Remove all media", action="store_true")



    export_parser = subparsers.add_parser("export")

    export_parser.add_argument("--url", help="canblog URL to be scraping", required=True)
    export_parser.add_argument("--directory",
                      default="backup",
                      help="backup file path")
    export_parser.add_argument("--no-css", help="No CSS", dest="css",  action="store_true")
    export_parser.add_argument("--no-js", help="No JS", dest="js",  action="store_true")
    export_parser.add_argument("--no-img", help="No img", dest="img",  action="store_true")
    export_parser.add_argument("--no-html", help="No HTML", dest="html", action="store_true")

    change_parser = subparsers.add_parser("change")
    change_parser.add_argument("--directory",
                      default="",
                      help="Directory")
    change_parser.add_argument("--file",
                      default="",
                      help="File")


    if not os.path.exists(TMP):
        os.mkdir(TMP)
    
    args = parser.parse_args()

    logger = logging.getLogger('import export canalblog')
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    ssl_canalblog = True
    ssl_wordpress = True

    for i in args.ssl.split(","):
        if i == "canalblog":
            ssl_canalblog = False
        if i == "wordpress":
            ssl_wordpress = False

    if args.quiet is False:
        ch = logging.StreamHandler()
        if args.debug is True:
            logger.setLevel(logging.DEBUG)
            ch.setLevel(logging.DEBUG)
        else:
            logger.setLevel(logging.INFO)
            ch.setLevel(logging.INFO)
        ch.setFormatter(formatter)
        logger.addHandler(ch)


    if len(args.logfile) > 0:
        fileHandler = logging.FileHandler(args.logfile)
        if args.debug is True:
            fileHandler.setLevel(logging.DEBUG)
        else:
            fileHandler.setLevel(logging.INFO)
        fileHandler.setFormatter(formatter)
        logger.addHandler(fileHandler)

    if args.command == "import" or args.command == "remove":
        password = args.password
        if len(args.password) == 0:
            password = getpass()
            if len(password) == 0:
                logger.error("No password error !!! ")
                exit(1)

        basic = HTTPBasicAuth(args.user, password)
    if args.command == "import":
        wordpress = args.wordpress.split(",")
        importWp = WPimport(basic=basic, wordpress="", logger=logger, parser=args.parser, ssl_wordpress=ssl_wordpress)
        if len(args.file) > 0:
            for i in wordpress:
                importWp.setUrl(i)
                importWp.fromFile(files=args.file.split(","))
        if len(args.directory) > 0:
            try:
                with futures.ThreadPoolExecutor(max_workers=int(args.parallel)) as ex:
                    wait_for = [ ex.submit(remove, i, args.parallel, args, basic, logger, ssl_wordpress) for i in range(0, int(args.parallel)) ]
                    wait(wait_for, return_when=ALL_COMPLETED)
                    wait_for = [
                    ex.submit(importDirectory, i, int(args.parallel), args.directory, logger, args.parser, args.wordpress, basic, args.serial, ssl_wordpress, args.create, args.update, args.image, TMP, revert)
                    for i in range(0, int(args.parallel))
                    ]
            except Exception as err:
                logger.error("Threading error : {0}".format(err))
        if len(args.canalblog) > 0:
            try:
                with futures.ThreadPoolExecutor(max_workers=int(args.parallel)) as ex:
                    wait_for = [ ex.submit(remove, i, args.parallel, args, basic, logger, ssl_wordpress) for i in range(0, int(args.parallel)) ]
                    wait(wait_for, return_when=ALL_COMPLETED)
                    wait_for = [
                    ex.submit(importUrl, i, int(args.parallel), args.canalblog, logger, args.parser, args.wordpress, basic, args.serial, ssl_wordpress, ssl_canalblog, args.create, args.update, args.image, args.revert, TMP)
                    for i in range(0, int(args.parallel))
                    ]


            except Exception as err:
                logger.error("Threading error : {0}".format(err))
        exit(0)


    if args.command == "export":
        canalblog = args.url.split(",")
        protocol = "https"
        if ssl_canalblog is False:
            protocol = "http"
        exportWp = WPExport(logger=logger, parser=args.parser, directory=args.directory, ssl_canalblog=ssl_canalblog)
        for canal in canalblog:
            try:
                o = urlparse(canal)
                o = o._replace(scheme=protocol)
                url = o.geturl().replace(":///", "://")
            except Exception as err:
                logger.error("parsing error : {0}".format(err))
                exit(1)
            exportWp.setUrl(url)
            if args.js is False:
                exportWp.downloadJs()
    
            if args.css is False:
                exportWp.downloadCss()
            del exportWp


    
            if args.html is False or args.img is False:
                try:
                    with futures.ThreadPoolExecutor(max_workers=int(args.parallel)) as ex:
                        wait_for = [
                        ex.submit(download, i, int(args.parallel), url, logger, args.parser, args.directory, args.html, args.img, ssl_canalblog, args.revert, TMP)
                        for i in range(0, int(args.parallel))
                        ]
                except Exception as err:
                    logger.error("Threading error : {0}".format(err))
        exit(0)


    if args.command == "remove":
        try:
            with futures.ThreadPoolExecutor(max_workers=int(args.parallel)) as ex:
                wait_for = [ ex.submit(remove, i, args.parallel, args, basic, logger, ssl_wordpress) for i in range(0, int(args.parallel)) ]
        except Exception as err:
            logger.error("Thread error for remove : {0}".format(err))
        exit(0)

    if args.command == "change":
        if len(args.directory) > 0:
            try:
                with futures.ThreadPoolExecutor(max_workers=int(args.parallel)) as ex:
                    wait_for = [ ex.submit(change, i, args.parallel, args, logger) for i in range(0, int(args.parallel)) ]
            except Exception as err:
                logger.error("Thread error for remove : {0}".format(err))
        if len(args.file) > 0:
            changeWp = WPChange(logger=logger)
            for filei in args.file.split(","):
                changeWp.fromFile(filei)
        exit(0)