import smtplib, os
from email.message import EmailMessage
from random import randrange
from webdriver_manager.firefox import GeckoDriverManager
from selenium import webdriver
import selenium
from utillc import *
import json
import time
from orange import *
from collections import defaultdict
import urllib.request
import tqdm, random
import ebooklib
from ebooklib import epub
from bs4 import BeautifulSoup

blacklist = [   '[document]',   'noscript', 'header',   'html', 'meta', 'head','input', 'script',   ]
# there may be more elements you don't want, such as "style", etc.

def thtml2ttext(thtml):
    Output = []
    for html in thtml:
        text =  chap2text(html)
        Output.append(text)
    return Output

def chap2text(chap):
    output = ''
    soup = BeautifulSoup(chap, 'html.parser')
    text = soup.find_all(text=True)
    for t in text:
        if t.parent.name not in blacklist:
            output += '{} '.format(t)
    return output.replace("\xa0","")

class Robot(RobotBase) :

    def save(self) :
        with open("bd.json", "w") as fd:
            fd.write(json.dumps(self.bd, indent=2, ensure_ascii=False))
    
    def __init__(self) :
        super().__init__()
        #self.load_bd()

    
    def get_text(self, path) :
        book = epub.read_epub(path)
        chapters = []
        for item in book.get_items():
            if item.get_type() == ebooklib.ITEM_DOCUMENT:
                #EKOX(item.get_content())
                chapters.append(item.get_content())
                
        return chapters

    def text(self) :
        with open("bd.json", "r") as fd :
            bd = json.load(fd)
            xx =  [ ae['livres'] for (a, ae) in bd.items()]
            links = [ (a, t, l['lien']) for (a, ae) in bd.items() for t, l in ae['livres'].items()  ]            
            for (a,t,l) in links :
                fn = "./books/%s_%s.epub" % (a, t)
                try :
                    chapters = self.get_text(fn)
                    ttext = thtml2ttext(chapters)
                    EKOX(ttext)
                    
                except :
                    pass
                    
    def download(self) :
        os.makedirs("./books", exist_ok=True)
        with open("bd.json", "r") as fd :
            bd = json.load(fd)
            xx =  [ ae['livres'] for (a, ae) in bd.items()]
            #EKOX(xx)
            links = [ (a, t, l['lien']) for (a, ae) in bd.items() for t, l in ae['livres'].items()  ]
            EKOX(len(links))
            
            random.shuffle(links)
            for (a,t,l) in tqdm.tqdm(links) :
                EKOX((a, t))
                try :
                    urllib.request.urlretrieve(l, "./books/%s_%s.epub" % (a, t))
                    time.sleep(5)
                except Exception as e :
                    EKOX(e)
                    pass
                    
                
            
            
    def load_bd(self) :
        self.build()
        url = "https://www.ebooksgratuits.com/ebooks.php"
        browser_test = self.driver.get(url)
        auteur_x = "/html/body/div/div[5]/form/select[2]"
        offset_x = "/html/body/div/div[5]/form/select[1]"
        rechercher_x = "/html/body/div/div[5]/form/input[2]"
        resultat_x = "/html/body/div/div[6]/ul"
        auteur_select = self.driver.find_element(By.XPATH, auteur_x)
        offset_select = self.driver.find_element(By.XPATH, offset_x)
        
        self.select(offset_x, "500")
        
        auteurs_sub = auteur_select.find_elements(By.XPATH, '*')
        EKOX(auteur_select.get_attribute("id"))
        self.auteurs = [ (e.text, e.get_attribute("value"))  for ie, e in enumerate(auteurs_sub) ]
        EKO()
        self.bd = {}
        for name, i in self.auteurs :
            self.bd[name] = {}

        """ BD
        {
             <auteur> : {
                 "nmb_livres" : 33,
                 "id" : 999,
                 "livres" : {
                         <titre> : {
                           "lien" : "http://xxx",
                           "texte" : "xxxx"
                         }
                  },
                  ...
        }
        """
        
        self.save()
        #auteur_field = self.driver.find_element(By.ID, "id_auteur")

        for i, (auteur, v) in enumerate(self.auteurs) :
            EKOX(auteur)
            #EKOX(v)
            time.sleep(5)
            self.select(auteur_x, str(v))
            #EKO()
            self.click(rechercher_x)
            #EKOI(self.screen(), sz=800)

            books = self.driver.find_element(By.XPATH, resultat_x)
            books_sub =  books.find_elements(By.XPATH, '*')
            self.bd[auteur]['nmb_livres'] = len(books_sub)
            self.bd[auteur]['id'] = v
            self.bd[auteur]['livres'] = {}
            for e in tqdm.tqdm(books_sub) :
                try :
                    #EKOX(e.get_attribute('innerHTML'))
                    resume = e.text
                    #auteur = e.find_element(By.XPATH, './span[1]').text
                    #EKOX(auteur)
                    #/html/body/div/div[6]/ul/li[1]/span[2]/span
                    titre  = e.find_element(By.XPATH, './span[2]/span').text
                    #EKOX(titre)
                    #EKOX(resume)
                    lien  = e.find_element(By.XPATH, './a[7]').get_attribute("href")
                    #urllib.request.urlretrieve(lien, "./books/%s_%s.epub" % (auteur, titre))
                    #EKO()
                    self.bd[auteur]['livres'][titre] = { 'lien' : lien, 'resume' : resume}
                    #break                    
                except Exception as e :
                    #EKOX(e)
                    pass
            self.save()
        
    def check(self) :
        pass

if __name__ == '__main__':
    EKO()
    robot = Robot()
    #robot.download()
    robot.text()