import os, tqdm, sys import requests import urllib.request import re #from bs4 import BeautifulSoup import multiprocessing import time from selenium import webdriver from selenium.webdriver.chrome.options import Options #from webdriver_manager.firefox import GeckoDriverManager from selenium import webdriver from utillc import * from selenium.webdriver.firefox.service import Service as ServiceF from selenium.webdriver.chrome.service import Service as ServiceC import PIL from selenium import webdriver from selenium.webdriver.common.keys import Keys from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.common.by import By from selenium.common.exceptions import TimeoutException import numpy as np #wd = browser = webdriver.Firefox(executable_path= GeckoDriverManager().install()) EKO() import sys from selenium.webdriver.firefox.options import Options as FirefoxOptions from selenium.webdriver.chrome.options import Options as ChromeOptions import time import cherrypy import argparse import PIL from PIL import Image import orange def tqdmtqdm(x) : return x class P(orange.RobotBase) : def __init__(self) : super().__init__() EKO() self.build(headless=True) EKO() pass def xx() : options = FirefoxOptions() options = ChromeOptions() options.add_argument('--headless') options.add_argument('--no-sandbox') options.add_argument('--disable-dev-shm-usage') # changing user-agent because etoro detects the automated browser somehow options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/86.0.4240.183 Safari/537.36") EKO() s = ServiceC('/usr/bin/chromedriver') bot = self.driver = webdriver.Chrome(service=s, options=options) #self.driver = webdriver.Chrome('/usr/lib/chromium-browser/chromedriver', options=options) #self.driver = webdriver.Firefox(options=options) #self.driver.get("https://pythonbasics.org") #wd = webdriver.Firefox(service=service, options=options) EKO() def load(self, outdir, unfold) : self.url = "https://podcloud.fr/podcast/franck-ferrand-raconte" bot = self.driver browser = self.driver.get(self.url) EKO() okay = '//*[@class="btn btn-primary btn-lg"]' okay = "//button[text()='Okay !']" EKOX(bot.find_element(By.XPATH, okay).get_attribute('innerHTML')) time.sleep(2) self.driver.implicitly_wait(5) EKO() okay = bot.find_element(By.XPATH, okay) okay.click() EKO() self.click(okay, absentOK=True) EKOX(self.driver.title) html_source = self.driver.page_source #with open("html.html", "w") as fd : fd.write(html_source) EKOX(hash(html_source)) #EKOX(dir(bot)) for i in range(unfold) : EKOX(i) elem = bot.find_element(By.XPATH, "//*[text()='Afficher plus']") EKOX(elem) try : elem.click() time.sleep(2) except Exception as ex: EKOX(ex) EKOT("tout déplié") break #//*[@id="outer-main"]/section/article/div[2]/div/div/div[2]/table/tbody/tr[3] # /html/body/div[3]/section/article/div[2]/div/div/div[2]/table/tbody/tr[2] # /html/body/div[3]/section/article/div[2]/div/div/div[2]/table/tbody/tr[3] xpath = "/html/body/section[2]/section/div[2]/section/div[2]/div[2]/div" #//*[@id="outer-main"]/section/article/div[2]/div/div/div[3]/table/tbody/tr[1] #//*[@id="outer-main"]/section/article/div[2]/div/div/div[3]/table/tbody/tr[3] xpath = '//*[@id="outer-main"]/section/article/div[2]/div/div/div[3]/table/tbody/tr' xpath = '//table/tbody' xpath = '//*[@id="content"]/div[2]/section/div[2]/div[2]/div' #elems = bot.find_elements(By.XPATH, xpath) xpath = "/html/body/section[2]/section/div[2]/div/div/section/div[2]/div[2]/div" elems = bot.find_elements(By.XPATH, xpath) EKOX(len(list(elems))) def loade(ie, e) : EKOX(ie) aa = bot.find_element(By.XPATH, xpath + "[%d]/a" % (ie+1)) #EKOX(e.get_attribute('innerHTML')) #EKOX(aa.get_attribute('innerHTML')) #EKOX(e.find_element(By.XPATH, '//span[2]/a').get_attribute('innerHTML')) title = e.find_element(By.XPATH, './a').get_attribute('innerHTML') #title = aa.text EKOX(title) #EKOX(e.find_element(By.XPATH, '//li[@role="presentation"]').get_attribute('innerHTML')) #EKOX(e.find_element(By.XPATH, '//a[@target="_blank"]').get_attribute('innerHTML')) bxp = './/div[@class="dropdown inline"]' bxp = './/div/a[@data-toggle="dropdown"]' #EKOX(e.find_element(By.XPATH, bxp).get_attribute('innerHTML')) bb = e.find_element(By.XPATH, bxp) time.sleep(1) bb.click() time.sleep(2) #EKOX(e.find_element(By.XPATH, './/ul/li/a[@target="_blank"]').get_attribute('innerHTML')) #EKOX(e.find_element(By.XPATH, './/ul/li/a[@target="_blank"]').get_attribute('href')) href = e.find_element(By.XPATH, './/ul/li/a[@target="_blank"]').get_attribute('href') EKOX(href) filename = title.replace("\n", "").replace("/", "_").strip() + ".mp3" EKOX(filename) # declick ce podcast bb.click() pth = os.path.join(outdir, filename) if not os.path.exists(pth) : EKOT("downloading") doc = requests.get(href) EKOT("writing %s" % pth) with open(pth, 'wb') as f: f.write(doc.content) EKOT("%s written" % filename) else : EKOT("%s already there" % pth) for ie, e in tqdmtqdm(enumerate(list(elems))) : try : loade(ie, e) except Exception as e: EKOX(e) time.sleep(1) bot.close() # shuts down the bot if __name__ == '__main__': parser = argparse.ArgumentParser( prog='ferrand aspirator', description='download ferrand podcast') parser.add_argument('--outdir') parser.add_argument('--nunfold', type=int, default=2) args = parser.parse_args() p = P() os.makedirs(args.outdir, exist_ok=True) p.load(args.outdir, args.nunfold)