import os, tqdm, sys import requests import urllib.request import re from bs4 import BeautifulSoup import multiprocessing import time from selenium import webdriver from selenium.webdriver.chrome.options import Options #from webdriver_manager.firefox import GeckoDriverManager from selenium import webdriver from utillc import * from selenium.webdriver.firefox.service import Service as ServiceF from selenium.webdriver.chrome.service import Service as ServiceC from selenium.webdriver.common.action_chains import ActionChains import PIL from selenium import webdriver from selenium.webdriver.common.keys import Keys from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.common.by import By from selenium.common.exceptions import TimeoutException import numpy as np #wd = browser = webdriver.Firefox(executable_path= GeckoDriverManager().install()) EKO() import sys from selenium.webdriver.firefox.options import Options as FirefoxOptions from selenium.webdriver.chrome.options import Options as ChromeOptions import time import cherrypy import argparse import PIL from PIL import Image import orange def tqdmtqdm(x) : return x class P(orange.RobotBase) : def __init__(self) : super().__init__() self.build(headless=True) EKO() pass def xx() : options = FirefoxOptions() options = ChromeOptions() #options.add_argument('--headless') options.add_argument('--no-sandbox') options.add_argument('--disable-dev-shm-usage') # changing user-agent because etoro detects the automated browser somehow options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/86.0.4240.183 Safari/537.36") EKO() s = ServiceC('/usr/bin/chromedriver') bot = self.driver = webdriver.Chrome(service=s, options=options) #self.driver = webdriver.Chrome('/usr/lib/chromium-browser/chromedriver', options=options) #self.driver = webdriver.Firefox(options=options) #self.driver.get("https://pythonbasics.org") #wd = webdriver.Firefox(service=service, options=options) EKO() def load(self, outdir, unfold, url, max=99999) : EKO(); bot = self.driver browser = self.driver.get(url) self.driver.implicitly_wait(5) EKOX(self.driver.title) html_source = self.driver.page_source #with open("html.html", "w") as fd : fd.write(html_source) EKOI(self.screen(), sz=800) ta = "Tout accepter" accs = [ ta, ta.lower(), ta.upper(), "Tout Accepter"] try : for acc in accs : EKOX(acc) accepts = self.driver.find_elements(By.XPATH, '//button/span[contains(text(), "%s")]' % acc) for ie, a in tqdmtqdm(enumerate(accepts)) : EKOX(a) if a.is_displayed() and a.is_enabled() : try : a.click() EKOT('clické') raise Exception('click') except Exception as ex1 : EKOX(ex1) if str(ex1) == 'click' : EKO() raise Exception(37) except Exception as ex : EKOX(ex) EKOI(self.screen(), sz=800) EKOX(hash(html_source)) #EKOX(dir(bot)) for i in range(unfold) : try : EKOX(i) button = bot.find_element(By.XPATH, '//button/span[@class="LoadMore-buttonText"]') EKOX(button) ActionChains(bot).move_to_element(button).click(button).perform() EKOT("clicked") #elem.click() time.sleep(2) EKO() except Exception as ex: EKOX(ex) EKOT("tout déplié ou autre") break xpath = '//li[@class="Collection-section-items-item svelte-1qwu1v2 separators"]' elems = bot.find_elements(By.XPATH, xpath) EKOX(len(list(elems))) ll = list(elems)[:max] for ie, e in tqdmtqdm(enumerate(ll)) : x2 = e.find_element(By.XPATH, './/div[@class="CardTitle qg-st4 svelte-1s1vx9z"]') EKOX(x2) EKOX(x2.text) EKOX(x2.get_attribute("textContent")) title = x2.text if len(x2.text) > 0 else x2.get_attribute("textContent") EKOX(title) x3 = e.find_element(By.XPATH, './/button[@type="button"]') EKOX(x3.text) EKOX(x3) time.sleep(1) EKOI(self.screen(), sz=800) x3.click() time.sleep(1) EKO() ignorer_xp = "/html/body/div[2]/footer/div[2]/div/div[1]/div[2]/span/button/span[2]/span[1]" ignorer_xp = "//*[contains(text(),'Ignorer')]"; try : ignorer = bot.find_element(By.XPATH, ignorer_xp) ignorer.click() time.sleep(1) except : pass EKOI(self.screen(), sz=800) p1 = bot.find_element(By.XPATH, "/html/body/audio") href = p1.get_attribute('src') EKOX(href) filename = title.replace("\n", "").replace("/", "_").replace("'", "_").strip() + ".mp3" ll = [ "'", "?", " ", ":"] for ee in ll : filename = filename.replace(ee, "_") EKOX(filename) # declick ce podcast if not os.path.exists(filename) : EKOT("downloading") doc = requests.get(href) ppp = os.path.join(outdir, filename) EKOT("writing %s" % ppp) with open(ppp, 'wb') as f: f.write(doc.content) EKOT("%s written" % filename) else : EKOT("%s already there" % filename) bot.close() # shuts down the bot def load(outdir, unfold, url, max=9999) : EKO(); p = P() EKOX(outdir) os.makedirs(outdir, exist_ok=True) EKO() p.load(outdir, unfold, url, max=max) def france_culture() : base = "https://www.radiofrance.fr/franceculture/podcasts/" l = ["le-cours-de-l-histoire", "bienvenue-au-book-club-part-2", "grandes-traversees"] l = ["le-cours-de-l-histoire"] for e in l : try : load(os.path.join("/media/usb-seagate2/Audio/", e), 10, base + e) except : pass if __name__ == '__main__': parser = argparse.ArgumentParser( prog='ferrand aspirator', description='download ferrand podcast') parser.add_argument('--outdir') parser.add_argument('--url') parser.add_argument('--nunfold', type=int, default=2) args = parser.parse_args() EKO(); load(args.outdir, args.nunfold, args.url)