#downloader.py
#This script downloads audio files from Wiktionary using Selenium Chrome
#Don't move your mouse when you run this program
#pip install requests
#pip install beautifulsoup4
#pip install selenium
#pip install pyautogui
import requests
import time
import pyautogui
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
def get_pronunciation_url(word):
base_url = f"https://en.wiktionary.org/api/rest_v1/page/media-list/{word}"
response = requests.get(base_url)
data = response.json()
# Check if there are media files available
if 'items' in data and data['items']:
k=0
for item in data['items']:
if 'title' in item and 'audio_type' in item and item['audio_type'] == 'generic':
if ('en-us' in item['title']) or ('En-us' in item['title']):
return item['title']
else:
k=1
if k==1:
for item in data['items']:
if 'title' in item and 'audio_type' in item and item['audio_type'] == 'generic':
if ('en-uk' in item['title']) or ('En-uk' in item['title']):
return item['title']
else:
k=2
if k==2:
for item in data['items']:
if 'title' in item and 'audio_type' in item and item['audio_type'] == 'generic':
if ('en-ca' in item['title']) or ('En-ca' in item['title']):
return item['title']
else:
k=3
if k==3:
for item in data['items']:
if 'title' in item and 'audio_type' in item and item['audio_type'] == 'generic':
if ('eng' in item['title']):
return item['title']
return None
def media_pages():
num_lines = sum(1 for _ in open("eng-words.txt"))
with open("eng-words.txt") as file_in:
lines = []
for line1 in file_in:
lines.append(line1.rstrip())
url2 = "https://en.wiktionary.org/wiki/"
media = []
notfound_media = []
for word_to_search in lines:
pronunciation_url = get_pronunciation_url(word_to_search)
if pronunciation_url:
url=url2+pronunciation_url
media.append(url.rstrip())
else:
media.append('\n')
notfound_media.append(word_to_search.rstrip())
print(f"Not found a link for '{word_to_search}'")
open("media-pages-text.txt", 'w').close()
open("notfound-media-pages-text.txt", 'w').close()
n = 0
with open("media-pages-text.txt", 'w') as fw:
for med in media:
n = n+1
if n<num_lines:
if med !='\n':
fw.write(med+'\n')
if med == '\n':
fw.write(med)
if n == num_lines:
fw.write(med)
with open("notfound-media-pages-text.txt", 'w') as fnw:
for notfound in notfound_media:
fnw.write(notfound+'\n')
def get_links():
num_lines = sum(1 for _ in open("media-pages-text.txt"))
with open("media-pages-text.txt") as file1:
lines1 = []
for line1 in file1:
if line1 !='':
lines1.append(line1.rstrip())
if line1 == '':
lines1.append(line1)
with open("eng-words.txt") as file2:
lines2 = []
for line2 in file2:
lines2.append(line2.rstrip())
links = []
notfound_links = []
for wiktionary_url, word in zip(lines1, lines2):
# Send an HTTP request and get the HTML content
if wiktionary_url !='':
try:
response = requests.get(wiktionary_url)
except:
pass
html_content = response.text
# Parse the HTML content with BeautifulSoup
soup = BeautifulSoup(html_content, "html.parser")
# Find the 'a' tag
audio_a_tag = soup.find("a", {"class": "internal"})
if audio_a_tag:
# Get the 'href' attribute from the 'a' tag
audio_link = audio_a_tag["href"]
links.append('https:'+audio_link.rstrip())
else:
links.append('\n')
notfound_links.append(word.rstrip())
if wiktionary_url == '':
links.append('\n')
notfound_links.append(word.rstrip())
open("sound-links-text.txt", 'w').close()
open("notfound-sound-links-text.txt", 'w').close()
n = 0
with open("sound-links-text.txt", 'w') as fw:
for link in links:
n = n+1
if n<num_lines:
if link != '\n':
fw.write(link+'\n')
if link == '\n':
fw.write(link)
if n == num_lines:
fw.write(link)
with open("notfound-sound-links-text.txt", 'w') as fnw:
for notfound in notfound_links:
fnw.write(notfound+'\n')
def download():
with open("sound-links-text.txt") as file_in:
lines1 = []
for line1 in file_in:
if line1 !='':
lines1.append(line1.rstrip())
if line1 == '':
lines1.append(line1)
with open("eng-words.txt") as file_in:
lines2 = []
for line2 in file_in:
lines2.append(line2.rstrip())
num_lines = sum(1 for _ in open("sound-links-text.txt"))
open("audio-files-text.txt", 'w').close()
m=0
with open("audio-files-text.txt", 'w') as f:
for line11, line21 in zip(lines1, lines2):
m = m+1
if line11 !='':
if 'ogg' in line11:
if m<num_lines:
f.write(line21+'.ogg'+'\n')
if m==num_lines:
f.write(line21+'.ogg')
if 'oga' in line11:
if m<num_lines:
f.write(line21+'.oga'+'\n')
if m==num_lines:
f.write(line21+'.oga')
if 'wav' in line11:
if m<num_lines:
f.write(line21+'.wav'+'\n')
if m==num_lines:
f.write(line21+'.wav')
if line11 == '':
pass
# Set the path to the chromedriver
chromedriver_path = "C:/Users/YourUsername/Desktop/selenium/chromedriver.exe" #Replace YourUsername with a real username
# Create ChromeOptions
options = Options()
# Your Options here....
options.add_argument("--start-maximized")
# Create a ChromeDriver
driver = webdriver.Chrome(options=options)
# This loop opens links to upload.wikimedia and downloaddownloads files to Downloads
for line11, line21 in zip(lines1, lines2):
if line11 !='':
driver.get(line11)
time.sleep(2)
pyautogui.click(button='right')
pyautogui.press('down', presses=3)
time.sleep(2)
pyautogui.press('enter')
time.sleep(1)
pyautogui.write(line21)
time.sleep(1)
pyautogui.press('enter')
time.sleep(3)
if line11=='':
pass
driver.quit()
def main():
media_pages()
get_links()
download()
if __name__ == "__main__":
main()
