Repository navigation
Expand file tree
/
Copy pathbs4Scraper.py
More file actions
76 lines (65 loc) · 2.7 KB
/
Copy pathbs4Scraper.py
File metadata and controls
76 lines (65 loc) · 2.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
from bs4 import BeautifulSoup
import urllib.request
import urllib.parse
import requests
from requests.exceptions import HTTPError, Timeout, RequestException
from .AnkiAudioTools import AnkiAudioObject, AnkiAudioGlobals
"""
Audio sources.
The primary source is the "forga" CDN (AnkiAudioGlobals.FORGA_BASE_URL).
When the CDN has no results we fall back to external sources:
- lingua libre (open audio records, any language)
Openrussian (Russian only) also exists as a last resort, but it lives in its
own module because it needs a logged-in session; see `openrussian.py`.
"""
def fetch_html(url):
try:
page = urllib.request.Request(url)
infile = urllib.request.urlopen(page).read()
data = infile.decode('UTF-8')
return BeautifulSoup(data, "html.parser")
except Exception as e:
print(str(e))
return None
def lookup_word_lingua_libre(word, languageCode):
audioList = []
wordEncoded = urllib.parse.quote(word)
page = fetch_html("https://lingualibre.org/index.php?search=" + wordEncoded)
if page is None:
return audioList
links = page.select("a[href^='/wiki/Q']")
pattern = word + " | audio record - " + languageCode
for link in links:
clean_title = link["title"].replace("\u200E", "").lower()
if clean_title.startswith(pattern):
try:
linkPage = fetch_html("https://lingualibre.org" + link['href'])
wordID = link['href'][link['href'].find("Q"):]
audioList.append(AnkiAudioObject(word, wordID, linkPage.select_one("source[type^='audio/ogg']")['src']))
except Exception as e:
print(str(e))
return audioList
def forga_lookup(word, language):
"""Look up audio on the forga CDN. `language` uses the full 'Language_code' format (e.g. 'English_en').
The CDN matches exactly and case-insensitively, so `word` must already be
normalized (see normalize.py). Results come back best-first."""
print(f"Forga: looking up '{word}' in '{language}'")
results = []
try:
response = requests.get(
AnkiAudioGlobals.FORGA_BASE_URL + "/audios",
params={"language": language, "value": word},
timeout=10,
)
response.raise_for_status()
data = response.json()
for audio in data:
results.append(AnkiAudioObject(word, audio['id'], audio['link']))
except (HTTPError, Timeout, RequestException) as err:
print(f"Forga: request failed: {err}")
except ValueError as err:
print(f"Forga: invalid response: {err}")
except Exception as err:
print(f"Forga: an error occurred: {err}")
print(f"Forga: Results count: {len(results)}")
return results