web_scraper/crawl.py at master · Cmepo/web_scraper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
from urllib.parse import urlsplit, urljoin
from bs4 import BeautifulSoup, Tag
from typing import TypedDict

class PageData(TypedDict):
    url:str
    heading:str
    first_paragraph:str
    outgoing_links:list[str]
    image_urls:list[str]

def normalize_url(url:str) -> str:
    split = urlsplit(url=url)
    try:
        if split.hostname != None:
            norm_url = split.hostname + split.path.rstrip("/")
            return norm_url
        else:
            raise Exception
    except:
        print("Error normalizing the url")

def get_heading_from_html(html:str) -> str:
    soup = BeautifulSoup(html, 'html.parser')
    h_tag = None
    if soup.find("h1") != None:
        h_tag = soup.find("h1")
    elif soup.find("h2") != None:
        h_tag = soup.find("h2")
    else:
        return ""

    return h_tag.get_text().strip() if isinstance(h_tag, Tag) else ""

def get_first_paragraph_from_html(html:str) -> str:
    soup = BeautifulSoup(html, 'html.parser')
    if soup.main:
        if soup.main.p:
            return soup.main.p.get_text().strip()
    elif soup.p:
        return soup.p.get_text().strip()
    else:
        return ""

def get_urls_from_html(html:str, base_url:str) -> list:
    soup = BeautifulSoup(html, 'html.parser')
    url_list = []
    for a in soup.find_all('a'):
        if not isinstance(a, Tag):
            continue
        href = a.get("href")
        if isinstance(href,str) and href:
            try:
                abs_url = urljoin(base_url,href)
                url_list.append(abs_url)
            except Exception as e:
                print(f"{e}: {href}")
    return url_list

def get_images_from_html(html:str, base_url:str) -> list:
    soup = BeautifulSoup(html, 'html.parser')
    url_list = []
    for img in soup.find_all('img'):
        if not isinstance(img, Tag):
            continue
        src = img.get("src")
        if isinstance(src,str) and src:
            try:
                abs_url = urljoin(base_url, src)
                url_list.append(abs_url)
            except Exception as e:
                print(f"{e}: {src}")
    return url_list

def extract_page_data(html: str, page_url: str):
    norm_url = normalize_url(page_url)
    dictionary:PageData = {
        "url":page_url,
        "heading":get_heading_from_html(html),
        "first_paragraph":get_first_paragraph_from_html(html),
        "outgoing_links":get_urls_from_html(html,page_url),
        "image_urls":get_images_from_html(html,page_url),
    }
    return dictionary