diff --git a/miti-scraper/.claude/settings.local.json b/miti-scraper/.claude/settings.local.json index 356627e..533b70a 100644 --- a/miti-scraper/.claude/settings.local.json +++ b/miti-scraper/.claude/settings.local.json @@ -1,7 +1,9 @@ { "permissions": { "allow": [ - "Bash(go build:*)" + "Bash(go build:*)", + "Bash(go get:*)", + "Bash(go run:*)" ], "deny": [], "ask": [] diff --git a/miti-scraper/go.mod b/miti-scraper/go.mod index 3b6600a..3cba084 100644 --- a/miti-scraper/go.mod +++ b/miti-scraper/go.mod @@ -1,6 +1,8 @@ module miti-scraper -go 1.21 +go 1.23.0 + +toolchain go1.23.4 require github.com/gocolly/colly/v2 v2.1.0 @@ -16,8 +18,8 @@ require ( github.com/kennygrant/sanitize v1.2.4 // indirect github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect github.com/temoto/robotstxt v1.1.1 // indirect - golang.org/x/net v0.0.0-20200602114024-627f9648deb9 // indirect - golang.org/x/text v0.3.2 // indirect + golang.org/x/net v0.43.0 // indirect + golang.org/x/text v0.28.0 // indirect google.golang.org/appengine v1.6.6 // indirect google.golang.org/protobuf v1.24.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/miti-scraper/go.sum b/miti-scraper/go.sum index bbd4d40..fab4f28 100644 --- a/miti-scraper/go.sum +++ b/miti-scraper/go.sum @@ -75,6 +75,8 @@ golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -86,6 +88,8 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= diff --git a/miti-scraper/main.go b/miti-scraper/main.go index 78167a1..d24a92d 100644 --- a/miti-scraper/main.go +++ b/miti-scraper/main.go @@ -13,6 +13,7 @@ import ( "time" "github.com/gocolly/colly/v2" + "golang.org/x/net/html" "gopkg.in/yaml.v3" ) @@ -127,6 +128,43 @@ func (ws *WebScraper) normalizeURLForFilename(urlStr string) string { return filename } +func (ws *WebScraper) extractTextFromHTML(htmlContent string) string { + doc, err := html.Parse(strings.NewReader(htmlContent)) + if err != nil { + log.Printf("Failed to parse HTML: %v", err) + return htmlContent + } + + var textContent strings.Builder + ws.extractText(doc, &textContent) + + text := textContent.String() + text = regexp.MustCompile(`\s+`).ReplaceAllString(text, " ") + text = strings.TrimSpace(text) + + return text +} + +func (ws *WebScraper) extractText(n *html.Node, textContent *strings.Builder) { + if n.Type == html.TextNode { + textContent.WriteString(n.Data) + textContent.WriteString(" ") + } + + if n.Type == html.ElementNode { + switch n.Data { + case "script", "style", "noscript", "head": + return + case "br", "p", "div", "h1", "h2", "h3", "h4", "h5", "h6": + textContent.WriteString(" ") + } + } + + for child := n.FirstChild; child != nil; child = child.NextSibling { + ws.extractText(child, textContent) + } +} + func (ws *WebScraper) saveContent(urlStr, content string) error { dataDir := "data" if err := os.MkdirAll(dataDir, 0755); err != nil { @@ -136,7 +174,8 @@ func (ws *WebScraper) saveContent(urlStr, content string) error { filename := ws.normalizeURLForFilename(urlStr) + ".txt" filePath := filepath.Join(dataDir, filename) - return os.WriteFile(filePath, []byte(content), 0644) + textContent := ws.extractTextFromHTML(content) + return os.WriteFile(filePath, []byte(textContent), 0644) } func (ws *WebScraper) processURL(url string) {