feat: only save content

miti-scraper/.claude/settings.local.json

@@ -1,7 +1,9 @@
 {
   "permissions": {
     "allow": [
-      "Bash(go build:*)"
+      "Bash(go build:*)",
+      "Bash(go get:*)",
+      "Bash(go run:*)"
     ],
     "deny": [],
     "ask": []

miti-scraper/go.mod

@@ -1,6 +1,8 @@
 module miti-scraper
 
-go 1.21
+go 1.23.0
+
+toolchain go1.23.4
 
 require github.com/gocolly/colly/v2 v2.1.0
 
@@ -16,8 +18,8 @@ require (
 	github.com/kennygrant/sanitize v1.2.4 // indirect
 	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
 	github.com/temoto/robotstxt v1.1.1 // indirect
-	golang.org/x/net v0.0.0-20200602114024-627f9648deb9 // indirect
-	golang.org/x/text v0.3.2 // indirect
+	golang.org/x/net v0.43.0 // indirect
+	golang.org/x/text v0.28.0 // indirect
 	google.golang.org/appengine v1.6.6 // indirect
 	google.golang.org/protobuf v1.24.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

miti-scraper/go.sum

@@ -75,6 +75,8 @@ golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM=
 golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE=
+golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -86,6 +88,8 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
+golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=

miti-scraper/main.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/gocolly/colly/v2"
+	"golang.org/x/net/html"
 	"gopkg.in/yaml.v3"
 )
 
@@ -127,6 +128,43 @@ func (ws *WebScraper) normalizeURLForFilename(urlStr string) string {
 	return filename
 }
 
+func (ws *WebScraper) extractTextFromHTML(htmlContent string) string {
+	doc, err := html.Parse(strings.NewReader(htmlContent))
+	if err != nil {
+		log.Printf("Failed to parse HTML: %v", err)
+		return htmlContent
+	}
+
+	var textContent strings.Builder
+	ws.extractText(doc, &textContent)
+	
+	text := textContent.String()
+	text = regexp.MustCompile(`\s+`).ReplaceAllString(text, " ")
+	text = strings.TrimSpace(text)
+	
+	return text
+}
+
+func (ws *WebScraper) extractText(n *html.Node, textContent *strings.Builder) {
+	if n.Type == html.TextNode {
+		textContent.WriteString(n.Data)
+		textContent.WriteString(" ")
+	}
+	
+	if n.Type == html.ElementNode {
+		switch n.Data {
+		case "script", "style", "noscript", "head":
+			return
+		case "br", "p", "div", "h1", "h2", "h3", "h4", "h5", "h6":
+			textContent.WriteString(" ")
+		}
+	}
+	
+	for child := n.FirstChild; child != nil; child = child.NextSibling {
+		ws.extractText(child, textContent)
+	}
+}
+
 func (ws *WebScraper) saveContent(urlStr, content string) error {
 	dataDir := "data"
 	if err := os.MkdirAll(dataDir, 0755); err != nil {
@@ -136,7 +174,8 @@ func (ws *WebScraper) saveContent(urlStr, content string) error {
 	filename := ws.normalizeURLForFilename(urlStr) + ".txt"
 	filePath := filepath.Join(dataDir, filename)
 
-	return os.WriteFile(filePath, []byte(content), 0644)
+	textContent := ws.extractTextFromHTML(content)
+	return os.WriteFile(filePath, []byte(textContent), 0644)
 }
 
 func (ws *WebScraper) processURL(url string) {