From 2e40ac48e7bb26cf741b39df642a0e9b5058f7c9 Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Mon, 25 Aug 2025 21:24:27 +0700 Subject: [PATCH] feat: simple version --- miti-scraper/.claude/settings.local.json | 9 ++ miti-scraper/.gitignore | 2 + miti-scraper/config.json | 9 -- miti-scraper/config.yaml | 9 ++ miti-scraper/go.mod | 1 + miti-scraper/go.sum | 3 + miti-scraper/main.go | 101 ++++++++++++++++++++--- 7 files changed, 112 insertions(+), 22 deletions(-) create mode 100644 miti-scraper/.claude/settings.local.json create mode 100644 miti-scraper/.gitignore delete mode 100644 miti-scraper/config.json create mode 100644 miti-scraper/config.yaml diff --git a/miti-scraper/.claude/settings.local.json b/miti-scraper/.claude/settings.local.json new file mode 100644 index 0000000..356627e --- /dev/null +++ b/miti-scraper/.claude/settings.local.json @@ -0,0 +1,9 @@ +{ + "permissions": { + "allow": [ + "Bash(go build:*)" + ], + "deny": [], + "ask": [] + } +} \ No newline at end of file diff --git a/miti-scraper/.gitignore b/miti-scraper/.gitignore new file mode 100644 index 0000000..32a2514 --- /dev/null +++ b/miti-scraper/.gitignore @@ -0,0 +1,2 @@ +data/ +*.exe diff --git a/miti-scraper/config.json b/miti-scraper/config.json deleted file mode 100644 index a75cdd0..0000000 --- a/miti-scraper/config.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "root_url": "https://example.com", - "whitelist": [ - "example.com", - "subdomain.example.com", - "blog.example.com" - ], - "data_file": "processed_urls.txt" -} \ No newline at end of file diff --git a/miti-scraper/config.yaml b/miti-scraper/config.yaml new file mode 100644 index 0000000..c768d3a --- /dev/null +++ b/miti-scraper/config.yaml @@ -0,0 +1,9 @@ +root_url: "https://iamminhnguyet.com/" + +whitelist: + - "^https?://([^/]*\\.)?iamminhnguyet\\.com(/[^?]*)?$" + +data_file: "processed_urls.txt" + +# Delay between requests in seconds (helps avoid rate limiting) +delay_seconds: 1 diff --git a/miti-scraper/go.mod b/miti-scraper/go.mod index 8fd4965..3b6600a 100644 --- a/miti-scraper/go.mod +++ b/miti-scraper/go.mod @@ -20,4 +20,5 @@ require ( golang.org/x/text v0.3.2 // indirect google.golang.org/appengine v1.6.6 // indirect google.golang.org/protobuf v1.24.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/miti-scraper/go.sum b/miti-scraper/go.sum index a96bd74..bbd4d40 100644 --- a/miti-scraper/go.sum +++ b/miti-scraper/go.sum @@ -114,5 +114,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/miti-scraper/main.go b/miti-scraper/main.go index 2b3d8ad..78167a1 100644 --- a/miti-scraper/main.go +++ b/miti-scraper/main.go @@ -2,20 +2,25 @@ package main import ( "bufio" - "encoding/json" "fmt" "log" + "net/url" "os" + "path/filepath" + "regexp" "strings" "sync" + "time" "github.com/gocolly/colly/v2" + "gopkg.in/yaml.v3" ) type Config struct { - RootURL string `json:"root_url"` - Whitelist []string `json:"whitelist"` - DataFile string `json:"data_file"` + RootURL string `yaml:"root_url"` + Whitelist []string `yaml:"whitelist"` + DataFile string `yaml:"data_file"` + Delay int `yaml:"delay_seconds"` } type WebScraper struct { @@ -23,13 +28,15 @@ type WebScraper struct { processedURL map[string]bool mutex sync.RWMutex dataFile string + delay time.Duration } -func NewWebScraper(whitelist []string, dataFile string) *WebScraper { +func NewWebScraper(whitelist []string, dataFile string, delay time.Duration) *WebScraper { return &WebScraper{ whitelist: whitelist, processedURL: make(map[string]bool), dataFile: dataFile, + delay: delay, } } @@ -74,7 +81,12 @@ func (ws *WebScraper) saveProcessedURLs() error { func (ws *WebScraper) isWhitelisted(url string) bool { for _, pattern := range ws.whitelist { - if strings.Contains(url, pattern) { + matched, err := regexp.MatchString(pattern, url) + if err != nil { + log.Printf("Invalid regex pattern '%s': %v", pattern, err) + continue + } + if matched { return true } } @@ -93,6 +105,40 @@ func (ws *WebScraper) markAsProcessed(url string) { ws.processedURL[url] = true } +func (ws *WebScraper) normalizeURLForFilename(urlStr string) string { + parsedURL, err := url.Parse(urlStr) + if err != nil { + return "invalid-url" + } + + filename := parsedURL.Host + parsedURL.Path + if parsedURL.RawQuery != "" { + filename += "_" + parsedURL.RawQuery + } + + reg := regexp.MustCompile(`[<>:"/\\|?*]`) + filename = reg.ReplaceAllString(filename, "_") + filename = strings.ReplaceAll(filename, ".", "_") + + if filename == "" || filename[len(filename)-1] == '_' { + filename += "index" + } + + return filename +} + +func (ws *WebScraper) saveContent(urlStr, content string) error { + dataDir := "data" + if err := os.MkdirAll(dataDir, 0755); err != nil { + return err + } + + filename := ws.normalizeURLForFilename(urlStr) + ".txt" + filePath := filepath.Join(dataDir, filename) + + return os.WriteFile(filePath, []byte(content), 0644) +} + func (ws *WebScraper) processURL(url string) { fmt.Printf("Processing URL: %s\n", url) ws.markAsProcessed(url) @@ -107,6 +153,18 @@ func (ws *WebScraper) Start(rootURL string) error { colly.AllowedDomains(), ) + c.OnResponse(func(r *colly.Response) { + urlStr := r.Request.URL.String() + if ws.isWhitelisted(urlStr) { + content := string(r.Body) + if err := ws.saveContent(urlStr, content); err != nil { + log.Printf("Failed to save content for %s: %v", urlStr, err) + } else { + fmt.Printf("Saved content: %s\n", ws.normalizeURLForFilename(urlStr)+".txt") + } + } + }) + c.OnHTML("a[href]", func(e *colly.HTMLElement) { link := e.Attr("href") absoluteURL := e.Request.AbsoluteURL(link) @@ -123,10 +181,28 @@ func (ws *WebScraper) Start(rootURL string) error { c.OnRequest(func(r *colly.Request) { fmt.Printf("Visiting: %s\n", r.URL.String()) + if ws.delay > 0 { + time.Sleep(ws.delay) + } }) c.OnError(func(r *colly.Response, err error) { - log.Printf("Error visiting %s: %v", r.Request.URL, err) + if r != nil { + switch r.StatusCode { + case 301, 302, 303, 307, 308: + log.Printf("REDIRECT: %s (Status: %d) -> Location: %s", r.Request.URL, r.StatusCode, r.Headers.Get("Location")) + case 403: + log.Printf("BLOCKED: Access forbidden to %s (Status: 403)", r.Request.URL) + case 429: + log.Printf("RATE_LIMITED: Too many requests to %s (Status: 429)", r.Request.URL) + case 404: + log.Printf("NOT_FOUND: %s (Status: 404)", r.Request.URL) + default: + log.Printf("ERROR: %s (Status: %d) - %v", r.Request.URL, r.StatusCode, err) + } + } else { + log.Printf("NETWORK_ERROR: Failed to connect to %s - %v", r.Request.URL, err) + } }) if ws.isWhitelisted(rootURL) && !ws.isProcessed(rootURL) { @@ -147,25 +223,24 @@ func (ws *WebScraper) Start(rootURL string) error { } func loadConfig(filename string) (*Config, error) { - file, err := os.Open(filename) + data, err := os.ReadFile(filename) if err != nil { return nil, err } - defer file.Close() var config Config - decoder := json.NewDecoder(file) - err = decoder.Decode(&config) + err = yaml.Unmarshal(data, &config) return &config, err } func main() { - config, err := loadConfig("config.json") + config, err := loadConfig("config.yaml") if err != nil { log.Fatal("Failed to load config:", err) } - scraper := NewWebScraper(config.Whitelist, config.DataFile) + delay := time.Duration(config.Delay) * time.Second + scraper := NewWebScraper(config.Whitelist, config.DataFile, delay) if err := scraper.Start(config.RootURL); err != nil { log.Fatal(err)