tiennm99/MTTools
1
0
Fork 0
mirror of https://github.com/tiennm99/MTTools.git synced 2026-08-13 17:22:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: simple version

Browse Source
This commit is contained in:
tiennm99
2025-08-25 21:24:27 +07:00
parent 314457f845
commit 2e40ac48e7
7 changed files with 112 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
miti-scraper/.claude/settings.local.json
+9
View File
@@ -0,0 +1,9 @@
{
"permissions": {
"allow": [
"Bash(go build:*)"
],
"deny": [],
"ask": []
}
}
miti-scraper/.gitignore
+2
View File
@@ -0,0 +1,2 @@
data/
*.exe
miti-scraper/config.json
-9
View File
@@ -1,9 +0,0 @@
{
"root_url": "https://example.com",
"whitelist": [
"example.com",
"subdomain.example.com",
"blog.example.com"
],
"data_file": "processed_urls.txt"
}
miti-scraper/config.yaml
+9
View File
@@ -0,0 +1,9 @@
root_url: "https://iamminhnguyet.com/"
whitelist:
- "^https?://([^/]*\\.)?iamminhnguyet\\.com(/[^?]*)?$"
data_file: "processed_urls.txt"
# Delay between requests in seconds (helps avoid rate limiting)
delay_seconds: 1
miti-scraper/go.mod
+1
View File
@@ -20,4 +20,5 @@ require (
golang.org/x/text v0.3.2 // indirect
google.golang.org/appengine v1.6.6 // indirect
google.golang.org/protobuf v1.24.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
miti-scraper/go.sum
+3
View File
@@ -114,5 +114,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA=
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
miti-scraper/main.go
+88 -13
View File
@@ -2,20 +2,25 @@ package main
import (
"bufio"
"encoding/json"
"fmt"
"log"
"net/url"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"time"
"github.com/gocolly/colly/v2"
"gopkg.in/yaml.v3"
)
type Config struct {
RootURL string `json:"root_url"`
Whitelist []string `json:"whitelist"`
DataFile string `json:"data_file"`
RootURL string `yaml:"root_url"`
Whitelist []string `yaml:"whitelist"`
DataFile string `yaml:"data_file"`
Delay int `yaml:"delay_seconds"`
}
type WebScraper struct {
@@ -23,13 +28,15 @@ type WebScraper struct {
processedURL map[string]bool
mutex sync.RWMutex
dataFile string
delay time.Duration
}
func NewWebScraper(whitelist []string, dataFile string) *WebScraper {
func NewWebScraper(whitelist []string, dataFile string, delay time.Duration) *WebScraper {
return &WebScraper{
whitelist: whitelist,
processedURL: make(map[string]bool),
dataFile: dataFile,
delay: delay,
}
}
@@ -74,7 +81,12 @@ func (ws *WebScraper) saveProcessedURLs() error {
func (ws *WebScraper) isWhitelisted(url string) bool {
for _, pattern := range ws.whitelist {
if strings.Contains(url, pattern) {
matched, err := regexp.MatchString(pattern, url)
if err != nil {
log.Printf("Invalid regex pattern '%s': %v", pattern, err)
continue
}
if matched {
return true
}
}
@@ -93,6 +105,40 @@ func (ws *WebScraper) markAsProcessed(url string) {
ws.processedURL[url] = true
}
func (ws *WebScraper) normalizeURLForFilename(urlStr string) string {
parsedURL, err := url.Parse(urlStr)
if err != nil {
return "invalid-url"
}
filename := parsedURL.Host + parsedURL.Path
if parsedURL.RawQuery != "" {
filename += "_" + parsedURL.RawQuery
}
reg := regexp.MustCompile(`[<>:"/\\|?*]`)
filename = reg.ReplaceAllString(filename, "_")
filename = strings.ReplaceAll(filename, ".", "_")
if filename == "" || filename[len(filename)-1] == '_' {
filename += "index"
}
return filename
}
func (ws *WebScraper) saveContent(urlStr, content string) error {
dataDir := "data"
if err := os.MkdirAll(dataDir, 0755); err != nil {
return err
}
filename := ws.normalizeURLForFilename(urlStr) + ".txt"
filePath := filepath.Join(dataDir, filename)
return os.WriteFile(filePath, []byte(content), 0644)
}
func (ws *WebScraper) processURL(url string) {
fmt.Printf("Processing URL: %s\n", url)
ws.markAsProcessed(url)
@@ -107,6 +153,18 @@ func (ws *WebScraper) Start(rootURL string) error {
colly.AllowedDomains(),
)
c.OnResponse(func(r *colly.Response) {
urlStr := r.Request.URL.String()
if ws.isWhitelisted(urlStr) {
content := string(r.Body)
if err := ws.saveContent(urlStr, content); err != nil {
log.Printf("Failed to save content for %s: %v", urlStr, err)
} else {
fmt.Printf("Saved content: %s\n", ws.normalizeURLForFilename(urlStr)+".txt")
}
}
})
c.OnHTML("a[href]", func(e *colly.HTMLElement) {
link := e.Attr("href")
absoluteURL := e.Request.AbsoluteURL(link)
@@ -123,10 +181,28 @@ func (ws *WebScraper) Start(rootURL string) error {
c.OnRequest(func(r *colly.Request) {
fmt.Printf("Visiting: %s\n", r.URL.String())
if ws.delay > 0 {
time.Sleep(ws.delay)
}
})
c.OnError(func(r *colly.Response, err error) {
log.Printf("Error visiting %s: %v", r.Request.URL, err)
if r != nil {
switch r.StatusCode {
case 301, 302, 303, 307, 308:
log.Printf("REDIRECT: %s (Status: %d) -> Location: %s", r.Request.URL, r.StatusCode, r.Headers.Get("Location"))
case 403:
log.Printf("BLOCKED: Access forbidden to %s (Status: 403)", r.Request.URL)
case 429:
log.Printf("RATE_LIMITED: Too many requests to %s (Status: 429)", r.Request.URL)
case 404:
log.Printf("NOT_FOUND: %s (Status: 404)", r.Request.URL)
default:
log.Printf("ERROR: %s (Status: %d) - %v", r.Request.URL, r.StatusCode, err)
}
} else {
log.Printf("NETWORK_ERROR: Failed to connect to %s - %v", r.Request.URL, err)
}
})
if ws.isWhitelisted(rootURL) && !ws.isProcessed(rootURL) {
@@ -147,25 +223,24 @@ func (ws *WebScraper) Start(rootURL string) error {
}
func loadConfig(filename string) (*Config, error) {
file, err := os.Open(filename)
data, err := os.ReadFile(filename)
if err != nil {
return nil, err
}
defer file.Close()
var config Config
decoder := json.NewDecoder(file)
err = decoder.Decode(&config)
err = yaml.Unmarshal(data, &config)
return &config, err
}
func main() {
config, err := loadConfig("config.json")
config, err := loadConfig("config.yaml")
if err != nil {
log.Fatal("Failed to load config:", err)
}
scraper := NewWebScraper(config.Whitelist, config.DataFile)
delay := time.Duration(config.Delay) * time.Second
scraper := NewWebScraper(config.Whitelist, config.DataFile, delay)
if err := scraper.Start(config.RootURL); err != nil {
log.Fatal(err)