From bbe168529181e2caba72ce1f2a2ac7f86847eb3d Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Sun, 30 Nov 2025 20:22:58 +0700 Subject: [PATCH] feat(go): rewrite in Go --- .gitignore | 37 +++++++++++- README.md | 35 +++++++----- go.mod | 12 ++++ go.sum | 71 +++++++++++++++++++++++ main.go | 145 +++++++++++++++++++++++++++++++++++++++++++++++ main.py | 54 ------------------ requirements.txt | 2 - 7 files changed, 285 insertions(+), 71 deletions(-) create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go delete mode 100644 main.py delete mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore index 723ef36..089a213 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,36 @@ -.idea \ No newline at end of file +.idea + + + +# If you prefer the allow list template instead of the deny list, see community template: +# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore +# +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Code coverage profiles and other test artifacts +*.out +coverage.* +*.coverprofile +profile.cov + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Go workspace file +go.work +go.work.sum + +# env file +.env + +# Editor/IDE +# .idea/ +# .vscode/ diff --git a/README.md b/README.md index afa2f88..89e07ed 100644 --- a/README.md +++ b/README.md @@ -2,17 +2,24 @@ Crawl data of "Anh trai nhân vật chính", a novel from https://ln.hako.vn/sang-tac/8476-kiep-nay-la-anh-trai-cua-nhan-vat-chinh -# How to run -1. Install requirements - ``` - pip install -r requirements.txt - ``` -2. Run main.py to crawl data - ``` - python main.py - ``` - __Note__: You may get `HTTP Error 429: Too Many Requests`. Then you can try again later, and skip downloaded chapters, example skip 50 first chapters like: - ``` - for chapter in chapters[50:]: - ``` -3. The data will be saved in `data` folder \ No newline at end of file +## The Original Python Version + +*In 2025, I rewrote this project using Go. The original Python version of this project can be found at the `feature/python` branch.* + +## Quick Start + +1. Install Go: https://go.dev/dl/ +2. `go mod init atnvc-crawler` (if needed) +3. `go get github.com/PuerkitoBio/goquery` +4. `go run main.go` + +Data saved in `./data/` as `.txt`. + +## Rate Limits (HTTP 429) +Skip crawled chapters and retry later. Uncomment line 104, replace 100 with number of chapters you want to skip. +```go +chapters = chapters.Slice(50, chapters.Length()) +``` + +## Customize +This version should work with other novels on Hako with some modifications. Feel free to try it! diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..95bba89 --- /dev/null +++ b/go.mod @@ -0,0 +1,12 @@ +module github.com/tiennm99/atnvc-crawler + +go 1.24.0 + +toolchain go1.24.10 + +require github.com/PuerkitoBio/goquery v1.11.0 + +require ( + github.com/andybalholm/cascadia v1.3.3 // indirect + golang.org/x/net v0.47.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..806bd54 --- /dev/null +++ b/go.sum @@ -0,0 +1,71 @@ +github.com/PuerkitoBio/goquery v1.11.0 h1:jZ7pwMQXIITcUXNH83LLk+txlaEy6NVOfTuP43xxfqw= +github.com/PuerkitoBio/goquery v1.11.0/go.mod h1:wQHgxUOU3JGuj3oD/QFfxUdlzW6xPHfqyHre6VMY4DQ= +github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= +github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/main.go b/main.go new file mode 100644 index 0000000..8f8173c --- /dev/null +++ b/main.go @@ -0,0 +1,145 @@ +package main + +import ( + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/PuerkitoBio/goquery" +) + +const DEBUG = false + +var sanitizeRe = regexp.MustCompile(`[\\\/:*?"<>|]`) + +func isDigit(id string) bool { + if id == "" { + return false + } + for _, r := range id { + if r < '0' || r > '9' { + return false + } + } + return true +} + +func getFromURL(url string) (string, error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return "", err + } + req.Header.Set("User-Agent", "Mozilla/5.0") + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("HTTP error: %s", resp.Status) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + return string(body), nil +} + +func writeTextToFile(text, filename string) error { + filename = sanitizeRe.ReplaceAllString(filename, "_") + dir := "data" + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + path := filepath.Join(dir, filename) + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + _, err = f.WriteString(text) + return err +} + +func readTextFromFile(filename string) (string, error) { + path := filepath.Join("data", filename) + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + return string(data), nil +} + +func main() { + var html string + var err error + if DEBUG { + html, err = readTextFromFile("_.txt") + } else { + html, err = getFromURL("https://ln.hako.vn/sang-tac/8476-kiep-nay-la-anh-trai-cua-nhan-vat-chinh") + // writeTextToFile(html, "_.txt") + } + if err != nil { + fmt.Printf("Error: %v\n", err) + return + } + + doc, err := goquery.NewDocumentFromReader(strings.NewReader(html)) + if err != nil { + fmt.Printf("Parse error: %v\n", err) + return + } + + chapters := doc.Find("div.chapter-name") + /* + you may get `HTTP Error 429: Too Many Requests` + you can try again later, and skip downloaded chapters + example skip 50 first chapters like: chapters = chapters.Slice(50, chapters.Length()) + */ + // chapters = chapters.Slice(100, chapters.Length()) + for i := 0; i < chapters.Length(); i++ { + chapterSel := chapters.Eq(i) + children := chapterSel.ChildrenFiltered("a") + if children.Length() == 0 { + continue + } + linkSel := children.First() + title, hasTitle := linkSel.Attr("title") + if !hasTitle { + continue + } + href, hasHref := linkSel.Attr("href") + if !hasHref { + continue + } + chapterURL := "https://ln.hako.vn" + href + chapterHTML, err := getFromURL(chapterURL) + if err != nil { + fmt.Printf("Error fetching %s: %v\n", chapterURL, err) + continue + } + chapterDoc, err := goquery.NewDocumentFromReader(strings.NewReader(chapterHTML)) + if err != nil { + fmt.Printf("Parse chapter %s: %v\n", title, err) + continue + } + content := chapterDoc.Find("#chapter-content") + var chapterData strings.Builder + content.Find("p").Each(func(_ int, s *goquery.Selection) { + id, exists := s.Attr("id") + if exists && isDigit(id) { + chapterData.WriteString(s.Text()) + chapterData.WriteString("\n") + } + }) + filename := title + ".txt" + if err := writeTextToFile(chapterData.String(), filename); err != nil { + fmt.Printf("Error writing %s: %v\n", filename, err) + } + } +} diff --git a/main.py b/main.py deleted file mode 100644 index 39859e9..0000000 --- a/main.py +++ /dev/null @@ -1,54 +0,0 @@ -import re -from urllib.request import Request, urlopen -from bs4 import BeautifulSoup - -DEBUG = False - - -def get_from_url(url): - request_site = Request(url, headers={"User-Agent": "Mozilla/5.0"}) - webpage = urlopen(request_site).read() - return webpage.decode("utf-8") - - -def write_text_to_file(text, filename): - filename = re.sub(r"[^\w_. -]", "_", filename) - f = open("data/{}".format(filename), "w", encoding="utf-8") - f.write(text) - f.close() - - -def read_text_from_file(filename): - f = open("data/{}".format(filename), "r", encoding="utf-8") - text = f.read() - f.close() - return text - - -if DEBUG: - html = read_text_from_file("_.txt") -else: - html = get_from_url("https://ln.hako.vn/sang-tac/8476-kiep-nay-la-anh-trai-cua-nhan-vat-chinh") - # write_text_to_file(html, "_.txt") - -soup = BeautifulSoup(html, "html.parser") -chapters = soup.find_all("div", {"class": "chapter-name"}) -""" -you may get `HTTP Error 429: Too Many Requests` -you can try again later, and skip downloaded chapters -example skip 50 first chapters like: `for chapter in chapters[50:]` -""" -for chapter in chapters: - children = chapter.find_all("a", recursive=False) - child = children[0] - chapterTitle = child.attrs["title"] - chapterUrl = "https://ln.hako.vn" + child.attrs["href"] - chapterHtml = get_from_url(chapterUrl) - chapterSoup = BeautifulSoup(chapterHtml, "html.parser") - chapterContent = chapterSoup.find("div", {"id": "chapter-content"}) - chapterData = "" - if chapterContent is not None: - paragraphs = chapterContent.find_all("p", id=lambda x: x and x.isdigit()) - for paragraph in paragraphs: - chapterData += paragraph.text + "\n" - write_text_to_file(chapterData, chapterTitle + ".txt") diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 5da028b..0000000 --- a/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -beautifulsoup4==4.12.3 -soupsieve==2.6