Firefly-Shelter/Hakushi_Crawl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
Azen Kain
2025-12-31 16:06:58 +07:00
commit ce230479c7
21 changed files with 797 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile Normal file
View File

@@ -0,0 +1,4 @@
build:
@echo Building windows binary...
go build -trimpath -ldflags="-s -w" .
@echo Done!

35
api/as.go Normal file
View File

@@ -0,0 +1,35 @@
package api
import (
"fmt"
"hakushi-crawl/utils"
"log"
)
func GetASEventInfoApi(eventId string, locale string) (*any, error) {
url := fmt.Sprintf(
"https://api.hakush.in/hsr/data/%s/boss/%s.json",
locale,
eventId,
)
data, err := utils.GetĐataJSON[any](url)
if err != nil {
log.Println(err)
return nil, nil
}
return data, nil
}
func GetListASEventApi() (map[string]*any, error) {
url := "https://api.hakush.in/hsr/data/maze_boss.json"
data, err := utils.GetĐataJSON[map[string]*any](url)
if err != nil {
log.Println(err)
return nil, nil
}
return *data, nil
}

35
api/character.go Normal file
View File

@@ -0,0 +1,35 @@
package api
import (
"fmt"
"hakushi-crawl/utils"
"log"
)
func GetCharacterInfoApi(avatarId string, locale string) (*any, error) {
url := fmt.Sprintf(
"https://api.hakush.in/hsr/data/%s/character/%s.json",
locale,
avatarId,
)
data, err := utils.GetĐataJSON[any](url)
if err != nil {
log.Println(err)
return nil, nil
}
return data, nil
}
func GetListCharacterApi() (map[string]*any, error) {
url := "https://api.hakush.in/hsr/data/character.json"
data, err := utils.GetĐataJSON[map[string]*any](url)
if err != nil {
log.Println(err)
return nil, nil
}
return *data, nil
}

35
api/lightcone.go Normal file
View File

@@ -0,0 +1,35 @@
package api
import (
"fmt"
"hakushi-crawl/utils"
"log"
)
func GetLightconeInfoApi(lightconeId string, locale string) (*any, error) {
url := fmt.Sprintf(
"https://api.hakush.in/hsr/data/%s/lightcone/%s.json",
locale,
lightconeId,
)
data, err := utils.GetĐataJSON[any](url)
if err != nil {
log.Println(err)
return nil, nil
}
return data, nil
}
func GetListLightconeApi() (map[string]*any, error) {
url := "https://api.hakush.in/hsr/data/lightcone.json"
data, err := utils.GetĐataJSON[map[string]*any](url)
if err != nil {
log.Println(err)
return nil, nil
}
return *data, nil
}

35
api/moc.go Normal file
View File

@@ -0,0 +1,35 @@
package api
import (
"fmt"
"hakushi-crawl/utils"
"log"
)
func GetMOCEventInfoApi(eventId string, locale string) (*any, error) {
url := fmt.Sprintf(
"https://api.hakush.in/hsr/data/%s/maze/%s.json",
locale,
eventId,
)
data, err := utils.GetĐataJSON[any](url)
if err != nil {
log.Println(err)
return nil, nil
}
return data, nil
}
func GetListMOCEventApi() (map[string]*any, error) {
url := "https://api.hakush.in/hsr/data/maze.json"
data, err := utils.GetĐataJSON[map[string]*any](url)
if err != nil {
log.Println(err)
return nil, nil
}
return *data, nil
}

35
api/monster.go Normal file
View File

@@ -0,0 +1,35 @@
package api
import (
"fmt"
"hakushi-crawl/utils"
"log"
)
func GetMonsterInfoApi(monsterId string, locale string) (*any, error) {
url := fmt.Sprintf(
"https://api.hakush.in/hsr/data/%s/monster/%s.json",
locale,
monsterId,
)
data, err := utils.GetĐataJSON[any](url)
if err != nil {
log.Println(err)
return nil, nil
}
return data, nil
}
func GetListMonsterApi() (map[string]*any, error) {
url := "https://api.hakush.in/hsr/data/monster.json"
data, err := utils.GetĐataJSON[map[string]*any](url)
if err != nil {
log.Println(err)
return nil, nil
}
return *data, nil
}

35
api/peak.go Normal file
View File

@@ -0,0 +1,35 @@
package api
import (
"fmt"
"hakushi-crawl/utils"
"log"
)
func GetPeakEventInfoApi(eventId string, locale string) (*any, error) {
url := fmt.Sprintf(
"https://api.hakush.in/hsr/data/%s/peak/%s.json",
locale,
eventId,
)
data, err := utils.GetĐataJSON[any](url)
if err != nil {
log.Println(err)
return nil, nil
}
return data, nil
}
func GetListPeakEventApi() (map[string]*any, error) {
url := "https://api.hakush.in/hsr/data/maze_peak.json"
data, err := utils.GetĐataJSON[map[string]*any](url)
if err != nil {
log.Println(err)
return nil, nil
}
return *data, nil
}

35
api/pf.go Normal file
View File

@@ -0,0 +1,35 @@
package api
import (
"fmt"
"hakushi-crawl/utils"
"log"
)
func GetPFEventInfoApi(eventId string, locale string) (*any, error) {
url := fmt.Sprintf(
"https://api.hakush.in/hsr/data/%s/story/%s.json",
locale,
eventId,
)
data, err := utils.GetĐataJSON[any](url)
if err != nil {
log.Println(err)
return nil, nil
}
return data, nil
}
func GetListPFEventApi() (map[string]*any, error) {
url := "https://api.hakush.in/hsr/data/maze_extra.json"
data, err := utils.GetĐataJSON[map[string]*any](url)
if err != nil {
log.Println(err)
return nil, nil
}
return *data, nil
}

35
api/relic.go Normal file
View File

@@ -0,0 +1,35 @@
package api
import (
"fmt"
"hakushi-crawl/utils"
"log"
)
func GetRelicInfoApi(relicId string, locale string) (*any, error) {
url := fmt.Sprintf(
"https://api.hakush.in/hsr/data/%s/relicset/%s.json",
locale,
relicId,
)
data, err := utils.GetĐataJSON[any](url)
if err != nil {
log.Println(err)
return nil, nil
}
return data, nil
}
func GetListRelicsetApi() (map[string]*any, error) {
url := "https://api.hakush.in/hsr/data/relicset.json"
data, err := utils.GetĐataJSON[map[string]*any](url)
if err != nil {
log.Println(err)
return nil, nil
}
return *data, nil
}

3
go.mod Normal file
View File

@@ -0,0 +1,3 @@
module hakushi-crawl
go 1.25.5

BIN
hakushi-crawl.exe Normal file
View File

Binary file not shown.

54
internal/as.go Normal file
View File

@@ -0,0 +1,54 @@
package internal
import (
"encoding/json"
"fmt"
"hakushi-crawl/api"
"os"
"sync"
)
func SaveDataASEvent(locale string) error {
listItem, err := api.GetListASEventApi()
if err != nil {
return err
}
mapItemDetail := make(map[string]*any)
var mu sync.Mutex
jobs := make(chan string)
wg := sync.WaitGroup{}
workerCount := 12
for range workerCount {
wg.Go(func() {
for itemId := range jobs {
itemDetail, err := api.GetASEventInfoApi(itemId, locale)
if err != nil || itemDetail == nil {
continue
}
mu.Lock()
mapItemDetail[itemId] = itemDetail
mu.Unlock()
}
})
}
for itemId := range listItem {
jobs <- itemId
}
close(jobs)
wg.Wait()
fileName := fmt.Sprintf("as.%s.json", locale)
data, err := json.Marshal(mapItemDetail)
if err != nil {
return err
}
return os.WriteFile(fileName, data, 0644)
}

54
internal/character.go Normal file
View File

@@ -0,0 +1,54 @@
package internal
import (
"encoding/json"
"fmt"
"hakushi-crawl/api"
"os"
"sync"
)
func SaveDataCharacter(locale string) error {
listItem, err := api.GetListCharacterApi()
if err != nil {
return err
}
mapItemDetail := make(map[string]*any)
var mu sync.Mutex
jobs := make(chan string)
wg := sync.WaitGroup{}
workerCount := 12
for range workerCount {
wg.Go(func() {
for itemId := range jobs {
itemDetail, err := api.GetCharacterInfoApi(itemId, locale)
if err != nil || itemDetail == nil {
continue
}
mu.Lock()
mapItemDetail[itemId] = itemDetail
mu.Unlock()
}
})
}
for itemId := range listItem {
jobs <- itemId
}
close(jobs)
wg.Wait()
fileName := fmt.Sprintf("characters.%s.json", locale)
data, err := json.Marshal(mapItemDetail)
if err != nil {
return err
}
return os.WriteFile(fileName, data, 0644)
}

55
internal/lightcone.go Normal file
View File

@@ -0,0 +1,55 @@
package internal
import (
"encoding/json"
"fmt"
"hakushi-crawl/api"
"os"
"sync"
)
func SaveDataLightcone(locale string) error {
listItem, err := api.GetListLightconeApi()
if err != nil {
return err
}
mapItemDetail := make(map[string]*any)
var mu sync.Mutex
jobs := make(chan string)
wg := sync.WaitGroup{}
workerCount := 12
for range workerCount {
wg.Go(func() {
for itemId := range jobs {
itemDetail, err := api.GetLightconeInfoApi(itemId, locale)
if err != nil || itemDetail == nil {
continue
}
mu.Lock()
mapItemDetail[itemId] = itemDetail
mu.Unlock()
}
})
}
for itemId := range listItem {
jobs <- itemId
}
close(jobs)
wg.Wait()
fileName := fmt.Sprintf("lightcones.%s.json", locale)
data, err := json.Marshal(mapItemDetail)
if err != nil {
return err
}
return os.WriteFile(fileName, data, 0644)
}

55
internal/moc.go Normal file
View File

@@ -0,0 +1,55 @@
package internal
import (
"encoding/json"
"fmt"
"hakushi-crawl/api"
"os"
"sync"
)
func SaveDataMOCEvent(locale string) error {
listItem, err := api.GetListMOCEventApi()
if err != nil {
return err
}
mapItemDetail := make(map[string]*any)
var mu sync.Mutex
jobs := make(chan string)
wg := sync.WaitGroup{}
workerCount := 12
for range workerCount {
wg.Go(func() {
for itemId := range jobs {
itemDetail, err := api.GetMOCEventInfoApi(itemId, locale)
if err != nil || itemDetail == nil {
continue
}
mu.Lock()
mapItemDetail[itemId] = itemDetail
mu.Unlock()
}
})
}
for itemId := range listItem {
jobs <- itemId
}
close(jobs)
wg.Wait()
fileName := fmt.Sprintf("moc.%s.json", locale)
data, err := json.Marshal(mapItemDetail)
if err != nil {
return err
}
return os.WriteFile(fileName, data, 0644)
}

55
internal/monster.go Normal file
View File

@@ -0,0 +1,55 @@
package internal
import (
"encoding/json"
"fmt"
"hakushi-crawl/api"
"os"
"sync"
)
func SaveDataMonster(locale string) error {
listItem, err := api.GetListMonsterApi()
if err != nil {
return err
}
mapItemDetail := make(map[string]*any)
var mu sync.Mutex
jobs := make(chan string)
wg := sync.WaitGroup{}
workerCount := 12
for range workerCount {
wg.Go(func() {
for itemId := range jobs {
itemDetail, err := api.GetMonsterInfoApi(itemId, locale)
if err != nil || itemDetail == nil {
continue
}
mu.Lock()
mapItemDetail[itemId] = itemDetail
mu.Unlock()
}
})
}
for itemId := range listItem {
jobs <- itemId
}
close(jobs)
wg.Wait()
fileName := fmt.Sprintf("monsters.%s.json", locale)
data, err := json.Marshal(mapItemDetail)
if err != nil {
return err
}
return os.WriteFile(fileName, data, 0644)
}

55
internal/peak.go Normal file
View File

@@ -0,0 +1,55 @@
package internal
import (
"encoding/json"
"fmt"
"hakushi-crawl/api"
"os"
"sync"
)
func SaveDataPeakEvent(locale string) error {
listItem, err := api.GetListPeakEventApi()
if err != nil {
return err
}
mapItemDetail := make(map[string]*any)
var mu sync.Mutex
jobs := make(chan string)
wg := sync.WaitGroup{}
workerCount := 12
for range workerCount {
wg.Go(func() {
for itemId := range jobs {
itemDetail, err := api.GetPeakEventInfoApi(itemId, locale)
if err != nil || itemDetail == nil {
continue
}
mu.Lock()
mapItemDetail[itemId] = itemDetail
mu.Unlock()
}
})
}
for itemId := range listItem {
jobs <- itemId
}
close(jobs)
wg.Wait()
fileName := fmt.Sprintf("peak.%s.json", locale)
data, err := json.Marshal(mapItemDetail)
if err != nil {
return err
}
return os.WriteFile(fileName, data, 0644)
}

55
internal/pf.go Normal file
View File

@@ -0,0 +1,55 @@
package internal
import (
"encoding/json"
"fmt"
"hakushi-crawl/api"
"os"
"sync"
)
func SaveDataPFEvent(locale string) error {
listItem, err := api.GetListPFEventApi()
if err != nil {
return err
}
mapItemDetail := make(map[string]*any)
var mu sync.Mutex
jobs := make(chan string)
wg := sync.WaitGroup{}
workerCount := 12
for range workerCount {
wg.Go(func() {
for itemId := range jobs {
itemDetail, err := api.GetPFEventInfoApi(itemId, locale)
if err != nil || itemDetail == nil {
continue
}
mu.Lock()
mapItemDetail[itemId] = itemDetail
mu.Unlock()
}
})
}
for itemId := range listItem {
jobs <- itemId
}
close(jobs)
wg.Wait()
fileName := fmt.Sprintf("pf.%s.json", locale)
data, err := json.Marshal(mapItemDetail)
if err != nil {
return err
}
return os.WriteFile(fileName, data, 0644)
}

55
internal/relic.go Normal file
View File

@@ -0,0 +1,55 @@
package internal
import (
"encoding/json"
"fmt"
"hakushi-crawl/api"
"os"
"sync"
)
func SaveDataRelic(locale string) error {
listItem, err := api.GetListRelicsetApi()
if err != nil {
return err
}
mapItemDetail := make(map[string]*any)
var mu sync.Mutex
jobs := make(chan string)
wg := sync.WaitGroup{}
workerCount := 12
for range workerCount {
wg.Go(func() {
for itemId := range jobs {
itemDetail, err := api.GetRelicInfoApi(itemId, locale)
if err != nil || itemDetail == nil {
continue
}
mu.Lock()
mapItemDetail[itemId] = itemDetail
mu.Unlock()
}
})
}
for itemId := range listItem {
jobs <- itemId
}
close(jobs)
wg.Wait()
fileName := fmt.Sprintf("relics.%s.json", locale)
data, err := json.Marshal(mapItemDetail)
if err != nil {
return err
}
return os.WriteFile(fileName, data, 0644)
}

38
main.go Normal file
View File

@@ -0,0 +1,38 @@
package main
import (
"fmt"
"hakushi-crawl/internal"
)
func main() {
listLocale := []string{
"en", "jp", "cn", "kr",
}
for _, locale := range listLocale {
fmt.Printf("Fetching character data for %s\n", locale)
internal.SaveDataCharacter(locale)
fmt.Printf("Fetching lightcone data for %s\n", locale)
internal.SaveDataLightcone(locale)
fmt.Printf("Fetching relic data for %s\n", locale)
internal.SaveDataRelic(locale)
fmt.Printf("Fetching monster data for %s\n", locale)
internal.SaveDataMonster(locale)
fmt.Printf("Fetching moc data for %s\n", locale)
internal.SaveDataMOCEvent(locale)
fmt.Printf("Fetching pf data for %s\n", locale)
internal.SaveDataPFEvent(locale)
fmt.Printf("Fetching as data for %s\n", locale)
internal.SaveDataASEvent(locale)
fmt.Printf("Fetching peak data for %s\n", locale)
internal.SaveDataPeakEvent(locale)
}
fmt.Println("DONE!")
}

34
utils/utils.go Normal file
View File

@@ -0,0 +1,34 @@
package utils
import (
"encoding/json"
"fmt"
"net/http"
)
func GetĐataJSON[T any](url string) (*T, error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("http status %d", resp.StatusCode)
}
var data T
if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
return nil, err
}
return &data, nil
}