mirrors/nali
1
0
Fork 0
mirror of https://github.com/nxtrace/nali synced 2024-11-21 16:49:39 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: Support update db by direct download

Browse Source 
Signed-off-by: zu1k <i@zu1k.com>
This commit is contained in:
zu1k 2022-05-24 15:16:48 +08:00
parent bd84bf86f0
commit f391935999
9 changed files with 93 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
internal/config/config.go
View File

@ -35,6 +35,7 @@ func ReadConfig(basePath string) {
if err != nil {
log.Fatalln("Config invalid:", err)
}
db.NameDBMap.From(dbList)
db.TypeDBMap.From(dbList)
}

25
internal/db/default.go
View File

@ -1,5 +1,10 @@
package db
import (
"github.com/zu1k/nali/pkg/cdn"
"github.com/zu1k/nali/pkg/ip2region"
)
func GetDefaultDBList() List {
return List{
&DB{
@ -57,10 +62,11 @@ func GetDefaultDBList() List {
NameAlias: []string{
"i2r",
},
Format: FormatIP2Region,
File: "ip2region.db",
Languages: LanguagesZH,
Types: TypesIPv4,
Format: FormatIP2Region,
File: "ip2region.db",
Languages: LanguagesZH,
Types: TypesIPv4,
DownloadUrls: ip2region.DownloadUrls,
},
&DB{
Name: "ip2location",
@ -71,11 +77,12 @@ func GetDefaultDBList() List {
},
&DB{
Name: "cdn",
Format: FormatCDNSkkYml,
File: "cdn.yml",
Languages: LanguagesZH,
Types: TypesCDN,
Name: "cdn",
Format: FormatCDNSkkYml,
File: "cdn.yml",
Languages: LanguagesZH,
Types: TypesCDN,
DownloadUrls: cdn.DownloadUrls,
},
}
}

6
internal/db/type.go
View File

@ -15,12 +15,14 @@ import (
type DB struct {
Name string
NameAlias []string `yaml:"name-alias,omitempty"`
NameAlias []string `yaml:"name-alias,omitempty" mapstructure:"name-alias"`
Format Format
File string
Languages []string
Types []Type
DownloadUrls []string `yaml:"download-urls,omitempty" mapstructure:"download-urls"`
}
func (d *DB) get() (db dbif.DB) {
@ -43,7 +45,7 @@ func (d *DB) get() (db dbif.DB) {
case FormatIP2Region:
db, err = ip2region.NewIp2Region(filePath)
case FormatIP2Location:
db, err = ip2locationdb.NewIP2LocationDB(filePath)
db, err = ip2location.NewIP2Location(filePath)
case FormatCDNSkkYml:
db, err = cdn.NewCDN(filePath)
default:

29
internal/db/update.go
View File

@ -6,6 +6,7 @@ import (
"time"
"github.com/zu1k/nali/pkg/cdn"
"github.com/zu1k/nali/pkg/download"
"github.com/zu1k/nali/pkg/ip2region"
"github.com/zu1k/nali/pkg/qqwry"
"github.com/zu1k/nali/pkg/zxipv6wry"
@ -38,13 +39,30 @@ var DbNameListForUpdate = []string{
func getUpdateFuncByName(name string) (func() error, string) {
name = strings.TrimSpace(name)
if db := getDbByName(name); db != nil {
// direct download if download-url not null
if len(db.DownloadUrls) > 0 {
return func() error {
log.Printf("正在下载最新 %s 数据库...\n", db.Name)
_, err := download.Download(db.File, db.DownloadUrls...)
if err != nil {
log.Printf("%s 数据库下载失败: %s\n", db.Name, db.File)
log.Println("error:", err)
return err
} else {
log.Printf("%s 数据库下载成功: %s\n", db.Name, db.File)
return nil
}
}, string(db.Format)
}
// intenel download func
switch db.Format {
case FormatQQWry:
return func() error {
log.Println("正在下载最新 纯真 IPv4数据库...")
_, err := qqwry.Download(getDbByName("qqwry").File)
if err != nil {
log.Fatalln("数据库 QQWry 下载失败:", err)
log.Println("数据库 QQWry 下载失败:", err)
}
return err
}, FormatQQWry
@ -53,7 +71,7 @@ func getUpdateFuncByName(name string) (func() error, string) {
log.Println("正在下载最新 ZX IPv6数据库...")
_, err := zxipv6wry.Download(getDbByName("zxipv6wry").File)
if err != nil {
log.Fatalln("数据库 ZXIPv6Wry 下载失败:", err)
log.Println("数据库 ZXIPv6Wry 下载失败:", err)
}
return err
}, FormatZXIPv6Wry
@ -62,7 +80,7 @@ func getUpdateFuncByName(name string) (func() error, string) {
log.Println("正在下载最新 Ip2Region 数据库...")
_, err := ip2region.Download(getDbByName("ip2region").File)
if err != nil {
log.Fatalln("数据库 Ip2Region 下载失败:", err)
log.Println("数据库 Ip2Region 下载失败:", err)
}
return err
}, FormatZXIPv6Wry
@ -71,13 +89,14 @@ func getUpdateFuncByName(name string) (func() error, string) {
log.Println("正在下载最新 CDN服务提供商数据库...")
_, err := cdn.Download(getDbByName("cdn").File)
if err != nil {
log.Fatalln("数据库 CDN 下载失败:", err)
log.Println("数据库 CDN 下载失败:", err)
}
return err
}, FormatZXIPv6Wry
default:
return func() error {
log.Fatalln("不支持该类型数据库的自动更新：", db.Format)
log.Println("暂不支持该类型数据库的自动更新")
log.Println("可通过指定数据库的 download-urls 从特定链接下载数据库文件")
return nil
}, time.Now().String()
}

18
pkg/cdn/update.go
View File

@ -6,19 +6,19 @@ import (
"github.com/zu1k/nali/pkg/common"
)
const (
githubUrl = "https://raw.githubusercontent.com/SukkaLab/cdn/master/src/cdn.yml"
jsdelivrUrl = "https://cdn.jsdelivr.net/gh/SukkaLab/cdn/src/cdn.yml"
githubUrl2 = "https://raw.githubusercontent.com/4ft35t/cdn/master/src/cdn.yml"
jsdelivrUrl2 = "https://cdn.jsdelivr.net/gh/4ft35t/cdn/src/cdn.yml"
)
var DownloadUrls = []string{
"https://cdn.jsdelivr.net/gh/SukkaLab/cdn/src/cdn.yml",
"https://raw.githubusercontent.com/SukkaLab/cdn/master/src/cdn.yml",
"https://cdn.jsdelivr.net/gh/4ft35t/cdn/src/cdn.yml",
"https://raw.githubusercontent.com/4ft35t/cdn/master/src/cdn.yml",
}
// Deprecated: This will be removed from 0.5.0, use package download instead
func Download(filePath ...string) (data []byte, err error) {
data, err = common.GetHttpClient().Get(jsdelivrUrl, githubUrl, jsdelivrUrl2, githubUrl2)
data, err = common.GetHttpClient().Get(DownloadUrls...)
if err != nil {
log.Printf("CDN数据库下载失败，请手动下载解压后保存到本地: %s \n", filePath)
log.Println("下载链接：", githubUrl)
log.Println("下载链接：", DownloadUrls)
return
}

2
pkg/dbif/db.go
View File

@ -30,6 +30,6 @@ var (
_ DB = &ipip.IPIPFree{}
_ DB = &geoip.GeoIP{}
_ DB = &ip2region.Ip2Region{}
_ DB = &ip2locationdb.IP2LocationDB{}
_ DB = &ip2location.IP2Location{}
_ DB = &cdn.CDN{}
)

25
pkg/download/download.go Normal file
View File

@ -0,0 +1,25 @@
package download
import (
"log"
"github.com/zu1k/nali/pkg/common"
)
func Download(filePath string, urls ...string) (data []byte, err error) {
_ = urls[0]
data, err = common.GetHttpClient().Get(urls...)
if err != nil {
log.Printf("文件下载失败，请手动下载解压后保存到本地: %s \n", filePath)
log.Println("下载链接：", urls)
return
}
if len(filePath) == 1 {
if err := common.SaveFile(filePath, data); err == nil {
log.Println("文件下载成功:", filePath)
}
}
return
}

12
pkg/ip2location/ip2locationdb.go → pkg/ip2location/ip2location.go
View File

@ -1,4 +1,4 @@
package ip2locationdb
package ip2location
import (
"errors"
@ -10,13 +10,13 @@ import (
"github.com/ip2location/ip2location-go/v9"
)
// IP2LocationDB
type IP2LocationDB struct {
// IP2Location
type IP2Location struct {
db *ip2location.DB
}
// new IP2Location from database file
func NewIP2LocationDB(filePath string) (*IP2LocationDB, error) {
func NewIP2Location(filePath string) (*IP2Location, error) {
_, err := os.Stat(filePath)
if err != nil && os.IsNotExist(err) {
log.Println("文件不存在，请自行下载 IP2Location 库，并保存在", filePath)
@ -27,11 +27,11 @@ func NewIP2LocationDB(filePath string) (*IP2LocationDB, error) {
if err != nil {
log.Fatal(err)
}
return &IP2LocationDB{db: db}, nil
return &IP2Location{db: db}, nil
}
}
func (x IP2LocationDB) Find(query string, params ...string) (result fmt.Stringer, err error) {
func (x IP2Location) Find(query string, params ...string) (result fmt.Stringer, err error) {
ip := net.ParseIP(query)
if ip == nil {
return nil, errors.New("Query should be valid IP")

13
pkg/ip2region/update.go
View File

@ -6,16 +6,17 @@ import (
"github.com/zu1k/nali/pkg/common"
)
const (
githubUrl = "https://raw.githubusercontent.com/lionsoul2014/ip2region/master/data/ip2region.db"
jsdelivrUrl = "https://cdn.jsdelivr.net/gh/lionsoul2014/ip2region/data/ip2region.db"
)
var DownloadUrls = []string{
"https://cdn.jsdelivr.net/gh/lionsoul2014/ip2region/data/ip2region.db",
"https://raw.githubusercontent.com/lionsoul2014/ip2region/master/data/ip2region.db",
}
// Deprecated: This will be removed from 0.5.0, use package download instead
func Download(filePath ...string) (data []byte, err error) {
data, err = common.GetHttpClient().Get(jsdelivrUrl, githubUrl)
data, err = common.GetHttpClient().Get(DownloadUrls...)
if err != nil {
log.Printf("CDN数据库下载失败，请手动下载解压后保存到本地: %s \n", filePath)
log.Println("下载链接：", githubUrl)
log.Println("下载链接：", DownloadUrls)
return
}