Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: fetch ranges from AS via bgp tools #8

Merged
merged 3 commits into from
Oct 2, 2024
Merged

feat: fetch ranges from AS via bgp tools #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
03f2188
feat: fetch ranges from AS via bgp tools
nohehf Oct 2, 2024
3794f9d
fix: alibaba ranges
nohehf Oct 2, 2024
c36d12d
Merge remote-tracking branch 'origin' into feat/fetch-ranges-from-AS-…
nohehf Oct 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 13 additions & 13 deletions internal/source/source_alibaba.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,26 +7,26 @@ import (

type Alibaba struct{}

const alibabaFileURL = "https://raw.githubusercontent.com/devanshbatham/ip2cloud/main/data/aliyun.txt"
var AlibabaASNs = []string{
// Alibaba cloud
"24429",
// Alibaba AS45102
"45102",
// Alibaba (china) AS37963
"37963",
}

func (a Alibaba) GetProvider() provider.Provider {
return provider.Alibaba
}

func (a Alibaba) GetIPRanges() []*IPRange {
log.Info("Using static Alibaba ip ranges")

ranges := make([]*IPRange, 0)
alibabaRanges, err := LoadTextURLToRange(alibabaFileURL)
if err != nil {
log.Fatal("Failed to load text url to range for Alibaba", err)
}
for _, cidr := range alibabaRanges {
network, cat := ParseCIDR(cidr)
ranges = append(ranges, &IPRange{
Network: network,
Cat: cat,
})
for _, asn := range AlibabaASNs {
log.Info("[Alibaba] - Using ranges from ASN list (AS%s)", asn)
_ranges := getRangesForAsn(asn)
ranges = append(ranges, _ranges...)
log.Info("[Alibaba] - Found %d ranges for AS%s", len(ranges), asn)
}
return ranges
}
26 changes: 12 additions & 14 deletions internal/source/source_linode.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,26 +7,24 @@ import (

type Linode struct{}

const LinodeFileURL = "https://raw.githubusercontent.com/devanshbatham/ip2cloud/main/data/linode.txt"

func (a Linode) GetProvider() provider.Provider {
return provider.Linode
}

func (a Linode) GetIPRanges() []*IPRange {
log.Info("Using static Linode ip ranges")
var LinodeASNs = []string{
// Linode AS63949
"63949",
// Linode CorpNet
"48337",
}

func (a Linode) GetIPRanges() []*IPRange {
ranges := make([]*IPRange, 0)
linodeRanges, err := LoadTextURLToRange(LinodeFileURL)
if err != nil {
log.Fatal("Failed to load text url to range for Linode", err)
}
for _, cidr := range linodeRanges {
network, cat := ParseCIDR(cidr)
ranges = append(ranges, &IPRange{
Network: network,
Cat: cat,
})
for _, asn := range LinodeASNs {
log.Info("[Linode] - Using ranges from ASN list (AS%s)", asn)
_ranges := getRangesForAsn(asn)
ranges = append(ranges, _ranges...)
log.Info("[Linode] - Found %d ranges for AS%s", len(ranges), asn)
}
return ranges
}
76 changes: 6 additions & 70 deletions internal/source/source_ovh.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,80 +7,16 @@ import (

type Ovh struct{}

// TODO: Dynamically fetch the sources
// Source: https://ipinfo.io/AS16276#block-ranges
var ovhRanges = [...]string{
"135.125.0.0/17",
"135.125.128.0/17",
"135.148.0.0/17",
"135.148.128.0/17",
"137.74.0.0/16",
"139.99.0.0/17",
"139.99.128.0/17",
"141.227.128.0/24",
"141.227.130.0/24",
"141.227.132.0/24",
"141.227.134.0/24",
"141.227.136.0/24",
"141.227.138.0/24",
"141.227.140.0/24",
"141.94.0.0/16",
"141.95.0.0/17",
"141.95.128.0/17",
"142.4.192.0/19",
"142.44.128.0/17",
"142.44.140.0/24",
"144.217.0.0/16",
"145.239.0.0/16",
"146.59.0.0/16",
"146.59.0.0/17",
"147.135.0.0/17",
"147.135.128.0/17",
"148.113.0.0/18",
"148.113.128.0/17",
"149.202.0.0/16",
"149.56.0.0/16",
"151.80.0.0/16",
"15.204.0.0/17",
"15.204.128.0/17",
"152.228.128.0/17",
"15.235.0.0/17",
"15.235.128.0/17",
"158.69.0.0/16",
"162.19.0.0/17",
"162.19.128.0/17",
"164.132.0.0/16",
"167.114.0.0/17",
"167.114.128.0/18",
"167.114.192.0/19",
"176.31.0.0/16",
"178.32.0.0/15",
"185.15.68.0/22",
"185.45.160.0/22",
"188.165.0.0/16",
"192.240.152.0/21",
"192.31.246.0/24",
"192.95.0.0/18",
"192.99.0.0/16",
"192.99.65.0/24",
"193.70.0.0/17",
}

func (a Ovh) GetProvider() provider.Provider {
return provider.Ovh
}

func (a Ovh) GetIPRanges() []*IPRange {
log.Info("Using static ovh ip ranges")

ranges := make([]*IPRange, 0)
for _, cidr := range ovhRanges {
network, cat := ParseCIDR(cidr)
ranges = append(ranges, &IPRange{
Network: network,
Cat: cat,
})
}
// Note: there is also AS22598 for OVHTelecom. But this probably doesn't expose hosting services
var OvhCloudASN = "16276"

func (a Ovh) GetIPRanges() []*IPRange {
log.Info("[Ovh] - Using ranges from ASN list (AS%s)", OvhCloudASN)
ranges := getRangesForAsn(OvhCloudASN)
log.Info("[Ovh] - Found %d ranges for AS%s", len(ranges), OvhCloudASN)
return ranges
}
35 changes: 5 additions & 30 deletions internal/source/source_scaleway.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,40 +7,15 @@ import (

type Scaleway struct{}

// Source: https://ipinfo.io/AS12876#block-ranges
var scalewayRanges = [...]string{
"151.115.0.0/18",
"151.115.64.0/18",
"163.172.0.0/16",
"163.172.208.0/20",
"195.154.0.0/16",
"212.129.0.0/18",
"212.47.224.0/19",
"212.83.128.0/19",
"212.83.160.0/19",
"51.15.0.0/16",
"51.15.0.0/17",
"51.158.0.0/15",
"51.158.128.0/17",
"62.210.0.0/16",
"62.4.0.0/19",
}

func (a Scaleway) GetProvider() provider.Provider {
return provider.Scaleway
}

func (a Scaleway) GetIPRanges() []*IPRange {
log.Info("Using static Scaleway ip ranges")

ranges := make([]*IPRange, 0)
for _, cidr := range scalewayRanges {
network, cat := ParseCIDR(cidr)
ranges = append(ranges, &IPRange{
Network: network,
Cat: cat,
})
}
var ScalewayASN = "12876"

func (a Scaleway) GetIPRanges() []*IPRange {
log.Info("[Scaleway] - Using ranges from ASN list (AS%s)", ScalewayASN)
ranges := getRangesForAsn(ScalewayASN)
log.Info("[Scaleway] - Found %d ranges for AS%s", len(ranges), ScalewayASN)
return ranges
}
28 changes: 14 additions & 14 deletions internal/source/source_tencent.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,26 +7,26 @@ import (

type Tencent struct{}

const tencentFileURL = "https://raw.githubusercontent.com/devanshbatham/ip2cloud/main/data/tencent.txt"

func (a Tencent) GetProvider() provider.Provider {
return provider.Tencent
}

func (a Tencent) GetIPRanges() []*IPRange {
log.Info("Using static Tencent ip ranges")
var TencentASNs = []string{
// Tencent Cloud
"132591",
// Tencent Global
"132203",
// Tencent-CN
"45090",
}

func (a Tencent) GetIPRanges() []*IPRange {
ranges := make([]*IPRange, 0)
tencentRanges, err := LoadTextURLToRange(tencentFileURL)
if err != nil {
log.Fatal("Failed to load text url to range for Tencent", err)
}
for _, cidr := range tencentRanges {
network, cat := ParseCIDR(cidr)
ranges = append(ranges, &IPRange{
Network: network,
Cat: cat,
})
for _, asn := range TencentASNs {
log.Info("[Tencent] - Using ranges from ASN list (AS%s)", asn)
_ranges := getRangesForAsn(asn)
ranges = append(ranges, _ranges...)
log.Info("[Tencent] - Found %d ranges for AS%s", len(ranges), asn)
}
return ranges
}
22 changes: 6 additions & 16 deletions internal/source/source_ucloud.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,26 +7,16 @@ import (

type Ucloud struct{}

const ucloudFileURL = "https://raw.githubusercontent.com/devanshbatham/ip2cloud/main/data/ucloud.txt"

func (a Ucloud) GetProvider() provider.Provider {
return provider.Ucloud
}

func (a Ucloud) GetIPRanges() []*IPRange {
log.Info("Using static Ucloud ip ranges")
// UCLOUD INFORMATION TECHNOLOGY (HK) LIMITED
var UcloudASN = "135377"

ranges := make([]*IPRange, 0)
ucloudRanges, err := LoadTextURLToRange(ucloudFileURL)
if err != nil {
log.Fatal("Failed to load text url to range for ucloud", err)
}
for _, cidr := range ucloudRanges {
network, cat := ParseCIDR(cidr)
ranges = append(ranges, &IPRange{
Network: network,
Cat: cat,
})
}
func (a Ucloud) GetIPRanges() []*IPRange {
log.Info("[Ucloud] - Using ranges from ASN list (AS%s)", UcloudASN)
ranges := getRangesForAsn(UcloudASN)
log.Info("[Ucloud] - Found %d ranges for AS%s", len(ranges), UcloudASN)
return ranges
}
1 change: 1 addition & 0 deletions internal/source/source_vercel.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ func (a Vercel) GetProvider() provider.Provider {
}

// TODO: Get this range dynamically
// NOTE: this is pretty flaky, as vercel is just a wrapper of AWS. Maybe this provider should be removed
// Source: https://networksdb.io/ip-addresses-of/vercel-inc
const vercelRange = "76.76.21.0/24"

Expand Down
68 changes: 68 additions & 0 deletions internal/source/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package source

import (
"bufio"
"context"
"encoding/json"
"errors"
Expand All @@ -9,7 +10,10 @@ import (
"net"
"net/http"
"strings"
"sync"
"time"

"github.com/Escape-Technologies/cloudfinder/internal/log"
)

const defaultHTTPTimeout = 30 * time.Second
Expand Down Expand Up @@ -128,3 +132,67 @@ func isPrivateNetwork(n *net.IPNet) bool {
}
return false
}

/// BGP TOOLS

// ensure we fill the map only once
var bgpToolsMutex = sync.Mutex{}

// ASN -> CIDR
var bgpToolsAsnRanges map[string][]*IPRange
var bgpToolsTableURL = "https://bgp.tools/table.txt"

// Fetches https://bgp.tools/table.txt and parses it into the ASN -> CIDR map
func getRangesForAsn(asn string) []*IPRange {
bgpToolsMutex.Lock()
if bgpToolsAsnRanges == nil {
bgpToolsAsnRanges = make(map[string][]*IPRange)

// Fill the map
log.Info("Fetching AS infos from %s", bgpToolsTableURL)
req, _ := http.NewRequest(http.MethodGet, bgpToolsTableURL, nil) // nolint: noctx
// bgp tools requires a descriptive user agent in case the program gets out of control
req.Header.Add("user-agent", "https://github.com/Escape-Technologies/cloudfinder - [email protected]")
res, err := http.DefaultClient.Do(req)
if err != nil {
log.Fatal("Error getting bgp tools table", err)
}
if res.StatusCode != http.StatusOK {
err := fmt.Errorf("status code: %d", res.StatusCode)
log.Fatal("Error getting bgp tools table", err)
}

scanner := bufio.NewScanner(res.Body)
// Read lines
for scanner.Scan() {
line := scanner.Text()
// Line is formatted as "<CIDR> <ASN>"
x := strings.Split(line, " ")
if len(x) != 2 { // nolint: mnd
continue
}
cidr, asn := x[0], x[1]

n, cat := ParseCIDR(cidr)
// Skip private networks
if isPrivateNetwork(n) {
continue
}
// Fill map
if _, ok := bgpToolsAsnRanges[asn]; !ok {
bgpToolsAsnRanges[asn] = make([]*IPRange, 0)
}
bgpToolsAsnRanges[asn] = append(bgpToolsAsnRanges[asn], &IPRange{
Network: n,
Cat: cat,
})
}
res.Body.Close()
log.Info("Got %d AS infos", len(bgpToolsAsnRanges))
}
bgpToolsMutex.Unlock()
if val, ok := bgpToolsAsnRanges[asn]; ok {
return val
}
return []*IPRange{}
}
2 changes: 1 addition & 1 deletion internal/static/hash.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
56bb287f7dbb22a8ba1ccad5e16fe86e8bd3632de2a056b37ed939cd3692ab7f
47c10bda97e1961c363626868c7aa32aace0cff5557877bbdcd0ea48a506804e
Binary file modified internal/static/ipv4.gob
View file
Open in desktop
Binary file not shown.
Binary file modified internal/static/ipv6.gob
View file
Open in desktop
Binary file not shown.
Loading
Loading