package main import ( "errors" "io/ioutil" "log" "net" "net/http" "os" "strings" ) var ( zone_file = "tmp" lists = []string{ "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts", "https://mirror1.malwaredomains.com/files/justdomains", "http://sysctl.org/cameleon/hosts", "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist", "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt", "https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt", "https://hosts-file.net/ad_servers.txt", } invalid = []string{ "localhost", "localhost.localdomain", "local", "broadcasthost", "ip6-localhost", "ip6-loopback", "ip6-localnet", "ip6-mcastprefix", "ip6-allnodes", "ip6-allrouters", "ip6-allhosts", } ) func main() { // Open the zone file, and create it if needed f, err := os.OpenFile(zone_file, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) if err != nil { log.Fatal(err) } defer f.Close() // Override the current content if err := f.Truncate(0); err != nil { log.Fatal(err) } // Fetch and parse the zone content from lists c := make(chan string) for _, list := range lists { go handle(list, c) } // Write the zones for range lists { if _, err = f.WriteString(<-c); err != nil { log.Fatal(err) } } } func handle(url string, zone chan string) { body, err := fetch(url) if err != nil { return } zone <- sanitize(body) } func fetch(url string) (string, error) { req, err := http.Get(url) if err != nil { return "", err } defer req.Body.Close() body, err := ioutil.ReadAll(req.Body) if err != nil { return "", err } return string(body), nil } func sanitize(records string) string { var sanitized []string for _, line := range strings.Split(records, "\n") { ip, domain, err := parse_record(line) if err != nil { continue } // Domain is invalid (e.g., "localhost") if is_invalid(domain) { continue } // Domain is an IP address if net.ParseIP(domain) != nil { continue } // Unify the records if ip != "127.0.0.1" { ip = "127.0.0.1" } sanitized = append(sanitized, ip + " " + domain) } return strings.Join(sanitized, "\n") } func parse_record(record string) (string, string, error) { if len(record) == 0 { return "", "", errors.New("Empty line") } if strings.HasPrefix(record, "#") { return "", "", errors.New("Comment") } words := strings.Fields(record) // Domain-only list if len(words) == 1 { return "", words[0], nil } return words[0], words[1], nil } func is_invalid(domain string) bool { for _, i := range invalid { if domain == i { return true } } return false }