ads-blacklist/main.go

package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"strings"
)

var (
	lists = []string{
		"https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts",
		"https://mirror1.malwaredomains.com/files/justdomains",
		"http://sysctl.org/cameleon/hosts",
		"https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist",
		"https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt",
		"https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt",
		"https://hosts-file.net/ad_servers.txt",
	}

	invalid = []string{
		"localhost", "localhost.localdomain", "local", "broadcasthost",
		"ip6-localhost", "ip6-loopback", "ip6-localnet",
		"ip6-mcastprefix", "ip6-allnodes", "ip6-allrouters",
		"ip6-allhosts",
	}
	zone_file = "blacklist.txt"
)

func main() {
	fmt.Print(generateList())
}

func generateList() (string, error) {
	c := make(chan string)
	for _, list := range lists {
		go handle(list, c)
	}

	var zones []string
	for range lists {
		zones = append(zones, <-c)
	}

	return strings.Join(zones, "\n"), nil
}

func handle(url string, zone chan string) {
	body, err := fetch(url)
	if err != nil {
		return
	}

	zone <- sanitize(body)
}

func fetch(url string) (string, error) {
	req, err := http.Get(url)
	if err != nil {
		return "", err
	}
	defer req.Body.Close()

	body, err := ioutil.ReadAll(req.Body)
	if err != nil {
		return "", err
	}

	return string(body), nil
}

func sanitize(records string) string {
	var sanitized []string

	for _, line := range strings.Split(records, "\n") {
		ip, domain, err := parse_record(line)
		if err != nil {
			continue
		}

		// Domain is invalid (e.g., "localhost")
		if is_invalid(domain) {
			continue
		}

		// Domain is an IP address
		if net.ParseIP(domain) != nil {
			continue
		}

		// Unify the records
		if ip != "127.0.0.1" {
			ip = "127.0.0.1"
		}

		sanitized = append(sanitized, ip + " " + domain)
	}

	return strings.Join(sanitized, "\n")
}

func parse_record(record string) (string, string, error) {
	if len(record) == 0 {
		return "", "", errors.New("Empty line")
	}

	if strings.HasPrefix(record, "#") {
		return "", "", errors.New("Comment")
	}

	words := strings.Fields(record)
	// Domain-only list
	if len(words) == 1 {
		return "", words[0], nil
	}

	return words[0], words[1], nil
}

func is_invalid(domain string) bool {
	for _, i := range invalid {
		if domain == i {
			return true
		}
	}
	return false
}