main.go 2.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144
  1. package main
  2. import (
  3. "errors"
  4. "io/ioutil"
  5. "log"
  6. "net"
  7. "net/http"
  8. "os"
  9. "strings"
  10. )
  11. var (
  12. lists = []string{
  13. "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts",
  14. "https://mirror1.malwaredomains.com/files/justdomains",
  15. "http://sysctl.org/cameleon/hosts",
  16. "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist",
  17. "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt",
  18. "https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt",
  19. "https://hosts-file.net/ad_servers.txt",
  20. }
  21. invalid = []string{
  22. "localhost", "localhost.localdomain", "local", "broadcasthost",
  23. "ip6-localhost", "ip6-loopback", "ip6-localnet",
  24. "ip6-mcastprefix", "ip6-allnodes", "ip6-allrouters",
  25. "ip6-allhosts",
  26. }
  27. )
  28. func main() {
  29. if len(os.Args) < 2 {
  30. log.Fatal("Please provide a zone file to write to.")
  31. }
  32. zone_file := os.Args[1]
  33. // Open the zone file, and create it if needed
  34. f, err := os.OpenFile(zone_file, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
  35. if err != nil {
  36. log.Fatal(err)
  37. }
  38. defer f.Close()
  39. // Override the current content
  40. if err := f.Truncate(0); err != nil {
  41. log.Fatal(err)
  42. }
  43. // Fetch and parse the zone content from lists
  44. c := make(chan string)
  45. for _, list := range lists {
  46. go handle(list, c)
  47. }
  48. // Write the zones
  49. for range lists {
  50. if _, err = f.WriteString(<-c); err != nil {
  51. log.Fatal(err)
  52. }
  53. }
  54. }
  55. func handle(url string, zone chan string) {
  56. body, err := fetch(url)
  57. if err != nil {
  58. return
  59. }
  60. zone <- sanitize(body)
  61. }
  62. func fetch(url string) (string, error) {
  63. req, err := http.Get(url)
  64. if err != nil {
  65. return "", err
  66. }
  67. defer req.Body.Close()
  68. body, err := ioutil.ReadAll(req.Body)
  69. if err != nil {
  70. return "", err
  71. }
  72. return string(body), nil
  73. }
  74. func sanitize(records string) string {
  75. var sanitized []string
  76. for _, line := range strings.Split(records, "\n") {
  77. ip, domain, err := parse_record(line)
  78. if err != nil {
  79. continue
  80. }
  81. // Domain is invalid (e.g., "localhost")
  82. if is_invalid(domain) {
  83. continue
  84. }
  85. // Domain is an IP address
  86. if net.ParseIP(domain) != nil {
  87. continue
  88. }
  89. // Unify the records
  90. if ip != "127.0.0.1" {
  91. ip = "127.0.0.1"
  92. }
  93. sanitized = append(sanitized, ip + " " + domain)
  94. }
  95. return strings.Join(sanitized, "\n")
  96. }
  97. func parse_record(record string) (string, string, error) {
  98. if len(record) == 0 {
  99. return "", "", errors.New("Empty line")
  100. }
  101. if strings.HasPrefix(record, "#") {
  102. return "", "", errors.New("Comment")
  103. }
  104. words := strings.Fields(record)
  105. // Domain-only list
  106. if len(words) == 1 {
  107. return "", words[0], nil
  108. }
  109. return words[0], words[1], nil
  110. }
  111. func is_invalid(domain string) bool {
  112. for _, i := range invalid {
  113. if domain == i {
  114. return true
  115. }
  116. }
  117. return false
  118. }