diff --git a/config.go b/config.go
index d7335bd..36f32e5 100644
--- a/config.go
+++ b/config.go
@@ -2,8 +2,10 @@ package main
 
 import (
 	"flag"
+	"net"
 
 	"github.com/vharitonsky/iniflags"
+	"github.com/sirupsen/logrus"
 )
 
 var (
@@ -21,7 +23,8 @@ var (
 	localCert         = flag.String("local_cert", "", "SSL certificate for STARTTLS/TLS")
 	localKey          = flag.String("local_key", "", "SSL private key for STARTTLS/TLS")
 	localForceTLS     = flag.Bool("local_forcetls", false, "Force STARTTLS (needs local_cert and local_key)")
-	allowedNets       = flag.String("allowed_nets", "127.0.0.1/8 ::1/128", "Networks allowed to send mails")
+	allowedNetsStr    = flag.String("allowed_nets", "127.0.0.0/8 ::1/128", "Networks allowed to send mails")
+	allowedNets       = []*net.IPNet{}
 	allowedSender     = flag.String("allowed_sender", "", "Regular expression for valid FROM EMail addresses")
 	allowedRecipients = flag.String("allowed_recipients", "", "Regular expression for valid TO EMail addresses")
 	allowedUsers      = flag.String("allowed_users", "", "Path to file with valid users/passwords")
@@ -33,6 +36,29 @@ var (
 	versionInfo       = flag.Bool("version", false, "Show version information")
 )
 
+
+func setupAllowedNetworks() {
+	for _, netstr := range splitstr(*allowedNetsStr, ' ') {
+		baseIP, allowedNet, err := net.ParseCIDR(netstr)
+		if err != nil {
+			log.WithField("netstr", netstr).
+				WithError(err).
+				Fatal("Invalid CIDR notation in allowed_nets")
+		}
+
+		// Reject any network specification where any host bits are set,
+		// meaning the address refers to a host and not a network.
+		if !allowedNet.IP.Equal(baseIP) {
+			log.WithFields(logrus.Fields{
+				"given_net": netstr,
+				"proper_net": allowedNet,
+			}).Fatal("Invalid network in allowed_nets (host bits set)")
+		}
+
+		allowedNets = append(allowedNets, allowedNet)
+	}
+}
+
 func ConfigLoad() {
 	iniflags.Parse()
 
@@ -42,4 +68,6 @@ func ConfigLoad() {
 	if (*remoteHost == "") {
 		log.Warn("remote_host not set; mail will not be forwarded!")
 	}
+
+	setupAllowedNetworks()
 }
diff --git a/main.go b/main.go
index de436c8..cbd5993 100644
--- a/main.go
+++ b/main.go
@@ -17,20 +17,15 @@ import (
 )
 
 func connectionChecker(peer smtpd.Peer) error {
-	var peerIP net.IP
-	if addr, ok := peer.Addr.(*net.TCPAddr); ok {
-		peerIP = net.ParseIP(addr.IP.String())
-	} else {
-		log.WithField("ip", addr.IP).
-			Warn("failed to parse IP")
-		return smtpd.Error{Code: 421, Message: "Denied"}
+	// This can't panic because we only have TCP listeners
+	peerIP := peer.Addr.(*net.TCPAddr).IP
+
+	if len(allowedNets) == 0 {
+		// Special case: empty string means allow everything
+		return nil
 	}
 
-	nets := strings.Split(*allowedNets, " ")
-
-	for i := range nets {
-		_, allowedNet, _ := net.ParseCIDR(nets[i])
-
+	for _, allowedNet := range allowedNets {
 		if allowedNet.Contains(peerIP) {
 			return nil
 		}
diff --git a/smtprelay.ini b/smtprelay.ini
index c1e90e8..0b97399 100644
--- a/smtprelay.ini
+++ b/smtprelay.ini
@@ -31,7 +31,8 @@
 ;local_forcetls = false
 
 ; Networks that are allowed to send mails to us
-;allowed_nets = 127.0.0.1/8 ::1/128
+; Defaults to localhost. If set to "", then any address is allowed.
+;allowed_nets = 127.0.0.0/8 ::1/128
 
 ; Regular expression for valid FROM EMail addresses
 ; Example: ^(.*)@localhost.localdomain$