diff --git a/config.go b/config.go
index 8e4894d..155069a 100644
--- a/config.go
+++ b/config.go
@@ -157,6 +157,14 @@ func splitProto(s string) protoAddr {
 func setupListeners() {
 	for _, listenAddr := range strings.Split(*listenStr, " ") {
 		pa := splitProto(listenAddr)
+
+		if localAuthRequired() && pa.protocol == "" {
+			log.WithField("address", pa.address).
+				Fatal("Local authentication (via allowed_users file) " +
+				      "not allowed with non-TLS listener")
+		}
+
+
 		listenAddrs = append(listenAddrs, pa)
 	}
 }