diff --git a/config.go b/config.go
index 5a29ba5..a1bdc53 100644
--- a/config.go
+++ b/config.go
@@ -28,7 +28,8 @@ var (
 	allowedNets       = []*net.IPNet{}
 	allowedSenderStr  = flag.String("allowed_sender", "", "Regular expression for valid FROM EMail addresses")
 	allowedSender     *regexp.Regexp
-	allowedRecipients = flag.String("allowed_recipients", "", "Regular expression for valid TO EMail addresses")
+	allowedRecipStr   = flag.String("allowed_recipients", "", "Regular expression for valid TO EMail addresses")
+	allowedRecipients *regexp.Regexp
 	allowedUsers      = flag.String("allowed_users", "", "Path to file with valid users/passwords")
 	remoteHost        = flag.String("remote_host", "", "Outgoing SMTP server")
 	remoteUser        = flag.String("remote_user", "", "Username for authentication on outgoing SMTP server")
@@ -61,17 +62,25 @@ func setupAllowedNetworks() {
 	}
 }
 
-func setupAllowedSender() {
-	if (*allowedSenderStr == "") {
-		return
+func setupAllowedPatterns() {
+	var err error
+
+	if (*allowedSenderStr != "") {
+		allowedSender, err = regexp.Compile(*allowedSenderStr)
+		if err != nil {
+			log.WithField("allowed_sender", *allowedSenderStr).
+				WithError(err).
+				Fatal("allowed_sender pattern invalid")
+		}
 	}
 
-	var err error
-	allowedSender, err = regexp.Compile(*allowedSenderStr)
-	if err != nil {
-		log.WithField("allowed_sender", *allowedSenderStr).
-			WithError(err).
-			Fatal("allowed_sender pattern invalid")
+	if (*allowedRecipStr != "") {
+		allowedRecipients, err = regexp.Compile(*allowedRecipStr)
+		if err != nil {
+			log.WithField("allowed_recipients", *allowedRecipStr).
+				WithError(err).
+				Fatal("allowed_recipients pattern invalid")
+		}
 	}
 }
 
@@ -86,5 +95,5 @@ func ConfigLoad() {
 	}
 
 	setupAllowedNetworks()
-	setupAllowedSender()
+	setupAllowedPatterns()
 }
diff --git a/main.go b/main.go
index c0e8b64..5c960ef 100644
--- a/main.go
+++ b/main.go
@@ -7,7 +7,6 @@ import (
 	"net/smtp"
 	"net/textproto"
 	"os"
-	"regexp"
 	"strings"
 	"time"
 
@@ -121,19 +120,13 @@ func senderChecker(peer smtpd.Peer, addr string) error {
 }
 
 func recipientChecker(peer smtpd.Peer, addr string) error {
-	if *allowedRecipients == "" {
+	if allowedRecipients == nil {
+		// Any recipient is permitted
 		return nil
 	}
 
-	re, err := regexp.Compile(*allowedRecipients)
-	if err != nil {
-		log.WithFields(logrus.Fields{
-			"allowed_recipients": *allowedRecipients,
-		}).WithError(err).Warn("allowed_recipients pattern invalid")
-		return smtpd.Error{Code: 451, Message: "Bad recipient address"}
-	}
-
-	if re.MatchString(addr) {
+	if allowedRecipients.MatchString(addr) {
+		// Permitted by regex
 		return nil
 	}