Merge pull request #20 from halverneus/dev

Add optional check for HTTP 'Referer' header

Dockerfile

@@ -3,7 +3,7 @@
 ################################################################################
 FROM golang:1.11.3 as builder
 
-ENV VERSION 1.5.2
+ENV VERSION 1.6.0
 ENV BUILD_DIR /build
 
 RUN mkdir -p ${BUILD_DIR}

Dockerfile.all

@@ -1,6 +1,6 @@
 FROM golang:1.11.3 as builder
 
-ENV VERSION 1.5.2
+ENV VERSION 1.6.0
 ENV BUILD_DIR /build
 
 RUN mkdir -p ${BUILD_DIR}

README.md

@@ -35,6 +35,13 @@ URL_PREFIX=
 # served using HTTP.
 TLS_CERT=
 TLS_KEY=
+# List of accepted HTTP referrers. Return 403 if HTTP header `Referer` does not
+# match prefixes provided in the list.
+# Examples:
+#   'REFERRERS=http://localhost,https://...,https://another.name'
+#   To accept missing referrer header, add a blank entry (start comma):
+#   'REFERRERS=,http://localhost,https://another.name'
+REFERRERS=
 ```
 
 ### YAML Configuration File
@@ -53,6 +60,9 @@ folder: /web
 url-prefix: ""
 tls-cert: ""
 tls-key: ""
+referrers:
+  - http://localhost
+  - https://my.site
 ```
 
 ## Deployment

cli/help/help.go

@@ -45,6 +45,17 @@ ENVIRONMENT VARIABLES
         to a client without regard for the hostname.
     PORT
         The port used for binding. If not supplied, defaults to port '8080'.
+    REFERRERS
+        A comma-separated list of acceped Referrers based on the 'Referer' HTTP
+        header. If incoming header value is not in the list, a 403 HTTP error is
+        returned. To accept requests without a 'Referer' HTTP header in addition
+        to the whitelisted values, include an empty value (either with a leading
+        comma in the environment variable or with an empty list item in the YAML
+        configuration file) as demonstrated in the second example. If not
+        supplied the 'Referer' HTTP header is ignored.
+        Examples:
+          REFERRERS='http://localhost,https://some.site,http://other.site:8080'
+          REFERRERS=',http://localhost,https://some.site,http://other.site:8080'
     SHOW_LISTING
         Automatically serve the index file for the directory if requested. For
         example, if the client requests 'http://127.0.0.1/' the 'index.html'
@@ -77,12 +88,23 @@ CONFIGURATION FILE
     folder: /web
     host: ""
     port: 8080
+    referrers: []
     show-listing: true
     tls-cert: ""
     tls-key: ""
     url-prefix: ""
     ----------------------------------------------------------------------------
 
+    Example config.yml with possible alternative values:
+    ----------------------------------------------------------------------------
+    debug: true
+    folder: /var/www
+    port: 80
+    referrers:
+      - http://localhost
+      - https://mydomain.com
+    ----------------------------------------------------------------------------
+
 USAGE
     FILE LAYOUT
        /var/www/sub/my.file
@@ -99,7 +121,7 @@ USAGE
         export PORT=80
         static-file-server
             Retrieve with: wget http://my.machine/sub/my.file
-        
+
         export FOLDER=/var/www
         static-file-server -c config.yml
             Result: Runs with values from config.yml, but with the folder being

cli/server/server.go

@@ -34,11 +34,18 @@ func Run() error {
 // configuration.
 func handlerSelector() (handler http.HandlerFunc) {
 	var serveFileHandler handle.FileServerFunc
+
 	serveFileHandler = http.ServeFile
 	if config.Get.Debug {
 		serveFileHandler = handle.WithLogging(serveFileHandler)
 	}
 
+	if 0 != len(config.Get.Referrers) {
+		serveFileHandler = handle.WithReferrers(
+			serveFileHandler, config.Get.Referrers,
+		)
+	}
+
 	// Choose and set the appropriate, optimized static file serving function.
 	if 0 == len(config.Get.URLPrefix) {
 		handler = handle.Basic(serveFileHandler, config.Get.Folder)

cli/server/server_test.go

@@ -32,6 +32,8 @@ func TestHandlerSelector(t *testing.T) {
 	// This test only exercises function branches.
 	testFolder := "/web"
 	testPrefix := "/url/prefix"
+	var ignoreReferrer []string
+	testReferrer := []string{"http://localhost"}
 
 	testCases := []struct {
 		name    string
@@ -39,15 +41,24 @@ func TestHandlerSelector(t *testing.T) {
 		prefix  string
 		listing bool
 		debug   bool
+		refer   []string
 	}{
-		{"Basic handler w/o debug", testFolder, "", true, false},
-		{"Prefix handler w/o debug", testFolder, testPrefix, true, false},
-		{"Basic and hide listing handler w/o debug", testFolder, "", false, false},
-		{"Prefix and hide listing handler w/o debug", testFolder, testPrefix, false, false},
-		{"Basic handler w/debug", testFolder, "", true, true},
-		{"Prefix handler w/debug", testFolder, testPrefix, true, true},
-		{"Basic and hide listing handler w/debug", testFolder, "", false, true},
-		{"Prefix and hide listing handler w/debug", testFolder, testPrefix, false, true},
+		{"Basic handler w/o debug", testFolder, "", true, false, ignoreReferrer},
+		{"Prefix handler w/o debug", testFolder, testPrefix, true, false, ignoreReferrer},
+		{"Basic and hide listing handler w/o debug", testFolder, "", false, false, ignoreReferrer},
+		{"Prefix and hide listing handler w/o debug", testFolder, testPrefix, false, false, ignoreReferrer},
+		{"Basic handler w/debug", testFolder, "", true, true, ignoreReferrer},
+		{"Prefix handler w/debug", testFolder, testPrefix, true, true, ignoreReferrer},
+		{"Basic and hide listing handler w/debug", testFolder, "", false, true, ignoreReferrer},
+		{"Prefix and hide listing handler w/debug", testFolder, testPrefix, false, true, ignoreReferrer},
+		{"Basic handler w/o debug w/refer", testFolder, "", true, false, testReferrer},
+		{"Prefix handler w/o debug w/refer", testFolder, testPrefix, true, false, testReferrer},
+		{"Basic and hide listing handler w/o debug w/refer", testFolder, "", false, false, testReferrer},
+		{"Prefix and hide listing handler w/o debug w/refer", testFolder, testPrefix, false, false, testReferrer},
+		{"Basic handler w/debug w/refer", testFolder, "", true, true, testReferrer},
+		{"Prefix handler w/debug w/refer", testFolder, testPrefix, true, true, testReferrer},
+		{"Basic and hide listing handler w/debug w/refer", testFolder, "", false, true, testReferrer},
+		{"Prefix and hide listing handler w/debug w/refer", testFolder, testPrefix, false, true, testReferrer},
 	}
 
 	for _, tc := range testCases {
@@ -56,6 +67,7 @@ func TestHandlerSelector(t *testing.T) {
 			config.Get.Folder = tc.folder
 			config.Get.ShowListing = tc.listing
 			config.Get.URLPrefix = tc.prefix
+			config.Get.Referrers = tc.refer
 
 			handlerSelector()
 		})

config/config.go

@@ -14,14 +14,15 @@ import (
 var (
 	// Get the desired configuration value.
 	Get struct {
-		Debug       bool   `yaml:"debug"`
-		Folder      string `yaml:"folder"`
-		Host        string `yaml:"host"`
-		Port        uint16 `yaml:"port"`
-		ShowListing bool   `yaml:"show-listing"`
-		TLSCert     string `yaml:"tls-cert"`
-		TLSKey      string `yaml:"tls-key"`
-		URLPrefix   string `yaml:"url-prefix"`
+		Debug       bool     `yaml:"debug"`
+		Folder      string   `yaml:"folder"`
+		Host        string   `yaml:"host"`
+		Port        uint16   `yaml:"port"`
+		ShowListing bool     `yaml:"show-listing"`
+		TLSCert     string   `yaml:"tls-cert"`
+		TLSKey      string   `yaml:"tls-key"`
+		URLPrefix   string   `yaml:"url-prefix"`
+		Referrers   []string `yaml:"referrers"`
 	}
 )
 
@@ -30,17 +31,19 @@ const (
 	folderKey      = "FOLDER"
 	hostKey        = "HOST"
 	portKey        = "PORT"
+	referrersKey   = "REFERRERS"
 	showListingKey = "SHOW_LISTING"
 	tlsCertKey     = "TLS_CERT"
 	tlsKeyKey      = "TLS_KEY"
 	urlPrefixKey   = "URL_PREFIX"
 )
 
-const (
+var (
 	defaultDebug       = false
 	defaultFolder      = "/web"
 	defaultHost        = ""
 	defaultPort        = uint16(8080)
+	defaultReferrers   = []string{}
 	defaultShowListing = true
 	defaultTLSCert     = ""
 	defaultTLSKey      = ""
@@ -57,6 +60,7 @@ func setDefaults() {
 	Get.Folder = defaultFolder
 	Get.Host = defaultHost
 	Get.Port = defaultPort
+	Get.Referrers = defaultReferrers
 	Get.ShowListing = defaultShowListing
 	Get.TLSCert = defaultTLSCert
 	Get.TLSKey = defaultTLSKey
@@ -108,6 +112,7 @@ func overrideWithEnvVars() {
 	Get.TLSCert = envAsStr(tlsCertKey, Get.TLSCert)
 	Get.TLSKey = envAsStr(tlsKeyKey, Get.TLSKey)
 	Get.URLPrefix = envAsStr(urlPrefixKey, Get.URLPrefix)
+	Get.Referrers = envAsStrSlice(referrersKey, Get.Referrers)
 }
 
 // validate the configuration.
@@ -150,6 +155,15 @@ func envAsStr(key, fallback string) string {
 	return fallback
 }
 
+// envAsStrSlice returns the value of the environment variable as a slice of
+// strings if set.
+func envAsStrSlice(key string, fallback []string) []string {
+	if value := os.Getenv(key); "" != value {
+		return strings.Split(value, ",")
+	}
+	return fallback
+}
+
 // envAsUint16 returns the value of the environment variable as a uint16 if set.
 func envAsUint16(key string, fallback uint16) uint16 {
 	// Retrieve the string value of the environment variable. If not set,

config/config_test.go

@@ -7,7 +7,7 @@ import (
 	"strconv"
 	"testing"
 
-	"gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v2"
 )
 
 func TestLoad(t *testing.T) {
@@ -245,6 +245,98 @@ func TestEnvAsStr(t *testing.T) {
 	}
 }
 
+func TestEnvAsStrSlice(t *testing.T) {
+	oe := "ONE_ENTRY"
+	oewc := "ONE_ENTRY_WITH_COMMA"
+	oewtc := "ONE_ENTRY_WITH_TRAILING_COMMA"
+	te := "TWO_ENTRY"
+	tewc := "TWO_ENTRY_WITH_COMMA"
+	oc := "ONLY_COMMA"
+	ev := "EMPTY_VALUE"
+	uv := "UNSET_VALUE"
+
+	fs := "http://my.site"
+	ts := "http://other.site"
+	fbr := []string{"one", "two"}
+	var efbr []string
+
+	oes := fs
+	oer := []string{fs}
+	oewcs := "," + fs
+	oewcr := []string{"", fs}
+	oewtcs := fs + ","
+	oewtcr := []string{fs, ""}
+	tes := fs + "," + ts
+	ter := []string{fs, ts}
+	tewcs := "," + fs + "," + ts
+	tewcr := []string{"", fs, ts}
+	ocs := ","
+	ocr := []string{"", ""}
+	evs := ""
+
+	os.Setenv(oe, oes)
+	os.Setenv(oewc, oewcs)
+	os.Setenv(oewtc, oewtcs)
+	os.Setenv(te, tes)
+	os.Setenv(tewc, tewcs)
+	os.Setenv(oc, ocs)
+	os.Setenv(ev, evs)
+
+	testCases := []struct {
+		name     string
+		key      string
+		fallback []string
+		result   []string
+	}{
+		{"One entry", oe, fbr, oer},
+		{"One entry w/comma", oewc, fbr, oewcr},
+		{"One entry w/trailing comma", oewtc, fbr, oewtcr},
+		{"Two entry", te, fbr, ter},
+		{"Two entry w/comma", tewc, fbr, tewcr},
+		{"Only comma", oc, fbr, ocr},
+		{"Empty value w/fallback", ev, fbr, fbr},
+		{"Empty value wo/fallback", ev, efbr, efbr},
+		{"Unset w/fallback", uv, fbr, fbr},
+		{"Unset wo/fallback", uv, efbr, efbr},
+	}
+
+	matches := func(a, b []string) bool {
+		if len(a) != len(b) {
+			return false
+		}
+		tally := make(map[int]bool)
+		for i := range a {
+			tally[i] = false
+		}
+		for _, val := range a {
+			for i, other := range b {
+				if other == val && !tally[i] {
+					tally[i] = true
+					break
+				}
+			}
+		}
+		for _, found := range tally {
+			if !found {
+				return false
+			}
+		}
+		return true
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := envAsStrSlice(tc.key, tc.fallback)
+			if !matches(tc.result, result) {
+				t.Errorf(
+					"For %s with a '%v' fallback expected '%v' but got '%v'",
+					tc.key, tc.fallback, tc.result, result,
+				)
+			}
+		})
+	}
+}
+
 func TestEnvAsUint16(t *testing.T) {
 	ubv := "UPPER_BOUNDS_VALUE"
 	lbv := "LOWER_BOUNDS_VALUE"

handle/handle.go

@@ -1,6 +1,7 @@
 package handle
 
 import (
+	"fmt"
 	"log"
 	"net/http"
 	"strings"
@@ -26,12 +27,31 @@ type ListenerFunc func(string, http.HandlerFunc) error
 // requesting client.
 type FileServerFunc func(http.ResponseWriter, *http.Request, string)
 
+// WithReferrers returns a function that evaluates the HTTP 'Referer' header
+// value and returns HTTP error 403 if the value is not found in the whitelist.
+// If one of the whitelisted referrers are an empty string, then it is allowed
+// for the 'Referer' HTTP header key to not be set.
+func WithReferrers(serveFile FileServerFunc, referrers []string) FileServerFunc {
+	return func(w http.ResponseWriter, r *http.Request, name string) {
+		if !validReferrer(referrers, r.Referer()) {
+			http.Error(
+				w,
+				fmt.Sprintf("Invalid source '%s'", r.Referer()),
+				http.StatusForbidden,
+			)
+			return
+		}
+		serveFile(w, r, name)
+	}
+}
+
 // WithLogging returns a function that logs information about the request prior
 // to serving the requested file.
 func WithLogging(serveFile FileServerFunc) FileServerFunc {
 	return func(w http.ResponseWriter, r *http.Request, name string) {
 		log.Printf(
-			"REQ: %s %s %s%s -> %s\n",
+			"REQ from '%s': %s %s %s%s -> %s\n",
+			r.Referer(),
 			r.Method,
 			r.Proto,
 			r.Host,
@@ -90,3 +110,29 @@ func TLSListening(tlsCert, tlsKey string) ListenerFunc {
 		return listenAndServeTLS(binding, tlsCert, tlsKey, nil)
 	}
 }
+
+// validReferrer returns true if the passed referrer can be resolved by the
+// passed list of referrers.
+func validReferrer(s []string, e string) bool {
+	// Whitelisted referer list is empty. All requests are allowed.
+	if 0 == len(s) {
+		return true
+	}
+
+	for _, a := range s {
+		// Handle blank HTTP Referer header, if configured
+		if a == "" {
+			if e == "" {
+				return true
+			}
+			// Continue loop (all strings start with "")
+			continue
+		}
+
+		// Compare header with allowed prefixes
+		if strings.HasPrefix(e, a) {
+			return true
+		}
+	}
+	return false
+}

handle/handle_test.go

@@ -84,6 +84,73 @@ func teardown() (err error) {
 	return os.RemoveAll("tmp")
 }
 
+func TestWithReferrers(t *testing.T) {
+	forbidden := http.StatusForbidden
+
+	ok1 := "http://valid.com"
+	ok2 := "https://valid.com"
+	ok3 := "http://localhost"
+	bad := "http://other.pl"
+
+	var noRefer []string
+	emptyRefer := []string{}
+	onlyNoRefer := []string{""}
+	refer := []string{ok1, ok2, ok3}
+	noWithRefer := []string{"", ok1, ok2, ok3}
+
+	testCases := []struct {
+		name   string
+		refers []string
+		refer  string
+		code   int
+	}{
+		{"Nil refer list", noRefer, bad, ok},
+		{"Empty refer list", emptyRefer, bad, ok},
+		{"Unassigned allowed & unassigned", onlyNoRefer, "", ok},
+		{"Unassigned allowed & assigned", onlyNoRefer, bad, forbidden},
+		{"Whitelist with unassigned", refer, "", forbidden},
+		{"Whitelist with bad", refer, bad, forbidden},
+		{"Whitelist with ok1", refer, ok1, ok},
+		{"Whitelist with ok2", refer, ok2, ok},
+		{"Whitelist with ok3", refer, ok3, ok},
+		{"Whitelist and none with unassigned", noWithRefer, "", ok},
+		{"Whitelist with bad", noWithRefer, bad, forbidden},
+		{"Whitelist with ok1", noWithRefer, ok1, ok},
+		{"Whitelist with ok2", noWithRefer, ok2, ok},
+		{"Whitelist with ok3", noWithRefer, ok3, ok},
+	}
+
+	success := func(w http.ResponseWriter, r *http.Request, name string) {
+		defer r.Body.Close()
+		w.WriteHeader(ok)
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			handler := WithReferrers(success, tc.refers)
+
+			fullpath := "http://localhost/" + tmpIndexName
+			req := httptest.NewRequest("GET", fullpath, nil)
+			req.Header.Add("Referer", tc.refer)
+			w := httptest.NewRecorder()
+
+			handler(w, req, "")
+
+			resp := w.Result()
+			_, err := ioutil.ReadAll(resp.Body)
+			if nil != err {
+				t.Errorf("While reading body got %v", err)
+			}
+			if tc.code != resp.StatusCode {
+				t.Errorf(
+					"With referer '%s' in '%v' expected status code %d but got %d",
+					tc.refer, tc.refers, tc.code, resp.StatusCode,
+				)
+			}
+		})
+	}
+}
+
 func TestBasicWithAndWithoutLogging(t *testing.T) {
 	testCases := []struct {
 		name     string
@@ -333,3 +400,50 @@ func TestTLSListening(t *testing.T) {
 		)
 	}
 }
+
+func TestValidReferrer(t *testing.T) {
+	ok1 := "http://valid.com"
+	ok2 := "https://valid.com"
+	ok3 := "http://localhost"
+	bad := "http://other.pl"
+
+	var noRefer []string
+	emptyRefer := []string{}
+	onlyNoRefer := []string{""}
+	refer := []string{ok1, ok2, ok3}
+	noWithRefer := []string{"", ok1, ok2, ok3}
+
+	testCases := []struct {
+		name   string
+		refers []string
+		refer  string
+		result bool
+	}{
+		{"Nil refer list", noRefer, bad, true},
+		{"Empty refer list", emptyRefer, bad, true},
+		{"Unassigned allowed & unassigned", onlyNoRefer, "", true},
+		{"Unassigned allowed & assigned", onlyNoRefer, bad, false},
+		{"Whitelist with unassigned", refer, "", false},
+		{"Whitelist with bad", refer, bad, false},
+		{"Whitelist with ok1", refer, ok1, true},
+		{"Whitelist with ok2", refer, ok2, true},
+		{"Whitelist with ok3", refer, ok3, true},
+		{"Whitelist and none with unassigned", noWithRefer, "", true},
+		{"Whitelist with bad", noWithRefer, bad, false},
+		{"Whitelist with ok1", noWithRefer, ok1, true},
+		{"Whitelist with ok2", noWithRefer, ok2, true},
+		{"Whitelist with ok3", noWithRefer, ok3, true},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := validReferrer(tc.refers, tc.refer)
+			if result != tc.result {
+				t.Errorf(
+					"With referrers of '%v' and a value of '%s' expected %t but got %t",
+					tc.refers, tc.refer, tc.result, result,
+				)
+			}
+		})
+	}
+}