implement gateway customization by allowing content-type allow and block lists, code cleanup, some refactoring

Author: patinthehat

README.md

@@ -125,6 +125,31 @@ tasks:
     platforms: ['linux', 'darwin'] # overrides the default
 ```
 
+#### Configuration: Settings: Gateway
+
+The `gateway` field of the `settings` section of the configuration file is used to customize the behavior of the http request gateway when accessing remote files or urls.  Both `blocked` and `allowed` content types can be specified as exact strings or wildcards, but both fields are optional.
+
+The default value for `blocked` is to not block any content types, and the default value for `allowed` is to allow all content types.
+
+```yaml
+settings:
+  gateway:
+    content-types:
+      blocked:
+        - audio/*
+        - font/*
+        - image/*
+        - multipart/*
+        - video/*
+      allowed:
+        - application/json
+        - application/vnd.api+json
+        - application/vnd.github.*
+        - application/xml
+        - application/yaml
+        - text/*
+```
+
 #### Configuration: Settings: Domains
 
 The `domains` section of the configuration file is used to specify a list of domain names that can be accessed when downloading remote files

go.mod

@@ -51,6 +51,7 @@ require (
 	github.com/posthog/posthog-go v0.0.0-20230801140217-d607812dee69
 	github.com/rs/zerolog v1.30.0
 	github.com/ryanuber/go-glob v1.0.0
+	github.com/stoewer/go-strcase v1.3.0
 	go.etcd.io/bbolt v1.3.7
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 )

go.sum

@@ -99,6 +99,8 @@ github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIH
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs=
+github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=

lib/app/workflow.go

@@ -45,6 +45,14 @@ type WorkflowSettings struct {
 	Cache                  *WorkflowSettingsCache    `yaml:"cache"`
 	Domains                *WorkflowSettingsDomains  `yaml:"domains"`
 	AnonymousStatistics    *bool                     `yaml:"anonymous-stats"`
+	Gateway                *WorkflowSettingsGateway  `yaml:"gateway"`
+}
+type GatewayContentTypes struct {
+	Blocked []string `yaml:"blocked"`
+	Allowed []string `yaml:"allowed"`
+}
+type WorkflowSettingsGateway struct {
+	ContentTypes *GatewayContentTypes `yaml:"content-types"`
 }
 
 type WorkflowSettingsDomains struct {
@@ -200,6 +208,20 @@ func (workflow *StackupWorkflow) configureDefaultSettings() {
 
 	App.Gateway.SetAllowedDomains(workflow.Settings.Domains.Allowed)
 
+	if workflow.Settings.Gateway == nil {
+		workflow.Settings.Gateway = &WorkflowSettingsGateway{
+			ContentTypes: &GatewayContentTypes{
+				Blocked: []string{},
+				Allowed: []string{},
+			},
+		}
+	}
+
+	if workflow.Settings.Gateway != nil && workflow.Settings.Gateway.ContentTypes != nil {
+		App.Gateway.SetDomainContentTypes("*", workflow.Settings.Gateway.ContentTypes.Allowed)
+		App.Gateway.SetBlockedContentTypes("*", workflow.Settings.Gateway.ContentTypes.Blocked)
+	}
+
 	if workflow.Settings.Cache.TtlMinutes <= 0 {
 		workflow.Settings.Cache.TtlMinutes = 5
 	}
@@ -249,6 +271,12 @@ func (workflow *StackupWorkflow) createMissingSettingsSection() {
 					Platforms: []string{"windows", "linux", "darwin"},
 				},
 			},
+			Gateway: &WorkflowSettingsGateway{
+				ContentTypes: &GatewayContentTypes{
+					Blocked: []string{},
+					Allowed: []string{"*"},
+				},
+			},
 		}
 	}
 }
@@ -390,6 +418,10 @@ func (workflow *StackupWorkflow) handleDataNotCached(found bool, data *cache.Cac
 			return nil
 		}
 
+		if err != nil {
+			return err
+		}
+
 		include.Hash = checksums.CalculateSha256Hash(include.Contents)
 		expires := carbon.Now().AddMinutes(App.Workflow.Settings.Cache.TtlMinutes)
 		now := carbon.Now()
@@ -451,8 +483,24 @@ func (workflow *StackupWorkflow) copySettingsFromIncludedTemplate(template *Incl
 		if template.Settings.Defaults != nil {
 			workflow.Settings.Defaults = template.Settings.Defaults
 		}
+		if workflow.Settings.Gateway != nil && workflow.Settings.Gateway.ContentTypes != nil {
+			for _, contentType := range workflow.Settings.Gateway.ContentTypes.Blocked {
+				workflow.Settings.Gateway.ContentTypes.Blocked = append(workflow.Settings.Gateway.ContentTypes.Blocked, contentType)
+			}
+			for _, contentType := range workflow.Settings.Gateway.ContentTypes.Allowed {
+				workflow.Settings.Gateway.ContentTypes.Allowed = append(workflow.Settings.Gateway.ContentTypes.Allowed, contentType)
+			}
+		}
+		if template.Settings.Gateway != nil {
+			workflow.Settings.Gateway = template.Settings.Gateway
+		}
 	}
 
 	workflow.Settings.Domains.Allowed = utils.GetUniqueStrings(workflow.Settings.Domains.Allowed)
 	App.Gateway.SetAllowedDomains(workflow.Settings.Domains.Allowed)
+
+	workflow.Settings.Gateway.ContentTypes.Allowed = utils.GetUniqueStrings(workflow.Settings.Gateway.ContentTypes.Allowed)
+	workflow.Settings.Gateway.ContentTypes.Blocked = utils.GetUniqueStrings(workflow.Settings.Gateway.ContentTypes.Blocked)
+	App.Gateway.SetDomainContentTypes("*", workflow.Settings.Gateway.ContentTypes.Allowed)
+	App.Gateway.SetBlockedContentTypes("*", workflow.Settings.Gateway.ContentTypes.Blocked)
 }

lib/app/workflowInclude.go

@@ -62,7 +62,7 @@ func (include *WorkflowInclude) Process() {
 	}
 
 	if err = include.Workflow.handleDataNotCached(include.FromCache, data, include); err != nil {
-		fmt.Println(err)
+		support.FailureMessageWithXMark("remote include (rejected: " + err.Error() + "): " + include.DisplayName())
 		return
 	}
 
@@ -71,7 +71,7 @@ func (include *WorkflowInclude) Process() {
 	}
 
 	if err = include.Workflow.loadRemoteFileInclude(include); err != nil {
-		fmt.Println(err)
+		support.FailureMessageWithXMark("remote include (rejected: " + err.Error() + "): " + include.DisplayName())
 		return
 	}
 

lib/gateway/gateway.go

@@ -17,22 +17,32 @@ type GatewayUrlRequestMiddleware struct {
 	Handler func(g *Gateway, link string) error
 }
 
+type GatewayUrlResponseMiddleware struct {
+	Name    string
+	Handler func(g *Gateway, resp *http.Response) error
+}
 type Gateway struct {
-	Enabled        bool
-	AllowedDomains []string
-	DeniedDomains  []string
-	Middleware     []*GatewayUrlRequestMiddleware
-	DomainHeaders  *sync.Map
+	Enabled             bool
+	AllowedDomains      []string
+	DeniedDomains       []string
+	Middleware          []*GatewayUrlRequestMiddleware
+	PostMiddleware      []*GatewayUrlResponseMiddleware
+	DomainHeaders       *sync.Map
+	DomainContentTypes  *sync.Map
+	BlockedContentTypes *sync.Map
 }
 
 // New initializes the gateway with deny/allow lists
 func New(deniedDomains, allowedDomains []string) *Gateway {
 	result := Gateway{
-		Enabled:        true,
-		DeniedDomains:  deniedDomains,
-		AllowedDomains: allowedDomains,
-		Middleware:     []*GatewayUrlRequestMiddleware{},
-		DomainHeaders:  &sync.Map{},
+		Enabled:             true,
+		DeniedDomains:       deniedDomains,
+		AllowedDomains:      allowedDomains,
+		Middleware:          []*GatewayUrlRequestMiddleware{},
+		PostMiddleware:      []*GatewayUrlResponseMiddleware{},
+		DomainHeaders:       &sync.Map{},
+		DomainContentTypes:  &sync.Map{},
+		BlockedContentTypes: &sync.Map{},
 	}
 
 	result.Initialize()
@@ -46,6 +56,7 @@ func (g *Gateway) Initialize() {
 
 	g.AddMiddleware(&ValidateUrlMiddleware)
 	g.AddMiddleware(&VerifyFileTypeMiddleware)
+	g.AddPostMiddleware(&VerifyContentType)
 
 	g.Enable()
 }
@@ -61,15 +72,64 @@ func (g *Gateway) SetDeniedDomains(domains []string) {
 }
 
 func (g *Gateway) SetDomainHeaders(domain string, headers []string) {
-	for _, header := range headers {
-		g.DomainHeaders.Store(domain, header)
-	}
+	g.DomainHeaders.Store(domain, headers)
+}
+
+func (g *Gateway) GetDomainHeaders(domain string) []string {
+	result := []string{}
+
+	g.DomainHeaders.Range(func(key, value any) bool {
+		if glob.Glob(key.(string), domain) {
+			result = append(result, value.([]string)...)
+		}
+		return true
+	})
+
+	return result
+}
+
+func (g *Gateway) SetBlockedContentTypes(domain string, contentTypes []string) {
+	g.BlockedContentTypes.Store(domain, contentTypes)
+}
+
+func (g *Gateway) GetBlockedContentTypes(domain string) []string {
+	result := []string{}
+
+	g.BlockedContentTypes.Range(func(key, value any) bool {
+		if glob.Glob(key.(string), domain) {
+			result = append(result, value.([]string)...)
+		}
+		return true
+	})
+
+	return result
+}
+
+func (g *Gateway) SetDomainContentTypes(domain string, contentTypes []string) {
+	g.DomainContentTypes.Store(domain, contentTypes)
+}
+
+func (g *Gateway) GetDomainContentTypes(domain string) []string {
+	result := []string{}
+
+	g.DomainContentTypes.Range(func(key, value any) bool {
+		if glob.Glob(key.(string), domain) {
+			result = append(result, value.([]string)...)
+		}
+		return true
+	})
+
+	return result
 }
 
 func (g *Gateway) AddMiddleware(mw *GatewayUrlRequestMiddleware) {
 	g.Middleware = append(g.Middleware, mw)
 }
 
+func (g *Gateway) AddPostMiddleware(mw *GatewayUrlResponseMiddleware) {
+	g.PostMiddleware = append(g.PostMiddleware, mw)
+}
+
 // The `runUrlRequestPipeline` function is a method of the `Gateway` struct. It iterates over the
 // `Middleware` slice of the `Gateway` struct and executes each middleware function in order. Each
 // middleware function takes a `Gateway` instance and a URL `link` as parameters and returns an error.
@@ -87,6 +147,16 @@ func (g *Gateway) runUrlRequestPipeline(link string) error {
 	return nil
 }
 
+func (g *Gateway) runResponsePipeline(resp *http.Response) error {
+	for _, mw := range g.PostMiddleware {
+		err := (*mw).Handler(g, resp)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (g *Gateway) Allowed(link string) bool {
 	return g.runUrlRequestPipeline(link) == nil
 }
@@ -142,14 +212,15 @@ func (g *Gateway) GetUrl(urlStr string, headers ...string) (string, error) {
 		return "", err
 	}
 
-	// remove the header items that are empty strings:
 	var tempHeaders []string = []string{"User-Agent: stackup/1.0"}
 
 	g.DomainHeaders.Range(func(key, value any) bool {
 		parsed, _ := url.Parse(urlStr)
 		if glob.Glob(key.(string), parsed.Hostname()) {
-			header := os.ExpandEnv(value.(string))
-			tempHeaders = append(tempHeaders, header)
+			for _, header := range value.([]string) {
+				header := os.ExpandEnv(header)
+				tempHeaders = append(tempHeaders, header)
+			}
 		}
 		return true
 	})
@@ -180,6 +251,11 @@ func (g *Gateway) GetUrl(urlStr string, headers ...string) (string, error) {
 	}
 	defer resp.Body.Close()
 
+	err = g.runResponsePipeline(resp)
+	if err != nil {
+		return "", err
+	}
+
 	if resp.StatusCode >= 400 {
 		return "", fmt.Errorf("HTTP error: %d", resp.StatusCode)
 	}

lib/gateway/middleware.go

@@ -2,6 +2,7 @@ package gateway
 
 import (
 	"errors"
+	"net/http"
 	"net/url"
 	"path"
 	"strings"
@@ -10,8 +11,41 @@ import (
 var (
 	ValidateUrlMiddleware    = GatewayUrlRequestMiddleware{Name: "validateUrl", Handler: validateUrlHandler}
 	VerifyFileTypeMiddleware = GatewayUrlRequestMiddleware{Name: "verifyFileType", Handler: verifyFileTypeHandler}
+	VerifyContentType        = GatewayUrlResponseMiddleware{Name: "verifyContentType", Handler: verifyContentTypeHandler}
 )
 
+func verifyContentTypeHandler(g *Gateway, resp *http.Response) error {
+	if !g.Enabled {
+		return nil
+	}
+
+	contentType, _, _ := strings.Cut(resp.Header.Get("Content-Type"), ";")
+
+	// allowedTypes := []string{
+	// 	"application/json", "application/javascript", "application/x-yaml", "application/x-yml", "application/yaml", "application/yml",
+	// 	"text/*",
+	// 	"application/octet-stream", "application/zip", "application/x-zip-compressed",
+	// 	"application/x-gzip", "application/gzip", "application/x-tar", "application/tar",
+	// 	"application/x-bzip2", "application/bzip2", "application/x-bzip", "application/bzip",
+	// 	"application/x-xz", "application/x-lzma", "application/lzma",
+	// 	"application/vnd.debian.binary-package", //.deb
+	// 	"application/pgp-signature",             // .sig
+	// }
+
+	blockedTypes := g.GetBlockedContentTypes(resp.Request.URL.Hostname())
+
+	if g.checkArrayForMatch(&blockedTypes, contentType) {
+		return errors.New("content type blocked")
+	}
+
+	allowedTypes := g.GetDomainContentTypes(resp.Request.URL.Hostname())
+	if g.checkArrayForMatch(&allowedTypes, contentType) || len(allowedTypes) == 0 {
+		return nil
+	}
+
+	return errors.New("content type '" + contentType + "' not allowed")
+}
+
 func validateUrlHandler(g *Gateway, link string) error {
 	if !g.Enabled {
 		return nil

lib/projectinfo/project.go

@@ -1,7 +1,6 @@
 package projectinfo
 
 import (
-	"fmt"
 	"os"
 	"path"
 	"path/filepath"
@@ -52,8 +51,6 @@ func (p *Project) Name() string {
 
 	baseDir, err := findProjectBaseDir(cwd)
 
-	fmt.Printf("baseDir: %s\n", baseDir)
-
 	if err != nil {
 		if absCwd, err := filepath.Abs(cwd); err == nil {
 			return p.cacheName(path.Base(absCwd))