feat: Add plugin system to controller (#86)

* feat: Add plugin system to controller

* add priority system and default empty tls connection policy
This commit is contained in:
Marc-Antoine 2022-04-15 13:53:58 +02:00 committed by GitHub
parent 2410d8b325
commit cce8e52ddd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 773 additions and 373 deletions

View File

@ -2,10 +2,11 @@ package main
import (
"flag"
"github.com/caddyserver/ingress/internal/controller"
"github.com/caddyserver/ingress/pkg/store"
"strings"
)
func parseFlags() controller.Options {
func parseFlags() store.Options {
var namespace string
flag.StringVar(&namespace, "namespace", "", "the namespace that you would like to observe kubernetes ingress resources in.")
@ -18,12 +19,16 @@ func parseFlags() controller.Options {
var verbose bool
flag.BoolVar(&verbose, "v", false, "set the log level to debug")
var pluginsOrder string
flag.StringVar(&pluginsOrder, "plugins-order", "", "defines the order plugins should be used")
flag.Parse()
return controller.Options{
return store.Options{
WatchNamespace: namespace,
ConfigMapName: configMapName,
Verbose: verbose,
LeaseId: leaseId,
PluginsOrder: strings.Split(pluginsOrder, ","),
}
}

2
go.mod
View File

@ -9,7 +9,7 @@ require (
github.com/mitchellh/mapstructure v1.4.3
github.com/pires/go-proxyproto v0.3.1
github.com/pkg/errors v0.9.1
go.uber.org/zap v1.19.0
go.uber.org/zap v1.21.0
gopkg.in/go-playground/assert.v1 v1.2.1 // indirect
gopkg.in/go-playground/pool.v3 v3.1.1
k8s.io/api v0.19.4

7
go.sum
View File

@ -1038,8 +1038,9 @@ go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0=
go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
@ -1051,8 +1052,9 @@ go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc=
go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ=
go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE=
go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8=
go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw=
go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
gocloud.dev v0.19.0/go.mod h1:SmKwiR8YwIMMJvQBKLsC3fHNyMwXLw3PMDO+VVteJMI=
golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
@ -1105,7 +1107,6 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu
golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug=
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=

View File

@ -1,107 +1,26 @@
package caddy
import (
"encoding/json"
"github.com/caddyserver/caddy/v2"
"github.com/caddyserver/caddy/v2/caddyconfig"
"github.com/caddyserver/caddy/v2/modules/caddyhttp"
"github.com/caddyserver/caddy/v2/modules/caddytls"
"github.com/caddyserver/ingress/internal/controller"
"github.com/caddyserver/ingress/pkg/converter"
"github.com/caddyserver/ingress/pkg/store"
// Load default plugins
_ "github.com/caddyserver/ingress/internal/caddy/global"
_ "github.com/caddyserver/ingress/internal/caddy/ingress"
)
// StorageValues represents the config for certmagic storage providers.
type StorageValues struct {
Namespace string `json:"namespace"`
LeaseId string `json:"leaseId"`
}
// Storage represents the certmagic storage configuration.
type Storage struct {
System string `json:"module"`
StorageValues
}
// Config represents a caddy2 config file.
type Config struct {
Admin caddy.AdminConfig `json:"admin,omitempty"`
Storage Storage `json:"storage"`
Apps map[string]interface{} `json:"apps"`
Logging caddy.Logging `json:"logging"`
}
type Converter struct{}
const (
HttpServer = "ingress_server"
MetricsServer = "metrics_server"
)
func (c Converter) ConvertToCaddyConfig(store *store.Store) (interface{}, error) {
cfg := converter.NewConfig()
func metricsServer(enabled bool) *caddyhttp.Server {
handler := json.RawMessage(`{ "handler": "static_response" }`)
if enabled {
handler = json.RawMessage(`{ "handler": "metrics" }`)
for _, p := range converter.Plugins(store.Options.PluginsOrder) {
if m, ok := p.(converter.GlobalMiddleware); ok {
err := m.GlobalHandler(cfg, store)
if err != nil {
return cfg, err
}
}
}
return &caddyhttp.Server{
Listen: []string{":9765"},
AutoHTTPS: &caddyhttp.AutoHTTPSConfig{Disabled: true},
Routes: []caddyhttp.Route{{
HandlersRaw: []json.RawMessage{handler},
MatcherSetsRaw: []caddy.ModuleMap{{
"path": caddyconfig.JSON(caddyhttp.MatchPath{"/metrics"}, nil),
}},
}},
}
}
func newConfig(namespace string, store *controller.Store) (*Config, error) {
cfg := &Config{
Logging: caddy.Logging{},
Apps: map[string]interface{}{
"tls": &caddytls.TLS{
CertificatesRaw: caddy.ModuleMap{},
},
"http": &caddyhttp.App{
Servers: map[string]*caddyhttp.Server{
MetricsServer: metricsServer(store.ConfigMap.Metrics),
HttpServer: {
AutoHTTPS: &caddyhttp.AutoHTTPSConfig{},
// Listen to both :80 and :443 ports in order
// to use the same listener wrappers (PROXY protocol use it)
Listen: []string{":80", ":443"},
},
},
},
},
Storage: Storage{
System: "secret_store",
StorageValues: StorageValues{
Namespace: namespace,
LeaseId: store.Options.LeaseId,
},
},
}
return cfg, nil
}
func (c Converter) ConvertToCaddyConfig(namespace string, store *controller.Store) (interface{}, error) {
cfg, err := newConfig(namespace, store)
err = LoadIngressConfig(cfg, store)
if err != nil {
return cfg, err
}
err = LoadConfigMapOptions(cfg, store)
if err != nil {
return cfg, err
}
err = LoadTLSConfig(cfg, store)
if err != nil {
return cfg, err
}
return cfg, err
}

View File

@ -1,20 +1,32 @@
package caddy
package global
import (
"encoding/json"
caddy2 "github.com/caddyserver/caddy/v2"
"github.com/caddyserver/caddy/v2/caddyconfig"
"github.com/caddyserver/caddy/v2/modules/caddyhttp"
"github.com/caddyserver/caddy/v2/modules/caddytls"
"github.com/caddyserver/ingress/internal/controller"
"github.com/caddyserver/ingress/pkg/converter"
"github.com/caddyserver/ingress/pkg/store"
)
// LoadConfigMapOptions load options from ConfigMap
func LoadConfigMapOptions(config *Config, store *controller.Store) error {
type ConfigMapPlugin struct{}
func init() {
converter.RegisterPlugin(ConfigMapPlugin{})
}
func (p ConfigMapPlugin) IngressPlugin() converter.PluginInfo {
return converter.PluginInfo{
Name: "configmap",
New: func() converter.Plugin { return new(ConfigMapPlugin) },
}
}
func (p ConfigMapPlugin) GlobalHandler(config *converter.Config, store *store.Store) error {
cfgMap := store.ConfigMap
tlsApp := config.Apps["tls"].(*caddytls.TLS)
httpServer := config.Apps["http"].(*caddyhttp.App).Servers[HttpServer]
tlsApp := config.GetTLSApp()
httpServer := config.GetHTTPServer()
if cfgMap.Debug {
config.Logging.Logs = map[string]*caddy2.CustomLog{"default": {Level: "DEBUG"}}
@ -64,3 +76,8 @@ func LoadConfigMapOptions(config *Config, store *controller.Store) error {
}
return nil
}
// Interface guards
var (
_ = converter.GlobalMiddleware(ConfigMapPlugin{})
)

View File

@ -0,0 +1,69 @@
package global
import (
"encoding/json"
"github.com/caddyserver/caddy/v2"
"github.com/caddyserver/caddy/v2/modules/caddyhttp"
"github.com/caddyserver/ingress/pkg/converter"
"github.com/caddyserver/ingress/pkg/store"
)
type IngressPlugin struct{}
func (p IngressPlugin) IngressPlugin() converter.PluginInfo {
return converter.PluginInfo{
Name: "ingress",
New: func() converter.Plugin { return new(IngressPlugin) },
}
}
func init() {
converter.RegisterPlugin(IngressPlugin{})
}
func (p IngressPlugin) GlobalHandler(config *converter.Config, store *store.Store) error {
ingressHandlers := make([]converter.IngressMiddleware, 0)
for _, plugin := range converter.Plugins(store.Options.PluginsOrder) {
if m, ok := plugin.(converter.IngressMiddleware); ok {
ingressHandlers = append(ingressHandlers, m)
}
}
// create a server route for each ingress route
var routes caddyhttp.RouteList
for _, ing := range store.Ingresses {
for _, rule := range ing.Spec.Rules {
for _, path := range rule.HTTP.Paths {
r := &caddyhttp.Route{
HandlersRaw: []json.RawMessage{},
MatcherSetsRaw: []caddy.ModuleMap{},
}
for _, middleware := range ingressHandlers {
newRoute, err := middleware.IngressHandler(converter.IngressMiddlewareInput{
Config: config,
Store: store,
Ingress: ing,
Rule: rule,
Path: path,
Route: r,
})
if err != nil {
return err
}
r = newRoute
}
routes = append(routes, *r)
}
}
}
config.GetHTTPServer().Routes = routes
return nil
}
// Interface guards
var (
_ = converter.GlobalMiddleware(IngressPlugin{})
)

View File

@ -0,0 +1,47 @@
package global
import (
"encoding/json"
"github.com/caddyserver/caddy/v2"
"github.com/caddyserver/caddy/v2/caddyconfig"
"github.com/caddyserver/caddy/v2/modules/caddyhttp"
"github.com/caddyserver/ingress/pkg/converter"
"github.com/caddyserver/ingress/pkg/store"
)
type MetricsPlugin struct{}
func (p MetricsPlugin) IngressPlugin() converter.PluginInfo {
return converter.PluginInfo{
Name: "metrics",
New: func() converter.Plugin { return new(MetricsPlugin) },
}
}
func init() {
converter.RegisterPlugin(MetricsPlugin{})
}
func (p MetricsPlugin) GlobalHandler(config *converter.Config, store *store.Store) error {
httpApp := config.Apps["http"].(*caddyhttp.App)
if store.ConfigMap.Metrics {
httpApp.Servers[converter.MetricsServer] = &caddyhttp.Server{
Listen: []string{":9765"},
AutoHTTPS: &caddyhttp.AutoHTTPSConfig{Disabled: true},
Routes: []caddyhttp.Route{{
HandlersRaw: []json.RawMessage{json.RawMessage(`{ "handler": "metrics" }`)},
MatcherSetsRaw: []caddy.ModuleMap{{
"path": caddyconfig.JSON(caddyhttp.MatchPath{"/metrics"}, nil),
}},
}},
}
}
return nil
}
// Interface guards
var (
_ = converter.GlobalMiddleware(MetricsPlugin{})
)

View File

@ -0,0 +1,36 @@
package global
import (
"github.com/caddyserver/ingress/pkg/converter"
"github.com/caddyserver/ingress/pkg/store"
)
type SecretsStorePlugin struct{}
func (p SecretsStorePlugin) IngressPlugin() converter.PluginInfo {
return converter.PluginInfo{
Name: "secrets_store",
New: func() converter.Plugin { return new(SecretsStorePlugin) },
}
}
func init() {
converter.RegisterPlugin(SecretsStorePlugin{})
}
func (p SecretsStorePlugin) GlobalHandler(config *converter.Config, store *store.Store) error {
config.Storage = converter.Storage{
System: "secret_store",
StorageValues: converter.StorageValues{
Namespace: store.CurrentPod.Namespace,
LeaseId: store.Options.LeaseId,
},
}
return nil
}
// Interface guards
var (
_ = converter.GlobalMiddleware(SecretsStorePlugin{})
)

View File

@ -0,0 +1,49 @@
package global
import (
"encoding/json"
"github.com/caddyserver/ingress/internal/controller"
"github.com/caddyserver/ingress/pkg/converter"
"github.com/caddyserver/ingress/pkg/store"
)
type TLSPlugin struct{}
func (p TLSPlugin) IngressPlugin() converter.PluginInfo {
return converter.PluginInfo{
Name: "tls",
New: func() converter.Plugin { return new(TLSPlugin) },
}
}
func init() {
converter.RegisterPlugin(TLSPlugin{})
}
func (p TLSPlugin) GlobalHandler(config *converter.Config, store *store.Store) error {
tlsApp := config.GetTLSApp()
httpServer := config.GetHTTPServer()
var hosts []string
// Get all Hosts subject to custom TLS certs
for _, ing := range store.Ingresses {
for _, tlsRule := range ing.Spec.TLS {
for _, h := range tlsRule.Hosts {
hosts = append(hosts, h)
}
}
}
if len(hosts) > 0 {
tlsApp.CertificatesRaw["load_folders"] = json.RawMessage(`["` + controller.CertFolder + `"]`)
// do not manage certificates for those hosts
httpServer.AutoHTTPS.SkipCerts = hosts
}
return nil
}
// Interface guards
var (
_ = converter.GlobalMiddleware(TLSPlugin{})
)

View File

@ -1,106 +0,0 @@
package caddy
import (
"encoding/json"
"fmt"
"github.com/caddyserver/caddy/v2"
"github.com/caddyserver/caddy/v2/caddyconfig"
"github.com/caddyserver/caddy/v2/modules/caddyhttp"
"github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy"
"github.com/caddyserver/caddy/v2/modules/caddyhttp/rewrite"
"github.com/caddyserver/ingress/internal/controller"
"k8s.io/api/networking/v1"
)
const (
annotationPrefix = "caddy.ingress.kubernetes.io"
rewriteToAnnotation = "rewrite-to"
rewriteStripPrefixAnnotation = "rewrite-strip-prefix"
disableSSLRedirect = "disable-ssl-redirect"
)
func getAnnotation(ing *v1.Ingress, rule string) string {
return ing.Annotations[annotationPrefix+"/"+rule]
}
// TODO :- configure log middleware for all routes
func generateRoute(ing *v1.Ingress, rule v1.IngressRule, path v1.HTTPIngressPath) caddyhttp.Route {
var handlers []json.RawMessage
// Generate handlers
rewriteTo := getAnnotation(ing, rewriteToAnnotation)
if rewriteTo != "" {
handlers = append(handlers, caddyconfig.JSONModuleObject(
rewrite.Rewrite{URI: rewriteTo},
"handler", "rewrite", nil,
))
}
rewriteStripPrefix := getAnnotation(ing, rewriteStripPrefixAnnotation)
if rewriteStripPrefix != "" {
handlers = append(handlers, caddyconfig.JSONModuleObject(
rewrite.Rewrite{StripPathPrefix: rewriteStripPrefix},
"handler", "rewrite", nil,
))
}
clusterHostName := fmt.Sprintf("%v.%v.svc.cluster.local:%d", path.Backend.Service.Name, ing.Namespace, path.Backend.Service.Port.Number)
handlers = append(handlers, caddyconfig.JSONModuleObject(
reverseproxy.Handler{
Upstreams: reverseproxy.UpstreamPool{
{Dial: clusterHostName},
},
},
"handler", "reverse_proxy", nil,
))
// Generate matchers
match := caddy.ModuleMap{}
if getAnnotation(ing, disableSSLRedirect) != "true" {
match["protocol"] = caddyconfig.JSON(caddyhttp.MatchProtocol("https"), nil)
}
if rule.Host != "" {
match["host"] = caddyconfig.JSON(caddyhttp.MatchHost{rule.Host}, nil)
}
if path.Path != "" {
p := path.Path
if *path.PathType == v1.PathTypePrefix {
p += "*"
}
match["path"] = caddyconfig.JSON(caddyhttp.MatchPath{p}, nil)
}
return caddyhttp.Route{
HandlersRaw: handlers,
MatcherSetsRaw: []caddy.ModuleMap{match},
}
}
// LoadIngressConfig creates a routelist based off of ingresses managed by this controller.
func LoadIngressConfig(config *Config, store *controller.Store) error {
// TODO :-
// when setting the upstream url we should should bypass kube-dns and get the ip address of
// the pod for the deployment we are proxying to so that we can proxy to that ip address port.
// this is good for session affinity and increases performance.
// create a server route for each ingress route
var routes caddyhttp.RouteList
for _, ing := range store.Ingresses {
for _, rule := range ing.Spec.Rules {
for _, path := range rule.HTTP.Paths {
r := generateRoute(ing, rule, path)
routes = append(routes, r)
}
}
}
httpApp := config.Apps["http"].(*caddyhttp.App)
httpApp.Servers[HttpServer].Routes = routes
return nil
}

View File

@ -0,0 +1,14 @@
package ingress
import v1 "k8s.io/api/networking/v1"
const (
annotationPrefix = "caddy.ingress.kubernetes.io"
rewriteToAnnotation = "rewrite-to"
rewriteStripPrefixAnnotation = "rewrite-strip-prefix"
disableSSLRedirect = "disable-ssl-redirect"
)
func getAnnotation(ing *v1.Ingress, rule string) string {
return ing.Annotations[annotationPrefix+"/"+rule]
}

View File

@ -0,0 +1,52 @@
package ingress
import (
"github.com/caddyserver/caddy/v2"
"github.com/caddyserver/caddy/v2/caddyconfig"
"github.com/caddyserver/caddy/v2/modules/caddyhttp"
"github.com/caddyserver/ingress/pkg/converter"
v1 "k8s.io/api/networking/v1"
)
type MatcherPlugin struct{}
func (p MatcherPlugin) IngressPlugin() converter.PluginInfo {
return converter.PluginInfo{
Name: "ingress.matcher",
New: func() converter.Plugin { return new(MatcherPlugin) },
}
}
// IngressHandler Generate matchers for the route.
func (p MatcherPlugin) IngressHandler(input converter.IngressMiddlewareInput) (*caddyhttp.Route, error) {
match := caddy.ModuleMap{}
if getAnnotation(input.Ingress, disableSSLRedirect) != "true" {
match["protocol"] = caddyconfig.JSON(caddyhttp.MatchProtocol("https"), nil)
}
if input.Rule.Host != "" {
match["host"] = caddyconfig.JSON(caddyhttp.MatchHost{input.Rule.Host}, nil)
}
if input.Path.Path != "" {
p := input.Path.Path
if *input.Path.PathType == v1.PathTypePrefix {
p += "*"
}
match["path"] = caddyconfig.JSON(caddyhttp.MatchPath{p}, nil)
}
input.Route.MatcherSetsRaw = append(input.Route.MatcherSetsRaw, match)
return input.Route, nil
}
func init() {
converter.RegisterPlugin(MatcherPlugin{})
}
// Interface guards
var (
_ = converter.IngressMiddleware(MatcherPlugin{})
)

View File

@ -0,0 +1,55 @@
package ingress
import (
"fmt"
"github.com/caddyserver/caddy/v2/caddyconfig"
"github.com/caddyserver/caddy/v2/modules/caddyhttp"
"github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy"
"github.com/caddyserver/ingress/pkg/converter"
)
type ReverseProxyPlugin struct{}
func (p ReverseProxyPlugin) IngressPlugin() converter.PluginInfo {
return converter.PluginInfo{
Name: "ingress.reverseproxy",
// Should always go last by default
Priority: -10,
New: func() converter.Plugin { return new(ReverseProxyPlugin) },
}
}
// IngressHandler Add a reverse proxy handler to the route
func (p ReverseProxyPlugin) IngressHandler(input converter.IngressMiddlewareInput) (*caddyhttp.Route, error) {
path := input.Path
ing := input.Ingress
// TODO :-
// when setting the upstream url we should bypass kube-dns and get the ip address of
// the pod for the deployment we are proxying to so that we can proxy to that ip address port.
// this is good for session affinity and increases performance.
clusterHostName := fmt.Sprintf("%v.%v.svc.cluster.local:%d", path.Backend.Service.Name, ing.Namespace, path.Backend.Service.Port.Number)
handler := reverseproxy.Handler{
Upstreams: reverseproxy.UpstreamPool{
{Dial: clusterHostName},
},
}
handlerModule := caddyconfig.JSONModuleObject(
handler,
"handler",
"reverse_proxy",
nil,
)
input.Route.HandlersRaw = append(input.Route.HandlersRaw, handlerModule)
return input.Route, nil
}
func init() {
converter.RegisterPlugin(ReverseProxyPlugin{})
}
// Interface guards
var (
_ = converter.IngressMiddleware(ReverseProxyPlugin{})
)

View File

@ -0,0 +1,53 @@
package ingress
import (
"github.com/caddyserver/caddy/v2/caddyconfig"
"github.com/caddyserver/caddy/v2/modules/caddyhttp"
"github.com/caddyserver/caddy/v2/modules/caddyhttp/rewrite"
"github.com/caddyserver/ingress/pkg/converter"
)
type RewritePlugin struct{}
func (p RewritePlugin) IngressPlugin() converter.PluginInfo {
return converter.PluginInfo{
Name: "ingress.rewrite",
Priority: 10,
New: func() converter.Plugin { return new(RewritePlugin) },
}
}
// IngressHandler Converts rewrite annotations to rewrite handler
func (p RewritePlugin) IngressHandler(input converter.IngressMiddlewareInput) (*caddyhttp.Route, error) {
ing := input.Ingress
rewriteTo := getAnnotation(ing, rewriteToAnnotation)
if rewriteTo != "" {
handler := caddyconfig.JSONModuleObject(
rewrite.Rewrite{URI: rewriteTo},
"handler", "rewrite", nil,
)
input.Route.HandlersRaw = append(input.Route.HandlersRaw, handler)
}
rewriteStripPrefix := getAnnotation(ing, rewriteStripPrefixAnnotation)
if rewriteStripPrefix != "" {
handler := caddyconfig.JSONModuleObject(
rewrite.Rewrite{StripPathPrefix: rewriteStripPrefix},
"handler", "rewrite", nil,
)
input.Route.HandlersRaw = append(input.Route.HandlersRaw, handler)
}
return input.Route, nil
}
func init() {
converter.RegisterPlugin(RewritePlugin{})
}
// Interface guards
var (
_ = converter.IngressMiddleware(RewritePlugin{})
)

View File

@ -1,33 +0,0 @@
package caddy
import (
"encoding/json"
"github.com/caddyserver/caddy/v2/modules/caddyhttp"
"github.com/caddyserver/caddy/v2/modules/caddytls"
"github.com/caddyserver/ingress/internal/controller"
)
// LoadTLSConfig configure caddy when some ingresses have TLS certs
func LoadTLSConfig(config *Config, store *controller.Store) error {
tlsApp := config.Apps["tls"].(*caddytls.TLS)
httpApp := config.Apps["http"].(*caddyhttp.App)
var hosts []string
// Get all Hosts subject to custom TLS certs
for _, ing := range store.Ingresses {
for _, tlsRule := range ing.Spec.TLS {
for _, h := range tlsRule.Hosts {
hosts = append(hosts, h)
}
}
}
if len(hosts) > 0 {
tlsApp.CertificatesRaw["load_folders"] = json.RawMessage(`["` + controller.CertFolder + `"]`)
// do not manage certificates for those hosts
httpApp.Servers[HttpServer].AutoHTTPS.SkipCerts = hosts
}
return nil
}

View File

@ -1,7 +1,7 @@
package controller
import (
"github.com/caddyserver/ingress/internal/k8s"
"github.com/caddyserver/ingress/pkg/store"
v1 "k8s.io/api/core/v1"
)
@ -46,7 +46,7 @@ func (c *CaddyController) onConfigMapDeleted(obj *v1.ConfigMap) {
func (r ConfigMapAddedAction) handle(c *CaddyController) error {
c.logger.Infof("ConfigMap created (%s/%s)", r.resource.Namespace, r.resource.Name)
cfg, err := k8s.ParseConfigMap(r.resource)
cfg, err := store.ParseConfigMap(r.resource)
if err == nil {
c.resourceStore.ConfigMap = cfg
}
@ -56,7 +56,7 @@ func (r ConfigMapAddedAction) handle(c *CaddyController) error {
func (r ConfigMapUpdatedAction) handle(c *CaddyController) error {
c.logger.Infof("ConfigMap updated (%s/%s)", r.resource.Namespace, r.resource.Name)
cfg, err := k8s.ParseConfigMap(r.resource)
cfg, err := store.ParseConfigMap(r.resource)
if err == nil {
c.resourceStore.ConfigMap = cfg
}

View File

@ -28,7 +28,7 @@ func (r SyncStatusAction) handle(c *CaddyController) error {
// syncStatus ensures that the ingress source address points to this ingress controller's IP address.
func (c *CaddyController) syncStatus(ings []*v1.Ingress) error {
addrs, err := k8s.GetAddresses(c.podInfo, c.kubeClient)
addrs, err := k8s.GetAddresses(c.resourceStore.CurrentPod, c.kubeClient)
if err != nil {
return err
}

View File

@ -114,14 +114,7 @@ func (c *CaddyController) watchTLSSecrets() error {
}
for _, secret := range secrets {
content := make([]byte, 0)
for _, cert := range secret.Data {
content = append(content, cert...)
}
err := ioutil.WriteFile(filepath.Join(CertFolder, secret.Name+".pem"), content, 0644)
if err != nil {
if err := writeFile(secret); err != nil {
return err
}
}

View File

@ -8,9 +8,9 @@ import (
"github.com/caddyserver/certmagic"
"github.com/caddyserver/ingress/internal/k8s"
"github.com/caddyserver/ingress/pkg/storage"
"github.com/caddyserver/ingress/pkg/store"
"go.uber.org/zap"
apiv1 "k8s.io/api/core/v1"
"k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/informers"
@ -41,21 +41,6 @@ type Action interface {
handle(c *CaddyController) error
}
// Options represents ingress controller config received through cli arguments.
type Options struct {
WatchNamespace string
ConfigMapName string
Verbose bool
LeaseId string
}
// Store contains resources used to generate Caddy config
type Store struct {
Options *Options
ConfigMap *k8s.ConfigMapOptions
Ingresses []*v1.Ingress
}
// Informer defines the required SharedIndexInformers that interact with the API server.
type Informer struct {
Ingress cache.SharedIndexInformer
@ -73,12 +58,12 @@ type InformerFactory struct {
}
type Converter interface {
ConvertToCaddyConfig(namespace string, store *Store) (interface{}, error)
ConvertToCaddyConfig(store *store.Store) (interface{}, error)
}
// CaddyController represents an caddy ingress controller.
// CaddyController represents a caddy ingress controller.
type CaddyController struct {
resourceStore *Store
resourceStore *store.Store
kubeClient *kubernetes.Clientset
@ -93,9 +78,6 @@ type CaddyController struct {
// informer contains the cache Informers
informers *Informer
// ingress controller pod infos
podInfo *k8s.Info
// save last applied caddy config
lastAppliedConfig []byte
@ -107,7 +89,7 @@ type CaddyController struct {
func NewCaddyController(
logger *zap.SugaredLogger,
kubeClient *kubernetes.Clientset,
opts Options,
opts store.Options,
converter Converter,
stopChan chan struct{},
) *CaddyController {
@ -125,13 +107,12 @@ func NewCaddyController(
if err != nil {
logger.Fatalf("Unexpected error obtaining pod information: %v", err)
}
controller.podInfo = podInfo
// Create informer factories
controller.factories.PodNamespace = informers.NewSharedInformerFactoryWithOptions(
kubeClient,
resourcesSyncInterval,
informers.WithNamespace(controller.podInfo.Namespace),
informers.WithNamespace(podInfo.Namespace),
)
controller.factories.WatchedNamespace = informers.NewSharedInformerFactoryWithOptions(
kubeClient,
@ -168,7 +149,7 @@ func NewCaddyController(
caddy.RegisterModule(storage.SecretStorage{})
// Create resource store
controller.resourceStore = NewStore(opts)
controller.resourceStore = store.NewStore(opts, podInfo)
return controller
}
@ -260,13 +241,14 @@ func (c *CaddyController) processNextItem() bool {
}
// handleErrs reports errors received from queue actions.
//goland:noinspection GoUnusedParameter
func (c *CaddyController) handleErr(err error, action interface{}) {
c.logger.Error(err.Error())
}
// reloadCaddy generate a caddy config from controller's store
func (c *CaddyController) reloadCaddy() error {
config, err := c.converter.ConvertToCaddyConfig(c.podInfo.Namespace, c.resourceStore)
config, err := c.converter.ConvertToCaddyConfig(c.resourceStore)
if err != nil {
return err
}

View File

@ -1,34 +1,15 @@
package k8s
import (
"github.com/caddyserver/caddy/v2"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
v12 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/client-go/informers"
"k8s.io/client-go/tools/cache"
"reflect"
"time"
)
// ConfigMapOptions represents global options set through a configmap
type ConfigMapOptions struct {
Debug bool `json:"debug,omitempty"`
AcmeCA string `json:"acmeCA,omitempty"`
Email string `json:"email,omitempty"`
ProxyProtocol bool `json:"proxyProtocol,omitempty"`
Metrics bool `json:"metrics,omitempty"`
OnDemandTLS bool `json:"onDemandTLS,omitempty"`
OnDemandRateLimitInterval caddy.Duration `json:"onDemandTLSRateLimitInterval,omitempty"`
OnDemandRateLimitBurst int `json:"onDemandTLSRateLimitBurst,omitempty"`
OnDemandAsk string `json:"onDemandTLSAsk,omitempty"`
OCSPCheckInterval caddy.Duration `json:"ocspCheckInterval,omitempty"`
}
type ConfigMapHandlers struct {
AddFunc func(obj *v12.ConfigMap)
UpdateFunc func(oldObj, newObj *v12.ConfigMap)
DeleteFunc func(obj *v12.ConfigMap)
AddFunc func(obj *v1.ConfigMap)
UpdateFunc func(oldObj, newObj *v1.ConfigMap)
DeleteFunc func(obj *v1.ConfigMap)
}
type ConfigMapParams struct {
@ -37,7 +18,7 @@ type ConfigMapParams struct {
ConfigMapName string
}
func isControllerConfigMap(cm *v12.ConfigMap, name string) bool {
func isControllerConfigMap(cm *v1.ConfigMap, name string) bool {
return cm.GetName() == name
}
@ -46,22 +27,22 @@ func WatchConfigMaps(options ConfigMapParams, funcs ConfigMapHandlers) cache.Sha
informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
cm, ok := obj.(*v12.ConfigMap)
cm, ok := obj.(*v1.ConfigMap)
if ok && isControllerConfigMap(cm, options.ConfigMapName) {
funcs.AddFunc(cm)
}
},
UpdateFunc: func(oldObj, newObj interface{}) {
oldCM, ok1 := oldObj.(*v12.ConfigMap)
newCM, ok2 := newObj.(*v12.ConfigMap)
oldCM, ok1 := oldObj.(*v1.ConfigMap)
newCM, ok2 := newObj.(*v1.ConfigMap)
if ok1 && ok2 && isControllerConfigMap(newCM, options.ConfigMapName) {
funcs.UpdateFunc(oldCM, newCM)
}
},
DeleteFunc: func(obj interface{}) {
cm, ok := obj.(*v12.ConfigMap)
cm, ok := obj.(*v1.ConfigMap)
if ok && isControllerConfigMap(cm, options.ConfigMapName) {
funcs.DeleteFunc(cm)
@ -71,40 +52,3 @@ func WatchConfigMaps(options ConfigMapParams, funcs ConfigMapHandlers) cache.Sha
return informer
}
func stringToCaddyDurationHookFunc() mapstructure.DecodeHookFunc {
return func(f reflect.Type, t reflect.Type, data interface{}) (interface{}, error) {
if f.Kind() != reflect.String {
return data, nil
}
if t != reflect.TypeOf(caddy.Duration(time.Second)) {
return data, nil
}
return caddy.ParseDuration(data.(string))
}
}
func ParseConfigMap(cm *v12.ConfigMap) (*ConfigMapOptions, error) {
// parse configmap
cfgMap := ConfigMapOptions{}
config := &mapstructure.DecoderConfig{
Metadata: nil,
WeaklyTypedInput: true,
Result: &cfgMap,
TagName: "json",
DecodeHook: mapstructure.ComposeDecodeHookFunc(
stringToCaddyDurationHookFunc(),
),
}
decoder, err := mapstructure.NewDecoder(config)
if err != nil {
return nil, errors.Wrap(err, "unexpected error creating decoder")
}
err = decoder.Decode(cm.Data)
if err != nil {
return nil, errors.Wrap(err, "unexpected error parsing configmap")
}
return &cfgMap, nil
}

View File

@ -3,6 +3,7 @@ package k8s
import (
"context"
"fmt"
"github.com/caddyserver/ingress/pkg/store"
"os"
apiv1 "k8s.io/api/core/v1"
@ -11,18 +12,9 @@ import (
"k8s.io/client-go/kubernetes"
)
// Info contains runtime information about the pod running the Ingress controller
type Info struct {
Name string
Namespace string
// Labels selectors of the running pod
// This is used to search for other Ingress controller pods
Labels map[string]string
}
// GetAddresses gets the ip address or name of the node in the cluster that the
// ingress controller is running on.
func GetAddresses(p *Info, kubeClient *kubernetes.Clientset) ([]string, error) {
func GetAddresses(p *store.PodInfo, kubeClient *kubernetes.Clientset) ([]string, error) {
var addrs []string
// Get services that may select this pod
@ -67,7 +59,7 @@ func GetAddressFromService(service *apiv1.Service) string {
// GetPodDetails returns runtime information about the pod:
// name, namespace and IP of the node where it is running
func GetPodDetails(kubeClient *kubernetes.Clientset) (*Info, error) {
func GetPodDetails(kubeClient *kubernetes.Clientset) (*store.PodInfo, error) {
podName := os.Getenv("POD_NAME")
podNs := os.Getenv("POD_NAMESPACE")
@ -80,7 +72,7 @@ func GetPodDetails(kubeClient *kubernetes.Clientset) (*Info, error) {
return nil, fmt.Errorf("unable to get POD information")
}
return &Info{
return &store.PodInfo{
Name: podName,
Namespace: podNs,
Labels: pod.GetLabels(),

57
pkg/converter/config.go Normal file
View File

@ -0,0 +1,57 @@
package converter
import (
"github.com/caddyserver/caddy/v2"
"github.com/caddyserver/caddy/v2/modules/caddyhttp"
"github.com/caddyserver/caddy/v2/modules/caddytls"
)
// StorageValues represents the config for certmagic storage providers.
type StorageValues struct {
Namespace string `json:"namespace"`
LeaseId string `json:"leaseId"`
}
// Storage represents the certmagic storage configuration.
type Storage struct {
System string `json:"module"`
StorageValues
}
// Config represents a caddy2 config file.
type Config struct {
Admin caddy.AdminConfig `json:"admin,omitempty"`
Storage Storage `json:"storage"`
Apps map[string]interface{} `json:"apps"`
Logging caddy.Logging `json:"logging"`
}
func (c Config) GetHTTPServer() *caddyhttp.Server {
return c.Apps["http"].(*caddyhttp.App).Servers[HttpServer]
}
func (c Config) GetTLSApp() *caddytls.TLS {
return c.Apps["tls"].(*caddytls.TLS)
}
func NewConfig() *Config {
return &Config{
Logging: caddy.Logging{},
Apps: map[string]interface{}{
"tls": &caddytls.TLS{CertificatesRaw: caddy.ModuleMap{}},
"http": &caddyhttp.App{
Servers: map[string]*caddyhttp.Server{
HttpServer: {
AutoHTTPS: &caddyhttp.AutoHTTPSConfig{},
// Listen to both :80 and :443 ports in order
// to use the same listener wrappers (PROXY protocol use it)
Listen: []string{":80", ":443"},
TLSConnPolicies: caddytls.ConnectionPolicies{
&caddytls.ConnectionPolicy{},
},
},
},
},
},
}
}

112
pkg/converter/converter.go Normal file
View File

@ -0,0 +1,112 @@
package converter
import (
"fmt"
"github.com/caddyserver/caddy/v2/modules/caddyhttp"
"github.com/caddyserver/ingress/pkg/store"
v1 "k8s.io/api/networking/v1"
"sort"
)
const (
HttpServer = "ingress_server"
MetricsServer = "metrics_server"
)
// GlobalMiddleware is called with a default caddy config
// already configured with:
// - Secret storage store
// - A TLS App (https://caddyserver.com/docs/json/apps/tls/)
// - A HTTP App with an HTTP server listening to 80 443 ports (https://caddyserver.com/docs/json/apps/http/)
type GlobalMiddleware interface {
GlobalHandler(config *Config, store *store.Store) error
}
type IngressMiddlewareInput struct {
Config *Config
Store *store.Store
Ingress *v1.Ingress
Rule v1.IngressRule
Path v1.HTTPIngressPath
Route *caddyhttp.Route
}
// IngressMiddleware is called for each Caddy route that is generated for a specific
// ingress. It allows anyone to manipulate caddy routes before sending it to caddy.
type IngressMiddleware interface {
IngressHandler(input IngressMiddlewareInput) (*caddyhttp.Route, error)
}
type Plugin interface {
IngressPlugin() PluginInfo
}
type PluginInfo struct {
Name string
Priority int
New func() Plugin
}
func RegisterPlugin(m Plugin) {
plugin := m.IngressPlugin()
if _, ok := plugins[plugin.Name]; ok {
panic(fmt.Sprintf("plugin already registered: %s", plugin.Name))
}
plugins[plugin.Name] = plugin
pluginInstances[plugin.Name] = plugin.New()
}
func getOrderIndex(order []string, plugin string) int {
for idx, o := range order {
if plugin == o {
return idx
}
}
return -1
}
func sortPlugins(plugins []PluginInfo, order []string) []PluginInfo {
sort.SliceStable(plugins, func(i, j int) bool {
iPlugin, jPlugin := plugins[i], plugins[j]
iSortedIdx := getOrderIndex(order, iPlugin.Name)
jSortedIdx := getOrderIndex(order, jPlugin.Name)
if iSortedIdx != jSortedIdx {
return iSortedIdx > jSortedIdx
}
if iPlugin.Priority != jPlugin.Priority {
return iPlugin.Priority > jPlugin.Priority
}
return iPlugin.Name < jPlugin.Name
})
return plugins
}
// Plugins return a sorted array of plugin instances.
// Sort is made following these rules:
// - Plugins specified in the order slice will always go first (in the order specified in the slice)
// - A Plugin with higher priority will go before a plugin with lower priority
// - If 2 plugins have the same priority (and not in order slice), they will be sorted by plugin name
func Plugins(order []string) []Plugin {
sortedPlugins := make([]PluginInfo, 0, len(plugins))
for _, p := range plugins {
sortedPlugins = append(sortedPlugins, p)
}
sortPlugins(sortedPlugins, order)
pluginArr := make([]Plugin, 0, len(plugins))
for _, p := range sortedPlugins {
pluginArr = append(pluginArr, pluginInstances[p.Name])
}
return pluginArr
}
var (
plugins = make(map[string]PluginInfo)
pluginInstances = make(map[string]Plugin)
)

View File

@ -0,0 +1,53 @@
package converter
import "testing"
func TestSortPlugins(t *testing.T) {
tests := []struct {
name string
order []string
plugins []PluginInfo
expect []string
}{
{
name: "default to alpha sort",
order: nil,
plugins: []PluginInfo{{Name: "b"}, {Name: "c"}, {Name: "a"}},
expect: []string{"a", "b", "c"},
},
{
name: "use priority when specified",
order: nil,
plugins: []PluginInfo{{Name: "b"}, {Name: "a", Priority: 20}, {Name: "c", Priority: 10}},
expect: []string{"a", "c", "b"},
},
{
name: "fallback to alpha when no priority",
order: nil,
plugins: []PluginInfo{{Name: "b"}, {Name: "a"}, {Name: "c", Priority: 20}},
expect: []string{"c", "a", "b"},
},
{
name: "specify order",
order: []string{"c"},
plugins: []PluginInfo{{Name: "b"}, {Name: "a"}, {Name: "c"}},
expect: []string{"c", "a", "b"},
},
{
name: "order overrides other settings",
order: []string{"c"},
plugins: []PluginInfo{{Name: "b", Priority: 10}, {Name: "a"}, {Name: "c"}},
expect: []string{"c", "b", "a"},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
sortPlugins(test.plugins, test.order)
for i, plugin := range test.plugins {
if test.expect[i] != plugin.Name {
t.Errorf("expected order to match %v: got %v, expected %v", test.expect, plugin.Name, test.expect[i])
}
}
})
}
}

View File

@ -17,4 +17,4 @@ func (Wrapper) CaddyModule() caddy.ModuleInfo {
ID: "caddy.listeners.proxy_protocol",
New: func() caddy.Module { return new(Wrapper) },
}
}
}

View File

@ -0,0 +1,61 @@
package store
import (
"github.com/caddyserver/caddy/v2"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
apiv1 "k8s.io/api/core/v1"
"reflect"
"time"
)
// ConfigMapOptions represents global options set through a configmap
type ConfigMapOptions struct {
Debug bool `json:"debug,omitempty"`
AcmeCA string `json:"acmeCA,omitempty"`
Email string `json:"email,omitempty"`
ProxyProtocol bool `json:"proxyProtocol,omitempty"`
Metrics bool `json:"metrics,omitempty"`
OnDemandTLS bool `json:"onDemandTLS,omitempty"`
OnDemandRateLimitInterval caddy.Duration `json:"onDemandTLSRateLimitInterval,omitempty"`
OnDemandRateLimitBurst int `json:"onDemandTLSRateLimitBurst,omitempty"`
OnDemandAsk string `json:"onDemandTLSAsk,omitempty"`
OCSPCheckInterval caddy.Duration `json:"ocspCheckInterval,omitempty"`
}
func stringToCaddyDurationHookFunc() mapstructure.DecodeHookFunc {
return func(f reflect.Type, t reflect.Type, data interface{}) (interface{}, error) {
if f.Kind() != reflect.String {
return data, nil
}
if t != reflect.TypeOf(caddy.Duration(time.Second)) {
return data, nil
}
return caddy.ParseDuration(data.(string))
}
}
func ParseConfigMap(cm *apiv1.ConfigMap) (*ConfigMapOptions, error) {
// parse configmap
cfgMap := ConfigMapOptions{}
config := &mapstructure.DecoderConfig{
Metadata: nil,
WeaklyTypedInput: true,
Result: &cfgMap,
TagName: "json",
DecodeHook: mapstructure.ComposeDecodeHookFunc(
stringToCaddyDurationHookFunc(),
),
}
decoder, err := mapstructure.NewDecoder(config)
if err != nil {
return nil, errors.Wrap(err, "unexpected error creating decoder")
}
err = decoder.Decode(cm.Data)
if err != nil {
return nil, errors.Wrap(err, "unexpected error parsing configmap")
}
return &cfgMap, nil
}

10
pkg/store/options.go Normal file
View File

@ -0,0 +1,10 @@
package store
// Options represents ingress controller config received through cli arguments.
type Options struct {
WatchNamespace string
ConfigMapName string
Verbose bool
LeaseId string
PluginsOrder []string
}

10
pkg/store/pod.go Normal file
View File

@ -0,0 +1,10 @@
package store
// PodInfo contains runtime information about the pod running the Ingress controller
type PodInfo struct {
Name string
Namespace string
// Labels selectors of the running pod
// This is used to search for other Ingress controller pods
Labels map[string]string
}

View File

@ -1,16 +1,24 @@
package controller
package store
import (
"github.com/caddyserver/ingress/internal/k8s"
"k8s.io/api/networking/v1"
v1 "k8s.io/api/networking/v1"
)
// Store contains resources used to generate Caddy config
type Store struct {
Options *Options
ConfigMap *ConfigMapOptions
Ingresses []*v1.Ingress
CurrentPod *PodInfo
}
// NewStore returns a new store that keeps track of K8S resources needed by the controller.
func NewStore(opts Options) *Store {
func NewStore(opts Options, podInfo *PodInfo) *Store {
s := &Store{
Options: &opts,
Ingresses: []*v1.Ingress{},
ConfigMap: &k8s.ConfigMapOptions{},
Options: &opts,
Ingresses: []*v1.Ingress{},
ConfigMap: &ConfigMapOptions{},
CurrentPod: podInfo,
}
return s
}