From abe4a47df64a453c360225f4ef83605f4fc57892 Mon Sep 17 00:00:00 2001 From: dev Date: Wed, 19 Jun 2019 14:42:46 -0400 Subject: [PATCH] allow caddy to be configured by config map --- internal/caddy/convert.go | 11 ++-- internal/controller/action.go | 27 ++++---- internal/controller/controller.go | 16 ++++- internal/store/store.go | 19 +++++- kubernetes/generated/configmap.yaml | 96 ++++++++++++++++++++++++++++ kubernetes/generated/deployment.yaml | 14 +++- 6 files changed, 155 insertions(+), 28 deletions(-) create mode 100644 kubernetes/generated/configmap.yaml diff --git a/internal/caddy/convert.go b/internal/caddy/convert.go index 7def613..8b57272 100644 --- a/internal/caddy/convert.go +++ b/internal/caddy/convert.go @@ -9,7 +9,9 @@ import ( ) // ConvertToCaddyConfig returns a new caddy routelist based off of ingresses managed by this controller. -func ConvertToCaddyConfig(ings []*v1beta1.Ingress) (caddyhttp.RouteList, []string, error) { +// This is not used when this ingress controller is configured with a config map, so that we don't +// override user defined routes. +func ConvertToCaddyConfig(ings []*v1beta1.Ingress) (caddyhttp.RouteList, error) { // ~~~~ // TODO :- // when setting the upstream url we should should bypass kube-dns and get the ip address of @@ -17,15 +19,10 @@ func ConvertToCaddyConfig(ings []*v1beta1.Ingress) (caddyhttp.RouteList, []strin // this is good for session affinity and increases performance. // ~~~~ - // record hosts for tls policies - var hosts []string - // create a server route for each ingress route var routes caddyhttp.RouteList for _, ing := range ings { for _, rule := range ing.Spec.Rules { - hosts = append(hosts, rule.Host) - for _, path := range rule.HTTP.Paths { clusterHostName := fmt.Sprintf("%v.%v.svc.cluster.local", path.Backend.ServiceName, ing.Namespace) r := baseRoute(clusterHostName) @@ -46,7 +43,7 @@ func ConvertToCaddyConfig(ings []*v1beta1.Ingress) (caddyhttp.RouteList, []strin } } - return routes, hosts, nil + return routes, nil } func baseRoute(upstream string) caddyhttp.ServerRoute { diff --git a/internal/controller/action.go b/internal/controller/action.go index 19e69af..e97e358 100644 --- a/internal/controller/action.go +++ b/internal/controller/action.go @@ -148,21 +148,22 @@ func (r ResourceDeletedAction) handle(c *CaddyController) error { func updateConfig(c *CaddyController) error { // update internal caddy config with new ingress info - // serverRoutes, hosts, err := caddy.ConvertToCaddyConfig(c.resourceStore.Ingresses) - serverRoutes, _, err := caddy.ConvertToCaddyConfig(c.resourceStore.Ingresses) - if err != nil { - return errors.Wrap(err, "converting ingress resources to caddy config") - } + if !c.usingConfigMap { + serverRoutes, err := caddy.ConvertToCaddyConfig(c.resourceStore.Ingresses) + if err != nil { + return errors.Wrap(err, "converting ingress resources to caddy config") + } - // set the http server routes - if c.resourceStore.CaddyConfig != nil { - c.resourceStore.CaddyConfig.Apps["http"].(caddyhttp.App).Servers["ingress_server"].Routes = serverRoutes - } + // set the http server routes + if c.resourceStore.CaddyConfig != nil { + c.resourceStore.CaddyConfig.Apps["http"].(caddyhttp.App).Servers["ingress_server"].Routes = serverRoutes + } - // reload caddy2 config with newConfig - err = c.reloadCaddy() - if err != nil { - return errors.Wrap(err, "caddy config reload") + // reload caddy2 config with newConfig + err = c.reloadCaddy() + if err != nil { + return errors.Wrap(err, "caddy config reload") + } } return nil diff --git a/internal/controller/controller.go b/internal/controller/controller.go index d577858..e93d43f 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io" + "io/ioutil" "log" "os" "time" @@ -71,16 +72,25 @@ func NewCaddyController(kubeClient *kubernetes.Clientset, restClient rest.Interf controller.podInfo = podInfo // load caddy config from file if mounted with config map + var caddyCfgMap caddy.Config cfgPath := "/etc/caddy/config.json" if _, err := os.Stat(cfgPath); !os.IsNotExist(err) { + controller.usingConfigMap = true + file, err := os.Open(cfgPath) if err != nil { log.Fatal(err) } defer file.Close() - controller.usingConfigMap = true - controller.syncQueue.Add(LoadConfigAction{config: file}) + b, err := ioutil.ReadAll(file) + if err != nil { + log.Fatal(err) + } + + // load config file into caddy + controller.syncQueue.Add(LoadConfigAction{config: bytes.NewReader(b)}) + json.Unmarshal(b, &caddyCfgMap) } // setup the ingress controller and start watching resources @@ -94,7 +104,7 @@ func NewCaddyController(kubeClient *kubernetes.Clientset, restClient rest.Interf controller.informer = informer // setup store to keep track of resources - controller.resourceStore = store.NewStore(controller.kubeClient, podInfo.Namespace, cfg) + controller.resourceStore = store.NewStore(controller.kubeClient, podInfo.Namespace, cfg, &caddyCfgMap) // attempt to do initial sync of status addresses with ingresses controller.dispatchSync() diff --git a/internal/store/store.go b/internal/store/store.go index fcaeb48..3b5737b 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -17,7 +17,7 @@ type Store struct { // NewStore returns a new store that keeps track of ingresses and secrets. It will attempt to get // all current ingresses before returning. -func NewStore(kubeClient *kubernetes.Clientset, namespace string, cfg caddy.ControllerConfig) *Store { +func NewStore(kubeClient *kubernetes.Clientset, namespace string, cfg caddy.ControllerConfig, cfgMapConfig *caddy.Config) *Store { ingresses, err := kubeClient.ExtensionsV1beta1().Ingresses("").List(v1.ListOptions{}) if err != nil { logrus.Errorf("could not get existing ingresses in cluster") @@ -25,8 +25,21 @@ func NewStore(kubeClient *kubernetes.Clientset, namespace string, cfg caddy.Cont } s := &Store{ - Ingresses: []*v1beta1.Ingress{}, - CaddyConfig: caddy.NewConfig(namespace, cfg), + Ingresses: []*v1beta1.Ingress{}, + } + + if cfgMapConfig == nil { + s.CaddyConfig = caddy.NewConfig(namespace, cfg) + } else { + // set cert-magic storage provider + cfgMapConfig.Storage = caddy.Storage{ + System: "secret_store", + StorageValues: caddy.StorageValues{ + Namespace: namespace, + }, + } + + s.CaddyConfig = cfgMapConfig } for _, i := range ingresses.Items { diff --git a/kubernetes/generated/configmap.yaml b/kubernetes/generated/configmap.yaml new file mode 100644 index 0000000..6ca4eda --- /dev/null +++ b/kubernetes/generated/configmap.yaml @@ -0,0 +1,96 @@ +# this is an example config map for the caddy ingress controller +apiVersion: v1 +kind: ConfigMap +metadata: + name: caddy-config + namespace: caddy-system +data: + config.json: ' + { + "storage": { + "system": "secret_store", + "namespace": "caddy-system" + }, + "apps": { + "http": { + "servers": { + "ingress_server": { + "listen": [ + ":80", + ":443" + ], + "routes": [ + { + "match": [ + { + "host": [ + "danny2.kubed.co" + ], + "path": [ + "/hello2" + ] + } + ], + "apply": [ + { + "middleware": "log", + "filename": "/etc/caddy/access.log" + } + ], + "respond": { + "responder": "reverse_proxy", + "load_balance_type": "random", + "upstreams": [ + { + "host": "http://example2.default.svc.cluster.local" + } + ] + } + }, + { + "match": [ + { + "host": [ + "danny2.kubed.co" + ], + "path": [ + "" + ] + } + ], + "apply": [ + { + "middleware": "log", + "filename": "/etc/caddy/access.log" + } + ], + "respond": { + "responder": "reverse_proxy", + "load_balance_type": "random", + "upstreams": [ + { + "host": "http://example.default.svc.cluster.local" + } + ] + } + } + ] + } + } + }, + "tls": { + "automation": { + "policies": [ + { + "management": { + "module": "acme", + "email": "test@test.com" + } + } + ] + }, + "session_tickets": {} + } + } + } + ' \ No newline at end of file diff --git a/kubernetes/generated/deployment.yaml b/kubernetes/generated/deployment.yaml index d6c175d..59b0727 100755 --- a/kubernetes/generated/deployment.yaml +++ b/kubernetes/generated/deployment.yaml @@ -1,3 +1,7 @@ +# uncomment the config map below +# if configuring caddy with a config map +# ensure that you update ./configmap.yaml + apiVersion: extensions/v1beta1 kind: Deployment metadata: @@ -31,13 +35,19 @@ spec: volumes: - name: tmp emptyDir: {} + # - name: config-volume + # configMap: + # name: caddy-config containers: - name: caddy-ingress-controller - image: "gcr.io/danny-239313/ingresscontroller:v0.1.2" + image: gcr.io/danny-239313/ingresscontroller imagePullPolicy: IfNotPresent volumeMounts: - name: tmp mountPath: /tmp + # - name: config-volume + # mountPath: /etc/caddy/config.json + # subPath: config.json securityContext: allowPrivilegeEscalation: true capabilities: @@ -64,4 +74,4 @@ spec: fieldPath: metadata.namespace args: - -tls - - -email=navdgo@gmail.com \ No newline at end of file + - -email=test@test.com \ No newline at end of file