diff --git a/cmd/caddy/main.go b/cmd/caddy/main.go index 43c4fef..3ac01c4 100644 --- a/cmd/caddy/main.go +++ b/cmd/caddy/main.go @@ -76,19 +76,6 @@ func (h *healthChecker) Check(_ *http.Request) error { func startMetricsServer(port int) { mux := http.NewServeMux() - // reg := prometheus.NewRegistry() - - // reg.MustRegister(prometheus.NewGoCollector()) - // reg.MustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{ - // PidFn: func() (int, error) { return os.Getpid(), nil }, - // ReportErrors: true, - // })) - - // // healthz.InstallHandler(mux, - // // healthz.PingHealthz, - // // healthChecker, - // // ) - server := &http.Server{ Addr: fmt.Sprintf(":%v", port), Handler: mux, diff --git a/internal/caddy/config.go b/internal/caddy/config.go index 1749370..c02a7c8 100644 --- a/internal/caddy/config.go +++ b/internal/caddy/config.go @@ -4,59 +4,10 @@ import ( "encoding/json" "fmt" + "github.com/caddyserver/caddy2/modules/caddyhttp" "github.com/caddyserver/caddy2/modules/caddytls" ) -type serverRoute struct { - Matchers map[string]json.RawMessage `json:"match"` - Apply []map[string]string `json:"apply"` - Respond proxyConfig `json:"respond"` -} - -type routeList []serverRoute - -type proxyConfig struct { - Module string `json:"responder"` - LoadBalanceType string `json:"load_balance_type"` - Upstreams []upstreamConfig `json:"upstreams"` -} - -type upstreamConfig struct { - Host string `json:"host"` -} - -type httpServerConfig struct { - Listen []string `json:"listen"` - ReadTimeout string `json:"read_timeout"` - DisableAutoHTTPS bool `json:"disable_auto_https"` - // ReadHeaderTimeout caddy2.Duration `json:"read_header_timeout"` - // HiddenFiles []string `json:"hidden_files"` // TODO:... experimenting with shared/common state - TLSConnPolicies caddytls.ConnectionPolicies `json:"tls_connection_policies"` - Routes routeList `json:"routes"` -} - -type httpErrorConfig struct { - Routes routeList `json:"routes"` -} - -type serverConfig struct { - Server httpServerConfig `json:"ingress_server"` -} - -type servers struct { - Servers serverConfig `json:"servers"` -} - -type TLSConfig struct { - Module string `json:"module"` - Automation caddytls.AutomationConfig `json:"automation"` -} - -type httpServer struct { - TLS TLSConfig `json:"tls"` - HTTP servers `json:"http"` -} - // StorageValues represents the config for certmagic storage providers. type StorageValues struct { Namespace string `json:"namespace"` @@ -70,8 +21,8 @@ type Storage struct { // Config represents a caddy2 config file. type Config struct { - Storage Storage `json:"storage"` - Modules httpServer `json:"apps"` + Storage Storage `json:"storage"` + Apps map[string]interface{} `json:"apps"` } // ControllerConfig represents ingress controller config received through cli arguments. @@ -93,9 +44,8 @@ func NewConfig(namespace string, cfg ControllerConfig) *Config { Namespace: namespace, }, }, - Modules: httpServer{ - TLS: TLSConfig{ - Module: "acme", + Apps: map[string]interface{}{ + "tls": caddytls.TLS{ Automation: caddytls.AutomationConfig{ Policies: []caddytls.AutomationPolicy{ caddytls.AutomationPolicy{ @@ -105,15 +55,11 @@ func NewConfig(namespace string, cfg ControllerConfig) *Config { }, }, }, - HTTP: servers{ - Servers: serverConfig{ - Server: httpServerConfig{ + "http": caddyhttp.App{ + Servers: map[string]*caddyhttp.Server{ + "ingress_server": &caddyhttp.Server{ DisableAutoHTTPS: !cfg.AutomaticTLS, - ReadTimeout: "30s", Listen: []string{":80", ":443"}, - TLSConnPolicies: caddytls.ConnectionPolicies{ - &caddytls.ConnectionPolicy{}, - }, }, }, }, diff --git a/internal/caddy/convert.go b/internal/caddy/convert.go index bf15d76..7def613 100644 --- a/internal/caddy/convert.go +++ b/internal/caddy/convert.go @@ -4,23 +4,24 @@ import ( "encoding/json" "fmt" + "github.com/caddyserver/caddy2/modules/caddyhttp" "k8s.io/api/extensions/v1beta1" ) // ConvertToCaddyConfig returns a new caddy routelist based off of ingresses managed by this controller. -func ConvertToCaddyConfig(ings []*v1beta1.Ingress) ([]serverRoute, []string, error) { +func ConvertToCaddyConfig(ings []*v1beta1.Ingress) (caddyhttp.RouteList, []string, error) { // ~~~~ // TODO :- - // when setting the upstream url we should should bypass kube-proxy and get the ip address of + // when setting the upstream url we should should bypass kube-dns and get the ip address of // the pod for the deployment we are proxying to so that we can proxy to that ip address port. - // this is good for session affinity and increases performance (since we don't have to hit dns). + // this is good for session affinity and increases performance. // ~~~~ // record hosts for tls policies var hosts []string // create a server route for each ingress route - var routes routeList + var routes caddyhttp.RouteList for _, ing := range ings { for _, rule := range ing.Spec.Rules { hosts = append(hosts, rule.Host) @@ -33,16 +34,10 @@ func ConvertToCaddyConfig(ings []*v1beta1.Ingress) ([]serverRoute, []string, err h := json.RawMessage(fmt.Sprintf(`["%v"]`, rule.Host)) p := json.RawMessage(fmt.Sprintf(`["%v"]`, path.Path)) - r.Matchers = map[string]json.RawMessage{ - "host": h, - "path": p, - } - - // add logging middleware to all routes - r.Apply = []map[string]string{ - map[string]string{ - "file": "access.log", - "middleware": "log", + r.MatcherSets = []map[string]json.RawMessage{ + { + "host": h, + "path": p, }, } @@ -54,22 +49,26 @@ func ConvertToCaddyConfig(ings []*v1beta1.Ingress) ([]serverRoute, []string, err return routes, hosts, nil } -func baseRoute(upstream string) serverRoute { - return serverRoute{ - Apply: []map[string]string{ - map[string]string{ - "middleware": "log", - "file": "access.log", - }, - }, - Respond: proxyConfig{ - Module: "reverse_proxy", - LoadBalanceType: "random", - Upstreams: []upstreamConfig{ - upstreamConfig{ - Host: fmt.Sprintf("http://%v", upstream), - }, - }, +func baseRoute(upstream string) caddyhttp.ServerRoute { + return caddyhttp.ServerRoute{ + Apply: []json.RawMessage{ + json.RawMessage(` + { + "middleware": "log", + "filename": "/etc/caddy/access.log" + } + `), }, + Respond: json.RawMessage(` + { + "responder": "reverse_proxy", + "load_balance_type": "random", + "upstreams": [ + { + "host": "` + fmt.Sprintf("http://%v", upstream) + `" + } + ] + } + `), } } diff --git a/internal/controller/action.go b/internal/controller/action.go index 8193424..19e69af 100644 --- a/internal/controller/action.go +++ b/internal/controller/action.go @@ -2,13 +2,22 @@ package controller import ( "fmt" + "io" + "github.com/caddyserver/caddy2/modules/caddyhttp" "github.com/caddyserver/ingress/internal/caddy" "github.com/pkg/errors" "github.com/sirupsen/logrus" "k8s.io/api/extensions/v1beta1" ) +// loadConfigMap runs when a config map with caddy config is loaded on app start. +func (c *CaddyController) onLoadConfig(obj io.Reader) { + c.syncQueue.Add(LoadConfigAction{ + config: obj, + }) +} + // onResourceAdded runs when an ingress resource is added to the cluster. func (c *CaddyController) onResourceAdded(obj interface{}) { c.syncQueue.Add(ResourceAddedAction{ @@ -36,27 +45,37 @@ func (c *CaddyController) onSyncStatus(obj interface{}) { c.syncQueue.Add(SyncStatusAction{}) } -// Action is an interface for ingress actions +// Action is an interface for ingress actions. type Action interface { handle(c *CaddyController) error } -// ResourceAddedAction provides an implementation of the action interface +// LoadConfigAction provides an implementation of the action interface. +type LoadConfigAction struct { + config io.Reader +} + +// ResourceAddedAction provides an implementation of the action interface. type ResourceAddedAction struct { resource interface{} } -// ResourceUpdatedAction provides an implementation of the action interface +// ResourceUpdatedAction provides an implementation of the action interface. type ResourceUpdatedAction struct { resource interface{} oldResource interface{} } -// ResourceDeletedAction provides an implementation of the action interface +// ResourceDeletedAction provides an implementation of the action interface. type ResourceDeletedAction struct { resource interface{} } +func (r LoadConfigAction) handle(c *CaddyController) error { + logrus.Info("Config file detected, updating Caddy config...") + return c.loadConfigFromFile(r.config) +} + func (r ResourceAddedAction) handle(c *CaddyController) error { logrus.Info("New ingress resource detected, updating Caddy config...") @@ -129,19 +148,15 @@ func (r ResourceDeletedAction) handle(c *CaddyController) error { func updateConfig(c *CaddyController) error { // update internal caddy config with new ingress info - serverRoutes, hosts, err := caddy.ConvertToCaddyConfig(c.resourceStore.Ingresses) + // serverRoutes, hosts, err := caddy.ConvertToCaddyConfig(c.resourceStore.Ingresses) + serverRoutes, _, err := caddy.ConvertToCaddyConfig(c.resourceStore.Ingresses) if err != nil { return errors.Wrap(err, "converting ingress resources to caddy config") } + // set the http server routes if c.resourceStore.CaddyConfig != nil { - c.resourceStore.CaddyConfig.Modules.HTTP.Servers.Server.Routes = serverRoutes - - // set tls policies - p := c.resourceStore.CaddyConfig.Modules.TLS.Automation.Policies - for i := range p { - p[i].Hosts = hosts - } + c.resourceStore.CaddyConfig.Apps["http"].(caddyhttp.App).Servers["ingress_server"].Routes = serverRoutes } // reload caddy2 config with newConfig diff --git a/internal/controller/controller.go b/internal/controller/controller.go index 4e3c425..d577858 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -3,9 +3,9 @@ package controller import ( "bytes" "encoding/json" - "errors" "fmt" - "net/http" + "io" + "log" "os" "time" @@ -29,7 +29,12 @@ import ( _ "github.com/caddyserver/caddy2/modules/caddyhttp" _ "github.com/caddyserver/caddy2/modules/caddyhttp/caddylog" _ "github.com/caddyserver/caddy2/modules/caddyhttp/fileserver" + _ "github.com/caddyserver/caddy2/modules/caddyhttp/headers" + _ "github.com/caddyserver/caddy2/modules/caddyhttp/requestbody" _ "github.com/caddyserver/caddy2/modules/caddyhttp/reverseproxy" + _ "github.com/caddyserver/caddy2/modules/caddyhttp/rewrite" + _ "github.com/caddyserver/caddy2/modules/caddytls" + _ "github.com/caddyserver/caddy2/modules/caddytls/standardstek" ) const ( @@ -39,19 +44,19 @@ const ( // CaddyController represents an caddy ingress controller. type CaddyController struct { - resourceStore *store.Store - kubeClient *kubernetes.Clientset - indexer cache.Indexer - syncQueue workqueue.RateLimitingInterface - statusQueue workqueue.RateLimitingInterface // statusQueue performs ingress status updates every 60 seconds but inserts the work into the sync queue - informer cache.Controller - podInfo *pod.Info - config caddy.ControllerConfig + resourceStore *store.Store + kubeClient *kubernetes.Clientset + indexer cache.Indexer + syncQueue workqueue.RateLimitingInterface + statusQueue workqueue.RateLimitingInterface // statusQueue performs ingress status updates every 60 seconds but inserts the work into the sync queue + informer cache.Controller + podInfo *pod.Info + config caddy.ControllerConfig + usingConfigMap bool } // NewCaddyController returns an instance of the caddy ingress controller. func NewCaddyController(kubeClient *kubernetes.Clientset, restClient rest.Interface, cfg caddy.ControllerConfig) *CaddyController { - // setup the ingress controller and start watching resources controller := &CaddyController{ kubeClient: kubeClient, syncQueue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()), @@ -59,27 +64,42 @@ func NewCaddyController(kubeClient *kubernetes.Clientset, restClient rest.Interf config: cfg, } + podInfo, err := pod.GetPodDetails(kubeClient) + if err != nil { + logrus.Fatalf("Unexpected error obtaining pod information: %v", err) + } + controller.podInfo = podInfo + + // load caddy config from file if mounted with config map + cfgPath := "/etc/caddy/config.json" + if _, err := os.Stat(cfgPath); !os.IsNotExist(err) { + file, err := os.Open(cfgPath) + if err != nil { + log.Fatal(err) + } + defer file.Close() + + controller.usingConfigMap = true + controller.syncQueue.Add(LoadConfigAction{config: file}) + } + + // setup the ingress controller and start watching resources ingressListWatcher := cache.NewListWatchFromClient(restClient, "ingresses", cfg.WatchNamespace, fields.Everything()) indexer, informer := cache.NewIndexerInformer(ingressListWatcher, &v1beta1.Ingress{}, 0, cache.ResourceEventHandlerFuncs{ AddFunc: controller.onResourceAdded, UpdateFunc: controller.onResourceUpdated, DeleteFunc: controller.onResourceDeleted, }, cache.Indexers{}) - - podInfo, err := pod.GetPodDetails(kubeClient) - if err != nil { - logrus.Fatalf("Unexpected error obtaining pod information: %v", err) - } - - controller.podInfo = podInfo controller.indexer = indexer controller.informer = informer + + // setup store to keep track of resources controller.resourceStore = store.NewStore(controller.kubeClient, podInfo.Namespace, cfg) - // attempt to do initial sync with ingresses - controller.syncQueue.Add(SyncStatusAction{}) + // attempt to do initial sync of status addresses with ingresses + controller.dispatchSync() - // Register caddy cert storage module. + // register kubernetes specific cert-magic storage module caddy2.RegisterModule(caddy2.Module{ Name: "caddy.storage.secret_store", New: func() interface{} { @@ -92,12 +112,6 @@ func NewCaddyController(kubeClient *kubernetes.Clientset, restClient rest.Interf }, }) - // start caddy2 - err = caddy2.StartAdmin("127.0.0.1:1234") - if err != nil { - logrus.Fatal(err) - } - return controller } @@ -181,23 +195,33 @@ func (c *CaddyController) handleErr(err error, action interface{}) { logrus.Error(err) } -// reloadCaddy reloads the internal caddy instance with new config. +// loadConfigFromFile loads caddy with a config defined by an io.Reader. +func (c *CaddyController) loadConfigFromFile(cfg io.Reader) error { + err := caddy2.Load(cfg) + if err != nil { + return fmt.Errorf("could not load caddy config %v", err.Error()) + } + + return nil +} + +// reloadCaddy reloads the internal caddy instance with config from the internal store. func (c *CaddyController) reloadCaddy() error { j, err := json.Marshal(c.resourceStore.CaddyConfig) if err != nil { return err } - fmt.Println(string(j)) + // DEBUG ONLY + // PRETTY PRINT CADDY CONFIG ON UPDATE + js, _ := json.MarshalIndent(c.resourceStore.CaddyConfig, "", "\t") + fmt.Println(string(js)) + // - // post to load endpoint - resp, err := http.Post("http://127.0.0.1:1234/load", "application/json", bytes.NewBuffer(j)) + r := bytes.NewReader(j) + err = caddy2.Load(r) if err != nil { - return err - } - - if resp.StatusCode != http.StatusOK { - return errors.New("could not reload caddy config") + return fmt.Errorf("could not reload caddy config %v", err.Error()) } return nil