elias/caddy-ingess
1
0
Fork 0
mirror of https://github.com/eliasstepanik/caddy-ingess.git synced 2026-01-10 20:18:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor internal config to use caddy types

Browse Source
This commit is contained in:
dev 2019-06-18 17:09:53 -04:00 committed by Danny Navarro
parent 9a07d46e31
commit a17f132e7d
5 changed files with 124 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
cmd/caddy/main.go
View File

@ -76,19 +76,6 @@ func (h *healthChecker) Check(_ *http.Request) error {
func startMetricsServer(port int) {
mux := http.NewServeMux()
// reg := prometheus.NewRegistry()
// reg.MustRegister(prometheus.NewGoCollector())
// reg.MustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{
// PidFn: func() (int, error) { return os.Getpid(), nil },
// ReportErrors: true,
// }))
// // healthz.InstallHandler(mux,
// // healthz.PingHealthz,
// // healthChecker,
// // )
server := &http.Server{
Addr: fmt.Sprintf(":%v", port),
Handler: mux,

70
internal/caddy/config.go
View File

@ -4,59 +4,10 @@ import (
"encoding/json"
"fmt"
"github.com/caddyserver/caddy2/modules/caddyhttp"
"github.com/caddyserver/caddy2/modules/caddytls"
)
type serverRoute struct {
Matchers map[string]json.RawMessage `json:"match"`
Apply []map[string]string `json:"apply"`
Respond proxyConfig `json:"respond"`
}
type routeList []serverRoute
type proxyConfig struct {
Module string `json:"responder"`
LoadBalanceType string `json:"load_balance_type"`
Upstreams []upstreamConfig `json:"upstreams"`
}
type upstreamConfig struct {
Host string `json:"host"`
}
type httpServerConfig struct {
Listen []string `json:"listen"`
ReadTimeout string `json:"read_timeout"`
DisableAutoHTTPS bool `json:"disable_auto_https"`
// ReadHeaderTimeout caddy2.Duration `json:"read_header_timeout"`
// HiddenFiles []string `json:"hidden_files"` // TODO:... experimenting with shared/common state
TLSConnPolicies caddytls.ConnectionPolicies `json:"tls_connection_policies"`
Routes routeList `json:"routes"`
}
type httpErrorConfig struct {
Routes routeList `json:"routes"`
}
type serverConfig struct {
Server httpServerConfig `json:"ingress_server"`
}
type servers struct {
Servers serverConfig `json:"servers"`
}
type TLSConfig struct {
Module string `json:"module"`
Automation caddytls.AutomationConfig `json:"automation"`
}
type httpServer struct {
TLS TLSConfig `json:"tls"`
HTTP servers `json:"http"`
}
// StorageValues represents the config for certmagic storage providers.
type StorageValues struct {
Namespace string `json:"namespace"`
@ -70,8 +21,8 @@ type Storage struct {
// Config represents a caddy2 config file.
type Config struct {
Storage Storage `json:"storage"`
Modules httpServer `json:"apps"`
Storage Storage `json:"storage"`
Apps map[string]interface{} `json:"apps"`
}
// ControllerConfig represents ingress controller config received through cli arguments.
@ -93,9 +44,8 @@ func NewConfig(namespace string, cfg ControllerConfig) *Config {
Namespace: namespace,
},
},
Modules: httpServer{
TLS: TLSConfig{
Module: "acme",
Apps: map[string]interface{}{
"tls": caddytls.TLS{
Automation: caddytls.AutomationConfig{
Policies: []caddytls.AutomationPolicy{
caddytls.AutomationPolicy{
@ -105,15 +55,11 @@ func NewConfig(namespace string, cfg ControllerConfig) *Config {
},
},
},
HTTP: servers{
Servers: serverConfig{
Server: httpServerConfig{
"http": caddyhttp.App{
Servers: map[string]*caddyhttp.Server{
"ingress_server": &caddyhttp.Server{
DisableAutoHTTPS: !cfg.AutomaticTLS,
ReadTimeout: "30s",
Listen: []string{":80", ":443"},
TLSConnPolicies: caddytls.ConnectionPolicies{
&caddytls.ConnectionPolicy{},
},
},
},
},

59
internal/caddy/convert.go
View File

@ -4,23 +4,24 @@ import (
"encoding/json"
"fmt"
"github.com/caddyserver/caddy2/modules/caddyhttp"
"k8s.io/api/extensions/v1beta1"
)
// ConvertToCaddyConfig returns a new caddy routelist based off of ingresses managed by this controller.
func ConvertToCaddyConfig(ings []*v1beta1.Ingress) ([]serverRoute, []string, error) {
func ConvertToCaddyConfig(ings []*v1beta1.Ingress) (caddyhttp.RouteList, []string, error) {
// ~~~~
// TODO :-
// when setting the upstream url we should should bypass kube-proxy and get the ip address of
// when setting the upstream url we should should bypass kube-dns and get the ip address of
// the pod for the deployment we are proxying to so that we can proxy to that ip address port.
// this is good for session affinity and increases performance (since we don't have to hit dns).
// this is good for session affinity and increases performance.
// ~~~~
// record hosts for tls policies
var hosts []string
// create a server route for each ingress route
var routes routeList
var routes caddyhttp.RouteList
for _, ing := range ings {
for _, rule := range ing.Spec.Rules {
hosts = append(hosts, rule.Host)
@ -33,16 +34,10 @@ func ConvertToCaddyConfig(ings []*v1beta1.Ingress) ([]serverRoute, []string, err
h := json.RawMessage(fmt.Sprintf(`["%v"]`, rule.Host))
p := json.RawMessage(fmt.Sprintf(`["%v"]`, path.Path))
r.Matchers = map[string]json.RawMessage{
"host": h,
"path": p,
}
// add logging middleware to all routes
r.Apply = []map[string]string{
map[string]string{
"file": "access.log",
"middleware": "log",
r.MatcherSets = []map[string]json.RawMessage{
{
"host": h,
"path": p,
},
}
@ -54,22 +49,26 @@ func ConvertToCaddyConfig(ings []*v1beta1.Ingress) ([]serverRoute, []string, err
return routes, hosts, nil
}
func baseRoute(upstream string) serverRoute {
return serverRoute{
Apply: []map[string]string{
map[string]string{
"middleware": "log",
"file": "access.log",
},
},
Respond: proxyConfig{
Module: "reverse_proxy",
LoadBalanceType: "random",
Upstreams: []upstreamConfig{
upstreamConfig{
Host: fmt.Sprintf("http://%v", upstream),
},
},
func baseRoute(upstream string) caddyhttp.ServerRoute {
return caddyhttp.ServerRoute{
Apply: []json.RawMessage{
json.RawMessage(`
{
"middleware": "log",
"filename": "/etc/caddy/access.log"
}
`),
},
Respond: json.RawMessage(`
{
"responder": "reverse_proxy",
"load_balance_type": "random",
"upstreams": [
{
"host": "` + fmt.Sprintf("http://%v", upstream) + `"
}
]
}
`),
}
}

39
internal/controller/action.go
View File

@ -2,13 +2,22 @@ package controller
import (
"fmt"
"io"
"github.com/caddyserver/caddy2/modules/caddyhttp"
"github.com/caddyserver/ingress/internal/caddy"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"k8s.io/api/extensions/v1beta1"
)
// loadConfigMap runs when a config map with caddy config is loaded on app start.
func (c *CaddyController) onLoadConfig(obj io.Reader) {
c.syncQueue.Add(LoadConfigAction{
config: obj,
})
}
// onResourceAdded runs when an ingress resource is added to the cluster.
func (c *CaddyController) onResourceAdded(obj interface{}) {
c.syncQueue.Add(ResourceAddedAction{
@ -36,27 +45,37 @@ func (c *CaddyController) onSyncStatus(obj interface{}) {
c.syncQueue.Add(SyncStatusAction{})
}
// Action is an interface for ingress actions
// Action is an interface for ingress actions.
type Action interface {
handle(c *CaddyController) error
}
// ResourceAddedAction provides an implementation of the action interface
// LoadConfigAction provides an implementation of the action interface.
type LoadConfigAction struct {
config io.Reader
}
// ResourceAddedAction provides an implementation of the action interface.
type ResourceAddedAction struct {
resource interface{}
}
// ResourceUpdatedAction provides an implementation of the action interface
// ResourceUpdatedAction provides an implementation of the action interface.
type ResourceUpdatedAction struct {
resource interface{}
oldResource interface{}
}
// ResourceDeletedAction provides an implementation of the action interface
// ResourceDeletedAction provides an implementation of the action interface.
type ResourceDeletedAction struct {
resource interface{}
}
func (r LoadConfigAction) handle(c *CaddyController) error {
logrus.Info("Config file detected, updating Caddy config...")
return c.loadConfigFromFile(r.config)
}
func (r ResourceAddedAction) handle(c *CaddyController) error {
logrus.Info("New ingress resource detected, updating Caddy config...")
@ -129,19 +148,15 @@ func (r ResourceDeletedAction) handle(c *CaddyController) error {
func updateConfig(c *CaddyController) error {
// update internal caddy config with new ingress info
serverRoutes, hosts, err := caddy.ConvertToCaddyConfig(c.resourceStore.Ingresses)
// serverRoutes, hosts, err := caddy.ConvertToCaddyConfig(c.resourceStore.Ingresses)
serverRoutes, _, err := caddy.ConvertToCaddyConfig(c.resourceStore.Ingresses)
if err != nil {
return errors.Wrap(err, "converting ingress resources to caddy config")
}
// set the http server routes
if c.resourceStore.CaddyConfig != nil {
c.resourceStore.CaddyConfig.Modules.HTTP.Servers.Server.Routes = serverRoutes
// set tls policies
p := c.resourceStore.CaddyConfig.Modules.TLS.Automation.Policies
for i := range p {
p[i].Hosts = hosts
}
c.resourceStore.CaddyConfig.Apps["http"].(caddyhttp.App).Servers["ingress_server"].Routes = serverRoutes
}
// reload caddy2 config with newConfig

96
internal/controller/controller.go
View File

@ -3,9 +3,9 @@ package controller
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"net/http"
"io"
"log"
"os"
"time"
@ -29,7 +29,12 @@ import (
_ "github.com/caddyserver/caddy2/modules/caddyhttp"
_ "github.com/caddyserver/caddy2/modules/caddyhttp/caddylog"
_ "github.com/caddyserver/caddy2/modules/caddyhttp/fileserver"
_ "github.com/caddyserver/caddy2/modules/caddyhttp/headers"
_ "github.com/caddyserver/caddy2/modules/caddyhttp/requestbody"
_ "github.com/caddyserver/caddy2/modules/caddyhttp/reverseproxy"
_ "github.com/caddyserver/caddy2/modules/caddyhttp/rewrite"
_ "github.com/caddyserver/caddy2/modules/caddytls"
_ "github.com/caddyserver/caddy2/modules/caddytls/standardstek"
)
const (
@ -39,19 +44,19 @@ const (
// CaddyController represents an caddy ingress controller.
type CaddyController struct {
resourceStore *store.Store
kubeClient *kubernetes.Clientset
indexer cache.Indexer
syncQueue workqueue.RateLimitingInterface
statusQueue workqueue.RateLimitingInterface // statusQueue performs ingress status updates every 60 seconds but inserts the work into the sync queue
informer cache.Controller
podInfo *pod.Info
config caddy.ControllerConfig
resourceStore *store.Store
kubeClient *kubernetes.Clientset
indexer cache.Indexer
syncQueue workqueue.RateLimitingInterface
statusQueue workqueue.RateLimitingInterface // statusQueue performs ingress status updates every 60 seconds but inserts the work into the sync queue
informer cache.Controller
podInfo *pod.Info
config caddy.ControllerConfig
usingConfigMap bool
}
// NewCaddyController returns an instance of the caddy ingress controller.
func NewCaddyController(kubeClient *kubernetes.Clientset, restClient rest.Interface, cfg caddy.ControllerConfig) *CaddyController {
// setup the ingress controller and start watching resources
controller := &CaddyController{
kubeClient: kubeClient,
syncQueue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
@ -59,27 +64,42 @@ func NewCaddyController(kubeClient *kubernetes.Clientset, restClient rest.Interf
config: cfg,
}
podInfo, err := pod.GetPodDetails(kubeClient)
if err != nil {
logrus.Fatalf("Unexpected error obtaining pod information: %v", err)
}
controller.podInfo = podInfo
// load caddy config from file if mounted with config map
cfgPath := "/etc/caddy/config.json"
if _, err := os.Stat(cfgPath); !os.IsNotExist(err) {
file, err := os.Open(cfgPath)
if err != nil {
log.Fatal(err)
}
defer file.Close()
controller.usingConfigMap = true
controller.syncQueue.Add(LoadConfigAction{config: file})
}
// setup the ingress controller and start watching resources
ingressListWatcher := cache.NewListWatchFromClient(restClient, "ingresses", cfg.WatchNamespace, fields.Everything())
indexer, informer := cache.NewIndexerInformer(ingressListWatcher, &v1beta1.Ingress{}, 0, cache.ResourceEventHandlerFuncs{
AddFunc: controller.onResourceAdded,
UpdateFunc: controller.onResourceUpdated,
DeleteFunc: controller.onResourceDeleted,
}, cache.Indexers{})
podInfo, err := pod.GetPodDetails(kubeClient)
if err != nil {
logrus.Fatalf("Unexpected error obtaining pod information: %v", err)
}
controller.podInfo = podInfo
controller.indexer = indexer
controller.informer = informer
// setup store to keep track of resources
controller.resourceStore = store.NewStore(controller.kubeClient, podInfo.Namespace, cfg)
// attempt to do initial sync with ingresses
controller.syncQueue.Add(SyncStatusAction{})
// attempt to do initial sync of status addresses with ingresses
controller.dispatchSync()
// Register caddy cert storage module.
// register kubernetes specific cert-magic storage module
caddy2.RegisterModule(caddy2.Module{
Name: "caddy.storage.secret_store",
New: func() interface{} {
@ -92,12 +112,6 @@ func NewCaddyController(kubeClient *kubernetes.Clientset, restClient rest.Interf
},
})
// start caddy2
err = caddy2.StartAdmin("127.0.0.1:1234")
if err != nil {
logrus.Fatal(err)
}
return controller
}
@ -181,23 +195,33 @@ func (c *CaddyController) handleErr(err error, action interface{}) {
logrus.Error(err)
}
// reloadCaddy reloads the internal caddy instance with new config.
// loadConfigFromFile loads caddy with a config defined by an io.Reader.
func (c *CaddyController) loadConfigFromFile(cfg io.Reader) error {
err := caddy2.Load(cfg)
if err != nil {
return fmt.Errorf("could not load caddy config %v", err.Error())
}
return nil
}
// reloadCaddy reloads the internal caddy instance with config from the internal store.
func (c *CaddyController) reloadCaddy() error {
j, err := json.Marshal(c.resourceStore.CaddyConfig)
if err != nil {
return err
}
fmt.Println(string(j))
// DEBUG ONLY
// PRETTY PRINT CADDY CONFIG ON UPDATE
js, _ := json.MarshalIndent(c.resourceStore.CaddyConfig, "", "\t")
fmt.Println(string(js))
//
// post to load endpoint
resp, err := http.Post("http://127.0.0.1:1234/load", "application/json", bytes.NewBuffer(j))
r := bytes.NewReader(j)
err = caddy2.Load(r)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return errors.New("could not reload caddy config")
return fmt.Errorf("could not reload caddy config %v", err.Error())
}
return nil