From 54bd76cb78b0282b986f00d3d4cd8638b399a9d1 Mon Sep 17 00:00:00 2001 From: dev Date: Thu, 25 Apr 2019 15:05:10 -0400 Subject: [PATCH] Manage CRUD of caddy routes for ingress resource changes --- hack/test/example-deployment2.yaml | 24 ++++++++ hack/test/example-ingress.yaml | 4 ++ hack/test/example-service2.yaml | 12 ++++ internal/caddy/config.go | 19 ------ internal/caddy/convert.go | 61 +++++++++++++++++++ internal/caddy/import.go | 20 ------- internal/controller/action.go | 88 +++++++++++++++++++--------- internal/controller/controller.go | 30 +++++----- internal/store/store.go | 30 +++++++++- kubernetes/generated/deployment.yaml | 1 - skaffold.yaml | 2 + 11 files changed, 207 insertions(+), 84 deletions(-) create mode 100644 hack/test/example-deployment2.yaml create mode 100644 hack/test/example-service2.yaml create mode 100644 internal/caddy/convert.go delete mode 100644 internal/caddy/import.go diff --git a/hack/test/example-deployment2.yaml b/hack/test/example-deployment2.yaml new file mode 100644 index 0000000..b2b8bb2 --- /dev/null +++ b/hack/test/example-deployment2.yaml @@ -0,0 +1,24 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: example2 + labels: + app: example2 +spec: + replicas: 1 + selector: + matchLabels: + app: example2 + template: + metadata: + labels: + app: example2 + spec: + containers: + - name: httpecho + image: hashicorp/http-echo + args: + - "-listen=:8080" + - "-text=hello world 2" + ports: + - containerPort: 8080 \ No newline at end of file diff --git a/hack/test/example-ingress.yaml b/hack/test/example-ingress.yaml index cfe1f28..c016590 100644 --- a/hack/test/example-ingress.yaml +++ b/hack/test/example-ingress.yaml @@ -7,6 +7,10 @@ spec: - host: hello-world.xyz http: paths: + - path: /hello2 + backend: + serviceName: example2 + servicePort: 8080 - path: /hello backend: serviceName: example diff --git a/hack/test/example-service2.yaml b/hack/test/example-service2.yaml new file mode 100644 index 0000000..e90d464 --- /dev/null +++ b/hack/test/example-service2.yaml @@ -0,0 +1,12 @@ +kind: Service +apiVersion: v1 +metadata: + name: example2 +spec: + type: NodePort + selector: + app: example2 + ports: + - protocol: TCP + port: 80 + targetPort: 8080 \ No newline at end of file diff --git a/internal/caddy/config.go b/internal/caddy/config.go index c83ae93..3a38690 100644 --- a/internal/caddy/config.go +++ b/internal/caddy/config.go @@ -63,25 +63,6 @@ func NewConfig() *Config { Servers: serverConfig{ Server: httpServerConfig{ Listen: []string{":80", ":443"}, - Routes: routeList{ - serverRoute{ - Apply: []map[string]string{ - map[string]string{ - "_module": "log", - "file": "access.log", - }, - }, - Respond: proxyConfig{ - Module: "reverse_proxy", - LoadBalanceType: "random", - Upstreams: []upstreamConfig{ - upstreamConfig{ - Host: "http://example", - }, - }, - }, - }, - }, }, }, }, diff --git a/internal/caddy/convert.go b/internal/caddy/convert.go new file mode 100644 index 0000000..0fee5e6 --- /dev/null +++ b/internal/caddy/convert.go @@ -0,0 +1,61 @@ +package caddy + +import ( + "encoding/json" + "fmt" + + "k8s.io/api/extensions/v1beta1" +) + +// ~~~~ +// TODO :- +// when setting the upstream url we should should bypass kube-proxy and get the ip address of +// the pod for the deployment we are proxying to so that we can proxy to that ip address port. +// this is good for session affinity and increases performance (since we don't have to hit dns). +// ~~~~ + +// ConvertToCaddyConfig returns a new caddy routelist based off of ingresses managed by this controller. +func ConvertToCaddyConfig(ings []*v1beta1.Ingress) ([]serverRoute, error) { + // create a server route for each ingress route + var routes routeList + for _, ing := range ings { + for _, rule := range ing.Spec.Rules { + for _, path := range rule.HTTP.Paths { + r := baseRoute(path.Backend.ServiceName) + + // create matchers for ingress host and path + h := json.RawMessage(fmt.Sprintf(`["%v"]`, rule.Host)) + p := json.RawMessage(fmt.Sprintf(`["%v"]`, path.Path)) + + r.Matchers = map[string]json.RawMessage{ + "host": h, + "path": p, + } + + routes = append(routes, r) + } + } + } + + return routes, nil +} + +func baseRoute(upstream string) serverRoute { + return serverRoute{ + Apply: []map[string]string{ + map[string]string{ + "_module": "log", + "file": "access.log", + }, + }, + Respond: proxyConfig{ + Module: "reverse_proxy", + LoadBalanceType: "random", + Upstreams: []upstreamConfig{ + upstreamConfig{ + Host: fmt.Sprintf("http://%v", upstream), + }, + }, + }, + } +} diff --git a/internal/caddy/import.go b/internal/caddy/import.go deleted file mode 100644 index 73abd79..0000000 --- a/internal/caddy/import.go +++ /dev/null @@ -1,20 +0,0 @@ -package caddy - -import ( - "k8s.io/api/extensions/v1beta1" -) - -// AddIngressConfig attempts to configure caddy2 for a new ingress resource. -func AddIngressConfig(c *Config, ing *v1beta1.Ingress) (*Config, error) { - return nil, nil -} - -// UpdateIngressConfig attempts to update caddy2 config for an ingress resource that has already been configured. -func UpdateIngressConfig(c *Config, ing *v1beta1.Ingress) (*Config, error) { - return nil, nil -} - -// DeleteIngressConfig attempts to update caddy2 config to remove an ingress resource. -func DeleteIngressConfig(c *Config, ing *v1beta1.Ingress) (*Config, error) { - return nil, nil -} diff --git a/internal/controller/action.go b/internal/controller/action.go index 5bcb55b..5e39fdf 100644 --- a/internal/controller/action.go +++ b/internal/controller/action.go @@ -2,10 +2,11 @@ package controller import ( "fmt" - "io/ioutil" "bitbucket.org/lightcodelabs/ingress/internal/caddy" + "github.com/pkg/errors" "k8s.io/api/extensions/v1beta1" + "k8s.io/klog" ) // onResourceAdded runs when an ingress resource is added to the cluster. @@ -57,58 +58,91 @@ type ResourceDeletedAction struct { } func (r ResourceAddedAction) handle(c *CaddyController) error { + klog.Info("New ingress resource detected, updating Caddy config...") + // configure caddy to handle this resource ing, ok := r.resource.(*v1beta1.Ingress) if !ok { return fmt.Errorf("ResourceAddedAction: incoming resource is not of type ingress") } - // get current caddy config for rollback purposes - oldConfig := *c.resourceStore.CaddyConfig - fmt.Fprint(ioutil.Discard, oldConfig) + // add this ingress to the internal store + c.resourceStore.AddIngress(ing) - // update internal caddy config with new ingress info - newConfig, err := caddy.AddIngressConfig(c.resourceStore.CaddyConfig, ing) + err := updateConfig(c) if err != nil { return err } - // TODO :- reload caddy2 config with newConfig - fmt.Fprint(ioutil.Discard, newConfig) - - // TODO :- if err rollback to old config - // ensure that ingress source is updated to point to this ingress controller's ip - c.syncStatus([]*v1beta1.Ingress{ing}) - - c.resourceStore.AddIngress(ing) - - // ~~~~ - // when updating caddy config the ingress controller should bypass kube-proxy and get the ip address of - // the pod that the deployment we are proxying to is running on so that we can proxy to that ip address port. - // this is good for session affinity and increases performance (since we don't have to hit dns). - - // example getting an ingress - // ingClient := c.kubeClient.ExtensionsV1beta1().Ingresses(c.namespace) // get a client to update the ingress - // ingClient.UpdateStatus(ing) // pass an ingress with the status.address field updated - // ~~~ + err = c.syncStatus([]*v1beta1.Ingress{ing}) + if err != nil { + return errors.Wrapf(err, "syncing ingress source address name: %v", ing.GetName()) + } + klog.Info("Caddy reloaded successfully.") return nil } func (r ResourceUpdatedAction) handle(c *CaddyController) error { - // find the caddy config related to the oldResource and update it + klog.Info("Ingress resource update detected, updating Caddy config...") - fmt.Printf("\nUpdated resource:\n +%v\n\nOld resource: \n %+v\n", r.resource, r.oldResource) + // update caddy config regarding this ingress + ing, ok := r.resource.(*v1beta1.Ingress) + if !ok { + return fmt.Errorf("ResourceAddedAction: incoming resource is not of type ingress") + } + // add or update this ingress in the internal store + c.resourceStore.AddIngress(ing) + + err := updateConfig(c) + if err != nil { + return err + } + + klog.Info("Caddy reloaded successfully.") return nil } func (r ResourceDeletedAction) handle(c *CaddyController) error { + klog.Info("Ingress resource deletion detected, updating Caddy config...") + // delete all resources from caddy config that are associated with this resource // reload caddy config + ing, ok := r.resource.(*v1beta1.Ingress) + if !ok { + return fmt.Errorf("ResourceAddedAction: incoming resource is not of type ingress") + } - fmt.Printf("\nDeleted resource:\n +%v\n", r.resource) + // add this ingress to the internal store + c.resourceStore.PluckIngress(ing) + + err := updateConfig(c) + if err != nil { + return err + } + + klog.Info("Caddy reloaded successfully.") + return nil +} + +func updateConfig(c *CaddyController) error { + // update internal caddy config with new ingress info + serverRoutes, err := caddy.ConvertToCaddyConfig(c.resourceStore.Ingresses) + if err != nil { + return errors.Wrap(err, "converting ingress resources to caddy config") + } + + if c.resourceStore.CaddyConfig != nil { + c.resourceStore.CaddyConfig.Modules.HTTP.Servers.Server.Routes = serverRoutes + } + + // reload caddy2 config with newConfig + err = c.reloadCaddy() + if err != nil { + return errors.Wrap(err, "caddy config reload") + } return nil } diff --git a/internal/controller/controller.go b/internal/controller/controller.go index efc4de8..48f2323 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/json" "fmt" - "log" "os" "time" @@ -24,6 +23,7 @@ import ( "k8s.io/klog" // load required caddy plugins + _ "bitbucket.org/lightcodelabs/caddy2/modules/caddyhttp" _ "bitbucket.org/lightcodelabs/caddy2/modules/caddyhttp/caddylog" _ "bitbucket.org/lightcodelabs/caddy2/modules/caddyhttp/staticfiles" _ "bitbucket.org/lightcodelabs/proxy" @@ -36,7 +36,7 @@ var ResourceMap = map[string]run.Object{ const ( // how often we should attempt to keep ingress resource's source address in sync - syncInterval = time.Second * 10 + syncInterval = time.Second * 30 ) // CaddyController represents an caddy ingress controller. @@ -87,13 +87,6 @@ func NewCaddyController(namespace string, kubeClient *kubernetes.Clientset, reso func (c *CaddyController) Shutdown() error { // remove this ingress controller's ip from ingress resources. c.updateIngStatuses([]apiv1.LoadBalancerIngress{apiv1.LoadBalancerIngress{}}, c.resourceStore.Ingresses) - - // shutdownCaddy server gracefully - // err := caddy2.StopAdmin() - // if err != nil { - // return err - // } - return nil } @@ -102,17 +95,26 @@ func (c *CaddyController) handleErr(err error, action interface{}) { klog.Error(err) } -// Run method starts the ingress controller. -func (c *CaddyController) Run(stopCh chan struct{}) { +func (c *CaddyController) reloadCaddy() error { j, err := json.Marshal(c.resourceStore.CaddyConfig) if err != nil { - log.Fatal(err) + return err } - cfgReader := bytes.NewReader(j) + cfgReader := bytes.NewReader(j) err = caddy2.Load(cfgReader) if err != nil { - log.Fatal(err) + return err + } + + return nil +} + +// Run method starts the ingress controller. +func (c *CaddyController) Run(stopCh chan struct{}) { + err := c.reloadCaddy() + if err != nil { + klog.Errorf("initial caddy config load failed, %v", err.Error()) } defer runtime.HandleCrash() diff --git a/internal/store/store.go b/internal/store/store.go index 8317a04..75b2eaa 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -36,13 +36,15 @@ func NewStore(kubeClient *kubernetes.Clientset) *Store { return s } -// AddIngress adds an ingress to the store +// AddIngress adds an ingress to the store. It updates the element at the given index if it is unique. func (s *Store) AddIngress(ing *v1beta1.Ingress) { isUniq := true - for _, i := range s.Ingresses { - if i.GetUID() == ing.GetUID() { + for i := range s.Ingresses { + in := s.Ingresses[i] + if in.GetUID() == ing.GetUID() { isUniq = false + s.Ingresses[i] = ing } } @@ -50,3 +52,25 @@ func (s *Store) AddIngress(ing *v1beta1.Ingress) { s.Ingresses = append(s.Ingresses, ing) } } + +// PluckIngress removes the ingress passed in as an argument from the stores list of ingresses. +func (s *Store) PluckIngress(ing *v1beta1.Ingress) { + id := ing.GetUID() + + var index int + var hasMatch bool + for i := range s.Ingresses { + if s.Ingresses[i].GetUID() == id { + index = i + hasMatch = true + break + } + } + + // since order is not important we can swap the element to delete with the one at the end of the slice + // and then set ingresses to the n-1 first elements + if hasMatch { + s.Ingresses[len(s.Ingresses)-1], s.Ingresses[index] = s.Ingresses[index], s.Ingresses[len(s.Ingresses)-1] + s.Ingresses = s.Ingresses[:len(s.Ingresses)-1] + } +} diff --git a/kubernetes/generated/deployment.yaml b/kubernetes/generated/deployment.yaml index be6c760..42205e4 100644 --- a/kubernetes/generated/deployment.yaml +++ b/kubernetes/generated/deployment.yaml @@ -24,7 +24,6 @@ spec: release: "release-name" heritage: "Tiller" version: v0.1.0 - spec: serviceAccountName: caddyingresscontroller containers: diff --git a/skaffold.yaml b/skaffold.yaml index c4bdcde..2dbdac4 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -8,6 +8,8 @@ deploy: manifests: - hack/test/example-deployment.yaml - hack/test/example-ingress.yaml + - hack/test/example-deployment2.yaml + - hack/test/example-service2.yaml - hack/test/example-service.yaml - kubernetes/generated/clusterrole.yaml - kubernetes/generated/clusterrolebinding.yaml