Manage CRUD of caddy routes for ingress resource changes

hack/test/example-deployment2.yaml

@@ -0,0 +1,24 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: example2
+  labels:
+    app: example2
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: example2
+  template:
+    metadata:
+      labels:
+        app: example2
+    spec:
+      containers:
+      - name: httpecho
+        image: hashicorp/http-echo
+        args:
+        - "-listen=:8080"
+        - "-text=hello world 2"
+        ports:
+        - containerPort: 8080

hack/test/example-ingress.yaml

@@ -7,6 +7,10 @@ spec:
   - host: hello-world.xyz
     http:
       paths:
+      - path: /hello2
+        backend:
+          serviceName: example2
+          servicePort: 8080
       - path: /hello
         backend:
           serviceName: example

hack/test/example-service2.yaml

@@ -0,0 +1,12 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: example2
+spec:
+  type: NodePort
+  selector:
+    app: example2
+  ports:
+  - protocol: TCP
+    port: 80
+    targetPort: 8080

internal/caddy/config.go

@@ -63,25 +63,6 @@ func NewConfig() *Config {
 				Servers: serverConfig{
 					Server: httpServerConfig{
 						Listen: []string{":80", ":443"},
-						Routes: routeList{
-							serverRoute{
-								Apply: []map[string]string{
-									map[string]string{
-										"_module": "log",
-										"file":    "access.log",
-									},
-								},
-								Respond: proxyConfig{
-									Module:          "reverse_proxy",
-									LoadBalanceType: "random",
-									Upstreams: []upstreamConfig{
-										upstreamConfig{
-											Host: "http://example",
-										},
-									},
-								},
-							},
-						},
 					},
 				},
 			},

internal/caddy/convert.go

@@ -0,0 +1,61 @@
+package caddy
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"k8s.io/api/extensions/v1beta1"
+)
+
+// ~~~~
+// TODO :-
+// when setting the upstream url we should should bypass kube-proxy and get the ip address of
+// the pod for the deployment we are proxying to so that we can proxy to that ip address port.
+// this is good for session affinity and increases performance (since we don't have to hit dns).
+// ~~~~
+
+// ConvertToCaddyConfig returns a new caddy routelist based off of ingresses managed by this controller.
+func ConvertToCaddyConfig(ings []*v1beta1.Ingress) ([]serverRoute, error) {
+	// create a server route for each ingress route
+	var routes routeList
+	for _, ing := range ings {
+		for _, rule := range ing.Spec.Rules {
+			for _, path := range rule.HTTP.Paths {
+				r := baseRoute(path.Backend.ServiceName)
+
+				// create matchers for ingress host and path
+				h := json.RawMessage(fmt.Sprintf(`["%v"]`, rule.Host))
+				p := json.RawMessage(fmt.Sprintf(`["%v"]`, path.Path))
+
+				r.Matchers = map[string]json.RawMessage{
+					"host": h,
+					"path": p,
+				}
+
+				routes = append(routes, r)
+			}
+		}
+	}
+
+	return routes, nil
+}
+
+func baseRoute(upstream string) serverRoute {
+	return serverRoute{
+		Apply: []map[string]string{
+			map[string]string{
+				"_module": "log",
+				"file":    "access.log",
+			},
+		},
+		Respond: proxyConfig{
+			Module:          "reverse_proxy",
+			LoadBalanceType: "random",
+			Upstreams: []upstreamConfig{
+				upstreamConfig{
+					Host: fmt.Sprintf("http://%v", upstream),
+				},
+			},
+		},
+	}
+}

internal/caddy/import.go

@@ -1,20 +0,0 @@
-package caddy
-
-import (
-	"k8s.io/api/extensions/v1beta1"
-)
-
-// AddIngressConfig attempts to configure caddy2 for a new ingress resource.
-func AddIngressConfig(c *Config, ing *v1beta1.Ingress) (*Config, error) {
-	return nil, nil
-}
-
-// UpdateIngressConfig attempts to update caddy2 config for an ingress resource that has already been configured.
-func UpdateIngressConfig(c *Config, ing *v1beta1.Ingress) (*Config, error) {
-	return nil, nil
-}
-
-// DeleteIngressConfig attempts to update caddy2 config to remove an ingress resource.
-func DeleteIngressConfig(c *Config, ing *v1beta1.Ingress) (*Config, error) {
-	return nil, nil
-}

internal/controller/action.go

@@ -2,10 +2,11 @@ package controller
 
 import (
 	"fmt"
-	"io/ioutil"
 
 	"bitbucket.org/lightcodelabs/ingress/internal/caddy"
+	"github.com/pkg/errors"
 	"k8s.io/api/extensions/v1beta1"
+	"k8s.io/klog"
 )
 
 // onResourceAdded runs when an ingress resource is added to the cluster.
@@ -57,58 +58,91 @@ type ResourceDeletedAction struct {
 }
 
 func (r ResourceAddedAction) handle(c *CaddyController) error {
+	klog.Info("New ingress resource detected, updating Caddy config...")
+
 	// configure caddy to handle this resource
 	ing, ok := r.resource.(*v1beta1.Ingress)
 	if !ok {
 		return fmt.Errorf("ResourceAddedAction: incoming resource is not of type ingress")
 	}
 
-	// get current caddy config for rollback purposes
-	oldConfig := *c.resourceStore.CaddyConfig
-	fmt.Fprint(ioutil.Discard, oldConfig)
+	// add this ingress to the internal store
+	c.resourceStore.AddIngress(ing)
 
-	// update internal caddy config with new ingress info
-	newConfig, err := caddy.AddIngressConfig(c.resourceStore.CaddyConfig, ing)
+	err := updateConfig(c)
 	if err != nil {
 		return err
 	}
 
-	// TODO :- reload caddy2 config with newConfig
-	fmt.Fprint(ioutil.Discard, newConfig)
-
-	// TODO :- if err rollback to old config
-
 	// ensure that ingress source is updated to point to this ingress controller's ip
-	c.syncStatus([]*v1beta1.Ingress{ing})
-
-	c.resourceStore.AddIngress(ing)
-
-	// ~~~~
-	// when updating caddy config the ingress controller should bypass kube-proxy and get the ip address of
-	// the pod that the deployment we are proxying to is running on so that we can proxy to that ip address port.
-	// this is good for session affinity and increases performance (since we don't have to hit dns).
-
-	// example getting an ingress
-	// ingClient := c.kubeClient.ExtensionsV1beta1().Ingresses(c.namespace) // get a client to update the ingress
-	// ingClient.UpdateStatus(ing) // pass an ingress with the status.address field updated
-	// ~~~
+	err = c.syncStatus([]*v1beta1.Ingress{ing})
+	if err != nil {
+		return errors.Wrapf(err, "syncing ingress source address name: %v", ing.GetName())
+	}
 
+	klog.Info("Caddy reloaded successfully.")
 	return nil
 }
 
 func (r ResourceUpdatedAction) handle(c *CaddyController) error {
-	// find the caddy config related to the oldResource and update it
+	klog.Info("Ingress resource update detected, updating Caddy config...")
 
-	fmt.Printf("\nUpdated resource:\n +%v\n\nOld resource: \n %+v\n", r.resource, r.oldResource)
+	// update caddy config regarding this ingress
+	ing, ok := r.resource.(*v1beta1.Ingress)
+	if !ok {
+		return fmt.Errorf("ResourceAddedAction: incoming resource is not of type ingress")
+	}
 
+	// add or update this ingress in the internal store
+	c.resourceStore.AddIngress(ing)
+
+	err := updateConfig(c)
+	if err != nil {
+		return err
+	}
+
+	klog.Info("Caddy reloaded successfully.")
 	return nil
 }
 
 func (r ResourceDeletedAction) handle(c *CaddyController) error {
+	klog.Info("Ingress resource deletion detected, updating Caddy config...")
+
 	// delete all resources from caddy config that are associated with this resource
 	// reload caddy config
+	ing, ok := r.resource.(*v1beta1.Ingress)
+	if !ok {
+		return fmt.Errorf("ResourceAddedAction: incoming resource is not of type ingress")
+	}
 
-	fmt.Printf("\nDeleted resource:\n +%v\n", r.resource)
+	// add this ingress to the internal store
+	c.resourceStore.PluckIngress(ing)
+
+	err := updateConfig(c)
+	if err != nil {
+		return err
+	}
+
+	klog.Info("Caddy reloaded successfully.")
+	return nil
+}
+
+func updateConfig(c *CaddyController) error {
+	// update internal caddy config with new ingress info
+	serverRoutes, err := caddy.ConvertToCaddyConfig(c.resourceStore.Ingresses)
+	if err != nil {
+		return errors.Wrap(err, "converting ingress resources to caddy config")
+	}
+
+	if c.resourceStore.CaddyConfig != nil {
+		c.resourceStore.CaddyConfig.Modules.HTTP.Servers.Server.Routes = serverRoutes
+	}
+
+	// reload caddy2 config with newConfig
+	err = c.reloadCaddy()
+	if err != nil {
+		return errors.Wrap(err, "caddy config reload")
+	}
 
 	return nil
 }

internal/controller/controller.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"log"
 	"os"
 	"time"
 
@@ -24,6 +23,7 @@ import (
 	"k8s.io/klog"
 
 	// load required caddy plugins
+	_ "bitbucket.org/lightcodelabs/caddy2/modules/caddyhttp"
 	_ "bitbucket.org/lightcodelabs/caddy2/modules/caddyhttp/caddylog"
 	_ "bitbucket.org/lightcodelabs/caddy2/modules/caddyhttp/staticfiles"
 	_ "bitbucket.org/lightcodelabs/proxy"
@@ -36,7 +36,7 @@ var ResourceMap = map[string]run.Object{
 
 const (
 	// how often we should attempt to keep ingress resource's source address in sync
-	syncInterval = time.Second * 10
+	syncInterval = time.Second * 30
 )
 
 // CaddyController represents an caddy ingress controller.
@@ -87,13 +87,6 @@ func NewCaddyController(namespace string, kubeClient *kubernetes.Clientset, reso
 func (c *CaddyController) Shutdown() error {
 	// remove this ingress controller's ip from ingress resources.
 	c.updateIngStatuses([]apiv1.LoadBalancerIngress{apiv1.LoadBalancerIngress{}}, c.resourceStore.Ingresses)
-
-	// shutdownCaddy server gracefully
-	// err := caddy2.StopAdmin()
-	// if err != nil {
-	// 	return err
-	// }
-
 	return nil
 }
 
@@ -102,17 +95,26 @@ func (c *CaddyController) handleErr(err error, action interface{}) {
 	klog.Error(err)
 }
 
-// Run method starts the ingress controller.
-func (c *CaddyController) Run(stopCh chan struct{}) {
+func (c *CaddyController) reloadCaddy() error {
 	j, err := json.Marshal(c.resourceStore.CaddyConfig)
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
-	cfgReader := bytes.NewReader(j)
 
+	cfgReader := bytes.NewReader(j)
 	err = caddy2.Load(cfgReader)
 	if err != nil {
-		log.Fatal(err)
+		return err
+	}
+
+	return nil
+}
+
+// Run method starts the ingress controller.
+func (c *CaddyController) Run(stopCh chan struct{}) {
+	err := c.reloadCaddy()
+	if err != nil {
+		klog.Errorf("initial caddy config load failed, %v", err.Error())
 	}
 
 	defer runtime.HandleCrash()

internal/store/store.go

@@ -36,13 +36,15 @@ func NewStore(kubeClient *kubernetes.Clientset) *Store {
 	return s
 }
 
-// AddIngress adds an ingress to the store
+// AddIngress adds an ingress to the store. It updates the element at the given index if it is unique.
 func (s *Store) AddIngress(ing *v1beta1.Ingress) {
 	isUniq := true
 
-	for _, i := range s.Ingresses {
-		if i.GetUID() == ing.GetUID() {
+	for i := range s.Ingresses {
+		in := s.Ingresses[i]
+		if in.GetUID() == ing.GetUID() {
 			isUniq = false
+			s.Ingresses[i] = ing
 		}
 	}
 
@@ -50,3 +52,25 @@ func (s *Store) AddIngress(ing *v1beta1.Ingress) {
 		s.Ingresses = append(s.Ingresses, ing)
 	}
 }
+
+// PluckIngress removes the ingress passed in as an argument from the stores list of ingresses.
+func (s *Store) PluckIngress(ing *v1beta1.Ingress) {
+	id := ing.GetUID()
+
+	var index int
+	var hasMatch bool
+	for i := range s.Ingresses {
+		if s.Ingresses[i].GetUID() == id {
+			index = i
+			hasMatch = true
+			break
+		}
+	}
+
+	// since order is not important we can swap the element to delete with the one at the end of the slice
+	// and then set ingresses to the n-1 first elements
+	if hasMatch {
+		s.Ingresses[len(s.Ingresses)-1], s.Ingresses[index] = s.Ingresses[index], s.Ingresses[len(s.Ingresses)-1]
+		s.Ingresses = s.Ingresses[:len(s.Ingresses)-1]
+	}
+}

kubernetes/generated/deployment.yaml

@@ -24,7 +24,6 @@ spec:
         release: "release-name"
         heritage: "Tiller"
         version: v0.1.0
-
     spec:
       serviceAccountName: caddyingresscontroller
       containers:

skaffold.yaml

@@ -8,6 +8,8 @@ deploy:
     manifests:
     - hack/test/example-deployment.yaml
     - hack/test/example-ingress.yaml
+    - hack/test/example-deployment2.yaml
+    - hack/test/example-service2.yaml
     - hack/test/example-service.yaml
     - kubernetes/generated/clusterrole.yaml
     - kubernetes/generated/clusterrolebinding.yaml