Implement event creation after terminating a pod

README.md

@@ -203,6 +203,7 @@ Use `UTC`, `Local` or pick a timezone name from the [(IANA) tz database](https:/
 | `--timezone`              | timezone from tz database, e.g. "America/New_York", "UTC" or "Local" | (UTC)                      |
 | `--minimum-age`           | Minimum age to filter pods by                                        | 0s (matches every pod)     |
 | `--dry-run`               | don't kill pods, only log what would have been done                  | true                       |
+| `--create-events`         | If true, create an event in victims namespace after termination      | true                       |
 
 ## Related work
 

chaoskube/chaoskube.go

@@ -14,6 +14,8 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/selection"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/tools/reference"
 
 	"github.com/linki/chaoskube/util"
 )
@@ -42,6 +44,8 @@ type Chaoskube struct {
 	Logger log.FieldLogger
 	// dry run will not allow any pod terminations
 	DryRun bool
+	// create event with deletion message in victims namespace
+	CreateEvent bool
 	// a function to retrieve the current time
 	Now func() time.Time
 }
@@ -66,7 +70,8 @@ var (
 // * a time zone to apply to the aforementioned time-based filters
 // * a logger implementing logrus.FieldLogger to send log output to
 // * whether to enable/disable dry-run mode
-func New(client kubernetes.Interface, labels, annotations, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool) *Chaoskube {
+// * whether to enable/disable event creation
+func New(client kubernetes.Interface, labels, annotations, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool, createEvent bool) *Chaoskube {
 	return &Chaoskube{
 		Client:             client,
 		Labels:             labels,
@@ -79,6 +84,7 @@ func New(client kubernetes.Interface, labels, annotations, namespaces labels.Sel
 		MinimumAge:         minimumAge,
 		Logger:             logger,
 		DryRun:             dryRun,
+		CreateEvent:        createEvent,
 		Now:                time.Now,
 	}
 }
@@ -191,7 +197,43 @@ func (c *Chaoskube) DeletePod(victim v1.Pod) error {
 		return nil
 	}
 
-	return c.Client.CoreV1().Pods(victim.Namespace).Delete(victim.Name, nil)
+	err := c.Client.CoreV1().Pods(victim.Namespace).Delete(victim.Name, nil)
+	if err != nil {
+		return err
+	}
+
+	err = c.CreateDeleteEvent(victim)
+	if err != nil {
+		c.Logger.WithField("err", err).Error("failed to create deletion event")
+	}
+
+	return nil
+}
+
+// CreateDeleteEvent creates an event in victims namespace with an deletion message.
+func (c *Chaoskube) CreateDeleteEvent(victim v1.Pod) error {
+	ref, err := reference.GetReference(scheme.Scheme, victim.DeepCopyObject())
+	if err != nil {
+		return err
+	}
+
+	t := metav1.Time{Time: c.Now()}
+	_, err = c.Client.CoreV1().Events(victim.Namespace).Create(&v1.Event{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fmt.Sprintf("%s.chaos.%x", victim.Name, t.UnixNano()),
+			Namespace: victim.Namespace,
+		},
+		InvolvedObject: *ref,
+		Reason:         "Chaos",
+		Message:        fmt.Sprintf("Deleted pod %s", victim.Name),
+		FirstTimestamp: t,
+		LastTimestamp:  t,
+		Count:          1,
+		Action:         "Deleted",
+		Type:           v1.EventTypeNormal,
+	})
+
+	return err
 }
 
 // filterByNamespaces filters a list of pods by a given namespace selector.

chaoskube/chaoskube_test.go

@@ -57,6 +57,7 @@ func (suite *Suite) TestNew() {
 		minimumAge,
 		logger,
 		false,
+		true,
 	)
 	suite.Require().NotNil(chaoskube)
 
@@ -85,6 +86,7 @@ func (suite *Suite) TestRunContextCanceled() {
 		time.UTC,
 		time.Duration(0),
 		false,
+		true,
 	)
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -134,6 +136,7 @@ func (suite *Suite) TestCandidates() {
 			time.UTC,
 			time.Duration(0),
 			false,
+			true,
 		)
 
 		suite.assertCandidates(chaoskube, tt.pods)
@@ -168,6 +171,7 @@ func (suite *Suite) TestVictim() {
 			time.UTC,
 			time.Duration(0),
 			false,
+			true,
 		)
 
 		suite.assertVictim(chaoskube, tt.victim)
@@ -186,6 +190,7 @@ func (suite *Suite) TestNoVictimReturnsError() {
 		time.UTC,
 		time.Duration(0),
 		false,
+		true,
 	)
 
 	_, err := chaoskube.Victim()
@@ -214,6 +219,7 @@ func (suite *Suite) TestDeletePod() {
 			time.UTC,
 			time.Duration(0),
 			tt.dryRun,
+			true,
 		)
 
 		victim := util.NewPod("default", "foo", v1.PodRunning)
@@ -444,6 +450,7 @@ func (suite *Suite) TestTerminateVictim() {
 			tt.timezone,
 			time.Duration(0),
 			false,
+			true,
 		)
 		chaoskube.Now = tt.now
 
@@ -457,6 +464,34 @@ func (suite *Suite) TestTerminateVictim() {
 	}
 }
 
+func (suite *Suite) TestTerminateVictimCreatesEvent() {
+	chaoskube := suite.setupWithPods(
+		labels.Everything(),
+		labels.Everything(),
+		labels.Everything(),
+		[]time.Weekday{},
+		[]util.TimePeriod{},
+		[]time.Time{},
+		time.UTC,
+		time.Duration(0),
+		false,
+		true,
+	)
+	chaoskube.Now = ThankGodItsFriday{}.Now
+
+	err := chaoskube.TerminateVictim()
+	suite.Require().NoError(err)
+
+	events, err := chaoskube.Client.CoreV1().Events(v1.NamespaceAll).List(metav1.ListOptions{})
+	suite.Require().NoError(err)
+
+	suite.Require().Len(events.Items, 1)
+	event := events.Items[0]
+
+	suite.Equal("foo.chaos.-2be96689beac4e00", event.Name)
+	suite.Equal("Deleted pod foo", event.Message)
+}
+
 // TestTerminateNoVictimLogsInfo tests that missing victim prints a log message
 func (suite *Suite) TestTerminateNoVictimLogsInfo() {
 	chaoskube := suite.setup(
@@ -469,6 +504,7 @@ func (suite *Suite) TestTerminateNoVictimLogsInfo() {
 		time.UTC,
 		time.Duration(0),
 		false,
+		true,
 	)
 
 	err := chaoskube.TerminateVictim()
@@ -517,7 +553,7 @@ func (suite *Suite) assertLog(level log.Level, msg string, fields log.Fields) {
 	}
 }
 
-func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool) *Chaoskube {
+func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, createEvent bool) *Chaoskube {
 	chaoskube := suite.setup(
 		labelSelector,
 		annotations,
@@ -528,6 +564,7 @@ func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations lab
 		timezone,
 		minimumAge,
 		dryRun,
+		createEvent,
 	)
 
 	pods := []v1.Pod{
@@ -544,7 +581,7 @@ func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations lab
 	return chaoskube
 }
 
-func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool) *Chaoskube {
+func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, createEvent bool) *Chaoskube {
 	logOutput.Reset()
 
 	return New(
@@ -559,6 +596,7 @@ func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Sele
 		minimumAge,
 		logger,
 		dryRun,
+		createEvent,
 	)
 }
 
@@ -658,6 +696,7 @@ func (suite *Suite) TestMinimumAge() {
 			time.UTC,
 			tt.minimumAge,
 			false,
+			true,
 		)
 		chaoskube.Now = tt.now
 

examples/rbac.yaml

@@ -6,9 +6,10 @@ rules:
 - apiGroups: [""]
   resources: ["pods"]
   verbs: ["list", "delete"]
-
+- apiGroups: [""]
+  resources: ["events"]
+  verbs: ["create"]
 ---
-
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:

main.go

@@ -40,6 +40,7 @@ var (
 	kubeconfig         string
 	interval           time.Duration
 	dryRun             bool
+	createEvent        bool
 	debug              bool
 	metricsAddress     string
 )
@@ -59,6 +60,7 @@ func init() {
 	kingpin.Flag("kubeconfig", "Path to a kubeconfig file").StringVar(&kubeconfig)
 	kingpin.Flag("interval", "Interval between Pod terminations").Default("10m").DurationVar(&interval)
 	kingpin.Flag("dry-run", "If true, don't actually do anything.").Default("true").BoolVar(&dryRun)
+	kingpin.Flag("create-events", "If true, create an event in victims namespace after termination.").Default("true").BoolVar(&createEvent)
 	kingpin.Flag("debug", "Enable debug logging.").BoolVar(&debug)
 	kingpin.Flag("metrics-address", "Listening address for metrics handler").Default(":8080").StringVar(&metricsAddress)
 }
@@ -86,6 +88,7 @@ func main() {
 		"dryRun":             dryRun,
 		"debug":              debug,
 		"metricsAddress":     metricsAddress,
+		"createEvent":        createEvent,
 	}).Debug("reading config")
 
 	log.WithFields(log.Fields{
@@ -161,6 +164,7 @@ func main() {
 		minimumAge,
 		log.StandardLogger(),
 		dryRun,
+		createEvent,
 	)
 
 	if metricsAddress != "" {

util/util.go

@@ -136,6 +136,7 @@ func NewPod(namespace, name string, phase v1.PodPhase) v1.Pod {
 			Annotations: map[string]string{
 				"chaos": name,
 			},
+			SelfLink: fmt.Sprintf("/api/v1/namespaces/%s/pods/%s", namespace, name),
 		},
 		Status: v1.PodStatus{
 			Phase: phase,