diff --git a/README.md b/README.md index f2e1bab2..9e9e2028 100644 --- a/README.md +++ b/README.md @@ -203,6 +203,7 @@ Use `UTC`, `Local` or pick a timezone name from the [(IANA) tz database](https:/ | `--timezone` | timezone from tz database, e.g. "America/New_York", "UTC" or "Local" | (UTC) | | `--minimum-age` | Minimum age to filter pods by | 0s (matches every pod) | | `--dry-run` | don't kill pods, only log what would have been done | true | +| `--create-events` | If true, create an event in victims namespace after termination | true | ## Related work diff --git a/chaoskube/chaoskube.go b/chaoskube/chaoskube.go index d734f8db..b6783358 100644 --- a/chaoskube/chaoskube.go +++ b/chaoskube/chaoskube.go @@ -14,6 +14,8 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/reference" "github.com/linki/chaoskube/util" ) @@ -42,6 +44,8 @@ type Chaoskube struct { Logger log.FieldLogger // dry run will not allow any pod terminations DryRun bool + // create event with deletion message in victims namespace + CreateEvent bool // a function to retrieve the current time Now func() time.Time } @@ -66,7 +70,8 @@ var ( // * a time zone to apply to the aforementioned time-based filters // * a logger implementing logrus.FieldLogger to send log output to // * whether to enable/disable dry-run mode -func New(client kubernetes.Interface, labels, annotations, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool) *Chaoskube { +// * whether to enable/disable event creation +func New(client kubernetes.Interface, labels, annotations, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, logger log.FieldLogger, dryRun bool, createEvent bool) *Chaoskube { return &Chaoskube{ Client: client, Labels: labels, @@ -79,6 +84,7 @@ func New(client kubernetes.Interface, labels, annotations, namespaces labels.Sel MinimumAge: minimumAge, Logger: logger, DryRun: dryRun, + CreateEvent: createEvent, Now: time.Now, } } @@ -191,7 +197,43 @@ func (c *Chaoskube) DeletePod(victim v1.Pod) error { return nil } - return c.Client.CoreV1().Pods(victim.Namespace).Delete(victim.Name, nil) + err := c.Client.CoreV1().Pods(victim.Namespace).Delete(victim.Name, nil) + if err != nil { + return err + } + + err = c.CreateDeleteEvent(victim) + if err != nil { + c.Logger.WithField("err", err).Error("failed to create deletion event") + } + + return nil +} + +// CreateDeleteEvent creates an event in victims namespace with an deletion message. +func (c *Chaoskube) CreateDeleteEvent(victim v1.Pod) error { + ref, err := reference.GetReference(scheme.Scheme, victim.DeepCopyObject()) + if err != nil { + return err + } + + t := metav1.Time{Time: c.Now()} + _, err = c.Client.CoreV1().Events(victim.Namespace).Create(&v1.Event{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s.chaos.%x", victim.Name, t.UnixNano()), + Namespace: victim.Namespace, + }, + InvolvedObject: *ref, + Reason: "Chaos", + Message: fmt.Sprintf("Deleted pod %s", victim.Name), + FirstTimestamp: t, + LastTimestamp: t, + Count: 1, + Action: "Deleted", + Type: v1.EventTypeNormal, + }) + + return err } // filterByNamespaces filters a list of pods by a given namespace selector. diff --git a/chaoskube/chaoskube_test.go b/chaoskube/chaoskube_test.go index 6566f5f0..a26bb25d 100644 --- a/chaoskube/chaoskube_test.go +++ b/chaoskube/chaoskube_test.go @@ -57,6 +57,7 @@ func (suite *Suite) TestNew() { minimumAge, logger, false, + true, ) suite.Require().NotNil(chaoskube) @@ -85,6 +86,7 @@ func (suite *Suite) TestRunContextCanceled() { time.UTC, time.Duration(0), false, + true, ) ctx, cancel := context.WithCancel(context.Background()) @@ -134,6 +136,7 @@ func (suite *Suite) TestCandidates() { time.UTC, time.Duration(0), false, + true, ) suite.assertCandidates(chaoskube, tt.pods) @@ -168,6 +171,7 @@ func (suite *Suite) TestVictim() { time.UTC, time.Duration(0), false, + true, ) suite.assertVictim(chaoskube, tt.victim) @@ -186,6 +190,7 @@ func (suite *Suite) TestNoVictimReturnsError() { time.UTC, time.Duration(0), false, + true, ) _, err := chaoskube.Victim() @@ -214,6 +219,7 @@ func (suite *Suite) TestDeletePod() { time.UTC, time.Duration(0), tt.dryRun, + true, ) victim := util.NewPod("default", "foo", v1.PodRunning) @@ -444,6 +450,7 @@ func (suite *Suite) TestTerminateVictim() { tt.timezone, time.Duration(0), false, + true, ) chaoskube.Now = tt.now @@ -457,6 +464,34 @@ func (suite *Suite) TestTerminateVictim() { } } +func (suite *Suite) TestTerminateVictimCreatesEvent() { + chaoskube := suite.setupWithPods( + labels.Everything(), + labels.Everything(), + labels.Everything(), + []time.Weekday{}, + []util.TimePeriod{}, + []time.Time{}, + time.UTC, + time.Duration(0), + false, + true, + ) + chaoskube.Now = ThankGodItsFriday{}.Now + + err := chaoskube.TerminateVictim() + suite.Require().NoError(err) + + events, err := chaoskube.Client.CoreV1().Events(v1.NamespaceAll).List(metav1.ListOptions{}) + suite.Require().NoError(err) + + suite.Require().Len(events.Items, 1) + event := events.Items[0] + + suite.Equal("foo.chaos.-2be96689beac4e00", event.Name) + suite.Equal("Deleted pod foo", event.Message) +} + // TestTerminateNoVictimLogsInfo tests that missing victim prints a log message func (suite *Suite) TestTerminateNoVictimLogsInfo() { chaoskube := suite.setup( @@ -469,6 +504,7 @@ func (suite *Suite) TestTerminateNoVictimLogsInfo() { time.UTC, time.Duration(0), false, + true, ) err := chaoskube.TerminateVictim() @@ -517,7 +553,7 @@ func (suite *Suite) assertLog(level log.Level, msg string, fields log.Fields) { } } -func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool) *Chaoskube { +func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, createEvent bool) *Chaoskube { chaoskube := suite.setup( labelSelector, annotations, @@ -528,6 +564,7 @@ func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations lab timezone, minimumAge, dryRun, + createEvent, ) pods := []v1.Pod{ @@ -544,7 +581,7 @@ func (suite *Suite) setupWithPods(labelSelector labels.Selector, annotations lab return chaoskube } -func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool) *Chaoskube { +func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Selector, namespaces labels.Selector, excludedWeekdays []time.Weekday, excludedTimesOfDay []util.TimePeriod, excludedDaysOfYear []time.Time, timezone *time.Location, minimumAge time.Duration, dryRun bool, createEvent bool) *Chaoskube { logOutput.Reset() return New( @@ -559,6 +596,7 @@ func (suite *Suite) setup(labelSelector labels.Selector, annotations labels.Sele minimumAge, logger, dryRun, + createEvent, ) } @@ -658,6 +696,7 @@ func (suite *Suite) TestMinimumAge() { time.UTC, tt.minimumAge, false, + true, ) chaoskube.Now = tt.now diff --git a/examples/rbac.yaml b/examples/rbac.yaml index 74c7997a..736f40c5 100644 --- a/examples/rbac.yaml +++ b/examples/rbac.yaml @@ -6,9 +6,10 @@ rules: - apiGroups: [""] resources: ["pods"] verbs: ["list", "delete"] - +- apiGroups: [""] + resources: ["events"] + verbs: ["create"] --- - apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: diff --git a/main.go b/main.go index 1a06581f..2f321f4e 100644 --- a/main.go +++ b/main.go @@ -40,6 +40,7 @@ var ( kubeconfig string interval time.Duration dryRun bool + createEvent bool debug bool metricsAddress string ) @@ -59,6 +60,7 @@ func init() { kingpin.Flag("kubeconfig", "Path to a kubeconfig file").StringVar(&kubeconfig) kingpin.Flag("interval", "Interval between Pod terminations").Default("10m").DurationVar(&interval) kingpin.Flag("dry-run", "If true, don't actually do anything.").Default("true").BoolVar(&dryRun) + kingpin.Flag("create-events", "If true, create an event in victims namespace after termination.").Default("true").BoolVar(&createEvent) kingpin.Flag("debug", "Enable debug logging.").BoolVar(&debug) kingpin.Flag("metrics-address", "Listening address for metrics handler").Default(":8080").StringVar(&metricsAddress) } @@ -86,6 +88,7 @@ func main() { "dryRun": dryRun, "debug": debug, "metricsAddress": metricsAddress, + "createEvent": createEvent, }).Debug("reading config") log.WithFields(log.Fields{ @@ -161,6 +164,7 @@ func main() { minimumAge, log.StandardLogger(), dryRun, + createEvent, ) if metricsAddress != "" { diff --git a/util/util.go b/util/util.go index a40b128f..6c58a528 100644 --- a/util/util.go +++ b/util/util.go @@ -136,6 +136,7 @@ func NewPod(namespace, name string, phase v1.PodPhase) v1.Pod { Annotations: map[string]string{ "chaos": name, }, + SelfLink: fmt.Sprintf("/api/v1/namespaces/%s/pods/%s", namespace, name), }, Status: v1.PodStatus{ Phase: phase,