From 78ec26ed3d4ead2cb36a7a078edb38d41f3729c9 Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Tue, 22 Jan 2019 08:54:16 -0800 Subject: [PATCH 1/2] Remove support for old falco integration Remove support for endpoints that managed the pre-secure integration of falco rules into the agent. This was removed from the agent in 0.70.0. --- sdcclient/_client.py | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/sdcclient/_client.py b/sdcclient/_client.py index fbe7bfed..fba9dc3d 100644 --- a/sdcclient/_client.py +++ b/sdcclient/_client.py @@ -1769,32 +1769,6 @@ def get_metrics(self): return [False, self.lasterr] return [True, res.json()] - def get_falco_rules(self): - res = requests.get(self.url + '/api/agents/falco_rules', headers=self.hdrs, verify=self.ssl_verify) - if not self._checkResponse(res): - return [False, self.lasterr] - data = res.json() - return [True, data] - - def set_falco_rules_content_raw(self, raw_payload): - res = requests.put(self.url + '/api/agents/falco_rules', headers=self.hdrs, data=json.dumps(raw_payload), verify=self.ssl_verify) - if not self._checkResponse(res): - return [False, self.lasterr] - return [True, res.json()] - - def set_falco_rules_content(self, filter, rules_content): - payload = { "files" : [ { "filter": filter, "content": rules_content} ] } - return self.set_falco_rules_content_raw(payload) - - def set_falco_rules_filename(self, filter, rules_filename): - with open(rules_filename, 'r') as f: - rules_content = f.read() - return self.set_falco_rules_content(filter, rules_content) - - def clear_falco_rules(self): - data = {'files' : []} - return self.set_falco_rules_content_raw(data) - # For backwards compatibility SdcClient = SdMonitorClient From ce6923bec05bdf456456d796b3aac5247b9c94c0 Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Thu, 24 Jan 2019 17:56:17 -0800 Subject: [PATCH 2/2] Methods/examples to set/get falco rules files Add the following methods to the python client: - {get,set}_default_falco_rules_files: a wrapper around the api endpoint /api/settings/defaultRulesFiles, handling PUT and GET. - load_default_falco_rules_files: load a collection of files on disk with a documented structure, returning a dict suitable for use in set_default_falco_rules_files() - save_default_falco_rules_files: given a dict from get_default_falco_rules_files, save it to a collection of files on disk with a documented structure Also add example programs {set,get}_secure_default_falco_rules.py. get_... has the ability to either print the returned set of files directly or save them using a --save option. Similarly, set_ has a --load option to load files from disk to a dict for the PUT /api/settings/defaultRulesFiles. set_ also has some easier-to-use command line options that allow setting a single file and tag. --- .../get_secure_default_falco_rules_files.py | 70 ++++++ .../set_secure_default_falco_rules_files.py | 104 ++++++++ sdcclient/_client.py | 235 ++++++++++++++++++ 3 files changed, 409 insertions(+) create mode 100755 examples/get_secure_default_falco_rules_files.py create mode 100755 examples/set_secure_default_falco_rules_files.py diff --git a/examples/get_secure_default_falco_rules_files.py b/examples/get_secure_default_falco_rules_files.py new file mode 100755 index 00000000..ea3d61d1 --- /dev/null +++ b/examples/get_secure_default_falco_rules_files.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# +# Get the sysdig secure default rules files. +# +# The _files programs and endpoints are a replacement for the +# system_file endpoints and allow for publishing multiple files +# instead of a single file as well as publishing multiple variants of +# a given file that are compatible with different agent versions. +# + +import os +import sys +import pprint +import getopt +import shutil +sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), '..')) +from sdcclient import SdSecureClient + +# +# Parse arguments +# +def usage(): + print 'usage: %s [-s|--save ] ' % sys.argv[0] + print '-s|--save: save the retrieved files to a set of files below using save_default_rules_files().' + print 'You can find your token at https://secure.sysdig.com/#/settings/user' + sys.exit(1) + +try: + opts, args = getopt.getopt(sys.argv[1:],"s:",["save="]) +except getopt.GetoptError: + usage() + +save_dir = "" +for opt, arg in opts: + if opt in ("-s", "--save"): + save_dir = arg + +# +# Parse arguments +# +if len(args) != 1: + usage() + +sdc_token = args[0] + +# +# Instantiate the SDC client +# +sdclient = SdSecureClient(sdc_token, 'https://secure.sysdig.com') + +# +# Get the configuration +# +res = sdclient.get_default_falco_rules_files() + +# +# Return the result +# +if res[0]: + if save_dir == "": + pp = pprint.PrettyPrinter(indent=4) + pp.pprint(res[1]) + else: + print "Saving falco rules files below {}...".format(save_dir) + sres = sdclient.save_default_falco_rules_files(res[1], save_dir) + if not sres[0]: + print sres[1] +else: + print res[1] + sys.exit(1) diff --git a/examples/set_secure_default_falco_rules_files.py b/examples/set_secure_default_falco_rules_files.py new file mode 100755 index 00000000..aed32198 --- /dev/null +++ b/examples/set_secure_default_falco_rules_files.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python +# +# Set the sysdig secure default rules files. +# +# The _files programs and endpoints are a replacement for the +# system_file endpoints and allow for publishing multiple files +# instead of a single file as well as publishing multiple variants of +# a given file that are compatible with different agent versions. +# + +import os +import sys +import pprint +import getopt +import shutil +import yaml +sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), '..')) +from sdcclient import SdSecureClient + +# +# Parse arguments +# +def usage(): + print 'usage: %s [-l|--load ] [-t|--tag ] [-c|--content ] ' % sys.argv[0] + print '-l|--load: load the files to set from a set of files below using load_default_rules_files().' + print '-t|--tag: Set a tag for the set of files' + print '-c|--content: the (single) file to set' + print 'if --load is specified, neither --tag nor --content can be specified' + print 'You can find your token at https://secure.sysdig.com/#/settings/user' + sys.exit(1) + +try: + opts, args = getopt.getopt(sys.argv[1:],"l:t:n:c:",["load=","tag=","name=","content="]) +except getopt.GetoptError: + usage() + +load_dir = "" +tag = "" +cpath = "" +for opt, arg in opts: + if opt in ("-l", "--load"): + load_dir = arg + elif opt in ("-t", "--tag"): + tag = arg + elif opt in ("-c", "--content"): + cpath = arg + +if load_dir != "" and (tag != "" or cpath != ""): + usage() +# +# Parse arguments +# +if len(args) != 1: + usage() + +sdc_token = args[0] + +# +# Instantiate the SDC client +# +sdclient = SdSecureClient(sdc_token, 'https://secure.sysdig.com') + +files_obj = {} +if load_dir != "": + print "Loading falco rules files from {}...".format(load_dir) + res = sdclient.load_default_falco_rules_files(load_dir) + if res[0]: + files_obj = res[1] + else: + print res[1] + sys.exit(1) +else: + with open(cpath, 'r') as content_file: + content = content_file.read() + required_engine_version = 0 + cyaml = yaml.load(content) + for obj in cyaml: + if "required_engine_version" in obj: + try: + required_engine_version = int(obj["required_engine_version"]) + except ValueError: + print("Required engine version \"{}\" in content {} must be a number".format(obj["required_engine_version"], cpath)) + sys.exit(1) + files_obj = { + "tag": tag, + "files": [{ + "name": os.path.basename(cpath), + "variants": { + "required_engine_version": required_engine_version, + "content": content + } + }] + } + +res = sdclient.set_default_falco_rules_files(files_obj) + +# +# Return the result +# +if res[0]: + print 'default falco rules files set successfully' +else: + print res[1] + sys.exit(1) diff --git a/sdcclient/_client.py b/sdcclient/_client.py index fba9dc3d..e4c0a3f7 100644 --- a/sdcclient/_client.py +++ b/sdcclient/_client.py @@ -3,6 +3,7 @@ import requests import copy import datetime +import shutil class _SdcCommon(object): '''Interact with the Sysdig Monitor/Secure API. @@ -1863,6 +1864,240 @@ def set_user_falco_rules(self, rules_content): ''' return self._set_falco_rules("user", rules_content) + # Only one kind for now called "default", but might add a "custom" kind later. + def _get_falco_rules_files(self, kind): + + res = requests.get(self.url + '/api/settings/falco/{}RulesFiles'.format(kind), headers=self.hdrs, verify=self.ssl_verify) + if not self._checkResponse(res): + return [False, self.lasterr] + data = res.json() + + return [True, data] + + def get_default_falco_rules_files(self): + '''**Description** + Get the set of falco rules files from the backend. The _files programs and endpoints are a + replacement for the system_file endpoints and allow for publishing multiple files instead + of a single file as well as publishing multiple variants of a given file that are compatible + with different agent versions. + + **Arguments** + - None + + **Success Return Value** + A dict with the following keys: + - tag: A string used to uniquely identify this set of rules. It is recommended that this tag change every time the set of rules is updated. + - files: An array of dicts. Each dict has the following keys: + - name: the name of the file + - variants: An array of dicts with the following keys: + - requiredEngineVersion: the minimum falco engine version that can read this file + - content: the falco rules content + An example would be: + {'tag': 'v1.5.9', + 'files': [ + { + 'name': 'falco_rules.yaml', + 'variants': [ + { + 'content': '- required_engine_version: 29\n\n- list: foo\n', + 'requiredEngineVersion': 29 + }, + { + 'content': '- required_engine_version: 1\n\n- list: foo\n', + 'requiredEngineVersion': 1 + } + ] + }, + { + 'name': 'k8s_audit_rules.yaml', + 'variants': [ + { + 'content': '# some comment\n', + 'requiredEngineVersion': 0 + } + ] + } + ] + } + + **Example** + `examples/get_default_falco_rules_files.py `_ + ''' + + res = self._get_falco_rules_files("default") + + if not res[0]: + return res + else: + res_obj = res[1]["defaultFalcoRulesFiles"] + + # Copy only the tag and files over + ret = {} + + if "tag" in res_obj: + ret["tag"] = res_obj["tag"] + + if "files" in res_obj: + ret["files"] = res_obj["files"] + + return [True, ret] + + def save_default_falco_rules_files(self, fsobj, save_dir): + '''**Description** + Given a dict returned from get_default_falco_rules_files, save those files to a set of files below save_dir. + The first level below save_dir is a directory with the tag name. The second level is a directory per file. + The third level is a directory per variant. Finally the files are at the lowest level, in a file called "content". + For example, using the example dict in get_default_falco_rules_files(), the directory layout would look like: + save_dir/ + v1.5.9/ + falco_rules.yaml/ + 29/ + content: a file containing "- required_engine_version: 29\n\n- list: foo\n" + 1/ + content: a file containing "- required_engine_version: 1\n\n- list: foo\n" + k8s_audit_rules.yaml/ + 0/ + content: a file containing "# some comment" + **Arguments** + - fsobj: a python dict matching the structure returned by get_default_falco_rules_files() + - save_dir: a directory path under which to save the files. If the path already exists, it will be removed first. + + **Success Return Value** + - None + + **Example** + `examples/get_default_falco_rules_files.py `_ + ''' + if os.path.exists(save_dir): + try: + if os.path.isdir(save_dir): + shutil.rmtree(save_dir) + else: + os.unlink(save_dir) + except Exception as e: + return [False, "Could not remove existing save dir {}: {}".format(save_dir, str(e))] + + prefix = os.path.join(save_dir, fsobj["tag"]) + try: + os.makedirs(prefix) + except Exception as e: + return [False, "Could not create tag directory {}: {}".format(prefix, str(e))] + + if "files" in fsobj: + for fobj in fsobj["files"]: + fprefix = os.path.join(prefix, fobj["name"]) + try: + os.makedirs(fprefix) + except Exception as e: + return [False, "Could not create file directory {}: {}".format(fprefix, str(e))] + for variant in fobj["variants"]: + vprefix = os.path.join(fprefix, str(variant["requiredEngineVersion"])) + try: + os.makedirs(vprefix) + except Exception as e: + return [False, "Could not create variant directory {}: {}".format(vprefix, str(e))] + cpath = os.path.join(vprefix, "content") + try: + with open(cpath, "w") as cfile: + cfile.write(variant["content"]) + except Exception as e: + return [False, "Could not write content to {}: {}".format(cfile, str(e))] + + return [True, None] + + # Only One kind for now, but might add a "custom" kind later. + def _set_falco_rules_files(self, kind, rules_files): + + payload = self._get_falco_rules_files(kind) + + if not payload[0]: + return payload + + obj = payload[1]["{}FalcoRulesFiles".format(kind)] # pylint: disable=unsubscriptable-object + + obj["tag"] = rules_files["tag"] + obj["files"] = rules_files["files"] + + res = requests.put(self.url + '/api/settings/falco/{}RulesFiles'.format(kind), headers=self.hdrs, data=json.dumps(payload[1]), verify=self.ssl_verify) + if not self._checkResponse(res): + return [False, self.lasterr] + return [True, res.json()] + + def set_default_falco_rules_files(self, rules_files): + '''**Description** + Update the set of falco rules files to the provided set of files. See the `Falco wiki `_ for documentation on the falco rules format. + The _files programs and endpoints are a replacement for the system_file endpoints and + allow for publishing multiple files instead of a single file as well as publishing + multiple variants of a given file that are compatible with different agent versions. + + **Arguments** + - rules_files: a dict with the same structure as returned by get_default_falco_rules_files. + + **Success Return Value** + The contents of the default falco rules files that were just updated. + + **Example** + `examples/set_default_falco_rules_files.py `_ + + ''' + + return self._set_falco_rules_files("default", rules_files) + + def load_default_falco_rules_files(self, save_dir): + '''**Description** + Given a file and directory layout as described in save_default_falco_rules_files(), load those files and + return a dict representing the contents. This dict is suitable for passing to set_default_falco_rules_files(). + + **Arguments** + - save_dir: a directory path from which to load the files. + + **Success Return Value** + - A dict matching the format described in get_default_falco_rules_files. + + **Example** + `examples/set_default_falco_rules_files.py `_ + ''' + + tags = os.listdir(save_dir) + if len(tags) != 1: + return [False, "Directory {} did not contain exactly 1 entry".format(save_dir)] + + tpath = os.path.join(save_dir, tags[0]) + + if not os.path.isdir(tpath): + return [False, "Tag path {} is not a directory".format(tpath)] + + ret = {"tag": os.path.basename(tpath), "files": []} + + for fdir in os.listdir(tpath): + fpath = os.path.join(tpath, fdir) + if not os.path.isdir(fpath): + return [False, "File path {} is not a directory".format(fpath)] + fobj = {"name": os.path.basename(fpath), "variants": []} + for vdir in os.listdir(fpath): + vpath = os.path.join(fpath, vdir) + if not os.path.isdir(vpath): + return [False, "Variant path {} is not a directory".format(vpath)] + cpath = os.path.join(vpath, "content") + try: + with open(cpath, 'r') as content_file: + try: + required_engine_version = int(os.path.basename(vpath)) + if vpath < 0: + return [False, "Variant directory {} must be a positive number".format(vpath)] + fobj["variants"].append({ + "requiredEngineVersion": required_engine_version, + "content": content_file.read() + }) + except ValueError: + return [False, "Variant directory {} must be a number".format(vpath)] + except Exception as e: + return [False, "Could not read content at {}: {}".format(cpath, str(e))] + + ret["files"].append(fobj) + + return [True, ret] + def _get_policy_events_int(self, ctx): policy_events_url = self.url + '/api/policyEvents?from={:d}&to={:d}&offset={}&limit={}'.format(int(ctx['from']), int(ctx['to']), ctx['offset'], ctx['limit']) if 'sampling' in ctx: