Rewrite MISP plugin to convert to/from STIX-2 Indicators and Sightings

CHANGELOG.md

@@ -10,22 +10,37 @@ Every entry has a category for which we use the following visual abbreviations:
 - 🧬 Experimental feature
 - 🐞 Bugfix
 
-<!-- ## Unreleased -->
+## Unreleased
+
+- 🎁 The MISP plugin now supports the
+  [STIX-2 (version 2.1)](https://docs.oasis-open.org/cti/stix/v2.1/stix-v2.1.html)
+  standard for
+  [Indicators](https://docs.oasis-open.org/cti/stix/v2.1/cs02/stix-v2.1-cs02.html#_muftrcpnf89v)
+  and [Sightings](https://docs.oasis-open.org/cti/stix/v2.1/cs02/stix-v2.1-cs02.html#_a795guqsap3r).
+  The plugin converts MISP attributes to valid STIX-2 Indicators on best-effort
+  basis before publishing them on Threat Bus topics. Likewise, the plugin
+  converts STIX-2 sightings to MISP sightings before sending them the MISP.
+  [#102](https://github.com/tenzir/threatbus/pull/102)
+
+- 🐞 We fixed a bug in the JSON (de-)serialization logic for `SnapshotEnvelope`s
+  and `SnapshotRequest`s that lead to a malformed `type` field in the JSON
+  representations of both types.
+  [#102](https://github.com/tenzir/threatbus/pull/102)
+
 
 ## [2021.02.24]
 
-- 🎁 Feature
-  The MISP plugin now uses [extra dependencies](https://www.python.org/dev/peps/pep-0508/#extras).
+- 🎁 The MISP plugin now uses
+  [extra dependencies](https://www.python.org/dev/peps/pep-0508/#extras).
   Users can now chose the wanted dependencies during installation by running
   `pip install threatbus-misp[zmq]` to install the ZeroMQ dependency, or
   `pip install threatbus-misp[kafka]` to install the Kafka dependency. The
   plugin throws a fatal error if none of these dependencies is installed and
   exits immediately.
   [#99](https://github.com/tenzir/threatbus/pull/99)
 
-- 🎁 Feature
-  The RabbitMQ backbone plugin, the In-memory backbone plugins, and the Zmq-app
-  plugin now support the
+- 🎁 The RabbitMQ backbone plugin, the In-memory backbone plugins, and the
+  Zmq-app plugin now support the
   [STIX-2 (version 2.1)](https://docs.oasis-open.org/cti/stix/v2.1/stix-v2.1.html)
   standard for
   [Indicators](https://docs.oasis-open.org/cti/stix/v2.1/cs02/stix-v2.1-cs02.html#_muftrcpnf89v)

Makefile

@@ -21,11 +21,11 @@ unit-tests:
 	$(MAKE) -C plugins/backbones/threatbus_inmem unit-tests
 	$(MAKE) -C plugins/backbones/threatbus_rabbitmq unit-tests
 	$(MAKE) -C plugins/apps/threatbus_zmq_app unit-tests
+	$(MAKE) -C plugins/apps/threatbus_misp unit-tests
   # Threat Bus is currently being migrated to use STIX-2 as internal format.
   # For the time being, all un-migrated plugins cannot be not tested against the
   # current master
 	#$(MAKE) -C plugins/apps/threatbus_zeek unit-tests
-	#$(MAKE) -C plugins/apps/threatbus_misp unit-tests
 	#$(MAKE) -C plugins/apps/threatbus_cif3 unit-tests
 	#$(MAKE) -C apps/vast unit-tests
 
@@ -34,10 +34,11 @@ integration-tests:
 	-docker kill rabbit-int > /dev/null 2>&1
 	docker pull rabbitmq$(:)3 > /dev/null 2>&1
 	docker run -d --rm --hostname=test-rabbit --name=rabbit-int -p 35672$(:)5672 rabbitmq$(:)3 > /dev/null 2>&1
-	-python -m unittest tests/integration/test_message_roundtrips.py
-	-python -m unittest tests/integration/test_zeek_app.py
-	-python -m unittest tests/integration/test_rabbitmq.py
+	-python -m unittest tests/integration/test_misp_inmem.py
 	-python -m unittest tests/integration/test_zmq_app_management.py
+	# -python -m unittest tests/integration/test_rabbitmq.py
+	# -python -m unittest tests/integration/test_zeek_app.py
+	# -python -m unittest tests/integration/test_message_roundtrips.py
 	-${RM} {broker,intel,reporter,weird}.log
 	docker kill rabbit-int > /dev/null 2>&1
 
@@ -91,10 +92,10 @@ dev-mode:
 	$(MAKE) -C plugins/apps/threatbus_zmq_app dev-mode
 	$(MAKE) -C plugins/backbones/threatbus_inmem dev-mode
 	$(MAKE) -C plugins/backbones/threatbus_rabbitmq dev-mode
+	$(MAKE) -C plugins/apps/threatbus_misp dev-mode
   # Threat Bus is currently being migrated to use STIX-2 as internal format.
   # For the time being, all un-migrated plugins cannot be run in conjunction
   # with current master
 	# $(MAKE) -C plugins/apps/threatbus_zeek dev-mode
-	# $(MAKE) -C plugins/apps/threatbus_misp dev-mode
 	# $(MAKE) -C plugins/apps/threatbus_cif3 dev-mode
 	# $(MAKE) -C apps/vast dev-mode

plugins/apps/threatbus_misp/setup.py

@@ -26,8 +26,9 @@
     description="A plugin to enable threatbus communication with MISP.",
     entry_points={"threatbus.app": ["misp = threatbus_misp.plugin"]},
     install_requires=[
-        "threatbus >= 2020.12.16, < 2021.2.24",
         "pymisp >= 2.4.120",
+        "stix2 >= 2.1",
+        "threatbus >= 2021.2.24",
     ],
     extras_require={"kafka": ["confluent-kafka>=1.3.0"], "zmq": ["pyzmq>=18.1.1"]},
     keywords=[