docs: integrate onvif schemas and validation matrix to swagger

doc/ONVIF-protocol.md

@@ -251,13 +251,13 @@ The following table shows the Onvif functions tested for various Onvif cameras:
 |                   | SetNetworkDefaultGateway | ✔                   | ❌         | ✔                                 | ✔                   |                     |
 
 ### System Function
-| Onvif Web Service | Onvif Function          | Hikvision DFI6256TE | Tapo C200 | BOSCH DINION IP starlight 6000 HD | GeoVision GV-BX8700 | Happytime Simulator |
-|-------------------|-------------------------|---------------------|-----------|-----------------------------------|---------------------|---------------------|
-| **Device**        | GetDeviceInformation    | ✔                   | ✔         | ✔                                 | ✔                   |                     |
-|                   | GetSystemDateAndTime    | ✔                   | ✔         | ✔                                 | ✔                   |                     |
-|                   | SetSystemDateAndTime    | ✔                   | ✔ [ⓘ](onvif-footnotes.md#tapo-c200---setsystemdateandtime)         | ✔                                 | ✔                   |                     |
-|                   | SetSystemFactoryDefault | ✔                   | ✔         | ✔                                 | ✔                   |                     |
-|                   | Reboot                  | ✔                   | ✔         | ✔                                 | ✔                   |                     |
+| Onvif Web Service | Onvif Function          | Hikvision DFI6256TE | Tapo C200                                                  | BOSCH DINION IP starlight 6000 HD | GeoVision GV-BX8700 | Happytime Simulator |
+|-------------------|-------------------------|---------------------|------------------------------------------------------------|-----------------------------------|---------------------|---------------------|
+| **Device**        | GetDeviceInformation    | ✔                   | ✔                                                          | ✔                                 | ✔                   |                     |
+|                   | GetSystemDateAndTime    | ✔                   | ✔                                                          | ✔                                 | ✔                   |                     |
+|                   | SetSystemDateAndTime    | ✔                   | ✔ [ⓘ](onvif-footnotes.md#tapo-c200---setsystemdateandtime) | ✔                                 | ✔                   |                     |
+|                   | SetSystemFactoryDefault | ✔                   | ✔                                                          | ✔                                 | ✔                   |                     |
+|                   | Reboot                  | ✔                   | ✔                                                          | ✔                                 | ✔                   |                     |
 
 ### User Handling
 | Onvif Web Service | Onvif Function | Hikvision DFI6256TE | Tapo C200                                             | BOSCH DINION IP starlight 6000 HD | GeoVision GV-BX8700 | Happytime Simulator |

doc/openapi/.gitignore

@@ -0,0 +1,5 @@
+tmp/
+ref/out/
+schemas/_extracted.json
+openapi.json
+python/__pycache__/

doc/openapi/Makefile

@@ -1,43 +1,40 @@
+# Copyright (C) 2022 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
 
-.PHONY: gen install schema-gen
+.PHONY: gen install clean
+
+SHELL := /bin/bash
 
 POSTMAN_COLLECTION = ../postman/device-onvif-camera.postman_collection.json
 OPENAPI_FILE = v2/device-onvif-camera.yaml
-OPENAPI_TEMP_FILE = ./tmp/device-onvif-camera.yaml
+OPENAPI_TEMP_FILE = tmp/device-onvif-camera.yaml
 POSTMAN_2_OPENAPI_OPTIONS = p2o-options.json
 SIDECAR_FILE = sidecar.yaml
 PROFILE_FILE = ../../cmd/res/profiles/camera.yaml
+ONVIF_TESTED_FILE = ../ONVIF-protocol.md
+ONVIF_FOOTNOTES_FILE = ../onvif-footnotes.md
 
-SCHEMA_POSTMAN = ./device-onvif-camera.postman_collection.json
-SCHEMA_TEMP = ../postman/openapi.json
-OPENAPI_SCHEMA_TEMP_FILE = ./tmp/device-onvif-camera.schema.yaml
-EXTRACTED_SCHEMA = ../postman/schemas/_extracted.json
+# list of all services in the ./ref folder
+SERVICES = analytics device event imaging media media2 ptz
 
 # todo: replace the API version number based on the actual source code version
 
-$(OPENAPI_FILE) gen: $(SIDECAR_FILE) $(OPENAPI_SCHEMA_TEMP_FILE) $(PROFILE_FILE) python/postprocess.py
-	python3 python/postprocess.py $(OPENAPI_SCHEMA_TEMP_FILE) $(SIDECAR_FILE) $(PROFILE_FILE) $(OPENAPI_FILE)
-	rm -rf ./tmp
-
+$(OPENAPI_FILE) gen: $(SIDECAR_FILE) $(OPENAPI_TEMP_FILE) $(PROFILE_FILE) $(SERVICES:%=ref/out/%.yaml) $(ONVIF_TESTED_FILE) $(ONVIF_FOOTNOTES_FILE) python/postprocess.py python/matrix.py python/cleaner.py Makefile
+	python3 python/postprocess.py $(OPENAPI_TEMP_FILE) $(SIDECAR_FILE) $(PROFILE_FILE) $(OPENAPI_FILE) $(ONVIF_TESTED_FILE) $(ONVIF_FOOTNOTES_FILE)
 
 $(OPENAPI_TEMP_FILE): $(POSTMAN_COLLECTION) $(POSTMAN_2_OPENAPI_OPTIONS) | ./tmp
-	p2o $(POSTMAN_COLLECTION) -f $(OPENAPI_TEMP_FILE) -o $(POSTMAN_2_OPENAPI_OPTIONS)
+	p2o $(POSTMAN_COLLECTION) -f $(OPENAPI_TEMP_FILE) -o $(POSTMAN_2_OPENAPI_OPTIONS) > /dev/null
 
-./tmp:
+./tmp ./ref/out:
 	mkdir -p $@
 
 install:
 	npm install -g postman-to-openapi
-	npm install -g postman-to-openapi-json-extended
 	pip3 install -r python/requirements.txt
-	snap install yq
 
-$(SCHEMA_TEMP) schema-gen:
-	cd ../postman && \
-	p2ojx extract $(SCHEMA_POSTMAN) && \
-	p2ojx build
-	rm -f $(EXTRACTED_SCHEMA)
+# generate the ref/out/<service>.yaml files from the ref/<service>.yaml files
+$(SERVICES:%=ref/out/%.yaml) : ref/out/%.yaml : ref/%.yaml python/xmlstrip.py | ./ref/out
+	python/xmlstrip.py "$*" "ref/$*.yaml" "ref/out/$*.yaml"
 
-$(OPENAPI_SCHEMA_TEMP_FILE): $(SCHEMA_TEMP) | ./tmp
-	yq . -o=yaml $(SCHEMA_TEMP) -P > $(OPENAPI_SCHEMA_TEMP_FILE)
-	rm -f $(SCHEMA_TEMP)
+clean:
+	rm -rf ref/out tmp $(SCHEMA_TEMP_JSON) $(EXTRACTED_SCHEMA)

doc/openapi/README.md

@@ -1,8 +1,35 @@
-This code generates OpenAPI spec based on the Postman Collection 
+# OpenAPI / Swagger Spec Files
+This code generates OpenAPI 3.0 spec based on the Postman Collection 
 [device-onvif-camera.postman_collection.json](../postman/device-onvif-camera.postman_collection.json). 
 It also does some automated find and replace to insert some missing example values.
 
+> **TODO:** update usage
+
 Usage:
 - Install `postman-to-openapi` by running `make install` from this directory.
 - Update the latest postman collection [device-onvif-camera.postman_collection.json](../postman/device-onvif-camera.postman_collection.json)
 - Run `make gen` to re-generate the OpenAPI files.
+
+## [python](python) folder
+Contains scripts for processing the input and output files when
+generating the OpenAPI spec file.
+
+### [xmlstrip.py](python/xmlstrip.py)
+This script cleans up the yaml files in the [ref](ref) folder by removing all the xml 
+references in the schema, and tweaking the schema values to clean them up for use in 
+the [postprocess.py](#postprocesspypythonpostprocesspy) script.
+
+### [postprocess.py](python/postprocess.py)
+This script takes in the preliminary OpenAPI file that was generated from the Postman collection
+and adds additional metadata to it, as well as cleaning up the data and format.
+
+
+## [ref](ref) folder
+This folder contains files generated from the official Onvif wsdl
+spec files. The wsdl files were imported into [apimatic.io](https://apimatic.io)
+and exported as OpenAPI 3.0 spec yaml files.
+
+## [v2](v2) folder
+This folder contains the final exported OpenAPI 3.0 spec file, to be
+exported to SwaggerHub.
+

doc/openapi/python/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (C) 2022 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0

doc/openapi/python/cleaner.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+
+# Copyright (C) 2022 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+from collections import defaultdict
+from dataclasses import dataclass, field
+from ruamel.yaml import YAML
+
+yaml = YAML()
+
+
+@dataclass
+class Schema:
+    uses: set = field(default_factory=set)
+    used_by: set = field(default_factory=set)
+
+
+@dataclass
+class SchemaCleaner:
+    """
+    Implementation of a schema cleaner to remove unused schema definitions.
+
+    Note: This is a one time use object, do not re-use it!
+    """
+    yml: any
+    schemas: dict[str, Schema] = field(default_factory=lambda: defaultdict(Schema))
+    unused: set[str] = field(default_factory=set)
+
+    def _inner_parse(self, current, obj):
+        """Inner recursive portion of _parse_schemas"""
+        if isinstance(obj, dict):
+            if '$ref' in obj:
+                name = obj['$ref'].split('/')[-1]
+                self.schemas[current].uses.add(name)
+                self.schemas[name].used_by.add(current)
+            for _, x_obj in obj.items():
+                self._inner_parse(current, x_obj)
+        elif isinstance(obj, list):
+            for x_obj in obj:
+                self._inner_parse(current, x_obj)
+
+    def _parse_schemas(self):
+        """Parse the yml for all schemas and recursively detect what they are used by, and who uses them"""
+        for path, path_obj in self.yml['paths'].items():
+            for method, method_obj in path_obj.items():
+                name = f'{path.split("/")[-1]}_{method}'
+                # this is a top level path, so add a used_by to prevent it from being detected as unused
+                self.schemas[name].used_by.add('TOP_LEVEL')
+                self._inner_parse(name, method_obj)
+
+        for name, schema in self.yml['components']['schemas'].items():
+            self._inner_parse(name, schema)
+
+    def _inner_find_unused(self, ref, name):
+        """Inner recursive part of _find_unused"""
+        if ref is not None and ref in self.schemas[name].used_by:
+            self.schemas[name].used_by.remove(ref)
+        if len(self.schemas[name].used_by) == 0:
+            for use in self.schemas[name].uses:
+                self._inner_find_unused(name, use)
+
+    def _find_unused(self):
+        """
+        Recursively find the schemas that are unused by removing references to a schema which itself is unused
+
+        For Example: If Schema A is unused, and it refers to Schema B, remove the usage from Schema B. Now if Schema B has no
+        more usages, it is considered unused as well.
+        """
+        for name in self.schemas.keys():
+            self._inner_find_unused(None, name)
+
+        # note: we need to loop through the original yml schema list (and NOT self.schemas) to find schemas that are
+        # unused which do not have any usages themselves
+        for name in self.yml['components']['schemas'].keys():
+            if len(self.schemas[name].used_by) == 0:
+                self.unused.add(name)
+
+    def _remove_unused(self):
+        """Remove all unused items from the schema portion of the yaml"""
+        for name in self.unused:
+            self.yml["components"]["schemas"].pop(name)
+
+    def remove_unused_schemas(self) -> any:
+        """
+        Recursively parses the schemas to determine which ones are used and which ones are not. The unused
+        ones are subsequently deleted from the input yaml data.
+        """
+        print('Removing unused schemas')
+        total = len(self.yml["components"]["schemas"])  # cache the total number as it will be modified by the end
+        self._parse_schemas()
+        self._find_unused()
+        self._remove_unused()
+        print(f'Removed {len(self.unused)} unused schemas of {total} total schemas!')
+        print(f'{len(self.yml["components"]["schemas"])} total schemas remain')
+

doc/openapi/python/matrix.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+# Copyright (C) 2022 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+from enum import Enum, auto
+from dataclasses import dataclass, field
+import sys
+import os
+
+from ruamel.yaml import YAML
+
+yaml = YAML()
+
+
+class State(Enum):
+    NotStarted = auto()
+    WantSection = auto()
+    NewSection = auto()
+    Commands = auto()
+
+
+@dataclass
+class CameraSupport:
+    camera: str
+    support: str
+    notes: str = ''
+
+
+@dataclass
+class Command:
+    section: str
+    service: str
+    func: str
+    cameras: dict = field(default_factory=dict)
+
+    @property
+    def qualified_name(self):
+        return f'{self.service}_{self.func}'
+
+
+@dataclass
+class Section:
+    name: str
+    cameras: list = field(default_factory=list)
+
+
+@dataclass
+class MarkdownMatrix:
+    tested_file: str
+    footnotes_file: str
+    section: Section = None
+    service: str = ''
+    state: State = State.NotStarted
+    validated: dict = field(default_factory=dict)
+    footnotes: dict = field(default_factory=dict)
+
+    def _load_footnotes(self):
+        basename = os.path.basename(self.footnotes_file)
+        current_key = None
+        current_data = ''
+        with open(self.footnotes_file) as f:
+            while line := f.readline():
+                if not line.startswith('### '):
+                    if current_key is None:
+                        continue
+                    current_data += line.strip() + ' '
+                else:
+                    if current_key is not None:
+                        self.footnotes[current_key] = current_data.strip()
+                    current_data = ''
+                    key = line.lstrip('### ').lower().replace(' ', '-').strip()
+                    current_key = f'{basename}#{key}'
+            if current_key is not None:
+                self.footnotes[current_key] = current_data.strip()
+
+    def parse(self):
+        self._load_footnotes()
+        with open(self.tested_file) as f:
+            i = 0
+            for line in f:
+                i += 1
+                line = line.strip()
+                if self.state == State.NotStarted:
+                    if line.startswith('## Tested Onvif Cameras'):
+                        self.state = State.WantSection
+                    continue
+
+                if line.startswith('### '):
+                    self.state = State.NewSection
+                    self.section = Section(line.lstrip('### ').strip())
+                    continue
+
+                if self.state == State.NewSection:
+                    if not line.startswith('| Onvif Web Service | Onvif Function'):
+                        raise RuntimeError(f'Invalid state! Expected New section table!. Line #{i}: "{line.strip()}"')
+                    tokens = line.lstrip('| Onvif Web Service | Onvif Function').split('|')
+                    cameras = [x.strip() for x in tokens if x.strip() != '']
+                    self.section.cameras = cameras
+                    self.state = State.Commands
+                elif self.state == State.Commands:
+                    if not line.startswith('|'):
+                        continue
+                    fields = [x.strip() for x in line.split('|')][1:]
+                    if fields[0].startswith('----'):
+                        continue  # skip post-header line
+
+                    if fields[0].startswith('**'):
+                        # the service name is only placed on lines where it changes. cache the
+                        # service name and only change it when a new one is found. Format: **ServiceName**
+                        self.service = fields[0].strip('*')
+
+                    command = Command(self.section.name, self.service, fields[1])
+                    for i in range(len(self.section.cameras)):
+                        camera = self.section.cameras[i]
+                        data = fields[i+2].strip()
+                        notes = ''
+                        if data == '':
+                            continue  # skip empty results
+
+                        # replace footnotes link with actual footnotes data
+                        if '[ⓘ](' in data:
+                            key = data[data.rfind('[ⓘ](')+4:-1]
+                            if key in self.footnotes:
+                                data = data[:data.rfind('[ⓘ]')]
+                                notes = self.footnotes[key]
+
+                        command.cameras[camera] = CameraSupport(camera, data.replace('✔', '✔️').strip(), notes)
+                    self.validated[command.qualified_name] = command