diff --git a/.gitignore b/.gitignore index 45295af17..c0a522458 100644 --- a/.gitignore +++ b/.gitignore @@ -1,42 +1,43 @@ -/*ccache* -/.vscode -/.vs -/out -/data +# Build directories/files +/*build* + +# Dependencies +/deps .pkg.mutex + +# Temp files +*.*~ +callgrind.out.* +*.callgrind +/.clangd +/compile_commands.json +/.clang-tidy + +# Local settings CMakeSettings.json +config.ini .settings .cproject .project -Bestandsstatistik.stat -Differenzstatistik.stat -/build* -/rohdaten* *.user -*.bin -*.*~ -*.zip *.sublime-project *.sublime-workspace -*.raw -*.db -*.sqlite -*.sqlite3 -*.sqlite3-journal -stats.csv -tracking-in-messages.txt -gmon.out -/bikesharing* -config.ini -/eval +/.vs + +# JetBrains IDEs +**/.idea/** +!**/.idea/jsonSchemas.xml + +# Visual Studio Code +**/.vscode/** +!**/.vscode/settings.json + +# Data files +/data *.mdb *.mdb-lock -/*build* -/.idea -/deps -pkg -/.clang-tidy -callgrind.out.* -*.callgrind -/.clangd -/compile_commands.json +*.raw +*.db +*.bin +*.zip +*.7z diff --git a/.idea/jsonSchemas.xml b/.idea/jsonSchemas.xml new file mode 100644 index 000000000..13b849e7e --- /dev/null +++ b/.idea/jsonSchemas.xml @@ -0,0 +1,62 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.pkg b/.pkg index d28af04fd..615158aa7 100644 --- a/.pkg +++ b/.pkg @@ -21,7 +21,7 @@ [flatbuffers] url=git@github.com:motis-project/flatbuffers.git branch=motis - commit=5cea97c2dc88f629978434505b0f4d902827a5ea + commit=fd953b0d0dee3f56d2a1388232531d6cf92adccf [fmt] url=git@github.com:motis-project/fmt.git branch=master diff --git a/.pkg.lock b/.pkg.lock index aff9f07e0..056ef874c 100644 --- a/.pkg.lock +++ b/.pkg.lock @@ -1,4 +1,4 @@ -13212942322912422357 +6948170663282086064 cista d14b70b9f45d249d1c9a20bcc13877cc4b771188 zlib fe8e13ffca867612951bc6baf114e5ac8b00f305 boost be5235eb2258d2ec19e32546ab767a62311d9b46 @@ -17,7 +17,7 @@ context 797dd16e2b5e959997ddcd5bdeac4f80931169b6 ctx aec268c603d5d2fde734c4cea83445369b835a1d res 7d97784ba785ce8a2677ea77164040fde484fb04 date 443af47fb86db22d2c93f78106f90493d2513a9b -flatbuffers 5cea97c2dc88f629978434505b0f4d902827a5ea +flatbuffers fd953b0d0dee3f56d2a1388232531d6cf92adccf doctest 70e8f76437b76dd5e9c0a2eb9b907df190ab71a0 geo f455b76d5894bb156e3bcb1f85c3e2915820fe3a lmdb 9bd01f14f549d8202413c4cd5f49b066b0a22b66 diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..543e66c2c --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,7 @@ +{ + "yaml.schemas": { + "./docs/schemas/paths.json": "/docs/api/paths.yaml", + "./docs/schemas/tags.json": "/docs/api/tags.yaml", + "./docs/schemas/schema.json": "/docs/api/schemas/**.yaml" + } +} diff --git a/base/launcher/src/batch_mode.cc b/base/launcher/src/batch_mode.cc index 0587a3ee9..afff359af 100644 --- a/base/launcher/src/batch_mode.cc +++ b/base/launcher/src/batch_mode.cc @@ -117,7 +117,7 @@ struct query_injector : std::enable_shared_from_this { } response->get()->mutate_id(id); - out_ << response->to_json(true) << "\n"; + out_ << response->to_json(json_format::SINGLE_LINE) << "\n"; out_.flush(); } diff --git a/base/launcher/src/web_server.cc b/base/launcher/src/web_server.cc index 132696785..aef6f1fd1 100644 --- a/base/launcher/src/web_server.cc +++ b/base/launcher/src/web_server.cc @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include "boost/beast/version.hpp" @@ -26,12 +28,13 @@ using namespace motis::module; namespace motis::launcher { -std::string encode_msg(msg_ptr const& msg, bool const binary) { +std::string encode_msg(msg_ptr const& msg, bool const binary, + json_format const jf = kDefaultOuputJsonFormat) { std::string b; if (binary) { b = std::string{reinterpret_cast(msg->data()), msg->size()}; } else { - b = msg->to_json(); + b = msg->to_json(jf); } return b; } @@ -48,11 +51,15 @@ msg_ptr build_reply(int const id, msg_ptr const& res, return m; } -msg_ptr decode_msg(std::string const& req_buf, bool const binary) { +std::pair decode_msg( + std::string const& req_buf, bool const binary, + std::string_view const target = std::string_view{}) { + auto jf = kDefaultOuputJsonFormat; if (binary) { - return make_msg(req_buf.data(), req_buf.size()); + return {make_msg(req_buf.data(), req_buf.size()), jf}; } else { - return make_msg(req_buf, true); + auto const msg = make_msg(req_buf, jf, true, target); + return {msg, jf}; } } @@ -73,7 +80,8 @@ struct ws_client : public client, ~ws_client() override = default; - void set_on_msg_cb(std::function&& cb) override { + void set_on_msg_cb( + std::function&& cb) override { if (auto const lock = session_.lock(); lock) { lock->on_msg([this, cb = std::move(cb)](std::string const& req_buf, net::ws_msg_type const type) { @@ -85,10 +93,10 @@ struct ws_client : public client, int req_id = 0; try { - auto const req = + auto const [req, jf] = decode_msg(req_buf, type == net::ws_msg_type::BINARY); req_id = req->get()->id(); - return cb(req); + return cb(req, jf); } catch (std::system_error const& e) { err = build_reply(req_id, nullptr, e.code()); } catch (std::exception const& e) { @@ -117,11 +125,11 @@ struct ws_client : public client, } } - void send(msg_ptr const& m) override { - ios_.post([self = shared_from_this(), m]() { + void send(msg_ptr const& m, json_format const jf) override { + ios_.post([self = shared_from_this(), m, jf]() { if (auto s = self->session_.lock()) { s->send( - encode_msg(m, self->binary_), + encode_msg(m, self->binary_, jf), self->binary_ ? net::ws_msg_type::BINARY : net::ws_msg_type::TEXT, [](boost::system::error_code, size_t) {}); } @@ -200,7 +208,9 @@ struct web_server::impl { net::web_server::http_res_cb_t const& cb) { using namespace boost::beast::http; - auto const build_response = [req](msg_ptr const& response) { + auto const build_response = [req](msg_ptr const& response, + std::optional jf = + std::nullopt) { net::web_server::string_res_t res{ response == nullptr ? status::ok : response->get()->content_type() == MsgContent_MotisError @@ -229,15 +239,19 @@ struct web_server::impl { } } else { res.set(field::content_type, "application/json"); - res.body() = response == nullptr ? "" : response->to_json(); + res.body() = + response == nullptr + ? "" + : response->to_json(jf.value_or(kDefaultOuputJsonFormat)); } res.prepare_payload(); return res; }; - auto const res_cb = [cb, build_response](msg_ptr const& response) { - cb(build_response(response)); + auto const res_cb = [cb, build_response](msg_ptr const& response, + std::optional jf) { + cb(build_response(response, jf)); }; std::string req_msg; @@ -251,7 +265,8 @@ struct web_server::impl { } req_msg = req.body(); if (req_msg.empty()) { - req_msg = make_no_msg(std::string{req.target()})->to_json(); + req_msg = make_no_msg(std::string{req.target()}) + ->to_json(kDefaultOuputJsonFormat); } break; } @@ -261,7 +276,8 @@ struct web_server::impl { net::serve_static_file(static_file_path_, req, cb)) { return; } else { - req_msg = make_no_msg(std::string{req.target()})->to_json(); + req_msg = make_no_msg(std::string{req.target()}) + ->to_json(kDefaultOuputJsonFormat); break; } default: @@ -269,7 +285,7 @@ struct web_server::impl { std::make_error_code(std::errc::operation_not_supported)))); } - return on_msg_req(req_msg, false, res_cb); + return on_msg_req(req_msg, false, to_sv(req.target()), res_cb); } void on_ws_open(net::ws_session_ptr session, std::string const& target) { @@ -288,27 +304,35 @@ struct web_server::impl { void on_ws_msg(net::ws_session_ptr const& session, std::string const& msg, net::ws_msg_type type) { auto const is_binary = type == net::ws_msg_type::BINARY; - return on_msg_req(msg, is_binary, - [session, type, is_binary](msg_ptr const& response) { - if (auto s = session.lock()) { - s->send(encode_msg(response, is_binary), type, - [](boost::system::error_code, size_t) {}); - } - }); + return on_msg_req( + msg, is_binary, {}, + [session, type, is_binary](msg_ptr const& response, + std::optional jf) { + if (auto s = session.lock()) { + s->send(encode_msg(response, is_binary, + jf.value_or(kDefaultOuputJsonFormat)), + type, [](boost::system::error_code, size_t) {}); + } + }); } - void on_msg_req(std::string const& request, bool binary, - std::function const& cb) { + void on_msg_req( + std::string const& request, bool binary, std::string_view const target, + std::function)> const& cb, + std::optional jf = std::nullopt) { msg_ptr err; int req_id = 0; try { - auto const req = decode_msg(request, binary); + auto const [req, detected_jf] = decode_msg(request, binary, target); + if (!jf) { + jf = detected_jf; + } log_request(req); req_id = req->get()->id(); return receiver_.on_msg( - req, ios_.wrap([&, cb, req_id](msg_ptr const& res, - std::error_code const& ec) { - cb(build_reply(req_id, res, ec)); + req, ios_.wrap([cb, req_id, jf](msg_ptr const& res, + std::error_code const& ec) { + cb(build_reply(req_id, res, ec), jf); })); } catch (std::system_error const& e) { err = build_reply(req_id, nullptr, e.code()); @@ -318,11 +342,13 @@ struct web_server::impl { err = make_unknown_error_msg("unknown"); } err->get()->mutate_id(req_id); - return cb(err); + return cb(err, jf); } - void on_generic_req(net::web_server::http_req_t const& req, - std::function const& cb) { + void on_generic_req( + net::web_server::http_req_t const& req, + std::function)> const& cb, + std::optional jf = std::nullopt) { using namespace boost::beast::http; auto const req_id = 1; @@ -347,7 +373,7 @@ struct web_server::impl { return receiver_.on_msg( msg, ios_.wrap([&, cb](msg_ptr const& res, std::error_code const& ec) { - cb(build_reply(req_id, res, ec)); + cb(build_reply(req_id, res, ec), jf); })); } catch (std::system_error const& e) { err = build_reply(req_id, nullptr, e.code()); @@ -357,7 +383,7 @@ struct web_server::impl { err = make_unknown_error_msg("unknown"); } LOG(logging::error) << err->to_json(); - return cb(err); + return cb(err, jf); } void log_request(msg_ptr const& msg) { @@ -366,8 +392,8 @@ struct web_server::impl { } try { - log_file_ << "[" << motis::logging::time() << "] " << msg->to_json(true) - << std::endl; + log_file_ << "[" << motis::logging::time() << "] " + << msg->to_json(json_format::SINGLE_LINE) << std::endl; } catch (std::ios_base::failure const& e) { LOG(logging::error) << "could not log request: " << e.what(); reset_logging(false); diff --git a/base/module/include/motis/module/client.h b/base/module/include/motis/module/client.h index 41c973613..696d834d9 100644 --- a/base/module/include/motis/module/client.h +++ b/base/module/include/motis/module/client.h @@ -3,6 +3,7 @@ #include #include +#include "motis/module/json_format.h" #include "motis/module/message.h" namespace motis::module { @@ -15,9 +16,10 @@ struct client { client const& operator=(client const&&) = delete; virtual ~client() = default; - virtual void set_on_msg_cb(std::function&&) = 0; + virtual void set_on_msg_cb( + std::function&&) = 0; virtual void set_on_close_cb(std::function&&) = 0; - virtual void send(msg_ptr const&) = 0; + virtual void send(msg_ptr const&, json_format = kDefaultOuputJsonFormat) = 0; }; using client_hdl = std::weak_ptr; diff --git a/base/module/include/motis/module/fix_json.h b/base/module/include/motis/module/fix_json.h index c6e0d5353..281a9c309 100644 --- a/base/module/include/motis/module/fix_json.h +++ b/base/module/include/motis/module/fix_json.h @@ -1,9 +1,18 @@ #pragma once #include +#include + +#include "motis/module/json_format.h" namespace motis::module { -std::string fix_json(std::string const&); +struct fix_json_result { + std::string fixed_json_; + json_format detected_format_{json_format::DEFAULT_FLATBUFFERS}; +}; + +fix_json_result fix_json(std::string const& json, + std::string_view target = std::string_view{}); -} // namespace motis::module \ No newline at end of file +} // namespace motis::module diff --git a/base/module/include/motis/module/json_format.h b/base/module/include/motis/module/json_format.h new file mode 100644 index 000000000..5e95b3b1e --- /dev/null +++ b/base/module/include/motis/module/json_format.h @@ -0,0 +1,17 @@ +#pragma once + +#include + +namespace motis::module { + +enum class json_format : std::uint8_t { + DEFAULT_FLATBUFFERS, + SINGLE_LINE, + TYPES_IN_UNIONS, + CONTENT_ONLY_TYPES_IN_UNIONS +}; + +constexpr auto const kDefaultOuputJsonFormat = + json_format::CONTENT_ONLY_TYPES_IN_UNIONS; + +} // namespace motis::module diff --git a/base/module/include/motis/module/message.h b/base/module/include/motis/module/message.h index 0ec49a920..4d69ce402 100644 --- a/base/module/include/motis/module/message.h +++ b/base/module/include/motis/module/message.h @@ -1,5 +1,8 @@ #pragma once +#include +#include + #include "flatbuffers/flatbuffers.h" #include "flatbuffers/reflection.h" @@ -12,6 +15,7 @@ #include "motis/core/common/typed_flatbuffer.h" #include "motis/module/error.h" +#include "motis/module/json_format.h" namespace motis::module { @@ -40,7 +44,7 @@ struct message : public typed_flatbuffer { int id() const { return get()->id(); } - std::string to_json(bool compact = false) const; + std::string to_json(json_format jf = json_format::DEFAULT_FLATBUFFERS) const; static reflection::Schema const& get_schema(); static reflection::Object const* get_objectref(char const* name); @@ -49,9 +53,11 @@ struct message : public typed_flatbuffer { using msg_ptr = std::shared_ptr; -msg_ptr make_msg(std::string const& json, bool fix = false, +msg_ptr make_msg(std::string const& json, json_format& jf, bool fix = false, + std::string_view const target = std::string_view{}, std::size_t fbs_max_depth = DEFAULT_FBS_MAX_DEPTH, std::size_t fbs_max_tables = DEFAULT_FBS_MAX_TABLES); +msg_ptr make_msg(std::string const& json); msg_ptr make_msg(message_creator& builder); msg_ptr make_msg(void const* buf, size_t len, std::size_t fbs_max_depth = DEFAULT_FBS_MAX_DEPTH, diff --git a/base/module/src/event_collector.cc b/base/module/src/event_collector.cc index 1cae3f3f0..e048f2245 100644 --- a/base/module/src/event_collector.cc +++ b/base/module/src/event_collector.cc @@ -51,9 +51,10 @@ event_collector* event_collector::require( LOG(logging::info) << "prevented double execution"; LOG(logging::info) << "previous import events\n"; for (auto const& [k, v] : dependencies_) { - LOG(logging::info) << k << ": " << v->to_json(true); + LOG(logging::info) << k << ": " << v->to_json(json_format::SINGLE_LINE); } - LOG(logging::info) << "new import event: " << msg->to_json(true); + LOG(logging::info) << "new import event: " + << msg->to_json(json_format::SINGLE_LINE); return nullptr; } diff --git a/base/module/src/fix_json.cc b/base/module/src/fix_json.cc index b53f8ab5f..118a9937e 100644 --- a/base/module/src/fix_json.cc +++ b/base/module/src/fix_json.cc @@ -5,103 +5,174 @@ #include #include "rapidjson/document.h" +#include "rapidjson/error/en.h" #include "rapidjson/stringbuffer.h" #include "rapidjson/writer.h" +#include "motis/core/common/logging.h" #include "motis/module/error.h" using namespace rapidjson; namespace motis::module { -void write_json_value(Value const& v, Writer& writer, - bool const is_root = false) { - auto const is_type_key = [](auto const& m, std::string_view& union_key) { - auto const key = m.name.GetString(); - auto const key_len = m.name.GetStringLength(); - if (key_len > 5 && std::strcmp(key + key_len - 5, "_type") == 0) { - union_key = {key, key_len - 5}; - return true; - } - return false; - }; +struct json_converter { + static constexpr auto const kTypeKey = "_type"; - switch (v.GetType()) { // NOLINT - case rapidjson::kObjectType: { - writer.StartObject(); + json_converter(Writer& writer, std::string_view target) + : writer_{writer}, target_{target} {} - // Set of already written members. - std::set written; + void fix(Value const& v) { write_json_value(v, std::string_view{}, true); } - for (auto const& m : v.GetObject()) { - std::string_view union_key; - if (!is_type_key(m, union_key)) { - continue; // Not a union key. - } + json_format detected_format() const { + if (content_only_detected_) { + return json_format::CONTENT_ONLY_TYPES_IN_UNIONS; + } else if (type_in_union_detected_) { + return json_format::TYPES_IN_UNIONS; + } else { + return json_format::DEFAULT_FLATBUFFERS; + } + } - auto const it = v.GetObject().FindMember( - Value(union_key.data(), static_cast(union_key.size()))); - if (it == v.MemberEnd()) { - continue; // Could be a union key but no union found. +private: + void write_json_value(Value const& v, + std::string_view current_key = std::string_view{}, + bool const is_root = false) { + auto const is_type_key = [](auto const& m, std::string_view& union_key) { + auto const key = m.name.GetString(); + auto const key_len = m.name.GetStringLength(); + if (key_len > 5 && std::strcmp(key + key_len - 5, "_type") == 0) { + union_key = {key, key_len - 5}; + return true; + } + return false; + }; + + // Check if the root element has destination + content fields + // or uses the compact format (content only, target taken from URL). + auto add_msg_wrapper = false; + if (is_root && v.IsObject()) { + if (!v.HasMember("destination") && !v.HasMember("content")) { + add_msg_wrapper = true; + content_only_detected_ = true; + writer_.StartObject(); + + writer_.String("destination"); + writer_.StartObject(); + writer_.String("target"); + writer_.String(target_.data(), static_cast(target_.size())); + writer_.EndObject(); + + current_key = "content"; + } + } + + if (!current_key.empty()) { + if (v.IsObject()) { + // We're inside an object field. Check if it has a type key. + if (auto const it = v.GetObject().FindMember(kTypeKey); + it != v.MemberEnd()) { + // Type key found, write it before the object field itself. + auto const union_key = std::string{current_key} + kTypeKey; + writer_.String(union_key.data(), + static_cast(union_key.size())); + write_json_value(it->value); + type_in_union_detected_ = true; } + } + writer_.String(current_key.data(), + static_cast(current_key.size())); + } - // Write union key ("_type"). - writer.String(m.name.GetString(), m.name.GetStringLength()); - write_json_value(m.value, writer); + switch (v.GetType()) { // NOLINT + case rapidjson::kObjectType: { + writer_.StartObject(); - // Write union. - writer.String(it->name.GetString(), it->name.GetStringLength()); - write_json_value(it->value, writer); + // Set of already written members. + std::set written; - // Remember written values. - written.emplace(m.name.GetString(), m.name.GetStringLength()); - written.emplace(union_key); - } + for (auto const& m : v.GetObject()) { + std::string_view union_key; + if (!is_type_key(m, union_key)) { + continue; // Not a union key. + } + + auto const it = v.GetObject().FindMember( + Value(union_key.data(), static_cast(union_key.size()))); + if (it == v.MemberEnd()) { + continue; // Could be a union key but no union found. + } + + // Write union key ("_type"). + writer_.String(m.name.GetString(), m.name.GetStringLength()); + write_json_value(m.value); - // Write remaining values - for (auto const& m : v.GetObject()) { - if (written.find({m.name.GetString(), m.name.GetStringLength()}) == - end(written)) { - writer.String(m.name.GetString(), m.name.GetStringLength()); - write_json_value(m.value, writer); + // Write union. + write_json_value(it->value, union_key); + + // Remember written values. + written.emplace(m.name.GetString(), m.name.GetStringLength()); + written.emplace(union_key); + } + + // Write remaining values. + for (auto const& m : v.GetObject()) { + auto const key = + std::string_view{m.name.GetString(), m.name.GetStringLength()}; + if (key != kTypeKey && written.find(key) == end(written)) { + write_json_value(m.value, key); + } } + + // Write message id if missing. + if (is_root && !v.HasMember("id")) { + writer_.String("id"); + writer_.Int(1); + } + + writer_.EndObject(); + break; } - // Write message id if missing. - if (is_root && !v.HasMember("id")) { - writer.String("id"); - writer.Int(1); + case rapidjson::kArrayType: { + writer_.StartArray(); + for (auto const& entry : v.GetArray()) { + write_json_value(entry); + } + writer_.EndArray(); + break; } - writer.EndObject(); - break; + default: v.Accept(writer_); // NOLINT } - case rapidjson::kArrayType: { - writer.StartArray(); - for (auto const& entry : v.GetArray()) { - write_json_value(entry, writer); - } - writer.EndArray(); - break; + if (add_msg_wrapper) { + writer_.EndObject(); } - - default: v.Accept(writer); // NOLINT } -} -std::string fix_json(std::string const& json) { + Writer& writer_; // NOLINT + std::string_view target_; + bool content_only_detected_{false}; + bool type_in_union_detected_{false}; +}; + +fix_json_result fix_json(std::string const& json, + std::string_view const target) { rapidjson::Document d; if (d.Parse(json.c_str()).HasParseError()) { // NOLINT + LOG(motis::logging::error) + << "JSON parse error (step 1): " << GetParseError_En(d.GetParseError()) + << " at offset " << d.GetErrorOffset(); throw std::system_error(module::error::unable_to_parse_msg); } rapidjson::StringBuffer buffer; rapidjson::Writer writer(buffer); - write_json_value(d, writer, true); - - return buffer.GetString(); + auto converter = json_converter{writer, target}; + converter.fix(d); + return {buffer.GetString(), converter.detected_format()}; } } // namespace motis::module diff --git a/base/module/src/message.cc b/base/module/src/message.cc index 4766e659e..85a085d54 100644 --- a/base/module/src/message.cc +++ b/base/module/src/message.cc @@ -17,14 +17,22 @@ using namespace flatbuffers; namespace motis::module { -std::unique_ptr init_parser(bool compact = false) { +std::unique_ptr init_parser( + json_format const jf = json_format::DEFAULT_FLATBUFFERS) { auto parser = std::make_unique(); parser->opts.strict_json = true; parser->opts.skip_unexpected_fields_in_json = true; parser->opts.output_default_scalars_in_json = true; - if (compact) { - parser->opts.indent_step = -1; + + switch (jf) { + case json_format::DEFAULT_FLATBUFFERS: break; + case json_format::SINGLE_LINE: parser->opts.indent_step = -1; break; + case json_format::TYPES_IN_UNIONS: + case json_format::CONTENT_ONLY_TYPES_IN_UNIONS: + parser->opts.type_tags_in_unions = true; + break; } + int message_symbol_index = -1; for (unsigned i = 0; i < number_of_symbols; ++i) { if (strcmp(filenames[i], "Message.fbs") == 0) { // NOLINT @@ -53,7 +61,12 @@ namespace { std::unique_ptr json_parser = init_parser(); // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) -std::unique_ptr compact_json_parser = init_parser(true); +std::unique_ptr single_line_json_parser = + init_parser(json_format::SINGLE_LINE); + +// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) +std::unique_ptr types_in_unions_json_parser = + init_parser(json_format::TYPES_IN_UNIONS); // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables) std::unique_ptr reflection_parser = init_parser(); @@ -63,10 +76,23 @@ reflection::Schema const& schema = init_schema(*reflection_parser); } // namespace -std::string message::to_json(bool compact) const { +std::string message::to_json(json_format const jf) const { std::string json; - flatbuffers::GenerateText(compact ? *compact_json_parser : *json_parser, - data(), &json); + switch (jf) { + case json_format::DEFAULT_FLATBUFFERS: + flatbuffers::GenerateText(*json_parser, data(), &json); + break; + case json_format::SINGLE_LINE: + flatbuffers::GenerateText(*single_line_json_parser, data(), &json); + break; + case json_format::TYPES_IN_UNIONS: + flatbuffers::GenerateText(*types_in_unions_json_parser, data(), &json); + break; + case json_format::CONTENT_ONLY_TYPES_IN_UNIONS: + flatbuffers::GenerateTextSingleField(*types_in_unions_json_parser, data(), + &json, "content"); + break; + } return json; } @@ -80,18 +106,26 @@ std::pair message::get_fbs_definitions() { return std::make_pair(symbols, number_of_symbols); } -msg_ptr make_msg(std::string const& json, bool const fix, - std::size_t const fbs_max_depth, +msg_ptr make_msg(std::string const& json, json_format& jf, bool const fix, + std::string_view const target, std::size_t const fbs_max_depth, std::size_t const fbs_max_tables) { if (json.empty()) { LOG(motis::logging::error) << "empty request"; throw std::system_error(error::unable_to_parse_msg); } - bool const parse_ok = - json_parser->Parse(fix ? fix_json(json).c_str() : json.c_str()); + auto parse_ok = false; + if (fix) { + auto const fix_result = fix_json(json, target); + parse_ok = json_parser->Parse(fix_result.fixed_json_.c_str()); + jf = fix_result.detected_format_; + } else { + parse_ok = json_parser->Parse(json.c_str()); + } + if (!parse_ok) { - LOG(motis::logging::error) << "parse error: " << json_parser->error_; + LOG(motis::logging::error) + << "JSON parse error (step 2): " << json_parser->error_; throw std::system_error(error::unable_to_parse_msg); } @@ -99,6 +133,8 @@ msg_ptr make_msg(std::string const& json, bool const fix, json_parser->builder_.GetSize(), fbs_max_depth, fbs_max_tables); if (!VerifyMessageBuffer(verifier)) { + LOG(motis::logging::error) + << "JSON parse error (step 3): verification failed"; throw std::system_error(error::malformed_msg); } auto size = json_parser->builder_.GetSize(); @@ -108,6 +144,11 @@ msg_ptr make_msg(std::string const& json, bool const fix, return std::make_shared(size, std::move(buffer)); } +msg_ptr make_msg(std::string const& json) { + auto jf = json_format::DEFAULT_FLATBUFFERS; + return make_msg(json, jf); +} + msg_ptr make_msg(message_creator& builder) { auto len = builder.GetSize(); auto mem = builder.ReleaseBufferPointer(); diff --git a/base/module/src/remote.cc b/base/module/src/remote.cc index f0514833b..09a7e98a1 100644 --- a/base/module/src/remote.cc +++ b/base/module/src/remote.cc @@ -202,4 +202,4 @@ void remote::stop() const { impl_->stop(); } void remote::start() const { impl_->start(); } -} // namespace motis::module \ No newline at end of file +} // namespace motis::module diff --git a/base/module/test/fix_json_test.cc b/base/module/test/fix_json_test.cc index 998734990..a7251e539 100644 --- a/base/module/test/fix_json_test.cc +++ b/base/module/test/fix_json_test.cc @@ -41,10 +41,11 @@ using namespace motis; using namespace motis::intermodal; TEST(fix_json, fix_json) { - EXPECT_THROW(make_msg(req, false), std::system_error); // NO_LINT - EXPECT_NO_THROW(make_msg(req, true)); // NO_LINT + auto jf = json_format::DEFAULT_FLATBUFFERS; + EXPECT_THROW(make_msg(req, jf, false), std::system_error); // NO_LINT + EXPECT_NO_THROW(make_msg(req, jf, true)); // NO_LINT - auto const msg = make_msg(req, true); + auto const msg = make_msg(req, jf, true); auto const r = motis_content(IntermodalRoutingRequest, msg); EXPECT_EQ(1, msg->id()); EXPECT_EQ("/intermodal", msg->get()->destination()->target()->str()); diff --git a/docs/.vscode/settings.json b/docs/.vscode/settings.json new file mode 100644 index 000000000..bf4c6ebb2 --- /dev/null +++ b/docs/.vscode/settings.json @@ -0,0 +1,7 @@ +{ + "yaml.schemas": { + "./schemas/paths.json": "/api/paths.yaml", + "./schemas/tags.json": "/api/tags.yaml", + "./schemas/schema.json": "/api/schemas/**.yaml" + } +} diff --git a/docs/api/paths.yaml b/docs/api/paths.yaml index ed6be9096..7710843a8 100644 --- a/docs/api/paths.yaml +++ b/docs/api/paths.yaml @@ -7,7 +7,24 @@ type: motis.address.AddressResponse description: A list of guesses -#/cc: +/api: + summary: /api + tags: + - internal + output: + type: motis.ApiDescription + description: List of available endpoints + +# Also accepts RoutingResponse, but the protocol tool doesn't support +# multiple input types per operation. +/cc: + summary: /cc (Connection) + tags: + - internal + input: motis.Connection + output: + type: motis.MotisSuccess + description: Valid connection /csa: summary: Public transport routing (CSA) @@ -37,12 +54,59 @@ type: motis.routing.RoutingResponse description: Routing response -#/lookup/geo_station: -#/lookup/geo_station_batch: -#/lookup/geo_station_id: -#/lookup/id_train: -#/lookup/meta_station: -#/lookup/meta_station_batch: +/lookup/geo_station: + summary: /lookup/geo_station + tags: + - internal + input: motis.lookup.LookupGeoStationRequest + output: + type: motis.lookup.LookupGeoStationResponse + description: Stations matching the request + +/lookup/geo_station_batch: + summary: /lookup/geo_station_batch + tags: + - internal + input: motis.lookup.LookupBatchGeoStationRequest + output: + type: motis.lookup.LookupBatchGeoStationResponse + description: Stations matching the request + +/lookup/geo_station_id: + summary: /lookup/geo_station_id + tags: + - internal + input: motis.lookup.LookupGeoStationIdRequest + output: + type: motis.lookup.LookupGeoStationResponse + description: Stations matching the request + +/lookup/id_train: + summary: /lookup/id_train + tags: + - internal + input: motis.lookup.LookupIdTrainRequest + output: + type: motis.lookup.LookupIdTrainResponse + description: Response + +/lookup/meta_station: + summary: /lookup/meta_station + tags: + - internal + input: motis.lookup.LookupMetaStationRequest + output: + type: motis.lookup.LookupMetaStationResponse + description: Response + +/lookup/meta_station_batch: + summary: /lookup/meta_station_batch + tags: + - internal + input: motis.lookup.LookupBatchMetaStationRequest + output: + type: motis.lookup.LookupBatchMetaStationResponse + description: Response /lookup/ribasis: summary: Retrieve a trip in RI Basis format @@ -70,7 +134,14 @@ type: motis.lookup.LookupScheduleInfoResponse description: Information about the currently loaded schedule -#/lookup/station_events: +/lookup/station_events: + summary: /lookup/station_events + tags: + - internal + input: motis.lookup.LookupStationEventsRequest + output: + type: motis.lookup.LookupStationEventsResponse + description: Response /lookup/station_info: summary: Station information @@ -87,13 +158,59 @@ type: motis.lookup.LookupStationInfoResponse description: Information about the requested stations -#/osrm/one_to_many: -#/osrm/smooth_via: -#/osrm/table: -#/osrm/via: +/osrm/one_to_many: + summary: /osrm/one_to_many + tags: + - internal + input: motis.osrm.OSRMManyToManyRequest + output: + type: motis.osrm.OSRMOneToManyResponse + description: Response + +/osrm/smooth_via: + summary: /osrm/smooth_via + tags: + - internal + input: motis.osrm.OSRMSmoothViaRouteRequest + output: + type: motis.osrm.OSRMSmoothViaRouteResponse + description: Response + +/osrm/table: + summary: /osrm/table + tags: + - internal + input: motis.osrm.OSRMManyToManyResponse + output: + type: motis.osrm.OSRMManyToManyResponse + description: Response + +/osrm/via: + summary: /osrm/via + tags: + - internal + input: motis.osrm.OSRMViaRouteRequest + output: + type: motis.osrm.OSRMViaRouteResponse + description: Response -#/parking/edge: -#/parking/edges: +/parking/edge: + summary: /parking/edge + tags: + - internal + input: motis.parking.ParkingEdgeRequest + output: + type: motis.parking.ParkingEdgeResponse + description: Response + +/parking/edges: + summary: /parking/edges + tags: + - internal + input: motis.parking.ParkingEdgesRequest + output: + type: motis.parking.ParkingEdgesResponse + description: Response /parking/geo: summary: Parking lots in an area @@ -104,7 +221,6 @@ type: motis.parking.ParkingGeoResponse description: A list of parking lots - /parking/lookup: summary: Lookup a parking lot by ID tags: @@ -130,8 +246,6 @@ type: motis.paxforecast.PaxForecastApplyMeasuresResponse description: Information about the simulation result -#/paxmon/add_groups: - /paxmon/destroy_universe: summary: Destroy a paxmon universe description: | @@ -332,6 +446,43 @@ type: motis.paxmon.PaxMonGetUniversesResponse description: A list of paxmon universes +/paxmon/add_groups: + summary: /paxmon/add_groups + description: Deprecated. DO NOT USE. + deprecated: true + tags: + - internal + input: motis.paxmon.PaxMonAddGroupsRequest + output: + type: motis.paxmon.PaxMonAddGroupsResponse + description: Response + +/paxmon/remove_groups: + summary: /paxmon/remove_groups + description: Deprecated. DO NOT USE. + deprecated: true + tags: + - internal + input: motis.paxmon.PaxMonRemoveGroupsRequest + +/paxmon/addressable_groups: + summary: /paxmon/addressable_groups + tags: + - internal + input: motis.paxmon.PaxMonGetAddressableGroupsRequest + output: + type: motis.paxmon.PaxMonGetAddressableGroupsResponse + description: Response + +/paxmon/debug_graph: + summary: /paxmon/debug_graph + tags: + - internal + input: motis.paxmon.PaxMonDebugGraphRequest + output: + type: motis.paxmon.PaxMonDebugGraphResponse + description: Response + /ppr/profiles: summary: List available pedestrian routing profiles tags: @@ -358,6 +509,41 @@ type: motis.railviz.RailVizStationResponse description: A list of trips arring and departing at the station +/railviz/get_trains: + summary: /railviz/get_trains + tags: + - internal + input: motis.railviz.RailVizTrainsRequest + output: + type: motis.railviz.RailVizTrainsResponse + description: Response + +/railviz/get_trips: + summary: /railviz/get_trips + tags: + - internal + input: motis.railviz.RailVizTripsRequest + output: + type: motis.railviz.RailVizTrainsResponse + description: Response + +/railviz/get_trip_guesses: + summary: /railviz/get_trip_guesses + tags: + - internal + input: motis.railviz.RailVizTripGuessRequest + output: + type: motis.railviz.RailVizTripGuessResponse + description: Response + +/railviz/map_config: + summary: /railviz/map_config + tags: + - internal + output: + type: motis.railviz.RailVizMapConfigResponse + description: Response + /raptor: summary: Public transport routing (RAPTOR) tags: @@ -376,6 +562,35 @@ type: motis.Connection description: The updated connection +/ris/apply: + summary: /ris/apply + tags: + - internal + input: motis.ris.RISApplyRequest + output: + type: motis.ris.RISApplyResponse + description: Response + +/ris/forward: + summary: /ris/forward + tags: + - internal + input: motis.ris.RISForwardTimeRequest + +/ris/purge: + summary: /ris/purge + tags: + - internal + input: motis.ris.RISPurgeRequest + +/ris/read: + summary: /ris/read + tags: + - internal + +# Custom HTTP requests are not supported by the protocol tool. +#/ris/upload: + /routing: summary: Public transport routing (Multi Criteria Pareto Dijkstra) tags: @@ -402,3 +617,12 @@ output: type: motis.routing.RoutingResponse description: Routing response + +/rt/single: + summary: /rt/single + tags: + - internal + input: motis.ris.Message + output: + type: motis.MotisSuccess + description: Response diff --git a/docs/api/schemas/motis.yaml b/docs/api/schemas/motis.yaml index 305c8c30d..fb1b435dd 100644 --- a/docs/api/schemas/motis.yaml +++ b/docs/api/schemas/motis.yaml @@ -1,28 +1,28 @@ Position: - description: TODO + description: A geographic coordinate. fields: lat: - description: TODO + description: Latitude (north-south position) lng: - description: TODO + description: Longitude (east-west position) SearchDir: description: TODO Station: description: TODO fields: id: - description: TODO + description: The unique station ID. name: - description: TODO + description: The human readable station name. pos: - description: TODO + description: The coordinates where the station is located. Interval: - description: TODO + description: A time interval. fields: begin: - description: TODO + description: The first time in the interval (unix timestamp). end: - description: TODO + description: The first time not in the interval (unix timestamp). EventType: description: TODO TripId: @@ -41,7 +41,24 @@ TripId: line_id: description: The line name. TimestampReason: - description: TODO + description: | + The information source for the real-time timestamp. The system is + conservative in the sense that it selects the maximum timestamp from + propagation, forecast or schedule time if there is no `IS` or `REPAIR` + timestamp given. + + - `SCHEDULE`: the real-time timestamp is the schedule time. No real-time + information. + - `REPAIR`: there are conflicting real-time information and this + timestamp needed to be repaired such that basic properties of the + timetable hold (i.e. timestamps need to correspond to the event + order). + - `IS`: the system received a real-time message that the event took + place with this timestamp. + - `PROPAGATION`: the timestamp was computed by propagating times (e.g. + from earlier stops of the stop sequence of a trip) + - `FORECAST`: the system received a message that forecasted the event + time. ConnectionStatus: description: TODO ProblemType: @@ -50,19 +67,33 @@ EventInfo: description: TODO fields: time: - description: TODO + description: Real-time timestamp (unix timestamp). schedule_time: - description: TODO + description: Schedule timestamp (unix timestamp). track: - description: TODO + description: The real-time track name. schedule_track: - description: TODO + description: The schedule track name. valid: - description: TODO + description: | + If this flag is `false`, the event is invalid. This is the case for the + arrival event at the first stop and the departure event at the last + stop. reason: description: TODO Stop: - description: TODO + description: | + The stops are the basic building block of the journey data structure. + Everything else references stops by their index. + + Note that there are four cases for `exit` and `enter`: when entering the + first train, only `enter` is set. When exiting the last train, only `exit` + is set. There may be walk segments before the first `enter` and after the + last `exit`. For a direct interchange between two trains, `enter` *and* + `exit` are set. For a walk between two stops, `exit` is set at the first + stop and `enter` is set at the second stop. There may be more than one walk + segments in between. For every stop where the train just stops, neither + `enter` nor `exit` is set. fields: station: description: TODO @@ -71,9 +102,11 @@ Stop: departure: description: TODO exit: - description: TODO + description: | + Indicates whether the passenger alights from the vehicle at this stop. enter: - description: TODO + description: | + Indicates whether the passenger enters the vehicle at this stop. Range: description: TODO fields: @@ -153,20 +186,38 @@ Problem: type: description: TODO Connection: - description: TODO + description: | + A connection is a journey from a coordinate/station to another + coordinate/station. The core of the journey is the stop sequence. Other + information (such as `transports` and `trips`) reference stops by their + index in the stop sequence. fields: stops: - description: TODO + description: | + The stop sequence including (real-time) event times and information + about alighting and entering of services. This is the core of the + journey. All other collections in the `Connection` reference stop + indices. transports: - description: TODO + description: | + Information about the transports that are used with their corresponding + stop range. trips: - description: TODO + description: | + Trip IDs for each trip that is used in the journey with their + corresponding stop range. attributes: - description: TODO + description: | + Attributes that are active within the corresponding stop range. free_texts: - description: TODO + description: | + Free text messages can be used by the operator to inform and guide + users. problems: - description: TODO + description: | + Problems with the journey. This is empty for fresh journeys but can be + populated when real-time information such as cancellations, etc. are + processed. night_penalty: description: TODO db_costs: diff --git a/docs/api/schemas/motis/routing.yaml b/docs/api/schemas/motis/routing.yaml index 8de890cb0..f884f0f33 100644 --- a/docs/api/schemas/motis/routing.yaml +++ b/docs/api/schemas/motis/routing.yaml @@ -1,10 +1,16 @@ InputStation: - description: TODO + description: | + A input station is a station from user input. If the user used the + auto-completion function and the station ID is available, then the `id` + field is used to resolve the station. If this is not the case (the user just + entered a string), the `name` field is filled with a (possibly incomplete or + misspelled) station name. In the latter case, MOTIS will use the first guess + from the station auto-complete to resolve the station `id`. fields: id: - description: TODO + description: The station ID if available. May be empty if `name` is set. name: - description: TODO + description: The station name if no ID is available. May be empty if `id` is set. MumoEdge: description: TODO fields: diff --git a/docs/api/tags.yaml b/docs/api/tags.yaml index 41d1af95b..1cd796b05 100644 --- a/docs/api/tags.yaml +++ b/docs/api/tags.yaml @@ -10,3 +10,5 @@ description: Routing - name: rsl description: Passenger flow monitoring and forecasting +- name: internal + description: Internal APIs diff --git a/docs/schemas/paths.json b/docs/schemas/paths.json new file mode 100644 index 000000000..e7ee56886 --- /dev/null +++ b/docs/schemas/paths.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "type": "object", + "patternProperties": { + "^/.*$": { + "type": "object", + "properties": { + "summary": { "type": "string" }, + "description": { "type": "string" }, + "tags": { "type": "array", "items": { "type": "string" } }, + "deprecated": { "type": "boolean" }, + "input": { "type": "string" }, + "output": { + "type": "object", + "properties": { + "type": { "type": "string" }, + "description": { "type": "string" } + }, + "required": ["type", "description"] + } + }, + "required": ["summary"] + } + } +} diff --git a/docs/schemas/schema.json b/docs/schemas/schema.json new file mode 100644 index 000000000..ebc947259 --- /dev/null +++ b/docs/schemas/schema.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "description": { "type": "string" }, + "tags": { "type": "array", "items": { "type": "string" } }, + "examples": { "type": "array" }, + "fields": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "description": { "type": "string" }, + "examples": { "type": "array" } + }, + "required": ["description"] + } + } + }, + "required": ["description"] + } +} diff --git a/docs/schemas/tags.json b/docs/schemas/tags.json new file mode 100644 index 000000000..077051ea6 --- /dev/null +++ b/docs/schemas/tags.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "description": { "type": "string" } + }, + "required": ["name", "description"] + } +} diff --git a/modules/intermodal/src/eval/comparator/intermodal_comparator.cc b/modules/intermodal/src/eval/comparator/intermodal_comparator.cc index 8641909cd..77fa3e357 100644 --- a/modules/intermodal/src/eval/comparator/intermodal_comparator.cc +++ b/modules/intermodal/src/eval/comparator/intermodal_comparator.cc @@ -202,18 +202,21 @@ bool check(int id, std::vector const& responses, return; } + auto const jf = pretty_print ? json_format::DEFAULT_FLATBUFFERS + : json_format::SINGLE_LINE; + std::ofstream out{ (fail_path / fmt::format("{}_{}.json", std::to_string(id), file_identifier(response_files[file_idx]))) .string()}; - out << responses[file_idx]->to_json(!pretty_print) << std::endl; + out << responses[file_idx]->to_json(jf) << std::endl; if (!queries.empty()) { std::ofstream query_out{ (fail_path / fmt::format("{}_{}.json", std::to_string(id), file_identifier(query_files[file_idx]))) .string()}; - query_out << queries[file_idx]->to_json(!pretty_print) << std::endl; + query_out << queries[file_idx]->to_json(jf) << std::endl; } }; diff --git a/modules/intermodal/src/eval/generator/intermodal_generator.cc b/modules/intermodal/src/eval/generator/intermodal_generator.cc index 598cc7a9f..17909c489 100644 --- a/modules/intermodal/src/eval/generator/intermodal_generator.cc +++ b/modules/intermodal/src/eval/generator/intermodal_generator.cc @@ -656,7 +656,7 @@ void write_query(schedule const& sched, point_generator& point_gen, int id, for (auto const& [out_file, fbbp] : utl::zip(out_files, fbbs)) { auto& fbb = *fbbp; - out_file << make_msg(fbb)->to_json(true) << "\n"; + out_file << make_msg(fbb)->to_json(json_format::SINGLE_LINE) << "\n"; } } diff --git a/modules/paxforecast/src/monitoring_update.cc b/modules/paxforecast/src/monitoring_update.cc index 90f85bf86..ae15022e9 100644 --- a/modules/paxforecast/src/monitoring_update.cc +++ b/modules/paxforecast/src/monitoring_update.cc @@ -419,7 +419,8 @@ void on_monitoring_update(paxforecast& mod, paxmon_data& data, MOTIS_START_TIMING(write_load_forecast); if (mod.forecast_file_.is_open() && uv.id_ == 0) { scoped_timer const load_forecast_msg_timer{"load forecast to json"}; - mod.forecast_file_ << forecast_msg->to_json(true) << std::endl; + mod.forecast_file_ << forecast_msg->to_json(json_format::SINGLE_LINE) + << std::endl; } MOTIS_STOP_TIMING(write_load_forecast); tick_stats.t_write_load_forecast_ = MOTIS_TIMING_MS(write_load_forecast); diff --git a/modules/routing/src/eval/query_rewriter.cc b/modules/routing/src/eval/query_rewriter.cc index 623502fcf..2c098b9b9 100644 --- a/modules/routing/src/eval/query_rewriter.cc +++ b/modules/routing/src/eval/query_rewriter.cc @@ -1,98 +1,99 @@ -#include "motis/routing/eval/commands.h" - -#include -#include - -#include "boost/algorithm/string.hpp" - -#include "motis/module/message.h" - -#include "conf/configuration.h" -#include "conf/options_parser.h" - -#include "version.h" - -using namespace motis; -using motis::module::make_msg; - -namespace motis::routing::eval { - -struct rewrite_options : public conf::configuration { - rewrite_options() : configuration{"Rewrite options"} { - param(in_path_, "in", "Input file path"); - param(out_path_, "out", "Output file path"); - param(new_target_, "target", "New target"); - } - std::string in_path_, out_path_, new_target_; -}; - -int rewrite_queries(int argc, char const** argv) { - rewrite_options opt; - - try { - conf::options_parser parser({&opt}); - parser.read_command_line_args(argc, argv, false); - - if (parser.help()) { - std::cout << "\n\tMOTIS v" << short_version() << "\n\n"; - parser.print_help(std::cout); - return 0; - } else if (parser.version()) { - std::cout << "MOTIS v" << long_version() << "\n"; - return 0; - } - - parser.read_configuration_file(true); - parser.print_used(std::cout); - } catch (std::exception const& e) { - std::cout << "options error: " << e.what() << "\n"; - return 1; - } - - std::ifstream in{opt.in_path_.c_str()}; - std::ofstream out{opt.out_path_.c_str()}; - - in.exceptions(std::ifstream::failbit | std::ifstream::badbit); - out.exceptions(std::ifstream::failbit | std::ifstream::badbit); - - auto count = 0U; - std::string json; - while (!in.eof() && in.peek() != EOF) { - std::getline(in, json); - auto const msg_in = make_msg(json); - - motis::module::message_creator fbb; - - flatbuffers::Offset content; - switch (msg_in->get()->content_type()) { - case MsgContent_RoutingResponse: { - content = - motis_copy_table(RoutingResponse, fbb, msg_in->get()->content()) - .Union(); - } break; - case MsgContent_RoutingRequest: { - content = - motis_copy_table(RoutingRequest, fbb, msg_in->get()->content()) - .Union(); - } break; - case MsgContent_IntermodalRoutingRequest: { - using motis::intermodal::IntermodalRoutingRequest; - content = motis_copy_table(IntermodalRoutingRequest, fbb, - msg_in->get()->content()) - .Union(); - } break; - default: std::cerr << "unsupported message content type\n"; return 1; - } - fbb.create_and_finish(msg_in->get()->content_type(), content.Union(), - opt.new_target_, DestinationType_Module, - msg_in->id()); - out << make_msg(fbb)->to_json(true) << "\n"; - - ++count; - } - - std::cout << "rewrote " << count << " queries\n"; - return 0; -} - -} // namespace motis::routing::eval +#include "motis/routing/eval/commands.h" + +#include +#include + +#include "boost/algorithm/string.hpp" + +#include "motis/module/message.h" + +#include "conf/configuration.h" +#include "conf/options_parser.h" + +#include "version.h" + +using namespace motis; +using motis::module::json_format; +using motis::module::make_msg; + +namespace motis::routing::eval { + +struct rewrite_options : public conf::configuration { + rewrite_options() : configuration{"Rewrite options"} { + param(in_path_, "in", "Input file path"); + param(out_path_, "out", "Output file path"); + param(new_target_, "target", "New target"); + } + std::string in_path_, out_path_, new_target_; +}; + +int rewrite_queries(int argc, char const** argv) { + rewrite_options opt; + + try { + conf::options_parser parser({&opt}); + parser.read_command_line_args(argc, argv, false); + + if (parser.help()) { + std::cout << "\n\tMOTIS v" << short_version() << "\n\n"; + parser.print_help(std::cout); + return 0; + } else if (parser.version()) { + std::cout << "MOTIS v" << long_version() << "\n"; + return 0; + } + + parser.read_configuration_file(true); + parser.print_used(std::cout); + } catch (std::exception const& e) { + std::cout << "options error: " << e.what() << "\n"; + return 1; + } + + std::ifstream in{opt.in_path_.c_str()}; + std::ofstream out{opt.out_path_.c_str()}; + + in.exceptions(std::ifstream::failbit | std::ifstream::badbit); + out.exceptions(std::ifstream::failbit | std::ifstream::badbit); + + auto count = 0U; + std::string json; + while (!in.eof() && in.peek() != EOF) { + std::getline(in, json); + auto const msg_in = make_msg(json); + + motis::module::message_creator fbb; + + flatbuffers::Offset content; + switch (msg_in->get()->content_type()) { + case MsgContent_RoutingResponse: { + content = + motis_copy_table(RoutingResponse, fbb, msg_in->get()->content()) + .Union(); + } break; + case MsgContent_RoutingRequest: { + content = + motis_copy_table(RoutingRequest, fbb, msg_in->get()->content()) + .Union(); + } break; + case MsgContent_IntermodalRoutingRequest: { + using motis::intermodal::IntermodalRoutingRequest; + content = motis_copy_table(IntermodalRoutingRequest, fbb, + msg_in->get()->content()) + .Union(); + } break; + default: std::cerr << "unsupported message content type\n"; return 1; + } + fbb.create_and_finish(msg_in->get()->content_type(), content.Union(), + opt.new_target_, DestinationType_Module, + msg_in->id()); + out << make_msg(fbb)->to_json(json_format::SINGLE_LINE) << "\n"; + + ++count; + } + + std::cout << "rewrote " << count << " queries\n"; + return 0; +} + +} // namespace motis::routing::eval diff --git a/modules/routing/src/eval/result_comparator.cc b/modules/routing/src/eval/result_comparator.cc index 424581fd8..bb9d06fd3 100644 --- a/modules/routing/src/eval/result_comparator.cc +++ b/modules/routing/src/eval/result_comparator.cc @@ -272,13 +272,14 @@ bool analyze_result(int i, std::tuple const& res, ++stats.matches_; } else { ++stats.mismatches_; - failed_queries << q->to_json(true) << "\n"; + failed_queries << q->to_json(json_format::SINGLE_LINE) << "\n"; failed_queries.flush(); - write_file(r1->to_json(true), + write_file(r1->to_json(json_format::SINGLE_LINE), "fail_responses/" + std::to_string(i) + "_1.json"); - write_file(r2->to_json(true), + write_file(r2->to_json(json_format::SINGLE_LINE), "fail_responses/" + std::to_string(i) + "_2.json"); - write_file(q->to_json(true), "fail_queries/" + std::to_string(i) + ".json"); + write_file(q->to_json(json_format::SINGLE_LINE), + "fail_queries/" + std::to_string(i) + ".json"); } return true; @@ -351,4 +352,4 @@ int compare(int argc, char const** argv) { return (stats.mismatches_ == 0 && stats.errors_ == 0) ? 0 : 1; } -} // namespace motis::routing::eval \ No newline at end of file +} // namespace motis::routing::eval diff --git a/tools/protocol/README.md b/tools/protocol/README.md index d3aa089c3..d446618bb 100644 --- a/tools/protocol/README.md +++ b/tools/protocol/README.md @@ -74,6 +74,12 @@ Configuration: - `strict-int-types` (bool): Add constraints for min/max values depending on the type width - `strict-unions` (bool): Force matching `_type` tags for unions - `number-formats` (bool): Add `format` annotations for numeric types +- `types-in-unions` (bool): If true, `_type` tags are generated in unions, otherwise the FlatBuffers default is used +- `tagged-type-suffix`: If `types-in-unions` is set to true and a type is used in a union, the generated type + with the type tag uses a name with this suffix added +- `explicit-additional-properties` (bool): If true, explicitly add `"additionalProperties": true` to all schema object + types. Although this is the implicit default, some tools incorrectly use `"additionalProperties": false` as the + default. ### OpenAPI @@ -88,7 +94,13 @@ Configuration: - `3.1.0` - `file`: Output file - `base-uri`: The JSON Schema Base URI -- `ids` (boolean): Include `$id` for schema types +- `ids` (bool): Include `$id` for schema types +- `types-in-unions` (bool): If true, `_type` tags are generated in unions, otherwise the FlatBuffers default is used +- `msg-content-only` (bool): If true, messages only consist of the message content + (compact format, requires `types-in-unions`) +- `explicit-additional-properties` (bool): If true, explicitly add `"additionalProperties": true` to all schema object + types. Although this is the implicit default, some tools incorrectly use `"additionalProperties": false` as the + default. - `info`: The info block for the OpenAPI file (must include at least `title` and `version`) - `externalDocs` (optional): The externalDocs block for the OpenAPI file - `servers` (optional): The servers block for the OpenAPI file @@ -97,7 +109,7 @@ Configuration: - Descriptions of fields with custom types are currently missing, because custom types are referenced using `$ref` and no other sibling elements are allowed in OpenAPI 3.0. -- Matching union type tags are not enforced by the schema. +- Matching union type tags are not enforced by the schema if `types-in-unions == false`. - Only one example per type is allowed (the first example is used, all others are ignored). ### TypeScript Type Definitions @@ -155,8 +167,9 @@ The top level keys are the API paths, and each path has the following properties - `summary`: Short summary of the operation - `description` (optional): Longer description of the operation - `tags` (optional): A list of tags for the operation +- `deprecated` (optional, bool): Marks the operation as deprecated. - `input` (optional): Full type name for the request. If missing, a `GET` request is used. -- `output`: +- `output` (optional): Non-error response. If missing, `motis.MotisSuccess` is generated. - `type`: Full type name for the response - `description`: Description for the response diff --git a/tools/protocol/protocol.config.yaml b/tools/protocol/protocol.config.yaml index f78c14610..7b149bcb2 100644 --- a/tools/protocol/protocol.config.yaml +++ b/tools/protocol/protocol.config.yaml @@ -13,6 +13,11 @@ output: strict-unions: true exclude: &public-exclude - motis.import.* + - motis.Message + - motis.Destination + - motis.DestinationType + - motis.MsgContent + - motis.HTTP* openapi-3.1: format: openapi version: 3.1.0 @@ -22,6 +27,12 @@ output: info: &openapi-info title: MOTIS API version: 0.0.0 + description: | + For more information see: + + - [MOTIS Project Website](https://motis-project.de/) + - [MOTIS GitHub Project](https://github.com/motis-project/motis) + - [MOTIS GitHub Wiki](https://github.com/motis-project/motis/wiki) contact: name: MOTIS Project url: https://github.com/motis-project diff --git a/tools/protocol/src/doc/inout.ts b/tools/protocol/src/doc/inout.ts index 51dbe52c1..cad692387 100644 --- a/tools/protocol/src/doc/inout.ts +++ b/tools/protocol/src/doc/inout.ts @@ -193,6 +193,7 @@ function readPaths(ctx: DocContext) { input: props.input, output: undefined, operationId: props.operationId, + deprecated: props.deprecated, }; if (dp.input) { if (!ctx.schema.types.has(dp.input)) { diff --git a/tools/protocol/src/doc/types.ts b/tools/protocol/src/doc/types.ts index cdf06c96d..3eecbf6ba 100644 --- a/tools/protocol/src/doc/types.ts +++ b/tools/protocol/src/doc/types.ts @@ -22,6 +22,7 @@ export interface DocPath { input?: string | undefined; output?: DocResponse | undefined; operationId?: string | undefined; + deprecated?: boolean | undefined; } export interface DocResponse { diff --git a/tools/protocol/src/output/json-schema/context.ts b/tools/protocol/src/output/json-schema/context.ts index e1ca9657e..caaada41a 100644 --- a/tools/protocol/src/output/json-schema/context.ts +++ b/tools/protocol/src/output/json-schema/context.ts @@ -7,8 +7,16 @@ export interface JSContext { typeFilter: TypeFilter; baseUri: URL; jsonSchema: Map; + taggedToUntaggedType: Map; // tagged -> untagged fqtn + untaggedToTaggedType: Map; // tagged -> untagged fqtn strictIntTypes: boolean; numberFormats: boolean; strictUnions: boolean; + typesInUnions: boolean; getRefUrl: (fqtn: string[]) => string; + getTaggedType: (fqtn: string[]) => string[]; + typeKey: string; + includeOpenApiDiscriminators: boolean; + constAsEnum: boolean; + explicitAdditionalProperties: boolean; } diff --git a/tools/protocol/src/output/json-schema/output.ts b/tools/protocol/src/output/json-schema/output.ts index 9925d86c3..8bcb40045 100644 --- a/tools/protocol/src/output/json-schema/output.ts +++ b/tools/protocol/src/output/json-schema/output.ts @@ -4,7 +4,7 @@ import path from "path"; import { TypeFilter, includeType } from "@/filter/type-filter"; import { JSContext } from "@/output/json-schema/context"; import { basicTypeToJS } from "@/output/json-schema/primitive-types"; -import { JSONSchema } from "@/output/json-schema/types"; +import { JSONDiscriminator, JSONSchema } from "@/output/json-schema/types"; import { FieldType, SchemaType, @@ -46,10 +46,15 @@ export function writeJsonSchemaOutput( null, !!config["strict-int-types"], !!config["number-formats"], - config["strict-unions"] !== false + config["strict-unions"] !== false, + config["types-in-unions"] !== false, + false, + false, + !!config["explicit-additional-properties"], + config["tagged-type-suffix"] || "T" ); - const defs = getJSONSchemaTypes(ctx); + const { types: defs } = getJSONSchemaTypes(ctx); if (outputDir) { console.log(`writing json schema files to: ${outputDir}`); @@ -94,29 +99,56 @@ export function createJSContext( getRefUrl: ((fqtn: string[]) => string) | null = null, strictIntTypes = false, numberFormats = false, - strictUnions = true + strictUnions = true, + typesInUnions = true, + includeOpenApiDiscriminators = false, + constAsEnum = false, + explicitAdditionalProperties = false, + taggedTypeFnOrSuffix: ((fqtn: string[]) => string[]) | string = "T", + typeKey = "_type" ): JSContext { const ctx: JSContext = { schema, typeFilter, baseUri, jsonSchema: new Map(), + taggedToUntaggedType: new Map(), + untaggedToTaggedType: new Map(), strictIntTypes, numberFormats, strictUnions, + typesInUnions, getRefUrl: getRefUrl || ((fqtn) => getDefaultRefUrl(ctx, fqtn)), + getTaggedType: + typeof taggedTypeFnOrSuffix === "function" + ? taggedTypeFnOrSuffix + : (fqtn) => getDefaultTaggedType(fqtn, taggedTypeFnOrSuffix), + typeKey, + includeOpenApiDiscriminators, + constAsEnum, + explicitAdditionalProperties, }; return ctx; } -export function getJSONSchemaTypes(ctx: JSContext): Record { +export interface JSONSchemaTypes { + types: Record; + taggedToUntaggedType: Map; // tagged -> untagged fqtn + untaggedToTaggedType: Map; // untagged -> tagged fqtn +} + +export function getJSONSchemaTypes(ctx: JSContext): JSONSchemaTypes { for (const [fqtn, type] of ctx.schema.types) { if (!includeType(ctx.typeFilter, fqtn)) { continue; } convertSchemaType(ctx, fqtn, type); } - return bundleDefs(ctx); + return { + types: bundleDefs(ctx), + taggedToUntaggedType: ctx.taggedToUntaggedType, + untaggedToTaggedType: ctx.untaggedToTaggedType, + }; } function convertSchemaType(ctx: JSContext, fqtn: string, type: SchemaType) { @@ -131,33 +163,83 @@ function convertSchemaType(ctx: JSContext, fqtn: string, type: SchemaType) { break; case "union": { const union: JSONSchema = { ...base }; - const unionTags: JSONSchema = { - $id: ctx.baseUri.href + [...type.ns, `${type.name}Type`].join("/"), - type: "string", - }; - union.anyOf = []; - unionTags.enum = []; - for (const value of type.values) { - const fqtn = value.typeRef.resolvedFqtn; - const fqtnStr = fqtn.join("."); - if (includeType(ctx.typeFilter, fqtnStr)) { + if (ctx.typesInUnions) { + union.oneOf = []; + const discriminator: JSONDiscriminator = { + propertyName: ctx.typeKey, + mapping: {}, + }; + for (const value of type.values) { const fqtn = value.typeRef.resolvedFqtn; - union.anyOf.push({ $ref: ctx.getRefUrl(fqtn) }); - unionTags.enum.push(fqtn[fqtn.length - 1]); + const fqtnStr = fqtn.join("."); + if (includeType(ctx.typeFilter, fqtnStr)) { + const taggedFqtn = ctx.getTaggedType(fqtn); + const refUrl = ctx.getRefUrl(taggedFqtn); + union.oneOf.push({ $ref: refUrl }); + discriminator.mapping[fqtn[fqtn.length - 1]] = refUrl; + } + } + if (ctx.includeOpenApiDiscriminators) { + union.discriminator = discriminator; } + ctx.jsonSchema.set(fqtn, union); + } else { + const unionTags: JSONSchema = { + $id: ctx.baseUri.href + [...type.ns, `${type.name}Type`].join("/"), + type: "string", + }; + union.anyOf = []; + unionTags.enum = []; + for (const value of type.values) { + const fqtn = value.typeRef.resolvedFqtn; + const fqtnStr = fqtn.join("."); + if (includeType(ctx.typeFilter, fqtnStr)) { + const fqtn = value.typeRef.resolvedFqtn; + union.anyOf.push({ $ref: ctx.getRefUrl(fqtn) }); + unionTags.enum.push(fqtn[fqtn.length - 1]); + } + } + ctx.jsonSchema.set(fqtn, union); + ctx.jsonSchema.set(`${fqtn}Type`, unionTags); } - ctx.jsonSchema.set(fqtn, union); - ctx.jsonSchema.set(`${fqtn}Type`, unionTags); break; } case "table": { - ctx.jsonSchema.set( - fqtn, - addTableProperties(ctx, type, { - ...base, - type: "object", - }) - ); + const untagged = !ctx.typesInUnions || type.usedInTable; + const tagged = ctx.typesInUnions && type.usedInUnion; + if (untagged) { + ctx.jsonSchema.set( + fqtn, + addTableProperties( + ctx, + type, + { + ...base, + type: "object", + }, + false + ) + ); + } + if (tagged) { + const taggedBase = getTaggedBaseJSProps(ctx, type); + const taggedType = ctx.getTaggedType(fqtn.split(".")); + const taggedTypeStr = taggedType.join("."); + ctx.jsonSchema.set( + taggedTypeStr, + addTableProperties( + ctx, + type, + { + ...taggedBase, + type: "object", + }, + true + ) + ); + ctx.taggedToUntaggedType.set(taggedTypeStr, fqtn); + ctx.untaggedToTaggedType.set(fqtn, taggedTypeStr); + } break; } } @@ -175,10 +257,21 @@ function fieldTypeToJS(ctx: JSContext, type: FieldType): JSONSchema { throw new Error(`unhandled field type: ${JSON.stringify(type)}`); } -function addTableProperties(ctx: JSContext, type: TableType, js: JSONSchema) { +function addTableProperties( + ctx: JSContext, + type: TableType, + js: JSONSchema, + tagged: boolean +) { const props: { [name: string]: JSONSchema } = {}; const unionCases: JSONSchema[] = []; const required: string[] = []; + + if (tagged) { + props[ctx.typeKey] = getConstString(ctx, type.name); + required.push(ctx.typeKey); + } + for (const field of type.fields) { if (field.type.c === "custom") { const fqtn = field.type.type.resolvedFqtn.join("."); @@ -190,7 +283,7 @@ function addTableProperties(ctx: JSContext, type: TableType, js: JSONSchema) { })` ); } - if (resolvedType.type === "union") { + if (resolvedType.type === "union" && !ctx.typesInUnions) { const tagName = `${field.name}_type`; required.push(tagName); if (ctx.strictUnions) { @@ -203,7 +296,7 @@ function addTableProperties(ctx: JSContext, type: TableType, js: JSONSchema) { const tag = fqtn[fqtn.length - 1]; unionCases.push({ if: { - properties: { [tagName]: { type: "string", const: tag } }, + properties: { [tagName]: getConstString(ctx, tag) }, }, then: { properties: { @@ -233,6 +326,9 @@ function addTableProperties(ctx: JSContext, type: TableType, js: JSONSchema) { if (required.length > 0) { js.required = required; } + if (ctx.explicitAdditionalProperties) { + js.additionalProperties = true; + } return js; } @@ -240,6 +336,13 @@ function getBaseJSProps(ctx: JSContext, type: TypeBase): JSONSchema { return { $id: ctx.baseUri.href + [...type.ns, type.name].join("/") }; } +function getTaggedBaseJSProps(ctx: JSContext, type: TypeBase): JSONSchema { + return { + $id: + ctx.baseUri.href + ctx.getTaggedType([...type.ns, type.name]).join("/"), + }; +} + function getDefaultRefUrl(ctx: JSContext, fqtn: string[], absolute = false) { return (absolute ? ctx.baseUri.href : ctx.baseUri.pathname) + fqtn.join("/"); } @@ -250,6 +353,20 @@ function getUnionTagRefUrl(ctx: JSContext, baseFqtn: string[]) { return ctx.getRefUrl(fqtn); } +function getDefaultTaggedType(baseFqtn: string[], taggedTypeSuffix: string) { + const fqtn = [...baseFqtn]; + fqtn[fqtn.length - 1] += taggedTypeSuffix; + return fqtn; +} + +function getConstString(ctx: JSContext, value: string): JSONSchema { + if (ctx.constAsEnum) { + return { type: "string", enum: [value] }; + } else { + return { type: "string", const: value }; + } +} + function bundleDefs(ctx: JSContext) { const defs: Record = {}; for (const [fqtn, schema] of ctx.jsonSchema) { diff --git a/tools/protocol/src/output/json-schema/types.ts b/tools/protocol/src/output/json-schema/types.ts index 7b9db63c0..9a2219ac6 100644 --- a/tools/protocol/src/output/json-schema/types.ts +++ b/tools/protocol/src/output/json-schema/types.ts @@ -14,6 +14,12 @@ export type JSONValue = | JSONValue[] | { [name: string]: JSONValue }; +// not json schema, openapi extension +export type JSONDiscriminator = { + propertyName: string; + mapping: { [name: string]: string }; +}; + // just a rough subset export type JSONSchema = { $schema?: string; @@ -26,6 +32,7 @@ export type JSONSchema = { properties?: { [name: string]: JSONSchema }; required?: string[]; + additionalProperties?: boolean | JSONSchema; items?: JSONSchema; @@ -53,4 +60,7 @@ export type JSONSchema = { readOnly?: boolean; writeOnly?: boolean; deprecated?: boolean; + + // openapi extension + discriminator?: JSONDiscriminator; }; diff --git a/tools/protocol/src/output/openapi/context.ts b/tools/protocol/src/output/openapi/context.ts index f5dceef35..a4269a0e6 100644 --- a/tools/protocol/src/output/openapi/context.ts +++ b/tools/protocol/src/output/openapi/context.ts @@ -2,7 +2,7 @@ import { Document } from "yaml"; import { Documentation } from "@/doc/types"; import { TypeFilter } from "@/filter/type-filter"; -import { JSONSchema } from "@/output/json-schema/types"; +import { JSONSchemaTypes } from "@/output/json-schema/output"; import { SchemaTypes } from "@/schema/types"; export const OPEN_API_VERSIONS = ["3.0.3", "3.1.0"] as const; @@ -14,8 +14,10 @@ export interface OpenApiContext { typeFilter: TypeFilter; baseUri: URL; openApiVersion: OpenAPIVersion; - jsonSchema: Record; + jsonSchema: JSONSchemaTypes; doc: Documentation; yd: Document; includeIds: boolean; + typesInUnions: boolean; + msgContentOnly: boolean; } diff --git a/tools/protocol/src/output/openapi/output.ts b/tools/protocol/src/output/openapi/output.ts index 7d4c0aed3..5f32fc099 100644 --- a/tools/protocol/src/output/openapi/output.ts +++ b/tools/protocol/src/output/openapi/output.ts @@ -46,6 +46,11 @@ export function writeOpenAPIOutput( baseUri.pathname += "/"; } + const typesInUnions = config["types-in-unions"] !== false; + const msgContentOnly = typesInUnions && config["msg-content-only"] !== false; + const explicitAdditionalProperties = + config["explicit-additional-properties"] !== false; + const jsCtx = createJSContext( schema, typeFilter, @@ -53,7 +58,11 @@ export function writeOpenAPIOutput( getRefUrl, false, true, - false + false, + typesInUnions, + typesInUnions, + true, + explicitAdditionalProperties ); const jsonSchema = getJSONSchemaTypes(jsCtx); @@ -68,6 +77,8 @@ export function writeOpenAPIOutput( doc, yd, includeIds: config["ids"] !== false, + typesInUnions, + msgContentOnly, }; yd.contents = yd.createNode({}); @@ -88,13 +99,21 @@ export function writeOpenAPIOutput( const oaSchemas = createMap(yd, yd, ["components", "schemas"]); - const types = Object.keys(jsonSchema); + const types = Object.keys(jsonSchema.types); sortTypes(types); for (const fqtn of types) { + const origFqtn = jsonSchema.taggedToUntaggedType.get(fqtn) || fqtn; const oaSchema = createMap(yd, oaSchemas, [fqtn]); - const typeDoc = ctx.doc.types.get(fqtn); - writeSchema(ctx, oaSchema, jsonSchema[fqtn], typeDoc); + const typeDoc = ctx.doc.types.get(origFqtn); + writeSchema( + ctx, + oaSchema, + jsonSchema.types[fqtn], + typeDoc, + undefined, + fqtn + ); } oaSchemas.items.sort((a, b) => @@ -141,26 +160,34 @@ function writeResponse( "application/json", "schema", ]); - oaResponseSchema.set("type", "object"); - oaResponseSchema.set("required", ["content_type", "content"]); - oaResponseSchema.set("properties", { - destination: { - type: "object", - required: ["target"], - properties: { - target: { type: "string", enum: [""] }, - type: { type: "string", enum: ["Module"] }, + if (ctx.msgContentOnly) { + const taggedType = ctx.jsonSchema.untaggedToTaggedType.get(fqtn); + if (!taggedType) { + throw new Error(`OpenAPI: No tagged type for ${fqtn} found`); + } + oaResponseSchema.set("$ref", getRefUrl(taggedType.split("."))); + } else { + oaResponseSchema.set("type", "object"); + oaResponseSchema.set("required", ["content_type", "content"]); + oaResponseSchema.set("properties", { + destination: { + type: "object", + required: ["target"], + properties: { + target: { type: "string", enum: [""] }, + type: { type: "string", enum: ["Module"] }, + }, + }, + content_type: { + type: "string", + enum: [resTypeName], + }, + content: { + $ref: getRefUrl(resType), }, - }, - content_type: { - type: "string", - enum: [resTypeName], - }, - content: { - $ref: getRefUrl(resType), - }, - id: { type: "integer", format: "int32" }, - }); + id: { type: "integer", format: "int32" }, + }); + } } function writePaths(ctx: OpenApiContext) { @@ -184,11 +211,12 @@ function writePaths(ctx: OpenApiContext) { if (path.tags.length > 0) { oaOperation.set("tags", path.tags); } + if (path.deprecated) { + oaOperation.set("deprecated", true); + } if (path.input) { const reqFqtn = path.input; - const reqType = reqFqtn.split("."); - const reqTypeName = reqType[reqType.length - 1]; const oaRequest = createMap(ctx.yd, oaOperation, ["requestBody"]); oaRequest.set("required", true); const oaRequestSchema = createMap(ctx.yd, oaRequest, [ @@ -196,30 +224,40 @@ function writePaths(ctx: OpenApiContext) { "application/json", "schema", ]); - oaRequestSchema.set("type", "object"); - oaRequestSchema.set("required", [ - "destination", - "content_type", - "content", - ]); - oaRequestSchema.set("properties", { - destination: { - type: "object", - required: ["target"], - properties: { - target: { type: "string", enum: [path.path] }, - type: { type: "string", enum: ["Module"] }, + if (ctx.msgContentOnly) { + const taggedType = ctx.jsonSchema.untaggedToTaggedType.get(reqFqtn); + if (!taggedType) { + throw new Error(`OpenAPI: No tagged type for ${reqFqtn} found`); + } + oaRequestSchema.set("$ref", getRefUrl(taggedType.split("."))); + } else { + const reqType = reqFqtn.split("."); + const reqTypeName = reqType[reqType.length - 1]; + oaRequestSchema.set("type", "object"); + oaRequestSchema.set("required", [ + "destination", + "content_type", + "content", + ]); + oaRequestSchema.set("properties", { + destination: { + type: "object", + required: ["target"], + properties: { + target: { type: "string", enum: [path.path] }, + type: { type: "string", enum: ["Module"] }, + }, }, - }, - content_type: { - type: "string", - enum: [reqTypeName], - }, - content: { - $ref: getRefUrl(reqType), - }, - id: { type: "integer", format: "int32" }, - }); + content_type: { + type: "string", + enum: [reqTypeName], + }, + content: { + $ref: getRefUrl(reqType), + }, + id: { type: "integer", format: "int32" }, + }); + } } const oaResponses = createMap(ctx.yd, oaOperation, ["responses"]); @@ -228,7 +266,7 @@ function writePaths(ctx: OpenApiContext) { ctx, oaResponses, "200", - path.output?.type ?? "motis.MotisNoMessage", + path.output?.type ?? "motis.MotisSuccess", path.output?.description ?? "Empty response" ); @@ -241,7 +279,8 @@ function writeSchema( oaSchema: YAMLMap, jsonSchema: JSONSchema, typeDoc?: DocType | undefined, - fieldDoc?: DocField | undefined + fieldDoc?: DocField | undefined, + fqtn?: string | undefined ) { function setKey(key: keyof JSONSchema) { if (key in jsonSchema) { @@ -259,6 +298,10 @@ function writeSchema( if (ctx.includeIds) { setKey("$id"); } + if (fqtn) { + oaSchema.set("title", fqtn); + } + setKey("type"); if (typeDoc) { @@ -304,7 +347,7 @@ function writeSchema( const jsProp = jsProps[key]; const oaProp = createMap(ctx.yd, oaProps, [key]); let fieldDoc = typeDoc?.fields?.get(key); - if (!fieldDoc && key.endsWith("_type")) { + if (!fieldDoc && !ctx.typesInUnions && key.endsWith("_type")) { fieldDoc = { name: key, description: `Type of the \`${key.replace(/_type$/, "")}\` field`, @@ -322,8 +365,10 @@ function writeSchema( setKey("allOf"); setKey("anyOf"); setKey("oneOf"); + setKey("discriminator"); setKey("not"); setKey("if"); setKey("then"); setKey("else"); + setKey("additionalProperties"); } diff --git a/tools/protocol/src/schema/resolver.ts b/tools/protocol/src/schema/resolver.ts index 3e1c1100a..de82dbb7b 100644 --- a/tools/protocol/src/schema/resolver.ts +++ b/tools/protocol/src/schema/resolver.ts @@ -3,7 +3,13 @@ import * as path from "path"; import { AstFieldType, AstTopLevel } from "@/fbs/ast"; import { schema } from "@/fbs/parser/schema"; -import { FieldType, SchemaType, SchemaTypes, TypeRef } from "@/schema/types"; +import { + FieldType, + SchemaType, + SchemaTypes, + TableType, + TypeRef, +} from "@/schema/types"; interface ResolverContext { schema: SchemaTypes; @@ -174,6 +180,8 @@ function extractTypes( metadata: a.metadata, }; }), + usedInUnion: false, + usedInTable: false, }); break; } @@ -207,6 +215,17 @@ function resolveType(ctx: ResolverContext, ref: TypeRef) { } } +function markResolvedType( + ctx: ResolverContext, + ref: TypeRef, + fn: (resolved: TableType) => void +) { + const resolved = ctx.schema.types.get(ref.resolvedFqtn.join(".")); + if (resolved && resolved.type === "table") { + fn(resolved); + } +} + function resolveFieldType(ctx: ResolverContext, ft: FieldType) { switch (ft.c) { case "basic": @@ -214,9 +233,15 @@ function resolveFieldType(ctx: ResolverContext, ft: FieldType) { case "vector": resolveFieldType(ctx, ft.type); break; - case "custom": + case "custom": { resolveType(ctx, ft.type); + markResolvedType( + ctx, + ft.type, + (resolved) => (resolved.usedInTable = true) + ); break; + } } } @@ -228,6 +253,11 @@ function resolveTypes(ctx: ResolverContext) { case "union": for (const uv of t.values) { resolveType(ctx, uv.typeRef); + markResolvedType( + ctx, + uv.typeRef, + (resolved) => (resolved.usedInUnion = true) + ); } break; case "table": @@ -239,5 +269,10 @@ function resolveTypes(ctx: ResolverContext) { } if (ctx.schema.rootType) { resolveType(ctx, ctx.schema.rootType); + markResolvedType( + ctx, + ctx.schema.rootType, + (resolved) => (resolved.usedInTable = true) + ); } } diff --git a/tools/protocol/src/schema/types.ts b/tools/protocol/src/schema/types.ts index 7a94293d2..bb0dcb41a 100644 --- a/tools/protocol/src/schema/types.ts +++ b/tools/protocol/src/schema/types.ts @@ -55,6 +55,8 @@ export interface TableType extends TypeBase { type: "table"; isStruct: boolean; fields: TableField[]; + usedInUnion: boolean; + usedInTable: boolean; } export type SchemaType = EnumType | UnionType | TableType; diff --git a/ui/web/js/main.js b/ui/web/js/main.js index 691137f64..174557b39 100644 --- a/ui/web/js/main.js +++ b/ui/web/js/main.js @@ -16,7 +16,7 @@ ); Promise.all([mapConfigPromise, windowLoadPromise]).then((promises) => { - const mapConfig = promises[0].content; + const mapConfig = promises[0].content ?? promises[0]; let tilesEndpoint = mapConfig.tiles_redirect; if (!tilesEndpoint) {