Skip to content

Commit

Permalink
Always emit filter functionality.
Browse files Browse the repository at this point in the history 
With this patch we decouple whether we emit filter functionality from
the parser being public or not.
  • Loading branch information
@bbannier
bbannier committed Oct 20, 2021
1 parent 125e75c commit d15281b
Show file tree
Hide file tree
Showing 17 changed files with 1,048 additions and 210 deletions.
9 changes: 0 additions & 9 deletions spicy/toolchain/include/ast/types/unit.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,15 +155,6 @@ class Unit : detail::AssignIndices,
*/
bool usesRandomAccess() const { return propertyItem("%random-access").has_value(); }

/**
* Returns true if this unit type supports connecting a filter.
*
* \todo Currently we tie this capability to unit types being public,
* which is just a hack until we get something better. Eventually we
* should support this automatically as needed, through static analysis.
*/
bool supportsFilters() const { return isPublic(); }

/**
* Returns true if this unit type can act as a filter.
*
Expand Down
18 changes: 5 additions & 13 deletions spicy/toolchain/src/compiler/codegen/parser-builder.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,7 @@ struct ProductionVisitor

builder()->addLocal("filtered", hilti::builder::strong_reference(type::Stream()));

if ( unit && unit->supportsFilters() ) {
if ( unit ) {
pb->guardFeatureCode(*unit, "supports_filters", [&]() {
// If we have a filter attached, we initialize it and change to parse from its output.
auto filtered = builder::assign(builder::id("filtered"),
Expand Down Expand Up @@ -377,7 +377,7 @@ struct ProductionVisitor
// stage1 method is already declared (but not
// implemented) by the struct that unit-builder is
// declaring.
if ( unit && unit->supportsFilters() ) {
if ( unit ) {
addParseMethod(id_stage1.str() != "__parse_stage1", id_stage1, build_parse_stage1(), addl_param,
p.location());
addParseMethod(true, id_stage2, build_parse_stage12_or_stage2(false), addl_param, p.location());
Expand Down Expand Up @@ -1724,9 +1724,8 @@ void ParserBuilder::finalizeUnit(bool success, const Location& l) {
else
builder()->addMemberCall(state().self, "__on_0x25_error", {}, l);

if ( unit.supportsFilters() )
guardFeatureCode(unit, "supports_filters",
[&]() { builder()->addCall("spicy_rt::filter_disconnect", {state().self}); });
guardFeatureCode(unit, "supports_filters",
[&]() { builder()->addCall("spicy_rt::filter_disconnect", {state().self}); });

if ( unit.isFilter() )
guardFeatureCode(unit, "is_filter",
Expand All @@ -1736,14 +1735,7 @@ void ParserBuilder::finalizeUnit(bool success, const Location& l) {
builder()->addMemberCall(builder::member(state().self, s.id()), "close", {}, l);
}

static Expression _filters(const ParserState& state) {
hilti::Expression filters;

if ( state.unit.get().supportsFilters() )
return builder::member(state.self, ID("__filters"));

return builder::null();
}
static Expression _filters(const ParserState& state) { return builder::member(state.self, ID("__filters")); }

Expression ParserBuilder::waitForInputOrEod() {
return builder::call("spicy_rt::waitForInputOrEod", {state().data, state().cur, _filters(state())});
Expand Down
21 changes: 8 additions & 13 deletions spicy/toolchain/src/compiler/codegen/unit-builder.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,13 +182,11 @@ Type CodeGen::compileUnit(const type::Unit& unit, bool declare_only) {
if ( unit.isFilter() )
addDeclaration(builder::constant(ID(fmt("__feat%%%s%%is_filter", typeID)), builder::bool_(true)));

if ( unit.supportsFilters() )
addDeclaration(builder::constant(ID(fmt("__feat%%%s%%supports_filters", typeID)), builder::bool_(true)));

addDeclaration(builder::constant(ID(fmt("__feat%%%s%%supports_sinks", typeID)), builder::bool_(true)));

if ( unit.usesRandomAccess() )
addDeclaration(builder::constant(ID(fmt("__feat%%%s%%uses_random_access", typeID)), builder::bool_(true)));

addDeclaration(builder::constant(ID(fmt("__feat%%%s%%supports_filters", typeID)), builder::bool_(true)));
addDeclaration(builder::constant(ID(fmt("__feat%%%s%%supports_sinks", typeID)), builder::bool_(true)));
}

add_hook("0x25_gap", {builder::parameter("seq", type::UnsignedInteger(64)),
Expand Down Expand Up @@ -240,14 +238,11 @@ Type CodeGen::compileUnit(const type::Unit& unit, bool declare_only) {
v.addField(std::move(sink));
}

if ( unit.supportsFilters() ) {
auto filters = hilti::declaration::Field(ID("__filters"),
hilti::type::StrongReference(builder::typeByID("spicy_rt::Filters")),
AttributeSet({Attribute("&internal"),
Attribute("&needed-by-feature",
builder::string("supports_filters"))}));
v.addField(std::move(filters));
}
auto filters =
hilti::declaration::Field(ID("__filters"), hilti::type::StrongReference(builder::typeByID("spicy_rt::Filters")),
AttributeSet({Attribute("&internal"),
Attribute("&needed-by-feature", builder::string("supports_filters"))}));
v.addField(std::move(filters));

if ( unit.isFilter() ) {
auto forward =
Expand Down
3 changes: 0 additions & 3 deletions spicy/toolchain/src/compiler/visitors/validator.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -801,9 +801,6 @@ struct VisitorPost : public hilti::visitor::PreOrder<void, VisitorPost>, public
}

void operator()(const operator_::unit::ConnectFilter& n, position_t p) {
if ( auto x = n.op0().type().tryAs<type::Unit>(); x && ! x->supportsFilters() )
error("unit type does not support filters", p);

if ( const auto& y =
methodArgument(n, 0).type().as<type::StrongReference>().dereferencedType().as<type::Unit>();
! y.isFilter() )
Expand Down
5 changes: 5 additions & 0 deletions tests/Baseline/spicy.optimization.default-parser-functions/log
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
[debug/optimizer] disabling feature 'supports_filters' of type 'foo::P0' since it is not used
[debug/optimizer] disabling feature 'supports_filters' of type 'foo::P1' since it is not used
[debug/optimizer] disabling feature 'supports_sinks' of type 'foo::P0' since it is not used
[debug/optimizer] disabling feature 'supports_sinks' of type 'foo::P1' since it is not used
[debug/optimizer] disabling feature 'supports_sinks' of type 'foo::P2' since it is not used
[debug/optimizer] disabling feature 'uses_random_access' of type 'foo::P2' since it is not used
[debug/optimizer] inlining constant 'foo::__feat%foo__P0%supports_filters'
[debug/optimizer] inlining constant 'foo::__feat%foo__P0%supports_filters'
[debug/optimizer] inlining constant 'foo::__feat%foo__P0%supports_filters'
[debug/optimizer] inlining constant 'foo::__feat%foo__P1%supports_filters'
[debug/optimizer] inlining constant 'foo::__feat%foo__P1%supports_filters'
[debug/optimizer] inlining constant 'foo::__feat%foo__P1%supports_filters'
Expand Down Expand Up @@ -85,6 +89,7 @@
[debug/optimizer] removing field for unused method foo::__register_foo_P2::<anon-struct-9>::parse1
[debug/optimizer] removing field for unused method foo::__register_foo_P2::<anon-struct-9>::parse2
[debug/optimizer] removing field for unused method foo::__register_foo_P2::<anon-struct-9>::parse3
[debug/optimizer] removing unused member 'foo::P0::__filters'
[debug/optimizer] removing unused member 'foo::P0::__sink'
[debug/optimizer] removing unused member 'foo::P1::__filters'
[debug/optimizer] removing unused member 'foo::P1::__sink'
Expand Down
44 changes: 40 additions & 4 deletions tests/Baseline/spicy.optimization.default-parser-functions/noopt.hlt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import hilti;
type P0 = struct {
spicy_rt::Parser __parser &static &internal &needed-by-feature="supports_sinks";
spicy_rt::SinkState __sink &internal &needed-by-feature="supports_sinks";
strong_ref<spicy_rt::Filters> __filters &internal &needed-by-feature="supports_filters";
hook void __on_0x25_init();
hook void __on_0x25_done();
hook void __on_0x25_error();
Expand All @@ -20,6 +21,7 @@ type P0 = struct {
method extern view<stream> parse1(inout value_ref<stream> data, optional<view<stream>> cur = Null, optional<spicy_rt::UnitContext> context) &static &always-emit;
method extern view<stream> parse2(inout value_ref<P0> unit, inout value_ref<stream> data, optional<view<stream>> cur = Null, optional<spicy_rt::UnitContext> context) &static &always-emit;
method extern view<stream> parse3(inout value_ref<spicy_rt::ParsedUnit> gunit, inout value_ref<stream> data, optional<view<stream>> cur = Null, optional<spicy_rt::UnitContext> context) &static &always-emit;
method tuple<view<stream>, int<64>, iterator<stream>> __parse_foo_P0_stage2(inout value_ref<stream> __data, copy view<stream> __cur, copy bool __trim, copy int<64> __lah, copy iterator<stream> __lahe);
} &on-heap;
public type P1 = struct {
spicy_rt::Parser __parser &static &internal &needed-by-feature="supports_sinks";
Expand Down Expand Up @@ -67,24 +69,46 @@ public type P2 = struct {
method tuple<view<stream>, int<64>, iterator<stream>> __parse_foo_P2_stage2(inout value_ref<stream> __data, copy view<stream> __cur, copy bool __trim, copy int<64> __lah, copy iterator<stream> __lahe);
} &on-heap;

const bool __feat%foo__P0%supports_filters = True;
const bool __feat%foo__P0%supports_sinks = True;
const bool __feat%foo__P1%supports_filters = True;
const bool __feat%foo__P1%supports_sinks = True;
const bool __feat%foo__P2%uses_random_access = True;
const bool __feat%foo__P2%supports_filters = True;
const bool __feat%foo__P2%supports_sinks = True;
const bool __feat%foo__P2%uses_random_access = True;

method method tuple<view<stream>, int<64>, iterator<stream>> foo::P0::__parse_stage1(inout value_ref<stream> __data, copy view<stream> __cur, copy bool __trim, copy int<64> __lah, copy iterator<stream> __lahe) &always-emit {
local tuple<view<stream>, int<64>, iterator<stream>> __result;
try {
hilti::debugIndent("spicy");
(*self).__on_0x25_init();
(*self).__on_0x25_done();
hilti::debugDedent("spicy");
__result = (__cur, __lah, __lahe);
local strong_ref<stream> filtered = Null;

if ( __feat%foo__P0%supports_filters )

if ( filtered = spicy_rt::filter_init(self, __data, __cur) ) {
local value_ref<stream> filtered_data = filtered;
(*self).__parse_foo_P0_stage2(filtered_data, (*filtered_data), __trim, __lah, __lahe);
__cur = __cur.advance(|__cur|);

if ( __trim )
(*__data).trim(begin(__cur));

__result = (__cur, __lah, __lahe);
}



if ( ! filtered )
__result = (*self).__parse_foo_P0_stage2(__data, __cur, __trim, __lah, __lahe);

}
catch {
(*self).__on_0x25_error();

if ( __feat%foo__P0%supports_filters )
spicy_rt::filter_disconnect(self);

(*self).__on_0x25_finally();
throw;
}
Expand All @@ -93,6 +117,18 @@ method method tuple<view<stream>, int<64>, iterator<stream>> foo::P0::__parse_st
return __result;
}

method method tuple<view<stream>, int<64>, iterator<stream>> foo::P0::__parse_foo_P0_stage2(inout value_ref<stream> __data, copy view<stream> __cur, copy bool __trim, copy int<64> __lah, copy iterator<stream> __lahe) &always-emit {
local tuple<view<stream>, int<64>, iterator<stream>> __result;
(*self).__on_0x25_done();

if ( __feat%foo__P0%supports_filters )
spicy_rt::filter_disconnect(self);

hilti::debugDedent("spicy");
__result = (__cur, __lah, __lahe);
return __result;
}

method extern method view<stream> foo::P0::parse1(inout value_ref<stream> data, optional<view<stream>> cur = Null, optional<spicy_rt::UnitContext> context) &static &always-emit {
local value_ref<P0> unit = value_ref(default<P0>())value_ref(default<P0>());
local view<stream> ncur = cur ? (*cur) : cast<view<stream>>((*data));
Expand Down
20 changes: 16 additions & 4 deletions tests/Baseline/spicy.optimization.default-parser-functions/opt.hlt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ type P0 = struct {
method extern view<stream> parse1(inout value_ref<stream> data, optional<view<stream>> cur = Null, optional<spicy_rt::UnitContext> context) &static &always-emit;
method extern view<stream> parse2(inout value_ref<P0> unit, inout value_ref<stream> data, optional<view<stream>> cur = Null, optional<spicy_rt::UnitContext> context) &static &always-emit;
method extern view<stream> parse3(inout value_ref<spicy_rt::ParsedUnit> gunit, inout value_ref<stream> data, optional<view<stream>> cur = Null, optional<spicy_rt::UnitContext> context) &static &always-emit;
method tuple<view<stream>, int<64>, iterator<stream>> __parse_foo_P0_stage2(inout value_ref<stream> __data, copy view<stream> __cur, copy bool __trim, copy int<64> __lah, copy iterator<stream> __lahe);
} &on-heap;
public type P1 = struct {
spicy_rt::Parser __parser &static &internal &needed-by-feature="supports_sinks";
Expand All @@ -33,21 +34,24 @@ public type P2 = struct {
method tuple<view<stream>, int<64>, iterator<stream>> __parse_foo_P2_stage2(inout value_ref<stream> __data, copy view<stream> __cur, copy bool __trim, copy int<64> __lah, copy iterator<stream> __lahe);
} &on-heap;

const bool __feat%foo__P0%supports_filters = False;
const bool __feat%foo__P0%supports_sinks = False;
const bool __feat%foo__P1%supports_filters = False;
const bool __feat%foo__P1%supports_sinks = False;
const bool __feat%foo__P2%uses_random_access = False;
const bool __feat%foo__P2%supports_filters = True;
const bool __feat%foo__P2%supports_sinks = False;
const bool __feat%foo__P2%uses_random_access = False;

method method tuple<view<stream>, int<64>, iterator<stream>> foo::P0::__parse_stage1(inout value_ref<stream> __data, copy view<stream> __cur, copy bool __trim, copy int<64> __lah, copy iterator<stream> __lahe) &always-emit {
local tuple<view<stream>, int<64>, iterator<stream>> __result;
try {
hilti::debugIndent("spicy");
default<void>();
default<void>();
hilti::debugDedent("spicy");
__result = (__cur, __lah, __lahe);
local strong_ref<stream> filtered = Null;

if ( ! filtered )
__result = (*self).__parse_foo_P0_stage2(__data, __cur, __trim, __lah, __lahe);

}
catch {
default<void>();
Expand All @@ -59,6 +63,14 @@ method method tuple<view<stream>, int<64>, iterator<stream>> foo::P0::__parse_st
return __result;
}

method method tuple<view<stream>, int<64>, iterator<stream>> foo::P0::__parse_foo_P0_stage2(inout value_ref<stream> __data, copy view<stream> __cur, copy bool __trim, copy int<64> __lah, copy iterator<stream> __lahe) &always-emit {
local tuple<view<stream>, int<64>, iterator<stream>> __result;
default<void>();
hilti::debugDedent("spicy");
__result = (__cur, __lah, __lahe);
return __result;
}

method extern method view<stream> foo::P0::parse1(inout value_ref<stream> data, optional<view<stream>> cur = Null, optional<spicy_rt::UnitContext> context) &static &always-emit {
local value_ref<P0> unit = value_ref(default<P0>())value_ref(default<P0>());
local view<stream> ncur = cur ? (*cur) : cast<view<stream>>((*data));
Expand Down
25 changes: 25 additions & 0 deletions tests/Baseline/spicy.optimization.feature_requirements/log
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,19 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
[debug/optimizer] disabling feature 'is_filter' of type 'foo::X3' since it is not used
[debug/optimizer] disabling feature 'supports_filters' of type 'foo::X1' since it is not used
[debug/optimizer] disabling feature 'supports_filters' of type 'foo::X2' since it is not used
[debug/optimizer] disabling feature 'supports_filters' of type 'foo::X3' since it is not used
[debug/optimizer] disabling feature 'supports_filters' of type 'foo::X4' since it is not used
[debug/optimizer] disabling feature 'supports_filters' of type 'foo::X6' since it is not used
[debug/optimizer] disabling feature 'supports_sinks' of type 'foo::X1' since it is not used
[debug/optimizer] disabling feature 'supports_sinks' of type 'foo::X2' since it is not used
[debug/optimizer] disabling feature 'supports_sinks' of type 'foo::X3' since it is not used
[debug/optimizer] disabling feature 'supports_sinks' of type 'foo::X4' since it is not used
[debug/optimizer] disabling feature 'supports_sinks' of type 'foo::X5' since it is not used
[debug/optimizer] disabling feature 'uses_random_access' of type 'foo::X2' since it is not used
[debug/optimizer] inlining constant 'foo::__feat%foo__X1%supports_filters'
[debug/optimizer] inlining constant 'foo::__feat%foo__X1%supports_filters'
[debug/optimizer] inlining constant 'foo::__feat%foo__X1%supports_filters'
[debug/optimizer] inlining constant 'foo::__feat%foo__X1%uses_random_access'
[debug/optimizer] inlining constant 'foo::__feat%foo__X1%uses_random_access'
[debug/optimizer] inlining constant 'foo::__feat%foo__X1%uses_random_access'
Expand All @@ -20,6 +28,9 @@
[debug/optimizer] inlining constant 'foo::__feat%foo__X1%uses_random_access'
[debug/optimizer] inlining constant 'foo::__feat%foo__X1%uses_random_access'
[debug/optimizer] inlining constant 'foo::__feat%foo__X1%uses_random_access'
[debug/optimizer] inlining constant 'foo::__feat%foo__X2%supports_filters'
[debug/optimizer] inlining constant 'foo::__feat%foo__X2%supports_filters'
[debug/optimizer] inlining constant 'foo::__feat%foo__X2%supports_filters'
[debug/optimizer] inlining constant 'foo::__feat%foo__X2%uses_random_access'
[debug/optimizer] inlining constant 'foo::__feat%foo__X2%uses_random_access'
[debug/optimizer] inlining constant 'foo::__feat%foo__X2%uses_random_access'
Expand All @@ -36,11 +47,20 @@
[debug/optimizer] inlining constant 'foo::__feat%foo__X2%uses_random_access'
[debug/optimizer] inlining constant 'foo::__feat%foo__X3%is_filter'
[debug/optimizer] inlining constant 'foo::__feat%foo__X3%is_filter'
[debug/optimizer] inlining constant 'foo::__feat%foo__X3%supports_filters'
[debug/optimizer] inlining constant 'foo::__feat%foo__X3%supports_filters'
[debug/optimizer] inlining constant 'foo::__feat%foo__X3%supports_filters'
[debug/optimizer] inlining constant 'foo::__feat%foo__X4%is_filter'
[debug/optimizer] inlining constant 'foo::__feat%foo__X4%is_filter'
[debug/optimizer] inlining constant 'foo::__feat%foo__X4%supports_filters'
[debug/optimizer] inlining constant 'foo::__feat%foo__X4%supports_filters'
[debug/optimizer] inlining constant 'foo::__feat%foo__X4%supports_filters'
[debug/optimizer] inlining constant 'foo::__feat%foo__X5%supports_filters'
[debug/optimizer] inlining constant 'foo::__feat%foo__X5%supports_filters'
[debug/optimizer] inlining constant 'foo::__feat%foo__X5%supports_filters'
[debug/optimizer] inlining constant 'foo::__feat%foo__X6%supports_filters'
[debug/optimizer] inlining constant 'foo::__feat%foo__X6%supports_filters'
[debug/optimizer] inlining constant 'foo::__feat%foo__X6%supports_filters'
[debug/optimizer] removing declaration for unused function hilti::abort
[debug/optimizer] removing declaration for unused function hilti::current_time
[debug/optimizer] removing declaration for unused function hilti::debug
Expand Down Expand Up @@ -129,15 +149,20 @@
[debug/optimizer] removing field for unused method foo::__register_foo_X6::<anon-struct-18>::parse1
[debug/optimizer] removing field for unused method foo::__register_foo_X6::<anon-struct-18>::parse2
[debug/optimizer] removing field for unused method foo::__register_foo_X6::<anon-struct-18>::parse3
[debug/optimizer] removing unused member 'foo::X1::__filters'
[debug/optimizer] removing unused member 'foo::X1::__sink'
[debug/optimizer] removing unused member 'foo::X2::__begin'
[debug/optimizer] removing unused member 'foo::X2::__filters'
[debug/optimizer] removing unused member 'foo::X2::__position'
[debug/optimizer] removing unused member 'foo::X2::__position_update'
[debug/optimizer] removing unused member 'foo::X2::__sink'
[debug/optimizer] removing unused member 'foo::X3::__filters'
[debug/optimizer] removing unused member 'foo::X3::__forward'
[debug/optimizer] removing unused member 'foo::X3::__sink'
[debug/optimizer] removing unused member 'foo::X4::__filters'
[debug/optimizer] removing unused member 'foo::X4::__sink'
[debug/optimizer] removing unused member 'foo::X5::__sink'
[debug/optimizer] removing unused member 'foo::X6::__filters'
[debug/optimizer] removing unused member 'spicy_rt::Sink::connect_mime_type'
[debug/optimizer] removing unused member 'spicy_rt::Sink::connect_mime_type'
[debug/optimizer] replacing call to unimplemented function foo::X1::__on_0x25_done with default value
Expand Down
Loading

0 comments on commit d15281b

Please # to comment.