wip actslib

src/cli/main.cpp

@@ -220,7 +220,13 @@ int main(int argc, const char *_argv[]) {
 	int output;
 	{
 		actslib::profiler::ProfiledSection ps{ profiler, tool.m_name ? tool.m_name : "no-tool-name" };
-		output = tool.m_func(proc, argc, argv);
+		try {
+			output = tool.m_func(proc, argc, argv);
+		}
+		catch (std::exception& e) {
+			LOG_ERROR("Unhandled exception: {}", e.what());
+			output = tool::BASIC_ERROR;
+		}
 	}
 
 	LOG_TRACE("Tool took {}s to run with output {}{}", (double)(clock() - beginTime) / CLOCKS_PER_SEC, output, 

src/cli/tools/lib/actslibtest.cpp

@@ -1,11 +1,169 @@
+#include <includes.hpp>
 #include <actslib/actslib.hpp>
 #include <actslib/profiler.hpp>
-#include <includes.hpp>
 #include <actslib/crc.hpp>
 #include <actslib/hdt.hpp>
+#include <actslib/data/kmerger.hpp>
+#include <actslib/data/iterator.hpp>
+#include <actslib/rdf/raio.hpp>
 #include <mio.hpp>
 
 namespace {
+	using namespace actslib;
+	class KMergerTest : public actslib::data::KMergerConfig {
+
+		actslib::rdf::RDFParser& parser;
+		size_t tripleId{ 1 };
+		std::string buffers[rdf::RDF_TRIPLE_COUNT]{};
+		size_t bufferOffsets[rdf::RDF_TRIPLE_COUNT]{};
+		std::vector<size_t> strings[rdf::RDF_TRIPLE_COUNT]{};
+	public:
+
+		KMergerTest(actslib::rdf::RDFParser& parser, size_t bufferSize) : parser(parser) {
+			for (auto& buff : buffers) {
+				buff.resize(bufferSize);
+			}
+		}
+
+		void CreateDefaultChunk(const std::filesystem::path& chunkLocation) override {
+			std::ofstream out{ chunkLocation, std::ios::binary };
+
+			if (!out) throw std::runtime_error("Can't create default chunk");
+
+			actslib::rdf::raio::CompressComponentWriter w{ out };
+
+			w.WriteEnd();
+
+			out.close();
+		}
+
+		inline bool WriteChunkData(const rdf::Triple& triple, bool force, size_t index) {
+			auto& stringss = strings[rdf::RDF_SUBJECT];
+			auto& bufferss = buffers[rdf::RDF_SUBJECT];
+
+			auto& stringsp = strings[rdf::RDF_PREDICATE];
+			auto& buffersp = buffers[rdf::RDF_PREDICATE];
+
+			auto& stringso = strings[rdf::RDF_OBJECT];
+			auto& bufferso = buffers[rdf::RDF_OBJECT];
+
+			// check if we can add the components (reallocate if we force them)
+			if (bufferOffsets[rdf::RDF_SUBJECT] + triple.subject->length + sizeof(size_t) + 1 > bufferss.size()) {
+				if (!force) {
+					return false;
+				}
+				bufferss.resize((size_t)((triple.subject->length + sizeof(size_t) + 1) * 1.5));
+			}
+			if (bufferOffsets[rdf::RDF_PREDICATE] + triple.predicate->length + sizeof(size_t) + 1 > bufferss.size()) {
+				if (!force) {
+					return false;
+				}
+				buffersp.resize((size_t)((triple.predicate->length + sizeof(size_t) + 1) * 1.5));
+			}
+			if (bufferOffsets[rdf::RDF_OBJECT] + triple.object->length + sizeof(size_t) + 1 > bufferso.size()) {
+				if (!force) {
+					return false;
+				}
+				bufferso.resize((size_t)((triple.object->length + sizeof(size_t) + 1) * 1.5));
+			}
+
+			size_t offs = bufferOffsets[rdf::RDF_SUBJECT];
+			stringss.push_back(offs);
+			bufferOffsets[rdf::RDF_SUBJECT] += triple.subject->length + sizeof(size_t) + 1;
+
+			size_t offp = bufferOffsets[rdf::RDF_PREDICATE];
+			stringsp.push_back(offp);
+			bufferOffsets[rdf::RDF_PREDICATE] += triple.predicate->length + sizeof(size_t) + 1;
+
+			size_t offo = bufferOffsets[rdf::RDF_OBJECT];
+			stringso.push_back(offo);
+			bufferOffsets[rdf::RDF_OBJECT] += triple.object->length + sizeof(size_t) + 1;
+
+			*reinterpret_cast<size_t*>(bufferss.data() + offs) = index;
+			*reinterpret_cast<size_t*>(buffersp.data() + offp) = index;
+			*reinterpret_cast<size_t*>(bufferso.data() + offo) = index;
+
+			memcpy(bufferss.data() + offs + sizeof(size_t), triple.subject->buffer + triple.subject->offset, triple.subject->length);
+			memcpy(buffersp.data() + offp + sizeof(size_t), triple.predicate->buffer + triple.predicate->offset, triple.predicate->length);
+			memcpy(bufferso.data() + offo + sizeof(size_t), triple.object->buffer + triple.object->offset, triple.object->length);
+
+			// end str
+			bufferss.data()[bufferOffsets[rdf::RDF_SUBJECT]] = 0;
+			buffersp.data()[bufferOffsets[rdf::RDF_PREDICATE]] = 0;
+			bufferso.data()[bufferOffsets[rdf::RDF_OBJECT]] = 0;
+
+			return true;
+		}
+
+		inline void SortBuffer(rdf::RDFComponentType comp) {
+			auto& str = strings[comp];
+			const auto& buff = buffers[comp];
+
+			std::sort(str.begin(), str.end(), [&buff](size_t off1, size_t off2) -> bool {
+				const char* s1 = &buff[off1 + sizeof(size_t)];
+				const char* s2 = &buff[off2 + sizeof(size_t)];
+				return strcmp(s1, s2) < 0;
+			});
+
+		}
+
+		bool CreateChunk(const std::filesystem::path& chunkLocation) override {
+			if (!parser) {
+				return false;
+			}
+			WriteChunkData(*parser, true, tripleId);
+
+			++parser;
+			tripleId++;
+
+			while (parser) {
+				if (!WriteChunkData(*parser, true, tripleId)) {
+					break;
+				}
+				++parser;
+				tripleId++;
+			}
+
+			// sort strings
+			SortBuffer(rdf::RDF_SUBJECT);
+			SortBuffer(rdf::RDF_PREDICATE);
+			SortBuffer(rdf::RDF_OBJECT);
+
+			// write strings
+
+
+			return true;
+		}
+
+		void MergeChunks(const std::vector<actslib::data::KMergerChunk>& chunks, const std::filesystem::path& chunkLocation) override {
+
+		}
+	};
+
+	void MergeNumbers() {
+		size_t a1[]{ 1, 4, 7, 9 };
+		size_t a2[]{ 2, 3, 5, 6, 8, 10 };
+
+		struct Arr {
+			size_t* start;
+			size_t* end;
+		};
+
+		std::vector<Arr> vec{
+			{ std::begin(a1), std::end(a1) },
+			{ std::begin(a2), std::end(a2) },
+		};
+
+		actslib::data::iterator::AllocatedMergeAIterator<size_t, Arr> merger{
+			vec,
+			[](Arr& e) { return std::make_shared<actslib::data::iterator::HandleAIterator<size_t, size_t*>>(e.start, e.end); }
+		};
+
+		for (; merger; ++merger) {
+			LOG_INFO("{}", *merger);
+		}
+	}
+
 	int actslibtest(Process& proc, int argc, const char* argv[]) {
 		const char* type;
 		if (argc < 3) {
@@ -57,7 +215,7 @@ namespace {
 			LOG_INFO("===============");
 			cookie.Save("testcookie.cookie");
 
-			std::ifstream is{ "testcookie.cookie", std::ios::binary};
+			std::ifstream is{ "testcookie.cookie", std::ios::binary };
 
 			if (!is) {
 				LOG_ERROR("Can't open {}", "testcookie.cookie");
@@ -80,6 +238,78 @@ namespace {
 			}
 			is.close();
 			std::filesystem::remove("testcookie.cookie");
+		} 
+		else if (!_strcmpi(type, "nt")) {
+
+			if (argc < 4) {
+				LOG_ERROR("{} {} nt file", argv[0], argv[1]);
+				return tool::BAD_USAGE;
+			}
+
+			std::filesystem::path path{ argv[3] };
+
+			auto format = actslib::rdf::GuessFormat(path);
+
+			LOG_INFO("Format: {}", actslib::rdf::FormatName(format));
+
+			std::ifstream is{ path };
+
+			if (!is) {
+				LOG_ERROR("Can't open {}", path.string());
+				return tool::BASIC_ERROR;
+			}
+
+			std::unique_ptr<actslib::rdf::RDFParser> parserPtr = actslib::rdf::CreateParser(format, is, "http://atesab.fr/#");
+
+			for (auto& parser = *parserPtr; parser; ++parser) {
+				const auto& triple = *parser;
+				LOG_INFO("triple: '{}'", triple);
+			}
+
+			is.close();
+		}
+		else if (!_strcmpi(type, "merge")) {
+			MergeNumbers();
+		}
+		else if (!_strcmpi(type, "raio")) {
+			std::ofstream out{ "test.bin", std::ios::binary };
+
+			if (!out) {
+				LOG_ERROR("Can't open");
+				return tool::BASIC_ERROR;
+			}
+
+			actslib::rdf::raio::CompressComponentWriter writer{ out };
+
+			writer.WriteNode(1, actslib::rdf::StringComponent{ "test1" });
+			writer.WriteNode(2, actslib::rdf::StringComponent{ "test     2" });
+			writer.WriteNode(3, actslib::rdf::StringComponent{ "aaatest3" });
+			writer.WriteEnd();
+
+			out.close();
+
+
+			std::ifstream in{ "test.bin", std::ios::binary };
+
+			if (!in) {
+				LOG_ERROR("Can't open 2");
+				return tool::BASIC_ERROR;
+			}
+
+			actslib::rdf::raio::CompressComponentReader reader{ in };
+
+			for (; reader; ++reader) {
+
+				auto& n = *reader;
+				std::cout << "'" << n.comp << "' " << n.id << "\n";
+			}
+
+			if (!reader.CheckEnd()) {
+				LOG_ERROR("Bad end");
+			}
+
+			in.close();
+
 		}
 		return tool::OK;
 	}

src/lib/actslib/actslib.cpp

@@ -1,2 +1,31 @@
 #include "actslib.hpp"
 
+
+namespace actslib {
+
+	template<size_t Buffers, size_t Size>
+	struct StrBuff {
+		char buffer[Buffers][Size]{};
+		size_t loc{};
+
+		constexpr size_t NextId() {
+			return loc = (loc + 1) % Buffers;
+		}
+
+	};
+
+	const char* va(const char* fmt, ...) {
+		static thread_local StrBuff<0x10, 0x200> buf{};
+
+		va_list va;
+
+        va_start(va, fmt);
+
+		auto& buff = buf.buffer[buf.NextId()];
+		sprintf_s(buff, fmt);
+
+		va_end(va);
+
+		return buff;
+	}
+}

src/lib/actslib/actslib.hpp

@@ -10,7 +10,34 @@
 #include <filesystem>
 #include <cassert>
 #include <functional>
+#include <stdarg.h>
 
 namespace actslib {
 
+	const char* va(const char* fmt, ...);
+
+	template<typename Type>
+	struct BasicFormatter {
+		template<class ParseContext>
+		constexpr ParseContext::iterator parse(ParseContext& ctx) {
+			auto it = ctx.begin();
+			if (it == ctx.end()) {
+				return it;
+			}
+			if (*it != '}') {
+				throw std::format_error("Invalid format args.");
+			}
+
+			return it;
+		}
+
+		template<class FmtContext>
+		FmtContext::iterator format(const Type& p, FmtContext& ctx) const {
+			std::ostringstream out;
+
+			out << p;
+
+			return std::ranges::copy(std::move(out).str(), ctx.out()).out;
+		}
+	};
 }