#include #include #include #include #include #include #include /* Metabinary is envisioned as a standard protocol for; * high performance networking, as well as a versatile storage format for data; * encoding both large blocks and small metadata tokens in a single binary file. * */ namespace metabinary { // TODO: Use std::vector for byte buffer manipulations instead of arrays (trivial to change size) // TODO: Implement byte_array_tag // TODO: Implement var_int_tag #pragma region Internal-ish Utilities #pragma region Write Primitives // Handles endianness of integer types // All* integer types are written to NetworkByteOrder (big endian) // and read back into HostByteOrder (lil endian for x86) // Writes an 8-bit unsigned int (1-byte) to the buffer at the given index static int write_uint8(uint8_t* buf, int index, uint8_t val) { // *No need to swap NetworkOrder for single byte + It Breaks int offset = index; uint8_t data = val; memcpy(buf+offset, &data, sizeof(uint8_t)); offset+=sizeof(uint8_t); return offset-index; } // Writes a 16-bit unsigned int (2-bytes) to the buffer at the given index static int write_uint16(uint8_t* buf, int index, uint16_t val) { int offset = index; uint16_t data = htons(val); memcpy(buf+offset, &data, sizeof(uint16_t)); offset += sizeof(uint16_t); return offset - index; } // Writes a 32-bit unsigned int (4 bytes) to the buffer at the given index static int write_uint32(uint8_t* buf, int index, uint32_t val) { int offset = index; uint32_t data = htonl(val); memcpy(buf+offset, &data, sizeof(uint32_t)); offset += sizeof(uint32_t); return offset - index; } // Writes a 64-bit unsigned int (8 bytes) to the buffer at the given index static int write_uint64(uint8_t* buf, int index, uint64_t val) { int offset = index; uint64_t data = htonl(val); memcpy(buf+offset, &data, sizeof(uint64_t)); offset += sizeof(uint64_t); return offset - index; } // Writes an 8-bit signed int (1-byte) to the buffer at the index static int write_int8(uint8_t* buf, int index, int8_t val) { int offset = index; int8_t data = val; memcpy(buf+offset, &data, sizeof(int8_t)); offset += sizeof(int8_t); return offset - index; } // Writes a 16-bit signed int (2-bytes) to the buffer at the index static int write_int16(uint8_t* buf, int index, int16_t val) { int offset = index; int16_t data = htons(val); memcpy(buf+offset, &data, sizeof(int16_t)); offset += sizeof(int16_t); return offset - index; } // Writes a 32-bit signed int (4-bytes) to the buffer at the index static int write_int32(uint8_t* buf, int index, int32_t val) { int offset = index; int32_t data = htonl(val); memcpy(buf+offset, &data, sizeof(int32_t)); offset += sizeof(int32_t); return offset - index; } // Writes a 64-bit signed int (8-bytes) to the buffer at the index static int write_int64(uint8_t* buf, int index, int64_t val) { int offset = index; int64_t data = htonl(val); memcpy(buf+offset, &data, sizeof(int64_t)); offset += sizeof(int64_t); return offset - index; } // Writes a 32-bit IEEE 754 float static int write_float(uint8_t* buf, int index, float val) { int offset = index; memcpy(buf+offset, &val, sizeof(val)); offset += sizeof(float); return offset - index; } // Writes a 64-bit IEEE 754 float static int write_double(uint8_t* buf, int index, double val) { int offset = index; memcpy(buf+offset, &val, sizeof(val)); offset += sizeof(double); return offset - index; } // Yeah I Gave Up std::string GetStringFromBytes(std::vector bytes) { return std::string(bytes.begin(), bytes.end()); } std::vector GetBytesFromString(const std::string& str) { std::vector bytes(str.begin(), str.end()); return bytes; } // Writes a UTF8 String with it's length prefixed as a 32-bit unsigned int static int write_string(uint8_t* buf, int index, std::string val) { int offset = index; std::vector string_bytes = GetBytesFromString(val); // Add Payload Length offset += write_uint32(buf, offset, string_bytes.size()); // Add Payload (As UTF8-Encoded String) //memcpy(buf+offset, &string_bytes, string_bytes.size()); for(uint8_t byte : string_bytes) { buf[offset] = byte; offset++; } //offset += sizeof(string_bytes); return offset - index; } #pragma endregion #pragma region Read Primitives static uint8_t read_uint8(uint8_t* buf, int index) { uint8_t outpt = 0; memcpy(&outpt, buf+index, sizeof(uint8_t)); return outpt; } static uint16_t read_uint16(uint8_t* buf, int index) { uint16_t outpt = 0; memcpy(&outpt, buf+index, sizeof(uint16_t)); return ntohs(outpt); } static uint32_t read_uint32(uint8_t* buf, int index) { uint32_t outpt = 0; memcpy(&outpt, buf+index, sizeof(uint32_t)); return ntohl(outpt); } static uint64_t read_uint64(uint8_t* buf, int index) { uint64_t outpt = 0; memcpy(&outpt, buf+index, sizeof(uint64_t)); return ntohl(outpt); } static int8_t read_int8(uint8_t* buf, int index) { int8_t outpt = 0; memcpy(&outpt, buf+index, sizeof(int8_t)); return outpt; } static int16_t read_int16(uint8_t* buf, int index) { int16_t outpt = 0; memcpy(&outpt, buf+index, sizeof(int16_t)); return ntohs(outpt); } static int32_t read_int32(uint8_t* buf, int index) { int32_t outpt = 0; memcpy(&outpt, buf+index, sizeof(int32_t)); return ntohl(outpt); } static int64_t read_int64(uint8_t* buf, int index) { int64_t outpt = 0; memcpy(&outpt, buf+index, sizeof(int64_t)); return ntohl(outpt); } static float read_float(uint8_t* buf, int index) { float outpt = 0; memcpy(&outpt, buf+index, sizeof(float)); return outpt; } static double read_double(uint8_t* buf, int index) { double outpt = 0; memcpy(&outpt, buf+index, sizeof(double)); return outpt; } static std::string read_string(uint8_t* buf, int index) { uint32_t str_len = read_uint32(buf, index); std::vector bytes(str_len); for (int i = 0; i < str_len; ++i) bytes[i] = buf[i+4]; return GetStringFromBytes(bytes); } static uint32_t read_string_size(uint8_t* buf, int index) { return read_uint32(buf, index); } #pragma endregion #pragma endregion typedef enum { tag_end = 0, tag_uint8, tag_uint16, tag_uint32, tag_uint64, tag_sint8, tag_sint16, tag_sint32, tag_sint64, tag_float, tag_double, tag_byte_array, tag_string, tag_list, tag_compound, tag_identifier = -1, tag_primitive = -2, } tag_type_t; class tag { public: std::string name; tag() {} tag(std::string name) { this->name = name; } // Deserialize Constructor tag(uint8_t* buf, int index) { name = read_string(buf, index); } virtual int serialize(uint8_t* buf, int startidx) { return 0; } //static tag deserialize(byte* data); // Encodes name length + utf8 string int write_name(uint8_t* buf, int startidx) { return write_string(buf, startidx, this->name); } static int write_type(uint8_t* buf, int startidx, tag_type_t tag_type) { int index = startidx; buf[index] = tag_type; index++; return index-startidx; } virtual int write_payload(uint8_t* buf, int startidx) {} private: protected: }; class end_tag : public tag { public: int serialize(uint8_t *buf, int startidx) override {} }; class uint8_tag : public tag { private: uint8_t payload; public: uint8_tag() {} uint8_tag(std::string name) {} uint8_tag(std::string name, uint8_t data) { this->name = name; payload = data; } uint8_tag(uint8_t *buffer, int startidx) : tag(buffer, startidx) { int name_size = read_string_size(buffer, startidx); startidx+=name_size; this->payload = read_uint8(buffer, startidx); } int serialize(uint8_t *buf, int startidx) override { int offset = startidx; offset += write_type(buf, offset, metabinary::tag_uint8); offset += write_name(buf, offset); offset += write_payload(buf, offset); return offset-startidx; } int write_payload(uint8_t *buf, int startidx) override { return write_uint8(buf, startidx, payload); } }; class uint16_tag : public tag { private: uint16_t payload; public: uint16_tag(std::string name, uint16_t data) { this->name = name; payload = data; } uint16_tag(uint8_t* buffer, int startidx) {} int serialize(uint8_t *buf, int startidx) override { int offset = startidx; offset += write_type(buf, offset, metabinary::tag_uint16); offset += write_name(buf, offset); offset += write_payload(buf, offset); return offset-startidx; } int write_payload(uint8_t *buf, int startidx) override { return write_uint16(buf, startidx, payload); } }; class uint32_tag : public tag { private: uint32_t payload; public: uint32_tag() {} uint32_tag(std::string name) {} uint32_tag(std::string name, uint32_t data) { this->name = name; payload = data; } int serialize(uint8_t *buf, int startidx) override { int offset = startidx; offset += write_type(buf, offset, metabinary::tag_uint32); offset += write_name(buf, offset); offset += write_payload(buf, offset); return offset-startidx; } int write_payload(uint8_t *buf, int startidx) override { return write_uint32(buf, startidx, payload); } }; class uint64_tag : public tag { public: uint64_tag() {} uint64_tag(std::string name) {} uint64_tag(std::string name, uint64_t data) { this->name = name; payload = data; } int serialize(uint8_t *buf, int startidx) override { int offset = startidx; offset += write_type(buf, offset, metabinary::tag_uint64); offset += write_name(buf, offset); offset += write_payload(buf, offset); return offset-startidx; } int write_payload(uint8_t *buf, int startidx) override { return write_uint64(buf, startidx, payload); } private: uint64_t payload; }; class sint8_tag : public tag { int8_t payload; public: sint8_tag() {} sint8_tag(std::string name) {} sint8_tag(std::string name, int8_t data) { this->name = name; this->payload = data; } int serialize(uint8_t *buf, int startidx) { int offset = startidx; offset += write_type(buf, offset, metabinary::tag_sint8); offset += write_name(buf, offset); offset += write_payload(buf, offset); return offset - startidx; } int write_payload(uint8_t *buf, int startidx) override { return write_int8(buf, startidx, payload); } }; class sint16_tag : public tag { int16_t payload; public: sint16_tag() { } sint16_tag(std::string name) { } sint16_tag(std::string name, int16_t data) { this->name; payload = data; } int serialize(uint8_t *buf, int startidx) { int offset = startidx; offset += write_type(buf, offset, metabinary::tag_sint16); offset += write_name(buf, offset); offset += write_payload(buf, offset); return offset-startidx; } int write_payload(uint8_t *buf, int startidx) override { return write_int16(buf, startidx, payload); } }; class sint32_tag : public tag { int32_t payload; public: sint32_tag() {} sint32_tag(std::string name) {} sint32_tag(std::string name, int32_t data) { this->name = name; payload = data; } int serialize(uint8_t *buf, int startidx) { int offset = startidx; offset += write_type(buf, offset, metabinary::tag_sint32); offset += write_name(buf, offset); offset += write_payload(buf, offset); return offset - startidx; } int write_payload(uint8_t *buf, int startidx) override { return write_int32(buf, startidx, payload); } }; class sint64_tag : public tag { int64_t payload; public: sint64_tag() {} sint64_tag(std::string name) {} sint64_tag(std::string name, int64_t data) { this->name = name; payload = data; } int serialize(uint8_t *buf, int startidx) { int offset = startidx; offset += write_type(buf, offset, metabinary::tag_sint64); offset += write_name(buf, offset); offset += write_payload(buf, offset); return offset - startidx; } int write_payload(uint8_t *buf, int index) override { return write_int64(buf, index, payload); } }; class float_tag : public tag { float payload; public: float_tag() {} float_tag(std::string name) {} float_tag(std::string name, float data) { this->name = name; payload = data; } int serialize(uint8_t *buf, int startidx) override { int offset = startidx; offset+=write_type(buf, offset, metabinary::tag_float); offset+=write_name(buf, offset); offset+=write_payload(buf, offset); return offset-startidx; } int write_payload(uint8_t *buf, int startidx) override { return write_float(buf, startidx, payload); } }; class double_tag : public tag { double payload; public: double_tag() {} double_tag(std::string name) {} double_tag(std::string name, double data) { this->name = name; payload = data; } int serialize(uint8_t *buf, int startidx) override { int offset = startidx; offset += write_type(buf, offset, metabinary::tag_double); offset += write_name(buf, offset); offset += write_payload(buf, offset); return offset-startidx; } int write_payload(uint8_t *buf, int startidx) override { return write_double(buf, startidx, payload); } }; class string_tag : public tag { std::string payload; public: string_tag() {} string_tag(std::string name) { this->name = name;} string_tag(std::string name, std::string value) { this->name = name; this->payload = value; } int serialize(uint8_t *buf, int startidx) override { int offset = startidx; offset += write_type(buf, offset, metabinary::tag_string); offset += write_name(buf, offset); offset += write_payload(buf, offset); return offset-startidx; } int write_payload(uint8_t *buf, int startidx) override { return write_string(buf, startidx, payload); } }; class list_tag : public tag { public: private: }; class compound_tag : public tag { protected: std::vector payload; public: compound_tag() {} compound_tag(std::string name) {} compound_tag(std::string name, std::vector tags) { this->name = name; payload = tags; } tag get(std::string name) const { for (auto& child : payload) { if (child.name == name) return child; } } int serialize(uint8_t* buffer, int startidx) { int offset = startidx; offset += write_type(buffer, offset, metabinary::tag_compound); offset += write_name(buffer, offset); // Add Serialized Child Tags for (auto& tag : payload) offset += tag.serialize(buffer, offset); // Add END Tag buffer[offset] = tag_end; offset++; // Return used space return offset-startidx; } void add_byte() {} void add_short() {} void add_int() {} void add_long() {} void add_float() {} void add_double() {} void add_string(std::string name, std::string value) { this->payload.push_back(string_tag(name, value)); } }; class custom_data_tag : public tag { }; class root_tag : public compound_tag { public: root_tag() : compound_tag() {}; root_tag(std::string name) : compound_tag(name) {} root_tag(std::string name, std::vector tags) : compound_tag(name, tags) {} }; tag deserialize(uint8_t* data, int index = 0) { tag_type_t tag_typ = static_cast(read_uint8(data, index)); switch(tag_typ) { case tag_end: case tag_compound: case tag_uint8: { return uint8_tag(data, index); } case tag_uint16: { return uint16_tag(data, index); break; } case tag_uint32: case tag_uint64: default: break; } } struct file { root_tag tag; }; } namespace tests { // TODO: Move tests to GTest Suite void uint64_roundtrip_test() { uint64_t begin = 40269; uint8_t buf[sizeof(uint64_t)]; metabinary::write_uint64(buf, 0, begin); uint64_t result = metabinary::read_uint64(buf, 0); assert(begin == result && "YE NOTHIN"); std::cout << "uint64_roundtrip_test passed" << std::endl; } void uint32_roundtrip_test() { uint32_t begin = 42069; uint8_t buf[sizeof(uint32_t)]; metabinary::write_uint32(buf, 0, begin); uint32_t result = metabinary::read_uint32(buf, 0); assert(begin == result && "DAWSH"); std::cout << "uint32_roundtrip_test passed" << std::endl; } void uint16_roundtrip_test() { uint16_t begin = 42069; uint8_t buf[sizeof(uint16_t)]; metabinary::write_uint16(buf, 0, begin); uint16_t result = metabinary::read_uint16(buf, 0); assert(begin == result && "DAWSH"); std::cout << "uint16_roundtrip_test passed" << std::endl; } void uint8_roundtrip_test() { uint8_t begin = 255; uint8_t buf[sizeof(uint8_t)]; metabinary::write_uint8(buf, 0, begin); uint8_t result = metabinary::read_uint8(buf, 0); assert(begin == result && "uint8_roundtrip_test failed"); std::cout << "uint8_roundtrip_test passed" << std::endl; } void int64_roundtrip_test() { int64_t begin = 666420; uint8_t buf[sizeof(int64_t)]; metabinary::write_int64(buf, 0, begin); uint64_t result = metabinary::read_int64(buf, 0); assert(begin == result && ""); std::cout << "int64_roundtrip_test passed" << std::endl; } void int32_roundtrip_test() { int32_t begin = 666; uint8_t buf[sizeof(int32_t)]; metabinary::write_int32(buf, 0, begin); int32_t result = metabinary::read_int32(buf, 0); assert(begin == result && ""); std::cout << "int32_roundtrip_test passed" << std::endl; } void int16_roundtrip_test() { int16_t begin = 666; uint8_t buf[sizeof(int16_t)]; metabinary::write_int16(buf, 0, begin); int16_t result = metabinary::read_int16(buf, 0); assert(begin == result && ""); std::cout << "int16_roundtrip_test passed" << std::endl; } void int8_roundtrip_test() { int8_t begin = 255; uint8_t buf[sizeof(int8_t)]; metabinary::write_int8(buf, 0, begin); int8_t result = metabinary::read_int8(buf, 0); assert(begin == result && ""); std::cout << "int8_roundtrip_test passed" << std::endl; } void float_roundtrip_test() { float begin = 420.69f; uint8_t buf[sizeof(float)]; metabinary::write_float(buf, 0, begin); float result = metabinary::read_float(buf, 0); assert(begin == result); std::cout << "float_roundtrip_test passed" << std::endl; } void double_roundtrip_test() { double begin = 3.14159; uint8_t buf[sizeof(double)]; metabinary::write_double(buf, 0, begin); double result = metabinary::read_double(buf, 0); assert(begin == result); std::cout << "double_roundtrip_test passed" << std::endl; } void string_roundtrip_test() { std::string begin = "AYYO WHATS UP BABY"; uint8_t buf[begin.size()+sizeof(uint32_t)]; metabinary::write_string(buf, 0, begin); std::string result = metabinary::read_string(buf, 0); //std::cout << begin << " : " << result << std::endl; assert(begin == result); std::cout << "string_roundtrip_test passed" << std::endl; } } int main() { // UNIT TESTING OF FUNCTIONS tests::uint64_roundtrip_test(); tests::uint32_roundtrip_test(); tests::uint16_roundtrip_test(); tests::uint8_roundtrip_test(); tests::int64_roundtrip_test(); tests::int32_roundtrip_test(); tests::int16_roundtrip_test(); tests::int8_roundtrip_test(); tests::float_roundtrip_test(); tests::double_roundtrip_test(); tests::string_roundtrip_test(); using namespace metabinary; root_tag test_file1{"Test File",{ uint32_tag("bruh", 42069), }}; uint8_t data[69]; test_file1.serialize(data, 0); tag reconstructed = metabinary::deserialize(data); auto tone = uint16_tag{"b", 123}; compound_tag item_meta = {"", { uint16_tag{"id", 64}, uint16_tag{"quantity", 64}, tone }}; item_meta.add_string("custom_name", "BALLIN"); root_tag demo_file { "DEMO METABINARY FILE", { string_tag{"MAP_NAME", "LEVEL1"}, string_tag{"MAP_AUTHOR", "brogrammer"}, uint64_tag{"MAP_EDIT_TIMESTAMP", 999999}, compound_tag{"SHADERCACHE"}, compound_tag{"ENTITIES",{ compound_tag{"1", { uint64_tag{"uuid", 42069}, compound_tag{"pos", { float_tag{"x", 0.25f}, float_tag{"y", 0.25f}, float_tag{"angle", 3.1415f}, }}, }}, compound_tag{"2", { uint64_tag{"uuid", 42044469}, compound_tag{"pos", { float_tag{"x", 0.25f}, float_tag{"y", 0.25f}, float_tag{"angle", 3.1415f}, }}, }}, compound_tag{"3", { uint64_tag{"uuid", 66642044469}, compound_tag{"pos", { float_tag{"x", 0.25f}, float_tag{"y", 0.25f}, float_tag{"angle", 3.1415f}, }}, }}, compound_tag{"4", { uint64_tag{"uuid", 696969}, compound_tag{"pos", { float_tag{"x", 0.25f}, float_tag{"y", 0.25f}, float_tag{"angle", 3.1415f}, }}, }}, }}, compound_tag{"boyz", { float_tag{"x", 0.25f}, float_tag{"y", 0.25f}, double_tag{"magic_number", 3.1414951}, }}, }}; tag t = demo_file.get("SHORTY"); std::cout << t.name << std::endl; uint8_t byte_buff[9999]; demo_file.serialize(byte_buff, 0); // copy byte to char buff for writing std::cout << byte_buff << std::endl; char output[sizeof(byte_buff)]; memcpy(output, &byte_buff, sizeof(byte_buff)); std::ofstream output_buff("test.bin", std::ios::out|std::ios::binary); output_buff.write(output, sizeof(output)); output_buff.close(); return 0; }