浅析avro序列化和反序列化源码及提供两种数据序列存储思路

本文主要是介绍浅析avro序列化和反序列化源码及提供两种数据序列存储思路，希望对大家解决编程问题提供一定的参考价值，需要的开发者们随着小编来一起学习吧！

以hadoop的avro实现原理做引子：

varo的测试样例程序

#include <complex>#include "avro/Encoder.hh"
#include "avro/Decoder.hh"
#include "avro/Specific.hh"namespace avro {
template<typename T>
struct codec_traits<std::complex<T> > {static void encode(Encoder& e, const std::complex<T>& c) {avro::encode(e, std::real(c));avro::encode(e, std::imag(c));}static void decode(Decoder& d, std::complex<T>& c) {T re, im;avro::decode(d, re);avro::decode(d, im);c = std::complex<T>(re, im);}
};}
int
main()
{std::auto_ptr<avro::OutputStream> out = avro::memoryOutputStream();avro::EncoderPtr e = avro::binaryEncoder();e->init(*out);std::complex<double> c1(1.0, 2.0);avro::encode(*e, c1);std::auto_ptr<avro::InputStream> in = avro::memoryInputStream(*out);avro::DecoderPtr d = avro::binaryDecoder();d->init(*in);std::complex<double> c2;avro::decode(*d, c2);std::cout << '(' << std::real(c2) << ", " << std::imag(c2) << ')' << std::endl;return 0;
}

这个样例相对简单。

其中template <typename T>

struct codec_traits { };

为avro对不同数据类型存储实现的模板类特化。

由于avro的基础库类未有提供std::complex<double>的特化说明，故在样例函数添加avro作用域的complex类型说明，并将复数类型的特化函数转化为avro提供的基础数据类型的特化

特化函数定义在avro api/specific.hh 头文件中定义。样例这里使用了 143行定义的double类型处理

template <> struct codec_traits<double> {
/**
* Encodes a given value.
*/
static void encode(Encoder& e, double d) {
e.encodeDouble(d);
}
/**
* Decodes into a given value.
*/
static void decode(Decoder& d, double& dbl) {
dbl = d.decodeDouble();
}

};

这里说明下avro内部存储数据的结构：

由于样例中使用的是二进制编制的内存存储形式（相对于文件存储）

std::auto_ptr<avro::OutputStream> out = avro::memoryOutputStream();

avro::EncoderPtr e = avro::binaryEncoder();

其中：memoryOutputStream 数据保存内存存储类

binaryEncoder 存储数据写入管理类 --- 对应Impl/binaryEncoder.cc文件

double写入内存函数实现：

void BinaryEncoder::encodeDouble(double d)
{
const uint8_t* p = reinterpret_cast<const uint8_t*>(&d);
out_.writeBytes(p, sizeof(double));
}

out为MemoryOutputStream管理类。对应文件实现impl/stream.cc 文件

其存储形式如下图。如上例中c1(1.0,2.0)。其中图示每一个方格为一个double存储单元。

写入逻辑实现为：

1.检测当前空间，如上图中一行是否足够double写入（注每次申请行为固定个数double存储单元）。否申请新的空间（即如上图一行）

2.写入double数据，并记录当前写入地址及最大可写入地址。

以上为数据写入的实现，反之取出，只需要按顺序读取数值出来即可。

难点：avro特化函数对指定数据类型的处理

另外提供两种序列化和反序列化实现：

第一种：来自哥们写的。特复杂也很难理解，（我也看了好几天才懂~~），仅给出部分关键文件源码。

想看完整源码可在github下载：https://github.com/xyz347/x2struct

以json序列化为例源码示例：

X2Struct.hh 这个文件是关键，以宏遍历的形式对结构体成员进行逐个成员展开处理。如果想实现非json存储数据类型的数据序列化和反序列化也以这个文件为基础构建。

特别注意：结构体成员展开为二层次展开，（怎么理解，看代码~~~~）。结构体成员最大展开个数为 32*32


#ifndef __X_TO_STRUCT_HPP
#define __X_TO_STRUCT_HPP#include <string>
#include <set>
#include <stdexcept>
#include "xobj.hpp"#include "xmlstr.hpp"
#include "jsoncfgstr.hpp"#ifdef XTOSTRUCT_GEN_GOLANG_CODE
#include <typeinfo>
#include "gostr.hpp"
#endif
//#include <iostream>
//using namespace std;namespace x2struct {#define X2STRUCT_OPT_ME     "me"    // must existclass X {
public:template <typename TYPE>static bool loadjson(const std::string&str, TYPE&t, bool isfile=true) {JsonObj obj(str, isfile);t.__x_to_struct(obj);return true;}   template <typename TYPE>static std::string tojson(const TYPE&t, const std::string&root, bool newline=true, int space=4) {JsonCfgStr obj(true, newline, space);t.__struct_to_str(obj, root, 0);return obj.toStr();}};#define X_STRUCT_FUNC_TOX_BEGIN                                             \
private:                                                                    \std::set<std::string> __x_has_string;                                   \
public:                                                                     \bool xhas(const std::string& name) const {                              \return __x_has_string.find(name)!=__x_has_string.end();             \}                                                                       \bool __x_condition(x2struct::XObj& obj, const std::string&name) {       \(void)obj;(void)name;                                               \return true;                                                        \}                                                                       \void __x_to_struct(x2struct::XObj& obj) {#define X_STRUCT_ACT_TOX_O(M)                                               \if (obj.has(#M)) {                                                  \obj[#M].convert(M);                                             \__x_has_string.insert(#M);                                      \}#define X_STRUCT_ACT_TOX_M(M)                                               \if (obj.has(#M)) {                                                  \obj[#M].convert(M);                                             \__x_has_string.insert(#M);                                      \} else {                                                            \obj.me_exception(#M);                                           \}// aliase name
#define X_STRUCT_ACT_TOX_A(M, A_NAME)                                       \{                                                                       \bool me = false;                                                    \std::string __alias__name__ = obj.hasa(#M, A_NAME, me);             \if (!__alias__name__.empty()) {                                     \obj[__alias__name__].convert(M);                                \__x_has_string.insert(#M);                                      \} else if (me) {                                                    \obj.me_exception(#M);                                           \}                                                                   \}#define X_STRUCT_FUNC_TOX_END }// struct to string
#define X_STRUCT_FUNC_TOS_BEGIN                                                     \template <class CLASS>                                                          \void __struct_to_str(CLASS& obj, const std::string&root, int splen) const {     \int index = 0;                                                              \obj.begin(root, splen);#define X_STRUCT_ACT_TOS_O(M)                                                       \obj.convert(#M, M, splen+obj.space(), index++);#define X_STRUCT_ACT_TOS_A(M, A_NAME)                                               \obj.convert(obj.fieldName(#M,A_NAME), M, splen+obj.space(), index++);#define X_STRUCT_FUNC_TOS_END                                                       \obj.end(root,splen);                                                        \}// struct to golang code
#define X_STRUCT_FUNC_TOG_BEGIN                                                     \std::string __struct_to_go(x2struct::GoCode &obj) const {                       \obj.set_type_id_name(typeid(*this).name());#define X_STRUCT_ACT_TOG_O(M)                                                       \obj.convert(M, #M);#define X_STRUCT_ACT_TOG_A(M, A_NAME)                                               \obj.convert(M, #M, A_NAME);#define X_STRUCT_FUNC_TOG_END                                                       \return obj.str();                                                           \}// lenght should equal X_STRUCT_N
#define X_DEC_LIST _32,_31,_30,_29,_28,_27,_26,_25,_24,_23,_22,_21,_20,_19,_18,_17,_16,_15,_14,_13,_12,_11,_10,_9,_8,_7,_6,_5,_4,_3,_2,_1
// expand macro depend on parameters number
#define X_STRUCT_N(ACTION,_1,_2,_3,_4,_5,_6,_7,_8,_9,_10,_11,_12,_13,_14,_15,_16,_17,_18,_19,_20,_21,_22,_23,_24,_25,_26,_27,_28,_29,_30,_31,_32,N,...) ACTION##N/*work with X_STRUCT_N to expand macrostruct MyX {int         a;std::string b;double      c;XTOSTRUCT(A(a,"_id"), O(b,c));};there'r two layer in this macro:l1: A(a,"_id"), O(b,c)l2: a,"_id"; b,cso we need expand twice
*/// wrap X_STRUCT_N. so we can use X_DEC_LIST
#define X_STRUCT_WRAP_L1(ACT, LIST, ...) X_STRUCT_N(X_STRUCT_L1, __VA_ARGS__, LIST)(ACT, __VA_ARGS__)
#define X_STRUCT_WRAP_L2(ACT, LIST, ...) X_STRUCT_N(X_STRUCT_L2, __VA_ARGS__, LIST)(ACT, __VA_ARGS__)#define X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_##ACT##M
#define X_STRUCT_L1_1(ACT, M)        X_STRUCT_L1_DEF(ACT, M)
#define X_STRUCT_L1_2(ACT, M,...)    X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_1(ACT, __VA_ARGS__)
#define X_STRUCT_L1_3(ACT, M,...)    X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_2(ACT, __VA_ARGS__)
#define X_STRUCT_L1_4(ACT, M,...)    X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_3(ACT, __VA_ARGS__)
#define X_STRUCT_L1_5(ACT, M,...)    X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_4(ACT, __VA_ARGS__)
#define X_STRUCT_L1_6(ACT, M,...)    X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_5(ACT, __VA_ARGS__)
#define X_STRUCT_L1_7(ACT, M,...)    X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_6(ACT, __VA_ARGS__)
#define X_STRUCT_L1_8(ACT, M,...)    X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_7(ACT, __VA_ARGS__)
#define X_STRUCT_L1_9(ACT, M,...)    X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_8(ACT, __VA_ARGS__)
#define X_STRUCT_L1_10(ACT, M,...)   X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_9(ACT, __VA_ARGS__)
#define X_STRUCT_L1_11(ACT, M,...)   X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_10(ACT, __VA_ARGS__)
#define X_STRUCT_L1_12(ACT, M,...)   X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_11(ACT, __VA_ARGS__)
#define X_STRUCT_L1_13(ACT, M,...)   X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_12(ACT, __VA_ARGS__)
#define X_STRUCT_L1_14(ACT, M,...)   X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_13(ACT, __VA_ARGS__)
#define X_STRUCT_L1_15(ACT, M,...)   X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_14(ACT, __VA_ARGS__)
#define X_STRUCT_L1_16(ACT, M,...)   X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_15(ACT, __VA_ARGS__)
#define X_STRUCT_L1_17(ACT, M,...)   X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_16(ACT, __VA_ARGS__)
#define X_STRUCT_L1_18(ACT, M,...)   X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_17(ACT, __VA_ARGS__)
#define X_STRUCT_L1_19(ACT, M,...)   X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_18(ACT, __VA_ARGS__)
#define X_STRUCT_L1_20(ACT, M,...)   X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_19(ACT, __VA_ARGS__)
#define X_STRUCT_L1_21(ACT, M,...)   X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_20(ACT, __VA_ARGS__)
#define X_STRUCT_L1_22(ACT, M,...)   X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_21(ACT, __VA_ARGS__)
#define X_STRUCT_L1_23(ACT, M,...)   X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_22(ACT, __VA_ARGS__)
#define X_STRUCT_L1_24(ACT, M,...)   X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_23(ACT, __VA_ARGS__)
#define X_STRUCT_L1_25(ACT, M,...)   X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_24(ACT, __VA_ARGS__)
#define X_STRUCT_L1_26(ACT, M,...)   X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_25(ACT, __VA_ARGS__)
#define X_STRUCT_L1_27(ACT, M,...)   X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_26(ACT, __VA_ARGS__)
#define X_STRUCT_L1_28(ACT, M,...)   X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_27(ACT, __VA_ARGS__)
#define X_STRUCT_L1_29(ACT, M,...)   X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_28(ACT, __VA_ARGS__)
#define X_STRUCT_L1_30(ACT, M,...)   X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_29(ACT, __VA_ARGS__)
#define X_STRUCT_L1_31(ACT, M,...)   X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_30(ACT, __VA_ARGS__)
#define X_STRUCT_L1_32(ACT, M,...)   X_STRUCT_L1_DEF(ACT, M)      X_STRUCT_L1_31(ACT, __VA_ARGS__)#define X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_ACT_##ACT(M)
#define X_STRUCT_L2_1(ACT, M)       X_STRUCT_L2_DEF(ACT, M)
#define X_STRUCT_L2_2(ACT, M,...)   X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_1(ACT, __VA_ARGS__)
#define X_STRUCT_L2_3(ACT, M,...)   X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_2(ACT, __VA_ARGS__)
#define X_STRUCT_L2_4(ACT, M,...)   X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_3(ACT, __VA_ARGS__)
#define X_STRUCT_L2_5(ACT, M,...)   X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_4(ACT, __VA_ARGS__)
#define X_STRUCT_L2_6(ACT, M,...)   X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_5(ACT, __VA_ARGS__)
#define X_STRUCT_L2_7(ACT, M,...)   X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_6(ACT, __VA_ARGS__)
#define X_STRUCT_L2_8(ACT, M,...)   X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_7(ACT, __VA_ARGS__)
#define X_STRUCT_L2_9(ACT, M,...)   X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_8(ACT, __VA_ARGS__)
#define X_STRUCT_L2_10(ACT, M,...)  X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_9(ACT, __VA_ARGS__)
#define X_STRUCT_L2_11(ACT, M,...)  X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_10(ACT, __VA_ARGS__)
#define X_STRUCT_L2_12(ACT, M,...)  X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_11(ACT, __VA_ARGS__)
#define X_STRUCT_L2_13(ACT, M,...)  X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_12(ACT, __VA_ARGS__)
#define X_STRUCT_L2_14(ACT, M,...)  X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_13(ACT, __VA_ARGS__)
#define X_STRUCT_L2_15(ACT, M,...)  X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_14(ACT, __VA_ARGS__)
#define X_STRUCT_L2_16(ACT, M,...)  X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_15(ACT, __VA_ARGS__)
#define X_STRUCT_L2_17(ACT, M,...)  X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_16(ACT, __VA_ARGS__)
#define X_STRUCT_L2_18(ACT, M,...)  X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_17(ACT, __VA_ARGS__)
#define X_STRUCT_L2_19(ACT, M,...)  X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_18(ACT, __VA_ARGS__)
#define X_STRUCT_L2_20(ACT, M,...)  X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_19(ACT, __VA_ARGS__)
#define X_STRUCT_L2_21(ACT, M,...)  X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_20(ACT, __VA_ARGS__)
#define X_STRUCT_L2_22(ACT, M,...)  X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_21(ACT, __VA_ARGS__)
#define X_STRUCT_L2_23(ACT, M,...)  X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_22(ACT, __VA_ARGS__)
#define X_STRUCT_L2_24(ACT, M,...)  X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_23(ACT, __VA_ARGS__)
#define X_STRUCT_L2_25(ACT, M,...)  X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_24(ACT, __VA_ARGS__)
#define X_STRUCT_L2_26(ACT, M,...)  X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_25(ACT, __VA_ARGS__)
#define X_STRUCT_L2_27(ACT, M,...)  X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_26(ACT, __VA_ARGS__)
#define X_STRUCT_L2_28(ACT, M,...)  X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_27(ACT, __VA_ARGS__)
#define X_STRUCT_L2_29(ACT, M,...)  X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_28(ACT, __VA_ARGS__)
#define X_STRUCT_L2_30(ACT, M,...)  X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_29(ACT, __VA_ARGS__)
#define X_STRUCT_L2_31(ACT, M,...)  X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_30(ACT, __VA_ARGS__)
#define X_STRUCT_L2_32(ACT, M,...)  X_STRUCT_L2_DEF(ACT, M)     X_STRUCT_L2_31(ACT, __VA_ARGS__)// L1 action define. O M A
// string to struct
#define X_STRUCT_L1_TOX_O(...)  X_STRUCT_WRAP_L2(TOX_O, X_DEC_LIST, __VA_ARGS__)
#define X_STRUCT_L1_TOX_M(...)  X_STRUCT_WRAP_L2(TOX_M, X_DEC_LIST, __VA_ARGS__)
#define X_STRUCT_L1_TOX_A(M,A)  X_STRUCT_ACT_TOX_A(M,A)// struct to string
#define X_STRUCT_L1_TOS_O(...)  X_STRUCT_WRAP_L2(TOS_O, X_DEC_LIST, __VA_ARGS__)
#define X_STRUCT_L1_TOS_M       X_STRUCT_L1_TOS_O
#define X_STRUCT_L1_TOS_A(M,A)  X_STRUCT_ACT_TOS_A(M,A)// struct to golang code
#define X_STRUCT_L1_TOG_O(...)  X_STRUCT_WRAP_L2(TOG_O, X_DEC_LIST, __VA_ARGS__)
#define X_STRUCT_L1_TOG_M       X_STRUCT_L1_TOG_O
#define X_STRUCT_L1_TOG_A(M,A)  X_STRUCT_ACT_TOG_A(M,A)#ifdef XTOSTRUCT_GEN_GOLANG_CODE
#define XTOSTRUCT(...)  \X_STRUCT_FUNC_TOX_BEGIN  X_STRUCT_WRAP_L1(TOX_, X_DEC_LIST, __VA_ARGS__) X_STRUCT_FUNC_TOX_END  \X_STRUCT_FUNC_TOS_BEGIN  X_STRUCT_WRAP_L1(TOS_, X_DEC_LIST, __VA_ARGS__) X_STRUCT_FUNC_TOS_END  \X_STRUCT_FUNC_TOG_BEGIN  X_STRUCT_WRAP_L1(TOG_, X_DEC_LIST, __VA_ARGS__) X_STRUCT_FUNC_TOG_END
#else
#define XTOSTRUCT(...)  \X_STRUCT_FUNC_TOX_BEGIN  X_STRUCT_WRAP_L1(TOX_, X_DEC_LIST, __VA_ARGS__) X_STRUCT_FUNC_TOX_END  \X_STRUCT_FUNC_TOS_BEGIN  X_STRUCT_WRAP_L1(TOS_, X_DEC_LIST, __VA_ARGS__) X_STRUCT_FUNC_TOS_END
#endif#define XTOSTRUCT_CONDITION()   bool __x_condition(x2struct::XObj& obj, const char* name)
#define XTOSTRUCT_CONDITION_EQ(attr1, attr2)                    \bool __x_condition(x2struct::XObj& obj, const char* name) { \return obj.attribute(attr1)==obj.attribute(attr2);      \}}#endif

jsonstrcfg.h json序列化和反序列化接口定义


* Copyright (C) 2017 YY Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); 
* you may not use this file except in compliance with the License. 
* You may obtain a copy of the License at
*
*	http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, 
* software distributed under the License is distributed on an "AS IS" BASIS, 
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
* See the License for the specific language governing permissions and 
* limitations under the License.
*/#ifndef __X_TO_STRUCT_JSON_CFG_STR_HPP
#define __X_TO_STRUCT_JSON_CFG_STR_HPP#include <stdint.h>
#include <string>
#include <map>
#include <vector>
#include <set>
#include <sstream>
#include <boost/lexical_cast.hpp>#include "xstr.hpp"namespace x2struct {class JsonCfgStr:public XStr {
public:JsonCfgStr(bool isjson, bool newline=false, int splen=4);~JsonCfgStr();
public:void begin(const std::string&root, int splen);void end(const std::string&root, int splen);int  space();std::string toStr() const;
public:virtual void convert(const std::string&name, const std::string& data, int space, int index);// base typevoid convert(const std::string&name, int16_t data, int splen, int index);void convert(const std::string&name, uint16_t data, int splen, int index);void convert(const std::string&name, int32_t data, int splen, int index);void convert(const std::string&name, uint32_t data, int splen, int index);void convert(const std::string&name, int64_t data, int splen, int index);void convert(const std::string&name, uint64_t data, int splen, int index);void convert(const std::string&name, float data, int splen, int index);void convert(const std::string&name, double data, int splen, int index);void convert(const std::string&name, bool data, int splen, int index);// base type vectorvoid convert(const std::string&name, const std::vector<int16_t>& data, int splen, int index);void convert(const std::string&name, const std::vector<uint16_t>& data, int splen, int index);void convert(const std::string&name, const std::vector<int32_t>& data, int splen, int index);void convert(const std::string&name, const std::vector<uint32_t>& data, int splen, int index);void convert(const std::string&name, const std::vector<int64_t>& data, int splen, int index);void convert(const std::string&name, const std::vector<uint64_t>& data, int splen, int index);void convert(const std::string&name, const std::vector<float>& data, int splen, int index);void convert(const std::string&name, const std::vector<double>& data, int splen, int index);void convert(const std::string&name, const std::vector<bool>& data, int splen, int index);void convert(const std::string&name, const std::vector<std::string>& data, int splen, int index);// base type setvoid convert(const std::string&name, const std::set<int16_t>& data, int splen, int index);void convert(const std::string&name, const std::set<uint16_t>& data, int splen, int index);void convert(const std::string&name, const std::set<int32_t>& data, int splen, int index);void convert(const std::string&name, const std::set<uint32_t>& data, int splen, int index);void convert(const std::string&name, const std::set<int64_t>& data, int splen, int index);void convert(const std::string&name, const std::set<uint64_t>& data, int splen, int index);void convert(const std::string&name, const std::set<float>& data, int splen, int index);void convert(const std::string&name, const std::set<double>& data, int splen, int index);void convert(const std::string&name, const std::set<bool>& data, int splen, int index);void convert(const std::string&name, const std::set<std::string>& data, int splen, int index);template <typename TYPE>void convert(const std::string&name, const std::vector<TYPE> &data, int splen, int index) {linefeed(index);std::string oldsep(_sep);//_ss<<std::string(splen, ' ')<<_key<<name<<_key<<_assign<<_vecb;if (!name.empty()) {_ss<<std::string(splen, ' ')<<_key<<name<<_key<<_assign<<_vecb;} else {_ss<<std::string(splen, ' ')<<_vecb;}for (size_t i=0; i<data.size(); ++i) {_sep=",";this->convert("", data[i], splen+space(), (int)i);_sep=oldsep;}_ss<<(_newline?"\n":"")<<std::string(splen, ' ')<<_vece;}template <typename TYPE>void convert(const std::string&name, const std::set<TYPE> &data, int splen, int index) {int i = 0;linefeed(index);std::string oldsep(_sep);//_ss<<std::string(splen, ' ')<<_key<<name<<_key<<_assign<<_vecb;if (!name.empty()) {_ss<<std::string(splen, ' ')<<_key<<name<<_key<<_assign<<_vecb;} else {_ss<<std::string(splen, ' ')<<_vecb;}for (typename std::set<TYPE>::const_iterator it=data.begin(); it!=data.end(); ++it) {_sep=",";this->convert("", *it, splen+space(), i++);_sep=oldsep;}_ss<<(_newline?"\n":"")<<std::string(splen, ' ')<<_vece;}template <typename TYPE>void convert(const std::string&name, const std::map<std::string,TYPE> &data, int splen, int index) {int i = 0;linefeed(index);std::string oldsep(_sep);if (!name.empty()) {_ss<<std::string(splen, ' ')<<_key<<name<<_key<<_assign<<"{";} else {_ss<<std::string(splen, ' ')<<"{";}for (typename std::map<std::string,TYPE>::const_iterator iter=data.begin(); iter!=data.end(); ++iter) {_sep=",";this->convert(iter->first, iter->second, splen+space(), i++);_sep=oldsep;}_ss<<(_newline?"\n":"")<<std::string(splen, ' ')<<"}";}template <typename KEYTYPE, typename TYPE>void convert(const std::string&name, const std::map<KEYTYPE, TYPE> &data, int splen, int index) {int i = 0;linefeed(index);std::string oldsep(_sep);if (!name.empty()) {_ss<<std::string(splen, ' ')<<_key<<name<<_key<<_assign<<"{";} else {_ss<<std::string(splen, ' ')<<"{";}for (typename std::map<KEYTYPE,TYPE>::const_iterator iter=data.begin(); iter!=data.end(); ++iter) {_sep=",";std::string _k = boost::lexical_cast<std::string>(iter->first);this->convert(_k, iter->second, splen+space(), i++);_sep=oldsep;}_ss<<(_newline?"\n":"")<<std::string(splen, ' ')<<"}";}template <typename TYPE>void convert(const std::string&name, const TYPE& data, int splen, int index) {linefeed(index);data.__struct_to_str(*this, name, splen);}
private:void linefeed(int index) {if (index>0) {_ss<<_sep;}if (_newline) {_ss<<'\n';}}template <typename TYPE>void base_convert(const std::string&name, const TYPE& data, int splen, int index) {linefeed(index);if (!name.empty()) {_ss<<std::string(splen, ' ')<<_key<<name<<_key<<_assign<<data;} else {_ss<<std::string(splen, ' ')<<data;}}template <typename TYPE>void base_vector(const std::string&name, const std::vector<TYPE>& data, int splen, int index) {linefeed(index);if (!name.empty()) {_ss<<std::string(splen, ' ')<<_key<<name<<_key<<_assign<<_vecb;} else {_ss<<std::string(splen, ' ')<<_vecb;}for (typename std::vector<TYPE>::const_iterator it=data.begin(); it!=data.end(); ++it) {if (it != data.begin()) {_ss<<',';}_ss<<*it;}_ss<<_vece;}template <typename TYPE>void base_set(const std::string&name, const std::set<TYPE>& data, int splen, int index) {linefeed(index);if (!name.empty()) {_ss<<std::string(splen, ' ')<<_key<<name<<_key<<_assign<<_vecb;} else {_ss<<std::string(splen, ' ')<<_vecb;}for (typename std::set<TYPE>::const_iterator it=data.begin(); it!=data.end(); ++it) {if (it != data.begin()) {_ss<<',';}_ss<<*it;}_ss<<_vece;}
private:bool _json;bool _newline;int  _space;std::string _vecb;  // json:[, cfg:(std::string _vece;  // json:], cfg:)std::string _key;   // json:", cfg:(empty)std::string _assign;// json::, cfg:=std::string _sep;   // json:,, cfg:;std::stringstream _ss;
};}#endif

jsonstrcfg.cpp json序列化反序列化定义


* Copyright (C) 2017 YY Inc. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); 
* you may not use this file except in compliance with the License. 
* You may obtain a copy of the License at
*
*	http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, 
* software distributed under the License is distributed on an "AS IS" BASIS, 
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
* See the License for the specific language governing permissions and 
* limitations under the License.
*/#include <json/json.h>#include "jsoncfgstr.hpp"namespace x2struct {using namespace std;JsonCfgStr::JsonCfgStr(bool isjson, bool newline, int splen):XStr(isjson?"json":"config"),_json(isjson), _newline(newline),_space(splen)
{if (isjson) {_vecb = "[";_vece = "]";_key = "\"";_assign = ":";_sep = ",";} else {_vecb = "(";_vece = ")";_assign = "=";_sep = ";";}
}JsonCfgStr::~JsonCfgStr()
{
}void JsonCfgStr::begin(const string&root, int splen)
{if (root.empty()) {_ss<<string(splen, ' ')<<"{";} else {_ss<<string(splen, ' ')<<_key<<root<<_key<<_assign<<"{";}
}void JsonCfgStr::end(const string&root, int splen)
{if (_newline) {_ss<<'\n';}_ss<<string(splen, ' ')<<"}";
}int  JsonCfgStr::space()
{return _newline?_space:0;
}string JsonCfgStr::toStr()const 
{return _ss.str();
}void JsonCfgStr::convert(const string&name, int16_t data, int splen, int index)
{base_convert(name, data, splen, index);
}void JsonCfgStr::convert(const string&name, uint16_t data, int splen, int index)
{base_convert(name, data, splen, index);
}void JsonCfgStr::convert(const string&name, int32_t data, int splen, int index)
{base_convert(name, data, splen, index);
}void JsonCfgStr::convert(const string&name, uint32_t data, int splen, int index)
{base_convert(name, data, splen, index);
}void JsonCfgStr::convert(const std::string&name, int64_t data, int splen, int index)
{base_convert(name, data, splen, index);
}void JsonCfgStr::convert(const std::string&name, uint64_t data, int splen, int index)
{base_convert(name, data, splen, index);
}void JsonCfgStr::convert(const std::string&name, float data, int splen, int index)
{base_convert(name, data, splen, index);
}void JsonCfgStr::convert(const std::string&name, double data, int splen, int index)
{base_convert(name, data, splen, index);
}void JsonCfgStr::convert(const std::string&name, bool data, int splen, int index)
{base_convert(name, data?"true":"false", splen, index);
}void JsonCfgStr::convert(const string&name, const string& data, int splen, int index)
{linefeed(index);if (_json) {_ss<<string(splen, ' ')<<_key<<name<<_key<<_assign<<Json::valueToQuotedString(data.c_str());} else {_ss<<string(splen, ' ')<<_key<<name<<_key<<_assign<<'"'<<data<<'"';}
}// base type vector
void JsonCfgStr::convert(const std::string&name, const std::vector<int16_t>& data, int splen, int index)
{base_vector(name, data, splen, index);
}void JsonCfgStr::convert(const std::string&name, const std::vector<uint16_t>& data, int splen, int index)
{base_vector(name, data, splen, index);
}void JsonCfgStr::convert(const std::string&name, const std::vector<int32_t>& data, int splen, int index)
{base_vector(name, data, splen, index);
}void JsonCfgStr::convert(const std::string&name, const std::vector<uint32_t>& data, int splen, int index)
{base_vector(name, data, splen, index);
}void JsonCfgStr::convert(const std::string&name, const std::vector<int64_t>& data, int splen, int index)
{base_vector(name, data, splen, index);
}void JsonCfgStr::convert(const std::string&name, const std::vector<uint64_t>& data, int splen, int index)
{base_vector(name, data, splen, index);
}void JsonCfgStr::convert(const std::string&name, const std::vector<float>& data, int splen, int index)
{base_vector(name, data, splen, index);
}void JsonCfgStr::convert(const std::string&name, const std::vector<double>& data, int splen, int index)
{base_vector(name, data, splen, index);
}void JsonCfgStr::convert(const std::string&name, const std::vector<bool>& data, int splen, int index)
{base_vector(name, data, splen, index);
}void JsonCfgStr::convert(const std::string&name, const std::vector<std::string>& data, int splen, int index)
{linefeed(index);_ss<<std::string(splen, ' ')<<_key<<name<<_key<<_assign<<_vecb;for (typename std::vector<std::string>::const_iterator it=data.begin(); it!=data.end(); ++it) {if (it != data.begin()) {_ss<<',';}_ss<<'"'<<*it<<'"';}_ss<<_vece;
}// base type set
void JsonCfgStr::convert(const std::string&name, const std::set<int16_t>& data, int splen, int index)
{base_set(name, data, splen, index);
}void JsonCfgStr::convert(const std::string&name, const std::set<uint16_t>& data, int splen, int index)
{base_set(name, data, splen, index);
}void JsonCfgStr::convert(const std::string&name, const std::set<int32_t>& data, int splen, int index)
{base_set(name, data, splen, index);
}void JsonCfgStr::convert(const std::string&name, const std::set<uint32_t>& data, int splen, int index)
{base_set(name, data, splen, index);
}void JsonCfgStr::convert(const std::string&name, const std::set<int64_t>& data, int splen, int index)
{base_set(name, data, splen, index);
}void JsonCfgStr::convert(const std::string&name, const std::set<uint64_t>& data, int splen, int index)
{base_set(name, data, splen, index);
}void JsonCfgStr::convert(const std::string&name, const std::set<float>& data, int splen, int index)
{base_set(name, data, splen, index);
}void JsonCfgStr::convert(const std::string&name, const std::set<double>& data, int splen, int index)
{base_set(name, data, splen, index);
}void JsonCfgStr::convert(const std::string&name, const std::set<bool>& data, int splen, int index)
{base_set(name, data, splen, index);
}void JsonCfgStr::convert(const std::string&name, const std::set<std::string>& data, int splen, int index)
{linefeed(index);_ss<<std::string(splen, ' ')<<_key<<name<<_key<<_assign<<_vecb;for (typename std::set<std::string>::const_iterator it=data.begin(); it!=data.end(); ++it) {if (it != data.begin()) {_ss<<',';}_ss<<'"'<<*it<<'"';}_ss<<_vece;
}
}

第二种。实际上没做序列化定义，支撑任意类型的快速写入和取出，尤其应用在于单生产者、单消费者线程的无锁共享。怎么把它改造成rpc形式的序列化和反序列化，甚至支撑任意类型存储。可以使用avro特化函数。也可以借用redis队列协议即（数据类型+数据）。只是给个引子，怎么实现各位可以自己思考

#ifndef __LOGBLOCKMANER_H__
#define __LOGBLOCKMANER_H__#include <stdio.h>
class LogBlockManer
{
public:~LogBlockManer(){}LogBlockManer(){_blockSize = 1;}
private:struct __BlockNode{int seq;int w_offset;int r_offset;int memsize;char * memory;struct __BlockNode * next;__BlockNode(){seq = 0;w_offset = 0;r_offset = 0;memsize  = 0;memory = NULL;next = NULL;}void reset(int size){w_offset = 0;r_offset = 0;memsize  = size;	next = NULL;return ;}};__BlockNode * _wrnode;__BlockNode * _rdnode;__BlockNode * _head;int           _blockSize;int           _threadmem;
public:int Init();int  writeLog(char * log,int size,__FileInfo *&file);bool readLog(char *& log,int &size,__FileInfo *&file);int  setThreadmemsize(int size);
};#endif

.cpp

#include "LogBlockManer.h"#include <stdio.h>
#include <stdlib.h>
#include <string.h>#define MEMSIZE 1024*1024int LogBlockManer::writeLog(char * log,int size,__FileInfo *&file)
{static int int_size = sizeof(int);static int charpoint_size = sizeof(char*);if(_wrnode->w_offset + size + charpoint_size + int_size>= _wrnode->memsize){if(!_wrnode->memory){_wrnode->memory = new char[MEMSIZE];if(!_wrnode->memory){return 0;}_wrnode->memsize = MEMSIZE;	}else{__BlockNode * node = new __BlockNode();if(!node){			return 0;}	node->memory = new char[MEMSIZE];if(!node->memory){delete node;return 1;}			_blockSize++;_wrnode->next = node;_wrnode = node;	_wrnode->memsize = MEMSIZE;}}}memcpy((void*)(_wrnode->memory+_wrnode->w_offset),(void*)&file,charpoint_size);memcpy((void*)(_wrnode->memory+_wrnode->w_offset+charpoint_size),(void*)&size,int_size);memcpy((void*)(_wrnode->memory+_wrnode->w_offset+(charpoint_size+int_size)),(void*)log,size);_wrnode->w_offset += (size+int_size+charpoint_size);return 0;
}bool LogBlockManer::readLog(char * & log,int & size,__FileInfo *&file)
{static int int_size2 = sizeof(int);static int charpoint_size = sizeof(char*);if(_rdnode->r_offset >= _rdnode->w_offset){if(_rdnode != _wrnode){__BlockNode * node = _rdnode;_rdnode = _rdnode->next;delete [] node->memory;delete node;if(_rdnode->r_offset >= _rdnode->w_offset){return false;}}else{return false;}}memcpy((void*)&file,(void*)(_rdnode->memory + _rdnode->r_offset),charpoint_size);memcpy((void*)&size,(void*)(_rdnode->memory + _rdnode->r_offset + charpoint_size),int_size2);log = _rdnode->memory + (charpoint_size + int_size2+_rdnode->r_offset);_rdnode->r_offset += (size+charpoint_size+int_size2);return true;
}int LogBlockManer::Init()
{_wrnode = new __BlockNode();if(!_wrnode){return -1;}_head   = _wrnode;_rdnode = _wrnode;return 0;
}int LogBlockManer::setThreadmemsize(int size)
{_threadmem = size;return 0;
}

这篇关于浅析avro序列化和反序列化源码及提供两种数据序列存储思路的文章就介绍到这儿，希望我们推荐的文章对编程师们有所帮助！