基于ZeroMQ程序库的分布式存储系统

原创 于 2014-01-22 18:45:39 发布 · 3.1k 阅读 · 8 ·
本内容遵循CC 4.0 BY-SA版权协议
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
收录于
分布式消息队列

当前文章被收录于：

分布式消息队列
4 篇文章 0 人学习
订阅专栏 查看详情
本文介绍了利用ZeroMQ构建的分布式存储系统，包括master和slave节点的设计，master节点维护slave节点负载信息和文件映射表，客户端通过master请求文件并直接与slave通讯。系统架构类似传统分布式文件系统，主从节点间的消息交互流程详细阐述，同时展示了关键函数的实现。目前，系统已完成基本通讯功能，代码将开源在GitHub上。
1.前言

最近在学习ZeroMQ，并准备用ZeroMQ写个简单版的类TFS系统，该系统的主要消息通讯和数据的传输采用ZeroMQ，目前这个的框架基本完成，整体的框架很简单，初步已经完成了master和slave节点的开发，这个版本中，master节点主要包含了两类东西:1>Slave 节点的负载信息，2>文件到salve的映射表，设计思路很简单，而slave节点主要存储文件内容，客户端直接通过指定的端口向master节点发送文件请求（发送文件和接收文件），master节点收到客户端发来的请求后，根据请求的类型，返回指定的slave节点的地址，而后客户端即可跟相应的slave节点通讯，而master节点则会实时地发送心跳信息给slave节点以获取每个slave节点的负载情况，并实时更新负载列表。

2.架构

    本系统所采取的架构类似与传统的分布式文件系统架构，即分为了主节点和从节点，而主节点主要负责收集从节点的基本信息，并且也接收客户端发送的相关请求，架构图如下所示

    

3. 消息交互流程如下：

     

4. 所使用的一些基本结构定义

struct command_t
{
    command_t()
    {
        bzero(command,sizeof(command));
        bzero(filename,sizeof(filename));
        nsize = 0;
    }
    char  command[COMMAND_SIZE];
    char  filename[FILENAME_SIZE];
    size_t nsize;
};

struct file_t
{
    file_t()
    {
        bzero(filename,sizeof(filename));
        nsize = 0;
    }
    char filename[FILENAME_SIZE];
    size_t nsize;
};

struct context_t
{
    context_t()
    {
        context = NULL;
        sender = NULL;
    }
    void* context;
    void* sender;
    //char  server[FILENAME_SIZE];
};

enum responseType
{
    DIST_NONE = 1,
    DIST_GET,
    DIST_ADD,
    DIST_DELETE
};
struct response_t
{
    response_t()
    {
        type = DIST_NONE;
        bzero(server,sizeof(server));
        bzero(filename,sizeof(filename));
    }
    responseType type;
    char server[FILENAME_SIZE];
    char filename[FILENAME_SIZE];
};

struct request_t
{
    request_t()
    {
        type = DIST_NONE;
        bzero(filename,sizeof(filename));
    }
    responseType type;
    char filename[FILENAME_SIZE];
};

enum slaveType_t
{
    SLAVE_NONE = 0,
    SLAVE_ONLINE = 1,
    SLAVE_OFFLINE = 2,
    SLAVE_HEARTBEAT = 3
};
struct slave_t
{
    slave_t()
    {
        type = SLAVE_NONE;
        nsize = 0;
        bzero(server,sizeof(server));
    }
    slaveType_t type;
    size_t nsize;
    char server[FILENAME_SIZE];
};
struct msg_t
{
    msg_t()
    {
        nsize = 0;
        total_size = 0;
        bzero(buffer,sizeof(buffer));
        bzero(server,sizeof(buffer));
    }
    union
    {
        slaveType_t slaveType;
        responseType responType;
    };
    size_t nsize;
    size_t total_size;
    char buffer[BUFFER_SIZE];
    char server[BUFFER_SIZE];
};

5.封装的一些基本函数

   这部分主要是参考ZeroMQ手册上的一些基本函数，并稍作修改，使其支持任意类型结构传输 

std::set<std::string> serverList;
typedef std::set<std::string>::iterator serverList_iter;
std::map<std::string,std::string> fileMap;
typedef std::map<std::string,std::string>::iterator fileMap_iter;
std::map<int,std::string> serverLoad;
typedef std::map<int,std::string>::iterator serverLoad_iter;

static char* s_recv(void* socket)
{
    zmq_msg_t message;
    zmq_msg_init(&message);
    zmq_msg_recv(&message,socket,0);
    int size = zmq_msg_size(&message);
    char* string =(char*)malloc(size+1);
    memcpy(string,(char*)zmq_msg_data(&message),size);
    zmq_msg_close(&message);
    string[size+1]= 0;
    return string;
}
static int s_send(void* socket,msg_t* buf)
{
    zmq_msg_t message;
    zmq_msg_init_size(&message,buf->nsize);
    memcpy((char*)zmq_msg_data(&message),(void*)buf,buf->nsize);
    int rc = zmq_msg_send(&message,socket,0);
    zmq_msg_close(&message);
    return rc;
}
static int s_sendmore(void* socket,msg_t* buf)
{
    zmq_msg_t message;
    zmq_msg_init_size(&message,buf->nsize);
    memcpy(zmq_msg_data(&message),(void*)buf,buf->nsize);
    int rc = zmq_msg_send(&message,socket,ZMQ_SNDMORE);
    zmq_msg_close(&message);
    return rc;
}

static void s_set_id(void* socket,const char* name)
{
    zmq_setsockopt(socket,ZMQ_IDENTITY,name,strlen(name));
}

6. Master节点的主函数

int main(int argc,char** argv)
{
    if(argc < 4)
    {
        printf("syntax:%s <requestpoint><responsepoint><moniter><slave>\n",argv[0]);
        exit(EXIT_SUCCESS);
    }
    void* context = zmq_init(1);
    void* requester = zmq_socket(context,ZMQ_REP);
    zmq_bind(requester,argv[1]);
    //void* responser = zmq_socket(context,ZMQ_REP);
    //zmq_bind(responser,argv[2]);
    void* slave = zmq_socket(context,ZMQ_PUSH);
    zmq_bind(slave,argv[4]);
    void* moniter = zmq_socket(context,ZMQ_SUB);
    zmq_setsockopt(moniter,ZMQ_SUBSCRIBE,"",0);
    zmq_bind(moniter,argv[3]);
    pthread_t heartThread;
    pthread_create(&heartThread,NULL,sendHeartBeat,slave);
    while(true)
    {
        zmq_pollitem_t items[]=
        {
            {requester,0,ZMQ_POLLIN,0},
            {moniter,0,ZMQ_POLLIN,0}
        };
        int rc = zmq_poll(items,2,-1);
        if(rc == -1)
            break;
        if(items[0].revents & ZMQ_POLLIN)
        {
            getClientRequest(requester);
        }
        else if(items[1].revents & ZMQ_POLLIN)
        {
            getSlaveRequest(moniter);
        }
    }
    pthread_kill(heartThread,SIGKILL);
    return 0;
}

7. Slave节点的主函数

int main(int argc,char** argv)
{
    if(argc < 3)
    {
        printf("syntax:%s <slavename><heartbeatname><mastername>\n",argv[0]);
        exit(EXIT_SUCCESS);
    }
    void* context = zmq_init(1);
    assert(context);
    void* receive = zmq_socket(context,ZMQ_REP);
    assert(receive);
    zmq_bind(receive,argv[1]);

    void* heartbeat = zmq_socket(context,ZMQ_PULL);
    assert(heartbeat);
    zmq_connect(heartbeat,argv[2]);
    void* master = zmq_socket(context,ZMQ_PUB);
    assert(master);
    zmq_connect(master,argv[3]);

    sendMsgToMoniter(master,argv[1],SLAVE_ONLINE);
    while(true)
    {
        zmq_pollitem_t items[] =
        {
            {receive,0,ZMQ_POLLIN,0},
            {heartbeat,0,ZMQ_POLLIN,0},
        };
        int rc = zmq_poll(items,2,-1);
        if(rc == -1)
                break;
        if(items[0].revents & ZMQ_POLLIN)
        {
            char* buffer = s_recv(receive);
            msg_t* msg = (msg_t*)buffer;
            assert(buffer);
            std::string tmp = FILE_PATH;
            std::string path = tmp + msg->buffer;
            switch(msg->slaveType)
            {
                case DIST_GET:
                    {
                        sendFile(receive,path.c_str());
                    }
                    break;
                case DIST_ADD:
                    {
                        recvFile(receive,path.c_str());
                    }
                    break;
            }
        }
        if(items[1].revents & ZMQ_POLLIN)
        {
            sendMsgToMoniter(master,argv[1],SLAVE_HEARTBEAT);
        }
    }
    return 0;
}

8. Client节点的主函数

int main(int argc,char** argv)
{
    if(argc< 3)
    {
        printf("syntax:%s <moniter><slave>\n",argv[0]);
        exit(EXIT_SUCCESS);
    }
    void* context = zmq_init(1);
    void* sender = zmq_socket(context,ZMQ_REQ);
    zmq_connect(sender,argv[1]);
    context_t contxt;
    contxt.sender = sender;
    contxt.context = context;
    pthread_t worker;
    pthread_create(&worker,NULL,worker_thread,&contxt);

    while(true)
    {
        msg_t msg;
        msg.nsize = sizeof(msg_t);
        std::cin.getline(msg.server,sizeof(msg.server));
        std::cin.getline(msg.buffer,sizeof(msg.buffer));
        if(!strcmp(msg.server,"quit"))
            break;
        s_send(sender,&msg);
    }
    pthread_join(worker,NULL);
    zmq_close(sender);
    zmq_term(context);
    return 0;
}

9.其他一些比较重要的函数

    1）Master节点  

void getClientRequest(void* requester)
{
    char* command = s_recv(requester);
    assert(command);
    msg_t* msg = (msg_t*)command;
    if(!strcmp(msg->server,"get"))
    {
        msg_t sndmsg;
        sndmsg.responType = DIST_GET;
        fileMap_iter iter = fileMap.find(msg->buffer);
        if(iter != fileMap.end())
        {
            memcpy(sndmsg.server,iter->second.c_str(),strlen(iter->second.c_str()));
            memcpy(sndmsg.buffer,msg->buffer,strlen(msg->buffer));
            s_send(requester,&sndmsg);
        }
        else
        {
            memcpy(sndmsg.server,"No",strlen("No"));
            s_send(requester,&sndmsg);
        }
    }
    else if(!strcmp(msg->server,"add"))
    {
        msg_t sndmsg;
        sndmsg.responType = DIST_ADD;
        sndmsg.nsize = sizeof(msg_t);
        serverLoad_iter iter = serverLoad.begin();
        if(iter != serverLoad.end())
        {
            memcpy(sndmsg.server,iter->second.c_str(),strlen(iter->second.c_str()));
            memcpy(sndmsg.buffer,msg->buffer,strlen(msg->buffer));
            s_send(requester,&sndmsg);
        }
        else
        {
            memcpy(sndmsg.server,"None",strlen("None"));
            s_send(requester,&sndmsg);
        }
    }
    else
    {
        msg_t sndmsg;
        sndmsg.responType = DIST_NONE;
        sndmsg.nsize = sizeof(msg_t);
        memcpy(sndmsg.server,"None",strlen("None"));
        s_send(requester,&sndmsg);
    }

}
void getSlaveRequest(void* moniter)
{
    assert(moniter);
    char* buffer = s_recv(moniter);
    assert(buffer);
    msg_t* msg = (msg_t*)buffer;

    switch(msg->slaveType)
    {
        case SLAVE_ONLINE:
            {
                serverList_iter iter = serverList.find(msg->server);
                if(iter == serverList.end())
                    serverList.insert(msg->server);
                serverLoad_iter serverload_iter = serverLoad.begin();
                while(serverload_iter != serverLoad.end())
                {
                    if(!strcmp(serverload_iter->second.c_str(),msg->server))
                    {
                        serverLoad.erase(serverload_iter);
                        break;
                    }
                    ++serverload_iter;
                }
                serverLoad.insert(std::make_pair(msg->total_size,msg->server));
            }
            break;
        case SLAVE_HEARTBEAT:
            {
                serverLoad_iter serverload_iter = serverLoad.begin();
                for(;serverload_iter != serverLoad.end();)
                {
                    if(!strcmp(serverload_iter->second.c_str(),msg->server))
                    {
                        serverLoad.erase(serverload_iter);
                        break;
                    }
                    ++serverload_iter;
                }
                serverLoad.insert(std::make_pair(msg->total_size,msg->server));

            }
            break;
    }
}

2）slave节点

void sendMsgToMoniter(void* sender,const char* slavename,slaveType_t type)
{
    int total_size = getDirSize(FILE_PATH);
    msg_t msg;
    msg.nsize = sizeof(msg_t);
    msg.total_size = total_size;
    msg.slaveType = type;
    memcpy((void*)msg.server,(void*)slavename,strlen(slavename));
    s_send(sender,&msg);
}

void recvFile(void* receiver,const char* filename)
{
    assert(receiver);
    assert(filename);
    FILE* fp = fopen(filename,"w+");
    assert(fp);
    char* buf = NULL;
    while((buf = s_recv(receiver))!=NULL)
    {
        fwrite(buf,strlen(buf),1,fp);
    }
    printf("recevie File:%s success\n",filename);
    msg_t msg;
    msg.nsize = sizeof(msg_t);
    s_send(receiver,&msg);
    fclose(fp);
}
void sendFile(void* sender,const char* filename)
{
    assert(sender);
    assert(filename);

    FILE* fp = fopen(filename,"r");
    if(NULL == fp)
    {
        printf("this is no such file\n");
        return;
    }
    fseek(fp,SEEK_SET,SEEK_END);
    size_t size = ftell(fp);
    msg_t msg;
    msg.nsize = sizeof(msg_t);
    msg.total_size = size;
    memcpy((void*)msg.buffer,filename,strlen(filename));
    s_send(sender,&msg);

    char buffer[BUFFER_SIZE];
    while(fread(buffer,sizeof(buffer),1,fp))
    {
        memcpy(msg.buffer,buffer,strlen(buffer));
        s_sendmore(sender,&msg);
        memset(buffer,0,sizeof(buffer));
        fflush(fp);
    }
    fclose(fp);
}

 3） Client节点

void read(void* context,const char* server,const char* filename)
{
    char* file;
    assert(context);
    assert(server);
    void* client = zmq_socket(context,ZMQ_REQ);
    zmq_connect(client,server);

    msg_t msg;
    msg.responType = DIST_GET;
    msg.nsize = sizeof(msg_t);
    memcpy(&msg.buffer,filename,strlen(filename));
    s_send(client,&msg);
    FILE* fp = fopen(filename,"w+");
    assert(fp);
    while((file = s_recv(client)))
    {
        fwrite(file,strlen(file),1,fp);
    }
    fclose(fp);
    zmq_close(&client);
}

void write(void* context,const char* server,const char* filename)
{
    assert(context);
    assert(server);
    void* sender = zmq_socket(context,ZMQ_REQ);
    zmq_connect(sender,server);

    printf("send file server:%s,filename:%s\n",server,filename);
    msg_t msg;
    msg.responType = DIST_ADD;
    msg.nsize = sizeof(msg_t);
    memcpy(msg.buffer,filename,strlen(filename));
    s_send(sender,&msg);

    FILE* fp = fopen(filename,"r");
    assert(fp);
    char buffer[BUFFER_SIZE];
    bzero(buffer,sizeof(buffer));
    while(fread(buffer,sizeof(buffer),1,fp))
    {
        msg.nsize = strlen(buffer);
        printf("msg.nsize=%d\n",msg.nsize);
        memcpy(msg.buffer,buffer,strlen(buffer));
        s_sendmore(sender,&msg);
        bzero(buffer,sizeof(buffer));
    }
    char* end = s_recv(sender);
    assert(end);
    fclose(fp);
    zmq_close(sender);
}

static void* worker_thread(void* contxt)
{
    assert(contxt);
    context_t* cont = (context_t*)contxt;
    while(true)
    {
        zmq_pollitem_t items[] =
        {
            {cont->sender,0,ZMQ_POLLIN,0}
        };
        int rc = zmq_poll(items,1,-1);
        if(rc == -1)
            break;
        if(items[0].revents & ZMQ_POLLIN)
        {
            char* response = s_recv(cont->sender);
            assert(response);
            msg_t* msg = (msg_t*)response;
            printf("response msg->server:%s,msg->buffer:%s\n",msg->server,msg->buffer);
            switch(msg->responType)
            {
                case DIST_NONE:
                    break;
                case DIST_GET:
                    {
                        if(!strcmp(msg->server,"No"))
                            printf("This is not such file\n");
                        else
                            read(cont->context,msg->server,msg->buffer);
                    }
                    break;
                case DIST_ADD:
                    {
                        if(!strcmp(msg->server,"None"))
                            printf("All Slave Server is Dead\n");
                        else
                            write(cont->context,msg->server,msg->buffer);
                    }
                    break;
            }
        }
    }
    return NULL;
}

10.总结

     目前整个框架已经开发完了，节点之间基本上完成了通讯需求，目前整个功能还在完善中，其他的代码将会放在github上，谢谢


标签
#分布式 #通讯 #框架 #云计算 #thread
评论 1
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值