Caffe生成的數(shù)據(jù)分為2種格式：Lmdb 和 Leveldb

• 它們都是鍵/值對（Key/Value Pair）嵌入式數(shù)據(jù)庫管理系統(tǒng)編程庫。
• 雖然lmdb的內(nèi)存消耗是leveldb的1.1倍，但是lmdb的速度比leveldb快10%至15%，更重要的是lmdb允許多種訓(xùn)練模型同時讀取同一組數(shù)據(jù)集。
• 因此lmdb取代了leveldb成為Caffe默認(rèn)的數(shù)據(jù)集生成格式。

create_babyface.sh調(diào)用的convertData的源代碼如下：

#include<sys/types.h> #include<sys/stat.h> #include<dirent.h> #include <stdio.h> #include<string.h>#include <fstream> // NOLINT(readability/streams) #include <string> #include <vector>#include "boost/scoped_ptr.hpp" #include "glog/logging.h" #include "google/protobuf/text_format.h" #include "stdint.h"#include "caffe/proto/caffe.pb.h" #include "caffe/util/db.hpp"#include <opencv/cv.h> #include <opencv/highgui.h>using caffe::Datum; using boost::scoped_ptr; using std::string; namespace db = caffe::db; using namespace std;const int kCIFARSize = 32; const int kCIFARChannelBytes = 1024; const int kCIFARImageNBytes = 3072; const int kCIFARBatchSize = 1000;//1000 for a batch! const int kCIFARTrainBatches = 5;void read_image(std::ifstream* file, int* label, char* buffer) {char label_char;file->read(&label_char, 1);*label = label_char;file->read(buffer, kCIFARImageNBytes);return; }//Read IPLimage to the buffer void read_image(IplImage* out, char* buffer,char* RC, char* GC, char* BC) {int x,y;int idx =0;for(y = 0; y<out->height; y++){char *ptr= out->imageData + y * out->widthStep;for( x = 0;x< out->width;x++){idx =y*out->height + x;BC[idx]= ptr[3*x];GC[idx]= ptr[3*x+1];RC[idx]= ptr[3*x+2]; //這樣就可以添加自己的操作，這里我使三通道顏色一樣，就彩色圖轉(zhuǎn)黑白圖了}}memcpy( buffer ,RC, kCIFARChannelBytes*sizeof(char) );memcpy( buffer+ kCIFARChannelBytes*sizeof(char) , GC,kCIFARChannelBytes*sizeof(char) );memcpy( buffer+ kCIFARChannelBytes*sizeof(char) *2, BC,kCIFARChannelBytes*sizeof(char) );return; }//Travel the folder and load the filelist! //使用linux dirent遍歷目錄int traveldir(char* path ,int depth, vector<string > &FileList) {DIR* d;// astruct dirent *file; struct stat sb;if( !(d=opendir(path ) ) ){printf("Read path %s error,wishchin! ", path);return -1;}while( (file= readdir(d ) ) != NULL ) {if(0== strncmp(file->d_name, ".", 1 ) ) continue;char filename[256];strcpy( filename , file->d_name );string Sfilename(filename);string Spath(path);Spath.append(Sfilename);FileList.push_back(Spath);}if( stat(file->d_name, &sb)>=0 && S_ISDIR(sb.st_mode) && depth <=4 )traveldir(file->d_name,depth+1,FileList);closedir(d);return 1; }// convert the data to the lmdb format ! void convert_dataset(const string& input_folder,const string& output_folder,const string& db_type) {scoped_ptr<db::DB> train_db(db::GetDB(db_type));train_db->Open(output_folder + "/babyface_train_" + db_type, db::NEW);scoped_ptr<db::Transaction> txn(train_db->NewTransaction());char* path=new char[256];int depth=2;vector<string > FileList(0);// Data bufferint label;IplImage* ImageS;char str_buffer[kCIFARImageNBytes];char* RC=new char[kCIFARChannelBytes];char* GC=new char[kCIFARChannelBytes];char* BC=new char[kCIFARChannelBytes];Datum datum;datum.set_channels(3);datum.set_height(kCIFARSize);datum.set_width(kCIFARSize);//"Writing Training data"//載入訓(xùn)練數(shù)據(jù)LOG(INFO) << "Writing Training data";strcpy(path,( input_folder+(string)("train1") ).c_str() );traveldir( path , depth, FileList);for (int fileid = 0; fileid < kCIFARTrainBatches; ++fileid) {// Open filesLOG(INFO) << "Training Batch " << fileid + 1;snprintf(str_buffer, kCIFARImageNBytes, "/data_batch_%d.bin", fileid + 1);//CHECK(data_file) << "Unable to open train file #" << fileid + 1;label=1;//The Batch has 10000 pics!for (int itemid = 0; itemid < kCIFARBatchSize; ++itemid) {ImageS =cvLoadImage( (FileList[ fileid*kCIFARTrainBatches + itemid] ).c_str() );read_image( ImageS, str_buffer, RC, GC, BC);datum.set_label(label);//datum.set_label(label);datum.set_data(str_buffer, kCIFARImageNBytes);int length = snprintf(str_buffer, kCIFARImageNBytes,"%05d", fileid * kCIFARBatchSize + itemid);string out;CHECK(datum.SerializeToString( &out) ) ;txn->Put(string(str_buffer, length), out);//The main sentence ,put data to the txn!}}strcpy(path,( input_folder+(string)("train0") ).c_str() );traveldir( path , depth, FileList);for (int fileid = 0; fileid < kCIFARTrainBatches; ++fileid) {LOG(INFO) << "Training Batch " << fileid + 1;snprintf(str_buffer, kCIFARImageNBytes, "/data_batch_%d.bin", fileid + 1);//CHECK(data_file) << "Unable to open train file #" << fileid + 1;label=0;//The Batch has 10000 pics!for (int itemid = 0; itemid < kCIFARBatchSize; ++itemid) {ImageS =cvLoadImage( (FileList[ fileid*kCIFARTrainBatches + itemid] ).c_str() );read_image( ImageS, str_buffer, RC, GC, BC);datum.set_label(label);//datum.set_label(label);datum.set_data(str_buffer, kCIFARImageNBytes);int length = snprintf(str_buffer, kCIFARImageNBytes,"%05d", fileid * kCIFARBatchSize + itemid);string out;CHECK(datum.SerializeToString( &out) ) ;txn->Put(string(str_buffer, length), out);//The main sentence ,put data to the txn!}}txn->Commit();train_db->Close();//寫入測試數(shù)據(jù)！LOG(INFO) << "Writing Testing data";scoped_ptr<db::DB> test_db(db::GetDB(db_type));test_db->Open(output_folder + "/babyface_test_" + db_type, db::NEW);txn.reset(test_db->NewTransaction());strcpy(path,( input_folder+(string)("test1") ).c_str() );traveldir( path , depth, FileList);for (int fileid = 0; fileid < 2; ++fileid) {LOG(INFO) << "Training Batch " << fileid + 1;snprintf(str_buffer, kCIFARImageNBytes, "/data_batch_%d.bin", fileid + 1);label=1;//The Batch has 10000 pics!for (int itemid = 0; itemid < kCIFARBatchSize; ++itemid) {ImageS =cvLoadImage( (FileList[ fileid*2 + itemid] ).c_str() );read_image( ImageS, str_buffer, RC, GC, BC);datum.set_label(label);//datum.set_label(label);datum.set_data(str_buffer, kCIFARImageNBytes);int length = snprintf(str_buffer, kCIFARImageNBytes,"%05d", fileid * kCIFARBatchSize + itemid);string out;CHECK(datum.SerializeToString( &out) ) ;txn->Put(string(str_buffer, length), out);//The main sentence ,put data to the txn!}}strcpy(path,( input_folder+(string)("test0") ).c_str() );traveldir( path , depth, FileList);for (int fileid = 0; fileid < 2; ++fileid) {LOG(INFO) << "Training Batch " << fileid + 1;snprintf(str_buffer, kCIFARImageNBytes, "/data_batch_%d.bin", fileid + 1);label=0;//The Batch has 10000 pics!for (int itemid = 0; itemid < kCIFARBatchSize; ++itemid) {ImageS =cvLoadImage( (FileList[ fileid*2 + itemid] ).c_str() );read_image( ImageS, str_buffer, RC, GC, BC);datum.set_label(label);//datum.set_label(label);datum.set_data(str_buffer, kCIFARImageNBytes);int length = snprintf(str_buffer, kCIFARImageNBytes,"%05d", fileid * kCIFARBatchSize + itemid);string out;CHECK(datum.SerializeToString( &out) ) ;txn->Put(string(str_buffer, length), out);//The main sentence ,put data to the txn!}}txn->Commit();test_db->Close();cvReleaseImage(&ImageS);delete [] RC;delete [] GC;delete [] BC; }int main(int argc, char** argv) {if (argc != 4) {printf("This script converts the CIFAR dataset to the leveldb format used\n""by caffe to perform classification.\n""Usage:\n"" convert_cifar_data input_folder output_folder db_type\n""Where the input folder should contain the binary batch files.\n""The CIFAR dataset could be downloaded at\n"" http://www.cs.toronto.edu/~kriz/cifar.html\n""You should gunzip them after downloading.\n");} else {google::InitGoogleLogging(argv[0]);convert_dataset(string(argv[1]), string(argv[2]), string(argv[3]));}return 0; }

??????? 后記：目的是載入32×32的三通道圖像，直接輸入3072維的char向量進(jìn)行訓(xùn)練，至于怎樣訓(xùn)練網(wǎng)絡(luò)，還得仔細(xì)查看一下。

????? 后記：代碼出現(xiàn) coredump 問題，利用 gcc path/...bin? -o coredemo -g ，出現(xiàn)caffe.pb.h 包含丟失現(xiàn)象，why？？？

總結(jié)

以上是生活随笔為你收集整理的caffe特征提取/C++数据格式转换的全部內(nèi)容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網(wǎng)站內(nèi)容還不錯，歡迎將生活随笔推薦給好友。