有時會用到將一組圖像存放成MNIST中那樣的數據格式，以便于用于網絡的訓練和測試，如MNSIT中的測試集標簽t10k-labels.idx1-ubyte和測試集圖像t10k-images.idx3-ubyte，各包含了10000個樣本，這里以此兩個測試集為例詳細說明下實現過程：

在http://yann.lecun.com/exdb/mnist/ ?中對MNIST的數據存放格式進行了介紹，存儲的數據都以大多數非英特爾處理器使用的MSB優先(高端)格式存儲,英特爾處理器和其他低端機器的用戶必須翻轉標頭的字節(All the integers in the files are stored in the MSB first(high endian) format used by most non-Intel processors. Users of Intel processors and other low-endian machines must flip the bytes of the header.)。

t10k-labels.idx1-ubyte(訓練集標簽train-labels.idx1-ubyte與此存放格式完全相同)：第1至第4個字節存放magic number(MSB first)；第5至第8個字節存放標簽數即10000;從第9個字節開始,每個字節存放一個標簽值(label value)，標簽值的范圍為0到9。

此處的magic number(MSB first)是一個四個字節的整數，是一個IDX文件格式；第1，第2個字節總是0；第3個字節值表示數據的類型，如0x08表示unsigned byte；0x09表示signed byte；0x0B表示short(2 bytes)；0x0C表示int(4 bytes)；0x0D表示float(4 bytes)；0x0E表示double(8 bytes);因為t10k-labels.idx1-ubyte中標簽值范圍為0到9，因此這里第3字節值為0x08；第4個字節表示向量/矩陣的維數，1表示向量，2表示矩陣等；這里的標簽為一維向量，因此第4字節為0x01。t10k-labels.idx1-ubyte中的前8個字節是兩個magic number。

打開t10k-labels.idx1-ubyte二進制文件，前8個字節數據是：00 00 08 01 00 00 27 10，這里需要注意的是，magic number是一個四字節int，在讀或寫時每次性讀取4個字節，高字節在后，低字節在前，與存儲時順序不同，高字節在前，低字節在后，因此在讀或寫magic number時，需要做個轉換，即高字節變低字節，低字節變高字節，實現見ReverseInt函數。

t10k-images.idx3-ubyte(訓練集圖像train-images.idx3-ubyte與此存放格式完全相同)：第1至第4個字節存放magic number(MSB first)；第5至第8個字節存放圖像數即10000；第9至第12個字節存放每個圖像的行數即高，這里為28；第13至第16個字節存放每個圖像的列數即寬，這里為28；從第17個字節開始，每個字節存放一個像素值，像素值的范圍為0到255，0表示背景，255表示前景，像素按行排列；每28*28個字節大小存放一幅圖像數據。

此處的magic number(MSB first)是一個四個字節的整數，是一個IDX文件格式；第1，第2個字節總是0；第3個字節值表示數據的類型，如0x08表示unsigned byte；0x09表示signed byte；0x0B表示short(2 bytes)；0x0C表示int(4 bytes)；0x0D表示float(4 bytes)；0x0E表示double(8 bytes);因為t10k-images.idx3-ubyte中圖像像素值范圍為0到255，因此這里第3字節值為0x08；第4個字節表示向量/矩陣的維數，1表示向量，2表示矩陣等；這里的圖像可看做三維即channels*height*width，因此第4字節為0x03。t10k-images.idx3-ubyte中的前16個字節是四個magic number。打開0x03.t10k-images.idx3-ubyte二進制文件，前16個字節數據是：00 00 08 03 00 00 27 10 00 00 00 1c 00 00 00 1c。

測試代碼如下：

#include "funset.hpp"
#include <iostream>
#include <fstream>
#include <vector>
#include <memory>
#include <opencv2/opencv.hpp>// MNIST /
namespace {
int ReverseInt(int i)
{unsigned char ch1, ch2, ch3, ch4;ch1 = i & 255;ch2 = (i >> 8) & 255;ch3 = (i >> 16) & 255;ch4 = (i >> 24) & 255;return((int)ch1 << 24) + ((int)ch2 << 16) + ((int)ch3 << 8) + ch4;
}void read_Mnist(std::string filename, std::vector<cv::Mat> &vec)
{std::ifstream file(filename, std::ios::binary);if (file.is_open()) {int magic_number = 0;int number_of_images = 0;int n_rows = 0;int n_cols = 0;file.read((char*)&magic_number, sizeof(magic_number));magic_number = ReverseInt(magic_number);file.read((char*)&number_of_images, sizeof(number_of_images));number_of_images = ReverseInt(number_of_images);file.read((char*)&n_rows, sizeof(n_rows));n_rows = ReverseInt(n_rows);file.read((char*)&n_cols, sizeof(n_cols));n_cols = ReverseInt(n_cols);for (int i = 0; i < number_of_images; ++i) {cv::Mat tp = cv::Mat::zeros(n_rows, n_cols, CV_8UC1);for (int r = 0; r < n_rows; ++r) {for (int c = 0; c < n_cols; ++c) {unsigned char temp = 0;file.read((char*)&temp, sizeof(temp));tp.at<uchar>(r, c) = (int)temp;}}vec.push_back(tp);}file.close();}
}void read_Mnist_Label(std::string filename, std::vector<int> &vec)
{std::ifstream file(filename, std::ios::binary);if (file.is_open()) {int magic_number = 0;int number_of_images = 0;int n_rows = 0;int n_cols = 0;file.read((char*)&magic_number, sizeof(magic_number));magic_number = ReverseInt(magic_number);file.read((char*)&number_of_images, sizeof(number_of_images));number_of_images = ReverseInt(number_of_images);for (int i = 0; i < number_of_images; ++i) {unsigned char temp = 0;file.read((char*)&temp, sizeof(temp));vec[i] = (int)temp;}file.close();}
}std::string GetImageName(int number, int arr[])
{std::string str1, str2;for (int i = 0; i < 10; i++) {if (number == i) {arr[i]++;str1 = std::to_string(arr[i]);if (arr[i] < 10) {str1 = "0000" + str1;} else if (arr[i] < 100) {str1 = "000" + str1;} else if (arr[i] < 1000) {str1 = "00" + str1;} else if (arr[i] < 10000) {str1 = "0" + str1;}break;}}str2 = std::to_string(number) + "_" + str1;return str2;
}int write_images_to_file(const std::string& file_name, const std::vector<cv::Mat>& image_data,int magic_number, int image_number, int image_rows, int image_cols)
{if (image_number > image_data.size()) {fprintf(stderr, "Error: image_number > image_data.size(): \image_number: %d, image_data.size: %d", image_number, image_data.size());return -1;}std::ofstream file(file_name, std::ios::binary);if (!file.is_open()) {fprintf(stderr, "Error: open file fail: %s\n", file_name.c_str());return -1;}int tmp = ReverseInt(magic_number);file.write((char*)&tmp, sizeof(int));tmp = ReverseInt(image_number);file.write((char*)&tmp, sizeof(int));tmp = ReverseInt(image_rows);file.write((char*)&tmp, sizeof(int));tmp = ReverseInt(image_cols);file.write((char*)&tmp, sizeof(int));int size = image_rows * image_cols;for (int i = 0; i < image_number; ++i) {file.write((char*)image_data[i].data, sizeof(unsigned char) * size);}file.close();return 0;
}int write_labels_to_file(const std::string& file_name, const std::vector<int>& label_data,int magic_number, int label_number)
{if (label_number > label_data.size()) {fprintf(stderr, "Error: label_number > label_data.size(): \label_number: %d, label_data.size: %d", label_number, label_data.size());return -1;}std::ofstream file(file_name, std::ios::binary);if (!file.is_open()) {fprintf(stderr, "Error: open file fail: %s\n", file_name.c_str());return -1;}int tmp = ReverseInt(magic_number);file.write((char*)&tmp, sizeof(int));tmp = ReverseInt(label_number);file.write((char*)&tmp, sizeof(int));std::unique_ptr<unsigned char[]> labels(new unsigned char[label_number]);for (int i = 0; i < label_number; ++i) {labels[i] = static_cast<unsigned char>(label_data[i]);}file.write((char*)labels.get(), sizeof(unsigned char) * label_number);file.close();return 0;
}
} // namespace //mnistint ImageToMNIST()
{// read images
#ifdef _MSC_VERstd::string filename_test_images = "E:/GitCode/NN_Test/data/database/MNIST/t10k-images.idx3-ubyte";
#elsestd::string filename_test_images = "data/database/MNIST/t10k-images.idx3-ubyte";
#endifconst int number_of_test_images = 10000;std::vector<cv::Mat> vec_test_images;read_Mnist(filename_test_images, vec_test_images);if (vec_test_images.size() != number_of_test_images) {fprintf(stderr, "Error: fail to parse t10k-images.idx3-ubyte file: %d\n", vec_test_images.size());return -1;}// read labels
#ifdef _MSC_VERstd::string filename_test_labels = "E:/GitCode/NN_Test/data/database/MNIST/t10k-labels.idx1-ubyte";
#elsestd::string filename_test_labels = "data/database/MNIST/t10k-labels.idx1-ubyte";
#endifstd::vector<int> vec_test_labels(number_of_test_images);read_Mnist_Label(filename_test_labels, vec_test_labels);// write imagesconst int image_magic_number = 2051; // 0x00000803const int image_number = 10000;const int image_rows = 28;const int image_cols = 28;
#ifdef _MSC_VERconst std::string images_save_file_name = "E:/GitCode/NN_Test/data/new_t10k-images.idx3-ubyte";
#elseconst std::string images_save_file_name = "data/new_t10k-images.idx3-ubyte";
#endifif (write_images_to_file(images_save_file_name, vec_test_images, image_magic_number,image_number, image_rows, image_cols) != 0) {fprintf(stderr, "Error: write images to file fail\n");return -1;}// write labelsconst int label_magic_number = 2049; // 0x00000801const int label_number = 10000;
#ifdef _MSC_VERconst std::string labels_save_file_name = "E:/GitCode/NN_Test/data/new_t10k-labels.idx1-ubyte";
#elseconst std::string labels_save_file_name = "data/new_t10k-labels.idx1-ubyte";
#endifif (write_labels_to_file(labels_save_file_name, vec_test_labels, label_magic_number, label_number) != 0) {fprintf(stderr, "Error: write labels to file fail\n");return -1;}return 0;
}

新生成的兩個數據文件為new_t10k-labels.idx1-ubyte和new_t10k-images.idx3-ubyte，通過md5可知，新生成的文件與原始文件完全相同，結果如下：

GitHub：?https://github.com/fengbingchun/NN_Test?

總結

以上是生活随笔為你收集整理的图像集存储成MNIST数据集格式实现的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。