refactor some code. add more optimizer.

Author: xylcbd

README.md

@@ -9,7 +9,7 @@ small, clean, easy to understand!
 * All in one: without any dependency, pure c++ implemented.
 * Basic layer: data layer, convolution layer, pooling layer, full connect layer, softmax layer, activation layers(sigmod, tanh, RELU)
 * Loss function: Cross Entropy, MSE.
-* Optimize method: SGD.
+* Optimize method: SGD, SGDWithMomentum.
 
 ## Examples
 * mnist demo, with ConvNet and MLP net,  [examples/mnist/mnist_train_test.cpp](./examples/mnist/mnist_train_test.cpp "mnist_train_test.cpp")  
@@ -19,7 +19,7 @@ small, clean, easy to understand!
 * ~~fix train error when batch > 1 issue.~~
 * ~~add load & save model function.~~
 * add more layer, such as batch normalization layer, dropout layer, etc.
-* add weight regular, gradient momentum.
+* add weight regular.
 * port to other platforms, such as linux, mac, android, iOS, etc.
 * optimize network train/test speed, use cuBLAS/OpenBLAS etc.
 * add more optimize method.

examples/digit/digit_train_test.cpp

@@ -197,6 +197,7 @@ static EasyCNN::NetWork buildMLPNet(const size_t batch, const size_t channels, c
 	network.setPhase(EasyCNN::Phase::Train);
 	network.setInputSize(EasyCNN::DataSize(batch, channels, width, height));
 	network.setLossFunctor(std::make_shared<EasyCNN::MSEFunctor>());
+	network.setOptimizer(std::make_shared<EasyCNN::SGDWithMomentum>(0.01f, 0.9f));
 	//input data layer
 	std::shared_ptr<EasyCNN::InputLayer> _0_inputLayer(std::make_shared<EasyCNN::InputLayer>());
 	network.addayer(_0_inputLayer);
@@ -273,7 +274,7 @@ static void train(const std::string& digit_train_images_dir,
 	EasyCNN::logCritical("load training data done. train set's size is %d,validate set's size is %d", train_images.size(), validate_images.size());
 
 	float learningRate = 0.1f;
-	const float decayRate = 0.001f;
+	const float decayRate = 0.2f;
 	const float minLearningRate = 0.00001f;
 	const size_t testAfterBatches = 200;
 	const size_t maxBatches = 100000000;
@@ -288,6 +289,7 @@ static void train(const std::string& digit_train_images_dir,
 
 	EasyCNN::logCritical("construct network begin...");
 	EasyCNN::NetWork network(buildConvNet(batch, channels, width, height));
+	network.setLearningRate(learningRate);
 	EasyCNN::logCritical("construct network done.");
 
 	//train
@@ -297,18 +299,18 @@ static void train(const std::string& digit_train_images_dir,
 	size_t epochIdx = 0;
 	while (epochIdx < max_epoch)
 	{
+		//before epoch start, shuffle all train data first
+		shuffle_data(images, labels);
 		size_t batchIdx = 0;
 		while (true)
 		{
 			if (!fetch_data(train_images, inputDataBucket, train_labels, labelDataBucket, batchIdx*batch, batch))
 			{
 				break;
 			}
-			const float loss = network.trainBatch(inputDataBucket,labelDataBucket, learningRate);
+			const float loss = network.trainBatch(inputDataBucket,labelDataBucket);
 			if (batchIdx > 0 && batchIdx % testAfterBatches == 0)
-			{
-				learningRate -= decayRate;
-				learningRate = std::max(learningRate, minLearningRate);
+			{				
 				const float accuracy = test_batch(network,128,validate_images, validate_labels);
 				EasyCNN::logCritical("sample : %d/%d , learningRate : %f , loss : %f , accuracy : %.4f%%", 
 					batchIdx*batch, train_images.size(), learningRate, loss, accuracy*100.0f);
@@ -325,6 +327,10 @@ static void train(const std::string& digit_train_images_dir,
 		}
 		const float accuracy = test_batch(network,128,validate_images, validate_labels);
 		EasyCNN::logCritical("epoch[%d] accuracy : %.4f%%", epochIdx++, accuracy*100.0f);
+		//update learning rate
+		learningRate *= decayRate;
+		learningRate = std::max(learningRate, minLearningRate);
+		network.setLearningRate(learningRate);
 		if (accuracy >= 0.99)
 		{
 			break;

examples/main.cpp

@@ -7,5 +7,5 @@ extern int digit_main(int argc, char* argv[]);
 
 int main(int argc, char* argv[])
 {
-	return digit_main(argc, argv);
+	return mnist_main(argc, argv);
 }

examples/mnist/mnist_train_test.cpp

@@ -152,6 +152,7 @@ static EasyCNN::NetWork buildConvNet(const size_t batch,const size_t channels,co
 	network.setPhase(EasyCNN::Phase::Train);
 	network.setInputSize(EasyCNN::DataSize(batch, channels, width, height));
 	network.setLossFunctor(std::make_shared<EasyCNN::CrossEntropyFunctor>());
+	network.setOptimizer(std::make_shared<EasyCNN::SGD>(0.01f));
 	//input data layer 0
 	std::shared_ptr<EasyCNN::InputLayer> _0_inputLayer(std::make_shared<EasyCNN::InputLayer>());
 	network.addayer(_0_inputLayer);
@@ -197,6 +198,7 @@ static EasyCNN::NetWork buildMLPNet(const size_t batch, const size_t channels, c
 	network.setPhase(EasyCNN::Phase::Train);
 	network.setInputSize(EasyCNN::DataSize(batch, channels, width, height));
 	network.setLossFunctor(std::make_shared<EasyCNN::MSEFunctor>());
+	network.setOptimizer(std::make_shared<EasyCNN::SGDWithMomentum>(0.01f,0.9f));
 	//input data layer
 	std::shared_ptr<EasyCNN::InputLayer> _0_inputLayer(std::make_shared<EasyCNN::InputLayer>());
 	network.addayer(_0_inputLayer);
@@ -277,7 +279,7 @@ static void train(const std::string& mnist_train_images_file,
 	EasyCNN::logCritical("load training data done. train set's size is %d,validate set's size is %d", train_images.size(), validate_images.size());
 
 	float learningRate = 0.1f;
-	const float decayRate = 0.001f;
+	const float decayRate = 0.2f;
 	const float minLearningRate = 0.001f;
 	const size_t testAfterBatches = 200;
 	const size_t maxBatches = 10000;
@@ -291,7 +293,8 @@ static void train(const std::string& mnist_train_images_file,
 	EasyCNN::logCritical("channels:%d , width:%d , height:%d", channels, width, height);
 
 	EasyCNN::logCritical("construct network begin...");
-	EasyCNN::NetWork network(buildConvNet(batch, channels, width, height));
+	EasyCNN::NetWork network(buildMLPNet(batch, channels, width, height));
+	network.setLearningRate(learningRate);
 	EasyCNN::logCritical("construct network done.");
 
 	//train
@@ -301,18 +304,18 @@ static void train(const std::string& mnist_train_images_file,
 	size_t epochIdx = 0;
 	while (epochIdx < max_epoch)
 	{
+		//before epoch start, shuffle all train data first
+		shuffle_data(images, labels);
 		size_t batchIdx = 0;
 		while (true)
 		{
 			if (!fetch_data(train_images, inputDataBucket, train_labels, labelDataBucket, batchIdx*batch, batch))
 			{
 				break;
 			}
-			const float loss = network.trainBatch(inputDataBucket,labelDataBucket, learningRate);
+			const float loss = network.trainBatch(inputDataBucket,labelDataBucket);
 			if (batchIdx > 0 && batchIdx % testAfterBatches == 0)
 			{
-				learningRate -= decayRate;
-				learningRate = std::max(learningRate, minLearningRate);
 				const float accuracy = test(network,128,validate_images, validate_labels);
 				EasyCNN::logCritical("sample : %d/%d , learningRate : %f , loss : %f , accuracy : %.4f%%", 
 					batchIdx*batch, train_images.size(), learningRate, loss, accuracy*100.0f);
@@ -326,8 +329,11 @@ static void train(const std::string& mnist_train_images_file,
 		if (batchIdx >= maxBatches)
 		{
 			break;
-		}
+		}		
 		const float accuracy = test(network,128,validate_images, validate_labels);
+		//update learning rate
+		learningRate = std::max(learningRate*decayRate, minLearningRate);
+		network.setLearningRate(learningRate);
 		EasyCNN::logCritical("epoch[%d] accuracy : %.4f%%", epochIdx++, accuracy*100.0f);
 	}
 	const float accuracy = test(network, 128, validate_images, validate_labels);
@@ -376,9 +382,9 @@ static void test(const std::string& mnist_test_images_file,
 	EasyCNN::logCritical("finished test.");
 }
 
-static std::shared_ptr<EasyCNN::DataBucket> loadImage(const std::vector<std::string>& filePaths)
+static std::shared_ptr<EasyCNN::DataBucket> loadImage(const std::vector<std::pair<int, cv::Mat>>& samples)
 {
-	const int number = filePaths.size();
+	const int number = samples.size();
 	const int channel = 1;
 	const int width = 20;
 	const int height = 20;
@@ -387,11 +393,11 @@ static std::shared_ptr<EasyCNN::DataBucket> loadImage(const std::vector<std::str
 	const float scaleRate = 1.0f / 255.0f;
 	for (size_t i = 0; i < (size_t)number; i++)
 	{
-		const cv::Mat srcGrayImg = cv::imread(filePaths[i], cv::IMREAD_GRAYSCALE);
+		const cv::Mat srcGrayImg = samples[i].second;
 		cv::Mat normalisedImg;
 		cv::resize(srcGrayImg, normalisedImg, cv::Size(width, height));
 		cv::Mat binaryImg;
-		cv::threshold(normalisedImg, binaryImg, 127, 255, CV_THRESH_BINARY_INV);
+		cv::threshold(normalisedImg, binaryImg, 127, 255, CV_THRESH_BINARY);
 
 		//image data
 		float* inputData = result->getData().get() + i*sizePerImage;
@@ -403,7 +409,7 @@ static std::shared_ptr<EasyCNN::DataBucket> loadImage(const std::vector<std::str
 	}
 	return result;
 }
-static void test_single(const std::vector<std::string>& filePaths, const std::string& modelFilePath)
+static void test_single(const std::vector<std::pair<int, cv::Mat>>& samples, const std::string& modelFilePath)
 {
 	bool success = false;
 
@@ -418,26 +424,55 @@ static void test_single(const std::vector<std::string>& filePaths, const std::st
 	//train
 	EasyCNN::logCritical("begin test...");
 
-	const std::shared_ptr<EasyCNN::DataBucket> inputDataBucket = loadImage(filePaths);
+	const std::shared_ptr<EasyCNN::DataBucket> inputDataBucket = loadImage(samples);
 	const std::shared_ptr<EasyCNN::DataBucket> probDataBucket = network.testBatch(inputDataBucket);
 	const size_t labelSize = probDataBucket->getSize()._3DSize();
 	const float* probData = probDataBucket->getData().get();
-	for (size_t j = 0; j < filePaths.size(); j++)
+	for (size_t i = 0; i < samples.size(); i++)
 	{
-		const uint8_t testProb = getMaxIdxInArray(probData + j*labelSize, probData + (j + 1) * labelSize);
+		const uint8_t testProb = getMaxIdxInArray(probData + i*labelSize, probData + (i + 1) * labelSize);
 		EasyCNN::logCritical("label : %d",testProb);
 
-		const cv::Mat srcGrayImg = cv::imread(filePaths[j], cv::IMREAD_GRAYSCALE);
+		const cv::Mat srcGrayImg = samples[i].second;
 		cv::destroyAllWindows();
 		cv::imshow("src", srcGrayImg);
 		cv::waitKey(0);
 	}
 	EasyCNN::logCritical("finished test.");
 }
+static cv::Mat image_to_cv(const image_t& img)
+{
+	assert(img.channels == 1);
+	cv::Mat result(img.height, img.width,CV_8UC1,(void*)(&img.data[0]),img.width);
+	return result.clone();
+}
+static std::vector<std::pair<int, cv::Mat>> export_random_mnist_image(const std::string& mnist_test_images_file,
+	const std::string& mnist_test_labels_file, 
+	const int test_size)
+{
+	std::vector<std::pair<int, cv::Mat>> result;
+	bool success = true;
+	std::vector<image_t> images;
+	success = load_mnist_images(mnist_test_images_file, images);
+	assert(success);
+	std::vector<label_t> labels;
+	success = load_mnist_labels(mnist_test_labels_file, labels);
+	assert(success);
+	std::default_random_engine generator;
+	std::uniform_int_distribution<int> dis(0, images.size());
+	for (int i = 0; i < test_size;i++)
+	{
+		const int idx = dis(generator);
+		const int label = labels[idx].data;
+		const cv::Mat image = image_to_cv(images[idx]);
+		result.push_back(std::make_pair(label, image));
+	}
+	return result;
+}
 int mnist_main(int argc, char* argv[])
 {
-	const std::string model_file = "../../res/model/mnist_conv.model";
-#if 0
+	const std::string model_file = "../../res/model/mnist_mlp.model";
+#if 1
 	const std::string mnist_train_images_file = "../../res/mnist_data/train-images.idx3-ubyte";
 	const std::string mnist_train_labels_file = "../../res/mnist_data/train-labels.idx1-ubyte";
 	train(mnist_train_images_file, mnist_train_labels_file, model_file);
@@ -447,8 +482,11 @@ int mnist_main(int argc, char* argv[])
 	const std::string mnist_test_images_file = "../../res/mnist_data/t10k-images.idx3-ubyte";
 	const std::string mnist_test_labels_file = "../../res/mnist_data/t10k-labels.idx1-ubyte";
 	test(mnist_test_images_file, mnist_test_labels_file, model_file);
+#else
+	const std::string mnist_test_images_file = "../../res/mnist_data/t10k-images.idx3-ubyte";
+	const std::string mnist_test_labels_file = "../../res/mnist_data/t10k-labels.idx1-ubyte";
+	std::vector<std::pair<int, cv::Mat>> samples = export_random_mnist_image(mnist_test_images_file, mnist_test_labels_file, 10);
+	test_single(samples, model_file);
 #endif
-
-	test_single(std::vector<std::string>{"d:/0.png", "d:/1.png", "d:/2.png"}, model_file);
 	return 0;
 }

header/EasyCNN/ActivationLayer.h

@@ -6,6 +6,10 @@ namespace EasyCNN
 {
 	class ActivationLayer : public Layer
 	{
+	public:
+		virtual std::string getLayerType() const = 0;
+		virtual void forward(const std::shared_ptr<DataBucket> prevDataBucket, std::shared_ptr<DataBucket> nextDataBucket) = 0;
+		virtual void backward(std::shared_ptr<DataBucket> prevDataBucket, const std::shared_ptr<DataBucket> nextDataBucket, std::shared_ptr<DataBucket>& nextDiffBucket) = 0;
 	};
 
 	class SigmodLayer : public ActivationLayer

header/EasyCNN/ConvolutionLayer.h

@@ -24,7 +24,9 @@ namespace EasyCNN
 		size_t widthStep = 0;
 		size_t heightStep = 0;
 		std::shared_ptr<ParamBucket> kernelData;
+		std::shared_ptr<ParamBucket> kernelDiffData;
 		bool enabledBias = false;
 		std::shared_ptr<ParamBucket> biasData;
+		std::shared_ptr<ParamBucket> biasDiffData;
 	};
 }

header/EasyCNN/DataBucket.h

@@ -1,5 +1,4 @@
 #pragma once
-#include <iostream>
 #include <memory>
 #include "EasyCNN/Configure.h"
 #include "EasyCNN/EasyLogger.h"
@@ -13,6 +12,7 @@ namespace EasyCNN
 		DataSize() = default;
 		DataSize(const size_t _number, const size_t _channels, const size_t _width, const size_t _height)
 			:number(_number),channels(_channels), width(_width), height(_height){}
+		inline size_t totalSize() const { return _4DSize(); }
 		inline size_t _4DSize() const { return number*channels*width*height; }
 		inline size_t _3DSize() const { return channels*width*height; }
 		inline size_t _2DSize() const { return width*height; }

header/EasyCNN/FullconnectLayer.h

@@ -19,11 +19,13 @@ namespace EasyCNN
 		virtual std::string getLayerType() const override;
 		virtual void solveInnerParams() override;
 		virtual void forward(const std::shared_ptr<DataBucket> prevDataBucket, std::shared_ptr<DataBucket> nextDataBucket) override;
-		virtual void backward(std::shared_ptr<DataBucket> prevDataBucket, const std::shared_ptr<DataBucket> nextDataBucket, std::shared_ptr<DataBucket>& nextDiffBucket) override;
+		virtual void backward(std::shared_ptr<DataBucket> prevDataBucket, const std::shared_ptr<DataBucket> nextDataBucket, std::shared_ptr<DataBucket>& nextDiffBucket) override;		
 	private:
 		ParamSize outMapSize;
 		std::shared_ptr<ParamBucket> weightsData;
+		std::shared_ptr<ParamBucket> weightsDiffData;
 		bool enabledBias = false;
-		std::shared_ptr<ParamBucket> biasData;
+		std::shared_ptr<ParamBucket> biasData;		
+		std::shared_ptr<ParamBucket> biasDiffData;
 	};
 }

header/EasyCNN/Layer.h

@@ -1,6 +1,7 @@
 #pragma once
 #include <memory>
 #include <string>
+#include <vector>
 #include "EasyCNN/Configure.h"
 #include "EasyCNN/DataBucket.h"
 #include "EasyCNN/ParamBucket.h"
@@ -29,6 +30,10 @@ namespace EasyCNN
 		//learning rate
 		inline void setLearningRate(const float learningRate){ this->learningRate = learningRate; }
 		inline float getLearningRate() const{ return learningRate; }	
+		//diff
+		inline std::vector<std::shared_ptr<ParamBucket>> getDiffData() const { return diff; }
+		//params
+		inline std::vector<std::shared_ptr<ParamBucket>> getParamData() const { return params; }
 		//size
 		inline void setInputBucketSize(const DataSize size){ inputSize = size; }
 		inline DataSize getInputBucketSize() const{ return inputSize; }
@@ -39,7 +44,11 @@ namespace EasyCNN
 		//data flow		
 		virtual void forward(const std::shared_ptr<DataBucket> prevDataBucket, std::shared_ptr<DataBucket> nextDataBucket) = 0;
 		virtual void backward(std::shared_ptr<DataBucket> prevDataBucket, const std::shared_ptr<DataBucket> nextDataBucket, std::shared_ptr<DataBucket>& nextDiffBucket) = 0;
-	private:
+	protected:
+		//subclass must add all diffs to diff
+		std::vector<std::shared_ptr<DataBucket>> diff;
+		//subclass must add all weight to params
+		std::vector<std::shared_ptr<ParamBucket>> params;
 		Phase phase = Phase::Train;
 		DataSize inputSize;
 		DataSize outputSize;

header/EasyCNN/NetWork.h

@@ -4,6 +4,7 @@
 #include "EasyCNN/Configure.h"
 #include "EasyCNN/Layer.h"
 #include "EasyCNN/LossFunction.h"
+#include "EasyCNN/Optimizer.h"
 
 namespace EasyCNN
 {
@@ -22,17 +23,19 @@ namespace EasyCNN
 		//train only!
 		void setInputSize(const DataSize size);
 		void setLossFunctor(std::shared_ptr<LossFunctor> lossFunctor);
+		void setOptimizer(std::shared_ptr<Optimizer> optimizer);
+		void setLearningRate(const float lr);
 		void addayer(std::shared_ptr<Layer> layer);
 		float trainBatch(const std::shared_ptr<DataBucket> inputDataBucket,
-			const std::shared_ptr<DataBucket> labelDataBucket, float learningRate);
+			const std::shared_ptr<DataBucket> labelDataBucket);
 		bool saveModel(const std::string& modelFile);
 	private:
 		std::string encrypt(const std::string& content);
 		std::string decrypt(const std::string& content);
 	private:
 		//common
 		std::shared_ptr<EasyCNN::DataBucket> forward(const std::shared_ptr<DataBucket> inputDataBucket);
-		float backward(const std::shared_ptr<DataBucket> labelDataBucket, float learningRate);
+		float backward(const std::shared_ptr<DataBucket> labelDataBucket);
 		std::string serializeToString() const;
 		std::vector<std::shared_ptr<EasyCNN::Layer>> serializeFromString(const std::string content);
 		std::shared_ptr<EasyCNN::Layer> createLayerByType(const std::string layerType);
@@ -41,5 +44,6 @@ namespace EasyCNN
 		std::vector<std::shared_ptr<Layer>> layers;
 		std::vector<std::shared_ptr<DataBucket>> dataBuckets;
 		std::shared_ptr<LossFunctor> lossFunctor;
+		std::shared_ptr<Optimizer> optimizer;
 	};
 }