Skip to content

Commit

Permalink
[Application] Add Mixed Precision build file and update application main
Browse files Browse the repository at this point in the history 
Add Mixed Precision Build file and Update Application's main
this Application will run only at fp16 enable

**Self evaluation:**
1. Build test:	 [X]Passed [ ]Failed [ ]Skipped
2. Run test:	 [X]Passed [ ]Failed [ ]Skipped

Signed-off-by: Donghak PARK <[email protected]>
  • Loading branch information
@DonghakPark
DonghakPark committed Nov 26, 2024
1 parent 363edde commit 15fe748
Show file tree
Hide file tree
Showing 3 changed files with 158 additions and 122 deletions.
269 changes: 152 additions & 117 deletions Applications/MixedPrecision/jni/main.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,59 +1,109 @@
// SPDX-License-Identifier: Apache-2.0
/**
* Copyright (C) 2020 Jijoong Moon <jijoong.moon@samsung.com>
* Copyright (C) 2024 Donghak Park <donghak.park@samsung.com>
*
* @file main.cpp
* @date 25 Jul 2022
* @date 01 Jul 2024
* @brief Resnet Mixed Precision example
* @see https://github.com/nnstreamer/nntrainer
* @author Jijoong Moon <[email protected]>
* @author Donghak Park <[email protected]>
* @bug No known bugs except for NYI items
* @brief This is AlexNet Example with CIFAR100
*
* @bug No known bugs except for NYI items
*/

#include <fstream>
#include <iomanip>
#include <array>
#include <chrono>
#include <ctime>
#include <iostream>
#include <memory>
#include <sstream>
#include <stdlib.h>
#include <vector>

#include <cifar_dataloader.h>
#include <layer.h>
#include <model.h>
#include <optimizer.h>

#include <cifar_dataloader.h>

using LayerHandle = std::shared_ptr<ml::train::Layer>;
using ModelHandle = std::unique_ptr<ml::train::Model>;
using UserDataType = std::unique_ptr<nntrainer::util::DataLoader>;

/** cache loss values post training for test */
float training_loss = 0.0;
float validation_loss = 0.0;

/**
* @brief Data size for each category
* @brief make "key=value" from key and value
*
* @tparam T type of a value
* @param key key
* @param value value
* @return std::string with "key=value"
*/
const unsigned int num_class = 100;
template <typename T>
static std::string withKey(const std::string &key, const T &value) {
std::stringstream ss;
ss << key << "=" << value;
return ss.str();
}

const unsigned int num_train = 100;
template <typename T>
static std::string withKey(const std::string &key,
std::initializer_list<T> value) {
if (std::empty(value)) {
throw std::invalid_argument("empty data cannot be converted");
}

const unsigned int num_val = 20;
std::stringstream ss;
ss << key << "=";

const unsigned int batch_size = 128;
auto iter = value.begin();
for (; iter != value.end() - 1; ++iter) {
ss << *iter << ',';
}
ss << *iter;

const unsigned int feature_size = 3072;
return ss.str();
}

unsigned int train_count = 0;
unsigned int val_count = 0;
std::vector<LayerHandle> createGraph() {
using ml::train::createLayer;

int width = 32;
std::vector<LayerHandle> layers;
layers.push_back(createLayer(
"input", {withKey("name", "input0"), withKey("input_shape", "3:32:32")}));

int height = 32;
layers.push_back(createLayer(
"conv2d", {withKey("name", "conv0"), withKey("filters", 64),
withKey("kernel_size", {3, 3}), withKey("stride", {1, 1}),
withKey("padding", "same"), withKey("bias_initializer", "zeros"),
withKey("weight_initializer", "xavier_uniform")}));

int channel = 3;
layers.push_back(createLayer(
"batch_normalization",
{withKey("name", "first_bn_relu"), withKey("activation", "relu"),
withKey("momentum", "0.9"), withKey("epsilon", "0.00001")}));

unsigned int seed;
layers.push_back(createLayer(
"pooling2d", {withKey("name", "last_p1"), withKey("pooling", "average"),
withKey("pool_size", {4, 4}), withKey("stride", "4,4")}));

std::string resource;
layers.push_back(createLayer("flatten", {withKey("name", "last_f1")}));
layers.push_back(
createLayer("fully_connected",
{withKey("unit", 100), withKey("activation", "softmax")}));

float training_loss = 0.0;
float validation_loss = 0.0;
float last_batch_loss = 0.0;
return layers;
}

using ModelHandle = std::unique_ptr<ml::train::Model>;
using UserDataType = std::unique_ptr<nntrainer::util::DataLoader>;
ModelHandle createModel() {
ModelHandle model = ml::train::createModel(ml::train::ModelType::NEURAL_NET,
{withKey("loss", "mse")});
for (auto &layer : createGraph()) {
model->addLayer(layer);
}
return model;
}

int trainData_cb(float **input, float **label, bool *last, void *user_data) {
auto data = reinterpret_cast<nntrainer::util::DataLoader *>(user_data);
Expand All @@ -69,120 +119,105 @@ int validData_cb(float **input, float **label, bool *last, void *user_data) {
return 0;
}

void createAndRun(unsigned int epochs, unsigned int batch_size,
UserDataType &train_user_data,
UserDataType &valid_user_data) {
// setup model
ModelHandle model = createModel();
model->setProperty({withKey("batch_size", batch_size),
withKey("epochs", epochs),
withKey("save_path", "mixed_model.bin")});

#ifdef ENABLE_FP16
model->setProperty({"model_tensor_type=FP16-FP16"});
model->setProperty({"loss_scale=17768"});
#endif

auto optimizer = ml::train::createOptimizer("adam", {"learning_rate=0.001"});
model->setOptimizer(std::move(optimizer));

int status = model->compile();
if (status) {
throw std::invalid_argument("model compilation failed!");
}

status = model->initialize();
if (status) {
throw std::invalid_argument("model initialization failed!");
}

auto dataset_train = ml::train::createDataset(
ml::train::DatasetType::GENERATOR, trainData_cb, train_user_data.get());
auto dataset_valid = ml::train::createDataset(
ml::train::DatasetType::GENERATOR, validData_cb, valid_user_data.get());

model->setDataset(ml::train::DatasetModeType::MODE_TRAIN,
std::move(dataset_train));
model->setDataset(ml::train::DatasetModeType::MODE_VALID,
std::move(dataset_valid));
model->train();
}

std::array<UserDataType, 2>
createRealDataGenerator(const std::string &directory, unsigned int batch_size,
createFakeDataGenerator(unsigned int batch_size,
unsigned int simulated_data_size,
unsigned int data_split) {

UserDataType train_data(new nntrainer::util::Cifar100DataLoader(
directory + "/alex_trainingSet.dat", batch_size, 1));
UserDataType valid_data(new nntrainer::util::Cifar100DataLoader(
directory + "/alex_valSet.dat", batch_size, 1));
UserDataType train_data(new nntrainer::util::RandomDataLoader(
{{batch_size, 3, 32, 32}}, {{batch_size, 1, 1, 100}},
simulated_data_size / data_split));
UserDataType valid_data(new nntrainer::util::RandomDataLoader(
{{batch_size, 3, 32, 32}}, {{batch_size, 1, 1, 100}},
simulated_data_size / data_split));

return {std::move(train_data), std::move(valid_data)};
}

int main(int argc, char *argv[]) {

if (argc < 3) {
std::cout << "./nntrainer_alex alex.ini resource\n";
exit(-1);
if (argc < 4) {
std::cerr << "usage: ./main [batchsize] [data_split] [epoch] \n"
<< "data size is assumed 512 for both train and validation\n";
return EXIT_FAILURE;
}

seed = time(NULL);
srand(seed);
auto start = std::chrono::system_clock::now();
std::time_t start_time = std::chrono::system_clock::to_time_t(start);
std::cout << "started computation at " << std::ctime(&start_time)
<< std::endl;

unsigned int batch_size = std::stoul(argv[1]);
unsigned int data_split = std::stoul(argv[2]);
unsigned int epoch = std::stoul(argv[3]);

const std::vector<std::string> args(argv + 1, argv + argc);
std::string config = args[0];
resource = args[1];
std::cout << "batch_size: " << batch_size << " data_split: " << data_split
<< " epoch: " << epoch << std::endl;

std::array<UserDataType, 2> user_datas;

try {
user_datas = createRealDataGenerator(resource, batch_size, 1);
user_datas = createFakeDataGenerator(batch_size, 512, data_split);
} catch (const std::exception &e) {
std::cerr << "uncaught error while creating data generator! details: "
<< e.what() << std::endl;
return 1;
return EXIT_FAILURE;
}

auto &[train_user_data, valid_user_data] = user_datas;

std::unique_ptr<ml::train::Dataset> dataset_train;
try {
dataset_train = ml::train::createDataset(
ml::train::DatasetType::GENERATOR, trainData_cb, train_user_data.get());
createAndRun(epoch, batch_size, train_user_data, valid_user_data);
} catch (const std::exception &e) {
std::cerr << "Error during create train dataset: " << e.what() << std::endl;
return 1;
std::cerr << "uncaught error while running! details: " << e.what()
<< std::endl;
return EXIT_FAILURE;
}
auto end = std::chrono::system_clock::now();

std::unique_ptr<ml::train::Dataset> dataset_valid;
try {
dataset_valid = ml::train::createDataset(
ml::train::DatasetType::GENERATOR, validData_cb, valid_user_data.get());
} catch (const std::exception &e) {
std::cerr << "Error during create valid dataset: " << e.what() << std::endl;
return 1;
}

/**
* @brief Neural Network Create & Initialization
*/

ModelHandle model;
try {
model = ml::train::createModel(ml::train::ModelType::NEURAL_NET);
} catch (const std::exception &e) {
std::cerr << "Error during create model: " << e.what() << std::endl;
return 1;
}

try {
model->load(config, ml::train::ModelFormat::MODEL_FORMAT_INI);
} catch (const std::exception &e) {
std::cerr << "Error during loadFromConfig: " << e.what() << std::endl;
return 1;
}

try {
model->compile();
} catch (const std::exception &e) {
std::cerr << "Error during compile: " << e.what() << std::endl;
return 1;
}
std::chrono::duration<double> elapsed_seconds = end - start;
std::time_t end_time = std::chrono::system_clock::to_time_t(end);

try {
model->initialize();
} catch (const std::exception &e) {
std::cerr << "Error during ininitialize: " << e.what() << std::endl;
return 1;
}

try {
model->setDataset(ml::train::DatasetModeType::MODE_TRAIN,
std::move(dataset_train));
} catch (const std::exception &e) {
std::cerr << "Error during set train dataset: " << e.what() << std::endl;
return 1;
}

try {
model->setDataset(ml::train::DatasetModeType::MODE_VALID,
std::move(dataset_valid));
} catch (const std::exception &e) {
std::cerr << "Error during set valid dataset: " << e.what() << std::endl;
return 1;
}

try {
model->train();
training_loss = model->getTrainingLoss();
validation_loss = model->getValidationLoss();
last_batch_loss = model->getLoss();
} catch (const std::exception &e) {
std::cerr << "Error during train: " << e.what() << std::endl;
return 1;
}
std::cout << "finished computation at " << std::ctime(&end_time)
<< "elapsed time: " << elapsed_seconds.count() << "s\n";

return 0;
int status = EXIT_SUCCESS;
return status;
}
10 changes: 5 additions & 5 deletions Applications/MixedPrecision/jni/meson.build
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
build_root = meson.build_root()
res_path = meson.current_source_dir() / '..' / 'res'

nntr_alex_resdir = nntr_app_resdir / 'AlexNet'
run_command('cp', '-lr', res_path, nntr_alex_resdir)
nntr_mixed_resdir = nntr_app_resdir / 'MixedPrecision'
run_command('cp', '-lr', res_path, nntr_mixed_resdir)

alex_sources = [
mixed_sources = [
'main.cpp',
cifar_path / 'cifar_dataloader.cpp'
]

executable('nntrainer_alexnet',
alex_sources,
executable('nntrainer_mixed_example',
mixed_sources,
dependencies: [iniparser_dep, nntrainer_dep, nntrainer_ccapi_dep, app_utils_dep],
include_directories: [include_directories('.'), cifar_include_dir],
install: get_option('install-app'),
Expand Down
1 change: 1 addition & 0 deletions Applications/meson.build
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,3 +26,4 @@ if get_option('enable-tflite-backbone')
subdir('SimpleShot')
endif
subdir('PicoGPT/jni')
subdir('MixedPrecision/jni')

0 comments on commit 15fe748

Please sign in to comment.