From 15fe74858044b31ef580f8815513e5437312805c Mon Sep 17 00:00:00 2001 From: Donghak PARK Date: Tue, 2 Jul 2024 17:20:08 +0900 Subject: [PATCH] [Application] Add Mixed Precision build file and update application main Add Mixed Precision Build file and Update Application's main this Application will run only at fp16 enable **Self evaluation:** 1. Build test: [X]Passed [ ]Failed [ ]Skipped 2. Run test: [X]Passed [ ]Failed [ ]Skipped Signed-off-by: Donghak PARK --- Applications/MixedPrecision/jni/main.cpp | 269 +++++++++++--------- Applications/MixedPrecision/jni/meson.build | 10 +- Applications/meson.build | 1 + 3 files changed, 158 insertions(+), 122 deletions(-) diff --git a/Applications/MixedPrecision/jni/main.cpp b/Applications/MixedPrecision/jni/main.cpp index 91a6926387..a5a8b2f850 100644 --- a/Applications/MixedPrecision/jni/main.cpp +++ b/Applications/MixedPrecision/jni/main.cpp @@ -1,59 +1,109 @@ // SPDX-License-Identifier: Apache-2.0 /** - * Copyright (C) 2020 Jijoong Moon + * Copyright (C) 2024 Donghak Park * * @file main.cpp - * @date 25 Jul 2022 + * @date 01 Jul 2024 + * @brief Resnet Mixed Precision example * @see https://github.com/nnstreamer/nntrainer - * @author Jijoong Moon * @author Donghak Park - * @bug No known bugs except for NYI items - * @brief This is AlexNet Example with CIFAR100 - * + * @bug No known bugs except for NYI items */ -#include -#include +#include +#include +#include #include +#include #include -#include #include -#include +#include #include +#include + +#include + +using LayerHandle = std::shared_ptr; +using ModelHandle = std::unique_ptr; +using UserDataType = std::unique_ptr; + +/** cache loss values post training for test */ +float training_loss = 0.0; +float validation_loss = 0.0; /** - * @brief Data size for each category + * @brief make "key=value" from key and value + * + * @tparam T type of a value + * @param key key + * @param value value + * @return std::string with "key=value" */ -const unsigned int num_class = 100; +template +static std::string withKey(const std::string &key, const T &value) { + std::stringstream ss; + ss << key << "=" << value; + return ss.str(); +} -const unsigned int num_train = 100; +template +static std::string withKey(const std::string &key, + std::initializer_list value) { + if (std::empty(value)) { + throw std::invalid_argument("empty data cannot be converted"); + } -const unsigned int num_val = 20; + std::stringstream ss; + ss << key << "="; -const unsigned int batch_size = 128; + auto iter = value.begin(); + for (; iter != value.end() - 1; ++iter) { + ss << *iter << ','; + } + ss << *iter; -const unsigned int feature_size = 3072; + return ss.str(); +} -unsigned int train_count = 0; -unsigned int val_count = 0; +std::vector createGraph() { + using ml::train::createLayer; -int width = 32; + std::vector layers; + layers.push_back(createLayer( + "input", {withKey("name", "input0"), withKey("input_shape", "3:32:32")})); -int height = 32; + layers.push_back(createLayer( + "conv2d", {withKey("name", "conv0"), withKey("filters", 64), + withKey("kernel_size", {3, 3}), withKey("stride", {1, 1}), + withKey("padding", "same"), withKey("bias_initializer", "zeros"), + withKey("weight_initializer", "xavier_uniform")})); -int channel = 3; + layers.push_back(createLayer( + "batch_normalization", + {withKey("name", "first_bn_relu"), withKey("activation", "relu"), + withKey("momentum", "0.9"), withKey("epsilon", "0.00001")})); -unsigned int seed; + layers.push_back(createLayer( + "pooling2d", {withKey("name", "last_p1"), withKey("pooling", "average"), + withKey("pool_size", {4, 4}), withKey("stride", "4,4")})); -std::string resource; + layers.push_back(createLayer("flatten", {withKey("name", "last_f1")})); + layers.push_back( + createLayer("fully_connected", + {withKey("unit", 100), withKey("activation", "softmax")})); -float training_loss = 0.0; -float validation_loss = 0.0; -float last_batch_loss = 0.0; + return layers; +} -using ModelHandle = std::unique_ptr; -using UserDataType = std::unique_ptr; +ModelHandle createModel() { + ModelHandle model = ml::train::createModel(ml::train::ModelType::NEURAL_NET, + {withKey("loss", "mse")}); + for (auto &layer : createGraph()) { + model->addLayer(layer); + } + return model; +} int trainData_cb(float **input, float **label, bool *last, void *user_data) { auto data = reinterpret_cast(user_data); @@ -69,120 +119,105 @@ int validData_cb(float **input, float **label, bool *last, void *user_data) { return 0; } +void createAndRun(unsigned int epochs, unsigned int batch_size, + UserDataType &train_user_data, + UserDataType &valid_user_data) { + // setup model + ModelHandle model = createModel(); + model->setProperty({withKey("batch_size", batch_size), + withKey("epochs", epochs), + withKey("save_path", "mixed_model.bin")}); + +#ifdef ENABLE_FP16 + model->setProperty({"model_tensor_type=FP16-FP16"}); + model->setProperty({"loss_scale=17768"}); +#endif + + auto optimizer = ml::train::createOptimizer("adam", {"learning_rate=0.001"}); + model->setOptimizer(std::move(optimizer)); + + int status = model->compile(); + if (status) { + throw std::invalid_argument("model compilation failed!"); + } + + status = model->initialize(); + if (status) { + throw std::invalid_argument("model initialization failed!"); + } + + auto dataset_train = ml::train::createDataset( + ml::train::DatasetType::GENERATOR, trainData_cb, train_user_data.get()); + auto dataset_valid = ml::train::createDataset( + ml::train::DatasetType::GENERATOR, validData_cb, valid_user_data.get()); + + model->setDataset(ml::train::DatasetModeType::MODE_TRAIN, + std::move(dataset_train)); + model->setDataset(ml::train::DatasetModeType::MODE_VALID, + std::move(dataset_valid)); + model->train(); +} + std::array -createRealDataGenerator(const std::string &directory, unsigned int batch_size, +createFakeDataGenerator(unsigned int batch_size, + unsigned int simulated_data_size, unsigned int data_split) { - - UserDataType train_data(new nntrainer::util::Cifar100DataLoader( - directory + "/alex_trainingSet.dat", batch_size, 1)); - UserDataType valid_data(new nntrainer::util::Cifar100DataLoader( - directory + "/alex_valSet.dat", batch_size, 1)); + UserDataType train_data(new nntrainer::util::RandomDataLoader( + {{batch_size, 3, 32, 32}}, {{batch_size, 1, 1, 100}}, + simulated_data_size / data_split)); + UserDataType valid_data(new nntrainer::util::RandomDataLoader( + {{batch_size, 3, 32, 32}}, {{batch_size, 1, 1, 100}}, + simulated_data_size / data_split)); return {std::move(train_data), std::move(valid_data)}; } int main(int argc, char *argv[]) { - - if (argc < 3) { - std::cout << "./nntrainer_alex alex.ini resource\n"; - exit(-1); + if (argc < 4) { + std::cerr << "usage: ./main [batchsize] [data_split] [epoch] \n" + << "data size is assumed 512 for both train and validation\n"; + return EXIT_FAILURE; } - seed = time(NULL); - srand(seed); + auto start = std::chrono::system_clock::now(); + std::time_t start_time = std::chrono::system_clock::to_time_t(start); + std::cout << "started computation at " << std::ctime(&start_time) + << std::endl; + + unsigned int batch_size = std::stoul(argv[1]); + unsigned int data_split = std::stoul(argv[2]); + unsigned int epoch = std::stoul(argv[3]); - const std::vector args(argv + 1, argv + argc); - std::string config = args[0]; - resource = args[1]; + std::cout << "batch_size: " << batch_size << " data_split: " << data_split + << " epoch: " << epoch << std::endl; std::array user_datas; try { - user_datas = createRealDataGenerator(resource, batch_size, 1); + user_datas = createFakeDataGenerator(batch_size, 512, data_split); } catch (const std::exception &e) { std::cerr << "uncaught error while creating data generator! details: " << e.what() << std::endl; - return 1; + return EXIT_FAILURE; } auto &[train_user_data, valid_user_data] = user_datas; - std::unique_ptr dataset_train; try { - dataset_train = ml::train::createDataset( - ml::train::DatasetType::GENERATOR, trainData_cb, train_user_data.get()); + createAndRun(epoch, batch_size, train_user_data, valid_user_data); } catch (const std::exception &e) { - std::cerr << "Error during create train dataset: " << e.what() << std::endl; - return 1; + std::cerr << "uncaught error while running! details: " << e.what() + << std::endl; + return EXIT_FAILURE; } + auto end = std::chrono::system_clock::now(); - std::unique_ptr dataset_valid; - try { - dataset_valid = ml::train::createDataset( - ml::train::DatasetType::GENERATOR, validData_cb, valid_user_data.get()); - } catch (const std::exception &e) { - std::cerr << "Error during create valid dataset: " << e.what() << std::endl; - return 1; - } - - /** - * @brief Neural Network Create & Initialization - */ - - ModelHandle model; - try { - model = ml::train::createModel(ml::train::ModelType::NEURAL_NET); - } catch (const std::exception &e) { - std::cerr << "Error during create model: " << e.what() << std::endl; - return 1; - } - - try { - model->load(config, ml::train::ModelFormat::MODEL_FORMAT_INI); - } catch (const std::exception &e) { - std::cerr << "Error during loadFromConfig: " << e.what() << std::endl; - return 1; - } - - try { - model->compile(); - } catch (const std::exception &e) { - std::cerr << "Error during compile: " << e.what() << std::endl; - return 1; - } + std::chrono::duration elapsed_seconds = end - start; + std::time_t end_time = std::chrono::system_clock::to_time_t(end); - try { - model->initialize(); - } catch (const std::exception &e) { - std::cerr << "Error during ininitialize: " << e.what() << std::endl; - return 1; - } - - try { - model->setDataset(ml::train::DatasetModeType::MODE_TRAIN, - std::move(dataset_train)); - } catch (const std::exception &e) { - std::cerr << "Error during set train dataset: " << e.what() << std::endl; - return 1; - } - - try { - model->setDataset(ml::train::DatasetModeType::MODE_VALID, - std::move(dataset_valid)); - } catch (const std::exception &e) { - std::cerr << "Error during set valid dataset: " << e.what() << std::endl; - return 1; - } - - try { - model->train(); - training_loss = model->getTrainingLoss(); - validation_loss = model->getValidationLoss(); - last_batch_loss = model->getLoss(); - } catch (const std::exception &e) { - std::cerr << "Error during train: " << e.what() << std::endl; - return 1; - } + std::cout << "finished computation at " << std::ctime(&end_time) + << "elapsed time: " << elapsed_seconds.count() << "s\n"; - return 0; + int status = EXIT_SUCCESS; + return status; } diff --git a/Applications/MixedPrecision/jni/meson.build b/Applications/MixedPrecision/jni/meson.build index cfc83e2c09..88e3e7020f 100644 --- a/Applications/MixedPrecision/jni/meson.build +++ b/Applications/MixedPrecision/jni/meson.build @@ -1,16 +1,16 @@ build_root = meson.build_root() res_path = meson.current_source_dir() / '..' / 'res' -nntr_alex_resdir = nntr_app_resdir / 'AlexNet' -run_command('cp', '-lr', res_path, nntr_alex_resdir) +nntr_mixed_resdir = nntr_app_resdir / 'MixedPrecision' +run_command('cp', '-lr', res_path, nntr_mixed_resdir) -alex_sources = [ +mixed_sources = [ 'main.cpp', cifar_path / 'cifar_dataloader.cpp' ] -executable('nntrainer_alexnet', - alex_sources, +executable('nntrainer_mixed_example', + mixed_sources, dependencies: [iniparser_dep, nntrainer_dep, nntrainer_ccapi_dep, app_utils_dep], include_directories: [include_directories('.'), cifar_include_dir], install: get_option('install-app'), diff --git a/Applications/meson.build b/Applications/meson.build index 7c8ef63cd4..074bfae21e 100644 --- a/Applications/meson.build +++ b/Applications/meson.build @@ -26,3 +26,4 @@ if get_option('enable-tflite-backbone') subdir('SimpleShot') endif subdir('PicoGPT/jni') +subdir('MixedPrecision/jni')