From f9d1d38def7a5af023d78245e75e84c9b4288ea8 Mon Sep 17 00:00:00 2001 From: Douman Date: Fri, 30 Nov 2018 19:12:15 +0300 Subject: [PATCH 1/2] Correct conversions errors from u64 to size_t(u32 on msvc32) --- .gitignore | 4 +++- src/core/analysis/rnn_scorer_gbeam.cc | 4 ++-- src/core/analysis/score_processor.cc | 2 +- src/rnn/mikolov_rnn.cc | 22 +++++++++++++--------- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index b3f097c6..e9d4f023 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,6 @@ VERSION __pycache__/ # Mac OS -.DS_Store \ No newline at end of file +.DS_Store + +build/ \ No newline at end of file diff --git a/src/core/analysis/rnn_scorer_gbeam.cc b/src/core/analysis/rnn_scorer_gbeam.cc index f6051e9c..7e57dcf9 100644 --- a/src/core/analysis/rnn_scorer_gbeam.cc +++ b/src/core/analysis/rnn_scorer_gbeam.cc @@ -319,9 +319,9 @@ Status RnnScorerGbeamFactory::make(StringPiece rnnModelPath, state_->resolver.build(dic, state_->config, state_->rnnReader.words())); auto& h = state_->rnnReader.header(); state_->embeddings = {state_->rnnReader.embeddings(), h.layerSize, - h.vocabSize}; + static_cast(h.vocabSize)}; state_->nceEmbeddings = {state_->rnnReader.nceEmbeddings(), h.layerSize, - h.vocabSize}; + static_cast(h.vocabSize)}; if (h.layerSize > 64 * 1024) { return JPPS_NOT_IMPLEMENTED << "we don't support embed sizes > 64k"; } diff --git a/src/core/analysis/score_processor.cc b/src/core/analysis/score_processor.cc index 15ad2a04..8e5403b8 100644 --- a/src/core/analysis/score_processor.cc +++ b/src/core/analysis/score_processor.cc @@ -195,7 +195,7 @@ util::ArraySlice processBeamCandidates( candidates = util::MutableArraySlice{candidates, 0, sz}; } std::sort(candidates.begin(), candidates.end(), comp); - auto size = std::min(maxBeam, candidates.size()); + auto size = std::min(maxBeam, candidates.size()); return util::ArraySlice{candidates, 0, size}; } diff --git a/src/rnn/mikolov_rnn.cc b/src/rnn/mikolov_rnn.cc index c192e166..2aae8d79 100644 --- a/src/rnn/mikolov_rnn.cc +++ b/src/rnn/mikolov_rnn.cc @@ -177,29 +177,33 @@ Status MikolovModelReader::parse() { auto maxBlock = std::max({hdr.layerSize * hdr.vocabSize, hdr.maxentSize, hdr.layerSize * hdr.layerSize}); // 3 comes from rounding to next value + sizeof(float) - auto pageSize = 1ULL << (static_cast(std::log2(maxBlock)) + 3); + auto pageSize = static_cast(1) << (static_cast(std::log2(maxBlock)) + 3); data_->memmgr.initialize(pageSize); data_->alloc = data_->memmgr.value().core(); auto& alloc = data_->alloc; + + const auto embedding_size = static_cast(hdr.layerSize) * static_cast(hdr.vocabSize); + const auto matrix_size = static_cast(hdr.layerSize) * static_cast(hdr.layerSize); + data_->embeddingData = - alloc->allocateBuf(hdr.layerSize * hdr.vocabSize, 64); + alloc->allocateBuf(embedding_size, 64); data_->nceEmbeddingData = - alloc->allocateBuf(hdr.layerSize * hdr.vocabSize, 64); + alloc->allocateBuf(embedding_size, 64); data_->matrixData = - alloc->allocateBuf(hdr.layerSize * hdr.layerSize, 64); - data_->maxentWeightData = alloc->allocateBuf(hdr.maxentSize, 64); + alloc->allocateBuf(matrix_size, 64); + data_->maxentWeightData = alloc->allocateBuf(static_cast(hdr.maxentSize), 64); JPP_RIE_MSG(copyArray(contents, data_->embeddingData, - hdr.layerSize * hdr.vocabSize, &start), + embedding_size, &start), "embeds"); JPP_RIE_MSG(copyArray(contents, data_->nceEmbeddingData, - hdr.layerSize * hdr.vocabSize, &start), + embedding_size, &start), "nce embeds"); JPP_RIE_MSG(copyArray(contents, data_->matrixData, - hdr.layerSize * hdr.layerSize, &start), + matrix_size, &start), "matrix"); JPP_RIE_MSG( - copyArray(contents, data_->maxentWeightData, hdr.maxentSize, &start), + copyArray(contents, data_->maxentWeightData, static_cast(hdr.maxentSize), &start), "maxent weights"); if (start != contents.size()) { return Status::InvalidState() << "did not read rnn model file fully"; From 091287c1f4b61a8926fda7006dbbb745256c30e4 Mon Sep 17 00:00:00 2001 From: Douman Date: Sat, 1 Dec 2018 09:09:09 +0300 Subject: [PATCH 2/2] Apply comments --- src/rnn/mikolov_rnn.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/rnn/mikolov_rnn.cc b/src/rnn/mikolov_rnn.cc index 2aae8d79..2bdebf58 100644 --- a/src/rnn/mikolov_rnn.cc +++ b/src/rnn/mikolov_rnn.cc @@ -182,25 +182,25 @@ Status MikolovModelReader::parse() { data_->alloc = data_->memmgr.value().core(); auto& alloc = data_->alloc; - const auto embedding_size = static_cast(hdr.layerSize) * static_cast(hdr.vocabSize); - const auto matrix_size = static_cast(hdr.layerSize) * static_cast(hdr.layerSize); + const auto embedding_matrix_size = static_cast(hdr.layerSize) * static_cast(hdr.vocabSize); + const auto transition_matrix_size = static_cast(hdr.layerSize) * static_cast(hdr.layerSize); data_->embeddingData = - alloc->allocateBuf(embedding_size, 64); + alloc->allocateBuf(embedding_matrix_size, 64); data_->nceEmbeddingData = - alloc->allocateBuf(embedding_size, 64); + alloc->allocateBuf(embedding_matrix_size, 64); data_->matrixData = - alloc->allocateBuf(matrix_size, 64); + alloc->allocateBuf(transition_matrix_size, 64); data_->maxentWeightData = alloc->allocateBuf(static_cast(hdr.maxentSize), 64); JPP_RIE_MSG(copyArray(contents, data_->embeddingData, - embedding_size, &start), + embedding_matrix_size, &start), "embeds"); JPP_RIE_MSG(copyArray(contents, data_->nceEmbeddingData, - embedding_size, &start), + embedding_matrix_size, &start), "nce embeds"); JPP_RIE_MSG(copyArray(contents, data_->matrixData, - matrix_size, &start), + transition_matrix_size, &start), "matrix"); JPP_RIE_MSG( copyArray(contents, data_->maxentWeightData, static_cast(hdr.maxentSize), &start), @@ -249,4 +249,4 @@ StringPiece MikolovRnn::maxentWeightsAsStringpiece() const { } // namespace mikolov } // namespace rnn -} // namespace jumanpp \ No newline at end of file +} // namespace jumanpp