Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[SharedCache] Use m_exportInfos as an export list cache #6197

Open
wants to merge 6 commits into
base: dev
Choose a base branch
from
Open

[SharedCache] Use m_exportInfos as an export list cache #6197

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b7ccf06
[SharedCache] Use `m_exportInfos` as an export list cache
Nov 25, 2024
64174ee
[SharedCache] Add parameter to `SharedCache::InitializeHeader` to che…
Nov 26, 2024
6edb250
[SharedCache] Improve the types for `m_exportInfos`
Nov 26, 2024
9e7d6e2
[SharedCache] Only setup undo actions and bulk modify in `SharedCache…
Nov 26, 2024
202a792
[SharedCache] Make `m_exportInfos` map's values a `shared_ptr`
Nov 26, 2024
c83ec00
[SharedCache] Remove commenting out of line from debugging
Nov 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
185 changes: 111 additions & 74 deletions view/sharedcache/core/SharedCache.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
#include "SharedCache.h"
#include "ObjC.h"
#include <filesystem>
#include <unordered_map>
#include <utility>
#include <fcntl.h>
#include <memory>
Expand Down Expand Up @@ -71,7 +72,7 @@ struct ViewStateCacheStore {

std::string m_baseFilePath;

std::unordered_map<uint64_t, std::vector<std::pair<uint64_t, std::pair<BNSymbolType, std::string>>>> m_exportInfos;
std::unordered_map<uint64_t, std::shared_ptr<std::unordered_map<uint64_t, Ref<Symbol>>>> m_exportInfos;
std::unordered_map<uint64_t, std::vector<std::pair<uint64_t, std::pair<BNSymbolType, std::string>>>> m_symbolInfos;
};

Expand Down Expand Up @@ -2634,48 +2635,51 @@ void SharedCache::InitializeHeader(

if (header.exportTriePresent && header.linkeditPresent && vm->AddressIsMapped(header.linkeditSegment.vmaddr))
{
auto symbols = SharedCache::ParseExportTrie(vm->MappingAtAddress(header.linkeditSegment.vmaddr).first.fileAccessor->lock(), header);
std::vector<std::pair<uint64_t, std::pair<BNSymbolType, std::string>>> exportMapping;
for (const auto& symbol : symbols)
auto symbols = GetExportListForHeader(header, [&]() {
return vm->MappingAtAddress(header.linkeditSegment.vmaddr).first.fileAccessor->lock();
});
if (symbols)
{
exportMapping.push_back({symbol->GetAddress(), {symbol->GetType(), symbol->GetRawName()}});
if (typeLib)
for (const auto& [symbolAddress, symbol] : *symbols)
{
auto type = m_dscView->ImportTypeLibraryObject(typeLib, {symbol->GetFullName()});

if (type)
if (typeLib)
{
view->DefineAutoSymbolAndVariableOrFunction(view->GetDefaultPlatform(), symbol, type);
}
else
view->DefineAutoSymbol(symbol);
auto type = m_dscView->ImportTypeLibraryObject(typeLib, symbol->GetRawName());

if (view->GetAnalysisFunction(view->GetDefaultPlatform(), symbol->GetAddress()))
{
auto func = view->GetAnalysisFunction(view->GetDefaultPlatform(), symbol->GetAddress());
if (symbol->GetFullName() == "_objc_msgSend")
if (type)
{
func->SetHasVariableArguments(false);
view->DefineAutoSymbolAndVariableOrFunction(view->GetDefaultPlatform(), symbol, type);
}
else if (symbol->GetFullName().find("_objc_retain_x") != std::string::npos || symbol->GetFullName().find("_objc_release_x") != std::string::npos)
else
view->DefineAutoSymbol(symbol);

if (view->GetAnalysisFunction(view->GetDefaultPlatform(), symbolAddress))
{
auto x = symbol->GetFullName().rfind("x");
auto num = symbol->GetFullName().substr(x + 1);
auto func = view->GetAnalysisFunction(view->GetDefaultPlatform(), symbolAddress);
auto name = symbol->GetFullName();
if (name == "_objc_msgSend")
{
func->SetHasVariableArguments(false);
}
else if (name.find("_objc_retain_x") != std::string::npos || name.find("_objc_release_x") != std::string::npos)
{
auto x = name.rfind("x");
auto num = name.substr(x + 1);

std::vector<BinaryNinja::FunctionParameter> callTypeParams;
auto cc = m_dscView->GetDefaultArchitecture()->GetCallingConventionByName("apple-arm64-objc-fast-arc-" + num);
std::vector<BinaryNinja::FunctionParameter> callTypeParams;
auto cc = m_dscView->GetDefaultArchitecture()->GetCallingConventionByName("apple-arm64-objc-fast-arc-" + num);

callTypeParams.push_back({"obj", m_dscView->GetTypeByName({ "id" }), true, BinaryNinja::Variable()});
callTypeParams.push_back({"obj", m_dscView->GetTypeByName({ "id" }), true, BinaryNinja::Variable()});

auto funcType = BinaryNinja::Type::FunctionType(m_dscView->GetTypeByName({ "id" }), cc, callTypeParams);
func->SetUserType(funcType);
auto funcType = BinaryNinja::Type::FunctionType(m_dscView->GetTypeByName({ "id" }), cc, callTypeParams);
func->SetUserType(funcType);
}
}
}
else
view->DefineAutoSymbol(symbol);
}
else
view->DefineAutoSymbol(symbol);
}
m_exportInfos[header.textBase] = exportMapping;
}
view->EndBulkModifySymbols();

Expand Down Expand Up @@ -2779,6 +2783,39 @@ std::vector<Ref<Symbol>> SharedCache::ParseExportTrie(std::shared_ptr<MMappedFil
return symbols;
}


std::shared_ptr<std::unordered_map<uint64_t, Ref<Symbol>>> SharedCache::GetExportListForHeader(SharedCacheMachOHeader header, std::function<std::shared_ptr<MMappedFileAccessor>()> provideLinkeditFile, bool* didModifyExportList)
{
if (auto it = m_exportInfos.find(header.textBase); it != m_exportInfos.end())
{
if (didModifyExportList)
*didModifyExportList = false;
return it->second;
}
else
{
std::shared_ptr<MMappedFileAccessor> linkeditFile = provideLinkeditFile();
if (!linkeditFile)
{
if (didModifyExportList)
*didModifyExportList = false;
return nullptr;
}

auto exportList = SharedCache::ParseExportTrie(linkeditFile, header);
auto exportMapping = std::make_shared<std::unordered_map<uint64_t, Ref<Symbol>>>(exportList.size());
for (const auto& sym : exportList)
{
exportMapping->insert_or_assign(sym->GetAddress(), sym);
}
m_exportInfos.emplace(header.textBase, exportMapping);
if (didModifyExportList)
*didModifyExportList = true;
return m_exportInfos[header.textBase];
}
}


std::vector<std::string> SharedCache::GetAvailableImages()
{
std::vector<std::string> installNames;
Expand All @@ -2794,30 +2831,33 @@ std::vector<std::pair<std::string, Ref<Symbol>>> SharedCache::LoadAllSymbolsAndW
{
std::unique_lock<std::mutex> initialLoadBlock(viewSpecificMutexes[m_dscView->GetFile()->GetSessionId()].viewOperationsThatInfluenceMetadataMutex);

bool doSave = false;
std::vector<std::pair<std::string, Ref<Symbol>>> symbols;
for (const auto& img : m_images)
{
auto header = HeaderForAddress(img.headerLocation);
std::shared_ptr<MMappedFileAccessor> mapping;
try {
mapping = MMappedFileAccessor::Open(m_dscView, m_dscView->GetFile()->GetSessionId(), header->exportTriePath)->lock();
}
catch (...)
{
m_logger->LogWarn("Serious Error: Failed to open export trie %s for %s", header->exportTriePath.c_str(), header->installName.c_str());
auto exportList = GetExportListForHeader(*header, [&]() {
try {
auto mapping = MMappedFileAccessor::Open(m_dscView, m_dscView->GetFile()->GetSessionId(), header->exportTriePath)->lock();
return mapping;
}
catch (...)
{
m_logger->LogWarn("Serious Error: Failed to open export trie %s for %s", header->exportTriePath.c_str(), header->installName.c_str());
return std::shared_ptr<MMappedFileAccessor>(nullptr);
}
}, &doSave);
if (!exportList)
continue;
}
auto exportList = SharedCache::ParseExportTrie(mapping, *header);
std::vector<std::pair<uint64_t, std::pair<BNSymbolType, std::string>>> exportMapping;
for (const auto& sym : exportList)
for (const auto& [_, symbol] : *exportList)
{
exportMapping.push_back({sym->GetAddress(), {sym->GetType(), sym->GetRawName()}});
symbols.push_back({img.installName, sym});
symbols.push_back({img.installName, symbol});
}
m_exportInfos[header->textBase] = exportMapping;
}

SaveToDSCView();
// Only save to DSC view if a header was actually loaded
if (doSave)
SaveToDSCView();

return symbols;
}
Expand Down Expand Up @@ -2874,17 +2914,6 @@ void SharedCache::FindSymbolAtAddrAndApplyToAddr(uint64_t symbolLocation, uint64
auto header = HeaderForAddress(symbolLocation);
if (header)
{
std::shared_ptr<MMappedFileAccessor> mapping;
try {
mapping = MMappedFileAccessor::Open(m_dscView, m_dscView->GetFile()->GetSessionId(), header->exportTriePath)->lock();
}
catch (...)
{
m_logger->LogWarn("Serious Error: Failed to open export trie for %s", header->installName.c_str());
return;
}
auto exportList = SharedCache::ParseExportTrie(mapping, *header);
std::vector<std::pair<uint64_t, std::pair<BNSymbolType, std::string>>> exportMapping;
std::unique_lock<std::mutex> lock(viewSpecificMutexes[m_dscView->GetFile()->GetSessionId()].typeLibraryLookupAndApplicationMutex);
auto typeLib = m_dscView->GetTypeLibrary(header->installName);
if (!typeLib)
Expand All @@ -2897,46 +2926,54 @@ void SharedCache::FindSymbolAtAddrAndApplyToAddr(uint64_t symbolLocation, uint64
}
}
lock.unlock();
id = m_dscView->BeginUndoActions();
m_dscView->BeginBulkModifySymbols();
for (const auto& sym : exportList)

auto exportList = GetExportListForHeader(*header, [&]() {
try {
return MMappedFileAccessor::Open(m_dscView, m_dscView->GetFile()->GetSessionId(), header->exportTriePath)->lock();
}
catch (...)
{
m_logger->LogWarn("Serious Error: Failed to open export trie %s for %s", header->exportTriePath.c_str(), header->installName.c_str());
return std::shared_ptr<MMappedFileAccessor>(nullptr);
}
});

if (exportList)
{
exportMapping.push_back({sym->GetAddress(), {sym->GetType(), sym->GetRawName()}});
if (sym->GetAddress() == symbolLocation)
if (auto it = exportList->find(symbolLocation); it != exportList->end())
{
if (auto func = m_dscView->GetAnalysisFunction(m_dscView->GetDefaultPlatform(), targetLocation))
id = m_dscView->BeginUndoActions();
m_dscView->BeginBulkModifySymbols();

auto func = m_dscView->GetAnalysisFunction(m_dscView->GetDefaultPlatform(), targetLocation);
if (func)
{
m_dscView->DefineUserSymbol(
new Symbol(FunctionSymbol, prefix + sym->GetFullName(), targetLocation));
new Symbol(FunctionSymbol, prefix + it->second->GetFullName(), targetLocation));

if (typeLib)
if (auto type = m_dscView->ImportTypeLibraryObject(typeLib, {sym->GetFullName()}))
if (auto type = m_dscView->ImportTypeLibraryObject(typeLib, {it->second->GetFullName()}))
func->SetUserType(type);
}
else
{
m_dscView->DefineUserSymbol(
new Symbol(sym->GetType(), prefix + sym->GetFullName(), targetLocation));
new Symbol(it->second->GetType(), prefix + it->second->GetFullName(), targetLocation));

if (typeLib)
if (auto type = m_dscView->ImportTypeLibraryObject(typeLib, {sym->GetFullName()}))
if (auto type = m_dscView->ImportTypeLibraryObject(typeLib, {it->second->GetFullName()}))
m_dscView->DefineUserDataVariable(targetLocation, type);
}
if (triggerReanalysis)
{
auto func = m_dscView->GetAnalysisFunction(m_dscView->GetDefaultPlatform(), targetLocation);
if (func)
func->Reanalyze();
}
break;

m_dscView->EndBulkModifySymbols();
m_dscView->ForgetUndoActions(id);
}
}
{
std::unique_lock<std::mutex> _lock(viewSpecificMutexes[m_dscView->GetFile()->GetSessionId()].viewOperationsThatInfluenceMetadataMutex);
m_exportInfos[header->textBase] = exportMapping;
}
m_dscView->EndBulkModifySymbols();
m_dscView->ForgetUndoActions(id);
}
}

Expand All @@ -2961,7 +2998,7 @@ bool SharedCache::SaveToDSCView()
c.m_dyldDataRegions = m_dyldDataRegions;
c.m_nonImageRegions = m_nonImageRegions;
c.m_baseFilePath = m_baseFilePath;
c.m_exportInfos = m_exportInfos;
//c.m_exportInfos = m_exportInfos;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess you commented this out while testing the std::unique_ptr approach.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yup 🤦

c.m_symbolInfos = m_symbolInfos;
viewStateCache[m_dscView->GetFile()->GetSessionId()] = c;

Expand Down
25 changes: 13 additions & 12 deletions view/sharedcache/core/SharedCache.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -956,20 +956,20 @@ namespace SharedCacheCore {
// std::vector<std::pair<uint64_t, std::vector<std::pair<uint64_t, std::pair<BNSymbolType, std::string>>>>> exportInfos;
rapidjson::Document exportInfos(rapidjson::kArrayType);

for (const auto& pair1 : m_exportInfos)
for (const auto& [headerAddress, symbolMap] : m_exportInfos)
{
rapidjson::Value subObj(rapidjson::kObjectType);
rapidjson::Value subArr(rapidjson::kArrayType);
for (const auto& pair2 : pair1.second)
for (const auto& [symbolAddress, symbol] : *symbolMap)
{
rapidjson::Value subSubObj(rapidjson::kObjectType);
subSubObj.AddMember("key", pair2.first, m_activeContext.allocator);
subSubObj.AddMember("val1", pair2.second.first, m_activeContext.allocator);
subSubObj.AddMember("val2", pair2.second.second, m_activeContext.allocator);
subSubObj.AddMember("key", symbolAddress, m_activeContext.allocator);
subSubObj.AddMember("val1", symbol->GetType(), m_activeContext.allocator);
subSubObj.AddMember("val2", symbol->GetRawName(), m_activeContext.allocator);
subArr.PushBack(subSubObj, m_activeContext.allocator);
}

subObj.AddMember("key", pair1.first, m_activeContext.allocator);
subObj.AddMember("key", headerAddress, m_activeContext.allocator);
subObj.AddMember("value", subArr, m_activeContext.allocator);

exportInfos.PushBack(subObj, m_activeContext.allocator);
Expand Down Expand Up @@ -1042,14 +1042,15 @@ namespace SharedCacheCore {
m_exportInfos.clear();
for (const auto& obj1 : m_activeDeserContext.doc["exportInfos"].GetArray())
{
std::vector<std::pair<uint64_t, std::pair<BNSymbolType, std::string>>> innerVec;
auto innerMap = std::make_shared<std::unordered_map<uint64_t, Ref<Symbol>>>();
for (const auto& obj2 : obj1["value"].GetArray())
{
std::pair<BNSymbolType, std::string> innerPair = { (BNSymbolType)obj2["val1"].GetUint64(), obj2["val2"].GetString() };
innerVec.push_back({ obj2["key"].GetUint64(), innerPair });
auto address = obj2["key"].GetUint64();
Ref<Symbol> symbol = new Symbol((BNSymbolType)obj2["val1"].GetUint64(), obj2["val2"].GetString(), address);
innerMap->insert_or_assign(address, symbol);
}

m_exportInfos[obj1["key"].GetUint64()] = innerVec;
m_exportInfos.emplace(obj1["key"].GetUint64(), innerMap);
}
m_symbolInfos.clear();
for (auto& symbolInfo : m_activeDeserContext.doc["symbolInfos"].GetArray())
Expand Down Expand Up @@ -1114,8 +1115,7 @@ namespace SharedCacheCore {

// Updated as the view is loaded further, more images are added, etc
DSCViewState m_viewState = DSCViewStateUnloaded;
std::unordered_map<uint64_t, std::vector<std::pair<uint64_t, std::pair<BNSymbolType, std::string>>>>
m_exportInfos;
std::unordered_map<uint64_t, std::shared_ptr<std::unordered_map<uint64_t, Ref<Symbol>>>> m_exportInfos;
std::unordered_map<uint64_t, std::vector<std::pair<uint64_t, std::pair<BNSymbolType, std::string>>>>
m_symbolInfos;
// ---
Expand Down Expand Up @@ -1196,6 +1196,7 @@ namespace SharedCacheCore {
const std::string& currentText, size_t cursor, uint32_t endGuard);
std::vector<Ref<Symbol>> ParseExportTrie(
std::shared_ptr<MMappedFileAccessor> linkeditFile, SharedCacheMachOHeader header);
std::shared_ptr<std::unordered_map<uint64_t, Ref<Symbol>>> GetExportListForHeader(SharedCacheMachOHeader header, std::function<std::shared_ptr<MMappedFileAccessor>()> provideLinkeditFile, bool* didModifyExportList = nullptr);
};


Expand Down