debugger: fix memory issues caused by overlayed debug symbols

Author: Crementif

src/Cafe/GraphicPack/GraphicPack2PatchesApply.cpp

@@ -420,7 +420,7 @@ void PatchEntryInstruction::undoPatch()
 	uint8* patchAddr = (uint8*)memory_base + addr;
 	memcpy(patchAddr, m_dataBackup, m_length);
 	PPCRecompiler_invalidateRange(addr, addr + m_length);
-	rplSymbolStorage_removeRange(addr, m_length);
+	rplSymbolStorage_removeRange(addr, m_length, RPL_STORED_SYMBOL_PATCH);
 	DebugSymbolStorage::ClearRange(addr, m_length);
 }
 
@@ -434,7 +434,7 @@ bool registerU32Variable(PatchContext_t& ctx, std::string& name, uint32 value, P
 	}
 	ctx.map_values[name] = value;
 	// keep track of address symbols for the debugger
-	rplSymbolStorage_store(ctx.graphicPack->GetName().data(), name.data(), value);
+	rplSymbolStorage_store(ctx.graphicPack->GetName().data(), name.data(), value, RPL_STORED_SYMBOL_PATCH);
 
 	return true;
 }

src/Cafe/OS/RPL/rpl_symbol_storage.cpp

@@ -63,7 +63,7 @@ char* rplSymbolStorage_storeLibname(const char* libName)
 	return libEntry->libName;
 }
 
-RPLStoredSymbol* rplSymbolStorage_store(const char* libName, const char* symbolName, MPTR address)
+RPLStoredSymbol* rplSymbolStorage_store(const char* libName, const char* symbolName, MPTR address, uint32 type)
 {
 	std::unique_lock<std::mutex> lck(rplSymbolStorage.m_symbolStorageMutex);
 	char* libNameStorage = rplSymbolStorage_storeLibname(libName);
@@ -72,7 +72,7 @@ RPLStoredSymbol* rplSymbolStorage_store(const char* libName, const char* symbolN
 	storedSymbol->address = address;
 	storedSymbol->libName = libNameStorage;
 	storedSymbol->symbolName = symbolNameStorage;
-	storedSymbol->flags = 0;
+	storedSymbol->flags = type;
 	storedSymbol->previous = nullptr;
 	auto it = rplSymbolStorage.map_symbolByAddress.find(address);
 	if (it != rplSymbolStorage.map_symbolByAddress.end())
@@ -130,13 +130,41 @@ void rplSymbolStorage_remove(RPLStoredSymbol* storedSymbol)
 	delete storedSymbol;
 }
 
-void rplSymbolStorage_removeRange(MPTR address, sint32 length)
+
+void rplSymbolStorage_removeRange(MPTR address, sint32 length, uint32 type)
 {
+	std::unique_lock<std::mutex> lck(rplSymbolStorage.m_symbolStorageMutex);
 	while (length > 0)
 	{
-		RPLStoredSymbol* symbol = rplSymbolStorage_getByAddress(address);
-		if (symbol)
-			rplSymbolStorage_remove(symbol);
+		auto it = rplSymbolStorage.map_symbolByAddress.find(address);
+		if (it != rplSymbolStorage.map_symbolByAddress.end())
+		{
+			RPLStoredSymbol* current = it->second;
+			RPLStoredSymbol* newHead = current;
+			RPLStoredSymbol* previousKept = nullptr;
+			while (current)
+			{
+				RPLStoredSymbol* next = current->previous;
+				if ((current->flags & type) == type)
+				{
+					if (previousKept)
+						previousKept->previous = next;
+					else
+						newHead = next;
+					delete current;
+				}
+				else
+				{
+					previousKept = current;
+				}
+				current = next;
+			}
+
+			if (newHead)
+				it->second = newHead;
+			else
+				rplSymbolStorage.map_symbolByAddress.erase(it);
+		}
 		address += 4;
 		length -= 4;
 	}

src/Cafe/OS/RPL/rpl_symbol_storage.h

@@ -1,3 +1,10 @@
+enum RPLStoredSymbolType : uint32
+{
+	RPL_STORED_SYMBOL_NONE = 0,
+	RPL_STORED_SYMBOL_MAP = 1 << 0,
+	RPL_STORED_SYMBOL_PATCH = 1 << 1,
+};
+
 struct RPLStoredSymbol
 {
 	MPTR address;
@@ -9,9 +16,9 @@ struct RPLStoredSymbol
 
 void rplSymbolStorage_init();
 void rplSymbolStorage_unloadAll();
-RPLStoredSymbol* rplSymbolStorage_store(const char* libName, const char* symbolName, MPTR address);
+RPLStoredSymbol* rplSymbolStorage_store(const char* libName, const char* symbolName, MPTR address, uint32 type = RPL_STORED_SYMBOL_NONE);
 void rplSymbolStorage_remove(RPLStoredSymbol* storedSymbol);
-void rplSymbolStorage_removeRange(MPTR address, sint32 length);
+void rplSymbolStorage_removeRange(MPTR address, sint32 length, uint32 type = RPL_STORED_SYMBOL_NONE);
 RPLStoredSymbol* rplSymbolStorage_getByAddress(MPTR address);
 RPLStoredSymbol* rplSymbolStorage_getByClosestAddress(MPTR address);
 void rplSymbolStorage_createJumpProxySymbol(MPTR jumpAddress, MPTR destAddress);

src/gui/wxgui/debugger/DebuggerWindow2.cpp

@@ -401,7 +401,7 @@ void DebuggerWindow2::LoadModuleMap(DebuggerModuleStorage& moduleStorage)
 			const auto relocatedAddress = GetRelocatedMapAddress(moduleStorage.moduleInfo, mapAddress);
 			if (relocatedAddress)
 			{
-				moduleStorage.loaded_map_symbols.emplace_back(rplSymbolStorage_store(moduleStorage.moduleInfo.moduleName.c_str(), symbolName.c_str(), *relocatedAddress));
+				moduleStorage.loaded_map_symbols.emplace_back(rplSymbolStorage_store(moduleStorage.moduleInfo.moduleName.c_str(), symbolName.c_str(), *relocatedAddress, RPL_STORED_SYMBOL_MAP));
 			}
 		}
 	}