DBG: (performance) improvements to SymbolSourceDIA

Author: mrexodia

src/dbg/module.cpp

@@ -1190,6 +1190,8 @@ bool MODINFO::loadSymbols()
     if(symbols == &EmptySymbolSource && SymbolSourceDIA::isLibraryAvailable())
     {
         // TODO: do something with searchPaths
+        std::string modname = name;
+        modname += extension;
         DiaValidationData_t validationData;
         memcpy(&validationData.guid, &pdbValidation.guid, sizeof(GUID));
         validationData.signature = pdbValidation.signature;
@@ -1201,7 +1203,7 @@ bool MODINFO::loadSymbols()
             {
                 GuiSymbolLogAdd(StringUtils::sprintf("[DIA] Skipping non-existent PDB: %s\n", pdbPath.c_str()).c_str());
             }
-            else if(symSource->loadPDB(pdbPath, base, size, bForceLoadSymbols ? nullptr : &validationData))
+            else if(symSource->loadPDB(pdbPath, modname, base, size, bForceLoadSymbols ? nullptr : &validationData))
             {
                 symSource->resizeSymbolBitmap(size);
 

src/dbg/symbolsourcedia.cpp

@@ -42,7 +42,7 @@ DWORD WINAPI SymbolSourceDIA::SourceLinesThread(void* parameter)
     return 0;
 }
 
-bool SymbolSourceDIA::loadPDB(const std::string & path, duint imageBase, duint imageSize, DiaValidationData_t* validationData)
+bool SymbolSourceDIA::loadPDB(const std::string & path, const std::string & modname, duint imageBase, duint imageSize, DiaValidationData_t* validationData)
 {
     PDBDiaFile pdb; // Instance used for validation only.
     _isOpen = pdb.open(path.c_str(), 0, validationData);
@@ -119,7 +119,7 @@ bool SymbolSourceDIA::loadSymbolsAsync()
     DWORD loadStart = GetTickCount();
 
     PDBDiaFile::Query_t query;
-    query.collectSize = true;
+    query.collectSize = false;
     query.collectUndecoratedNames = true;
     query.callback = [&](DiaSymbol_t & sym) -> bool
     {
@@ -221,7 +221,7 @@ bool SymbolSourceDIA::loadSymbolsAsync()
     DWORD ms = GetTickCount() - loadStart;
     double secs = (double)ms / 1000.0;
 
-    GuiSymbolLogAdd(StringUtils::sprintf("[%p] Loaded %d symbols in %.03fs\n", _imageBase, _symAddrs.size(), secs).c_str());
+    GuiSymbolLogAdd(StringUtils::sprintf("[%p, %s] Loaded %d symbols in %.03fs\n", _imageBase, _modname.c_str(), _symAddrs.size(), secs).c_str());
 
     GuiInvalidateSymbolSource(_imageBase);
 
@@ -245,48 +245,76 @@ bool SymbolSourceDIA::loadSourceLinesAsync()
 
     const uint32_t rangeSize = 1024 * 1024 * 10;
 
+    std::vector<DiaLineInfo_t> lines;
     std::map<DWORD, String> files;
 
-    for(duint rva = 0; rva < _imageSize; rva += rangeSize)
-    {
-        if(_requiresShutdown)
-            return false;
-
-        std::vector<DiaLineInfo_t> lines;
+    if(!pdb.enumerateLineNumbers(0, uint32_t(_imageSize), lines, files))
+        return false;
 
-        bool res = pdb.enumerateLineNumbers(uint32_t(rva), rangeSize, lines, files);
-        for(const auto & line : lines)
+    if(files.size() == 1)
+    {
+        GuiSymbolLogAdd(StringUtils::sprintf("[%p, %s] Since there is only one file, attempting line overflow detection..\n", _imageBase, _modname.c_str()).c_str());
+
+        // This is a super hack to adjust for the (undocumented) limit of 16777215 lines (unsigned 24 bits maximum).
+        // It is unclear at this point if yasm/coff/link/pdb is causing this issue.
+        // We can fix this because there is only a single source file and the returned result is sorted by *both* rva/line (IMPORTANT!).
+        // For supporting multiple source files in the future we could need multiple 'lineOverflow' variables.
+        uint32_t maxLine = 0, maxRva = 0, lineOverflows = 0;
+        for(auto & line : lines)
         {
-            if(_requiresShutdown)
-                return false;
-
-            const auto & info = line;
-            auto it = _lines.find(info.rva);
-            if(it != _lines.end())
-                continue;
-
-            CachedLineInfo lineInfo;
-            lineInfo.rva = info.rva;
-            lineInfo.lineNumber = info.lineNumber;
+            uint32_t overflowValue = 0x1000000 * (lineOverflows + 1) - 1; //0xffffff, 0x1ffffff, 0x2ffffff, etc
+            if((line.lineNumber & 0xfffff0) == 0 && (maxLine & 0xfffffff0) == (overflowValue & 0xfffffff0))  // allow 16 lines of play, perhaps there is a label/comment on line 0xffffff+1
+            {
+                GuiSymbolLogAdd(StringUtils::sprintf("[%p, %s] Line number overflow detected (%u -> %u), adjusting with hacks...\n", _imageBase, _modname.c_str(), maxLine, line.lineNumber).c_str());
+                lineOverflows++;
+            }
 
-            auto sourceFileId = info.sourceFileId;
-            auto found = _sourceIdMap.find(sourceFileId);
-            if(found == _sourceIdMap.end())
+            line.lineNumber += lineOverflows * 0xffffff + lineOverflows;
+            if(!(line.lineNumber > maxLine))
             {
-                auto idx = _sourceFiles.size();
-                _sourceFiles.push_back(files[sourceFileId]);
-                found = _sourceIdMap.insert({ sourceFileId, uint32_t(idx) }).first;
+                GuiSymbolLogAdd(StringUtils::sprintf("[%p, %s] The line information is not sorted by line (violated assumption)! lineNumber: %u, maxLine: %u\n", _imageBase, _modname.c_str(), line.lineNumber, maxLine).c_str());
             }
-            lineInfo.sourceFileIdx = found->second;
+            maxLine = line.lineNumber;
+            if(!(line.rva > maxRva))
+            {
+                GuiSymbolLogAdd(StringUtils::sprintf("[%p, %s] The line information is not sorted by rva (violated assumption)! rva: 0x%x, maxRva: 0x%x\n", _imageBase, _modname.c_str(), line.rva, maxRva).c_str());
+            }
+            maxRva = line.rva;
+        }
+    }
+
+    _linesData.reserve(lines.size());
+    _sourceFiles.reserve(files.size());
+    for(const auto & line : lines)
+    {
+        if(_requiresShutdown)
+            return false;
 
-            _lockLines.lock();
+        const auto & info = line;
+        auto it = _lines.find(info.rva);
+        if(it != _lines.end())
+            continue;
 
-            _linesData.push_back(lineInfo);
-            _lines.insert({ lineInfo.rva, _linesData.size() - 1 });
+        CachedLineInfo lineInfo;
+        lineInfo.rva = info.rva;
+        lineInfo.lineNumber = info.lineNumber;
 
-            _lockLines.unlock();
+        auto sourceFileId = info.sourceFileId;
+        auto found = _sourceIdMap.find(sourceFileId);
+        if(found == _sourceIdMap.end())
+        {
+            auto idx = _sourceFiles.size();
+            _sourceFiles.push_back(files[sourceFileId]);
+            found = _sourceIdMap.insert({ sourceFileId, uint32_t(idx) }).first;
         }
+        lineInfo.sourceFileIdx = found->second;
+
+        _lockLines.lock();
+
+        _linesData.push_back(lineInfo);
+        _lines.insert({ lineInfo.rva, _linesData.size() - 1 });
 
+        _lockLines.unlock();
     }
 
     _sourceLines.resize(_sourceFiles.size());
@@ -302,7 +330,7 @@ bool SymbolSourceDIA::loadSourceLinesAsync()
     DWORD ms = GetTickCount() - lineLoadStart;
     double secs = (double)ms / 1000.0;
 
-    GuiSymbolLogAdd(StringUtils::sprintf("[%p] Loaded %d line infos in %.03fs\n", _imageBase, _lines.size(), secs).c_str());
+    GuiSymbolLogAdd(StringUtils::sprintf("[%p, %s] Loaded %d line infos in %.03fs\n", _imageBase, _modname.c_str(), _lines.size(), secs).c_str());
 
     GuiUpdateAllViews();
 

src/dbg/symbolsourcedia.h

@@ -106,6 +106,7 @@ class SymbolSourceDIA : public SymbolSourceBase
 
     bool _isOpen;
     std::string _path;
+    std::string _modname;
     duint _imageBase;
     duint _imageSize;
     SpinLock _lockSymbols;
@@ -158,7 +159,7 @@ class SymbolSourceDIA : public SymbolSourceBase
     virtual bool findSymbolsByPrefix(const std::string & prefix, const std::function<bool(const SymbolInfo &)> & cbSymbol, bool caseSensitive) override;
 
 public:
-    bool loadPDB(const std::string & path, duint imageBase, duint imageSize, DiaValidationData_t* validationData);
+    bool loadPDB(const std::string & path, const std::string & modname, duint imageBase, duint imageSize, DiaValidationData_t* validationData);
 
 private:
     void loadPDBAsync();