Writing a debugger mini series: A GUI source code viewer, Load, Wrap, Select, Highlight, Scroll and Center vertically on RichEdit (C/C++)

This is a side topic from the Debugger mini series. The debugger is a Windbg extension (because the debuggee is kernel). The goal is to write a simple source code browser in a GUI window that would display source code while you work on windbg’s main window.

I’m going to outline below how to do that (roughly, the code is below draft standard).

1. Create a thread for the UI. This is necessary because you need a dedicated loop to process window messages

DWORD WINAPI uiThread(LPVOID lpParam)
{
    // You need to load the richedit dll before using it
    LoadLibrary(TEXT("Riched32.dll"));
    
    // stuff needed for creating a Windows window
    WNDCLASS wndClass;
    HWND hWnd = 0;
    MSG msg;
    BOOL hasMsg;
    LPVOID lpMsgBuf;
    static BOOL registeredWindow = FALSE;

    if(!registeredWindow) {
        // regsiter the window class
        memset (&wndClass, 0, sizeof(wndClass));
        wndClass.cbWndExtra    = 4;
        wndClass.hCursor       = LoadCursor( NULL, IDC_ARROW );
        wndClass.hIcon         = LoadIcon(GetModuleHandle(NULL), MAKEINTRESOURCE(IDI_APPLICATION));
        wndClass.hInstance     = GetModuleHandle(NULL);
        wndClass.lpfnWndProc   = WndProcFlcndbg;
        wndClass.lpszClassName = TEXT("FlcndbgCodeWindow");
        wndClass.style         = CS_HREDRAW | CS_VREDRAW;
        if(!RegisterClass(&wndClass))
        {
            // decode the error message from GetLastError()
            wndClass.lpfnWndProc = NULL;
            FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER |
              FORMAT_MESSAGE_FROM_SYSTEM |
              FORMAT_MESSAGE_IGNORE_INSERTS,
              NULL,
              GetLastError(),
              MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), // Default language
              (LPTSTR) &lpMsgBuf,
              0,
              NULL);

            MessageBoxA(NULL, (LPCTSTR)lpMsgBuf, "Error", MB_OK | MB_ICONINFORMATION);
            // Free the buffer.
            LocalFree(lpMsgBuf);

            return 0;
        }
        registeredWindow = TRUE;
    }

    // create the window
    hWnd = CreateWindow(TEXT("XXXXX"), TEXT("XXXXX"),
                WS_BORDER | WS_CAPTION | WS_POPUP,
                50, 50, 600, 600, NULL, NULL, wndClass.hInstance, 0);
    if(!hWnd)
    {
        MessageBoxA( NULL, "Invalid hWnd", "Error", MB_OK | MB_ICONINFORMATION );
        if (wndClass.lpfnWndProc) {
            UnregisterClass(wndClass.lpszClassName, wndClass.hInstance);
        }
        return;
    }

    // display the window
    ShowWindow(hWnd, SW_NORMAL);
    UpdateWindow(hWnd);

    while(1) {
        // process Window messages
        hasMsg = GetMessage(&msg, NULL, 0, 0);
        if(hasMsg)
        {
            TranslateMessage(&msg);
            DispatchMessage(&msg);
        }
    }
}

2. Create the thread

// variables required for thread
HANDLE uiH;
DWORD uiId;

// open a window to display source code (this has to be recreate each time we
// re-enter the debugger (otherwise it would freeze anyway since the message queue
// isnt being processed when this debugger isnt running
uiH = CreateThread(NULL, 0, uiThread, NULL, 0, &uiId);
if(uiH == NULL) {
    // error creating thread
}

3. To communicate with the thread (such as telling it to terminate cleanly, sending it commands like what source to load etc) is another matter not discussed here. But you can look up how to pass messages between threads

4. Creating the Message handlers for the new Window + creating a RichEdit text box for doing the source displaying stuff

HWND childWin;

LRESULT CALLBACK WndProcFlcndbg(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam)
{
    switch(message)
    {
    case WM_CREATE:
        // find the inner area of the window
        GetClientRect(hWnd, &r);

        // create a RichEdit v1 text box with ID 1, fill the entire window
        childWin = CreateWindow(TEXT("RichEdit"), TEXT("Hello?"),
            // Google these flags to see what they do if you are interested
            WS_VISIBLE | WS_CHILD | WS_BORDER | ES_MULTILINE | ES_READONLY | WS_VSCROLL | WS_HSCROLL,
            // random sizes as examples
            0, 0, r.right, r.bottom,
            hWnd, (HMENU) 1, NULL, NULL);
        break;

    default:
        break;
    }
    // default handler for handling all the messages that we did not handle explicitly above
    return DefWindowProc(hWnd, message, wParam, lParam);
}

5. Actually using the RichEdit. I think this is by far the most annoying, and you should avoid using it if at all possible. There are alternatives like the MFC and WPF classes that are much better. Here I do not have a choice due to toolchain constraints…

I will describe here a list of messages that can be used to load text, highlight a line of text, and scrolling the editor to center on the line of text. The code below are not compilable examples and most variable names are pretty bad.

A note here is the variable CHARFORMAT2 cformat; needs to be outside of function scope, because SendMessage returns immediately and cformat may be destroyed before the message is actually processed (if this happens then the color is simply not set).

// the line of text we will be loading
char* lines = "This is a text\nlets see if this new line comeskljdasjdhkjaldhlkasdhgklajdhgaljsdhg up\n\nline 3\nline4\n\n\nline 3\nline4\n\n\nline 3\nline4\n\n\nline 3\nline4\n\nline4\n\n\nline 3\nline4\n\n\nline 3\nline4\n";
// character position thats needed for various messages
int lineChar;
// holder
int len;
// holds the line # for the top most visible in the editor
int topLine;
// the line # for the current selected line
int thisLine;
// bottom most line # that is visible
int bottomLine;
// total line # in the editor
int lastLine;
// # of lines to scroll
int scroll;
// direction of scroll
int scrollDir;
// tmp
int i;
// for getting the rectangle area of the text editor
RECT r;
// required by some messages
POINTL p;
// for setting the color of the current line of text
CHARFORMAT2 cformat;

// load text into the editor
SendMessage(childWin, WM_SETTEXT, NULL, lines);

// disable line wrap, see http://ucla.jamesyxu.com/?p=238
SendMessage(childWin, EM_SETTARGETDEVICE, NULL, 1);

// here we arbitrarily try to get the editor to scroll to line # 15, highlight it with red background and place that in the middle of the editor        
// what is the character # for the first char on line 15?
lineChar = SendMessage(childWin, EM_LINEINDEX, 15, 0);
// what is the length of the line at 15?
len = SendMessage(childWin, EM_LINELENGTH, lineChar, 0);
// select line 15 (by selecting the first char + len        
SendMessage(childWin, EM_SETSEL, lineChar, lineChar+len);
// set color
cformat.dwMask = CFM_BACKCOLOR;
cformat.cbSize = sizeof(cformat);
cformat.crBackColor = RGB(255,0,0);
SendMessage(childWin, EM_SETCHARFORMAT, SCF_SELECTION, &cformat);
// scroll to selection (so it is at least on screen)
SendMessage(childWin, EM_SCROLLCARET, 0, 0); 

// compute how many lines we need to scroll to adjust the target line to the middle (vertically)
topLine = SendMessage(childWin, EM_GETFIRSTVISIBLELINE, 0, 0);
thisLine = SendMessage(childWin, EM_LINEFROMCHAR, lineChar, 0);
// the way to get the last visible line on RichEdit is weird:
SendMessage(childWin, EM_GETRECT, 0, &r);
p.x = r.right;
p.y = r.bottom;
bottomLine = SendMessage(childWin, EM_CHARFROMPOS, 0, &p);
bottomLine = SendMessage(childWin, EM_LINEFROMCHAR, bottomLine, 0);
// last line of the file
lastLine = SendMessage(childWin, EM_GETLINECOUNT, 0, 0);

// We want thisLine to be the center of the screen.
// bottomLine - topLine / 2 is the target line, with a initial offset of topLine #
scroll = thisLine - ((bottomLine - topLine)/2) - topLine;

printf("Top is: %d, Bottom is %d, this is %d, scroll %d\n", topLine, bottomLine, thisLine, scroll);

// find the scroll direction
if (scroll > 0)
    scrollDir = SB_LINEDOWN;
else
    scrollDir = SB_LINEUP;
//  do the actual scroll
for(i = 0; i<abs(scroll); i++)
{
    SendMessage(childWin, WM_VSCROLL, scrollDir, 0);
}