Multithreaded http server crashes under load #3314

sikalyur · 2025-01-16T17:17:06Z

I've implemented simple multithreaded http server and built it under Windows OS with using pthreads4w library, the compiler is Visual Studio 2022 and build type is Release with Debug Information:

#include <libwebsockets.h>
#include <string.h>
#include <signal.h>

#include <string>
#include <vector>

#include <pthread.h>

#define COUNT_THREADS 4

static struct lws_context *context;
static volatile int interrupted;

void *thread_service(void *threadid)
{
    while (lws_service_tsi(context, 10000,
                   (int)(lws_intptr_t)threadid) >= 0 &&
           !interrupted)
        ;

    pthread_exit(NULL);

    return NULL;
}

void signal_cb(void *handle, int signum)
{
    interrupted = 1;

    switch (signum) {
    case SIGTERM:
    case SIGINT:
        break;
    default:
        lwsl_err("%s: signal %d\n", __func__, signum);
        break;
    }
    lws_context_destroy(context);
}

void sigint_handler(int sig)
{
    signal_cb(NULL, sig);
}

static const char* html_content = "<!DOCTYPE html><html><body><h1>Hello world!</h1></body></html>\r\n";

extern "C" int lws_http_handler(struct lws* wsi, enum lws_callback_reasons reason, void* user, void* in, size_t len)
{
    switch (reason)
    {
    case LWS_CALLBACK_HTTP: {
        lws_callback_on_writable(wsi);
        break;
    }
    case LWS_CALLBACK_HTTP_WRITEABLE: {
        int status = 200;
        std::string body = html_content;
        std::string contentType = "text/html";
        const static size_t reservedTitleSize = 512;
        const auto bufferSize = LWS_PRE + reservedTitleSize + contentType.size() + body.length();
        std::vector<unsigned char> buffer(bufferSize);
        unsigned char* p = &buffer[LWS_PRE];
        unsigned char* end = &buffer[bufferSize - LWS_PRE];
        unsigned char* start = p;
        // Add title and common headers (e.g., Content-Type)
        if (lws_add_http_common_headers(
                wsi, static_cast<int>(status), contentType.c_str(), body.size(), &p, end))
        {
            return 1;
        }
        // Finalize and send the HTTP header
        if (lws_finalize_write_http_header(wsi, start, &p, end))
        {
            return 1;
        }
        // Send body
        memcpy(p, body.c_str(), body.length());
        if (lws_write(wsi, p, body.length(), LWS_WRITE_HTTP_FINAL) != static_cast<int>(body.length()))
        {
            return 1;
        }
        if (lws_http_transaction_completed(wsi))
        {
            return -1;  // Close the connection due to lws internal error
        }
        break;
    }
    default:
        break;
    }
    return 0;
}

static struct lws_protocols protocols[] = {
    {"http", lws_http_handler, 0, 0},
    LWS_PROTOCOL_LIST_TERM  // Terminator
};


int main(int argc, const char **argv)
{
    pthread_t pthread_service[COUNT_THREADS];
    struct lws_context_creation_info info;
    void *retval;
    int n, logs = LLL_ERR | LLL_WARN
            /* for LLL_ verbosity above NOTICE to be built into lws,
             * lws must have been configured and built with
             * -DCMAKE_BUILD_TYPE=DEBUG instead of =RELEASE */
            /* | LLL_INFO */ /* | LLL_PARSER */ /* | LLL_HEADER */
            /* | LLL_EXT */ /* | LLL_CLIENT */ /* | LLL_LATENCY */
            /* | LLL_DEBUG */;

    lws_set_log_level(logs, NULL);

    memset(&info, 0, sizeof(info));

    info.port = CONTEXT_PORT_NO_LISTEN;
    info.options = LWS_SERVER_OPTION_EXPLICIT_VHOSTS;
    info.protocols = protocols;
    info.gid = -1;
    info.uid = -1;
    info.user = &context;

    info.count_threads = COUNT_THREADS;
    info.timeout_secs = 500;
    info.keepalive_timeout = 500;
    info.connect_timeout_secs = 160;
    info.timeout_secs_ah_idle = 150;

    context = lws_create_context(&info);
    if (!context) {
        lwsl_err("lws init failed\n");
        return 1;
    }

    info.iface = "0.0.0.0";
    info.port = 8080;
    if (lws_create_vhost(context, &info) == nullptr)
    {
        lwsl_err("lws vhost init failed\n");
        return 1;
    }

    lwsl_notice("  Service threads: %d\n", lws_get_count_threads(context));

    /* start all the service threads */

    for (n = 0; n < lws_get_count_threads(context); n++)
        if (pthread_create(&pthread_service[n], NULL, thread_service,
                   (void *)(lws_intptr_t)n))
            lwsl_err("Failed to start service thread\n");

    /* wait for all the service threads to exit */

    while ((--n) >= 0)
        pthread_join(pthread_service[n], &retval);

    lwsl_notice("%s: calling external context destroy\n", __func__);
    lws_context_destroy(context);

    return 0;
}

I use the h2load as a multiple clients load and run h2load in loop for spawning and closing connections in script:

#!/bin/bash
# Check for sufficient arguments
if [ "$#" -lt 1 ]; then
    echo "Usage: $0 <h2load_arguments>"
    echo "Example: $0 http://10.211.55.9:8080/hello -n100000 -c100 -t16 --h1"
    exit 1
fi

# Combine all passed arguments into the COMMAND
COMMAND="h2load $@"

echo "Starting endless h2load execution with: $COMMAND. Press Ctrl+C to stop."

# Infinite loop to execute the command
while true; do
    $COMMAND
    if [ $? -ne 0 ]; then
        echo "Command execution failed. Exiting."
        exit 1
    fi
done

I run script with options:

./h2load.sh http://<server_ip>:8080/ -n10000 -c100 -t16 --h1

The script runs sometime with good rate but after several minutes it starts freezing sometimes cause of server. Finally server crashes at line

libwebsockets/lib/plat/windows/windows-fds.c

Line 37 in 1fccae4

if (context->fd_hashtable[h].wsi[n]->desc.sockfd == fd)

with

error <path>\.conan2\p\b\libwe16e0083cfb612\b\src\lib\plat\windows\windows-fds.c:37: error: Debugger encountered an exception: Exception at 0x7ff7874e4973, code: 0xc0000005: read access violation at: 0x478, flags=0x0 (first chance)

.

May you please help me to resolve this issue?

The text was updated successfully, but these errors were encountered:

lws-team · 2025-01-16T19:40:31Z

Crash backtraces won't really help with that... if you want to help debug it, running it under linux + valgrind might provide useful information about what the exact code path that's wrong, eg, lock missing.

sikalyur · 2025-01-17T08:22:15Z

I've run this multithreaded http server under the valgrind and the CentOS Stream 9. It is built with GCC 11.5. No crash is observed in this case but it freezes forever and doesn't respond on any new request. Seems like there is dead lock in this case.
valgrind_output.txt

sikalyur · 2025-01-18T18:02:46Z

Could you confirm this implementation of multithread http server is correct?
Isn't there some implementation mistakes what leads to crash and dead lock?
Or it is good implemenation and there is some issue in LWS?

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Multithreaded http server crashes under load #3314

Multithreaded http server crashes under load #3314

sikalyur commented Jan 16, 2025

lws-team commented Jan 16, 2025

sikalyur commented Jan 17, 2025 •

edited

Loading

sikalyur commented Jan 18, 2025

Multithreaded http server crashes under load #3314

Multithreaded http server crashes under load #3314

Comments

sikalyur commented Jan 16, 2025

lws-team commented Jan 16, 2025

sikalyur commented Jan 17, 2025 • edited Loading

sikalyur commented Jan 18, 2025

sikalyur commented Jan 17, 2025 •

edited

Loading