Changeset 6919


Ignore:
Timestamp:
Oct 16, 2008, 5:24:57 AM (13 years ago)
Author:
charles
Message:

Try to hammer out some of the bugs in web.c. This still feels like a work in progress. Also, add a queue for tracker requests so that we don't wind up with 100s of concurrent http requests when there are a lot of torrents.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • trunk/libtransmission/web.c

    r6915 r6919  
    2323#include "web.h"
    2424
     25#define MAX_CONCURRENT_TASKS 16
     26
    2527#if LIBCURL_VERSION_NUM < 0x071003
    2628#define curl_multi_socket_action(m,fd,mask,i) curl_multi_socket((m),(fd),(i))
     
    2830
    2931#define dbgmsg( ... )  tr_deepLog( __FILE__, __LINE__, "web", __VA_ARGS__ )
     32/* #define dbgmsg(...)  do { fprintf( stderr, __VA_ARGS__ ); fprintf( stderr, "\n" ); } while( 0 ) */
    3033
    3134struct tr_web
     
    3740    tr_session * session;
    3841    struct event timer_event;
     42    tr_list * easy_queue;
    3943};
    4044
     
    5963    const size_t byteCount = size * nmemb;
    6064    evbuffer_add( ((struct tr_web_task*)task)->response, ptr, byteCount );
     65    dbgmsg( "wrote %zu bytes to task %p's buffer", byteCount, task );
    6166    return byteCount;
    6267}
     
    8287    {
    8388        struct tr_web * web = session->web;
    84         CURL * ch;
    85         CURLMcode rc;
     89        CURL * easy;
    8690
    8791        dbgmsg( "adding task #%lu [%s]", task->tag, task->url );
    8892
    89         ch = curl_easy_init( );
     93        easy = curl_easy_init( );
    9094
    9195        if( !task->range && session->isProxyEnabled ) {
    92             curl_easy_setopt( ch, CURLOPT_PROXY, session->proxy );
    93             curl_easy_setopt( ch, CURLOPT_PROXYPORT, session->proxyPort );
    94             curl_easy_setopt( ch, CURLOPT_PROXYTYPE, getCurlProxyType( session->proxyType ) );
    95             curl_easy_setopt( ch, CURLOPT_PROXYAUTH, CURLAUTH_ANY );
     96            curl_easy_setopt( easy, CURLOPT_PROXY, session->proxy );
     97            curl_easy_setopt( easy, CURLOPT_PROXYPORT, session->proxyPort );
     98            curl_easy_setopt( easy, CURLOPT_PROXYTYPE, getCurlProxyType( session->proxyType ) );
     99            curl_easy_setopt( easy, CURLOPT_PROXYAUTH, CURLAUTH_ANY );
    96100        }
    97101        if( !task->range && session->isProxyAuthEnabled ) {
    98102            char * str = tr_strdup_printf( "%s:%s", session->proxyUsername, session->proxyPassword );
    99             curl_easy_setopt( ch, CURLOPT_PROXYUSERPWD, str );
     103            curl_easy_setopt( easy, CURLOPT_PROXYUSERPWD, str );
    100104            tr_free( str );
    101105        }
    102106
    103         curl_easy_setopt( ch, CURLOPT_PRIVATE, task );
    104         curl_easy_setopt( ch, CURLOPT_URL, task->url );
    105         curl_easy_setopt( ch, CURLOPT_WRITEFUNCTION, writeFunc );
    106         curl_easy_setopt( ch, CURLOPT_WRITEDATA, task );
    107         curl_easy_setopt( ch, CURLOPT_USERAGENT, TR_NAME "/" LONG_VERSION_STRING );
    108         curl_easy_setopt( ch, CURLOPT_SSL_VERIFYHOST, 0 );
    109         curl_easy_setopt( ch, CURLOPT_SSL_VERIFYPEER, 0 );
    110         curl_easy_setopt( ch, CURLOPT_FORBID_REUSE, 1 );
    111         curl_easy_setopt( ch, CURLOPT_NOSIGNAL, 1 );
    112         curl_easy_setopt( ch, CURLOPT_FOLLOWLOCATION, 1 );
    113         curl_easy_setopt( ch, CURLOPT_MAXREDIRS, 5 );
    114         curl_easy_setopt( ch, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4 );
    115         curl_easy_setopt( ch, CURLOPT_VERBOSE, getenv( "TR_CURL_VERBOSE" ) != NULL );
     107        curl_easy_setopt( easy, CURLOPT_PRIVATE, task );
     108        curl_easy_setopt( easy, CURLOPT_URL, task->url );
     109        curl_easy_setopt( easy, CURLOPT_WRITEFUNCTION, writeFunc );
     110        curl_easy_setopt( easy, CURLOPT_WRITEDATA, task );
     111        curl_easy_setopt( easy, CURLOPT_USERAGENT, TR_NAME "/" LONG_VERSION_STRING );
     112        curl_easy_setopt( easy, CURLOPT_SSL_VERIFYHOST, 0 );
     113        curl_easy_setopt( easy, CURLOPT_SSL_VERIFYPEER, 0 );
     114        //curl_easy_setopt( easy, CURLOPT_FORBID_REUSE, 1 );
     115        curl_easy_setopt( easy, CURLOPT_NOSIGNAL, 1 );
     116        curl_easy_setopt( easy, CURLOPT_FOLLOWLOCATION, 1 );
     117        curl_easy_setopt( easy, CURLOPT_MAXREDIRS, 5 );
     118        curl_easy_setopt( easy, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4 );
     119        curl_easy_setopt( easy, CURLOPT_VERBOSE, getenv( "TR_CURL_VERBOSE" ) != NULL );
    116120        if( task->range )
    117             curl_easy_setopt( ch, CURLOPT_RANGE, task->range );
     121            curl_easy_setopt( easy, CURLOPT_RANGE, task->range );
    118122        else /* don't set encoding if range is sent; it messes up binary data */
    119             curl_easy_setopt( ch, CURLOPT_ENCODING, "" );
    120 
    121         rc = curl_multi_add_handle( web->multi, ch );
    122         if( rc != CURLM_OK )
    123             tr_err( "%s", curl_multi_strerror( rc ) );
     123            curl_easy_setopt( easy, CURLOPT_ENCODING, "" );
     124
     125        if( web->still_running >= MAX_CONCURRENT_TASKS ) {
     126            tr_list_append( &web->easy_queue, easy );
     127            dbgmsg( " >> adding a task to the curl queue... size is now %d", tr_list_size( web->easy_queue ) );
     128        } else {
     129            CURLMcode rc = curl_multi_add_handle( web->multi, easy );
     130            if( rc == CURLM_OK )
     131                ++web->still_running;
     132            else
     133                tr_err( "%s", curl_multi_strerror( rc ) );
     134        }
    124135    }
    125136}
     
    165176check_run_count( tr_web * g )
    166177{
     178    int added_from_queue = FALSE;
     179
    167180    dbgmsg( "check_run_count: prev_running %d, still_running %d",
    168181            g->prev_running, g->still_running );
    169182
    170     if( g->prev_running > g->still_running )
     183    if( 1 )
    171184    {
    172185        CURLMsg * msg;
     
    198211    g->prev_running = g->still_running;
    199212
    200     if( g->still_running <= 0 ) {
    201         if( evtimer_pending( &g->timer_event, NULL ) ) {
    202             dbgmsg( "deleting the pending global timer" );
    203             evtimer_del( &g->timer_event );
     213dbgmsg( "--> still running: %d ... max: %d ... queue size: %d", g->still_running, MAX_CONCURRENT_TASKS, tr_list_size( g->easy_queue ) );
     214
     215    while( ( g->still_running < MAX_CONCURRENT_TASKS )
     216        && ( tr_list_size( g->easy_queue ) > 0 ) )
     217    {
     218        CURL * easy = tr_list_pop_front( &g->easy_queue );
     219        if( easy )
     220        {
     221            const CURLMcode rc = curl_multi_add_handle( g->multi, easy );
     222            if( rc != CURLM_OK ) {
     223                tr_err( "%s", curl_multi_strerror( rc ) );
     224            } else {
     225                dbgmsg( "pumped a task out of the curl queue... %d remain", tr_list_size( g->easy_queue ) );
     226                added_from_queue = TRUE;
     227                ++g->still_running;
     228            }
    204229        }
    205230    }
    206231
    207     if( g->dying && ( g->still_running < 1 ) ) {
    208         dbgmsg( "destroying the web global now that all the tasks are done" );
    209         webDestroy( g );
     232    if( !added_from_queue )
     233    {
     234        if( g->still_running <= 0 ) {
     235            if( evtimer_pending( &g->timer_event, NULL ) ) {
     236                dbgmsg( "deleting the pending global timer" );
     237                evtimer_del( &g->timer_event );
     238            }
     239        }
     240
     241        if( g->dying && ( g->still_running < 1 ) ) {
     242            dbgmsg( "destroying the web global now that all the tasks are done" );
     243            webDestroy( g );
     244        }
    210245    }
    211246}
     
    233268        dbgmsg( "event_cb calling socket_action fd %d, mask %d", fd, mask );
    234269        rc = curl_multi_socket_action( g->multi, fd, mask, &g->still_running );
     270        dbgmsg( "event_cb(): still_running is %d", g->still_running );
    235271    } while( rc == CURLM_CALL_MULTI_PERFORM );
    236272    if( rc != CURLM_OK )
     
    252288        rc = curl_multi_socket_action( g->multi, CURL_SOCKET_TIMEOUT, 0,
    253289                                       &g->still_running );
     290        dbgmsg( "timer_cb(): still_running is %d", g->still_running );
    254291    } while( rc == CURLM_CALL_MULTI_PERFORM );
    255292
     
    277314         struct tr_web_sockinfo * f )
    278315{
    279     const int kind = (action & CURL_POLL_IN ? EV_READ : 0)
     316    const int kind = EV_PERSIST
     317                   | (action & CURL_POLL_IN ? EV_READ : 0)
    280318                   | (action & CURL_POLL_OUT ? EV_WRITE : 0);
    281319    dbgmsg( "setsock: fd is %d, curl action is %d, libevent action is %d", sockfd, action, kind );
Note: See TracChangeset for help on using the changeset viewer.