Changeset 11814


Ignore:
Timestamp:
Feb 2, 2011, 9:17:16 PM (11 years ago)
Author:
jordan
Message:

(trunk libT) #2955 "verify pieces only when necessary, or when the user requests it." -- improvements to .resume file

As pointed out by longinus00 and ijuxda, storing per-piece timestamps in the .resume file can involve a lot of overhead. This commit reduces the overhead by adding a couple of optimizations: (1) in cases where *all* or *none* of the files' pieces were checked after the file's mtime, we can safely fold all the pieces' mtimes into a single per-file mtime. (2) since unix time takes up a lot of space when rendered as a benc integer, find a common per-file "baseline" number, then store the pieces' timestamps as offsets from that number. Also add documentation explaining this new format, and also better explaining the pre-2.20 progress format.

Location:
trunk/libtransmission
Files:
3 edited

Legend:

Unmodified
Added
Removed
  • trunk/libtransmission/resume.c

    r11763 r11814  
    408408
    409409static void
    410 saveProgress( tr_benc *          dict,
    411               const tr_torrent * tor )
    412 {
    413     size_t              i, n;
    414     tr_benc *           p;
    415     tr_benc *           m;
    416     const tr_bitfield * bitfield;
    417 
    418     p = tr_bencDictAdd( dict, KEY_PROGRESS );
    419     tr_bencInitDict( p, 2 );
    420 
    421     /* add each piece's timeChecked */
    422     n = tor->info.pieceCount;
    423     m = tr_bencDictAddList( p, KEY_PROGRESS_CHECKTIME, n );
    424     for( i=0; i<n; ++i )
    425         tr_bencListAddInt( m, tor->info.pieces[i].timeChecked );
     410saveProgress( tr_benc * dict, const tr_torrent * tor )
     411{
     412    tr_benc * l;
     413    tr_benc * prog;
     414    tr_file_index_t fi;
     415    const struct tr_bitfield * bitfield;
     416    const tr_info * inf = tr_torrentInfo( tor );
     417    const time_t now = tr_time( );
     418
     419    prog = tr_bencDictAddDict( dict, KEY_PROGRESS, 3 );
     420
     421    /* add the file/piece check timestamps... */
     422    l = tr_bencDictAddList( prog, KEY_PROGRESS_CHECKTIME, inf->fileCount );
     423    for( fi=0; fi<inf->fileCount; ++fi )
     424    {
     425        const tr_piece * p;
     426        const tr_piece * pend;
     427        time_t oldest_nonzero = now;
     428        time_t newest = 0;
     429        tr_bool has_zero = FALSE;
     430        const time_t mtime = tr_torrentGetFileMTime( tor, fi );
     431        const tr_file * f = &inf->files[fi];
     432
     433        /* get the oldest and newest nonzero timestamps for pieces in this file */
     434        for( p=&inf->pieces[f->firstPiece], pend=&inf->pieces[f->lastPiece]; p!=pend; ++p )
     435        {
     436            if( !p->timeChecked )
     437                has_zero = TRUE;
     438            else if( oldest_nonzero > p->timeChecked )
     439                oldest_nonzero = p->timeChecked;
     440            if( newest < p->timeChecked )
     441                newest = p->timeChecked;
     442        }
     443
     444        /* If some of a file's pieces have been checked more recently than
     445           the file's mtime, and some lest recently, then that file will
     446           have a list containing timestamps for each piece.
     447           
     448           However, the most common use case is that the file doesn't change
     449           after it's downloaded. To reduce overhead in the .resume file,
     450           only a single timestamp is saved for the file if *all* or *none*
     451           of the pieces were tested more recently than the file's mtime. */
     452
     453        if( !has_zero && ( mtime <= oldest_nonzero ) ) /* all checked */
     454            tr_bencListAddInt( l, oldest_nonzero );
     455        else if( newest < mtime ) /* none checked */
     456            tr_bencListAddInt( l, newest );
     457        else { /* some are checked, some aren't... so list piece by piece */
     458            const int offset = oldest_nonzero - 1;
     459            tr_benc * ll = tr_bencListAddList( l, 2 + f->lastPiece - f->firstPiece );
     460            tr_bencListAddInt( ll, offset );
     461            for( p=&inf->pieces[f->firstPiece], pend=&inf->pieces[f->lastPiece]; p!=pend; ++p )
     462                tr_bencListAddInt( ll, p->timeChecked ? p->timeChecked - offset : 0 );
     463        }
     464    }
    426465
    427466    /* add the progress */
    428467    if( tor->completeness == TR_SEED )
    429         tr_bencDictAddStr( p, KEY_PROGRESS_HAVE, "all" );
     468        tr_bencDictAddStr( prog, KEY_PROGRESS_HAVE, "all" );
     469
     470    /* add the pieces bitfield */
    430471    bitfield = tr_cpBlockBitfield( &tor->completion );
    431     tr_bencDictAddRaw( p, KEY_PROGRESS_BITFIELD,
    432                        bitfield->bits, bitfield->byteCount );
    433 }
    434 
    435 static uint64_t
    436 loadProgress( tr_benc *    dict,
    437               tr_torrent * tor )
    438 {
    439     size_t    i, n;
    440     uint64_t  ret = 0;
    441     tr_benc * p;
    442 
    443     for( i=0, n=tor->info.pieceCount; i<n; ++i )
    444         tor->info.pieces[i].timeChecked = 0;
    445 
    446     if( tr_bencDictFindDict( dict, KEY_PROGRESS, &p ) )
     472    tr_bencDictAddRaw( prog, KEY_PROGRESS_BITFIELD, bitfield->bits,
     473                                                   bitfield->byteCount );
     474}
     475
     476static uint64_t
     477loadProgress( tr_benc * dict, tr_torrent * tor )
     478{
     479    size_t i, n;
     480    uint64_t ret = 0;
     481    tr_benc * prog;
     482    const tr_info * inf = tr_torrentInfo( tor );
     483
     484    for( i=0, n=inf->pieceCount; i<n; ++i )
     485        inf->pieces[i].timeChecked = 0;
     486
     487    if( tr_bencDictFindDict( dict, KEY_PROGRESS, &prog ) )
    447488    {
    448489        const char * err;
    449490        const char * str;
    450491        const uint8_t * raw;
    451         size_t          rawlen;
    452         tr_benc *       m;
    453         int64_t  timeChecked;
    454 
    455         if( tr_bencDictFindList( p, KEY_PROGRESS_CHECKTIME, &m ) )
    456         {
    457             /* This key was added in 2.20.
    458                Load in the timestamp of when we last checked each piece */
    459             for( i=0, n=tor->info.pieceCount; i<n; ++i )
    460                 if( tr_bencGetInt( tr_bencListChild( m, i ), &timeChecked ) )
    461                     tor->info.pieces[i].timeChecked = (time_t)timeChecked;
    462         }
    463         else if( tr_bencDictFindList( p, KEY_PROGRESS_MTIMES, &m ) )
    464         {
    465             /* This is how it was done pre-2.20... per file. */
    466             for( i=0, n=tr_bencListSize(m); i<n; ++i )
     492        size_t rawlen;
     493        tr_benc * l;
     494
     495        if( tr_bencDictFindList( prog, KEY_PROGRESS_CHECKTIME, &l ) )
     496        {
     497            /* per-piece timestamps were added in 2.20.
     498             
     499               If some of a file's pieces have been checked more recently than
     500               the file's mtime, and some lest recently, then that file will
     501               have a list containing timestamps for each piece.
     502             
     503               However, the most common use case is that the file doesn't change
     504               after it's downloaded. To reduce overhead in the .resume file,
     505               only a single timestamp is saved for the file if *all* or *none*
     506               of the pieces were tested more recently than the file's mtime. */
     507
     508            tr_file_index_t fi;
     509
     510            for( fi=0; fi<inf->fileCount; ++fi )
    467511            {
    468                 /* get the timestamp of file #i */
    469                 if( tr_bencGetInt( tr_bencListChild( m, i ), &timeChecked ) )
     512                tr_benc * b = tr_bencListChild( l, fi );
     513                const tr_file * f = &inf->files[fi];
     514                tr_piece * p = &inf->pieces[f->firstPiece];
     515                const tr_piece * pend = &inf->pieces[f->lastPiece];
     516
     517                if( tr_bencIsInt( b ) )
    470518                {
    471                     /* walk through all the pieces that are in that file... */
    472                     tr_piece_index_t j;
    473                     tr_file * file = &tor->info.files[i];
    474                     for( j=file->firstPiece; j<=file->lastPiece; ++j )
     519                    int64_t t;
     520                    tr_bencGetInt( b, &t );
     521                    for( ; p!=pend; ++p )
     522                        p->timeChecked = (time_t)t;
     523                }
     524                else if( tr_bencIsList( b ) )
     525                {
     526                    int i = 0;
     527                    int64_t offset = 0;
     528                    const int pieces = f->lastPiece + 1 - f->firstPiece;
     529
     530                    tr_bencGetInt( tr_bencListChild( b, 0 ), &offset );
     531
     532                    for( i=0; i<pieces; ++i )
    475533                    {
    476                         tr_piece * piece = &tor->info.pieces[j];
    477 
    478                         /* If the piece's timestamp is unset from earlier,
    479                          * set it here. */
    480                         if( piece->timeChecked == 0 )
    481                             piece->timeChecked = timeChecked;
    482 
    483                         /* If the piece's timestamp is *newer* timeChecked,
    484                          * the piece probably spans more than one file.
    485                          * To be safe, let's use the older timestamp. */
    486                         if( piece->timeChecked > timeChecked )
    487                             piece->timeChecked = timeChecked;
     534                        int64_t t = 0;
     535                        tr_bencGetInt( tr_bencListChild( b, i+1 ), &t );
     536                        inf->pieces[f->firstPiece+i].timeChecked = (time_t)(t + offset);
    488537                    }
    489538                }
    490539            }
    491540        }
     541        else if( tr_bencDictFindList( prog, KEY_PROGRESS_MTIMES, &l ) )
     542        {
     543            tr_file_index_t fi;
     544
     545            /* Before 2.20, we stored the files' mtimes in the .resume file.
     546               When loading the .resume file, a torrent's file would be flagged
     547               as untested if its stored mtime didn't match its real mtime. */
     548
     549            for( fi=0; fi<inf->fileCount; ++fi )
     550            {
     551                int64_t t;
     552
     553                if( tr_bencGetInt( tr_bencListChild( l, fi ), &t ) )
     554                {
     555                    const tr_file * f = &inf->files[fi];
     556                    tr_piece * p = &inf->pieces[f->firstPiece];
     557                    const tr_piece * pend = &inf->pieces[f->lastPiece];
     558                    const time_t mtime = tr_torrentGetFileMTime( tor, fi );
     559                    const time_t timeChecked = mtime==t ? mtime : 0;
     560
     561                    for( ; p!=pend; ++p )
     562                        p->timeChecked = timeChecked;
     563                }
     564            }
     565        }
    492566
    493567        err = NULL;
    494         if( tr_bencDictFindStr( p, KEY_PROGRESS_HAVE, &str ) )
     568        if( tr_bencDictFindStr( prog, KEY_PROGRESS_HAVE, &str ) )
    495569        {
    496570            if( !strcmp( str, "all" ) )
     
    499573                err = "Invalid value for HAVE";
    500574        }
    501         else if( tr_bencDictFindRaw( p, KEY_PROGRESS_BITFIELD, &raw, &rawlen ) )
     575        else if( tr_bencDictFindRaw( prog, KEY_PROGRESS_BITFIELD, &raw, &rawlen ) )
    502576        {
    503577            tr_bitfield tmp;
  • trunk/libtransmission/torrent.c

    r11813 r11814  
    23892389}
    23902390
    2391 static time_t
    2392 getFileMTime( const tr_torrent * tor, tr_file_index_t i )
     2391time_t
     2392tr_torrentGetFileMTime( const tr_torrent * tor, tr_file_index_t i )
    23932393{
    23942394    struct stat sb;
     
    24262426    for( ; f < inf->fileCount && pieceHasFile( p, &inf->files[f] ); ++f )
    24272427        if( tr_cpFileIsComplete( &tor->completion, f ) )
    2428             if( getFileMTime( tor, f ) > inf->pieces[p].timeChecked )
     2428            if( tr_torrentGetFileMTime( tor, f ) > inf->pieces[p].timeChecked )
    24292429                return TRUE;
    24302430
  • trunk/libtransmission/torrent.h

    r11807 r11814  
    430430tr_bool tr_torrentCheckPiece( tr_torrent * tor, tr_piece_index_t pieceIndex );
    431431
     432time_t tr_torrentGetFileMTime( const tr_torrent * tor, tr_file_index_t i );
     433
    432434uint64_t tr_torrentGetCurrentSizeOnDisk( const tr_torrent * tor );
    433435
Note: See TracChangeset for help on using the changeset viewer.