source: trunk/libtransmission/makemeta.c @ 2229

Last change on this file since 2229 was 2229, checked in by charles, 15 years ago

fix divide by zero bug in makemeta code, calculating number of pieces for a zero-sized file. (SoftwareElves?)

File size: 13.2 KB
Line 
1/*
2 * This file Copyright (C) 2007 Charles Kerr <charles@rebelbase.com>
3 *
4 * This file is licensed by the GPL version 2.  Works owned by the
5 * Transmission project are granted a special exemption to clause 2(b)
6 * so that the bulk of its code can remain under the MIT license.
7 * This exemption does not extend to derived works not owned by
8 * the Transmission project.
9 */
10
11#include <math.h>
12#include <sys/types.h>
13#include <sys/stat.h>
14#include <unistd.h>
15#include <libgen.h>
16#include <dirent.h>
17#include <stdio.h> /* FILE, snprintf, stderr */
18
19#include "transmission.h"
20#include "internal.h" /* for tr_torrent_t */
21#include "bencode.h"
22#include "makemeta.h"
23#include "platform.h" /* threads, locks */
24#include "shared.h" /* shared lock */
25#include "version.h"
26
27/****
28*****
29****/
30
31struct FileList
32{
33    struct FileList * next;
34    uint64_t size;
35    char * filename;
36};
37
38static struct FileList*
39getFiles( const char        * dir,
40          const char        * base,
41          struct FileList   * list )
42{
43    int i;
44    char buf[MAX_PATH_LENGTH];
45    struct stat sb;
46    DIR * odir = NULL;
47    sb.st_size = 0;
48
49    tr_buildPath( buf, sizeof(buf), dir, base, NULL );
50    i = stat( buf, &sb );
51    if( i ) {
52        tr_err("makemeta couldn't stat \"%s\"; skipping. (%s)", buf, strerror(errno));
53        return list;
54    }
55
56    if ( S_ISDIR( sb.st_mode ) && (( odir = opendir ( buf ) )) )
57    {
58        struct dirent *d;
59        for (d = readdir( odir ); d!=NULL; d=readdir( odir ) )
60            if( d->d_name && d->d_name[0]!='.' ) /* skip dotfiles, ., and .. */
61                list = getFiles( buf, d->d_name, list );
62        closedir( odir );
63    }
64    else if( S_ISREG( sb.st_mode ) )
65    {
66        struct FileList * node = tr_new( struct FileList, 1 );
67        node->size = sb.st_size;
68        node->filename = tr_strdup( buf );
69        node->next = list;
70        list = node;
71    }
72
73    return list;
74}
75
76static int
77bestPieceSize( uint64_t totalSize, int fileCount )
78{
79    const int minPieceSize = 65536; /* arbitrary; 2^16 */
80    const int maxPieceSize = 16777216; /* arbitrary; 2^24 */
81    const int desiredMinPiecesPerFile = 10; /* arbitrary */
82    const unsigned int desiredPieces = fileCount * desiredMinPiecesPerFile;
83
84    /* a good starting range is to have (1024..2048] pieces... */
85    int pieceSize = 1;
86    uint64_t log = totalSize;
87    while( log > 2048 ) {
88        log >>= 1;
89        pieceSize <<= 1;
90    }
91
92    /* now try to have N pieces per average file size...
93       this will reduce overhead on selective downloading
94       and increase swarm speed. */
95    while( ( totalSize / pieceSize ) < desiredPieces )
96        pieceSize >>= 1;
97
98    /* clamp to our desired range to make sure we
99       haven't gone too far astray... */
100    if( pieceSize < minPieceSize ) pieceSize = minPieceSize;
101    if( pieceSize > maxPieceSize ) pieceSize = maxPieceSize;
102
103    return pieceSize;
104}
105
106static int
107builderFileCompare ( const void * va, const void * vb)
108{
109    const tr_metainfo_builder_file_t * a = (const tr_metainfo_builder_file_t*) va;
110    const tr_metainfo_builder_file_t * b = (const tr_metainfo_builder_file_t*) vb;
111    return strcmp( a->filename, b->filename );
112}
113
114tr_metainfo_builder_t*
115tr_metaInfoBuilderCreate( tr_handle_t * handle, const char * topFile )
116{
117    int i;
118    struct FileList * files;
119    struct FileList * walk;
120    tr_metainfo_builder_t * ret = tr_new0( tr_metainfo_builder_t, 1 );
121    ret->top = tr_strdup( topFile );
122    ret->handle = handle; 
123    if (1) {
124        struct stat sb;
125        stat( topFile, &sb );
126        ret->isSingleFile = !S_ISDIR( sb.st_mode );
127    }
128
129    /* build a list of files containing topFile and,
130       if it's a directory, all of its children */
131    if (1) {
132        char *dir, *base;
133        char dirbuf[MAX_PATH_LENGTH];
134        char basebuf[MAX_PATH_LENGTH];
135        strlcpy( dirbuf, topFile, sizeof( dirbuf ) );
136        strlcpy( basebuf, topFile, sizeof( basebuf ) );
137        dir = dirname( dirbuf );
138        base = basename( basebuf );
139        files = getFiles( dir, base, NULL );
140    }
141
142    for( walk=files; walk!=NULL; walk=walk->next )
143        ++ret->fileCount;
144
145    ret->files = tr_new0( tr_metainfo_builder_file_t, ret->fileCount );
146
147    for( i=0, walk=files; walk!=NULL; ++i )
148    {
149        struct FileList * tmp = walk;
150        tr_metainfo_builder_file_t * file = &ret->files[i];
151        walk = walk->next;
152        file->filename = tmp->filename;
153        file->size = tmp->size;
154        ret->totalSize += tmp->size;
155        tr_free( tmp );
156    }
157
158    qsort( ret->files,
159           ret->fileCount,
160           sizeof(tr_metainfo_builder_file_t),
161           builderFileCompare );
162
163    ret->pieceSize = bestPieceSize( ret->totalSize, ret->fileCount );
164    ret->pieceCount = ret->pieceSize
165        ? (int)( ret->totalSize / ret->pieceSize)
166        : 0;
167    if( ret->totalSize % ret->pieceSize )
168        ++ret->pieceCount;
169
170    return ret;
171}
172
173void
174tr_metaInfoBuilderFree( tr_metainfo_builder_t * builder )
175{
176    if( builder != NULL )
177    {
178        int i;
179        for( i=0; i<builder->fileCount; ++i )
180            tr_free( builder->files[i].filename );
181        tr_free( builder->files );
182        tr_free( builder->top );
183        tr_free( builder->comment );
184        tr_free( builder->announce );
185        tr_free( builder->outputFile );
186        tr_free( builder );
187    }
188}
189
190/****
191*****
192****/
193
194static uint8_t*
195getHashInfo ( tr_metainfo_builder_t * b )
196{
197    int fileIndex = 0;
198    uint8_t *ret = tr_new( uint8_t, SHA_DIGEST_LENGTH * b->pieceCount );
199    uint8_t *walk = ret;
200    uint8_t *buf = tr_new( uint8_t, b->pieceSize );
201    uint64_t totalRemain;
202    uint64_t off = 0;
203    FILE * fp;
204
205    b->pieceIndex = 0;
206    totalRemain = b->totalSize;
207    fp = fopen( b->files[fileIndex].filename, "rb" );
208    while ( totalRemain )
209    {
210        uint8_t *bufptr = buf;
211        const uint64_t thisPieceSize =
212            MIN( (uint32_t)b->pieceSize, totalRemain );
213        uint64_t pieceRemain = thisPieceSize;
214
215        assert( b->pieceIndex < b->pieceCount );
216
217        while( pieceRemain )
218        {
219            const uint64_t n_this_pass =
220                MIN( (b->files[fileIndex].size - off), pieceRemain );
221            fread( bufptr, 1, n_this_pass, fp );
222            bufptr += n_this_pass;
223            off += n_this_pass;
224            pieceRemain -= n_this_pass;
225            if( off == b->files[fileIndex].size ) {
226                off = 0;
227                fclose( fp );
228                fp = NULL;
229                if( ++fileIndex < b->fileCount ) {
230                    fp = fopen( b->files[fileIndex].filename, "rb" );
231                }
232            }
233        }
234
235        assert( bufptr-buf == (int)thisPieceSize );
236        assert( pieceRemain == 0 );
237        SHA1( buf, thisPieceSize, walk );
238        walk += SHA_DIGEST_LENGTH;
239
240        if( b->abortFlag ) {
241            b->failed = 1;
242            break;
243        }
244
245        totalRemain -= thisPieceSize;
246        ++b->pieceIndex;
247    }
248    assert( b->abortFlag || (walk-ret == (int)(SHA_DIGEST_LENGTH*b->pieceCount)) );
249    assert( b->abortFlag || !totalRemain );
250    assert( b->abortFlag || fp == NULL );
251
252    if( fp != NULL )
253        fclose( fp );
254
255    tr_free( buf );
256    return ret;
257}
258
259static void
260getFileInfo( const char                        * topFile,
261             const tr_metainfo_builder_file_t  * file,
262             benc_val_t                        * uninitialized_length,
263             benc_val_t                        * uninitialized_path )
264{
265    benc_val_t *sub;
266    const char *pch, *prev;
267    const size_t topLen = strlen(topFile) + 1; /* +1 for '/' */
268    int n;
269
270    /* get the file size */
271    tr_bencInitInt( uninitialized_length, file->size );
272
273    /* the path list */
274    n = 1;
275    for( pch=file->filename+topLen; *pch; ++pch )
276        if (*pch == TR_PATH_DELIMITER)
277            ++n;
278    tr_bencInit( uninitialized_path, TYPE_LIST );
279    tr_bencListReserve( uninitialized_path, n );
280    for( prev=pch=file->filename+topLen; ; ++pch )
281    {
282        char buf[MAX_PATH_LENGTH];
283
284        if (*pch && *pch!=TR_PATH_DELIMITER )
285            continue;
286
287        memcpy( buf, prev, pch-prev );
288        buf[pch-prev] = '\0';
289
290        sub = tr_bencListAdd( uninitialized_path );
291        tr_bencInitStrDup( sub, buf );
292
293        prev = pch + 1;
294        if (!*pch)
295           break;
296    }
297}
298
299static void
300makeFilesList( benc_val_t                 * list,
301               const tr_metainfo_builder_t  * builder )
302{
303    int i = 0;
304
305    tr_bencListReserve( list, builder->fileCount );
306
307    for( i=0; i<builder->fileCount; ++i )
308    {
309        benc_val_t * dict = tr_bencListAdd( list );
310        benc_val_t *length, *pathVal;
311
312        tr_bencInit( dict, TYPE_DICT );
313        tr_bencDictReserve( dict, 2 );
314        length = tr_bencDictAdd( dict, "length" );
315        pathVal = tr_bencDictAdd( dict, "path" );
316        getFileInfo( builder->top, &builder->files[i], length, pathVal );
317    }
318}
319
320static void
321makeInfoDict ( benc_val_t             * dict,
322               tr_metainfo_builder_t  * builder )
323{
324    uint8_t * pch;
325    benc_val_t * val;
326    char base[MAX_PATH_LENGTH];
327
328    tr_bencDictReserve( dict, 5 );
329
330    val = tr_bencDictAdd( dict, "name" );
331    strlcpy( base, builder->top, sizeof( base ) );
332    tr_bencInitStrDup ( val, basename( base ) );
333
334    val = tr_bencDictAdd( dict, "piece length" );
335    tr_bencInitInt( val, builder->pieceSize );
336
337    pch = getHashInfo( builder );
338    val = tr_bencDictAdd( dict, "pieces" );
339    tr_bencInitStr( val, pch, SHA_DIGEST_LENGTH * builder->pieceCount, 0 );
340
341    if ( builder->isSingleFile )
342    {
343        val = tr_bencDictAdd( dict, "length" );
344        tr_bencInitInt( val, builder->files[0].size );
345    }
346    else
347    {
348        val = tr_bencDictAdd( dict, "files" );
349        tr_bencInit( val, TYPE_LIST );
350        makeFilesList( val, builder );
351    }
352
353    val = tr_bencDictAdd( dict, "private" );
354    tr_bencInitInt( val, builder->isPrivate ? 1 : 0 );
355}
356
357static void tr_realMakeMetaInfo ( tr_metainfo_builder_t * builder )
358{
359    int n = 5;
360    benc_val_t top, *val;
361
362    tr_bencInit ( &top, TYPE_DICT );
363    if ( builder->comment && *builder->comment ) ++n;
364    tr_bencDictReserve( &top, n );
365
366        val = tr_bencDictAdd( &top, "announce" );
367        tr_bencInitStrDup( val, builder->announce );
368
369        val = tr_bencDictAdd( &top, "created by" );
370        tr_bencInitStrDup( val, TR_NAME " " VERSION_STRING );
371
372        val = tr_bencDictAdd( &top, "creation date" );
373        tr_bencInitInt( val, time(0) );
374
375        val = tr_bencDictAdd( &top, "encoding" );
376        tr_bencInitStrDup( val, "UTF-8" );
377
378        if( builder->comment && *builder->comment ) {
379            val = tr_bencDictAdd( &top, "comment" );
380            tr_bencInitStrDup( val, builder->comment );
381        }
382
383        val = tr_bencDictAdd( &top, "info" );
384        tr_bencInit( val, TYPE_DICT );
385        tr_bencDictReserve( val, 666 );
386        makeInfoDict( val, builder );
387
388    /* save the file */
389    if ( !builder->abortFlag ) {
390        size_t nmemb;
391        char * pch = tr_bencSaveMalloc( &top, &n );
392        FILE * fp = fopen( builder->outputFile, "wb+" );
393        nmemb = n;
394        if( fp == NULL )
395            builder->failed = 1;
396        else if( fwrite( pch, 1, nmemb, fp ) != nmemb )
397            builder->failed = 1;
398        tr_free( pch );
399        fclose( fp );
400    }
401
402    /* cleanup */
403    tr_bencFree( & top );
404    builder->failed |= builder->abortFlag;
405    builder->isDone = 1;
406}
407
408/***
409****
410****  A threaded builder queue
411****
412***/
413
414static tr_metainfo_builder_t * queue = NULL;
415
416static int workerIsRunning = 0;
417
418static tr_thread_t workerThread;
419
420static tr_lock_t* getQueueLock( tr_handle_t * h )
421{
422    static tr_lock_t * lock = NULL;
423
424    tr_sharedLock( h->shared );
425    if( lock == NULL )
426    {
427        lock = tr_new0( tr_lock_t, 1 );
428        tr_lockInit( lock );
429    }
430    tr_sharedUnlock( h->shared );
431
432    return lock;
433}
434
435static void workerFunc( void * user_data )
436{
437    tr_handle_t * handle = (tr_handle_t *) user_data;
438
439    for (;;)
440    {
441        tr_metainfo_builder_t * builder = NULL;
442
443        /* find the next builder to process */
444        tr_lock_t * lock = getQueueLock ( handle );
445        tr_lockLock( lock );
446        if( queue != NULL ) {
447            builder = queue;
448            queue = queue->nextBuilder;
449        }
450        tr_lockUnlock( lock );
451
452        /* if no builders, this worker thread is done */
453        if( builder == NULL )
454          break;
455
456        tr_realMakeMetaInfo ( builder );
457    }
458
459    workerIsRunning = 0;
460}
461
462void
463tr_makeMetaInfo( tr_metainfo_builder_t  * builder,
464                 const char             * outputFile,
465                 const char             * announce,
466                 const char             * comment,
467                 int                      isPrivate )
468{
469    tr_lock_t * lock;
470
471    builder->abortFlag = 0;
472    builder->isDone = 0;
473    builder->announce = tr_strdup( announce );
474    builder->comment = tr_strdup( comment );
475    builder->isPrivate = isPrivate;
476    if( outputFile && *outputFile )
477        builder->outputFile = tr_strdup( outputFile );
478    else {
479        char out[MAX_PATH_LENGTH];
480        snprintf( out, sizeof(out), "%s.torrent", builder->top);
481        builder->outputFile = tr_strdup( out );
482    }
483
484    /* enqueue the builder */
485    lock = getQueueLock ( builder->handle );
486    tr_lockLock( lock );
487    builder->nextBuilder = queue;
488    queue = builder;
489    if( !workerIsRunning ) {
490        workerIsRunning = 1;
491        tr_threadCreate( &workerThread, workerFunc, builder->handle, "makeMeta" );
492    }
493    tr_lockUnlock( lock );
494}
495
Note: See TracBrowser for help on using the repository browser.