source: trunk/libtransmission/makemeta.c @ 2206

Last change on this file since 2206 was 2206, checked in by charles, 15 years ago

add tr_torrentCanAdd() as per BentMyWookie?'s request... clear tracker error string when restarting... fix r2202 "completed" announce bug.

File size: 13.2 KB
Line 
1/*
2 * This file Copyright (C) 2007 Charles Kerr <charles@rebelbase.com>
3 *
4 * This file is licensed by the GPL version 2.  Works owned by the
5 * Transmission project are granted a special exemption to clause 2(b)
6 * so that the bulk of its code can remain under the MIT license.
7 * This exemption does not extend to derived works not owned by
8 * the Transmission project.
9 */
10
11#include <math.h>
12#include <sys/types.h>
13#include <sys/stat.h>
14#include <unistd.h>
15#include <libgen.h>
16#include <dirent.h>
17#include <stdio.h> /* FILE, snprintf, stderr */
18
19#include "transmission.h"
20#include "internal.h" /* for tr_torrent_t */
21#include "bencode.h"
22#include "makemeta.h"
23#include "platform.h" /* threads, locks */
24#include "shared.h" /* shared lock */
25#include "version.h"
26
27/****
28*****
29****/
30
31struct FileList
32{
33    struct FileList * next;
34    uint64_t size;
35    char * filename;
36};
37
38static struct FileList*
39getFiles( const char        * dir,
40          const char        * base,
41          struct FileList   * list )
42{
43    int i;
44    char buf[MAX_PATH_LENGTH];
45    struct stat sb;
46    DIR * odir = NULL;
47    sb.st_size = 0;
48
49    tr_buildPath( buf, sizeof(buf), dir, base, NULL );
50    i = stat( buf, &sb );
51    if( i ) {
52        tr_err("makemeta couldn't stat \"%s\"; skipping. (%s)", buf, strerror(errno));
53        return list;
54    }
55
56    if ( S_ISDIR( sb.st_mode ) && (( odir = opendir ( buf ) )) )
57    {
58        struct dirent *d;
59        for (d = readdir( odir ); d!=NULL; d=readdir( odir ) )
60            if( d->d_name && d->d_name[0]!='.' ) /* skip dotfiles, ., and .. */
61                list = getFiles( buf, d->d_name, list );
62        closedir( odir );
63    }
64    else if( S_ISREG( sb.st_mode ) )
65    {
66        struct FileList * node = tr_new( struct FileList, 1 );
67        node->size = sb.st_size;
68        node->filename = tr_strdup( buf );
69        node->next = list;
70        list = node;
71    }
72
73    return list;
74}
75
76static int
77bestPieceSize( uint64_t totalSize, int fileCount )
78{
79    const int minPieceSize = 65536; /* arbitrary; 2^16 */
80    const int maxPieceSize = 16777216; /* arbitrary; 2^24 */
81    const int desiredMinPiecesPerFile = 10; /* arbitrary */
82    const unsigned int desiredPieces = fileCount * desiredMinPiecesPerFile;
83
84    /* a good starting range is to have (1024..2048] pieces... */
85    int pieceSize = 1;
86    uint64_t log = totalSize;
87    while( log > 2048 ) {
88        log >>= 1;
89        pieceSize <<= 1;
90    }
91
92    /* now try to have N pieces per average file size...
93       this will reduce overhead on selective downloading
94       and increase swarm speed. */
95    while( ( totalSize / pieceSize ) < desiredPieces )
96        pieceSize >>= 1;
97
98    /* clamp to our desired range to make sure we
99       haven't gone too far astray... */
100    if( pieceSize < minPieceSize ) pieceSize = minPieceSize;
101    if( pieceSize > maxPieceSize ) pieceSize = maxPieceSize;
102
103    return pieceSize;
104}
105
106static int
107builderFileCompare ( const void * va, const void * vb)
108{
109    const tr_metainfo_builder_file_t * a = (const tr_metainfo_builder_file_t*) va;
110    const tr_metainfo_builder_file_t * b = (const tr_metainfo_builder_file_t*) vb;
111    return strcmp( a->filename, b->filename );
112}
113
114tr_metainfo_builder_t*
115tr_metaInfoBuilderCreate( tr_handle_t * handle, const char * topFile )
116{
117    int i;
118    struct FileList * files;
119    struct FileList * walk;
120    tr_metainfo_builder_t * ret = tr_new0( tr_metainfo_builder_t, 1 );
121    ret->top = tr_strdup( topFile );
122    ret->handle = handle; 
123    if (1) {
124        struct stat sb;
125        stat( topFile, &sb );
126        ret->isSingleFile = !S_ISDIR( sb.st_mode );
127    }
128
129    /* build a list of files containing topFile and,
130       if it's a directory, all of its children */
131    if (1) {
132        char *dir, *base;
133        char dirbuf[MAX_PATH_LENGTH];
134        char basebuf[MAX_PATH_LENGTH];
135        strlcpy( dirbuf, topFile, sizeof( dirbuf ) );
136        strlcpy( basebuf, topFile, sizeof( basebuf ) );
137        dir = dirname( dirbuf );
138        base = basename( basebuf );
139        files = getFiles( dir, base, NULL );
140    }
141
142    for( walk=files; walk!=NULL; walk=walk->next )
143        ++ret->fileCount;
144
145    ret->files = tr_new0( tr_metainfo_builder_file_t, ret->fileCount );
146
147    for( i=0, walk=files; walk!=NULL; ++i )
148    {
149        struct FileList * tmp = walk;
150        tr_metainfo_builder_file_t * file = &ret->files[i];
151        walk = walk->next;
152        file->filename = tmp->filename;
153        file->size = tmp->size;
154        ret->totalSize += tmp->size;
155        tr_free( tmp );
156    }
157
158    qsort( ret->files,
159           ret->fileCount,
160           sizeof(tr_metainfo_builder_file_t),
161           builderFileCompare );
162
163    ret->pieceSize = bestPieceSize( ret->totalSize, ret->fileCount );
164    ret->pieceCount = (int)( ret->totalSize / ret->pieceSize);
165    if( ret->totalSize % ret->pieceSize )
166        ++ret->pieceCount;
167
168    return ret;
169}
170
171void
172tr_metaInfoBuilderFree( tr_metainfo_builder_t * builder )
173{
174    if( builder != NULL )
175    {
176        int i;
177        for( i=0; i<builder->fileCount; ++i )
178            tr_free( builder->files[i].filename );
179        tr_free( builder->files );
180        tr_free( builder->top );
181        tr_free( builder->comment );
182        tr_free( builder->announce );
183        tr_free( builder->outputFile );
184        tr_free( builder );
185    }
186}
187
188/****
189*****
190****/
191
192static uint8_t*
193getHashInfo ( tr_metainfo_builder_t * b )
194{
195    int fileIndex = 0;
196    uint8_t *ret = tr_new( uint8_t, SHA_DIGEST_LENGTH * b->pieceCount );
197    uint8_t *walk = ret;
198    uint8_t *buf = tr_new( uint8_t, b->pieceSize );
199    uint64_t totalRemain;
200    uint64_t off = 0;
201    FILE * fp;
202
203    b->pieceIndex = 0;
204    totalRemain = b->totalSize;
205    fp = fopen( b->files[fileIndex].filename, "rb" );
206    while ( totalRemain )
207    {
208        uint8_t *bufptr = buf;
209        const uint64_t thisPieceSize =
210            MIN( (uint32_t)b->pieceSize, totalRemain );
211        uint64_t pieceRemain = thisPieceSize;
212
213        assert( b->pieceIndex < b->pieceCount );
214
215        while( pieceRemain )
216        {
217            const uint64_t n_this_pass =
218                MIN( (b->files[fileIndex].size - off), pieceRemain );
219            fread( bufptr, 1, n_this_pass, fp );
220            bufptr += n_this_pass;
221            off += n_this_pass;
222            pieceRemain -= n_this_pass;
223            if( off == b->files[fileIndex].size ) {
224                off = 0;
225                fclose( fp );
226                fp = NULL;
227                if( ++fileIndex < b->fileCount ) {
228                    fp = fopen( b->files[fileIndex].filename, "rb" );
229                }
230            }
231        }
232
233        assert( bufptr-buf == (int)thisPieceSize );
234        assert( pieceRemain == 0 );
235        SHA1( buf, thisPieceSize, walk );
236        walk += SHA_DIGEST_LENGTH;
237
238        if( b->abortFlag ) {
239            b->failed = 1;
240            break;
241        }
242
243        totalRemain -= thisPieceSize;
244        ++b->pieceIndex;
245    }
246    assert( b->abortFlag || (walk-ret == (int)(SHA_DIGEST_LENGTH*b->pieceCount)) );
247    assert( b->abortFlag || !totalRemain );
248    assert( b->abortFlag || fp == NULL );
249
250    if( fp != NULL )
251        fclose( fp );
252
253    tr_free( buf );
254    return ret;
255}
256
257static void
258getFileInfo( const char                        * topFile,
259             const tr_metainfo_builder_file_t  * file,
260             benc_val_t                        * uninitialized_length,
261             benc_val_t                        * uninitialized_path )
262{
263    benc_val_t *sub;
264    const char *pch, *prev;
265    const size_t topLen = strlen(topFile) + 1; /* +1 for '/' */
266    int n;
267
268    /* get the file size */
269    tr_bencInitInt( uninitialized_length, file->size );
270
271    /* the path list */
272    n = 1;
273    for( pch=file->filename+topLen; *pch; ++pch )
274        if (*pch == TR_PATH_DELIMITER)
275            ++n;
276    tr_bencInit( uninitialized_path, TYPE_LIST );
277    tr_bencListReserve( uninitialized_path, n );
278    for( prev=pch=file->filename+topLen; ; ++pch )
279    {
280        char buf[MAX_PATH_LENGTH];
281
282        if (*pch && *pch!=TR_PATH_DELIMITER )
283            continue;
284
285        memcpy( buf, prev, pch-prev );
286        buf[pch-prev] = '\0';
287
288        sub = tr_bencListAdd( uninitialized_path );
289        tr_bencInitStrDup( sub, buf );
290
291        prev = pch + 1;
292        if (!*pch)
293           break;
294    }
295}
296
297static void
298makeFilesList( benc_val_t                 * list,
299               const tr_metainfo_builder_t  * builder )
300{
301    int i = 0;
302
303    tr_bencListReserve( list, builder->fileCount );
304
305    for( i=0; i<builder->fileCount; ++i )
306    {
307        benc_val_t * dict = tr_bencListAdd( list );
308        benc_val_t *length, *pathVal;
309
310        tr_bencInit( dict, TYPE_DICT );
311        tr_bencDictReserve( dict, 2 );
312        length = tr_bencDictAdd( dict, "length" );
313        pathVal = tr_bencDictAdd( dict, "path" );
314        getFileInfo( builder->top, &builder->files[i], length, pathVal );
315    }
316}
317
318static void
319makeInfoDict ( benc_val_t             * dict,
320               tr_metainfo_builder_t  * builder )
321{
322    uint8_t * pch;
323    benc_val_t * val;
324    char base[MAX_PATH_LENGTH];
325
326    tr_bencDictReserve( dict, 5 );
327
328    val = tr_bencDictAdd( dict, "name" );
329    strlcpy( base, builder->top, sizeof( base ) );
330    tr_bencInitStrDup ( val, basename( base ) );
331
332    val = tr_bencDictAdd( dict, "piece length" );
333    tr_bencInitInt( val, builder->pieceSize );
334
335    pch = getHashInfo( builder );
336    val = tr_bencDictAdd( dict, "pieces" );
337    tr_bencInitStr( val, pch, SHA_DIGEST_LENGTH * builder->pieceCount, 0 );
338
339    if ( builder->isSingleFile )
340    {
341        val = tr_bencDictAdd( dict, "length" );
342        tr_bencInitInt( val, builder->files[0].size );
343    }
344    else
345    {
346        val = tr_bencDictAdd( dict, "files" );
347        tr_bencInit( val, TYPE_LIST );
348        makeFilesList( val, builder );
349    }
350
351    val = tr_bencDictAdd( dict, "private" );
352    tr_bencInitInt( val, builder->isPrivate ? 1 : 0 );
353}
354
355static void tr_realMakeMetaInfo ( tr_metainfo_builder_t * builder )
356{
357    int n = 5;
358    benc_val_t top, *val;
359
360    tr_bencInit ( &top, TYPE_DICT );
361    if ( builder->comment && *builder->comment ) ++n;
362    tr_bencDictReserve( &top, n );
363
364        val = tr_bencDictAdd( &top, "announce" );
365        tr_bencInitStrDup( val, builder->announce );
366
367        val = tr_bencDictAdd( &top, "created by" );
368        tr_bencInitStrDup( val, TR_NAME " " VERSION_STRING );
369
370        val = tr_bencDictAdd( &top, "creation date" );
371        tr_bencInitInt( val, time(0) );
372
373        val = tr_bencDictAdd( &top, "encoding" );
374        tr_bencInitStrDup( val, "UTF-8" );
375
376        if( builder->comment && *builder->comment ) {
377            val = tr_bencDictAdd( &top, "comment" );
378            tr_bencInitStrDup( val, builder->comment );
379        }
380
381        val = tr_bencDictAdd( &top, "info" );
382        tr_bencInit( val, TYPE_DICT );
383        tr_bencDictReserve( val, 666 );
384        makeInfoDict( val, builder );
385
386    /* save the file */
387    if ( !builder->abortFlag ) {
388        size_t nmemb;
389        char * pch = tr_bencSaveMalloc( &top, &n );
390        FILE * fp = fopen( builder->outputFile, "wb+" );
391        nmemb = n;
392        if( fp == NULL )
393            builder->failed = 1;
394        else if( fwrite( pch, 1, nmemb, fp ) != nmemb )
395            builder->failed = 1;
396        tr_free( pch );
397        fclose( fp );
398    }
399
400    /* cleanup */
401    tr_bencFree( & top );
402    builder->failed |= builder->abortFlag;
403    builder->isDone = 1;
404}
405
406/***
407****
408****  A threaded builder queue
409****
410***/
411
412static tr_metainfo_builder_t * queue = NULL;
413
414static int workerIsRunning = 0;
415
416static tr_thread_t workerThread;
417
418static tr_lock_t* getQueueLock( tr_handle_t * h )
419{
420    static tr_lock_t * lock = NULL;
421
422    tr_sharedLock( h->shared );
423    if( lock == NULL )
424    {
425        lock = tr_new0( tr_lock_t, 1 );
426        tr_lockInit( lock );
427    }
428    tr_sharedUnlock( h->shared );
429
430    return lock;
431}
432
433static void workerFunc( void * user_data )
434{
435    tr_handle_t * handle = (tr_handle_t *) user_data;
436
437    for (;;)
438    {
439        tr_metainfo_builder_t * builder = NULL;
440
441        /* find the next builder to process */
442        tr_lock_t * lock = getQueueLock ( handle );
443        tr_lockLock( lock );
444        if( queue != NULL ) {
445            builder = queue;
446            queue = queue->nextBuilder;
447        }
448        tr_lockUnlock( lock );
449
450        /* if no builders, this worker thread is done */
451        if( builder == NULL )
452          break;
453
454        tr_realMakeMetaInfo ( builder );
455    }
456
457    workerIsRunning = 0;
458}
459
460void
461tr_makeMetaInfo( tr_metainfo_builder_t  * builder,
462                 const char             * outputFile,
463                 const char             * announce,
464                 const char             * comment,
465                 int                      isPrivate )
466{
467    tr_lock_t * lock;
468
469    builder->abortFlag = 0;
470    builder->isDone = 0;
471    builder->announce = tr_strdup( announce );
472    builder->comment = tr_strdup( comment );
473    builder->isPrivate = isPrivate;
474    if( outputFile && *outputFile )
475        builder->outputFile = tr_strdup( outputFile );
476    else {
477        char out[MAX_PATH_LENGTH];
478        snprintf( out, sizeof(out), "%s.torrent", builder->top);
479        builder->outputFile = tr_strdup( out );
480    }
481
482    /* enqueue the builder */
483    lock = getQueueLock ( builder->handle );
484    tr_lockLock( lock );
485    builder->nextBuilder = queue;
486    queue = builder;
487    if( !workerIsRunning ) {
488        workerIsRunning = 1;
489        tr_threadCreate( &workerThread, workerFunc, builder->handle, "makeMeta" );
490    }
491    tr_lockUnlock( lock );
492}
493
Note: See TracBrowser for help on using the repository browser.