source: branches/file_selection/libtransmission/makemeta.c @ 2143

Last change on this file since 2143 was 2143, checked in by charles, 15 years ago
  • fix 'piece size is always 1MB on 32 bit machines' bug.
  • fix label typo in gtk+ client
File size: 12.7 KB
Line 
1/*
2 * This file Copyright (C) 2007 Charles Kerr <charles@rebelbase.com>
3 *
4 * This file is licensed by the GPL version 2.  Works owned by the
5 * Transmission project are granted a special exemption to clause 2(b)
6 * so that the bulk of its code can remain under the MIT license.
7 * This exemption does not extend to derived works not owned by
8 * the Transmission project.
9 */
10
11#include <sys/types.h>
12#include <sys/stat.h>
13#include <unistd.h>
14#include <libgen.h>
15#include <dirent.h>
16#include <stdio.h> /* FILE, snprintf, stderr */
17#include <stdlib.h> /* malloc, calloc */
18
19#include "transmission.h"
20#include "internal.h" /* for tr_torrent_t */
21#include "bencode.h"
22#include "makemeta.h"
23#include "platform.h" /* threads, locks */
24#include "shared.h" /* shared lock */
25#include "version.h"
26
27/****
28*****
29****/
30
31struct FileList
32{
33    struct FileList * next;
34    char filename[MAX_PATH_LENGTH];
35};
36
37static struct FileList*
38getFiles( const char        * dir,
39          const char        * base,
40          struct FileList   * list )
41{
42    int i;
43    char buf[MAX_PATH_LENGTH];
44    struct stat sb;
45    DIR * odir = NULL;
46    sb.st_size = 0;
47
48    snprintf( buf, sizeof(buf), "%s"TR_PATH_DELIMITER_STR"%s", dir, base );
49    i = stat( buf, &sb );
50    if( i ) {
51        tr_err("makemeta couldn't stat \"%s\"; skipping. (%s)", buf, strerror(errno));
52        return list;
53    }
54
55    if ( S_ISDIR( sb.st_mode ) && (( odir = opendir ( buf ) )) )
56    {
57        struct dirent *d;
58        for (d = readdir( odir ); d!=NULL; d=readdir( odir ) )
59            if( strcmp( d->d_name,"." ) && strcmp( d->d_name,".." ) )
60                list = getFiles( buf, d->d_name, list );
61        closedir( odir );
62    }
63    else if( S_ISREG( sb.st_mode ) )
64    {
65        struct FileList * node = malloc( sizeof( struct FileList ) );
66        snprintf( node->filename, sizeof( node->filename ), "%s", buf );
67        node->next = list;
68        list = node;
69    }
70
71    return list;
72}
73
74static void
75freeFileList( struct FileList * list )
76{
77    while( list ) {
78        struct FileList * tmp = list->next;
79        free( list );
80        list = tmp;
81    }
82}
83
84static off_t
85getFileSize ( const char * filename )
86{
87    struct stat sb;
88    sb.st_size = 0;
89    stat( filename, &sb );
90    return sb.st_size;
91}
92
93#define MiB 1048576ul
94#define GiB 1073741824ul
95
96static int
97bestPieceSize( uint64_t totalSize )
98{
99    /* almost always best to have a piee size of 512 or 256 kb.
100       common practice seems to be to bump up to 1MB pieces at
101       at total size of around 8GiB or so */
102
103    if (totalSize >= (8 * GiB) )
104        return MiB;
105
106    if (totalSize >= GiB )
107        return MiB / 2;
108
109    return MiB / 4;
110}
111
112/****
113*****
114****/
115
116static int pstrcmp( const void * va, const void * vb)
117{
118    const char * a = *(const char**) va;
119    const char * b = *(const char**) vb;
120    return strcmp( a, b );
121}
122
123tr_metainfo_builder_t*
124tr_metaInfoBuilderCreate( tr_handle_t * handle, const char * topFile )
125{
126    int i;
127    struct FileList * files;
128    const struct FileList * walk;
129    tr_metainfo_builder_t * ret = calloc( 1, sizeof(tr_metainfo_builder_t) );
130    ret->top = tr_strdup( topFile );
131    ret->handle = handle;
132
133    if (1) {
134        struct stat sb;
135        stat( topFile, &sb );
136        ret->isSingleFile = !S_ISDIR( sb.st_mode );
137    }
138
139    /* build a list of files containing topFile and,
140       if it's a directory, all of its children */
141    if (1) {
142        char *dir, *base;
143        char dirbuf[MAX_PATH_LENGTH];
144        char basebuf[MAX_PATH_LENGTH];
145        strlcpy( dirbuf, topFile, sizeof( dirbuf ) );
146        strlcpy( basebuf, topFile, sizeof( basebuf ) );
147        dir = dirname( dirbuf );
148        base = basename( basebuf );
149        files = getFiles( dir, base, NULL );
150    }
151
152    for( walk=files; walk!=NULL; walk=walk->next )
153        ++ret->fileCount;
154    ret->files = calloc( ret->fileCount, sizeof(char*) );
155    ret->fileLengths = calloc( ret->fileCount, sizeof(uint64_t) );
156
157    for( i=0, walk=files; walk!=NULL; walk=walk->next, ++i )
158        ret->files[i] = tr_strdup( walk->filename );
159
160    qsort( ret->files, ret->fileCount, sizeof(char*), pstrcmp );
161    for( i=0; i<ret->fileCount; ++i ) {
162        ret->fileLengths[i] = getFileSize( ret->files[i] );
163        ret->totalSize += ret->fileLengths[i];
164    }
165
166    freeFileList( files );
167   
168    ret->pieceSize = bestPieceSize( ret->totalSize );
169    ret->pieceCount = (int)( ret->totalSize / ret->pieceSize);
170    if( ret->totalSize % ret->pieceSize )
171        ++ret->pieceCount;
172
173    return ret;
174}
175
176void
177tr_metaInfoBuilderFree( tr_metainfo_builder_t * builder )
178{
179    if( builder != NULL )
180    {
181        int i;
182        for( i=0; i<builder->fileCount; ++i )
183            tr_free( builder->files[i] );
184        tr_free( builder->files );
185        tr_free( builder->fileLengths );
186        tr_free( builder->top );
187        tr_free( builder->comment );
188        tr_free( builder->announce );
189        tr_free( builder->outputFile );
190        tr_free( builder );
191    }
192}
193
194/****
195*****
196****/
197
198static uint8_t*
199getHashInfo ( tr_metainfo_builder_t * b )
200{
201    int fileIndex = 0;
202    uint8_t *ret = (uint8_t*) malloc ( SHA_DIGEST_LENGTH * b->pieceCount );
203    uint8_t *walk = ret;
204    uint8_t *buf = malloc( b->pieceSize );
205    uint64_t totalRemain;
206    uint64_t off = 0;
207    FILE * fp;
208
209    b->pieceIndex = 0;
210    totalRemain = b->totalSize;
211    fp = fopen( b->files[fileIndex], "rb" );
212    while ( totalRemain )
213    {
214        uint8_t *bufptr = buf;
215        const uint64_t thisPieceSize = MIN( (uint32_t)b->pieceSize, totalRemain );
216        uint64_t pieceRemain = thisPieceSize;
217
218        assert( b->pieceIndex < b->pieceCount );
219
220        while( pieceRemain )
221        {
222            const uint64_t n_this_pass = MIN( (b->fileLengths[fileIndex] - off), pieceRemain );
223            fread( bufptr, 1, n_this_pass, fp );
224            bufptr += n_this_pass;
225            off += n_this_pass;
226            pieceRemain -= n_this_pass;
227            if( off == b->fileLengths[fileIndex] ) {
228                off = 0;
229                fclose( fp );
230                fp = NULL;
231                if( ++fileIndex < b->fileCount ) {
232                    fp = fopen( b->files[fileIndex], "rb" );
233                }
234            }
235        }
236
237        assert( bufptr-buf == (int)thisPieceSize );
238        assert( pieceRemain == 0 );
239        SHA1( buf, thisPieceSize, walk );
240        walk += SHA_DIGEST_LENGTH;
241
242        if( b->abortFlag ) {
243            b->failed = 1;
244            break;
245        }
246
247        totalRemain -= thisPieceSize;
248        ++b->pieceIndex;
249    }
250    assert( b->abortFlag || (walk-ret == (int)(SHA_DIGEST_LENGTH*b->pieceCount)) );
251    assert( b->abortFlag || !totalRemain );
252    assert( b->abortFlag || fp == NULL );
253
254    if( fp != NULL )
255        fclose( fp );
256
257    free( buf );
258    return ret;
259}
260
261static void
262getFileInfo( const char * topFile,
263             const char * filename,
264             benc_val_t * uninitialized_length,
265             benc_val_t * uninitialized_path )
266{
267    benc_val_t *sub;
268    const char *pch, *prev;
269    const size_t topLen = strlen(topFile) + 1; /* +1 for '/' */
270    int n;
271
272    /* get the file size */
273    tr_bencInitInt( uninitialized_length, getFileSize(filename) );
274
275    /* the path list */
276    n = 1;
277    for( pch=filename+topLen; *pch; ++pch )
278        if (*pch == TR_PATH_DELIMITER)
279            ++n;
280    tr_bencInit( uninitialized_path, TYPE_LIST );
281    tr_bencListReserve( uninitialized_path, n );
282    for( prev=pch=filename+topLen; ; ++pch )
283    {
284        char buf[MAX_PATH_LENGTH];
285
286        if (*pch && *pch!=TR_PATH_DELIMITER )
287            continue;
288
289        memcpy( buf, prev, pch-prev );
290        buf[pch-prev] = '\0';
291
292        sub = tr_bencListAdd( uninitialized_path );
293        tr_bencInitStrDup( sub, buf );
294
295        prev = pch + 1;
296        if (!*pch)
297           break;
298    }
299}
300
301static void
302makeFilesList( benc_val_t                 * list,
303               const tr_metainfo_builder_t  * builder )
304{
305    int i = 0;
306
307    tr_bencListReserve( list, builder->fileCount );
308
309    for( i=0; i<builder->fileCount; ++i )
310    {
311        benc_val_t * dict = tr_bencListAdd( list );
312        benc_val_t *length, *pathVal;
313
314        tr_bencInit( dict, TYPE_DICT );
315        tr_bencDictReserve( dict, 2 );
316        length = tr_bencDictAdd( dict, "length" );
317        pathVal = tr_bencDictAdd( dict, "path" );
318        getFileInfo( builder->top, builder->files[i], length, pathVal );
319    }
320}
321
322static void
323makeInfoDict ( benc_val_t             * dict,
324               tr_metainfo_builder_t  * builder )
325{
326    uint8_t * pch;
327    benc_val_t * val;
328    char base[MAX_PATH_LENGTH];
329
330    tr_bencDictReserve( dict, 5 );
331
332    val = tr_bencDictAdd( dict, "name" );
333    strlcpy( base, builder->top, sizeof( base ) );
334    tr_bencInitStrDup ( val, basename( base ) );
335
336    val = tr_bencDictAdd( dict, "piece length" );
337    tr_bencInitInt( val, builder->pieceSize );
338
339    pch = getHashInfo( builder );
340    val = tr_bencDictAdd( dict, "pieces" );
341    tr_bencInitStr( val, pch, SHA_DIGEST_LENGTH * builder->pieceCount, 0 );
342
343    if ( builder->isSingleFile )
344    {
345        val = tr_bencDictAdd( dict, "length" );
346        tr_bencInitInt( val, builder->fileLengths[0] );
347    }
348    else
349    {
350        val = tr_bencDictAdd( dict, "files" );
351        tr_bencInit( val, TYPE_LIST );
352        makeFilesList( val, builder );
353    }
354
355    val = tr_bencDictAdd( dict, "private" );
356    tr_bencInitInt( val, builder->isPrivate ? 1 : 0 );
357}
358
359static void tr_realMakeMetaInfo ( tr_metainfo_builder_t * builder )
360{
361    int n = 5;
362    benc_val_t top, *val;
363
364    tr_bencInit ( &top, TYPE_DICT );
365    if ( builder->comment && *builder->comment ) ++n;
366    tr_bencDictReserve( &top, n );
367
368        val = tr_bencDictAdd( &top, "announce" );
369        tr_bencInitStrDup( val, builder->announce );
370
371        val = tr_bencDictAdd( &top, "created by" );
372        tr_bencInitStrDup( val, TR_NAME " " VERSION_STRING );
373
374        val = tr_bencDictAdd( &top, "creation date" );
375        tr_bencInitInt( val, time(0) );
376
377        val = tr_bencDictAdd( &top, "encoding" );
378        tr_bencInitStrDup( val, "UTF-8" );
379
380        if( builder->comment && *builder->comment ) {
381            val = tr_bencDictAdd( &top, "comment" );
382            tr_bencInitStrDup( val, builder->comment );
383        }
384
385        val = tr_bencDictAdd( &top, "info" );
386        tr_bencInit( val, TYPE_DICT );
387        tr_bencDictReserve( val, 666 );
388        makeInfoDict( val, builder );
389
390    /* save the file */
391    if ( !builder->abortFlag ) {
392        size_t nmemb;
393        char * pch = tr_bencSaveMalloc( &top, &n );
394        FILE * fp = fopen( builder->outputFile, "wb+" );
395        nmemb = n;
396        if( fp == NULL )
397            builder->failed = 1;
398        else if( fwrite( pch, 1, nmemb, fp ) != nmemb )
399            builder->failed = 1;
400        free( pch );
401        fclose( fp );
402    }
403
404    /* cleanup */
405    tr_bencFree( & top );
406    builder->failed |= builder->abortFlag;
407    builder->isDone = 1;
408}
409
410/***
411****
412****  A threaded builder queue
413****
414***/
415
416static tr_metainfo_builder_t * queue = NULL;
417
418static int workerIsRunning = 0;
419
420static tr_thread_t workerThread;
421
422static tr_lock_t* getQueueLock( tr_handle_t * h )
423{
424    static tr_lock_t * lock = NULL;
425
426    tr_sharedLock( h->shared );
427    if( lock == NULL )
428    {
429        lock = calloc( 1, sizeof( tr_lock_t ) );
430        tr_lockInit( lock );
431    }
432    tr_sharedUnlock( h->shared );
433
434    return lock;
435}
436
437static void workerFunc( void * user_data )
438{
439    tr_handle_t * handle = (tr_handle_t *) user_data;
440
441    for (;;)
442    {
443        tr_metainfo_builder_t * builder = NULL;
444
445        /* find the next builder to process */
446        tr_lock_t * lock = getQueueLock ( handle );
447        tr_lockLock( lock );
448        if( queue != NULL ) {
449            builder = queue;
450            queue = queue->nextBuilder;
451        }
452        tr_lockUnlock( lock );
453
454        /* if no builders, this worker thread is done */
455        if( builder == NULL )
456          break;
457
458        tr_realMakeMetaInfo ( builder );
459    }
460
461    workerIsRunning = 0;
462}
463
464void
465tr_makeMetaInfo( tr_metainfo_builder_t  * builder,
466                 const char             * outputFile,
467                 const char             * announce,
468                 const char             * comment,
469                 int                      isPrivate )
470{
471    tr_lock_t * lock;
472
473    builder->abortFlag = 0;
474    builder->isDone = 0;
475    builder->announce = tr_strdup( announce );
476    builder->comment = tr_strdup( comment );
477    builder->isPrivate = isPrivate;
478    if( outputFile && *outputFile )
479        builder->outputFile = tr_strdup( outputFile );
480    else {
481        char out[MAX_PATH_LENGTH];
482        snprintf( out, sizeof(out), "%s.torrent", builder->top);
483        builder->outputFile = tr_strdup( out );
484    }
485
486    /* enqueue the builder */
487    lock = getQueueLock ( builder->handle );
488    tr_lockLock( lock );
489    builder->nextBuilder = queue;
490    queue = builder;
491    if( !workerIsRunning ) {
492        workerIsRunning = 1;
493        tr_threadCreate( &workerThread, workerFunc, builder->handle, "makeMeta" );
494    }
495    tr_lockUnlock( lock );
496}
497
Note: See TracBrowser for help on using the repository browser.