source: trunk/libtransmission/makemeta.c @ 2154

Last change on this file since 2154 was 2154, checked in by charles, 15 years ago
  • fix error checking large files reported by Gimp_
  • portability changes to pathname/filename building
  • small gratuitous changes
File size: 12.7 KB
Line 
1/*
2 * This file Copyright (C) 2007 Charles Kerr <charles@rebelbase.com>
3 *
4 * This file is licensed by the GPL version 2.  Works owned by the
5 * Transmission project are granted a special exemption to clause 2(b)
6 * so that the bulk of its code can remain under the MIT license.
7 * This exemption does not extend to derived works not owned by
8 * the Transmission project.
9 */
10
11#include <sys/types.h>
12#include <sys/stat.h>
13#include <unistd.h>
14#include <libgen.h>
15#include <dirent.h>
16#include <stdio.h> /* FILE, snprintf, stderr */
17#include <stdlib.h> /* malloc, calloc */
18
19#include "transmission.h"
20#include "internal.h" /* for tr_torrent_t */
21#include "bencode.h"
22#include "makemeta.h"
23#include "platform.h" /* threads, locks */
24#include "shared.h" /* shared lock */
25#include "version.h"
26
27/****
28*****
29****/
30
31struct FileList
32{
33    struct FileList * next;
34    char filename[MAX_PATH_LENGTH];
35};
36
37static struct FileList*
38getFiles( const char        * dir,
39          const char        * base,
40          struct FileList   * list )
41{
42    int i;
43    char buf[MAX_PATH_LENGTH];
44    struct stat sb;
45    DIR * odir = NULL;
46    sb.st_size = 0;
47
48    tr_buildPath( buf, sizeof(buf), dir, base, NULL );
49    i = stat( buf, &sb );
50    if( i ) {
51        tr_err("makemeta couldn't stat \"%s\"; skipping. (%s)", buf, strerror(errno));
52        return list;
53    }
54
55    if ( S_ISDIR( sb.st_mode ) && (( odir = opendir ( buf ) )) )
56    {
57        struct dirent *d;
58        for (d = readdir( odir ); d!=NULL; d=readdir( odir ) )
59            if( strcmp( d->d_name,"." ) && strcmp( d->d_name,".." ) )
60                list = getFiles( buf, d->d_name, list );
61        closedir( odir );
62    }
63    else if( S_ISREG( sb.st_mode ) )
64    {
65        struct FileList * node = malloc( sizeof( struct FileList ) );
66        snprintf( node->filename, sizeof( node->filename ), "%s", buf );
67        node->next = list;
68        list = node;
69    }
70
71    return list;
72}
73
74static void
75freeFileList( struct FileList * list )
76{
77    while( list ) {
78        struct FileList * tmp = list->next;
79        free( list );
80        list = tmp;
81    }
82}
83
84static off_t
85getFileSize ( const char * filename )
86{
87    struct stat sb;
88    sb.st_size = 0;
89    stat( filename, &sb );
90    return sb.st_size;
91}
92
93
94static int
95bestPieceSize( uint64_t totalSize )
96{
97    const int MiB = 1048576;
98    const int GiB = totalSize / (uint64_t)1073741824;
99
100    /* almost always best to have a piee size of 512 or 256 kb.
101       common practice seems to be to bump up to 1MB pieces at
102       at total size of around 8GiB or so */
103
104    if( GiB >= 8 )
105        return MiB;
106
107    if( GiB >= 1 )
108        return MiB / 2;
109
110    return MiB / 4;
111}
112
113/****
114*****
115****/
116
117static int pstrcmp( const void * va, const void * vb)
118{
119    const char * a = *(const char**) va;
120    const char * b = *(const char**) vb;
121    return strcmp( a, b );
122}
123
124tr_metainfo_builder_t*
125tr_metaInfoBuilderCreate( tr_handle_t * handle, const char * topFile )
126{
127    int i;
128    struct FileList * files;
129    const struct FileList * walk;
130    tr_metainfo_builder_t * ret = calloc( 1, sizeof(tr_metainfo_builder_t) );
131    ret->top = tr_strdup( topFile );
132    ret->handle = handle;
133
134    if (1) {
135        struct stat sb;
136        stat( topFile, &sb );
137        ret->isSingleFile = !S_ISDIR( sb.st_mode );
138    }
139
140    /* build a list of files containing topFile and,
141       if it's a directory, all of its children */
142    if (1) {
143        char *dir, *base;
144        char dirbuf[MAX_PATH_LENGTH];
145        char basebuf[MAX_PATH_LENGTH];
146        strlcpy( dirbuf, topFile, sizeof( dirbuf ) );
147        strlcpy( basebuf, topFile, sizeof( basebuf ) );
148        dir = dirname( dirbuf );
149        base = basename( basebuf );
150        files = getFiles( dir, base, NULL );
151    }
152
153    for( walk=files; walk!=NULL; walk=walk->next )
154        ++ret->fileCount;
155    ret->files = calloc( ret->fileCount, sizeof(char*) );
156    ret->fileLengths = calloc( ret->fileCount, sizeof(uint64_t) );
157
158    for( i=0, walk=files; walk!=NULL; walk=walk->next, ++i )
159        ret->files[i] = tr_strdup( walk->filename );
160
161    qsort( ret->files, ret->fileCount, sizeof(char*), pstrcmp );
162    for( i=0; i<ret->fileCount; ++i ) {
163        ret->fileLengths[i] = getFileSize( ret->files[i] );
164        ret->totalSize += ret->fileLengths[i];
165    }
166
167    freeFileList( files );
168   
169    ret->pieceSize = bestPieceSize( ret->totalSize );
170    ret->pieceCount = (int)( ret->totalSize / ret->pieceSize);
171    if( ret->totalSize % ret->pieceSize )
172        ++ret->pieceCount;
173
174    return ret;
175}
176
177void
178tr_metaInfoBuilderFree( tr_metainfo_builder_t * builder )
179{
180    if( builder != NULL )
181    {
182        int i;
183        for( i=0; i<builder->fileCount; ++i )
184            tr_free( builder->files[i] );
185        tr_free( builder->files );
186        tr_free( builder->fileLengths );
187        tr_free( builder->top );
188        tr_free( builder->comment );
189        tr_free( builder->announce );
190        tr_free( builder->outputFile );
191        tr_free( builder );
192    }
193}
194
195/****
196*****
197****/
198
199static uint8_t*
200getHashInfo ( tr_metainfo_builder_t * b )
201{
202    int fileIndex = 0;
203    uint8_t *ret = (uint8_t*) malloc ( SHA_DIGEST_LENGTH * b->pieceCount );
204    uint8_t *walk = ret;
205    uint8_t *buf = malloc( b->pieceSize );
206    uint64_t totalRemain;
207    uint64_t off = 0;
208    FILE * fp;
209
210    b->pieceIndex = 0;
211    totalRemain = b->totalSize;
212    fp = fopen( b->files[fileIndex], "rb" );
213    while ( totalRemain )
214    {
215        uint8_t *bufptr = buf;
216        const uint64_t thisPieceSize = MIN( (uint32_t)b->pieceSize, totalRemain );
217        uint64_t pieceRemain = thisPieceSize;
218
219        assert( b->pieceIndex < b->pieceCount );
220
221        while( pieceRemain )
222        {
223            const uint64_t n_this_pass = MIN( (b->fileLengths[fileIndex] - off), pieceRemain );
224            fread( bufptr, 1, n_this_pass, fp );
225            bufptr += n_this_pass;
226            off += n_this_pass;
227            pieceRemain -= n_this_pass;
228            if( off == b->fileLengths[fileIndex] ) {
229                off = 0;
230                fclose( fp );
231                fp = NULL;
232                if( ++fileIndex < b->fileCount ) {
233                    fp = fopen( b->files[fileIndex], "rb" );
234                }
235            }
236        }
237
238        assert( bufptr-buf == (int)thisPieceSize );
239        assert( pieceRemain == 0 );
240        SHA1( buf, thisPieceSize, walk );
241        walk += SHA_DIGEST_LENGTH;
242
243        if( b->abortFlag ) {
244            b->failed = 1;
245            break;
246        }
247
248        totalRemain -= thisPieceSize;
249        ++b->pieceIndex;
250    }
251    assert( b->abortFlag || (walk-ret == (int)(SHA_DIGEST_LENGTH*b->pieceCount)) );
252    assert( b->abortFlag || !totalRemain );
253    assert( b->abortFlag || fp == NULL );
254
255    if( fp != NULL )
256        fclose( fp );
257
258    free( buf );
259    return ret;
260}
261
262static void
263getFileInfo( const char * topFile,
264             const char * filename,
265             benc_val_t * uninitialized_length,
266             benc_val_t * uninitialized_path )
267{
268    benc_val_t *sub;
269    const char *pch, *prev;
270    const size_t topLen = strlen(topFile) + 1; /* +1 for '/' */
271    int n;
272
273    /* get the file size */
274    tr_bencInitInt( uninitialized_length, getFileSize(filename) );
275
276    /* the path list */
277    n = 1;
278    for( pch=filename+topLen; *pch; ++pch )
279        if (*pch == TR_PATH_DELIMITER)
280            ++n;
281    tr_bencInit( uninitialized_path, TYPE_LIST );
282    tr_bencListReserve( uninitialized_path, n );
283    for( prev=pch=filename+topLen; ; ++pch )
284    {
285        char buf[MAX_PATH_LENGTH];
286
287        if (*pch && *pch!=TR_PATH_DELIMITER )
288            continue;
289
290        memcpy( buf, prev, pch-prev );
291        buf[pch-prev] = '\0';
292
293        sub = tr_bencListAdd( uninitialized_path );
294        tr_bencInitStrDup( sub, buf );
295
296        prev = pch + 1;
297        if (!*pch)
298           break;
299    }
300}
301
302static void
303makeFilesList( benc_val_t                 * list,
304               const tr_metainfo_builder_t  * builder )
305{
306    int i = 0;
307
308    tr_bencListReserve( list, builder->fileCount );
309
310    for( i=0; i<builder->fileCount; ++i )
311    {
312        benc_val_t * dict = tr_bencListAdd( list );
313        benc_val_t *length, *pathVal;
314
315        tr_bencInit( dict, TYPE_DICT );
316        tr_bencDictReserve( dict, 2 );
317        length = tr_bencDictAdd( dict, "length" );
318        pathVal = tr_bencDictAdd( dict, "path" );
319        getFileInfo( builder->top, builder->files[i], length, pathVal );
320    }
321}
322
323static void
324makeInfoDict ( benc_val_t             * dict,
325               tr_metainfo_builder_t  * builder )
326{
327    uint8_t * pch;
328    benc_val_t * val;
329    char base[MAX_PATH_LENGTH];
330
331    tr_bencDictReserve( dict, 5 );
332
333    val = tr_bencDictAdd( dict, "name" );
334    strlcpy( base, builder->top, sizeof( base ) );
335    tr_bencInitStrDup ( val, basename( base ) );
336
337    val = tr_bencDictAdd( dict, "piece length" );
338    tr_bencInitInt( val, builder->pieceSize );
339
340    pch = getHashInfo( builder );
341    val = tr_bencDictAdd( dict, "pieces" );
342    tr_bencInitStr( val, pch, SHA_DIGEST_LENGTH * builder->pieceCount, 0 );
343
344    if ( builder->isSingleFile )
345    {
346        val = tr_bencDictAdd( dict, "length" );
347        tr_bencInitInt( val, builder->fileLengths[0] );
348    }
349    else
350    {
351        val = tr_bencDictAdd( dict, "files" );
352        tr_bencInit( val, TYPE_LIST );
353        makeFilesList( val, builder );
354    }
355
356    val = tr_bencDictAdd( dict, "private" );
357    tr_bencInitInt( val, builder->isPrivate ? 1 : 0 );
358}
359
360static void tr_realMakeMetaInfo ( tr_metainfo_builder_t * builder )
361{
362    int n = 5;
363    benc_val_t top, *val;
364
365    tr_bencInit ( &top, TYPE_DICT );
366    if ( builder->comment && *builder->comment ) ++n;
367    tr_bencDictReserve( &top, n );
368
369        val = tr_bencDictAdd( &top, "announce" );
370        tr_bencInitStrDup( val, builder->announce );
371
372        val = tr_bencDictAdd( &top, "created by" );
373        tr_bencInitStrDup( val, TR_NAME " " VERSION_STRING );
374
375        val = tr_bencDictAdd( &top, "creation date" );
376        tr_bencInitInt( val, time(0) );
377
378        val = tr_bencDictAdd( &top, "encoding" );
379        tr_bencInitStrDup( val, "UTF-8" );
380
381        if( builder->comment && *builder->comment ) {
382            val = tr_bencDictAdd( &top, "comment" );
383            tr_bencInitStrDup( val, builder->comment );
384        }
385
386        val = tr_bencDictAdd( &top, "info" );
387        tr_bencInit( val, TYPE_DICT );
388        tr_bencDictReserve( val, 666 );
389        makeInfoDict( val, builder );
390
391    /* save the file */
392    if ( !builder->abortFlag ) {
393        size_t nmemb;
394        char * pch = tr_bencSaveMalloc( &top, &n );
395        FILE * fp = fopen( builder->outputFile, "wb+" );
396        nmemb = n;
397        if( fp == NULL )
398            builder->failed = 1;
399        else if( fwrite( pch, 1, nmemb, fp ) != nmemb )
400            builder->failed = 1;
401        free( pch );
402        fclose( fp );
403    }
404
405    /* cleanup */
406    tr_bencFree( & top );
407    builder->failed |= builder->abortFlag;
408    builder->isDone = 1;
409}
410
411/***
412****
413****  A threaded builder queue
414****
415***/
416
417static tr_metainfo_builder_t * queue = NULL;
418
419static int workerIsRunning = 0;
420
421static tr_thread_t workerThread;
422
423static tr_lock_t* getQueueLock( tr_handle_t * h )
424{
425    static tr_lock_t * lock = NULL;
426
427    tr_sharedLock( h->shared );
428    if( lock == NULL )
429    {
430        lock = calloc( 1, sizeof( tr_lock_t ) );
431        tr_lockInit( lock );
432    }
433    tr_sharedUnlock( h->shared );
434
435    return lock;
436}
437
438static void workerFunc( void * user_data )
439{
440    tr_handle_t * handle = (tr_handle_t *) user_data;
441
442    for (;;)
443    {
444        tr_metainfo_builder_t * builder = NULL;
445
446        /* find the next builder to process */
447        tr_lock_t * lock = getQueueLock ( handle );
448        tr_lockLock( lock );
449        if( queue != NULL ) {
450            builder = queue;
451            queue = queue->nextBuilder;
452        }
453        tr_lockUnlock( lock );
454
455        /* if no builders, this worker thread is done */
456        if( builder == NULL )
457          break;
458
459        tr_realMakeMetaInfo ( builder );
460    }
461
462    workerIsRunning = 0;
463}
464
465void
466tr_makeMetaInfo( tr_metainfo_builder_t  * builder,
467                 const char             * outputFile,
468                 const char             * announce,
469                 const char             * comment,
470                 int                      isPrivate )
471{
472    tr_lock_t * lock;
473
474    builder->abortFlag = 0;
475    builder->isDone = 0;
476    builder->announce = tr_strdup( announce );
477    builder->comment = tr_strdup( comment );
478    builder->isPrivate = isPrivate;
479    if( outputFile && *outputFile )
480        builder->outputFile = tr_strdup( outputFile );
481    else {
482        char out[MAX_PATH_LENGTH];
483        snprintf( out, sizeof(out), "%s.torrent", builder->top);
484        builder->outputFile = tr_strdup( out );
485    }
486
487    /* enqueue the builder */
488    lock = getQueueLock ( builder->handle );
489    tr_lockLock( lock );
490    builder->nextBuilder = queue;
491    queue = builder;
492    if( !workerIsRunning ) {
493        workerIsRunning = 1;
494        tr_threadCreate( &workerThread, workerFunc, builder->handle, "makeMeta" );
495    }
496    tr_lockUnlock( lock );
497}
498
Note: See TracBrowser for help on using the repository browser.