source: branches/file_selection/libtransmission/makemeta.c @ 2124

Last change on this file since 2124 was 2124, checked in by charles, 15 years ago

license the files I wrote under the creative commons Attribution-NonCommercial?-ShareAlike? 3.0 license, not the MIT license.

File size: 13.0 KB
Line 
1/*
2 * This file Copyright (C) 2007 Charles Kerr <charles@rebelbase.com>
3 *
4 * This file is available under the
5 * Attribution-Noncommercial-Share Alike 3.0 License.
6 * Overview:  http://creativecommons.org/licenses/by-nc-sa/3.0/
7 * Full text: http://creativecommons.org/licenses/by-nc-sa/3.0/legalcode
8 *
9 * Loosely speaking, you're free to use and adapt this work
10 * provided that it's for noncommercial uses and that you share your
11 * changes under a compatible license.
12 */
13
14#include <sys/types.h>
15#include <sys/stat.h>
16#include <unistd.h>
17#include <libgen.h>
18#include <dirent.h>
19#include <stdio.h> /* FILE, snprintf, stderr */
20#include <stdlib.h> /* malloc, calloc */
21
22#include "transmission.h"
23#include "internal.h" /* for tr_torrent_t */
24#include "bencode.h"
25#include "makemeta.h"
26#include "platform.h" /* threads, locks */
27#include "shared.h" /* shared lock */
28#include "version.h"
29
30/****
31*****
32****/
33
34struct FileList
35{
36    struct FileList * next;
37    char filename[MAX_PATH_LENGTH];
38};
39
40static struct FileList*
41getFiles( const char        * dir,
42          const char        * base,
43          struct FileList   * list )
44{
45    int i;
46    char buf[MAX_PATH_LENGTH];
47    struct stat sb;
48    sb.st_size = 0;
49
50    snprintf( buf, sizeof(buf), "%s"TR_PATH_DELIMITER_STR"%s", dir, base );
51    i = stat( buf, &sb );
52    if( i ) {
53        tr_err("makemeta couldn't stat \"%s\"; skipping. (%s)", buf, strerror(errno));
54        return list;
55    }
56
57    if ( S_ISDIR( sb.st_mode ) )
58    {
59        DIR * odir = opendir( buf );
60        struct dirent *d;
61        for (d = readdir( odir ); d!=NULL; d=readdir( odir ) )
62            if( strcmp( d->d_name,"." ) && strcmp( d->d_name,".." ) )
63                list = getFiles( buf, d->d_name, list );
64        closedir( odir );
65    }
66    else if( S_ISREG( sb.st_mode ) )
67    {
68        struct FileList * node = malloc( sizeof( struct FileList ) );
69        snprintf( node->filename, sizeof( node->filename ), "%s", buf );
70        node->next = list;
71        list = node;
72    }
73
74    return list;
75}
76
77static void
78freeFileList( struct FileList * list )
79{
80    while( list ) {
81        struct FileList * tmp = list->next;
82        free( list );
83        list = tmp;
84    }
85}
86
87static off_t
88getFileSize ( const char * filename )
89{
90    struct stat sb;
91    sb.st_size = 0;
92    stat( filename, &sb );
93    return sb.st_size;
94}
95
96#define MiB 1048576ul
97#define GiB 1073741824ul
98
99static size_t
100bestPieceSize( size_t totalSize )
101{
102    /* almost always best to have a piee size of 512 or 256 kb.
103       common practice seems to be to bump up to 1MB pieces at
104       at total size of around 8GiB or so */
105
106    if (totalSize >= (8 * GiB) )
107        return MiB;
108
109    if (totalSize >= GiB )
110        return MiB / 2;
111
112    return MiB / 4;
113}
114
115/****
116*****
117****/
118
119static int pstrcmp( const void * va, const void * vb)
120{
121    const char * a = *(const char**) va;
122    const char * b = *(const char**) vb;
123    return strcmp( a, b );
124}
125
126tr_metainfo_builder_t*
127tr_metaInfoBuilderCreate( tr_handle_t * handle, const char * topFile )
128{
129    size_t i;
130    struct FileList * files;
131    const struct FileList * walk;
132    tr_metainfo_builder_t * ret = calloc( 1, sizeof(tr_metainfo_builder_t) );
133    ret->top = tr_strdup( topFile );
134    ret->handle = handle;
135
136    if (1) {
137        struct stat sb;
138        stat( topFile, &sb );
139        ret->isSingleFile = !S_ISDIR( sb.st_mode );
140    }
141
142    /* build a list of files containing topFile and,
143       if it's a directory, all of its children */
144    if (1) {
145        char *dir, *base;
146        char dirbuf[MAX_PATH_LENGTH];
147        char basebuf[MAX_PATH_LENGTH];
148        strlcpy( dirbuf, topFile, sizeof( dirbuf ) );
149        strlcpy( basebuf, topFile, sizeof( basebuf ) );
150        dir = dirname( dirbuf );
151        base = basename( basebuf );
152        files = getFiles( dir, base, NULL );
153    }
154
155    for( walk=files; walk!=NULL; walk=walk->next )
156        ++ret->fileCount;
157    ret->files = calloc( ret->fileCount, sizeof(char*) );
158    ret->fileLengths = calloc( ret->fileCount, sizeof(size_t) );
159
160    for( i=0, walk=files; walk!=NULL; walk=walk->next, ++i )
161        ret->files[i] = tr_strdup( walk->filename );
162
163    qsort( ret->files, ret->fileCount, sizeof(char*), pstrcmp );
164    for( i=0; i<ret->fileCount; ++i ) {
165        ret->fileLengths[i] = getFileSize( ret->files[i] );
166        ret->totalSize += ret->fileLengths[i];
167    }
168
169    freeFileList( files );
170   
171    ret->pieceSize = bestPieceSize( ret->totalSize );
172    ret->pieceCount = ret->totalSize / ret->pieceSize;
173    if( ret->totalSize % ret->pieceSize )
174        ++ret->pieceCount;
175
176    return ret;
177}
178
179void
180tr_metaInfoBuilderFree( tr_metainfo_builder_t * builder )
181{
182    if( builder != NULL )
183    {
184        size_t i;
185        for( i=0; i<builder->fileCount; ++i )
186            tr_free( builder->files[i] );
187        tr_free( builder->files );
188        tr_free( builder->fileLengths );
189        tr_free( builder->top );
190        tr_free( builder->comment );
191        tr_free( builder->announce );
192        tr_free( builder->outputFile );
193        tr_free( builder );
194    }
195}
196
197/****
198*****
199****/
200
201static uint8_t*
202getHashInfo ( tr_metainfo_builder_t * b )
203{
204    size_t off = 0;
205    size_t fileIndex = 0;
206    uint8_t *ret = (uint8_t*) malloc ( SHA_DIGEST_LENGTH * b->pieceCount );
207    uint8_t *walk = ret;
208    uint8_t *buf = malloc( b->pieceSize );
209    size_t totalRemain;
210    FILE * fp;
211
212    b->pieceIndex = 0;
213    totalRemain = b->totalSize;
214    fp = fopen( b->files[fileIndex], "rb" );
215    while ( totalRemain )
216    {
217        uint8_t *bufptr = buf;
218        const size_t thisPieceSize = MIN( b->pieceSize, totalRemain );
219        size_t pieceRemain = thisPieceSize;
220
221        assert( b->pieceIndex < b->pieceCount );
222
223        while( pieceRemain )
224        {
225            const size_t n_this_pass = MIN( (b->fileLengths[fileIndex] - off), pieceRemain );
226            fread( bufptr, 1, n_this_pass, fp );
227            bufptr += n_this_pass;
228            off += n_this_pass;
229            pieceRemain -= n_this_pass;
230            if( off == b->fileLengths[fileIndex] ) {
231                off = 0;
232                fclose( fp );
233                fp = NULL;
234                if( ++fileIndex < b->fileCount ) {
235                    fp = fopen( b->files[fileIndex], "rb" );
236                }
237            }
238        }
239
240        assert( bufptr-buf == (int)thisPieceSize );
241        assert( pieceRemain == 0 );
242        SHA1( buf, thisPieceSize, walk );
243        walk += SHA_DIGEST_LENGTH;
244
245        if( b->abortFlag ) {
246            b->failed = 1;
247            break;
248        }
249
250        totalRemain -= thisPieceSize;
251        ++b->pieceIndex;
252    }
253    assert( b->abortFlag || (walk-ret == (int)(SHA_DIGEST_LENGTH*b->pieceCount)) );
254    assert( b->abortFlag || !totalRemain );
255    assert( b->abortFlag || fp == NULL );
256
257    if( fp != NULL )
258        fclose( fp );
259
260    free( buf );
261    return ret;
262}
263
264static void
265getFileInfo( const char * topFile,
266             const char * filename,
267             benc_val_t * uninitialized_length,
268             benc_val_t * uninitialized_path )
269{
270    benc_val_t *sub;
271    const char *pch, *prev;
272    const size_t topLen = strlen(topFile) + 1; /* +1 for '/' */
273    int n;
274
275    /* get the file size */
276    tr_bencInitInt( uninitialized_length, getFileSize(filename) );
277
278    /* the path list */
279    n = 1;
280    for( pch=filename+topLen; *pch; ++pch )
281        if (*pch == TR_PATH_DELIMITER)
282            ++n;
283    tr_bencInit( uninitialized_path, TYPE_LIST );
284    tr_bencListReserve( uninitialized_path, n );
285    for( prev=pch=filename+topLen; ; ++pch )
286    {
287        char buf[MAX_PATH_LENGTH];
288
289        if (*pch && *pch!=TR_PATH_DELIMITER )
290            continue;
291
292        memcpy( buf, prev, pch-prev );
293        buf[pch-prev] = '\0';
294        /*fprintf ( stderr, "adding [%s] to the list of paths\n", buf );*/
295
296        sub = tr_bencListAdd( uninitialized_path );
297        tr_bencInitStrDup( sub, buf );
298
299        prev = pch + 1;
300        if (!*pch)
301           break;
302    }
303}
304
305static void
306makeFilesList( benc_val_t                 * list,
307               const tr_metainfo_builder_t  * builder )
308{
309    size_t i = 0;
310
311    tr_bencListReserve( list, builder->fileCount );
312
313    for( i=0; i<builder->fileCount; ++i )
314    {
315        benc_val_t * dict = tr_bencListAdd( list );
316        benc_val_t *length, *pathVal;
317
318        tr_bencInit( dict, TYPE_DICT );
319        tr_bencDictReserve( dict, 2 );
320        length = tr_bencDictAdd( dict, "length" );
321        pathVal = tr_bencDictAdd( dict, "path" );
322        getFileInfo( builder->top, builder->files[i], length, pathVal );
323    }
324}
325
326static void
327makeInfoDict ( benc_val_t             * dict,
328               tr_metainfo_builder_t  * builder )
329{
330    uint8_t * pch;
331    benc_val_t * val;
332    char base[MAX_PATH_LENGTH];
333
334    tr_bencDictReserve( dict, 5 );
335
336    val = tr_bencDictAdd( dict, "name" );
337    strlcpy( base, builder->top, sizeof( base ) );
338    tr_bencInitStrDup ( val, basename( base ) );
339
340    val = tr_bencDictAdd( dict, "piece length" );
341    tr_bencInitInt( val, builder->pieceSize );
342
343    pch = getHashInfo( builder );
344    val = tr_bencDictAdd( dict, "pieces" );
345    tr_bencInitStr( val, pch, SHA_DIGEST_LENGTH * builder->pieceCount, 0 );
346
347    if ( builder->isSingleFile )
348    {
349        val = tr_bencDictAdd( dict, "length" );
350        tr_bencInitInt( val, builder->fileLengths[0] );
351    }
352    else
353    {
354        val = tr_bencDictAdd( dict, "files" );
355        tr_bencInit( val, TYPE_LIST );
356        makeFilesList( val, builder );
357    }
358
359    val = tr_bencDictAdd( dict, "private" );
360    tr_bencInitInt( val, builder->isPrivate ? 1 : 0 );
361}
362
363static void tr_realMakeMetaInfo ( tr_metainfo_builder_t * builder )
364{
365    int n = 5;
366    benc_val_t top, *val;
367fprintf( stderr, "builder %p in realMakeMetaInfo\n", builder );
368
369    tr_bencInit ( &top, TYPE_DICT );
370    if ( builder->comment && *builder->comment ) ++n;
371    tr_bencDictReserve( &top, n );
372
373        val = tr_bencDictAdd( &top, "announce" );
374        tr_bencInitStrDup( val, builder->announce );
375
376        val = tr_bencDictAdd( &top, "created by" );
377        tr_bencInitStrDup( val, TR_NAME " " VERSION_STRING );
378
379        val = tr_bencDictAdd( &top, "creation date" );
380        tr_bencInitInt( val, time(0) );
381
382        val = tr_bencDictAdd( &top, "encoding" );
383        tr_bencInitStrDup( val, "UTF-8" );
384
385        if( builder->comment && *builder->comment ) {
386            val = tr_bencDictAdd( &top, "comment" );
387            tr_bencInitStrDup( val, builder->comment );
388        }
389
390        val = tr_bencDictAdd( &top, "info" );
391        tr_bencInit( val, TYPE_DICT );
392        tr_bencDictReserve( val, 666 );
393        makeInfoDict( val, builder );
394
395    /* save the file */
396    if (1) {
397        FILE * fp = fopen( builder->outputFile, "wb+" );
398        char * pch = tr_bencSaveMalloc( &top, &n );
399        fwrite( pch, n, 1, fp );
400        free( pch );
401        fclose( fp );
402    }
403
404    /* cleanup */
405    tr_bencFree( & top );
406    builder->isDone = 1;
407    builder->failed = builder->abortFlag; /* FIXME: doesn't catch all failures */
408}
409
410/***
411****
412****  A threaded builder queue
413****
414***/
415
416static tr_metainfo_builder_t * queue = NULL;
417
418static int workerIsRunning = 0;
419
420static tr_thread_t workerThread;
421
422static tr_lock_t* getQueueLock( tr_handle_t * h )
423{
424    static tr_lock_t * lock = NULL;
425
426    tr_sharedLock( h->shared );
427    if( lock == NULL )
428    {
429        lock = calloc( 1, sizeof( tr_lock_t ) );
430        tr_lockInit( lock );
431    }
432    tr_sharedUnlock( h->shared );
433
434    return lock;
435}
436
437static void workerFunc( void * user_data )
438{
439    tr_handle_t * handle = (tr_handle_t *) user_data;
440
441    for (;;)
442    {
443        tr_metainfo_builder_t * builder = NULL;
444
445        /* find the next builder to process */
446        tr_lock_t * lock = getQueueLock ( handle );
447        tr_lockLock( lock );
448        if( queue != NULL ) {
449            builder = queue;
450            queue = queue->nextBuilder;
451        }
452        tr_lockUnlock( lock );
453
454        /* if no builders, this worker thread is done */
455        if( builder == NULL )
456          break;
457
458fprintf( stderr, "worker thread got builder %p\n", builder);
459        tr_realMakeMetaInfo ( builder );
460    }
461
462fprintf( stderr, "worker thread exiting\n" );
463    workerIsRunning = 0;
464}
465
466void
467tr_makeMetaInfo( tr_metainfo_builder_t  * builder,
468                 const char             * outputFile,
469                 const char             * announce,
470                 const char             * comment,
471                 int                      isPrivate )
472{
473    tr_lock_t * lock;
474    builder->announce = tr_strdup( announce );
475    builder->comment = tr_strdup( comment );
476    builder->isPrivate = isPrivate;
477    if( outputFile && *outputFile )
478        builder->outputFile = tr_strdup( outputFile );
479    else {
480        char out[MAX_PATH_LENGTH];
481        snprintf( out, sizeof(out), "%s.torrent", builder->top);
482        builder->outputFile = tr_strdup( out );
483    }
484
485fprintf( stderr, "enqueuing a builder %p\n", builder);
486    /* enqueue the builder */
487    lock = getQueueLock ( builder->handle );
488    tr_lockLock( lock );
489    builder->nextBuilder = queue;
490    queue = builder;
491    if( !workerIsRunning ) {
492        workerIsRunning = 1;
493fprintf( stderr, "making new worker thread\n" );
494        tr_threadCreate( &workerThread, workerFunc, builder->handle, "makeMeta" );
495    }
496    tr_lockUnlock( lock );
497}
498
Note: See TracBrowser for help on using the repository browser.