source: trunk/libtransmission/fdlimit.c @ 8291

Last change on this file since 8291 was 8291, checked in by charles, 13 years ago

(trunk libT) need feedback from Mac users on this change. On Linux, it gets rid of the inactive-memory-grows-during-torrent-verification behavior that's often reported as a bug.

  • Property svn:keywords set to Date Rev Author Id
File size: 16.6 KB
Line 
1/******************************************************************************
2 * $Id: fdlimit.c 8291 2009-04-26 00:51:51Z charles $
3 *
4 * Copyright (c) 2005-2008 Transmission authors and contributors
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 *****************************************************************************/
24
25#ifndef WIN32
26 #define HAVE_GETRLIMIT
27#endif
28
29#ifdef HAVE_POSIX_FADVISE
30 #ifdef _XOPEN_SOURCE
31  #undef _XOPEN_SOURCE
32 #endif
33 #define _XOPEN_SOURCE 600
34#endif
35
36#include <assert.h>
37#include <errno.h>
38#include <inttypes.h>
39#include <stdio.h>
40#include <stdlib.h>
41#include <string.h>
42#ifdef SYS_DARWIN
43 #include <fcntl.h>
44#endif
45
46#ifdef HAVE_FALLOCATE
47 #include <linux/falloc.h>
48#endif
49
50#ifdef HAVE_XFS_XFS_H
51 #include <xfs/xfs.h>
52#endif
53
54#include <sys/types.h>
55#include <sys/stat.h>
56#ifdef HAVE_GETRLIMIT
57 #include <sys/time.h> /* getrlimit */
58 #include <sys/resource.h> /* getrlimit */
59#endif
60#include <unistd.h>
61#include <fcntl.h> /* O_LARGEFILE posix_fadvise */
62
63#include <evutil.h>
64
65#include "transmission.h"
66#include "fdlimit.h"
67#include "list.h"
68#include "net.h"
69#include "platform.h" /* tr_lock */
70#include "utils.h"
71
72#define dbgmsg( ... ) \
73    do { \
74        if( tr_deepLoggingIsActive( ) ) \
75            tr_deepLog( __FILE__, __LINE__, NULL, __VA_ARGS__ ); \
76    } while( 0 )
77
78/**
79***
80**/
81
82enum
83{
84    NOFILE_BUFFER = 512, /* the process' number of open files is
85                            globalMaxPeers + NOFILE_BUFFER */
86};
87
88struct tr_openfile
89{
90    tr_bool    isCheckedOut;
91    tr_bool    isWritable;
92    tr_bool    closeWhenDone;
93    char       filename[MAX_PATH_LENGTH];
94    int        fd;
95    uint64_t   date;
96};
97
98struct tr_fd_s
99{
100    int                   socketCount;
101    int                   socketLimit;
102
103    struct tr_openfile  * openFiles;
104    int                   openFileLimit;
105
106    tr_lock             * lock;
107};
108
109static struct tr_fd_s * gFd = NULL;
110
111/***
112****
113****  Local Files
114****
115***/
116
117#ifndef O_LARGEFILE
118 #define O_LARGEFILE 0
119#endif
120
121static tr_bool
122preallocateFileSparse( int fd, uint64_t length )
123{
124    const char zero = '\0';
125
126    if( length == 0 )
127        return TRUE;
128
129    if( lseek( fd, length-1, SEEK_SET ) == -1 )
130        return FALSE;
131    if( write( fd, &zero, 1 ) == -1 )
132        return FALSE;
133    if( ftruncate( fd, length ) == -1 )
134        return FALSE;
135
136    return TRUE;
137}
138
139static tr_bool
140preallocateFileFull( const char * filename, uint64_t length )
141{
142    tr_bool success = 0;
143
144#ifdef WIN32
145
146    HANDLE hFile = CreateFile( filename, GENERIC_WRITE, 0, 0, CREATE_NEW, 0, 0 );
147    if( hFile != INVALID_HANDLE_VALUE )
148    {
149        LARGE_INTEGER li;
150        li.QuadPart = length;
151        success = SetFilePointerEx( hFile, li, NULL, FILE_BEGIN ) && SetEndOfFile( hFile );
152        CloseHandle( hFile );
153    }
154
155#else
156
157    int flags = O_RDWR | O_CREAT | O_LARGEFILE;
158    int fd = open( filename, flags, 0666 );
159    if( fd >= 0 )
160    {
161# ifdef HAVE_XFS_XFS_H
162        if( !success && platform_test_xfs_fd( fd ) )
163        {
164            xfs_flock64_t fl;
165            fl.l_whence = 0;
166            fl.l_start = 0;
167            fl.l_len = length;
168            success = !xfsctl( NULL, fd, XFS_IOC_RESVSP64, &fl );
169        }
170# endif
171# ifdef SYS_DARWIN
172        if( !success )
173        {
174            fstore_t fst;
175            fst.fst_flags = F_ALLOCATECONTIG;
176            fst.fst_posmode = F_PEOFPOSMODE;
177            fst.fst_offset = 0;
178            fst.fst_length = length;
179            fst.fst_bytesalloc = 0;
180            success = !fcntl( fd, F_PREALLOCATE, &fst );
181        }
182# endif
183# ifdef HAVE_FALLOCATE
184        if( !success )
185        {
186            success = !fallocate( fd, FALLOC_FL_KEEP_SIZE, 0, length );
187        }
188# endif
189# ifdef HAVE_POSIX_FALLOCATE
190        if( !success )
191        {
192            success = !posix_fallocate( fd, 0, length );
193        }
194# endif
195
196        close( fd );
197    }
198
199#endif
200
201    return success;
202}
203
204FILE*
205tr_open_file_for_scanning( const char * filename )
206{
207    int fd;
208    int flags;
209
210    /* build the flags */
211    flags = O_RDONLY;
212#ifdef O_SEQUENTIAL
213    if( sequential ) flags |= O_SEQUENTIAL;
214#endif
215#ifdef O_RANDOM
216    if( !sequential ) flags |= O_RANDOM
217#endif
218#ifdef O_BINARY
219    flags |= O_BINARY;
220#endif
221#ifdef O_LARGEFILE
222    flags |= O_LARGEFILE;
223#endif
224
225    /* open the file */
226    fd = open( filename, flags, 0666 );
227    if( fd < 0 )
228        return NULL;
229
230#ifdef HAVE_POSIX_FADVISE
231    posix_fadvise( fd, 0, 0, POSIX_FADV_SEQUENTIAL );
232#endif
233
234    return fdopen( fd, "r" );
235}
236
237static void
238flush_before_closing( int fd )
239{
240#if defined(HAVE_POSIX_FADVISE)
241    posix_fadvise( fd, 0, 0, POSIX_FADV_DONTNEED );
242#elif defined(SYS_DARWIN)
243    fcntl( fd, F_NOCACHE, 1 );
244#endif
245}
246
247void
248tr_close_file( FILE * fp )
249{
250    flush_before_closing( fileno( fp ) );
251    fclose( fp );
252}
253
254/**
255 * returns 0 on success, or an errno value on failure.
256 * errno values include ENOENT if the parent folder doesn't exist,
257 * plus the errno values set by tr_mkdirp() and open().
258 */
259static int
260TrOpenFile( int                      i,
261            const char             * folder,
262            const char             * torrentFile,
263            tr_bool                  doWrite,
264            tr_preallocation_mode    preallocationMode,
265            uint64_t                 desiredFileSize )
266{
267    struct tr_openfile * file = &gFd->openFiles[i];
268    int                  flags;
269    char               * filename;
270    struct stat          sb;
271    int                  alreadyExisted;
272
273    /* confirm the parent folder exists */
274    if( stat( folder, &sb ) || !S_ISDIR( sb.st_mode ) )
275    {
276        tr_err( _( "Couldn't create \"%1$s\": parent folder \"%2$s\" does not exist" ), torrentFile, folder );
277        return ENOENT;
278    }
279
280    /* create subfolders, if any */
281    filename = tr_buildPath( folder, torrentFile, NULL );
282    if( doWrite )
283    {
284        char * tmp = tr_dirname( filename );
285        const int err = tr_mkdirp( tmp, 0777 ) ? errno : 0;
286        if( err ) {
287            tr_err( _( "Couldn't create \"%1$s\": %2$s" ), tmp, tr_strerror( err ) );
288            tr_free( tmp );
289            tr_free( filename );
290            return err;
291        }
292        tr_free( tmp );
293    }
294
295    alreadyExisted = !stat( filename, &sb ) && S_ISREG( sb.st_mode );
296
297    if( doWrite && !alreadyExisted && ( preallocationMode == TR_PREALLOCATE_FULL ) )
298        if( preallocateFileFull( filename, desiredFileSize ) )
299            tr_inf( _( "Preallocated file \"%s\"" ), filename );
300   
301    /* open the file */
302    flags = doWrite ? ( O_RDWR | O_CREAT ) : O_RDONLY;
303#ifdef O_RANDOM
304    flags |= O_RANDOM
305#endif
306#ifdef O_LARGEFILE
307    flags |= O_LARGEFILE;
308#endif
309#ifdef WIN32
310    flags |= O_BINARY;
311#endif
312    file->fd = open( filename, flags, 0666 );
313    if( file->fd == -1 )
314    {
315        const int err = errno;
316        tr_err( _( "Couldn't open \"%1$s\": %2$s" ), filename, tr_strerror( err ) );
317        tr_free( filename );
318        return err;
319    }
320
321    if( doWrite && !alreadyExisted && ( preallocationMode == TR_PREALLOCATE_SPARSE ) )
322        preallocateFileSparse( file->fd, desiredFileSize );
323
324#ifdef HAVE_POSIX_FADVISE
325    posix_fadvise( file->fd, 0, 0, POSIX_FADV_RANDOM );
326#endif
327
328    tr_free( filename );
329    return 0;
330}
331
332static int
333fileIsOpen( const struct tr_openfile * o )
334{
335    return o->fd >= 0;
336}
337
338static void
339TrCloseFile( int i )
340{
341    struct tr_openfile * o = &gFd->openFiles[i];
342
343    assert( i >= 0 );
344    assert( i < gFd->openFileLimit );
345    assert( fileIsOpen( o ) );
346
347    flush_before_closing( o->fd );
348    close( o->fd );
349    o->fd = -1;
350    o->isCheckedOut = 0;
351}
352
353static int
354fileIsCheckedOut( const struct tr_openfile * o )
355{
356    return fileIsOpen( o ) && o->isCheckedOut;
357}
358
359/* returns an fd on success, or a -1 on failure and sets errno */
360int
361tr_fdFileCheckout( const char             * folder,
362                   const char             * torrentFile,
363                   tr_bool                  doWrite,
364                   tr_preallocation_mode    preallocationMode,
365                   uint64_t                 desiredFileSize )
366{
367    int i, winner = -1;
368    struct tr_openfile * o;
369    char filename[MAX_PATH_LENGTH];
370
371    assert( folder && *folder );
372    assert( torrentFile && *torrentFile );
373    assert( doWrite == 0 || doWrite == 1 );
374
375    tr_snprintf( filename, sizeof( filename ), "%s%c%s", folder, TR_PATH_DELIMITER, torrentFile );
376    dbgmsg( "looking for file '%s', writable %c", filename, doWrite ? 'y' : 'n' );
377
378    tr_lockLock( gFd->lock );
379
380    /* Is it already open? */
381    for( i = 0; i < gFd->openFileLimit; ++i )
382    {
383        o = &gFd->openFiles[i];
384
385        if( !fileIsOpen( o ) )
386            continue;
387
388        if( strcmp( filename, o->filename ) )
389            continue;
390
391        if( fileIsCheckedOut( o ) )
392        {
393            dbgmsg( "found it!  it's open, but checked out.  waiting..." );
394            tr_lockUnlock( gFd->lock );
395            tr_wait( 200 );
396            tr_lockLock( gFd->lock );
397            i = -1; /* reloop */
398            continue;
399        }
400
401        if( doWrite && !o->isWritable )
402        {
403            dbgmsg(
404                "found it!  it's open and available, but isn't writable. closing..." );
405            TrCloseFile( i );
406            break;
407        }
408
409        dbgmsg( "found it!  it's ready for use!" );
410        winner = i;
411        break;
412    }
413
414    dbgmsg(
415        "it's not already open.  looking for an open slot or an old file." );
416    while( winner < 0 )
417    {
418        uint64_t date = tr_date( ) + 1;
419
420        /* look for the file that's been open longest */
421        for( i = 0; i < gFd->openFileLimit; ++i )
422        {
423            o = &gFd->openFiles[i];
424
425            if( !fileIsOpen( o ) )
426            {
427                winner = i;
428                dbgmsg( "found an empty slot in %d", winner );
429                break;
430            }
431
432            if( date > o->date )
433            {
434                date = o->date;
435                winner = i;
436            }
437        }
438
439        if( winner >= 0 )
440        {
441            if( fileIsOpen( &gFd->openFiles[winner] ) )
442            {
443                dbgmsg( "closing file '%s', slot #%d",
444                        gFd->openFiles[winner].filename,
445                        winner );
446                TrCloseFile( winner );
447            }
448        }
449        else
450        {
451            dbgmsg( "everything's full!  waiting for someone else to finish something" );
452            tr_lockUnlock( gFd->lock );
453            tr_wait( 200 );
454            tr_lockLock( gFd->lock );
455        }
456    }
457
458    assert( winner >= 0 );
459    o = &gFd->openFiles[winner];
460    if( !fileIsOpen( o ) )
461    {
462        const int err = TrOpenFile( winner, folder, torrentFile, doWrite, preallocationMode, desiredFileSize );
463        if( err ) {
464            tr_lockUnlock( gFd->lock );
465            errno = err;
466            return -1;
467        }
468
469        dbgmsg( "opened '%s' in slot %d, doWrite %c", filename, winner,
470                doWrite ? 'y' : 'n' );
471        tr_strlcpy( o->filename, filename, sizeof( o->filename ) );
472        o->isWritable = doWrite;
473    }
474
475    dbgmsg( "checking out '%s' in slot %d", filename, winner );
476    o->isCheckedOut = 1;
477    o->closeWhenDone = 0;
478    o->date = tr_date( );
479    tr_lockUnlock( gFd->lock );
480    return o->fd;
481}
482
483void
484tr_fdFileReturn( int fd )
485{
486    int i;
487
488    tr_lockLock( gFd->lock );
489
490    for( i = 0; i < gFd->openFileLimit; ++i )
491    {
492        struct tr_openfile * o = &gFd->openFiles[i];
493        if( o->fd != fd )
494            continue;
495
496        dbgmsg( "releasing file '%s' in slot #%d", o->filename, i );
497        o->isCheckedOut = 0;
498        if( o->closeWhenDone )
499            TrCloseFile( i );
500
501        break;
502    }
503
504    tr_lockUnlock( gFd->lock );
505}
506
507void
508tr_fdFileClose( const char * filename )
509{
510    int i;
511
512    tr_lockLock( gFd->lock );
513
514    for( i = 0; i < gFd->openFileLimit; ++i )
515    {
516        struct tr_openfile * o = &gFd->openFiles[i];
517        if( !fileIsOpen( o ) || strcmp( filename, o->filename ) )
518            continue;
519
520        dbgmsg( "tr_fdFileClose closing '%s'", filename );
521
522        if( !o->isCheckedOut )
523        {
524            dbgmsg( "not checked out, so closing it now... '%s'", filename );
525            TrCloseFile( i );
526        }
527        else
528        {
529            dbgmsg(
530                "flagging file '%s', slot #%d to be closed when checked in",
531                gFd->openFiles[i].filename, i );
532            o->closeWhenDone = 1;
533        }
534    }
535
536    tr_lockUnlock( gFd->lock );
537}
538
539/***
540****
541****  Sockets
542****
543***/
544
545static int
546getSocketMax( struct tr_fd_s * gFd )
547{
548    return gFd->socketLimit;
549}
550
551int
552tr_fdSocketCreate( int domain, int type )
553{
554    int s = -1;
555
556    tr_lockLock( gFd->lock );
557
558    if( gFd->socketCount < getSocketMax( gFd ) )
559        if( ( s = socket( domain, type, 0 ) ) < 0 )
560        {
561#ifdef SYS_DARWIN
562            if( sockerrno != EAFNOSUPPORT )
563#endif
564            tr_err( _( "Couldn't create socket: %s" ),
565                   tr_strerror( sockerrno ) );
566            s = -sockerrno;
567        }
568
569    if( s > -1 )
570        ++gFd->socketCount;
571
572    assert( gFd->socketCount >= 0 );
573
574    tr_lockUnlock( gFd->lock );
575    return s;
576}
577
578int
579tr_fdSocketAccept( int           b,
580                   tr_address  * addr,
581                   tr_port     * port )
582{
583    int                s = -1;
584    unsigned int       len;
585    struct sockaddr_storage sock;
586
587    assert( addr );
588    assert( port );
589
590    tr_lockLock( gFd->lock );
591    if( gFd->socketCount < getSocketMax( gFd ) )
592    {
593        len = sizeof( struct sockaddr_storage );
594        s = accept( b, (struct sockaddr *) &sock, &len );
595    }
596    if( s > -1 )
597    {
598        /* "The ss_family field of the sockaddr_storage structure will always
599         * align with the family field of any protocol-specific structure." */ 
600        if( sock.ss_family == AF_INET ) 
601        { 
602            struct sockaddr_in * sock4 = (struct sockaddr_in *)&sock; 
603            addr->type = TR_AF_INET; 
604            addr->addr.addr4.s_addr = sock4->sin_addr.s_addr; 
605            *port = sock4->sin_port; 
606        } 
607        else 
608        { 
609            struct sockaddr_in6 * sock6 = (struct sockaddr_in6 *)&sock; 
610            addr->type = TR_AF_INET6; 
611            addr->addr.addr6 = sock6->sin6_addr;
612            *port = sock6->sin6_port; 
613        } 
614        ++gFd->socketCount;
615    }
616    tr_lockUnlock( gFd->lock );
617
618    return s;
619}
620
621static void
622socketClose( int fd )
623{
624    EVUTIL_CLOSESOCKET( fd );
625}
626
627void
628tr_fdSocketClose( int s )
629{
630    tr_lockLock( gFd->lock );
631
632    if( s >= 0 )
633    {
634        socketClose( s );
635        --gFd->socketCount;
636    }
637
638    assert( gFd->socketCount >= 0 );
639
640    tr_lockUnlock( gFd->lock );
641}
642
643/***
644****
645****  Startup / Shutdown
646****
647***/
648
649void
650tr_fdInit( size_t openFileLimit, size_t socketLimit )
651{
652    int i;
653
654    assert( gFd == NULL );
655    gFd = tr_new0( struct tr_fd_s, 1 );
656    gFd->openFiles = tr_new0( struct tr_openfile, openFileLimit );
657    gFd->openFileLimit = openFileLimit;
658    gFd->lock = tr_lockNew( );
659
660#ifdef HAVE_GETRLIMIT
661    {
662        struct rlimit rlim;
663        getrlimit( RLIMIT_NOFILE, &rlim );
664        rlim.rlim_cur = MIN( rlim.rlim_max,
665                            (rlim_t)( socketLimit + NOFILE_BUFFER ) );
666        setrlimit( RLIMIT_NOFILE, &rlim );
667        gFd->socketLimit = rlim.rlim_cur - NOFILE_BUFFER;
668        tr_dbg( "setrlimit( RLIMIT_NOFILE, %d )", (int)rlim.rlim_cur );
669    }
670#else
671    gFd->socketLimit = socketLimit;
672#endif
673    tr_dbg( "%zu usable file descriptors", socketLimit );
674
675    for( i = 0; i < gFd->openFileLimit; ++i )
676        gFd->openFiles[i].fd = -1;
677}
678
679void
680tr_fdClose( void )
681{
682    int i = 0;
683
684    for( i = 0; i < gFd->openFileLimit; ++i )
685        if( fileIsOpen( &gFd->openFiles[i] ) )
686            TrCloseFile( i );
687
688    tr_lockFree( gFd->lock );
689
690    tr_free( gFd->openFiles );
691    tr_free( gFd );
692    gFd = NULL;
693}
694
695void
696tr_fdSetPeerLimit( uint16_t n )
697{
698    assert( gFd != NULL && "tr_fdInit() must be called first!" );
699    gFd->socketLimit = n;
700}
701
702uint16_t
703tr_fdGetPeerLimit( void )
704{
705    return gFd ? gFd->socketLimit : -1;
706}
707
Note: See TracBrowser for help on using the repository browser.