source: trunk/libtransmission/makemeta.c @ 14327

Last change on this file since 14327 was 14327, checked in by jordan, 8 years ago

(trunk, libt) #4160 - the slow slog to catch trunk up to mike.dld's 4160 diff continues. This step applies 4160-03b-file.patch, which replaces native file operations with the tr_sys_file_*() portability wrappers added in r14321.

  • Property svn:keywords set to Date Rev Author Id
File size: 14.9 KB
Line 
1/*
2 * This file Copyright (C) 2010-2014 Mnemosyne LLC
3 *
4 * It may be used under the GNU GPL versions 2 or 3
5 * or any future license endorsed by Mnemosyne LLC.
6 *
7 * $Id: makemeta.c 14327 2014-07-28 04:13:38Z jordan $
8 */
9
10#include <assert.h>
11#include <errno.h>
12#include <stdlib.h> /* qsort */
13#include <string.h> /* strcmp, strlen */
14
15#include <dirent.h>
16
17#include <event2/util.h> /* evutil_ascii_strcasecmp () */
18
19#include "transmission.h"
20#include "crypto.h" /* tr_sha1 */
21#include "error.h"
22#include "file.h"
23#include "log.h"
24#include "session.h"
25#include "makemeta.h"
26#include "platform.h" /* threads, locks */
27#include "utils.h" /* buildpath */
28#include "variant.h"
29#include "version.h"
30
31/****
32*****
33****/
34
35struct FileList
36{
37  uint64_t size;
38  char * filename;
39  struct FileList * next;
40};
41
42static struct FileList *
43getFiles (const char      * dir,
44          const char      * base,
45          struct FileList * list)
46{
47  DIR * odir;
48  char * buf;
49  tr_sys_path_info info;
50  tr_error * error = NULL;
51
52  buf = tr_buildPath (dir, base, NULL);
53  if (!tr_sys_path_get_info (buf, 0, &info, &error))
54    {
55      tr_logAddError (_("Torrent Creator is skipping file \"%s\": %s"),
56                      buf, error->message);
57      tr_free (buf);
58      tr_error_free (error);
59      return list;
60    }
61
62  if (info.type == TR_SYS_PATH_IS_DIRECTORY && ((odir = opendir (buf))))
63    {
64      struct dirent *d;
65      for (d = readdir (odir); d != NULL; d = readdir (odir))
66        if (d->d_name && d->d_name[0] != '.') /* skip dotfiles */
67          list = getFiles (buf, d->d_name, list);
68      closedir (odir);
69    }
70  else if (info.type == TR_SYS_PATH_IS_FILE && info.size > 0)
71    {
72      struct FileList * node = tr_new (struct FileList, 1);
73      node->size = info.size;
74      node->filename = tr_strdup (buf);
75      node->next = list;
76      list = node;
77    }
78
79  tr_free (buf);
80  return list;
81}
82
83static uint32_t
84bestPieceSize (uint64_t totalSize)
85{
86  const uint32_t KiB = 1024;
87  const uint32_t MiB = 1048576;
88  const uint32_t GiB = 1073741824;
89
90  if (totalSize >= (2 * GiB)) return   2 * MiB;
91  if (totalSize >= (1 * GiB)) return   1 * MiB;
92  if (totalSize >= (512 * MiB)) return 512 * KiB;
93  if (totalSize >= (350 * MiB)) return 256 * KiB;
94  if (totalSize >= (150 * MiB)) return 128 * KiB;
95  if (totalSize >= (50 * MiB)) return  64 * KiB;
96  return 32 * KiB;  /* less than 50 meg */
97}
98
99static int
100builderFileCompare (const void * va, const void * vb)
101{
102  const tr_metainfo_builder_file * a = va;
103  const tr_metainfo_builder_file * b = vb;
104
105  return evutil_ascii_strcasecmp (a->filename, b->filename);
106}
107
108tr_metainfo_builder*
109tr_metaInfoBuilderCreate (const char * topFileArg)
110{
111  int i;
112  struct FileList * files;
113  struct FileList * walk;
114  tr_metainfo_builder * ret = tr_new0 (tr_metainfo_builder, 1);
115
116  ret->top = tr_sys_path_resolve (topFileArg, NULL);
117
118  {
119    tr_sys_path_info info;
120    ret->isFolder = tr_sys_path_get_info (ret->top, 0, &info, NULL) &&
121                    info.type == TR_SYS_PATH_IS_DIRECTORY;
122  }
123
124  /* build a list of files containing top file and,
125     if it's a directory, all of its children */
126  {
127    char * dir = tr_sys_path_dirname (ret->top, NULL);
128    char * base = tr_sys_path_basename (ret->top, NULL);
129    files = getFiles (dir, base, NULL);
130    tr_free (base);
131    tr_free (dir);
132  }
133
134  for (walk=files; walk!=NULL; walk=walk->next)
135    ++ret->fileCount;
136
137  ret->files = tr_new0 (tr_metainfo_builder_file, ret->fileCount);
138
139  for (i=0, walk=files; walk!=NULL; ++i)
140    {
141      struct FileList *  tmp = walk;
142      tr_metainfo_builder_file * file = &ret->files[i];
143      walk = walk->next;
144      file->filename = tmp->filename;
145      file->size = tmp->size;
146      ret->totalSize += tmp->size;
147      tr_free (tmp);
148    }
149
150  qsort (ret->files,
151         ret->fileCount,
152         sizeof (tr_metainfo_builder_file),
153         builderFileCompare);
154
155  tr_metaInfoBuilderSetPieceSize (ret, bestPieceSize (ret->totalSize));
156
157  return ret;
158}
159
160static bool
161isValidPieceSize (uint32_t n)
162{
163  const bool isPowerOfTwo = !(n == 0) && !(n & (n - 1));
164
165  return isPowerOfTwo;
166}
167
168bool
169tr_metaInfoBuilderSetPieceSize (tr_metainfo_builder * b,
170                                uint32_t              bytes)
171{
172  if (!isValidPieceSize (bytes))
173    {
174      char wanted[32];
175      char gotten[32];
176      tr_formatter_mem_B (wanted, bytes, sizeof(wanted));
177      tr_formatter_mem_B (gotten, b->pieceSize, sizeof(gotten));
178      tr_logAddError (_("Failed to set piece size to %s, leaving it at %s"),
179                      wanted,
180                      gotten);
181      return false;
182    }
183
184  b->pieceSize = bytes;
185
186  b->pieceCount = (int)(b->totalSize / b->pieceSize);
187  if (b->totalSize % b->pieceSize)
188    ++b->pieceCount;
189
190  return true;
191}
192
193
194void
195tr_metaInfoBuilderFree (tr_metainfo_builder * builder)
196{
197  if (builder)
198    {
199      int i;
200      tr_file_index_t t;
201
202      for (t=0; t<builder->fileCount; ++t)
203        tr_free (builder->files[t].filename);
204      tr_free (builder->files);
205      tr_free (builder->top);
206      tr_free (builder->comment);
207      for (i=0; i<builder->trackerCount; ++i)
208        tr_free (builder->trackers[i].announce);
209      tr_free (builder->trackers);
210      tr_free (builder->outputFile);
211      tr_free (builder);
212    }
213}
214
215/****
216*****
217****/
218
219static uint8_t*
220getHashInfo (tr_metainfo_builder * b)
221{
222  uint32_t fileIndex = 0;
223  uint8_t *ret = tr_new0 (uint8_t, SHA_DIGEST_LENGTH * b->pieceCount);
224  uint8_t *walk = ret;
225  uint8_t *buf;
226  uint64_t totalRemain;
227  uint64_t off = 0;
228  tr_sys_file_t fd;
229  tr_error * error = NULL;
230
231  if (!b->totalSize)
232    return ret;
233
234  buf = tr_valloc (b->pieceSize);
235  b->pieceIndex = 0;
236  totalRemain = b->totalSize;
237  fd = tr_sys_file_open (b->files[fileIndex].filename, TR_SYS_FILE_READ |
238                         TR_SYS_FILE_SEQUENTIAL, 0, &error);
239  if (fd == TR_BAD_SYS_FILE)
240    {
241      b->my_errno = error->code;
242      tr_strlcpy (b->errfile,
243                  b->files[fileIndex].filename,
244                  sizeof (b->errfile));
245      b->result = TR_MAKEMETA_IO_READ;
246      tr_free (buf);
247      tr_free (ret);
248      tr_error_free (error);
249      return NULL;
250    }
251
252  while (totalRemain)
253    {
254      uint8_t * bufptr = buf;
255      const uint32_t thisPieceSize = (uint32_t) MIN (b->pieceSize, totalRemain);
256      uint64_t leftInPiece = thisPieceSize;
257
258      assert (b->pieceIndex < b->pieceCount);
259
260      while (leftInPiece)
261        {
262          const uint64_t n_this_pass = MIN (b->files[fileIndex].size - off, leftInPiece);
263          uint64_t n_read = 0;
264          tr_sys_file_read (fd, bufptr, n_this_pass, &n_read, NULL);
265          bufptr += n_read;
266          off += n_read;
267          leftInPiece -= n_read;
268          if (off == b->files[fileIndex].size)
269            {
270              off = 0;
271              tr_sys_file_close (fd, NULL);
272              fd = TR_BAD_SYS_FILE;
273              if (++fileIndex < b->fileCount)
274                {
275                  fd = tr_sys_file_open (b->files[fileIndex].filename, TR_SYS_FILE_READ |
276                                         TR_SYS_FILE_SEQUENTIAL, 0, &error);
277                  if (fd == TR_BAD_SYS_FILE)
278                    {
279                      b->my_errno = error->code;
280                      tr_strlcpy (b->errfile,
281                                  b->files[fileIndex].filename,
282                                  sizeof (b->errfile));
283                      b->result = TR_MAKEMETA_IO_READ;
284                      tr_free (buf);
285                      tr_free (ret);
286                      tr_error_free (error);
287                      return NULL;
288                    }
289                }
290            }
291        }
292
293      assert (bufptr - buf == (int)thisPieceSize);
294      assert (leftInPiece == 0);
295      tr_sha1 (walk, buf, thisPieceSize, NULL);
296      walk += SHA_DIGEST_LENGTH;
297
298      if (b->abortFlag)
299        {
300          b->result = TR_MAKEMETA_CANCELLED;
301          break;
302        }
303
304      totalRemain -= thisPieceSize;
305      ++b->pieceIndex;
306    }
307
308  assert (b->abortFlag
309        || (walk - ret == (int)(SHA_DIGEST_LENGTH * b->pieceCount)));
310  assert (b->abortFlag || !totalRemain);
311
312  if (fd != TR_BAD_SYS_FILE)
313    tr_sys_file_close (fd, NULL);
314
315  tr_free (buf);
316  return ret;
317}
318
319static void
320getFileInfo (const char                      * topFile,
321             const tr_metainfo_builder_file  * file,
322             tr_variant                      * uninitialized_length,
323             tr_variant                      * uninitialized_path)
324{
325  size_t offset;
326
327  /* get the file size */
328  tr_variantInitInt (uninitialized_length, file->size);
329
330  /* how much of file->filename to walk past */
331  offset = strlen (topFile);
332  if (offset>0 && topFile[offset-1]!=TR_PATH_DELIMITER)
333    ++offset; /* +1 for the path delimiter */
334
335  /* build the path list */
336  tr_variantInitList (uninitialized_path, 0);
337  if (strlen (file->filename) > offset)
338    {
339      char * filename = tr_strdup (file->filename + offset);
340      char * walk = filename;
341      const char * token;
342      while ((token = tr_strsep (&walk, TR_PATH_DELIMITER_STR)))
343        if (*token)
344          tr_variantListAddStr (uninitialized_path, token);
345      tr_free (filename);
346    }
347}
348
349static void
350makeInfoDict (tr_variant          * dict,
351              tr_metainfo_builder * builder)
352{
353  char * base;
354  uint8_t * pch;
355
356  tr_variantDictReserve (dict, 5);
357
358  if (builder->isFolder) /* root node is a directory */
359    {
360      uint32_t  i;
361      tr_variant * list = tr_variantDictAddList (dict, TR_KEY_files,
362                                                 builder->fileCount);
363      for (i=0; i<builder->fileCount; ++i)
364        {
365          tr_variant * d = tr_variantListAddDict (list, 2);
366          tr_variant * length = tr_variantDictAdd (d, TR_KEY_length);
367          tr_variant * pathVal = tr_variantDictAdd (d, TR_KEY_path);
368          getFileInfo (builder->top, &builder->files[i], length, pathVal);
369        }
370    }
371  else
372    {
373      tr_variantDictAddInt (dict, TR_KEY_length, builder->files[0].size);
374    }
375
376  base = tr_sys_path_basename (builder->top, NULL);
377  tr_variantDictAddStr (dict, TR_KEY_name, base);
378  tr_free (base);
379
380  tr_variantDictAddInt (dict, TR_KEY_piece_length, builder->pieceSize);
381
382  if ((pch = getHashInfo (builder)))
383    {
384      tr_variantDictAddRaw (dict, TR_KEY_pieces,
385                            pch,
386                            SHA_DIGEST_LENGTH * builder->pieceCount);
387      tr_free (pch);
388    }
389
390  tr_variantDictAddInt (dict, TR_KEY_private, builder->isPrivate ? 1 : 0);
391}
392
393static void
394tr_realMakeMetaInfo (tr_metainfo_builder * builder)
395{
396  int i;
397  tr_variant top;
398
399  /* allow an empty set, but if URLs *are* listed, verify them. #814, #971 */
400  for (i=0; i<builder->trackerCount && !builder->result; ++i)
401    {
402      if (!tr_urlIsValidTracker (builder->trackers[i].announce))
403        {
404          tr_strlcpy (builder->errfile, builder->trackers[i].announce,
405                      sizeof (builder->errfile));
406          builder->result = TR_MAKEMETA_URL;
407        }
408    }
409
410  tr_variantInitDict (&top, 6);
411
412  if (!builder->fileCount || !builder->totalSize ||
413      !builder->pieceSize || !builder->pieceCount)
414    {
415      builder->errfile[0] = '\0';
416      builder->my_errno = ENOENT;
417      builder->result = TR_MAKEMETA_IO_READ;
418      builder->isDone = true;
419    }
420
421  if (!builder->result && builder->trackerCount)
422    {
423      int prevTier = -1;
424      tr_variant * tier = NULL;
425
426      if (builder->trackerCount > 1)
427        {
428          tr_variant * annList = tr_variantDictAddList (&top, TR_KEY_announce_list, 0);
429          for (i=0; i<builder->trackerCount; ++i)
430            {
431              if (prevTier != builder->trackers[i].tier)
432                {
433                  prevTier = builder->trackers[i].tier;
434                  tier = tr_variantListAddList (annList, 0);
435                }
436              tr_variantListAddStr (tier, builder->trackers[i].announce);
437            }
438        }
439
440      tr_variantDictAddStr (&top, TR_KEY_announce, builder->trackers[0].announce);
441    }
442
443  if (!builder->result && !builder->abortFlag)
444    {
445      if (builder->comment && *builder->comment)
446        tr_variantDictAddStr (&top, TR_KEY_comment, builder->comment);
447      tr_variantDictAddStr (&top, TR_KEY_created_by,
448                            TR_NAME "/" LONG_VERSION_STRING);
449      tr_variantDictAddInt (&top, TR_KEY_creation_date, time (NULL));
450      tr_variantDictAddStr (&top, TR_KEY_encoding, "UTF-8");
451      makeInfoDict (tr_variantDictAddDict (&top, TR_KEY_info, 666), builder);
452    }
453
454  /* save the file */
455  if (!builder->result && !builder->abortFlag)
456    {
457      if (tr_variantToFile (&top, TR_VARIANT_FMT_BENC, builder->outputFile))
458        {
459          builder->my_errno = errno;
460          tr_strlcpy (builder->errfile, builder->outputFile,
461                     sizeof (builder->errfile));
462          builder->result = TR_MAKEMETA_IO_WRITE;
463        }
464    }
465
466  /* cleanup */
467  tr_variantFree (&top);
468  if (builder->abortFlag)
469    builder->result = TR_MAKEMETA_CANCELLED;
470  builder->isDone = true;
471}
472
473/***
474****
475****  A threaded builder queue
476****
477***/
478
479static tr_metainfo_builder * queue = NULL;
480
481static tr_thread * workerThread = NULL;
482
483static tr_lock*
484getQueueLock (void)
485{
486  static tr_lock * lock = NULL;
487
488  if (!lock)
489    lock = tr_lockNew ();
490
491  return lock;
492}
493
494static void
495makeMetaWorkerFunc (void * unused UNUSED)
496{
497  for (;;)
498    {
499      tr_metainfo_builder * builder = NULL;
500
501      /* find the next builder to process */
502      tr_lock * lock = getQueueLock ();
503      tr_lockLock (lock);
504      if (queue)
505        {
506          builder = queue;
507          queue = queue->nextBuilder;
508        }
509      tr_lockUnlock (lock);
510
511      /* if no builders, this worker thread is done */
512      if (builder == NULL)
513        break;
514
515      tr_realMakeMetaInfo (builder);
516    }
517
518  workerThread = NULL;
519}
520
521void
522tr_makeMetaInfo (tr_metainfo_builder   * builder,
523                 const char            * outputFile,
524                 const tr_tracker_info * trackers,
525                 int                     trackerCount,
526                 const char            * comment,
527                 bool                    isPrivate)
528{
529  int i;
530  tr_lock * lock;
531
532  assert (tr_isBool (isPrivate));
533
534  /* free any variables from a previous run */
535  for (i=0; i<builder->trackerCount; ++i)
536    tr_free (builder->trackers[i].announce);
537  tr_free (builder->trackers);
538  tr_free (builder->comment);
539  tr_free (builder->outputFile);
540
541  /* initialize the builder variables */
542  builder->abortFlag = false;
543  builder->result = 0;
544  builder->isDone = false;
545  builder->pieceIndex = 0;
546  builder->trackerCount = trackerCount;
547  builder->trackers = tr_new0 (tr_tracker_info, builder->trackerCount);
548  for (i=0; i<builder->trackerCount; ++i)
549    {
550      builder->trackers[i].tier = trackers[i].tier;
551      builder->trackers[i].announce = tr_strdup (trackers[i].announce);
552    }
553  builder->comment = tr_strdup (comment);
554  builder->isPrivate = isPrivate;
555  if (outputFile && *outputFile)
556    builder->outputFile = tr_strdup (outputFile);
557  else
558    builder->outputFile = tr_strdup_printf ("%s.torrent", builder->top);
559
560  /* enqueue the builder */
561  lock = getQueueLock ();
562  tr_lockLock (lock);
563  builder->nextBuilder = queue;
564  queue = builder;
565  if (!workerThread)
566    workerThread = tr_threadNew (makeMetaWorkerFunc, NULL);
567  tr_lockUnlock (lock);
568}
569
Note: See TracBrowser for help on using the repository browser.