Changeset 14651

Timestamp:

Jan 2, 2016, 2:28:59 PM (6 years ago)

Author:

mikedld

Message:

#5663: Rework directory watching in daemon

Implement BSD/Darwin (kqueue) and Windows (ReadDirectoryChanges?) mechanisms
for receiving directory change notifications. Use events instead of polling
for changes. Retry file parsing up to 3 times before giving up.

Huge thanks to missionsix for preparing first two versions of the patch.

Location:

trunk

Files:

• Transmission.xcodeproj/project.pbxproj (modified) (11 diffs)
• configure.ac (modified) (1 diff)
• daemon/CMakeLists.txt (modified) (3 diffs)
• daemon/Makefile.am (modified) (1 diff)
• daemon/daemon.c (modified) (14 diffs)
• daemon/watch.c (deleted)
• daemon/watch.h (deleted)
• libtransmission/CMakeLists.txt (modified) (5 diffs)
• libtransmission/Makefile.am (modified) (4 diffs)
• libtransmission/watchdir-common.h (added)
• libtransmission/watchdir-generic.c (added)
• libtransmission/watchdir-inotify.c (added)
• libtransmission/watchdir-kqueue.c (added)
• libtransmission/watchdir-test.c (added)
• libtransmission/watchdir-win32.c (added)
• libtransmission/watchdir.c (added)
• libtransmission/watchdir.h (added)

trunk/Transmission.xcodeproj/project.pbxproj

@@ -294 +294 @@
                 A2D307A40D9EC6870051FD27 /* BlocklistDownloader.m in Sources */ = {isa = PBXBuildFile; fileRef = A2D307A30D9EC6870051FD27 /* BlocklistDownloader.m */; };
                 A2D307B10D9EC9F50051FD27 /* BlocklistStatusWindow.xib in Resources */ = {isa = PBXBuildFile; fileRef = A2D307B00D9EC9F50051FD27 /* BlocklistStatusWindow.xib */; };
-                A2D5972A0F5AE49E0001AB3C /* watch.c in Sources */ = {isa = PBXBuildFile; fileRef = A2D597280F5AE49E0001AB3C /* watch.c */; };
                 A2D77451154CC25700A62B93 /* WebSeedTableView.h in Headers */ = {isa = PBXBuildFile; fileRef = A2D7744F154CC25700A62B93 /* WebSeedTableView.h */; };
                 A2D77452154CC25700A62B93 /* WebSeedTableView.m in Sources */ = {isa = PBXBuildFile; fileRef = A2D77450154CC25700A62B93 /* WebSeedTableView.m */; };
@@ -474 +473 @@
                 C1639A7D1A55F57200E42033 /* cencode.h in Headers */ = {isa = PBXBuildFile; fileRef = C1639A7B1A55F57200E42033 /* cencode.h */; };
                 C1F690FD1AD0627500D95CF0 /* daemon-posix.c in Sources */ = {isa = PBXBuildFile; fileRef = C1F690FC1AD0627500D95CF0 /* daemon-posix.c */; };
+                C1FEE5771C3223CC00D62832 /* watchdir-common.h in Headers */ = {isa = PBXBuildFile; fileRef = C1FEE5721C3223CC00D62832 /* watchdir-common.h */; };
+                C1FEE5781C3223CC00D62832 /* watchdir-generic.c in Sources */ = {isa = PBXBuildFile; fileRef = C1FEE5731C3223CC00D62832 /* watchdir-generic.c */; };
+                C1FEE5791C3223CC00D62832 /* watchdir-kqueue.c in Sources */ = {isa = PBXBuildFile; fileRef = C1FEE5741C3223CC00D62832 /* watchdir-kqueue.c */; };
+                C1FEE57A1C3223CC00D62832 /* watchdir.c in Sources */ = {isa = PBXBuildFile; fileRef = C1FEE5751C3223CC00D62832 /* watchdir.c */; };
+                C1FEE57B1C3223CC00D62832 /* watchdir.h in Headers */ = {isa = PBXBuildFile; fileRef = C1FEE5761C3223CC00D62832 /* watchdir.h */; };
                 D4AF3B2F0C41F7A500D46B6B /* list.c in Sources */ = {isa = PBXBuildFile; fileRef = D4AF3B2D0C41F7A500D46B6B /* list.c */; };
                 D4AF3B300C41F7A600D46B6B /* list.h in Headers */ = {isa = PBXBuildFile; fileRef = D4AF3B2E0C41F7A500D46B6B /* list.h */; };
@@ -1035 +1039 @@
                 A2D307A30D9EC6870051FD27 /* BlocklistDownloader.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; name = BlocklistDownloader.m; path = macosx/BlocklistDownloader.m; sourceTree = "<group>"; };
                 A2D307B00D9EC9F50051FD27 /* BlocklistStatusWindow.xib */ = {isa = PBXFileReference; lastKnownFileType = file.xib; name = BlocklistStatusWindow.xib; path = macosx/BlocklistStatusWindow.xib; sourceTree = "<group>"; };
-                A2D597280F5AE49E0001AB3C /* watch.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; name = watch.c; path = daemon/watch.c; sourceTree = "<group>"; };
-                A2D597290F5AE49E0001AB3C /* watch.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = watch.h; path = daemon/watch.h; sourceTree = "<group>"; };
                 A2D7744F154CC25700A62B93 /* WebSeedTableView.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = WebSeedTableView.h; path = macosx/WebSeedTableView.h; sourceTree = "<group>"; };
                 A2D77450154CC25700A62B93 /* WebSeedTableView.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; name = WebSeedTableView.m; path = macosx/WebSeedTableView.m; sourceTree = "<group>"; };
@@ -1231 +1233 @@
                 C1F690FC1AD0627500D95CF0 /* daemon-posix.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; name = "daemon-posix.c"; path = "daemon/daemon-posix.c"; sourceTree = "<group>"; };
                 C1F690FE1AD0628400D95CF0 /* daemon.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = daemon.h; path = daemon/daemon.h; sourceTree = "<group>"; };
+                C1FEE5721C3223CC00D62832 /* watchdir-common.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = "watchdir-common.h"; path = "libtransmission/watchdir-common.h"; sourceTree = "<group>"; };
+                C1FEE5731C3223CC00D62832 /* watchdir-generic.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = "watchdir-generic.c"; path = "libtransmission/watchdir-generic.c"; sourceTree = "<group>"; };
+                C1FEE5741C3223CC00D62832 /* watchdir-kqueue.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = "watchdir-kqueue.c"; path = "libtransmission/watchdir-kqueue.c"; sourceTree = "<group>"; };
+                C1FEE5751C3223CC00D62832 /* watchdir.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; name = watchdir.c; path = libtransmission/watchdir.c; sourceTree = "<group>"; };
+                C1FEE5761C3223CC00D62832 /* watchdir.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = watchdir.h; path = libtransmission/watchdir.h; sourceTree = "<group>"; };
                 D4AF3B2D0C41F7A500D46B6B /* list.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; name = list.c; path = libtransmission/list.c; sourceTree = "<group>"; };
                 D4AF3B2E0C41F7A500D46B6B /* list.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = list.h; path = libtransmission/list.h; sourceTree = "<group>"; };
@@ -1811 +1818 @@
                                 A2A4EA0A0DE106E8000CE197 /* ConvertUTF.c */,
                                 4DB74F070E8CD75100AEB1A8 /* wildmat.c */,
+                                C1FEE5751C3223CC00D62832 /* watchdir.c */,
+                                C1FEE5761C3223CC00D62832 /* watchdir.h */,
+                                C1FEE5731C3223CC00D62832 /* watchdir-generic.c */,
+                                C1FEE5741C3223CC00D62832 /* watchdir-kqueue.c */,
+                                C1FEE5721C3223CC00D62832 /* watchdir-common.h */,
                         );
                         name = libtransmission;
@@ -2013 +2025 @@
                                 C1F690FC1AD0627500D95CF0 /* daemon-posix.c */,
                                 BEFC1C140C07756200B0BB3C /* remote.c */,
-                                A2D597280F5AE49E0001AB3C /* watch.c */,
-                                A2D597290F5AE49E0001AB3C /* watch.h */,
                         );
                         name = daemon;
@@ -2123 +2133 @@
                                 BEFC1E490C07861A00B0BB3C /* metainfo.h in Headers */,
                                 BEFC1E4D0C07861A00B0BB3C /* session.h in Headers */,
+                                C1FEE5771C3223CC00D62832 /* watchdir-common.h in Headers */,
                                 BEFC1E4E0C07861A00B0BB3C /* inout.h in Headers */,
                                 BEFC1E520C07861A00B0BB3C /* fdlimit.h in Headers */,
@@ -2141 +2152 @@
                                 A29DF8BB0DB2544C00D04E5A /* torrent.h in Headers */,
                                 A29DF8BE0DB2545F00D04E5A /* verify.h in Headers */,
+                                C1FEE57B1C3223CC00D62832 /* watchdir.h in Headers */,
                                 A2AAB6650DE0D08B00E04DDA /* blocklist.h in Headers */,
                                 A2A4E9210DE0F7E9000CE197 /* web.h in Headers */,
@@ -2754 +2766 @@
                                 BEFC1E4F0C07861A00B0BB3C /* inout.c in Sources */,
                                 BEFC1E530C07861A00B0BB3C /* fdlimit.c in Sources */,
+                                C1FEE5781C3223CC00D62832 /* watchdir-generic.c in Sources */,
                                 BEFC1E560C07861A00B0BB3C /* completion.c in Sources */,
                                 BEFC1E580C07861A00B0BB3C /* clients.c in Sources */,
@@ -2787 +2800 @@
                                 A209EE5C1144B51E002B02D1 /* history.c in Sources */,
                                 A220EC5B118C8A060022B4BE /* tr-lpd.c in Sources */,
+                                C1FEE57A1C3223CC00D62832 /* watchdir.c in Sources */,
                                 A23547E211CD0B090046EAE6 /* cache.c in Sources */,
                                 A284214412DA663E00FBDDBB /* tr-udp.c in Sources */,
                                 A2679294130E00A000CB7464 /* tr-utp.c in Sources */,
                                 A23F29A2132A447400E9A83B /* announcer-http.c in Sources */,
+                                C1FEE5791C3223CC00D62832 /* watchdir-kqueue.c in Sources */,
                                 A2AA9BE1132CAC8E00FA131E /* announcer-udp.c in Sources */,
                                 A2D77452154CC25700A62B93 /* WebSeedTableView.m in Sources */,
@@ -2967 +2982 @@
                         files = (
                                 BEFC1C1A0C07756200B0BB3C /* daemon.c in Sources */,
-                                A2D5972A0F5AE49E0001AB3C /* watch.c in Sources */,
                                 C1F690FD1AD0627500D95CF0 /* daemon-posix.c in Sources */,
                         );

trunk/configure.ac

@@ -205 +205 @@
 dnl file monitoring for the daemon
 
-AC_CHECK_HEADER([sys/inotify.h],
-                [AC_CHECK_FUNC([inotify_init],[have_inotify="yes"],[have_inotify="no"])],
-                [have_inotify="no"])
 AC_ARG_WITH([inotify],
-            [AS_HELP_STRING([--with-inotify],[Enable inotify support (default=auto)])],
-            [want_inotify=${withval}],
-            [want_inotify=${have_inotify}])
-if test "x$want_inotify" = "xyes" ; then
-    if test "x$have_inotify" = "xyes"; then
-      AC_DEFINE([WITH_INOTIFY],[1])
-    else
-      AC_MSG_ERROR("inotify not found!")
-    fi
-fi
-
-AC_CHECK_HEADER([sys/event.h],
-                [AC_CHECK_FUNC([kqueue],[have_kqueue="yes"],[have_kqueue="no"])],
-                [have_kqueue="no"])
+            [AS_HELP_STRING([--with-inotify], [Enable inotify support (default=auto)])],
+            [WANT_INOTIFY=${withval}],
+            [WANT_INOTIFY=auto])
+HAVE_INOTIFY=0
+AS_IF([test "x$WANT_INOTIFY" != "xno"],
+      [AC_CHECK_HEADER([sys/inotify.h],
+                       [AC_CHECK_FUNC([inotify_init],
+                                      [HAVE_INOTIFY=1])],
+                       [AS_IF([test "x$WANT_INOTIFY" = "xyes"],
+                              [AC_MSG_ERROR("inotify not found!")])])])
+AM_CONDITIONAL([USE_INOTIFY], [test "x$WANT_INOTIFY" != "xno" -a $HAVE_INOTIFY -eq 1])
+
 AC_ARG_WITH([kqueue],
             [AS_HELP_STRING([--with-kqueue],[Enable kqueue support (default=auto)])],
-            [want_kqueue=${withval}],
-            [want_kqueue=${have_kqueue}])
-if test "x$want_kqueue" = "xyes" ; then
-    if test "x$have_kqueue" = "xyes"; then
-      AC_DEFINE([WITH_KQUEUE],[1])
-    else
-      AC_MSG_ERROR("kqueue not found!")
-    fi
-fi
+            [WITH_KQUEUE=${withval}],
+            [WITH_KQUEUE=auto])
+HAVE_KQUEUE=0
+AS_IF([test "x$WITH_KQUEUE" != "xno"],
+      [AC_CHECK_HEADER([sys/event.h],
+                       [AC_CHECK_FUNC([kqueue],
+                                      [HAVE_KQUEUE=1])],
+                       [AS_IF([test "x$WANT_KQUEUE" = "xyes"],
+                              [AC_MSG_ERROR("kqueue not found!")])])])
+AM_CONDITIONAL([USE_KQUEUE], [test "x$WITH_KQUEUE" != "xno" -a $HAVE_KQUEUE -eq 1])
+
 
 AC_CHECK_HEADERS([sys/statvfs.h \

trunk/daemon/CMakeLists.txt

@@ -1 +1 @@
 project(trdaemon)
-
-if(WITH_INOTIFY)
-    add_definitions(-DWITH_INOTIFY)
-endif()
-
-if(WITH_KQUEUE)
-    add_definitions(-DWITH_KQUEUE)
-endif()
 
 if(WITH_SYSTEMD)
@@ -23 +15 @@
     daemon-posix.c
     daemon-win32.c
-    watch.c
 )
 
@@ -34 +25 @@
 set(${PROJECT_NAME}_HEADERS
     daemon.h
-    watch.h
 )
 

trunk/daemon/Makefile.am

@@ -35 +35 @@
     ${LIBM}
 
-noinst_HEADERS = \
-    daemon.h \
-    watch.h
+noinst_HEADERS = daemon.h
 
-transmission_daemon_SOURCES = daemon.c watch.c
+transmission_daemon_SOURCES = daemon.c
 transmission_remote_SOURCES = remote.c
 

trunk/daemon/daemon.c

@@ -22 +22 @@
 #endif
 
-#include <event2/buffer.h>
 #include <event2/event.h>
 
@@ -33 +32 @@
 #include <libtransmission/variant.h>
 #include <libtransmission/version.h>
+#include <libtransmission/watchdir.h>
 
 #ifdef USE_SYSTEMD_DAEMON
@@ -42 +42 @@
 
 #include "daemon.h"
-#include "watch.h"
 
 #define MY_NAME "transmission-daemon"
@@ -186 +185 @@
 }
 
-static void
-onFileAdded (tr_session * session, const char * dir, const char * file)
-{
-    char * filename = tr_buildPath (dir, file, NULL);
+static tr_watchdir_status
+onFileAdded (tr_watchdir_t   dir,
+             const char    * name,
+             void          * context)
+{
+    tr_session * session = context;
+
+    if (!tr_str_has_suffix (name, ".torrent"))
+        return TR_WATCHDIR_IGNORE;
+
+    char * filename = tr_buildPath (tr_watchdir_get_path (dir), name, NULL);
     tr_ctor * ctor = tr_ctorNew (session);
     int err = tr_ctorSetMetainfoFromFile (ctor, filename);
@@ -198 +204 @@
 
         if (err == TR_PARSE_ERR)
-            tr_logAddError ("Error parsing .torrent file \"%s\"", file);
+            tr_logAddError ("Error parsing .torrent file \"%s\"", name);
         else
         {
@@ -204 +210 @@
             const bool test = tr_ctorGetDeleteSource (ctor, &trash);
 
-            tr_logAddInfo ("Parsing .torrent file successful \"%s\"", file);
+            tr_logAddInfo ("Parsing .torrent file successful \"%s\"", name);
 
             if (test && trash)
@@ -210 +216 @@
                 tr_error * error = NULL;
 
-                tr_logAddInfo ("Deleting input .torrent file \"%s\"", file);
+                tr_logAddInfo ("Deleting input .torrent file \"%s\"", name);
                 if (!tr_sys_path_remove (filename, &error))
                 {
@@ -225 +231 @@
         }
     }
+    else
+    {
+        err = TR_PARSE_ERR;
+    }
 
     tr_ctorFree (ctor);
     tr_free (filename);
+
+    return err == TR_PARSE_ERR ? TR_WATCHDIR_RETRY : TR_WATCHDIR_ACCEPT;
 }
 
@@ -294 +306 @@
 
 static void
-periodicUpdate (evutil_socket_t fd UNUSED, short what UNUSED, void *watchdir)
-{
-    dtr_watchdir_update (watchdir);
-
+periodicUpdate (evutil_socket_t   fd UNUSED,
+                short             what UNUSED,
+                void            * context UNUSED)
+{
     pumpLogMessages (logfile);
-
     reportStatus ();
 }
@@ -480 +491 @@
     bool boolVal;
     const char * pid_filename;
-    dtr_watchdir * watchdir = NULL;
     bool pidfile_created = false;
     tr_session * session = NULL;
-    struct event *status_ev;
+    struct event * status_ev = NULL;
+    tr_watchdir_t watchdir = NULL;
 
     struct daemon_data * const arg = raw_arg;
@@ -559 +570 @@
         {
             tr_logAddInfo ("Watching \"%s\" for new .torrent files", dir);
-            watchdir = dtr_watchdir_new (mySession, dir, onFileAdded);
+            if ((watchdir = tr_watchdir_new (dir, &onFileAdded, mySession, ev_base)) == NULL)
+                goto cleanup;
         }
     }
@@ -582 +594 @@
     {
         struct timeval one_sec = { 1, 0 };
-        status_ev = event_new(ev_base, -1, EV_PERSIST, &periodicUpdate, watchdir);
+        status_ev = event_new(ev_base, -1, EV_PERSIST, &periodicUpdate, NULL);
         if (status_ev == NULL)
         {
@@ -608 +620 @@
     printf ("Closing transmission session...");
 
+    tr_watchdir_free (watchdir);
+
     if (status_ev)
     {
@@ -616 +630 @@
 
     tr_sessionSaveSettings (mySession, configDir, settings);
-    dtr_watchdir_free (watchdir);
     tr_sessionClose (mySession);
     pumpLogMessages (logfile);

trunk/libtransmission/CMakeLists.txt

@@ -63 +63 @@
     variant-json.c
     verify.c
+    watchdir.c
+    watchdir-generic.c
+    watchdir-inotify.c
+    watchdir-kqueue.c
+    watchdir-win32.c
     web.c
     webseed.c
@@ -75 +80 @@
 endforeach()
 
+if(WITH_INOTIFY)
+    add_definitions(-DWITH_INOTIFY)
+else()
+    set_source_files_properties(watchdir-inotify.c PROPERTIES HEADER_FILE_ONLY ON)
+endif()
+
+if(WITH_KQUEUE)
+    add_definitions(-DWITH_KQUEUE)
+else()
+    set_source_files_properties(watchdir-kqueue.c PROPERTIES HEADER_FILE_ONLY ON)
+endif()
+
 if(WIN32)
     set_source_files_properties(file-posix.c PROPERTIES HEADER_FILE_ONLY ON)
 else()
-    set_source_files_properties(file-win32.c PROPERTIES HEADER_FILE_ONLY ON)
+    set_source_files_properties(file-win32.c watchdir-win32.c PROPERTIES HEADER_FILE_ONLY ON)
 endif()
 
@@ -93 +110 @@
     utils.h
     variant.h
+    watchdir.h
     web.h
     ${PROJECT_BINARY_DIR}/version.h
@@ -141 +159 @@
     verify.h
     version.h
+    watchdir-common.h
     webseed.h
 )
@@ -241 +260 @@
     set(crypto-test_ADD_SOURCES crypto-test-ref.h)
 
-    foreach(T bitfield blocklist clients crypto error file history json magnet metainfo move peer-msgs quark rename rpc session tr-getopt utils variant)
+    foreach(T bitfield blocklist clients crypto error file history json magnet metainfo move peer-msgs quark rename rpc session
+              tr-getopt utils variant watchdir)
         set(TP ${TR_NAME}-test-${T})
         add_executable(${TP} ${T}-test.c ${${T}-test_ADD_SOURCES})

trunk/libtransmission/Makefile.am

@@ -73 +73 @@
   variant-json.c \
   verify.c \
+  watchdir.c \
+  watchdir-generic.c \
   web.c \
   webseed.c \
   wildmat.c
 
+if USE_INOTIFY
+libtransmission_a_SOURCES += watchdir-inotify.c
+endif
+
+if USE_KQUEUE
+libtransmission_a_SOURCES += watchdir-kqueue.c
+endif
+
 if WIN32
-libtransmission_a_SOURCES += file-win32.c
+libtransmission_a_SOURCES += file-win32.c watchdir-win32.c
 else
 libtransmission_a_SOURCES += file-posix.c
@@ -151 +161 @@
   verify.h \
   version.h \
+  watchdir.h \
+  watchdir-common.h \
   web.h \
   webseed.h
@@ -174 +186 @@
   tr-getopt-test \
   utils-test \
-  variant-test
+  variant-test \
+  watchdir-test
 
 noinst_PROGRAMS = $(TESTS)
@@ -271 +284 @@
 variant_test_LDFLAGS = ${apps_ldflags}
 
+watchdir_test_SOURCES = watchdir-test.c $(TEST_SOURCES)
+watchdir_test_LDADD = ${apps_ldadd}
+watchdir_test_LDFLAGS = ${apps_ldflags}
+
 rename_test_SOURCES = rename-test.c $(TEST_SOURCES)
 rename_test_LDADD = ${apps_ldadd}