diff --git a/.gitignore b/.gitignore index 8186834..474bb95 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ config.status # Binary outputs cloudfuse hubicfuse + +project-vc/ \ No newline at end of file diff --git a/.hubicfuse.sample b/.hubicfuse.sample new file mode 100644 index 0000000..ee8dc5b --- /dev/null +++ b/.hubicfuse.sample @@ -0,0 +1,13 @@ +client_id = api_hubic_xxxxx +client_secret = xxxxxx +refresh_token = xxxx +cache_timeout = 600 +temp_dir = /tmp/hubicfuse + +get_extended_metadata = true +curl_verbose = false +curl_progress_state = true +cache_statfs_timeout = 15 +debug_level = 0 +enable_chmod = false +enable_chown = false diff --git a/Makefile.in b/Makefile.in index 29c7354..54f2548 100644 --- a/Makefile.in +++ b/Makefile.in @@ -11,8 +11,8 @@ prefix = @prefix@ exec_prefix = @exec_prefix@ bindir = $(DESTDIR)$(exec_prefix)/bin -SOURCES=cloudfsapi.c cloudfuse.c -HEADERS=cloudfsapi.h +SOURCES=cloudfsapi.c cloudfuse.c commonfs.c +HEADERS=cloudfsapi.h commonfs.h all: hubicfuse @@ -29,7 +29,7 @@ hubicfuse: $(SOURCES) $(HEADERS) $(CC) $(CFLAGS) -o hubicfuse $(SOURCES) $(LIBS) $(LDFLAGS) clean: - /bin/rm -f hubicfuse + /bin/rm -fr hubicfuse *.orig config.h *~ aclocal.m4 autom4te.cache distclean: clean /bin/rm -f Makefile config.h config.status config.cache config.log \ @@ -39,6 +39,9 @@ mostlyclean: clean maintainer-clean: clean +debug: CFLAGS += -g +debug: hubicfuse + config.h.in: stamp-h.in stamp-h.in: configure.in autoheader @@ -52,3 +55,18 @@ Makefile: Makefile.in config.status config.status: configure ./config.status --recheck +reformat: + astyle --style=allman \ + --convert-tabs \ + --indent=spaces=2 \ + --lineend=linux \ + --max-code-length=79 \ + --pad-oper \ + --pad-header \ + --align-pointer=type \ + --align-reference=name \ + --break-closing-brackets \ + --min-conditional-indent=0 \ + --remove-brackets \ + --remove-comment-prefix \ + *.c *.h diff --git a/README b/README index 083efc8..e345d16 100644 --- a/README +++ b/README @@ -39,6 +39,15 @@ USE: client_secret=[Hubic client secret for the registered application] refresh_token=[A refresh token you got from the script] + Optional variables: + get_extended_metadata=[true/false, force download of additional file metadata like atime and mtime on first directory list] + curl_verbose=[true/false, enable verbose output for curl HTTP requests] + curl_progress_state=[true/false, enable verbose progress output for curl HTTP requests. Used for debugging.] + cache_statfs_timeout=[value in seconds, large timeout increases the file access speed] + debug_level=[0 default, 1 extremely verbose for debugging purposes] + enable_chmod=[true/false, false by default, still experimental feature] + enable_chown=[true/false, false by default, still experimental feature] + client_id, client_secret can be retrieved from the hubic web interface: https://hubic.com/home/browser/developers/ @@ -83,7 +92,20 @@ BUGS/SHORTCOMINGS: "application/directory". * Cloud Files limits container and object listings to 10,000 items. cloudfuse won't list more than that many files in a single directory. + * File copy progress when uploading does not work, progress is shown when file + is copied in local cache, then upload operation happens at 100% +Recent additions/fixes: + * Support for atime, mtime, chmod, chown. + * Large files (segmented) have correct size listed (was 0 before). + * Multiple speed improvements, minimised the number of HTTP calls and added more caching features. + * Fixed many segmentation faults. + * Cached files are deleted on cache expiration when using a custom temp folder. + * Files copied have attributes preserved. + * Working well with rsync due to mtime support and proper copy operations. + * Debugging support for http progress (track upload / download speed etc.) + * Reduced traffic, skips file uploads to cloud if content does not change (using md5sum compare) + * Major code refactoring, code documentation, extensive debugging, additional config options AWESOME CONTRIBUTORS: @@ -97,6 +119,7 @@ AWESOME CONTRIBUTORS: * Mike Lundy https://github.com/novas0x2a * justinb https://github.com/justinsb * Matt Greenway https://github.com/LabAdvComp + * Dan Cristian https://github.com/dan-cristian Thanks, and I hope you find it useful. diff --git a/cloudfsapi.c b/cloudfsapi.c index 18be34e..c52ce17 100644 --- a/cloudfsapi.c +++ b/cloudfsapi.c @@ -20,21 +20,15 @@ #include #include #include +#include "commonfs.h" #include "cloudfsapi.h" #include "config.h" #include - #define RHEL5_LIBCURL_VERSION 462597 #define RHEL5_CERTIFICATE_FILE "/etc/pki/tls/certs/ca-bundle.crt" - -#define REQUEST_RETRIES 4 - +#define REQUEST_RETRIES 3 #define MAX_FILES 10000 - -// 64 bit time + nanoseconds -#define TIME_CHARS 32 - // size of buffer for writing to disk look at ioblksize.h in coreutils // and try some values on your own system if you want the best performance #define DISK_BUFF_SIZE 32768 @@ -42,12 +36,20 @@ static char storage_url[MAX_URL_SIZE]; static char storage_token[MAX_HEADER_SIZE]; static pthread_mutex_t pool_mut; -static CURL *curl_pool[1024]; +static CURL* curl_pool[1024]; static int curl_pool_count = 0; -static int debug = 0; -static int verify_ssl = 2; +extern int debug; +extern int verify_ssl; +extern bool option_get_extended_metadata; +extern bool option_curl_verbose; +extern int option_curl_progress_state; +extern int option_cache_statfs_timeout; +extern bool option_extensive_debug; +extern bool option_enable_chown; +extern bool option_enable_chmod; static int rhel5_mode = 0; -static struct statvfs statcache = { +static struct statvfs statcache = +{ .f_bsize = 4096, .f_frsize = 4096, .f_blocks = INT_MAX, @@ -58,13 +60,19 @@ static struct statvfs statcache = { .f_favail = 0, .f_namemax = INT_MAX }; - +//used to compute statfs cache interval +static time_t last_stat_read_time = 0; extern FuseOptions options; +struct MemoryStruct +{ + char* memory; + size_t size; +}; #ifdef HAVE_OPENSSL #include -static pthread_mutex_t *ssl_lockarray; -static void lock_callback(int mode, int type, char *file, int line) +static pthread_mutex_t* ssl_lockarray; +static void lock_callback(int mode, int type, char* file, int line) { if (mode & CRYPTO_LOCK) pthread_mutex_lock(&(ssl_lockarray[type])); @@ -78,47 +86,72 @@ static unsigned long thread_id() } #endif -static size_t xml_dispatch(void *ptr, size_t size, size_t nmemb, void *stream) +static size_t xml_dispatch(void* ptr, size_t size, size_t nmemb, void* stream) { - xmlParseChunk((xmlParserCtxtPtr)stream, (char *)ptr, size * nmemb, 0); + xmlParseChunk((xmlParserCtxtPtr)stream, (char*)ptr, size * nmemb, 0); return size * nmemb; } -static CURL *get_connection(const char *path) +static CURL* get_connection(const char* path) { pthread_mutex_lock(&pool_mut); - CURL *curl = curl_pool_count ? curl_pool[--curl_pool_count] : curl_easy_init(); + CURL* curl = curl_pool_count ? curl_pool[--curl_pool_count] : curl_easy_init(); if (!curl) { - debugf("curl alloc failed"); + debugf(DBG_LEVEL_NORM, KRED"curl alloc failed"); abort(); } pthread_mutex_unlock(&pool_mut); return curl; } -static void return_connection(CURL *curl) +static void return_connection(CURL* curl) { pthread_mutex_lock(&pool_mut); curl_pool[curl_pool_count++] = curl; pthread_mutex_unlock(&pool_mut); } -static void add_header(curl_slist **headers, const char *name, - const char *value) +static void add_header(curl_slist** headers, const char* name, + const char* value) { char x_header[MAX_HEADER_SIZE]; - snprintf(x_header, sizeof(x_header), "%s: %s", name, value); + char safe_value[256]; + const char* value_ptr; + + debugf(DBG_LEVEL_EXT, "add_header(%s:%s)", name, value); + if (strlen(value) > 256) + { + debugf(DBG_LEVEL_NORM, KRED"add_header: warning, value size > 256 (%s:%s) ", + name, value); + //hubic will throw an HTTP 400 error on X-Copy-To operation if X-Object-Meta-FilePath header value is larger than 256 chars + //fix for issue #95 https://github.com/TurboGit/hubicfuse/issues/95 + if (!strcasecmp(name, "X-Object-Meta-FilePath")) + { + debugf(DBG_LEVEL_NORM, + KRED"add_header: trimming header (%s) value to max allowed", name); + //trim header size to max allowed + strncpy(safe_value, value, 256 - 1); + safe_value[255] = '\0'; + value_ptr = safe_value; + } + else + value_ptr = value; + } + else + value_ptr = value; + + snprintf(x_header, sizeof(x_header), "%s: %s", name, value_ptr); *headers = curl_slist_append(*headers, x_header); } -static size_t header_dispatch(void *ptr, size_t size, size_t nmemb, void *stream) +static size_t header_dispatch(void* ptr, size_t size, size_t nmemb, + void* dir_entry) { - debugf("Dispatching response headers"); - char *header = (char *)alloca(size * nmemb + 1); - char *head = (char *)alloca(size * nmemb + 1); - char *value = (char *)alloca(size * nmemb + 1); - memcpy(header, (char *)ptr, size * nmemb); + char* header = (char*)alloca(size * nmemb + 1); + char* head = (char*)alloca(size * nmemb + 1); + char* value = (char*)alloca(size * nmemb + 1); + memcpy(header, (char*)ptr, size * nmemb); header[size * nmemb] = '\0'; if (sscanf(header, "%[^:]: %[^\r\n]", head, value) == 2) { @@ -127,10 +160,13 @@ static size_t header_dispatch(void *ptr, size_t size, size_t nmemb, void *stream if (!strncasecmp(head, "x-storage-url", size * nmemb)) strncpy(storage_url, value, sizeof(storage_url)); if (!strncasecmp(head, "x-account-meta-quota", size * nmemb)) - statcache.f_blocks = (unsigned long) (strtoull(value, NULL, 10)/statcache.f_frsize); + statcache.f_blocks = (unsigned long) (strtoull(value, NULL, + 10) / statcache.f_frsize); if (!strncasecmp(head, "x-account-bytes-used", size * nmemb)) - statcache.f_bfree = statcache.f_bavail = statcache.f_blocks - (unsigned long) (strtoull(value, NULL, 10)/statcache.f_frsize); - if (!strncasecmp(head, "x-account-object-count", size * nmemb)) { + statcache.f_bfree = statcache.f_bavail = statcache.f_blocks - (unsigned long) ( + strtoull(value, NULL, 10) / statcache.f_frsize); + if (!strncasecmp(head, "x-account-object-count", size * nmemb)) + { unsigned long object_count = strtoul(value, NULL, 10); statcache.f_ffree = MAX_FILES - object_count; statcache.f_favail = MAX_FILES - object_count; @@ -139,77 +175,292 @@ static size_t header_dispatch(void *ptr, size_t size, size_t nmemb, void *stream return size * nmemb; } -static size_t rw_callback(size_t (*rw)(void*, size_t, size_t, FILE*), void *ptr, - size_t size, size_t nmemb, void *userp) +static void header_set_time_from_str(char* time_str, + struct timespec* time_entry) { - struct segment_info *info = (struct segment_info *)userp; - size_t mem = size * nmemb; + char sec_value[TIME_CHARS] = { 0 }; + char nsec_value[TIME_CHARS] = { 0 }; + time_t sec; + long nsec; + sscanf(time_str, "%[^.].%[^\n]", sec_value, nsec_value); + sec = strtoll(sec_value, NULL, 10);//to allow for larger numbers + nsec = atol(nsec_value); + debugf(DBG_LEVEL_EXTALL, "Received time=%s.%s / %li.%li, existing=%li.%li", + sec_value, nsec_value, sec, nsec, time_entry->tv_sec, time_entry->tv_nsec); + if (sec != time_entry->tv_sec || nsec != time_entry->tv_nsec) + { + debugf(DBG_LEVEL_EXTALL, + "Time changed, setting new time=%li.%li, existing was=%li.%li", + sec, nsec, time_entry->tv_sec, time_entry->tv_nsec); + time_entry->tv_sec = sec; + time_entry->tv_nsec = nsec; - if (mem < 1 || info->size < 1) - return 0; + char time_str_local[TIME_CHARS] = ""; + get_time_as_string((time_t)sec, nsec, time_str_local, sizeof(time_str_local)); + debugf(DBG_LEVEL_EXTALL, "header_set_time_from_str received time=[%s]", + time_str_local); - size_t amt_read = rw(ptr, 1, info->size < mem ? info->size : mem, info->fp); - info->size -= amt_read; - - return amt_read; + get_timespec_as_str(time_entry, time_str_local, sizeof(time_str_local)); + debugf(DBG_LEVEL_EXTALL, "header_set_time_from_str set time=[%s]", + time_str_local); + } } -size_t fwrite2(void *ptr, size_t size, size_t nmemb, FILE *filep) +static size_t header_get_meta_dispatch(void* ptr, size_t size, size_t nmemb, + void* userdata) { - return fwrite((const void*)ptr, size, nmemb, filep); + char* header = (char*)alloca(size * nmemb + 1); + char* head = (char*)alloca(size * nmemb + 1); + char* value = (char*)alloca(size * nmemb + 1); + memcpy(header, (char*)ptr, size * nmemb); + header[size * nmemb] = '\0'; + static char storage[MAX_HEADER_SIZE]; + if (sscanf(header, "%[^:]: %[^\r\n]", head, value) == 2) + { + strncpy(storage, head, sizeof(storage)); + dir_entry* de = (dir_entry*)userdata; + if (de != NULL) + { + if (!strncasecmp(head, HEADER_TEXT_ATIME, size * nmemb)) + header_set_time_from_str(value, &de->atime); + if (!strncasecmp(head, HEADER_TEXT_CTIME, size * nmemb)) + header_set_time_from_str(value, &de->ctime); + if (!strncasecmp(head, HEADER_TEXT_MTIME, size * nmemb)) + header_set_time_from_str(value, &de->mtime); + if (!strncasecmp(head, HEADER_TEXT_CHMOD, size * nmemb)) + de->chmod = atoi(value); + if (!strncasecmp(head, HEADER_TEXT_GID, size * nmemb)) + de->gid = atoi(value); + if (!strncasecmp(head, HEADER_TEXT_UID, size * nmemb)) + de->uid = atoi(value); + } + else + debugf(DBG_LEVEL_EXT, + "Unexpected NULL dir_entry on header(%s), file should be in cache already", + storage); + } + else + { + //debugf(DBG_LEVEL_NORM, "Received unexpected header line"); + } + return size * nmemb; } -static size_t read_callback(void *ptr, size_t size, size_t nmemb, void *userp) +static size_t rw_callback(size_t (*rw)(void*, size_t, size_t, FILE*), + void* ptr, + size_t size, size_t nmemb, void* userp) { - return rw_callback(fread, ptr, size, nmemb, userp); + struct segment_info* info = (struct segment_info*)userp; + size_t mem = size * nmemb; + if (mem < 1 || info->size < 1) + return 0; + + size_t amt_read = rw(ptr, 1, info->size < mem ? info->size : mem, info->fp); + info->size -= amt_read; + return amt_read; } -static size_t write_callback(void *ptr, size_t size, size_t nmemb, void *userp) +size_t fwrite2(void* ptr, size_t size, size_t nmemb, FILE* filep) { - return rw_callback(fwrite2, ptr, size, nmemb, userp); + return fwrite((const void*)ptr, size, nmemb, filep); } -static int send_request_size(const char *method, const char *path, void *fp, - xmlParserCtxtPtr xmlctx, curl_slist *extra_headers, - off_t file_size, int is_segment) +static size_t read_callback(void* ptr, size_t size, size_t nmemb, void* userp) { + return rw_callback(fread, ptr, size, nmemb, userp); +} + +static size_t write_callback(void* ptr, size_t size, size_t nmemb, void* userp) +{ + return rw_callback(fwrite2, ptr, size, nmemb, userp); +} + +//http://curl.haxx.se/libcurl/c/CURLOPT_XFERINFOFUNCTION.html +int progress_callback_xfer(void* clientp, curl_off_t dltotal, curl_off_t dlnow, + curl_off_t ultotal, curl_off_t ulnow) +{ + struct curl_progress* myp = (struct curl_progress*)clientp; + CURL* curl = myp->curl; + double curtime = 0; + double dspeed = 0, uspeed = 0; + + curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &curtime); + curl_easy_getinfo(curl, CURLINFO_SPEED_DOWNLOAD, &dspeed); + curl_easy_getinfo(curl, CURLINFO_SPEED_UPLOAD, &uspeed); + + /* under certain circumstances it may be desirable for certain functionality + to only run every N seconds, in order to do this the transaction time can + be used */ + //http://curl.haxx.se/cvssource/src/tool_cb_prg.c + if ((curtime - myp->lastruntime) >= MINIMAL_PROGRESS_FUNCTIONALITY_INTERVAL) + { + myp->lastruntime = curtime; + curl_off_t total; + curl_off_t point; + double frac, percent; + total = dltotal + ultotal; + point = dlnow + ulnow; + frac = (double)point / (double)total; + percent = frac * 100.0f; + debugf(DBG_LEVEL_EXT, "TOTAL TIME: %.0f sec Down=%.0f Kbps UP=%.0f Kbps", + curtime, dspeed / 1024, uspeed / 1024); + debugf(DBG_LEVEL_EXT, "UP: %lld of %lld DOWN: %lld/%lld Completion %.1f %%", + ulnow, ultotal, dlnow, dltotal, percent); + } + return 0; +} + +//http://curl.haxx.se/libcurl/c/CURLOPT_PROGRESSFUNCTION.html +int progress_callback(void* clientp, double dltotal, double dlnow, + double ultotal, double ulnow) +{ + return progress_callback_xfer(clientp, (curl_off_t)dltotal, (curl_off_t)dlnow, + (curl_off_t)ultotal, (curl_off_t)ulnow); +} + + +//get the response from HTTP requests, mostly for debug purposes +// http://stackoverflow.com/questions/2329571/c-libcurl-get-output-into-a-string +// http://curl.haxx.se/libcurl/c/getinmemory.html +size_t writefunc_callback(void* contents, size_t size, size_t nmemb, + void* userp) +{ + size_t realsize = size * nmemb; + struct MemoryStruct* mem = (struct MemoryStruct*)userp; + + mem->memory = realloc(mem->memory, mem->size + realsize + 1); + if (mem->memory == NULL) + { + /* out of memory! */ + debugf(DBG_LEVEL_NORM, KRED"writefunc_callback: realloc() failed"); + return 0; + } + memcpy(&(mem->memory[mem->size]), contents, realsize); + mem->size += realsize; + mem->memory[mem->size] = 0; + return realsize; +} + +// de_cached_entry must be NULL when the file is already in global cache +// otherwise point to a new dir_entry that will be added to the cache (usually happens on first dir load) +static int send_request_size(const char* method, const char* path, void* fp, + xmlParserCtxtPtr xmlctx, curl_slist* extra_headers, + off_t file_size, int is_segment, + dir_entry* de_cached_entry, const char* unencoded_path) +{ + debugf(DBG_LEVEL_EXT, "send_request_size(%s) (%s)", method, path); char url[MAX_URL_SIZE]; - char *slash; + char orig_path[MAX_URL_SIZE]; + char header_data[MAX_HEADER_SIZE]; + + char* slash; long response = -1; int tries = 0; + //needed to keep the response data, for debug purposes + struct MemoryStruct chunk; + if (!storage_url[0]) { - debugf("send_request with no storage_url?"); + debugf(DBG_LEVEL_NORM, KRED"send_request with no storage_url?"); abort(); } + //char *encoded_path = curl_escape(path, 0); while ((slash = strstr(path, "%2F")) || (slash = strstr(path, "%2f"))) { *slash = '/'; - memmove(slash+1, slash+3, strlen(slash+3)+1); + memmove(slash + 1, slash + 3, strlen(slash + 3) + 1); } while (*path == '/') path++; snprintf(url, sizeof(url), "%s/%s", storage_url, path); + snprintf(orig_path, sizeof(orig_path), "/%s", path); // retry on failures for (tries = 0; tries < REQUEST_RETRIES; tries++) { - CURL *curl = get_connection(path); + chunk.memory = malloc(1); /* will be grown as needed by the realloc above */ + chunk.size = 0; /* no data at this point */ + CURL* curl = get_connection(path); if (rhel5_mode) curl_easy_setopt(curl, CURLOPT_CAINFO, RHEL5_CERTIFICATE_FILE); - curl_slist *headers = NULL; + curl_slist* headers = NULL; curl_easy_setopt(curl, CURLOPT_URL, url); curl_easy_setopt(curl, CURLOPT_HEADER, 0); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); - curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1); + //reversed logic, 0=to enable curl progress + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, option_curl_progress_state ? 0 : 1); curl_easy_setopt(curl, CURLOPT_USERAGENT, USER_AGENT); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, verify_ssl ? 1 : 0); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, verify_ssl); curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 10); - curl_easy_setopt(curl, CURLOPT_VERBOSE, debug); + curl_easy_setopt(curl, CURLOPT_VERBOSE, option_curl_verbose ? 1 : 0); add_header(&headers, "X-Auth-Token", storage_token); + dir_entry* de; + if (de_cached_entry == NULL) + de = check_path_info(unencoded_path); + else + { + // updating metadata on a file about to be added to cache (for x-copy, dest meta = src meta) + de = de_cached_entry; + debugf(DBG_LEVEL_EXTALL, "send_request_size: using param dir_entry(%s)", + orig_path); + } + if (!de) + debugf(DBG_LEVEL_EXTALL, + "send_request_size: "KYEL"file not in cache (%s)(%s)(%s)", orig_path, path, + unencoded_path); + else + { + // add headers to save utimens attribs only on upload + if (!strcasecmp(method, "PUT") || !strcasecmp(method, "MKDIR")) + { + debugf(DBG_LEVEL_EXTALL, "send_request_size: Saving utimens for file %s", + orig_path); + debugf(DBG_LEVEL_EXTALL, + "send_request_size: Cached utime for path=%s ctime=%li.%li mtime=%li.%li atime=%li.%li", + orig_path, + de->ctime.tv_sec, de->ctime.tv_nsec, de->mtime.tv_sec, de->mtime.tv_nsec, + de->atime.tv_sec, de->atime.tv_nsec); + + char atime_str_nice[TIME_CHARS] = "", mtime_str_nice[TIME_CHARS] = "", + ctime_str_nice[TIME_CHARS] = ""; + get_timespec_as_str(&(de->atime), atime_str_nice, sizeof(atime_str_nice)); + debugf(DBG_LEVEL_EXTALL, KCYN"send_request_size: atime=[%s]", atime_str_nice); + get_timespec_as_str(&(de->mtime), mtime_str_nice, sizeof(mtime_str_nice)); + debugf(DBG_LEVEL_EXTALL, KCYN"send_request_size: mtime=[%s]", mtime_str_nice); + get_timespec_as_str(&(de->ctime), ctime_str_nice, sizeof(ctime_str_nice)); + debugf(DBG_LEVEL_EXTALL, KCYN"send_request_size: ctime=[%s]", ctime_str_nice); + + char mtime_str[TIME_CHARS], atime_str[TIME_CHARS], ctime_str[TIME_CHARS]; + char string_float[TIME_CHARS]; + snprintf(mtime_str, TIME_CHARS, "%lu.%lu", de->mtime.tv_sec, + de->mtime.tv_nsec); + snprintf(atime_str, TIME_CHARS, "%lu.%lu", de->atime.tv_sec, + de->atime.tv_nsec); + snprintf(ctime_str, TIME_CHARS, "%lu.%lu", de->ctime.tv_sec, + de->ctime.tv_nsec); + add_header(&headers, HEADER_TEXT_FILEPATH, orig_path); + add_header(&headers, HEADER_TEXT_MTIME, mtime_str); + add_header(&headers, HEADER_TEXT_ATIME, atime_str); + add_header(&headers, HEADER_TEXT_CTIME, ctime_str); + add_header(&headers, HEADER_TEXT_MTIME_DISPLAY, mtime_str_nice); + add_header(&headers, HEADER_TEXT_ATIME_DISPLAY, atime_str_nice); + add_header(&headers, HEADER_TEXT_CTIME_DISPLAY, ctime_str_nice); + + char gid_str[INT_CHAR_LEN], uid_str[INT_CHAR_LEN], chmod_str[INT_CHAR_LEN]; + snprintf(gid_str, INT_CHAR_LEN, "%d", de->gid); + snprintf(uid_str, INT_CHAR_LEN, "%d", de->uid); + snprintf(chmod_str, INT_CHAR_LEN, "%d", de->chmod); + add_header(&headers, HEADER_TEXT_GID, gid_str); + add_header(&headers, HEADER_TEXT_UID, uid_str); + add_header(&headers, HEADER_TEXT_CHMOD, chmod_str); + } + else + debugf(DBG_LEVEL_EXTALL, "send_request_size: not setting utimes (%s)", + orig_path); + } if (!strcasecmp(method, "MKDIR")) { curl_easy_setopt(curl, CURLOPT_UPLOAD, 1); @@ -220,104 +471,204 @@ static int send_request_size(const char *method, const char *path, void *fp, { rewind(fp); curl_easy_setopt(curl, CURLOPT_UPLOAD, 1); - //curl_easy_setopt(curl, CURLOPT_INFILESIZE, 0); curl_easy_setopt(curl, CURLOPT_INFILESIZE, file_size); curl_easy_setopt(curl, CURLOPT_READDATA, fp); add_header(&headers, "Content-Type", "application/link"); } - else if (!strcasecmp(method, "PUT") && fp) + else if (!strcasecmp(method, "PUT")) { + //http://blog.chmouel.com/2012/02/06/anatomy-of-a-swift-put-query-to-object-server/ + debugf(DBG_LEVEL_EXT, "send_request_size: PUT (%s)", orig_path); curl_easy_setopt(curl, CURLOPT_UPLOAD, 1); - curl_easy_setopt(curl, CURLOPT_INFILESIZE, file_size); - curl_easy_setopt(curl, CURLOPT_READDATA, fp); + if (fp) + { + curl_easy_setopt(curl, CURLOPT_INFILESIZE, file_size); + curl_easy_setopt(curl, CURLOPT_READDATA, fp); + } + else + curl_easy_setopt(curl, CURLOPT_INFILESIZE, 0); if (is_segment) curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_callback); + //enable progress reporting + //http://curl.haxx.se/libcurl/c/progressfunc.html + struct curl_progress prog; + prog.lastruntime = 0; + prog.curl = curl; + curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progress_callback); + /* pass the struct pointer into the progress function */ + curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, &prog); + //get the response for debug purposes + /* send all data to this function */ + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writefunc_callback); + /* we pass our 'chunk' struct to the callback function */ + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&chunk); } else if (!strcasecmp(method, "GET")) { if (is_segment) { + debugf(DBG_LEVEL_EXT, "send_request_size: GET SEGMENT (%s)", orig_path); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); } else if (fp) { + debugf(DBG_LEVEL_EXT, "send_request_size: GET FP (%s)", orig_path); rewind(fp); // make sure the file is ready for a-writin' fflush(fp); if (ftruncate(fileno(fp), 0) < 0) { - debugf("ftruncate failed. I don't know what to do about that."); + debugf(DBG_LEVEL_NORM, + KRED"ftruncate failed. I don't know what to do about that."); abort(); } curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, &header_get_meta_dispatch); + // sample by UThreadCurl.cpp, https://bitbucket.org/pamungkas5/bcbcurl/src + // and http://www.codeproject.com/Articles/838366/BCBCurl-a-LibCurl-based-download-manager + curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void*)de); + + struct curl_progress prog; + prog.lastruntime = 0; + prog.curl = curl; + curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progress_callback); + curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, &prog); } else if (xmlctx) { + debugf(DBG_LEVEL_EXT, "send_request_size: GET XML (%s)", orig_path); curl_easy_setopt(curl, CURLOPT_WRITEDATA, xmlctx); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &xml_dispatch); } + else + { + //asumming retrieval of headers only + debugf(DBG_LEVEL_EXT, "send_request_size: GET HEADERS only(%s)"); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, &header_get_meta_dispatch); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void*)de); + curl_easy_setopt(curl, CURLOPT_NOBODY, 1); + } } else { + debugf(DBG_LEVEL_EXT, "send_request_size: catch_all (%s)"); + // this posts an HEAD request (e.g. for statfs) curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, method); curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, &header_dispatch); } /* add the headers from extra_headers if any */ - curl_slist *extra; + curl_slist* extra; for (extra = extra_headers; extra; extra = extra->next) { - debugf("adding header: %s", extra->data); + debugf(DBG_LEVEL_EXT, "adding header: %s", extra->data); headers = curl_slist_append(headers, extra->data); } curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + debugf(DBG_LEVEL_EXT, "status: send_request_size(%s) started HTTP REQ:%s", + orig_path, url); curl_easy_perform(curl); + double total_time; + char* effective_url; curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response); + curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &effective_url); + curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &total_time); + debugf(DBG_LEVEL_EXT, + "status: send_request_size(%s) completed HTTP REQ:%s total_time=%.1f seconds", + orig_path, effective_url, total_time); curl_slist_free_all(headers); curl_easy_reset(curl); return_connection(curl); - if ((response >= 200 && response < 400) || (!strcasecmp(method, "DELETE") && response == 409)) + + if (response != 404 && (response >= 400 || response < 200)) + { + /* + Now, our chunk.memory points to a memory block that is chunk.size + bytes big and contains the remote file. + */ + debugf(DBG_LEVEL_NORM, + KRED"send_request_size: error message, size=%lu, [HTTP %d] (%s)(%s)", + (long)chunk.size, response, method, path); + debugf(DBG_LEVEL_NORM, KRED"send_request_size: error message=[%s]", + chunk.memory); + } + free(chunk.memory); + + if ((response >= 200 && response < 400) || (!strcasecmp(method, "DELETE") + && response == 409)) + { + debugf(DBG_LEVEL_NORM, + "exit 0: send_request_size(%s) speed=%.1f sec "KCYN"(%s) "KGRN"[HTTP OK]", + orig_path, total_time, method); return response; - sleep(8 << tries); // backoff - if (response == 401 && !cloudfs_connect()) // re-authenticate on 401s + } + //handle cases when file is not found, no point in retrying, will exit + if (response == 404) + { + debugf(DBG_LEVEL_NORM, + "send_request_size: not found error for (%s)(%s), ignored "KYEL"[HTTP 404].", + method, path); return response; + } + else + { + debugf(DBG_LEVEL_NORM, + "send_request_size: httpcode=%d (%s)(%s), retrying "KRED"[HTTP ERR]", response, + method, path); + //todo: try to list response content for debug purposes + sleep(8 << tries); // backoff + } + if (response == 401 && !cloudfs_connect()) + { + // re-authenticate on 401s + debugf(DBG_LEVEL_NORM, KYEL"exit 1: send_request_size(%s) (%s) [HTTP REAUTH]", + path, method); + return response; + } if (xmlctx) xmlCtxtResetPush(xmlctx, NULL, 0, NULL, NULL); } + debugf(DBG_LEVEL_NORM, "exit 2: send_request_size(%s)"KCYN"(%s) response=%d", + path, method, response); return response; } -static int send_request(char *method, const char *path, FILE *fp, - xmlParserCtxtPtr xmlctx, curl_slist *extra_headers) +int send_request(char* method, const char* path, FILE* fp, + xmlParserCtxtPtr xmlctx, curl_slist* extra_headers, dir_entry* de_cached_entry, + const char* unencoded_path) { - - long flen = 0; - if (fp) { - // if we don't flush the size will probably be zero - fflush(fp); - flen = cloudfs_file_size(fileno(fp)); - } - - return send_request_size(method, path, fp, xmlctx, extra_headers, flen, 0); - + long flen = 0; + if (fp) + { + // if we don't flush the size will probably be zero + fflush(fp); + flen = cloudfs_file_size(fileno(fp)); + } + return send_request_size(method, path, fp, xmlctx, extra_headers, flen, 0, + de_cached_entry, unencoded_path); } -void *upload_segment(void *seginfo) +//thread that downloads or uploads large file segments +void* upload_segment(void* seginfo) { - struct segment_info *info = (struct segment_info *)seginfo; - char seg_path[MAX_URL_SIZE] = { 0 }; + struct segment_info* info = (struct segment_info*)seginfo; + char seg_path[MAX_URL_SIZE] = { 0 }; + //set pointer to the segment start index in the complete large file (several threads will write to same large file) fseek(info->fp, info->part * info->segment_size, SEEK_SET); setvbuf(info->fp, NULL, _IOFBF, DISK_BUFF_SIZE); snprintf(seg_path, MAX_URL_SIZE, "%s%08i", info->seg_base, info->part); - char *encoded = curl_escape(seg_path, 0); + char* encoded = curl_escape(seg_path, 0); + + debugf(DBG_LEVEL_EXT, KCYN"upload_segment(%s) part=%d size=%d seg_size=%d %s", + info->method, info->part, info->size, info->segment_size, seg_path); int response = send_request_size(info->method, encoded, info, NULL, NULL, - info->size, 1); + info->size, 1, NULL, seg_path); if (!(response >= 200 && response < 300)) fprintf(stderr, "Segment upload %s failed with response %d", seg_path, - response); + response); curl_free(encoded); fclose(info->fp); @@ -326,101 +677,137 @@ void *upload_segment(void *seginfo) // segment_size is the globabl config variable and size_of_segment is local //TODO: return whether the upload/download failed or not -void run_segment_threads(const char *method, int segments, int full_segments, int remaining, - FILE *fp, char *seg_base, int size_of_segments) +void run_segment_threads(const char* method, int segments, int full_segments, + int remaining, + FILE* fp, char* seg_base, int size_of_segments) { - char file_path[PATH_MAX] = { 0 }; - struct segment_info *info = (struct segment_info *) - malloc(segments * sizeof(struct segment_info)); + debugf(DBG_LEVEL_EXT, "run_segment_threads(%s)", method); + char file_path[PATH_MAX] = { 0 }; + struct segment_info* info = (struct segment_info*) + malloc(segments * sizeof(struct segment_info)); - pthread_t *threads = (pthread_t *)malloc(segments * sizeof(pthread_t)); + pthread_t* threads = (pthread_t*)malloc(segments * sizeof(pthread_t)); #ifdef __linux__ - snprintf(file_path, PATH_MAX, "/proc/self/fd/%d", fileno(fp)); + snprintf(file_path, PATH_MAX, "/proc/self/fd/%d", fileno(fp)); + debugf(DBG_LEVEL_NORM, "On run segment filepath=%s", file_path); #else - //TODO: I haven't actually tested this - if (fcntl(fileno(fp), F_GETPATH, file_path) == -1) - fprintf(stderr, "couldn't get the path name\n"); + //TODO: I haven't actually tested this + if (fcntl(fileno(fp), F_GETPATH, file_path) == -1) + fprintf(stderr, "couldn't get the path name\n"); #endif - int i, ret; - for (i = 0; i < segments; i++) { - info[i].method = method; - info[i].fp = fopen(file_path, method[0] == 'G' ? "r+" : "r"); - info[i].part = i; - info[i].segment_size = size_of_segments; - info[i].size = i < full_segments ? size_of_segments : remaining; - info[i].seg_base = seg_base; - pthread_create(&threads[i], NULL, upload_segment, (void *)&(info[i])); - } - - for (i = 0; i < segments; i++) { - if ((ret = pthread_join(threads[i], NULL)) != 0) - fprintf(stderr, "error waiting for thread %d, status = %d\n", i, ret); - } - free(info); - free(threads); -} - -void split_path(const char *path, char *seg_base, char *container, - char *object) -{ - char *string = strdup(path); - - snprintf(seg_base, MAX_URL_SIZE, "%s", strsep(&string, "/")); - - strncat(container, strsep(&string, "/"), - MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE)); - - char *_object = strsep(&string, "/"); - - char *remstr; - - while (remstr = strsep(&string, "/")) { - strncat(container, "/", - MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE)); - strncat(container, _object, - MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE)); - _object = remstr; + int i, ret; + for (i = 0; i < segments; i++) + { + info[i].method = method; + info[i].fp = fopen(file_path, method[0] == 'G' ? "r+" : "r"); + info[i].part = i; + info[i].segment_size = size_of_segments; + info[i].size = i < full_segments ? size_of_segments : remaining; + info[i].seg_base = seg_base; + pthread_create(&threads[i], NULL, upload_segment, (void*) & (info[i])); } - strncpy(object, _object, MAX_URL_SIZE); + for (i = 0; i < segments; i++) + { + if ((ret = pthread_join(threads[i], NULL)) != 0) + fprintf(stderr, "error waiting for thread %d, status = %d\n", i, ret); + } + free(info); + free(threads); + debugf(DBG_LEVEL_EXT, "exit: run_segment_threads(%s)", method); +} + +void split_path(const char* path, char* seg_base, char* container, + char* object) +{ + char* string = strdup(path); + snprintf(seg_base, MAX_URL_SIZE, "%s", strsep(&string, "/")); + strncat(container, strsep(&string, "/"), + MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE)); + char* _object = strsep(&string, "/"); + char* remstr; + + while (remstr = strsep(&string, "/")) + { + strncat(container, "/", + MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE)); + strncat(container, _object, + MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE)); + _object = remstr; + } + //fixed: when removing root folders this will generate a segfault + //issue #83, https://github.com/TurboGit/hubicfuse/issues/83 + if (_object == NULL) + _object = object; + else + strncpy(object, _object, MAX_URL_SIZE); free(string); } -int internal_is_segmented(const char *seg_path, const char *object) +//checks on the cloud if this file (seg_path) have an associated segment folder +int internal_is_segmented(const char* seg_path, const char* object, + const char* parent_path) { - dir_entry *seg_dir; - if (cloudfs_list_directory(seg_path, &seg_dir)) { - if (seg_dir && seg_dir->isdir) { - do { - if (!strncmp(seg_dir->name, object, MAX_URL_SIZE)) { - return 1; - } - } while ((seg_dir = seg_dir->next)); + debugf(DBG_LEVEL_EXT, "internal_is_segmented(%s)", seg_path); + //try to avoid an additional http request for small files + bool potentially_segmented; + dir_entry* de = check_path_info(parent_path); + if (!de) + { + //when files in folders are first loaded the path will not be yet in cache, so need + //to force segment meta download for segmented files + potentially_segmented = true; + } + else + { + //potentially segmented, assumption is that 0 size files are potentially segmented + //while size>0 is for sure not segmented, so no point in making an expensive HTTP GET call + potentially_segmented = (de->size == 0 && !de->isdir) ? true : false; + } + debugf(DBG_LEVEL_EXT, "internal_is_segmented: potentially segmented=%d", + potentially_segmented); + dir_entry* seg_dir; + if (potentially_segmented && cloudfs_list_directory(seg_path, &seg_dir)) + { + if (seg_dir && seg_dir->isdir) + { + do + { + if (!strncmp(seg_dir->name, object, MAX_URL_SIZE)) + { + debugf(DBG_LEVEL_EXT, "exit 0: internal_is_segmented(%s) "KGRN"TRUE", + seg_path); + return 1; + } + } + while ((seg_dir = seg_dir->next)); } } + debugf(DBG_LEVEL_EXT, "exit 1: internal_is_segmented(%s) "KYEL"FALSE", + seg_path); return 0; } -int is_segmented(const char *path) +int is_segmented(const char* path) { - char container[MAX_URL_SIZE] = ""; - char object[MAX_URL_SIZE] = ""; - char seg_base[MAX_URL_SIZE] = ""; - + debugf(DBG_LEVEL_EXT, "is_segmented(%s)", path); + char container[MAX_URL_SIZE] = { 0 }; + char object[MAX_URL_SIZE] = { 0 }; + char seg_base[MAX_URL_SIZE] = { 0 }; split_path(path, seg_base, container, object); - char seg_path[MAX_URL_SIZE]; snprintf(seg_path, MAX_URL_SIZE, "%s/%s_segments", seg_base, container); - - return internal_is_segmented(seg_path, object); + return internal_is_segmented(seg_path, object, path); } - -int format_segments(const char *path, char * seg_base, long *segments, - long *full_segments, long *remaining, long *size_of_segments) +//returns segmented file properties by parsing and retrieving the folder structure on the cloud +//added totalsize as parameter to return the file size on list directory for segmented files +//old implementation returns file size=0 (issue #91) +int format_segments(const char* path, char* seg_base, long* segments, + long* full_segments, long* remaining, long* size_of_segments, long* total_size) { - + debugf(DBG_LEVEL_EXT, "format_segments(%s)", path); char container[MAX_URL_SIZE] = ""; char object[MAX_URL_SIZE] = ""; @@ -429,56 +816,79 @@ int format_segments(const char *path, char * seg_base, long *segments, char seg_path[MAX_URL_SIZE]; snprintf(seg_path, MAX_URL_SIZE, "%s/%s_segments", seg_base, container); - if (internal_is_segmented(seg_path, object)) { + if (internal_is_segmented(seg_path, object, path)) + { char manifest[MAX_URL_SIZE]; - dir_entry *seg_dir; + dir_entry* seg_dir; snprintf(manifest, MAX_URL_SIZE, "%s/%s", seg_path, object); + debugf(DBG_LEVEL_EXT, KMAG"format_segments manifest(%s)", manifest); if (!cloudfs_list_directory(manifest, &seg_dir)) + { + debugf(DBG_LEVEL_EXT, "exit 0: format_segments(%s)", path); return 0; + } // snprintf seesaw between manifest and seg_path to get // the total_size and the segment size as well as the actual objects - char *timestamp = seg_dir->name; + char* timestamp = seg_dir->name; snprintf(seg_path, MAX_URL_SIZE, "%s/%s", manifest, timestamp); + debugf(DBG_LEVEL_EXT, KMAG"format_segments seg_path(%s)", seg_path); if (!cloudfs_list_directory(seg_path, &seg_dir)) + { + debugf(DBG_LEVEL_EXT, "exit 1: format_segments(%s)", path); return 0; + } - char *str_size = seg_dir->name; + char* str_size = seg_dir->name; snprintf(manifest, MAX_URL_SIZE, "%s/%s", seg_path, str_size); + debugf(DBG_LEVEL_EXT, KMAG"format_segments manifest2(%s) size=%s", manifest, + str_size); if (!cloudfs_list_directory(manifest, &seg_dir)) + { + debugf(DBG_LEVEL_EXT, "exit 2: format_segments(%s)", path); return 0; + } - char *str_segment = seg_dir->name; + //following folder name actually represents the parent file size + char* str_segment = seg_dir->name; snprintf(seg_path, MAX_URL_SIZE, "%s/%s", manifest, str_segment); + debugf(DBG_LEVEL_EXT, KMAG"format_segments seg_path2(%s)", seg_path); + //here is where we get a list with all segment files composing the parent large file if (!cloudfs_list_directory(seg_path, &seg_dir)) + { + debugf(DBG_LEVEL_EXT, "exit 3: format_segments(%s)", path); return 0; + } - long total_size = strtoll(str_size, NULL, 10); + *total_size = strtoll(str_size, NULL, 10); *size_of_segments = strtoll(str_segment, NULL, 10); - - *remaining = total_size % *size_of_segments; - *full_segments = total_size / *size_of_segments; + *remaining = *total_size % *size_of_segments; + *full_segments = *total_size / *size_of_segments; *segments = *full_segments + (*remaining > 0); snprintf(manifest, MAX_URL_SIZE, "%s_segments/%s/%s/%s/%s/", - container, object, timestamp, str_size, str_segment); + container, object, timestamp, str_size, str_segment); char tmp[MAX_URL_SIZE]; strncpy(tmp, seg_base, MAX_URL_SIZE); snprintf(seg_base, MAX_URL_SIZE, "%s/%s", tmp, manifest); - + debugf(DBG_LEVEL_EXT, KMAG"format_segments seg_base(%s)", seg_base); + debugf(DBG_LEVEL_EXT, + KMAG"exit 4: format_segments(%s) total=%d size_of_segments=%d remaining=%d, full_segments=%d segments=%d", + path, &total_size, &size_of_segments, &remaining, &full_segments, &segments); return 1; } - - else { + else + { + debugf(DBG_LEVEL_EXT, KMAG"exit 5: format_segments(%s) not segmented?", path); return 0; } } /* - * Public interface - */ + Public interface +*/ void cloudfs_init() { @@ -486,26 +896,26 @@ void cloudfs_init() xmlXPathInit(); curl_global_init(CURL_GLOBAL_ALL); pthread_mutex_init(&pool_mut, NULL); - curl_version_info_data *cvid = curl_version_info(CURLVERSION_NOW); + curl_version_info_data* cvid = curl_version_info(CURLVERSION_NOW); // CentOS/RHEL 5 get stupid mode, because they have a broken libcurl if (cvid->version_num == RHEL5_LIBCURL_VERSION) { - debugf("RHEL5 mode enabled."); + debugf(DBG_LEVEL_NORM, "RHEL5 mode enabled."); rhel5_mode = 1; } if (!strncasecmp(cvid->ssl_version, "openssl", 7)) { - #ifdef HAVE_OPENSSL +#ifdef HAVE_OPENSSL int i; - ssl_lockarray = (pthread_mutex_t *)OPENSSL_malloc(CRYPTO_num_locks() * - sizeof(pthread_mutex_t)); + ssl_lockarray = (pthread_mutex_t*)OPENSSL_malloc(CRYPTO_num_locks() * + sizeof(pthread_mutex_t)); for (i = 0; i < CRYPTO_num_locks(); i++) pthread_mutex_init(&(ssl_lockarray[i]), NULL); CRYPTO_set_id_callback((unsigned long (*)())thread_id); CRYPTO_set_locking_callback((void (*)())lock_callback); - #endif +#endif } else if (!strncasecmp(cvid->ssl_version, "nss", 3)) { @@ -514,56 +924,73 @@ void cloudfs_init() } } -int file_is_readable(const char *fname) +void cloudfs_free() { - FILE *file; - if ( file = fopen( fname, "r" ) ) - { - fclose( file ); - return 1; - } - return 0; + debugf(DBG_LEVEL_EXT, "Destroy mutex"); + pthread_mutex_destroy(&pool_mut); + int n; + for (n = 0; n < curl_pool_count; ++n) + { + debugf(DBG_LEVEL_EXT, "Cleaning curl conn %d", n); + curl_easy_cleanup(curl_pool[n]); + } } -const char * get_file_mimetype ( const char *path ) +int file_is_readable(const char* fname) { - if( file_is_readable( path ) == 1 ) - { - magic_t magic; - const char *mime; - - magic = magic_open( MAGIC_MIME_TYPE ); - magic_load( magic, NULL ); - magic_compile( magic, NULL ); - mime = magic_file( magic, path ); - magic_close( magic ); - - return mime; + FILE* file; + if ( file = fopen( fname, "r" ) ) + { + fclose( file ); + return 1; } + return 0; +} - const char *error = "application/octet-stream"; +const char* get_file_mimetype ( const char* path ) +{ + if ( file_is_readable( path ) == 1 ) + { + magic_t magic; + const char* mime; + + magic = magic_open( MAGIC_MIME_TYPE ); + magic_load( magic, NULL ); + magic_compile( magic, NULL ); + mime = magic_file( magic, path ); + magic_close( magic ); + + return mime; + } + const char* error = "application/octet-stream"; return error; } -int cloudfs_object_read_fp(const char *path, FILE *fp) +int cloudfs_object_read_fp(const char* path, FILE* fp) { - + debugf(DBG_LEVEL_EXT, "cloudfs_object_read_fp(%s)", path); long flen; fflush(fp); - const char *filemimetype = get_file_mimetype( path ); + const char* filemimetype = get_file_mimetype( path ); // determine the size of the file and segment if it is above the threshhold fseek(fp, 0, SEEK_END); flen = ftell(fp); // delete the previously uploaded segments - if (is_segmented(path)) { + if (is_segmented(path)) + { if (!cloudfs_delete_object(path)) - debugf("Couldn't delete one of the existing files while uploading."); + debugf(DBG_LEVEL_NORM, + KRED"cloudfs_object_read_fp: couldn't delete existing file"); + else + debugf(DBG_LEVEL_EXT, KYEL"cloudfs_object_read_fp: deleted existing file"); } - if (flen >= segment_above) { + struct timespec now; + if (flen >= segment_above) + { int i; long remaining = flen % segment_size; int full_segments = flen / segment_size; @@ -571,118 +998,161 @@ int cloudfs_object_read_fp(const char *path, FILE *fp) // The best we can do here is to get the current time that way tools that // use the mtime can at least check if the file was changing after now - struct timespec now; clock_gettime(CLOCK_REALTIME, &now); - char string_float[TIME_CHARS]; snprintf(string_float, TIME_CHARS, "%lu.%lu", now.tv_sec, now.tv_nsec); - char meta_mtime[TIME_CHARS]; snprintf(meta_mtime, TIME_CHARS, "%f", atof(string_float)); - char seg_base[MAX_URL_SIZE] = ""; - char container[MAX_URL_SIZE] = ""; char object[MAX_URL_SIZE] = ""; - split_path(path, seg_base, container, object); - char manifest[MAX_URL_SIZE]; snprintf(manifest, MAX_URL_SIZE, "%s_segments", container); - // create the segments container cloudfs_create_directory(manifest); - // reusing manifest + // TODO: check how addition of meta_mtime in manifest impacts utimens implementation snprintf(manifest, MAX_URL_SIZE, "%s_segments/%s/%s/%ld/%ld/", - container, object, meta_mtime, flen, segment_size); - + container, object, meta_mtime, flen, segment_size); char tmp[MAX_URL_SIZE]; strncpy(tmp, seg_base, MAX_URL_SIZE); snprintf(seg_base, MAX_URL_SIZE, "%s/%s", tmp, manifest); run_segment_threads("PUT", segments, full_segments, remaining, fp, - seg_base, segment_size); + seg_base, segment_size); - char *encoded = curl_escape(path, 0); - curl_slist *headers = NULL; + char* encoded = curl_escape(path, 0); + curl_slist* headers = NULL; add_header(&headers, "x-object-manifest", manifest); - add_header(&headers, "x-object-meta-mtime", meta_mtime); add_header(&headers, "Content-Length", "0"); add_header(&headers, "Content-Type", filemimetype); - - int response = send_request_size("PUT", encoded, NULL, NULL, headers, 0, 0); + int response = send_request_size("PUT", encoded, NULL, NULL, headers, 0, 0, + NULL, path); curl_slist_free_all(headers); - curl_free(encoded); + debugf(DBG_LEVEL_EXT, + "exit 0: cloudfs_object_read_fp(%s) uploaded ok, response=%d", path, response); return (response >= 200 && response < 300); - } - + else + { + // assume enters here when file is composed of only one segment (small files) + debugf(DBG_LEVEL_EXT, "cloudfs_object_read_fp(%s) "KYEL"unknown state", path); + } rewind(fp); - char *encoded = curl_escape(path, 0); - int response = send_request("PUT", encoded, fp, NULL, NULL); + char* encoded = curl_escape(path, 0); + dir_entry* de = path_info(path); + if (!de) + debugf(DBG_LEVEL_EXT, "cloudfs_object_read_fp(%s) not in cache", path); + else + debugf(DBG_LEVEL_EXT, "cloudfs_object_read_fp(%s) found in cache", path); + int response = send_request("PUT", encoded, fp, NULL, NULL, NULL, path); curl_free(encoded); + debugf(DBG_LEVEL_EXT, "exit 1: cloudfs_object_read_fp(%s)", path); return (response >= 200 && response < 300); } - -int cloudfs_object_write_fp(const char *path, FILE *fp) +//write file downloaded from cloud to local file +int cloudfs_object_write_fp(const char* path, FILE* fp) { - char *encoded = curl_escape(path, 0); + debugf(DBG_LEVEL_EXT, "cloudfs_object_write_fp(%s)", path); + char* encoded = curl_escape(path, 0); char seg_base[MAX_URL_SIZE] = ""; long segments; long full_segments; long remaining; long size_of_segments; + long total_size; + //checks if this file is a segmented one if (format_segments(path, seg_base, &segments, &full_segments, &remaining, - &size_of_segments)) { - + &size_of_segments, &total_size)) + { rewind(fp); fflush(fp); - if (ftruncate(fileno(fp), 0) < 0) { - debugf("ftruncate failed. I don't know what to do about that."); + debugf(DBG_LEVEL_NORM, + KRED"ftruncate failed. I don't know what to do about that."); abort(); } - run_segment_threads("GET", segments, full_segments, remaining, fp, - seg_base, size_of_segments); + seg_base, size_of_segments); + debugf(DBG_LEVEL_EXT, "exit 0: cloudfs_object_write_fp(%s)", path); return 1; } - int response = send_request("GET", encoded, fp, NULL, NULL); + int response = send_request("GET", encoded, fp, NULL, NULL, NULL, path); curl_free(encoded); fflush(fp); if ((response >= 200 && response < 300) || ftruncate(fileno(fp), 0)) + { + debugf(DBG_LEVEL_EXT, "exit 1: cloudfs_object_write_fp(%s)", path); return 1; + } rewind(fp); + debugf(DBG_LEVEL_EXT, "exit 2: cloudfs_object_write_fp(%s)", path); return 0; } -int cloudfs_object_truncate(const char *path, off_t size) +int cloudfs_object_truncate(const char* path, off_t size) { - char *encoded = curl_escape(path, 0); + char* encoded = curl_escape(path, 0); int response; if (size == 0) { - FILE *fp = fopen("/dev/null", "r"); - response = send_request("PUT", encoded, fp, NULL, NULL); + FILE* fp = fopen("/dev/null", "r"); + response = send_request("PUT", encoded, fp, NULL, NULL, NULL, path); fclose(fp); } else - {//TODO: this is busted - response = send_request("GET", encoded, NULL, NULL, NULL); + { + //TODO: this is busted + response = send_request("GET", encoded, NULL, NULL, NULL, NULL, path); } curl_free(encoded); return (response >= 200 && response < 300); } -int cloudfs_list_directory(const char *path, dir_entry **dir_list) +//get metadata from cloud, like time attribs. create new entry if not cached yet. +void get_file_metadata(dir_entry* de) { + if (de->size == 0 && !de->isdir && !de->metadata_downloaded) + { + //this can be a potential segmented file, try to read segments size + debugf(DBG_LEVEL_EXT, KMAG"ZERO size file=%s", de->full_name); + char seg_base[MAX_URL_SIZE] = ""; + long segments; + long full_segments; + long remaining; + long size_of_segments; + long total_size; + if (format_segments(de->full_name, seg_base, &segments, &full_segments, + &remaining, + &size_of_segments, &total_size)) + de->size = total_size; + } + if (option_get_extended_metadata) + { + debugf(DBG_LEVEL_EXT, KCYN "get_file_metadata(%s)", de->full_name); + //retrieve additional file metadata with a quick HEAD query + char* encoded = curl_escape(de->full_name, 0); + de->metadata_downloaded = true; + int response = send_request("GET", encoded, NULL, NULL, NULL, de, + de->full_name); + curl_free(encoded); + debugf(DBG_LEVEL_EXT, KCYN "exit: get_file_metadata(%s)", de->full_name); + } + return; +} + +//get list of folders from cloud +// return 1 for OK, 0 for error +int cloudfs_list_directory(const char* path, dir_entry** dir_list) +{ + debugf(DBG_LEVEL_EXT, "cloudfs_list_directory(%s)", path); char container[MAX_PATH_SIZE * 3] = ""; char object[MAX_PATH_SIZE] = ""; char last_subdir[MAX_PATH_SIZE] = ""; @@ -692,7 +1162,7 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) int entry_count = 0; *dir_list = NULL; - xmlNode *onode = NULL, *anode = NULL, *text_node = NULL; + xmlNode* onode = NULL, *anode = NULL, *text_node = NULL; xmlParserCtxtPtr xmlctx = xmlCreatePushParserCtxt(NULL, NULL, "", 0, NULL); if (!strcmp(path, "") || !strcmp(path, "/")) { @@ -702,11 +1172,11 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) else { sscanf(path, "/%[^/]/%[^\n]", container, object); - char *encoded_container = curl_escape(container, 0); - char *encoded_object = curl_escape(object, 0); + char* encoded_container = curl_escape(container, 0); + char* encoded_object = curl_escape(object, 0); // The empty path doesn't get a trailing slash, everything else does - char *trailing_slash; + char* trailing_slash; prefix_length = strlen(object); if (object[0] == 0) trailing_slash = ""; @@ -715,9 +1185,8 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) trailing_slash = "/"; prefix_length++; } - snprintf(container, sizeof(container), "%s?format=xml&delimiter=/&prefix=%s%s", - encoded_container, encoded_object, trailing_slash); + encoded_container, encoded_object, trailing_slash); curl_free(encoded_container); curl_free(encoded_object); } @@ -725,101 +1194,127 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) if ((!strcmp(path, "") || !strcmp(path, "/")) && *override_storage_url) response = 404; else - response = send_request("GET", container, NULL, xmlctx, NULL); + { + // this was generating 404 err on non segmented files (small files) + response = send_request("GET", container, NULL, xmlctx, NULL, NULL, path); + } if (response >= 200 && response < 300) xmlParseChunk(xmlctx, "", 0, 1); if (response >= 200 && response < 300 && xmlctx->wellFormed ) { - xmlNode *root_element = xmlDocGetRootElement(xmlctx->myDoc); + xmlNode* root_element = xmlDocGetRootElement(xmlctx->myDoc); for (onode = root_element->children; onode; onode = onode->next) { if (onode->type != XML_ELEMENT_NODE) continue; - - char is_object = !strcasecmp((const char *)onode->name, "object"); - char is_container = !strcasecmp((const char *)onode->name, "container"); - char is_subdir = !strcasecmp((const char *)onode->name, "subdir"); + char is_object = !strcasecmp((const char*)onode->name, "object"); + char is_container = !strcasecmp((const char*)onode->name, "container"); + char is_subdir = !strcasecmp((const char*)onode->name, "subdir"); if (is_object || is_container || is_subdir) { entry_count++; - - dir_entry *de = (dir_entry *)malloc(sizeof(dir_entry)); - de->next = NULL; - de->size = 0; - de->last_modified = time(NULL); + dir_entry* de = init_dir_entry(); + // useful docs on nodes here: http://developer.openstack.org/api-ref-objectstorage-v1.html if (is_container || is_subdir) de->content_type = strdup("application/directory"); for (anode = onode->children; anode; anode = anode->next) { - char *content = ""; + char* content = ""; for (text_node = anode->children; text_node; text_node = text_node->next) + { if (text_node->type == XML_TEXT_NODE) - content = (char *)text_node->content; - if (!strcasecmp((const char *)anode->name, "name")) + { + content = (char*)text_node->content; + //debugf(DBG_LEVEL_NORM, "List dir anode=%s content=%s", (const char *)anode->name, content); + } + else + { + //debugf(DBG_LEVEL_NORM, "List dir anode=%s", (const char *)anode->name); + } + } + if (!strcasecmp((const char*)anode->name, "name")) { de->name = strdup(content + prefix_length); - // Remove trailing slash - char *slash = strrchr(de->name, '/'); + char* slash = strrchr(de->name, '/'); if (slash && (0 == *(slash + 1))) *slash = 0; - if (asprintf(&(de->full_name), "%s/%s", path, de->name) < 0) de->full_name = NULL; } - if (!strcasecmp((const char *)anode->name, "bytes")) + if (!strcasecmp((const char*)anode->name, "bytes")) de->size = strtoll(content, NULL, 10); - if (!strcasecmp((const char *)anode->name, "content_type")) + if (!strcasecmp((const char*)anode->name, "content_type")) { de->content_type = strdup(content); - char *semicolon = strchr(de->content_type, ';'); + char* semicolon = strchr(de->content_type, ';'); if (semicolon) *semicolon = '\0'; } - if (!strcasecmp((const char *)anode->name, "last_modified")) + if (!strcasecmp((const char*)anode->name, "hash")) + de->md5sum = strdup(content); + if (!strcasecmp((const char*)anode->name, "last_modified")) { - struct tm last_modified; - strptime(content, "%FT%T", &last_modified); - de->last_modified = mktime(&last_modified); + time_t last_modified_t = get_time_from_str_as_gmt(content); + char local_time_str[64]; + time_t local_time_t = get_time_as_local(last_modified_t, local_time_str, + sizeof(local_time_str)); + de->last_modified = local_time_t; + de->ctime.tv_sec = local_time_t; + de->ctime.tv_nsec = 0; + //initialise all fields with hubic last modified date in case the file does not have extended attributes set + de->mtime.tv_sec = local_time_t; + de->mtime.tv_nsec = 0; + de->atime.tv_sec = local_time_t; + de->atime.tv_nsec = 0; + //todo: how can we retrieve and set nanoseconds, are stored by hubic? } } de->isdir = de->content_type && - ((strstr(de->content_type, "application/folder") != NULL) || - (strstr(de->content_type, "application/directory") != NULL)); + ((strstr(de->content_type, "application/folder") != NULL) || + (strstr(de->content_type, "application/directory") != NULL)); de->islink = de->content_type && - ((strstr(de->content_type, "application/link") != NULL)); + ((strstr(de->content_type, "application/link") != NULL)); if (de->isdir) { + //i guess this will remove a dir_entry from cache if is there already if (!strncasecmp(de->name, last_subdir, sizeof(last_subdir))) { - cloudfs_free_dir_list(de); + //todo: check why is needed and if memory is freed properly, seems to generate many missed delete operations + //cloudfs_free_dir_list(de); + debugf(DBG_LEVEL_EXT, + "cloudfs_list_directory: "KYEL"ignore "KNRM"cloudfs_free_dir_list(%s) command", + de->name); continue; } strncpy(last_subdir, de->name, sizeof(last_subdir)); } de->next = *dir_list; *dir_list = de; + char time_str[TIME_CHARS] = { 0 }; + get_timespec_as_str(&(de->mtime), time_str, sizeof(time_str)); + debugf(DBG_LEVEL_EXT, KCYN"new dir_entry %s size=%d %s dir=%d lnk=%d mod=[%s]", + de->full_name, de->size, de->content_type, de->isdir, de->islink, time_str); } else - { - debugf("unknown element: %s", onode->name); - } + debugf(DBG_LEVEL_EXT, "unknown element: %s", onode->name); } retval = 1; } - else if ((!strcmp(path, "") || !strcmp(path, "/")) && *override_storage_url) { + else if ((!strcmp(path, "") || !strcmp(path, "/")) && *override_storage_url) + { entry_count = 1; - - dir_entry *de = (dir_entry *)malloc(sizeof(dir_entry)); + debugf(DBG_LEVEL_NORM, "Init cache entry container=[%s]", public_container); + dir_entry* de = init_dir_entry(); de->name = strdup(public_container); struct tm last_modified; + //todo: check what this default time means? strptime("1388434648.01238", "%FT%T", &last_modified); de->last_modified = mktime(&last_modified); de->content_type = strdup("application/directory"); if (asprintf(&(de->full_name), "%s/%s", path, de->name) < 0) de->full_name = NULL; - de->isdir = 1; de->islink = 0; de->size = 4096; @@ -827,100 +1322,150 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) *dir_list = de; retval = 1; } - - debugf("entry count: %d", entry_count); - xmlFreeDoc(xmlctx->myDoc); xmlFreeParserCtxt(xmlctx); + debugf(DBG_LEVEL_EXT, "exit: cloudfs_list_directory(%s)", path); return retval; } -void cloudfs_free_dir_list(dir_entry *dir_list) +int cloudfs_delete_object(const char* path) { - while (dir_list) - { - dir_entry *de = dir_list; - dir_list = dir_list->next; - free(de->name); - free(de->full_name); - free(de->content_type); - free(de); - } -} - -int cloudfs_delete_object(const char *path) -{ - + debugf(DBG_LEVEL_EXT, "cloudfs_delete_object(%s)", path); char seg_base[MAX_URL_SIZE] = ""; long segments; long full_segments; long remaining; long size_of_segments; + long total_size; if (format_segments(path, seg_base, &segments, &full_segments, &remaining, - &size_of_segments)) { + &size_of_segments, &total_size)) + { int response; int i; char seg_path[MAX_URL_SIZE] = ""; - for (i = 0; i < segments; i++) { + for (i = 0; i < segments; i++) + { snprintf(seg_path, MAX_URL_SIZE, "%s%08i", seg_base, i); - char *encoded = curl_escape(seg_path, 0); - response = send_request("DELETE", encoded, NULL, NULL, NULL); + char* encoded = curl_escape(seg_path, 0); + response = send_request("DELETE", encoded, NULL, NULL, NULL, NULL, seg_path); if (response < 200 || response >= 300) + { + debugf(DBG_LEVEL_EXT, "exit 1: cloudfs_delete_object(%s) response=%d", path, + response); return 0; + } } } - char *encoded = curl_escape(path, 0); - int response = send_request("DELETE", encoded, NULL, NULL, NULL); + char* encoded = curl_escape(path, 0); + int response = send_request("DELETE", encoded, NULL, NULL, NULL, NULL, path); curl_free(encoded); int ret = (response >= 200 && response < 300); + debugf(DBG_LEVEL_EXT, "status: cloudfs_delete_object(%s) response=%d", path, + response); if (response == 409) + { + debugf(DBG_LEVEL_EXT, "status: cloudfs_delete_object(%s) NOT EMPTY", path); ret = -1; + } return ret; } -int cloudfs_copy_object(const char *src, const char *dst) +//fixme: this op does not preserve src attributes (e.g. will make rsync not work well) +// https://ask.openstack.org/en/question/14307/is-there-a-way-to-moverename-an-object/ +// this operation also causes an HTTP 400 error if X-Object-Meta-FilePath value is larger than 256 chars +int cloudfs_copy_object(const char* src, const char* dst) { - char *dst_encoded = curl_escape(dst, 0); - curl_slist *headers = NULL; - add_header(&headers, "X-Copy-From", src); + debugf(DBG_LEVEL_EXT, "cloudfs_copy_object(%s, %s) lensrc=%d, lendst=%d", src, + dst, strlen(src), strlen(dst)); + + char* dst_encoded = curl_escape(dst, 0); + char* src_encoded = curl_escape(src, 0); + + //convert encoded string (slashes are encoded as well) to encoded string with slashes + char* slash; + while ((slash = strstr(src_encoded, "%2F")) + || (slash = strstr(src_encoded, "%2f"))) + { + *slash = '/'; + memmove(slash + 1, slash + 3, strlen(slash + 3) + 1); + } + + curl_slist* headers = NULL; + add_header(&headers, "X-Copy-From", src_encoded); add_header(&headers, "Content-Length", "0"); - int response = send_request("PUT", dst_encoded, NULL, NULL, headers); + //get source file entry + dir_entry* de_src = check_path_info(src); + if (de_src) + debugf(DBG_LEVEL_EXT, "status cloudfs_copy_object(%s, %s): src file found", + src, dst); + else + debugf(DBG_LEVEL_NORM, + KRED"status cloudfs_copy_object(%s, %s): src file NOT found", src, dst); + //pass src metadata so that PUT will set time attributes of the src file + int response = send_request("PUT", dst_encoded, NULL, NULL, headers, de_src, + dst); curl_free(dst_encoded); + curl_free(src_encoded); curl_slist_free_all(headers); + debugf(DBG_LEVEL_EXT, "exit: cloudfs_copy_object(%s,%s) response=%d", src, dst, + response); return (response >= 200 && response < 300); } -int cloudfs_statfs(const char *path, struct statvfs *stat) +// http://developer.openstack.org/api-ref-objectstorage-v1.html#updateObjectMeta +int cloudfs_update_meta(dir_entry* de) { - int response = send_request("HEAD", "/", NULL, NULL, NULL); - - debugf("Assigning statvfs values from cache."); - *stat = statcache; - return (response >= 200 && response < 300); + int response = cloudfs_copy_object(de->full_name, de->full_name); + return response; } -int cloudfs_create_symlink(const char *src, const char *dst) +//optimised with cache +int cloudfs_statfs(const char* path, struct statvfs* stat) { - char *dst_encoded = curl_escape(dst, 0); - - FILE *lnk = tmpfile(); + time_t now = get_time_now(); + int lapsed = now - last_stat_read_time; + if (lapsed > option_cache_statfs_timeout) + { + //todo: check why stat head request is always set to /, why not path? + int response = send_request("HEAD", "/", NULL, NULL, NULL, NULL, "/"); + *stat = statcache; + debugf(DBG_LEVEL_EXT, + "exit: cloudfs_statfs (new recent values, was cached since %d seconds)", + lapsed); + last_stat_read_time = now; + return (response >= 200 && response < 300); + } + else + { + debugf(DBG_LEVEL_EXT, + "exit: cloudfs_statfs (old values, cached since %d seconds)", lapsed); + return 1; + } +} +int cloudfs_create_symlink(const char* src, const char* dst) +{ + char* dst_encoded = curl_escape(dst, 0); + FILE* lnk = tmpfile(); fwrite(src, 1, strlen(src), lnk); fwrite("\0", 1, 1, lnk); - int response = send_request("MKLINK", dst_encoded, lnk, NULL, NULL); + int response = send_request("MKLINK", dst_encoded, lnk, NULL, NULL, NULL, dst); curl_free(dst_encoded); fclose(lnk); return (response >= 200 && response < 300); } -int cloudfs_create_directory(const char *path) +int cloudfs_create_directory(const char* path) { - char *encoded = curl_escape(path, 0); - int response = send_request("MKDIR", encoded, NULL, NULL, NULL); + debugf(DBG_LEVEL_EXT, "cloudfs_create_directory(%s)", path); + char* encoded = curl_escape(path, 0); + int response = send_request("MKDIR", encoded, NULL, NULL, NULL, NULL, path); curl_free(encoded); + debugf(DBG_LEVEL_EXT, "cloudfs_create_directory(%s) response=%d", path, + response); return (response >= 200 && response < 300); } @@ -931,127 +1476,141 @@ off_t cloudfs_file_size(int fd) return buf.st_size; } -void cloudfs_debug(int dbg) -{ - debug = dbg; -} - void cloudfs_verify_ssl(int vrfy) { verify_ssl = vrfy ? 2 : 0; } -static struct { +void cloudfs_option_get_extended_metadata(int option) +{ + option_get_extended_metadata = option ? true : false; +} + +void cloudfs_option_curl_verbose(int option) +{ + option_curl_verbose = option ? true : false; +} + +static struct +{ char client_id [MAX_HEADER_SIZE]; char client_secret[MAX_HEADER_SIZE]; char refresh_token[MAX_HEADER_SIZE]; } reconnect_args; -void cloudfs_set_credentials(char *client_id, char *client_secret, char *refresh_token) +void cloudfs_set_credentials(char* client_id, char* client_secret, + char* refresh_token) { - strncpy(reconnect_args.client_id , client_id , sizeof(reconnect_args.client_id )); - strncpy(reconnect_args.client_secret, client_secret, sizeof(reconnect_args.client_secret)); - strncpy(reconnect_args.refresh_token, refresh_token, sizeof(reconnect_args.refresh_token)); + strncpy(reconnect_args.client_id , client_id , + sizeof(reconnect_args.client_id )); + strncpy(reconnect_args.client_secret, client_secret, + sizeof(reconnect_args.client_secret)); + strncpy(reconnect_args.refresh_token, refresh_token, + sizeof(reconnect_args.refresh_token)); } -struct htmlString { - char *text; - size_t size; +struct htmlString +{ + char* text; + size_t size; }; -static size_t writefunc_string(void *contents, size_t size, size_t nmemb, void *data) +static size_t writefunc_string(void* contents, size_t size, size_t nmemb, + void* data) { - struct htmlString *mem = (struct htmlString *) data; - size_t realsize = size * nmemb; - mem->text = realloc(mem->text, mem->size + realsize + 1); - if (mem->text == NULL) { /* out of memory! */ - perror(__FILE__); - exit(EXIT_FAILURE); - } + struct htmlString* mem = (struct htmlString*) data; + size_t realsize = size * nmemb; + mem->text = realloc(mem->text, mem->size + realsize + 1); + if (mem->text == NULL) /* out of memory! */ + { + perror(__FILE__); + exit(EXIT_FAILURE); + } - memcpy(&(mem->text[mem->size]), contents, realsize); - mem->size += realsize; - return realsize; + memcpy(&(mem->text[mem->size]), contents, realsize); + mem->size += realsize; + return realsize; } -char* htmlStringGet(CURL *curl) +char* htmlStringGet(CURL* curl) { - struct htmlString chunk; - chunk.text = malloc(sizeof(char)); - chunk.size = 0; + struct htmlString chunk; + chunk.text = malloc(sizeof(char)); + chunk.size = 0; + chunk.text[0] = '\0';//added to avoid valgrind unitialised warning - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &chunk); - do { - curl_easy_perform(curl); - } while (chunk.size == 0); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &chunk); + do + { + curl_easy_perform(curl); + } + while (chunk.size == 0); - chunk.text[chunk.size] = '\0'; - return chunk.text; + chunk.text[chunk.size] = '\0'; + return chunk.text; } /* thanks to http://devenix.wordpress.com */ -char *unbase64(unsigned char *input, int length) +char* unbase64(unsigned char* input, int length) { - BIO *b64, *bmem; + BIO* b64, *bmem; - char *buffer = (char *)malloc(length); - memset(buffer, 0, length); + char* buffer = (char*)malloc(length); + memset(buffer, 0, length); - b64 = BIO_new(BIO_f_base64()); - bmem = BIO_new_mem_buf(input, length); - bmem = BIO_push(b64, bmem); - BIO_set_flags(bmem, BIO_FLAGS_BASE64_NO_NL); + b64 = BIO_new(BIO_f_base64()); + bmem = BIO_new_mem_buf(input, length); + bmem = BIO_push(b64, bmem); + BIO_set_flags(bmem, BIO_FLAGS_BASE64_NO_NL); - BIO_read(bmem, buffer, length); + BIO_read(bmem, buffer, length); - BIO_free_all(bmem); + BIO_free_all(bmem); - return buffer; + return buffer; } -int safe_json_string(json_object *jobj, char *buffer, char *name) +int safe_json_string(json_object* jobj, char* buffer, char* name) { int result = 0; if (jobj) + { + json_object* o; + int found; + found = json_object_object_get_ex(jobj, name, &o); + if (found) { - json_object *o; - int found; - found = json_object_object_get_ex(jobj, name, &o); - if (found) - { - strcpy (buffer, json_object_get_string(o)); - result = 1; - } + strcpy (buffer, json_object_get_string(o)); + result = 1; } + } if (!result) - debugf("HUBIC cannot get json field '%s'\n", name); + debugf(DBG_LEVEL_NORM, KRED"HUBIC cannot get json field '%s'\n", name); return result; } int cloudfs_connect() { - #define HUBIC_TOKEN_URL "https://api.hubic.com/oauth/token" - #define HUBIC_CRED_URL "https://api.hubic.com/1.0/account/credentials" - #define HUBIC_CLIENT_ID (reconnect_args.client_id) - #define HUBIC_CLIENT_SECRET (reconnect_args.client_secret) - #define HUBIC_REFRESH_TOKEN (reconnect_args.refresh_token) - #define HUBIC_OPTIONS_SIZE 2048 +#define HUBIC_TOKEN_URL "https://api.hubic.com/oauth/token" +#define HUBIC_CRED_URL "https://api.hubic.com/1.0/account/credentials" +#define HUBIC_CLIENT_ID (reconnect_args.client_id) +#define HUBIC_CLIENT_SECRET (reconnect_args.client_secret) +#define HUBIC_REFRESH_TOKEN (reconnect_args.refresh_token) +#define HUBIC_OPTIONS_SIZE 2048 long response = -1; char url[HUBIC_OPTIONS_SIZE]; char payload[HUBIC_OPTIONS_SIZE]; - struct json_object *json_obj; + struct json_object* json_obj; pthread_mutex_lock(&pool_mut); - - debugf("Authenticating... (client_id = '%s')", HUBIC_CLIENT_ID); - + debugf(DBG_LEVEL_NORM, "Authenticating... (client_id = '%s')", + HUBIC_CLIENT_ID); storage_token[0] = storage_url[0] = '\0'; - - CURL *curl = curl_easy_init(); + CURL* curl = curl_easy_init(); /* curl default options */ curl_easy_setopt(curl, CURLOPT_VERBOSE, debug); @@ -1068,36 +1627,30 @@ int cloudfs_connect() curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writefunc_string); /* Step 1 : request a token - Not needed anymore with refresh_token */ - - /* Step 2 : get request code - Not needed anymore with refresh_token */ - - /* Step 3 : get access token */ - sprintf(payload, "refresh_token=%s&grant_type=refresh_token", HUBIC_REFRESH_TOKEN); - + sprintf(payload, "refresh_token=%s&grant_type=refresh_token", + HUBIC_REFRESH_TOKEN); curl_easy_setopt(curl, CURLOPT_URL, HUBIC_TOKEN_URL); curl_easy_setopt(curl, CURLOPT_POST, 1L); curl_easy_setopt(curl, CURLOPT_HEADER, 0); - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, payload); curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, strlen(payload)); - curl_easy_setopt(curl, CURLOPT_USERNAME, HUBIC_CLIENT_ID); curl_easy_setopt(curl, CURLOPT_PASSWORD, HUBIC_CLIENT_SECRET); curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); - char *json_str = htmlStringGet(curl); + char* json_str = htmlStringGet(curl); json_obj = json_tokener_parse(json_str); - debugf ("HUBIC TOKEN_URL result: '%s'\n", json_str); + debugf(DBG_LEVEL_NORM, "HUBIC TOKEN_URL result: '%s'\n", json_str); free(json_str); char access_token[HUBIC_OPTIONS_SIZE]; char token_type[HUBIC_OPTIONS_SIZE]; int expire_sec; int found; - json_object *o; + json_object* o; if (!safe_json_string(json_obj, access_token, "access_token")) return 0; @@ -1105,21 +1658,19 @@ int cloudfs_connect() return 0; found = json_object_object_get_ex(json_obj, "expires_in", &o); - expire_sec = json_object_get_int(o); - debugf ("HUBIC Access token: %s\n", access_token); - debugf ("HUBIC Token type : %s\n", token_type); - debugf ("HUBIC Expire in : %d\n", expire_sec); + debugf(DBG_LEVEL_NORM, "HUBIC Access token: %s\n", access_token); + debugf(DBG_LEVEL_NORM, "HUBIC Token type : %s\n", token_type); + debugf(DBG_LEVEL_NORM, "HUBIC Expire in : %d\n", expire_sec); /* Step 4 : request OpenStack storage URL */ - curl_easy_setopt(curl, CURLOPT_URL, HUBIC_CRED_URL); curl_easy_setopt(curl, CURLOPT_POST, 0L); curl_easy_setopt(curl, CURLOPT_HEADER, 0); curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_NONE); /* create the Bearer authentication header */ - curl_slist *headers = NULL; + curl_slist* headers = NULL; sprintf (payload, "Bearer %s", access_token); add_header(&headers, "Authorization", payload); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); @@ -1127,10 +1678,9 @@ int cloudfs_connect() char token[HUBIC_OPTIONS_SIZE]; char endpoint[HUBIC_OPTIONS_SIZE]; char expires[HUBIC_OPTIONS_SIZE]; - json_str = htmlStringGet(curl); json_obj = json_tokener_parse(json_str); - debugf ("CRED_URL result: '%s'\n", json_str); + debugf(DBG_LEVEL_NORM, "CRED_URL result: '%s'\n", json_str); free(json_str); if (!safe_json_string(json_obj, token, "token")) @@ -1143,25 +1693,9 @@ int cloudfs_connect() /* set the global storage_url and storage_token, the only parameters needed for swift */ strcpy (storage_url, endpoint); strcpy (storage_token, token); - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response); - curl_easy_cleanup(curl); - pthread_mutex_unlock(&pool_mut); - - return (response >= 200 && response < 300 && storage_token[0] && storage_url[0]); -} - -void debugf(char *fmt, ...) -{ - if (debug) - { - va_list args; - va_start(args, fmt); - fputs("!!! ", stderr); - vfprintf(stderr, fmt, args); - va_end(args); - putc('\n', stderr); - } + return (response >= 200 && response < 300 && storage_token[0] + && storage_url[0]); } diff --git a/cloudfsapi.h b/cloudfsapi.h index 14f83ce..2dfbc76 100644 --- a/cloudfsapi.h +++ b/cloudfsapi.h @@ -4,77 +4,86 @@ #include #include #include +#include #define BUFFER_INITIAL_SIZE 4096 #define MAX_HEADER_SIZE 8192 -#define MAX_PATH_SIZE (1024 + 256 + 3) + #define MAX_URL_SIZE (MAX_PATH_SIZE * 3) #define USER_AGENT "CloudFuse" #define OPTION_SIZE 1024 typedef struct curl_slist curl_slist; -typedef struct dir_entry +#define MINIMAL_PROGRESS_FUNCTIONALITY_INTERVAL 5 +struct curl_progress { - char *name; - char *full_name; - char *content_type; - off_t size; - time_t last_modified; - int isdir; - int islink; - struct dir_entry *next; -} dir_entry; + double lastruntime; + CURL* curl; +}; -typedef struct options { - char cache_timeout[OPTION_SIZE]; - char verify_ssl[OPTION_SIZE]; - char segment_size[OPTION_SIZE]; - char segment_above[OPTION_SIZE]; - char storage_url[OPTION_SIZE]; - char container[OPTION_SIZE]; - char temp_dir[OPTION_SIZE]; - char client_id[OPTION_SIZE]; - char client_secret[OPTION_SIZE]; - char refresh_token[OPTION_SIZE]; +typedef struct options +{ + char cache_timeout[OPTION_SIZE]; + char verify_ssl[OPTION_SIZE]; + char segment_size[OPTION_SIZE]; + char segment_above[OPTION_SIZE]; + char storage_url[OPTION_SIZE]; + char container[OPTION_SIZE]; + char temp_dir[OPTION_SIZE]; + char client_id[OPTION_SIZE]; + char client_secret[OPTION_SIZE]; + char refresh_token[OPTION_SIZE]; } FuseOptions; +typedef struct extra_options +{ + char get_extended_metadata[OPTION_SIZE]; + char curl_verbose[OPTION_SIZE]; + char cache_statfs_timeout[OPTION_SIZE]; + char debug_level[OPTION_SIZE]; + char curl_progress_state[OPTION_SIZE]; + char enable_chmod[OPTION_SIZE]; + char enable_chown[OPTION_SIZE]; +} ExtraFuseOptions; + void cloudfs_init(void); -void cloudfs_set_credentials(char *client_id, char *client_secret, char *refresh_token); +void cloudfs_free(void); +void cloudfs_set_credentials(char* client_id, char* client_secret, + char* refresh_token); int cloudfs_connect(void); struct segment_info { - FILE *fp; - int part; - long size; - long segment_size; - char *seg_base; - const char *method; + FILE* fp; + int part; + long size; + long segment_size; + char* seg_base; + const char* method; }; long segment_size; long segment_above; -char *override_storage_url; -char *public_container; +char* override_storage_url; +char* public_container; -int file_is_readable(const char *fname); -const char * get_file_mimetype ( const char *filename ); - -int cloudfs_object_read_fp(const char *path, FILE *fp); -int cloudfs_object_write_fp(const char *path, FILE *fp); -int cloudfs_list_directory(const char *path, dir_entry **); -int cloudfs_delete_object(const char *path); -int cloudfs_copy_object(const char *src, const char *dst); -int cloudfs_create_symlink(const char *src, const char *dst); -int cloudfs_create_directory(const char *label); -int cloudfs_object_truncate(const char *path, off_t size); +int file_is_readable(const char* fname); +const char* get_file_mimetype ( const char* filename ); +int cloudfs_object_read_fp(const char* path, FILE* fp); +int cloudfs_object_write_fp(const char* path, FILE* fp); +int cloudfs_list_directory(const char* path, dir_entry**); +int cloudfs_delete_object(const char* path); +int cloudfs_copy_object(const char* src, const char* dst); +int cloudfs_create_symlink(const char* src, const char* dst); +int cloudfs_create_directory(const char* label); +int cloudfs_object_truncate(const char* path, off_t size); off_t cloudfs_file_size(int fd); -void cloudfs_debug(int dbg); +int cloudfs_statfs(const char* path, struct statvfs* stat); void cloudfs_verify_ssl(int dbg); -void cloudfs_free_dir_list(dir_entry *dir_list); -int cloudfs_statfs(const char *path, struct statvfs *stat); - -void debugf(char *fmt, ...); +void cloudfs_option_get_extended_metadata(int option); +void cloudfs_option_curl_verbose(int option); +void get_file_metadata(dir_entry* de); +int cloudfs_update_meta(dir_entry* de); #endif diff --git a/cloudfuse.c b/cloudfuse.c index 90b6680..1900a32 100644 --- a/cloudfuse.c +++ b/cloudfuse.c @@ -14,21 +14,23 @@ #include #include #include +#include +#include "commonfs.h" #include "cloudfsapi.h" #include "config.h" -static int cache_timeout; -static char *temp_dir; - -typedef struct dir_cache -{ - char *path; - dir_entry *entries; - time_t cached; - struct dir_cache *next, *prev; -} dir_cache; -static dir_cache *dcache; -static pthread_mutex_t dmut; +extern char* temp_dir; +extern pthread_mutex_t dcachemut; +extern pthread_mutexattr_t mutex_attr; +extern int debug; +extern int cache_timeout; +extern int option_cache_statfs_timeout; +extern int option_debug_level; +extern bool option_get_extended_metadata; +extern bool option_curl_progress_state; +extern bool option_enable_chown; +extern bool option_enable_chmod; +extern size_t file_buffer_size; typedef struct { @@ -36,193 +38,83 @@ typedef struct int flags; } openfile; - -static void dir_for(const char *path, char *dir) +static int cfs_getattr(const char* path, struct stat* stbuf) { - strncpy(dir, path, MAX_PATH_SIZE); - char *slash = strrchr(dir, '/'); - if (slash) - *slash = '\0'; -} + debugf(DBG_LEVEL_NORM, KBLU "cfs_getattr(%s)", path); -static dir_cache *new_cache(const char *path) -{ - dir_cache *cw = (dir_cache *)calloc(sizeof(dir_cache), 1); - cw->path = strdup(path); - cw->prev = NULL; - cw->entries = NULL; - cw->cached = time(NULL); - if (dcache) - dcache->prev = cw; - cw->next = dcache; - return (dcache = cw); -} -static int caching_list_directory(const char *path, dir_entry **list) -{ - pthread_mutex_lock(&dmut); - if (!strcmp(path, "/")) - path = ""; - dir_cache *cw; - for (cw = dcache; cw; cw = cw->next) - if (!strcmp(cw->path, path)) - break; - if (!cw) - { - if (!cloudfs_list_directory(path, list)) - return 0; - cw = new_cache(path); - } - else if (cache_timeout > 0 && (time(NULL) - cw->cached > cache_timeout)) - { - if (!cloudfs_list_directory(path, list)) - return 0; - cloudfs_free_dir_list(cw->entries); - cw->cached = time(NULL); - } - else - *list = cw->entries; - cw->entries = *list; - pthread_mutex_unlock(&dmut); - return 1; -} - -static void update_dir_cache(const char *path, off_t size, int isdir, int islink) -{ - pthread_mutex_lock(&dmut); - dir_cache *cw; - dir_entry *de; - char dir[MAX_PATH_SIZE]; - dir_for(path, dir); - for (cw = dcache; cw; cw = cw->next) - { - if (!strcmp(cw->path, dir)) - { - for (de = cw->entries; de; de = de->next) - { - if (!strcmp(de->full_name, path)) - { - de->size = size; - pthread_mutex_unlock(&dmut); - return; - } - } - de = (dir_entry *)malloc(sizeof(dir_entry)); - de->size = size; - de->isdir = isdir; - de->islink = islink; - de->name = strdup(&path[strlen(cw->path)+1]); - de->full_name = strdup(path); - - if (isdir) - { - de->content_type = strdup("application/link"); - } - if(islink) - { - de->content_type = strdup("application/directory"); - } - else - { - - de->content_type = strdup("application/octet-stream"); - } - de->last_modified = time(NULL); - de->next = cw->entries; - cw->entries = de; - if (isdir) - new_cache(path); - break; - } - } - pthread_mutex_unlock(&dmut); -} - -static void dir_decache(const char *path) -{ - dir_cache *cw; - pthread_mutex_lock(&dmut); - dir_entry *de, *tmpde; - char dir[MAX_PATH_SIZE]; - dir_for(path, dir); - for (cw = dcache; cw; cw = cw->next) - { - if (!strcmp(cw->path, path)) - { - if (cw == dcache) - dcache = cw->next; - if (cw->prev) - cw->prev->next = cw->next; - if (cw->next) - cw->next->prev = cw->prev; - cloudfs_free_dir_list(cw->entries); - free(cw->path); - free(cw); - } - else if (cw->entries && !strcmp(dir, cw->path)) - { - if (!strcmp(cw->entries->full_name, path)) - { - de = cw->entries; - cw->entries = de->next; - de->next = NULL; - cloudfs_free_dir_list(de); - } - else for (de = cw->entries; de->next; de = de->next) - { - if (!strcmp(de->next->full_name, path)) - { - tmpde = de->next; - de->next = de->next->next; - tmpde->next = NULL; - cloudfs_free_dir_list(tmpde); - break; - } - } - } - } - pthread_mutex_unlock(&dmut); -} - -static dir_entry *path_info(const char *path) -{ - char dir[MAX_PATH_SIZE]; - dir_for(path, dir); - dir_entry *tmp; - if (!caching_list_directory(dir, &tmp)) - return NULL; - for (; tmp; tmp = tmp->next) - { - if (!strcmp(tmp->full_name, path)) - return tmp; - } - return NULL; -} - -static int cfs_getattr(const char *path, struct stat *stbuf) -{ - stbuf->st_uid = geteuid(); - stbuf->st_gid = getegid(); + //return standard values for root folder if (!strcmp(path, "/")) { + stbuf->st_uid = geteuid(); + stbuf->st_gid = getegid(); stbuf->st_mode = S_IFDIR | 0755; stbuf->st_nlink = 2; + debug_list_cache_content(); + debugf(DBG_LEVEL_NORM, KBLU "exit 0: cfs_getattr(%s)", path); return 0; } - dir_entry *de = path_info(path); + //get file. if not in cache will be downloaded. + dir_entry* de = path_info(path); if (!de) + { + debug_list_cache_content(); + debugf(DBG_LEVEL_NORM, KBLU"exit 1: cfs_getattr(%s) "KYEL"not-in-cache/cloud", + path); return -ENOENT; - stbuf->st_ctime = stbuf->st_mtime = de->last_modified; + } + + //lazzy download of file metadata, only when really needed + if (option_get_extended_metadata && !de->metadata_downloaded) + get_file_metadata(de); + if (option_enable_chown) + { + stbuf->st_uid = de->uid; + stbuf->st_gid = de->gid; + } + else + { + stbuf->st_uid = geteuid(); + stbuf->st_gid = getegid(); + } + // change needed due to utimens + stbuf->st_atime = de->atime.tv_sec; + stbuf->st_atim.tv_nsec = de->atime.tv_nsec; + stbuf->st_mtime = de->mtime.tv_sec; + stbuf->st_mtim.tv_nsec = de->mtime.tv_nsec; + stbuf->st_ctime = de->ctime.tv_sec; + stbuf->st_ctim.tv_nsec = de->ctime.tv_nsec; + char time_str[TIME_CHARS] = ""; + get_timespec_as_str(&(de->atime), time_str, sizeof(time_str)); + debugf(DBG_LEVEL_EXT, KCYN"cfs_getattr: atime=[%s]", time_str); + get_timespec_as_str(&(de->mtime), time_str, sizeof(time_str)); + debugf(DBG_LEVEL_EXT, KCYN"cfs_getattr: mtime=[%s]", time_str); + get_timespec_as_str(&(de->ctime), time_str, sizeof(time_str)); + debugf(DBG_LEVEL_EXT, KCYN"cfs_getattr: ctime=[%s]", time_str); + + int default_mode_dir, default_mode_file; + + if (option_enable_chmod) + { + default_mode_dir = de->chmod; + default_mode_file = de->chmod; + } + else + { + default_mode_dir = 0755; + default_mode_file = 0666; + } + if (de->isdir) { stbuf->st_size = 0; - stbuf->st_mode = S_IFDIR | 0755; + stbuf->st_mode = S_IFDIR | default_mode_dir; stbuf->st_nlink = 2; } else if (de->islink) { stbuf->st_size = 1; - stbuf->st_mode = S_IFLNK | 0755; + stbuf->st_mode = S_IFLNK | default_mode_dir; stbuf->st_nlink = 1; stbuf->st_size = de->size; /* calc. blocks as if 4K blocksize filesystem; stat uses units of 512B */ @@ -233,105 +125,193 @@ static int cfs_getattr(const char *path, struct stat *stbuf) stbuf->st_size = de->size; /* calc. blocks as if 4K blocksize filesystem; stat uses units of 512B */ stbuf->st_blocks = ((4095 + de->size) / 4096) * 8; - stbuf->st_mode = S_IFREG | 0666; + stbuf->st_mode = S_IFREG | default_mode_file; stbuf->st_nlink = 1; } + debugf(DBG_LEVEL_NORM, KBLU "exit 2: cfs_getattr(%s)", path); return 0; } -static int cfs_fgetattr(const char *path, struct stat *stbuf, struct fuse_file_info *info) +static int cfs_fgetattr(const char* path, struct stat* stbuf, + struct fuse_file_info* info) { - openfile *of = (openfile *)(uintptr_t)info->fh; + debugf(DBG_LEVEL_NORM, KBLU "cfs_fgetattr(%s)", path); + openfile* of = (openfile*)(uintptr_t)info->fh; if (of) { + //get file. if not in cache will be downloaded. + dir_entry* de = path_info(path); + if (!de) + { + debug_list_cache_content(); + debugf(DBG_LEVEL_NORM, KBLU"exit 1: cfs_fgetattr(%s) "KYEL"not-in-cache/cloud", + path); + return -ENOENT; + } + int default_mode_file; + if (option_enable_chmod) + default_mode_file = de->chmod; + else + default_mode_file = 0666; + stbuf->st_size = cloudfs_file_size(of->fd); - stbuf->st_mode = S_IFREG | 0666; + stbuf->st_mode = S_IFREG | default_mode_file; stbuf->st_nlink = 1; + debugf(DBG_LEVEL_NORM, KBLU "exit 0: cfs_fgetattr(%s)", path); return 0; } + debugf(DBG_LEVEL_NORM, KRED "exit 1: cfs_fgetattr(%s)", path); return -ENOENT; } -static int cfs_readdir(const char *path, void *buf, fuse_fill_dir_t filldir, off_t offset, struct fuse_file_info *info) +static int cfs_readdir(const char* path, void* buf, fuse_fill_dir_t filldir, + off_t offset, struct fuse_file_info* info) { - dir_entry *de; + debugf(DBG_LEVEL_NORM, KBLU "cfs_readdir(%s)", path); + dir_entry* de; if (!caching_list_directory(path, &de)) + { + debug_list_cache_content(); + debugf(DBG_LEVEL_NORM, KRED "exit 0: cfs_readdir(%s)", path); return -ENOLINK; + } filldir(buf, ".", NULL, 0); filldir(buf, "..", NULL, 0); for (; de; de = de->next) filldir(buf, de->name, NULL, 0); + debug_list_cache_content(); + debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_readdir(%s)", path); return 0; } -static int cfs_mkdir(const char *path, mode_t mode) +static int cfs_mkdir(const char* path, mode_t mode) { - if (cloudfs_create_directory(path)) + debugf(DBG_LEVEL_NORM, KBLU "cfs_mkdir(%s)", path); + int response = cloudfs_create_directory(path); + if (response) { update_dir_cache(path, 0, 1, 0); + debug_list_cache_content(); + debugf(DBG_LEVEL_NORM, KBLU "exit 0: cfs_mkdir(%s)", path); return 0; } + debugf(DBG_LEVEL_NORM, KRED "exit 1: cfs_mkdir(%s) response=%d", path, + response); return -ENOENT; } -static int cfs_create(const char *path, mode_t mode, struct fuse_file_info *info) +static int cfs_create(const char* path, mode_t mode, + struct fuse_file_info* info) { - FILE *temp_file; + debugf(DBG_LEVEL_NORM, KBLU "cfs_create(%s)", path); + FILE* temp_file; + int errsv; + char file_path_safe[NAME_MAX] = ""; if (*temp_dir) + { + get_safe_cache_file_path(path, file_path_safe, temp_dir); + temp_file = fopen(file_path_safe, "w+b"); + errsv = errno; + if (temp_file == NULL) { - char tmp_path[PATH_MAX]; - strncpy(tmp_path, path, PATH_MAX); - - char *pch; - while((pch = strchr(tmp_path, '/'))) { - *pch = '.'; - } - - char file_path[PATH_MAX]; - snprintf(file_path, PATH_MAX, "%s/.cloudfuse%ld-%s", temp_dir, - (long)getpid(), tmp_path); - temp_file = fopen(file_path, "w+b"); + debugf(DBG_LEVEL_NORM, KRED + "exit 0: cfs_create cannot open temp file %s.error %s\n", file_path_safe, + strerror(errsv)); + return -EIO; } + } else + { temp_file = tmpfile(); - - openfile *of = (openfile *)malloc(sizeof(openfile)); + errsv = errno; + if (temp_file == NULL) + { + debugf(DBG_LEVEL_NORM, KRED + "exit 1: cfs_create cannot open tmp file for path %s.error %s\n", path, + strerror(errsv)); + return -EIO; + } + } + openfile* of = (openfile*)malloc(sizeof(openfile)); of->fd = dup(fileno(temp_file)); fclose(temp_file); of->flags = info->flags; info->fh = (uintptr_t)of; update_dir_cache(path, 0, 0, 0); info->direct_io = 1; + dir_entry* de = check_path_info(path); + if (de) + { + debugf(DBG_LEVEL_EXT, KCYN"cfs_create(%s): found in cache", path); + struct timespec now; + clock_gettime(CLOCK_REALTIME, &now); + debugf(DBG_LEVEL_EXT, KCYN"cfs_create(%s) set utimes as now", path); + de->atime.tv_sec = now.tv_sec; + de->atime.tv_nsec = now.tv_nsec; + de->mtime.tv_sec = now.tv_sec; + de->mtime.tv_nsec = now.tv_nsec; + de->ctime.tv_sec = now.tv_sec; + de->ctime.tv_nsec = now.tv_nsec; + + char time_str[TIME_CHARS] = ""; + get_timespec_as_str(&(de->atime), time_str, sizeof(time_str)); + debugf(DBG_LEVEL_EXT, KCYN"cfs_create: atime=[%s]", time_str); + get_timespec_as_str(&(de->mtime), time_str, sizeof(time_str)); + debugf(DBG_LEVEL_EXT, KCYN"cfs_create: mtime=[%s]", time_str); + get_timespec_as_str(&(de->ctime), time_str, sizeof(time_str)); + debugf(DBG_LEVEL_EXT, KCYN"cfs_create: ctime=[%s]", time_str); + + //set chmod & chown + de->chmod = mode; + de->uid = geteuid(); + de->gid = getegid(); + } + else + debugf(DBG_LEVEL_EXT, KBLU "cfs_create(%s) "KYEL"dir-entry not found", path); + debugf(DBG_LEVEL_NORM, KBLU "exit 2: cfs_create(%s)=(%s) result=%d:%s", path, + file_path_safe, errsv, strerror(errsv)); return 0; } -static int cfs_open(const char *path, struct fuse_file_info *info) +// open (download) file from cloud +// todo: implement etag optimisation, download only if content changed, http://www.17od.com/2012/12/19/ten-useful-openstack-swift-features/ +static int cfs_open(const char* path, struct fuse_file_info* info) { - FILE *temp_file; - dir_entry *de = path_info(path); + debugf(DBG_LEVEL_NORM, KBLU "cfs_open(%s)", path); + FILE* temp_file = NULL; + int errsv; + dir_entry* de = path_info(path); if (*temp_dir) { - char tmp_path[PATH_MAX]; - strncpy(tmp_path, path, PATH_MAX); + char file_path_safe[NAME_MAX]; + get_safe_cache_file_path(path, file_path_safe, temp_dir); - char *pch; - while((pch = strchr(tmp_path, '/'))) { - *pch = '.'; - } - - char file_path[PATH_MAX]; - snprintf(file_path, PATH_MAX, "%s/.cloudfuse%ld-%s", temp_dir, - (long)getpid(), tmp_path); - - if(access(file_path, F_OK) != -1) + debugf(DBG_LEVEL_EXT, "cfs_open: try open (%s)", file_path_safe); + if (access(file_path_safe, F_OK) != -1) { // file exists - temp_file = fopen(file_path, "r"); + temp_file = fopen(file_path_safe, "r"); + errsv = errno; + if (temp_file == NULL) + { + debugf(DBG_LEVEL_NORM, + KRED"exit 0: cfs_open can't open temp_file=[%s] err=%d:%s", file_path_safe, + errsv, strerror(errsv)); + return -ENOENT; + } + else + debugf(DBG_LEVEL_EXT, "cfs_open: file exists"); } - else if (!(info->flags & O_WRONLY)) + else { + errsv = errno; + debugf(DBG_LEVEL_EXT, "cfs_open: file not in cache, err=%s", strerror(errsv)); + //FIXME: commented out as this condition will not be meet in some odd cases and program will crash + //if (!(info->flags & O_WRONLY)) { + debugf(DBG_LEVEL_EXT, "cfs_open: opening for write"); + // we need to lock on the filename another process could open the file // while we are writing to it and then only read part of the file @@ -350,11 +330,21 @@ static int cfs_open(const char *path, struct fuse_file_info *info) // that file. // TODO: just to prevent this craziness for now - temp_file = fopen(file_path, "w+b"); + temp_file = fopen(file_path_safe, "w+b"); + errsv = errno; + if (temp_file == NULL) + { + debugf(DBG_LEVEL_NORM, + KRED"exit 1: cfs_open cannot open temp_file=[%s] err=%d:%s", file_path_safe, + errsv, strerror(errsv)); + return -ENOENT; + } if (!cloudfs_object_write_fp(path, temp_file)) { fclose(temp_file); + debugf(DBG_LEVEL_NORM, KRED "exit 2: cfs_open(%s) cannot download/write", + path); return -ENOENT; } } @@ -362,211 +352,439 @@ static int cfs_open(const char *path, struct fuse_file_info *info) else { temp_file = tmpfile(); + if (temp_file == NULL) + { + debugf(DBG_LEVEL_NORM, KRED"exit 3: cfs_open cannot create temp_file err=%s", + strerror(errno)); + return -ENOENT; + } + if (!(info->flags & O_TRUNC)) { if (!cloudfs_object_write_fp(path, temp_file) && !(info->flags & O_CREAT)) { fclose(temp_file); + debugf(DBG_LEVEL_NORM, KRED"exit 4: cfs_open(%s) cannot download/write", path); return -ENOENT; } } } update_dir_cache(path, (de ? de->size : 0), 0, 0); - - openfile *of = (openfile *)malloc(sizeof(openfile)); + openfile* of = (openfile*)malloc(sizeof(openfile)); of->fd = dup(fileno(temp_file)); if (of->fd == -1) + { + //FIXME: potential leak if free not used? + free(of); + debugf(DBG_LEVEL_NORM, KRED "exit 5: cfs_open(%s) of->fd", path); return -ENOENT; + } fclose(temp_file); + //TODO: why this allocation to of? of->flags = info->flags; info->fh = (uintptr_t)of; info->direct_io = 1; info->nonseekable = 1; + //FIXME: potential leak if free(of) not used? although if free(of) is used will generate bad descriptor errors + debugf(DBG_LEVEL_NORM, KBLU "exit 6: cfs_open(%s)", path); return 0; } -static int cfs_read(const char *path, char *buf, size_t size, off_t offset, struct fuse_file_info *info) +static int cfs_read(const char* path, char* buf, size_t size, off_t offset, + struct fuse_file_info* info) { - return pread(((openfile *)(uintptr_t)info->fh)->fd, buf, size, offset); + debugf(DBG_LEVEL_EXTALL, KBLU "cfs_read(%s) buffsize=%lu offset=%lu", path, + size, offset); + file_buffer_size = size; + debug_print_descriptor(info); + int result = pread(((openfile*)(uintptr_t)info->fh)->fd, buf, size, offset); + debugf(DBG_LEVEL_EXTALL, KBLU "exit: cfs_read(%s) result=%s", path, + strerror(errno)); + return result; } -static int cfs_flush(const char *path, struct fuse_file_info *info) +//todo: flush will upload a file again even if just file attributes are changed. +//optimisation needed to detect if content is changed and to only save meta when just attribs are modified. +static int cfs_flush(const char* path, struct fuse_file_info* info) { - openfile *of = (openfile *)(uintptr_t)info->fh; + debugf(DBG_LEVEL_NORM, KBLU "cfs_flush(%s)", path); + debug_print_descriptor(info); + openfile* of = (openfile*)(uintptr_t)info->fh; + int errsv = 0; + if (of) { update_dir_cache(path, cloudfs_file_size(of->fd), 0, 0); if (of->flags & O_RDWR || of->flags & O_WRONLY) { - FILE *fp = fdopen(dup(of->fd), "r"); - rewind(fp); - if (!cloudfs_object_read_fp(path, fp)) + FILE* fp = fdopen(dup(of->fd), "r"); + errsv = errno; + if (fp != NULL) { + rewind(fp); + //calculate md5 hash, compare with cloud hash to determine if file content is changed + char md5_file_hash_str[MD5_DIGEST_LENGTH + 1] = "\0"; + file_md5(fp, md5_file_hash_str); + dir_entry* de = check_path_info(path); + if (de && de->md5sum != NULL && (!strcasecmp(md5_file_hash_str, de->md5sum))) + { + //file content is identical, no need to upload entire file, just update metadata + debugf(DBG_LEVEL_NORM, KBLU + "cfs_flush(%s): skip full upload as content did not change", path); + cloudfs_update_meta(de); + } + else + { + rewind(fp); + debugf(DBG_LEVEL_NORM, KBLU + "cfs_flush(%s): perform full upload as content changed (or no file found in cache)", + path); + if (!cloudfs_object_read_fp(path, fp)) + { + fclose(fp); + errsv = errno; + debugf(DBG_LEVEL_NORM, KRED"exit 0: cfs_flush(%s) result=%d:%s", path, errsv, + strerror(errno)); + return -ENOENT; + } + } fclose(fp); - return -ENOENT; + errsv = errno; } - fclose(fp); + else + debugf(DBG_LEVEL_EXT, KRED "status: cfs_flush, err=%d:%s", errsv, + strerror(errno)); + } + } + debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_flush(%s) result=%d:%s", path, errsv, + strerror(errno)); + return 0; +} + +static int cfs_release(const char* path, struct fuse_file_info* info) +{ + debugf(DBG_LEVEL_NORM, KBLU "cfs_release(%s)", path); + close(((openfile*)(uintptr_t)info->fh)->fd); + debugf(DBG_LEVEL_NORM, KBLU "exit: cfs_release(%s)", path); + return 0; +} + +static int cfs_rmdir(const char* path) +{ + debugf(DBG_LEVEL_NORM, KBLU "cfs_rmdir(%s)", path); + int success = cloudfs_delete_object(path); + if (success == -1) + { + debugf(DBG_LEVEL_NORM, KBLU "exit 0: cfs_rmdir(%s)", path); + return -ENOTEMPTY; + } + if (success) + { + dir_decache(path); + debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_rmdir(%s)", path); + return 0; + } + debugf(DBG_LEVEL_NORM, KBLU "exit 2: cfs_rmdir(%s)", path); + return -ENOENT; +} + +static int cfs_ftruncate(const char* path, off_t size, + struct fuse_file_info* info) +{ + debugf(DBG_LEVEL_NORM, KBLU "cfs_ftruncate(%s)", path); + file_buffer_size = size; + openfile* of = (openfile*)(uintptr_t)info->fh; + if (ftruncate(of->fd, size)) + return -errno; + lseek(of->fd, 0, SEEK_SET); + update_dir_cache(path, size, 0, 0); + debugf(DBG_LEVEL_NORM, KBLU "exit: cfs_ftruncate(%s)", path); + return 0; +} + +static int cfs_write(const char* path, const char* buf, size_t length, + off_t offset, struct fuse_file_info* info) +{ + debugf(DBG_LEVEL_EXTALL, KBLU "cfs_write(%s) bufflength=%lu offset=%lu", path, + length, offset); + // FIXME: Potential inconsistent cache update if pwrite fails? + update_dir_cache(path, offset + length, 0, 0); + //int result = pwrite(info->fh, buf, length, offset); + int result = pwrite(((openfile*)(uintptr_t)info->fh)->fd, buf, length, offset); + int errsv = errno; + if (errsv == 0) + debugf(DBG_LEVEL_EXTALL, KBLU "exit 0: cfs_write(%s) result=%d:%s", path, + errsv, strerror(errsv)); + else + debugf(DBG_LEVEL_EXTALL, KBLU "exit 1: cfs_write(%s) "KRED"result=%d:%s", path, + errsv, strerror(errsv)); + return result; +} + +static int cfs_unlink(const char* path) +{ + debugf(DBG_LEVEL_NORM, KBLU "cfs_unlink(%s)", path); + int success = cloudfs_delete_object(path); + if (success == -1) + { + debugf(DBG_LEVEL_NORM, KRED "exit 0: cfs_unlink(%s)", path); + return -EACCES; + } + if (success) + { + dir_decache(path); + debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_unlink(%s)", path); + return 0; + } + debugf(DBG_LEVEL_NORM, KRED "exit 2: cfs_unlink(%s)", path); + return -ENOENT; +} + +static int cfs_fsync(const char* path, int idunno, struct fuse_file_info* info) +{ + debugf(DBG_LEVEL_NORM, "cfs_fsync(%s)", path); + return 0; +} + +static int cfs_truncate(const char* path, off_t size) +{ + debugf(DBG_LEVEL_NORM, "cfs_truncate(%s)", path); + cloudfs_object_truncate(path, size); + debugf(DBG_LEVEL_NORM, "exit: cfs_truncate(%s)", path); + return 0; +} + +//this is called regularly on copy (via mc), is optimised (cached) +static int cfs_statfs(const char* path, struct statvfs* stat) +{ + debugf(DBG_LEVEL_NORM, KBLU "cfs_statfs(%s)", path); + if (cloudfs_statfs(path, stat)) + { + debugf(DBG_LEVEL_NORM, KBLU "exit 0: cfs_statfs(%s)", path); + return 0; + } + else + { + debugf(DBG_LEVEL_NORM, KRED"exit 1: cfs_statfs(%s) not-found", path); + return -EIO; + } +} + +static int cfs_chown(const char* path, uid_t uid, gid_t gid) +{ + debugf(DBG_LEVEL_NORM, KBLU "cfs_chown(%s,%d,%d)", path, uid, gid); + dir_entry* de = check_path_info(path); + if (de) + { + if (de->uid != uid || de->gid != gid) + { + debugf(DBG_LEVEL_NORM, "cfs_chown(%s): change from uid:gid %d:%d to %d:%d", + path, de->uid, de->gid, uid, gid); + de->uid = uid; + de->gid = gid; + //issue a PUT request to update metadata (quick request just to update headers) + int response = cloudfs_update_meta(de); } } return 0; } -static int cfs_release(const char *path, struct fuse_file_info *info) +static int cfs_chmod(const char* path, mode_t mode) { - close(((openfile *)(uintptr_t)info->fh)->fd); - return 0; -} - -static int cfs_rmdir(const char *path) -{ - int success = cloudfs_delete_object(path); - if (success == -1) - return -ENOTEMPTY; - if (success) + debugf(DBG_LEVEL_NORM, KBLU"cfs_chmod(%s,%d)", path, mode); + dir_entry* de = check_path_info(path); + if (de) { - dir_decache(path); - return 0; + if (de->chmod != mode) + { + debugf(DBG_LEVEL_NORM, "cfs_chmod(%s): change mode from %d to %d", path, + de->chmod, mode); + de->chmod = mode; + //todo: issue a PUT request to update metadata (empty request just to update headers?) + int response = cloudfs_update_meta(de); + } } - return -ENOENT; -} - -static int cfs_ftruncate(const char *path, off_t size, struct fuse_file_info *info) -{ - openfile *of = (openfile *)(uintptr_t)info->fh; - if (ftruncate(of->fd, size)) - return -errno; - lseek(of->fd, 0, SEEK_SET); - update_dir_cache(path, size, 0, 0); return 0; } -static int cfs_write(const char *path, const char *buf, size_t length, off_t offset, struct fuse_file_info *info) +static int cfs_rename(const char* src, const char* dst) { - update_dir_cache(path, offset + length, 0, 0); - return pwrite(((openfile *)(uintptr_t)info->fh)->fd, buf, length, offset); -} - -static int cfs_unlink(const char *path) -{ - int success = cloudfs_delete_object(path); - if (success == -1) - return -EACCES; - if (success) - { - dir_decache(path); - return 0; - } - return -ENOENT; -} - -static int cfs_fsync(const char *path, int idunno, struct fuse_file_info *info) -{ - return 0; -} - -static int cfs_truncate(const char *path, off_t size) -{ - cloudfs_object_truncate(path, size); - return 0; -} - -static int cfs_statfs(const char *path, struct statvfs *stat) -{ - if (cloudfs_statfs(path, stat)){ - return 0; - } - else - return -EIO; -} - -static int cfs_chown(const char *path, uid_t uid, gid_t gid) -{ - return 0; -} - -static int cfs_chmod(const char *path, mode_t mode) -{ - return 0; -} - -static int cfs_rename(const char *src, const char *dst) -{ - dir_entry *src_de = path_info(src); + debugf(DBG_LEVEL_NORM, KBLU"cfs_rename(%s, %s)", src, dst); + dir_entry* src_de = path_info(src); if (!src_de) - return -ENOENT; + { + debugf(DBG_LEVEL_NORM, KRED"exit 0: cfs_rename(%s,%s) not-found", src, dst); + return -ENOENT; + } if (src_de->isdir) + { + debugf(DBG_LEVEL_NORM, KRED"exit 1: cfs_rename(%s,%s) cannot rename dirs!", + src, dst); return -EISDIR; + } if (cloudfs_copy_object(src, dst)) { /* FIXME this isn't quite right as doesn't preserve last modified */ + //fix done in cloudfs_copy_object() update_dir_cache(dst, src_de->size, 0, 0); - return cfs_unlink(src); + int result = cfs_unlink(src); + + dir_entry* dst_de = path_info(dst); + if (!dst_de) + debugf(DBG_LEVEL_NORM, KRED"cfs_rename(%s,%s) dest-not-found-in-cache", src, + dst); + else + { + debugf(DBG_LEVEL_NORM, KBLU"cfs_rename(%s,%s) upload ok", src, dst); + //copy attributes, shortcut, rather than forcing a download from cloud + copy_dir_entry(src_de, dst_de); + } + + debugf(DBG_LEVEL_NORM, KBLU"exit 3: cfs_rename(%s,%s)", src, dst); + return result; } + debugf(DBG_LEVEL_NORM, KRED"exit 4: cfs_rename(%s,%s) io error", src, dst); return -EIO; } -static int cfs_symlink(const char *src, const char *dst) +static int cfs_symlink(const char* src, const char* dst) { - if(cloudfs_create_symlink(src, dst)) + debugf(DBG_LEVEL_NORM, KBLU"cfs_symlink(%s, %s)", src, dst); + if (cloudfs_create_symlink(src, dst)) { update_dir_cache(dst, 1, 0, 1); + debugf(DBG_LEVEL_NORM, KBLU"exit0: cfs_symlink(%s, %s)", src, dst); return 0; } + debugf(DBG_LEVEL_NORM, KRED"exit1: cfs_symlink(%s, %s) io error", src, dst); return -EIO; } static int cfs_readlink(const char* path, char* buf, size_t size) { - FILE *temp_file = tmpfile(); + debugf(DBG_LEVEL_NORM, KBLU"cfs_readlink(%s)", path); + //fixme: use temp file specified in config + FILE* temp_file = tmpfile(); int ret = 0; if (!cloudfs_object_write_fp(path, temp_file)) { - ret = -ENOENT; + debugf(DBG_LEVEL_NORM, KRED"exit 1: cfs_readlink(%s) not found", path); + ret = -ENOENT; } if (!pread(fileno(temp_file), buf, size, 0)) { - ret = -ENOENT; + debugf(DBG_LEVEL_NORM, KRED"exit 2: cfs_readlink(%s) not found", path); + ret = -ENOENT; } fclose(temp_file); + debugf(DBG_LEVEL_NORM, KBLU"exit 3: cfs_readlink(%s)", path); return ret; } -static void *cfs_init(struct fuse_conn_info *conn) +static void* cfs_init(struct fuse_conn_info* conn) { signal(SIGPIPE, SIG_IGN); return NULL; } -char *get_home_dir() +//http://man7.org/linux/man-pages/man2/utimensat.2.html +static int cfs_utimens(const char* path, const struct timespec times[2]) { - char *home; - if ((home = getenv("HOME")) && !access(home, R_OK)) - return home; - struct passwd *pwd = getpwuid(geteuid()); - if ((home = pwd->pw_dir) && !access(home, R_OK)) - return home; - return "~"; + debugf(DBG_LEVEL_NORM, KBLU "cfs_utimens(%s)", path); + dir_entry* path_de = path_info(path); + if (!path_de) + { + debugf(DBG_LEVEL_NORM, KRED"exit 0: cfs_utimens(%s) file not in cache", path); + return -ENOENT; + } + struct timespec now; + clock_gettime(CLOCK_REALTIME, &now); + + if (path_de->atime.tv_sec != times[0].tv_sec + || path_de->atime.tv_nsec != times[0].tv_nsec || + path_de->mtime.tv_sec != times[1].tv_sec + || path_de->mtime.tv_nsec != times[1].tv_nsec) + { + debugf(DBG_LEVEL_EXT, KCYN + "cfs_utimens: change for %s, prev: atime=%li.%li mtime=%li.%li, new: atime=%li.%li mtime=%li.%li", + path, + path_de->atime.tv_sec, path_de->atime.tv_nsec, path_de->mtime.tv_sec, + path_de->mtime.tv_nsec, + times[0].tv_sec, times[0].tv_nsec, times[1].tv_sec, times[1].tv_nsec); + char time_str[TIME_CHARS] = ""; + get_timespec_as_str(×[1], time_str, sizeof(time_str)); + debugf(DBG_LEVEL_EXT, KCYN"cfs_utimens: set mtime=[%s]", time_str); + get_timespec_as_str(×[0], time_str, sizeof(time_str)); + debugf(DBG_LEVEL_EXT, KCYN"cfs_utimens: set atime=[%s]", time_str); + path_de->atime = times[0]; + path_de->mtime = times[1]; + // not sure how to best obtain ctime from fuse source file. just record current date. + path_de->ctime = now; + //calling a meta cloud update here is not always needed. + //touch for example opens and closes/flush the file. + //worth implementing a meta cache functionality to avoid multiple uploads on meta changes + //when changing timestamps on very large files, touch command will trigger 2 x long and useless file uploads on cfs_flush() + } + else + debugf(DBG_LEVEL_EXT, KCYN"cfs_utimens: a/m/time not changed"); + debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_utimens(%s)", path); + return 0; } -FuseOptions options = { - .cache_timeout = "600", - .verify_ssl = "true", - .segment_size = "1073741824", - .segment_above = "2147483647", - .storage_url = "", - .container = "", - //.temp_dir = "/tmp/", - .temp_dir = "", - .client_id = "", - .client_secret = "", - .refresh_token = "" + +int cfs_setxattr(const char* path, const char* name, const char* value, + size_t size, int flags) +{ + return 0; +} + +int cfs_getxattr(const char* path, const char* name, char* value, size_t size) +{ + return 0; +} + +int cfs_removexattr(const char* path, const char* name) +{ + return 0; +} + +int cfs_listxattr(const char* path, char* list, size_t size) +{ + return 0; +} + +FuseOptions options = +{ + .cache_timeout = "600", + .verify_ssl = "true", + .segment_size = "1073741824", + .segment_above = "2147483647", + .storage_url = "", + .container = "", + //.temp_dir = "/tmp/", + .temp_dir = "", + .client_id = "", + .client_secret = "", + .refresh_token = "" }; -int parse_option(void *data, const char *arg, int key, struct fuse_args *outargs) +ExtraFuseOptions extra_options = +{ + .get_extended_metadata = "false", + .curl_verbose = "false", + .cache_statfs_timeout = 0, + .debug_level = 0, + .curl_progress_state = "false", + .enable_chown = "false", + .enable_chmod = "false" +}; + +int parse_option(void* data, const char* arg, int key, + struct fuse_args* outargs) { if (sscanf(arg, " cache_timeout = %[^\r\n ]", options.cache_timeout) || sscanf(arg, " verify_ssl = %[^\r\n ]", options.verify_ssl) || @@ -577,20 +795,71 @@ int parse_option(void *data, const char *arg, int key, struct fuse_args *outargs sscanf(arg, " temp_dir = %[^\r\n ]", options.temp_dir) || sscanf(arg, " client_id = %[^\r\n ]", options.client_id) || sscanf(arg, " client_secret = %[^\r\n ]", options.client_secret) || - sscanf(arg, " refresh_token = %[^\r\n ]", options.refresh_token)) + sscanf(arg, " refresh_token = %[^\r\n ]", options.refresh_token) || + + sscanf(arg, " get_extended_metadata = %[^\r\n ]", + extra_options.get_extended_metadata) || + sscanf(arg, " curl_verbose = %[^\r\n ]", extra_options.curl_verbose) || + sscanf(arg, " cache_statfs_timeout = %[^\r\n ]", + extra_options.cache_statfs_timeout) || + sscanf(arg, " debug_level = %[^\r\n ]", extra_options.debug_level) || + sscanf(arg, " curl_progress_state = %[^\r\n ]", + extra_options.curl_progress_state) || + sscanf(arg, " enable_chmod = %[^\r\n ]", extra_options.enable_chmod) || + sscanf(arg, " enable_chown = %[^\r\n ]", extra_options.enable_chown) + ) return 0; if (!strcmp(arg, "-f") || !strcmp(arg, "-d") || !strcmp(arg, "debug")) cloudfs_debug(1); return 1; } -int main(int argc, char **argv) +//allows memory leaks inspections +void interrupt_handler(int sig) { + debugf(DBG_LEVEL_NORM, "Got interrupt signal %d, cleaning memory", sig); + //TODO: clean memory allocations + //http://www.cprogramming.com/debugging/valgrind.html + cloudfs_free(); + //TODO: clear dir cache + pthread_mutex_destroy(&dcachemut); + exit(0); +} + +void initialise_options() +{ + //todo: handle param init consistently, quite heavy implementation + cloudfs_verify_ssl(!strcasecmp(options.verify_ssl, "true")); + cloudfs_option_get_extended_metadata(!strcasecmp( + extra_options.get_extended_metadata, "true")); + cloudfs_option_curl_verbose(!strcasecmp(extra_options.curl_verbose, "true")); + //lean way to init params, to be used as reference + if (*extra_options.debug_level) + option_debug_level = atoi(extra_options.debug_level); + if (*extra_options.cache_statfs_timeout) + option_cache_statfs_timeout = atoi(extra_options.cache_statfs_timeout); + if (*extra_options.curl_progress_state) + option_curl_progress_state = !strcasecmp(extra_options.curl_progress_state, + "true"); + if (*extra_options.enable_chmod) + option_enable_chmod = !strcasecmp(extra_options.enable_chmod, "true"); + if (*extra_options.enable_chown) + option_enable_chown = !strcasecmp(extra_options.enable_chown, "true"); +} + +int main(int argc, char** argv) +{ + if (debug) + fprintf(stderr, "Starting hubicfuse on homedir %s!\n", get_home_dir()); + + signal(SIGINT, interrupt_handler); + char settings_filename[MAX_PATH_SIZE] = ""; - FILE *settings; + FILE* settings; struct fuse_args args = FUSE_ARGS_INIT(argc, argv); - snprintf(settings_filename, sizeof(settings_filename), "%s/.hubicfuse", get_home_dir()); + snprintf(settings_filename, sizeof(settings_filename), "%s/.hubicfuse", + get_home_dir()); if ((settings = fopen(settings_filename, "r"))) { char line[OPTION_SIZE]; @@ -600,7 +869,6 @@ int main(int argc, char **argv) } fuse_opt_parse(&args, &options, NULL, parse_option); - cache_timeout = atoi(options.cache_timeout); segment_size = atoll(options.segment_size); segment_above = atoll(options.segment_above); @@ -611,42 +879,66 @@ int main(int argc, char **argv) if (!*options.client_id || !*options.client_secret || !*options.refresh_token) { - fprintf(stderr, "Unable to determine client_id, client_secret or refresh_token.\n\n"); + fprintf(stderr, + "Unable to determine client_id, client_secret or refresh_token.\n\n"); fprintf(stderr, "These can be set either as mount options or in " - "a file named %s\n\n", settings_filename); + "a file named %s\n\n", settings_filename); fprintf(stderr, " client_id=[App's id]\n"); fprintf(stderr, " client_secret=[App's secret]\n"); fprintf(stderr, " refresh_token=[Get it running hubic_token]\n"); fprintf(stderr, "The following settings are optional:\n\n"); - fprintf(stderr, " cache_timeout=[Seconds for directory caching, default 600]\n"); - fprintf(stderr, " verify_ssl=[False to disable SSL cert verification]\n"); - fprintf(stderr, " segment_size=[Size to use when creating DLOs, default 1073741824]\n"); - fprintf(stderr, " segment_above=[File size at which to use segments, defult 2147483648]\n"); - fprintf(stderr, " storage_url=[Storage URL for other tenant to view container]\n"); - fprintf(stderr, " container=[Public container to view of tenant specified by storage_url]\n"); + fprintf(stderr, + " cache_timeout=[Seconds for directory caching, default 600]\n"); + fprintf(stderr, " verify_ssl=[false to disable SSL cert verification]\n"); + fprintf(stderr, + " segment_size=[Size to use when creating DLOs, default 1073741824]\n"); + fprintf(stderr, + " segment_above=[File size at which to use segments, defult 2147483648]\n"); + fprintf(stderr, + " storage_url=[Storage URL for other tenant to view container]\n"); + fprintf(stderr, + " container=[Public container to view of tenant specified by storage_url]\n"); fprintf(stderr, " temp_dir=[Directory to store temp files]\n"); - + fprintf(stderr, + " get_extended_metadata=[true to enable download of utime, chmod, chown file attributes (but slower)]\n"); + fprintf(stderr, + " curl_verbose=[true to debug info on curl requests (lots of output)]\n"); + fprintf(stderr, + " curl_progress_state=[true to enable progress callback enabled. Mostly used for debugging]\n"); + fprintf(stderr, + " cache_statfs_timeout=[number of seconds to cache requests to statfs (cloud statistics), 0 for no cache]\n"); + fprintf(stderr, + " debug_level=[0 to 2, 0 for minimal verbose debugging. No debug if -d or -f option is not provided.]\n"); + fprintf(stderr, " enable_chmod=[true to enable chmod support on fuse]\n"); + fprintf(stderr, " enable_chown=[true to enable chown support on fuse]\n"); return 1; } - cloudfs_init(); - - cloudfs_verify_ssl(!strcasecmp(options.verify_ssl, "true")); - - cloudfs_set_credentials(options.client_id, options.client_secret, options.refresh_token); + initialise_options(); + if (debug) + { + fprintf(stderr, "debug_level = %d\n", option_debug_level); + fprintf(stderr, "get_extended_metadata = %d\n", option_get_extended_metadata); + fprintf(stderr, "curl_progress_state = %d\n", option_curl_progress_state); + fprintf(stderr, "enable_chmod = %d\n", option_enable_chmod); + fprintf(stderr, "enable_chown = %d\n", option_enable_chown); + } + cloudfs_set_credentials(options.client_id, options.client_secret, + options.refresh_token); if (!cloudfs_connect()) { fprintf(stderr, "Failed to authenticate.\n"); return 1; } - - #ifndef HAVE_OPENSSL - #warning Compiling without libssl, will run single-threaded. + //todo: check why in some cases the define below is not available (when running the binary on symbolic linked folders) +#ifndef HAVE_OPENSSL +#warning Compiling without libssl, will run single-threaded. fuse_opt_add_arg(&args, "-s"); - #endif +#endif - struct fuse_operations cfs_oper = { + struct fuse_operations cfs_oper = + { .readdir = cfs_readdir, .mkdir = cfs_mkdir, .read = cfs_read, @@ -669,8 +961,17 @@ int main(int argc, char **argv) .symlink = cfs_symlink, .readlink = cfs_readlink, .init = cfs_init, + .utimens = cfs_utimens, +#ifdef HAVE_SETXATTR + .setxattr = cfs_setxattr, + .getxattr = cfs_getxattr, + .listxattr = cfs_listxattr, + .removexattr = cfs_removexattr, +#endif }; - pthread_mutex_init(&dmut, NULL); + pthread_mutexattr_init(&mutex_attr); + pthread_mutexattr_settype(&mutex_attr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&dcachemut, &mutex_attr); return fuse_main(args.argc, args.argv, &cfs_oper, &options); } diff --git a/commonfs.c b/commonfs.c new file mode 100644 index 0000000..5d54e1b --- /dev/null +++ b/commonfs.c @@ -0,0 +1,684 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#ifdef __linux__ +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "commonfs.h" +#include "config.h" + +pthread_mutex_t dcachemut; +pthread_mutexattr_t mutex_attr; +dir_cache* dcache; +char* temp_dir; +int cache_timeout; +int debug = 0; +int verify_ssl = 2; +bool option_get_extended_metadata = false; +bool option_curl_verbose = false; +int option_cache_statfs_timeout = 0; +int option_debug_level = 0; +int option_curl_progress_state = 1;//1 to disable curl progress +bool option_enable_chown = false; +bool option_enable_chmod = false; +bool option_enable_progressive_upload = false; +bool option_enable_progressive_download = false; +size_t file_buffer_size = 0; + +// needed to get correct GMT / local time +// hubic stores time as GMT so we have to do conversions +// http://zhu-qy.blogspot.ro/2012/11/ref-how-to-convert-from-utc-to-local.html +time_t my_timegm(struct tm* tm) +{ + time_t epoch = 0; + time_t offset = mktime(gmtime(&epoch)); + time_t utc = mktime(tm); + return difftime(utc, offset); +} + +//expect time_str as a friendly string format +time_t get_time_from_str_as_gmt(char* time_str) +{ + struct tm val_time_tm; + time_t val_time_t; + strptime(time_str, "%FT%T", &val_time_tm); + val_time_tm.tm_isdst = -1; + val_time_t = my_timegm(&val_time_tm); + return val_time_t; +} + +time_t get_time_as_local(time_t time_t_val, char time_str[], int char_buf_size) +{ + struct tm loc_time_tm; + loc_time_tm = *localtime(&time_t_val); + if (time_str != NULL) + { + //debugf(DBG_LEVEL_NORM, 0,"Local len=%d size=%d pass=%d", strlen(time_str), sizeof(time_str), char_buf_size); + strftime(time_str, char_buf_size, "%c", &loc_time_tm); + //debugf(DBG_LEVEL_NORM, 0,"Local timestr=[%s] size=%d", time_str, strlen(time_str)); + } + //debugf(DBG_LEVEL_NORM, 0,"Local time_t %li", mktime(&loc_time_tm)); + return mktime(&loc_time_tm); +} + +int get_time_as_string(time_t time_t_val, long nsec, char* time_str, + int time_str_len) +{ + struct tm time_val_tm; + time_t safe_input_time; + //if time is incorrect (too long) you get segfault, need to check length and trim + if (time_t_val > INT_MAX) + { + debugf(DBG_LEVEL_NORM, + KRED"get_time_as_string: input time length too long, %lu > max=%lu, trimming!", + time_t_val, INT_MAX); + safe_input_time = 0;//(int)time_t_val; + } + else + safe_input_time = time_t_val; + time_val_tm = *gmtime(&safe_input_time); + int str_len = strftime(time_str, time_str_len, HUBIC_DATE_FORMAT, + &time_val_tm); + char nsec_str[TIME_CHARS]; + sprintf(nsec_str, "%d", nsec); + strcat(time_str, nsec_str); + return str_len + strlen(nsec_str); +} + +time_t get_time_now() +{ + struct timespec now; + clock_gettime(CLOCK_REALTIME, &now); + return now.tv_sec; +} + +size_t get_time_now_as_str(char* time_str, int time_str_len) +{ + time_t now = time(0); + struct tm tstruct; + tstruct = *localtime(&now); + // Visit http://en.cppreference.com/w/cpp/chrono/c/strftime + // for more information about date/time format + size_t result = strftime(time_str, time_str_len, HUBIC_DATE_FORMAT, &tstruct); + return result; +} + +int get_timespec_as_str(const struct timespec* times, char* time_str, + int time_str_len) +{ + return get_time_as_string(times->tv_sec, times->tv_nsec, time_str, + time_str_len); +} + +char* str2md5(const char* str, int length) +{ + int n; + MD5_CTX c; + unsigned char digest[16]; + char* out = (char*)malloc(33); + + MD5_Init(&c); + while (length > 0) + { + if (length > 512) + MD5_Update(&c, str, 512); + else + MD5_Update(&c, str, length); + length -= 512; + str += 512; + } + MD5_Final(digest, &c); + for (n = 0; n < 16; ++n) + snprintf(&(out[n * 2]), 16 * 2, "%02x", (unsigned int)digest[n]); + return out; +} + +// http://stackoverflow.com/questions/10324611/how-to-calculate-the-md5-hash-of-a-large-file-in-c +int file_md5(FILE* file_handle, char* md5_file_str) +{ + if (file_handle == NULL) + { + debugf(DBG_LEVEL_NORM, KRED"file_md5: NULL file handle"); + return 0; + } + if (file_buffer_size == 0) + { + debugf(DBG_LEVEL_NORM, KYEL"file_md5: 0 file buffer size, using default"); + file_buffer_size = 1024; + } + unsigned char c[MD5_DIGEST_LENGTH]; + int i; + MD5_CTX mdContext; + int bytes; + char mdchar[3];//2 chars for md5 + null string terminator + unsigned char* data_buf = malloc(file_buffer_size * sizeof(unsigned char)); + MD5_Init(&mdContext); + while ((bytes = fread(data_buf, 1, file_buffer_size, file_handle)) != 0) + MD5_Update(&mdContext, data_buf, bytes); + MD5_Final(c, &mdContext); + for (i = 0; i < MD5_DIGEST_LENGTH; i++) + { + snprintf(mdchar, 3, "%02x", c[i]); + strcat(md5_file_str, mdchar); + } + free(data_buf); + return 0; +} + +int get_safe_cache_file_path(const char* path, char* file_path_safe, + char* temp_dir) +{ + char tmp_path[PATH_MAX]; + strncpy(tmp_path, path, PATH_MAX); + char* pch; + while ((pch = strchr(tmp_path, '/'))) + * pch = '.'; + char file_path[PATH_MAX] = ""; + //temp file name had process pid in it, removed as on restart files are left in cache (pid changes) + snprintf(file_path, PATH_MAX, TEMP_FILE_NAME_FORMAT, temp_dir, tmp_path); + //fixme check if sizeof or strlen is suitable + int file_path_len = sizeof(file_path); + //the file path name using this format can go beyond NAME_MAX size and will generate error on fopen + //solution: cap file length to NAME_MAX, use a prefix from original path for debug purposes and add md5 id + char* md5_path = str2md5(file_path, file_path_len); + int md5len = strlen(md5_path); + size_t safe_len_prefix = min(NAME_MAX - md5len, file_path_len); + strncpy(file_path_safe, file_path, safe_len_prefix); + strncpy(file_path_safe + safe_len_prefix, md5_path, md5len); + //sometimes above copy process produces longer strings that NAME_MAX, force a null terminated string + file_path_safe[safe_len_prefix + md5len - 1] = '\0'; + free(md5_path); + return strlen(file_path_safe); +} + +void get_file_path_from_fd(int fd, char* path, int size_path) +{ + char proc_path[MAX_PATH_SIZE]; + /* Read out the link to our file descriptor. */ + sprintf(proc_path, "/proc/self/fd/%d", fd); + memset(path, 0, size_path); + readlink(proc_path, path, size_path - 1); +} + +//for file descriptor debugging +void debug_print_flags(int flags) +{ + int accmode, val; + accmode = flags & O_ACCMODE; + if (accmode == O_RDONLY) debugf(DBG_LEVEL_EXTALL, KYEL"read only"); + else if (accmode == O_WRONLY) debugf(DBG_LEVEL_EXTALL, KYEL"write only"); + else if (accmode == O_RDWR) debugf(DBG_LEVEL_EXTALL, KYEL"read write"); + else debugf(DBG_LEVEL_EXT, KYEL"unknown access mode"); + + if (val & O_APPEND) debugf(DBG_LEVEL_EXTALL, KYEL", append"); + if (val & O_NONBLOCK) debugf(DBG_LEVEL_EXTALL, KYEL", nonblocking"); +#if !defined(_POSIX_SOURCE) && defined(O_SYNC) + if (val & O_SYNC) debugf(DBG_LEVEL_EXT, 0, + KRED ", synchronous writes"); +#endif + +} + +//for file descriptor debugging +void debug_print_descriptor(struct fuse_file_info* info) +{ + char file_path[MAX_PATH_SIZE]; + get_file_path_from_fd(info->fh, file_path, sizeof(file_path)); + debugf(DBG_LEVEL_EXT, KCYN "descriptor localfile=[%s] fd=%d", file_path, + info->fh); + debug_print_flags(info->flags); +} + +void dir_for(const char* path, char* dir) +{ + strncpy(dir, path, MAX_PATH_SIZE); + char* slash = strrchr(dir, '/'); + if (slash) + *slash = '\0'; +} + +//prints cache content for debug purposes +void debug_list_cache_content() +{ + return;//disabled + dir_cache* cw; + dir_entry* de; + for (cw = dcache; cw; cw = cw->next) + { + debugf(DBG_LEVEL_EXT, "LIST-CACHE: DIR[%s]", cw->path); + for (de = cw->entries; de; de = de->next) + debugf(DBG_LEVEL_EXT, "LIST-CACHE: FOLDER[%s]", de->full_name); + } +} + +int delete_file(char* path) +{ + debugf(DBG_LEVEL_NORM, KYEL"delete_file(%s)", path); + char file_path_safe[NAME_MAX] = ""; + get_safe_cache_file_path(path, file_path_safe, temp_dir); + int result = unlink(file_path_safe); + debugf(DBG_LEVEL_EXT, KYEL"delete_file(%s) (%s) result=%s", path, + file_path_safe, strerror(result)); + return result; +} + +//adding a directory in cache +dir_cache* new_cache(const char* path) +{ + debugf(DBG_LEVEL_NORM, KCYN"new_cache(%s)", path); + dir_cache* cw = (dir_cache*)calloc(sizeof(dir_cache), 1); + cw->path = strdup(path); + cw->prev = NULL; + cw->entries = NULL; + cw->cached = time(NULL); + //added cache by access + cw->accessed_in_cache = time(NULL); + cw->was_deleted = false; + if (dcache) + dcache->prev = cw; + cw->next = dcache; + dir_cache* result; + result = (dcache = cw); + debugf(DBG_LEVEL_EXT, "exit: new_cache(%s)", path); + return result; +} + +//todo: check if the program behaves ok when free_dir +//is made on a folder that has an operation in progress +void cloudfs_free_dir_list(dir_entry* dir_list) +{ + //check for NULL as dir might be already removed from cache by other thread + debugf(DBG_LEVEL_NORM, "cloudfs_free_dir_list(%s)", dir_list->full_name); + while (dir_list) + { + dir_entry* de = dir_list; + dir_list = dir_list->next; + //remove file from disk cache, fix for issue #89, https://github.com/TurboGit/hubicfuse/issues/89 + delete_file(de->full_name); + free(de->name); + free(de->full_name); + free(de->content_type); + //TODO free all added fields + free(de->md5sum); + free(de); + } +} + +void dir_decache(const char* path) +{ + dir_cache* cw; + debugf(DBG_LEVEL_NORM, "dir_decache(%s)", path); + pthread_mutex_lock(&dcachemut); + dir_entry* de, *tmpde; + char dir[MAX_PATH_SIZE]; + dir_for(path, dir); + for (cw = dcache; cw; cw = cw->next) + { + debugf(DBG_LEVEL_EXT, "dir_decache: parse(%s)", cw->path); + if (!strcmp(cw->path, path)) + { + if (cw == dcache) + dcache = cw->next; + if (cw->prev) + cw->prev->next = cw->next; + if (cw->next) + cw->next->prev = cw->prev; + debugf(DBG_LEVEL_EXT, "dir_decache: free_dir1(%s)", cw->path); + //fixme: this sometimes is NULL and generates segfaults, checking first + if (cw->entries != NULL) + cloudfs_free_dir_list(cw->entries); + free(cw->path); + free(cw); + } + else if (cw->entries && !strcmp(dir, cw->path)) + { + if (!strcmp(cw->entries->full_name, path)) + { + de = cw->entries; + cw->entries = de->next; + de->next = NULL; + debugf(DBG_LEVEL_EXT, "dir_decache: free_dir2()"); + cloudfs_free_dir_list(de); + } + else for (de = cw->entries; de->next; de = de->next) + { + if (!strcmp(de->next->full_name, path)) + { + tmpde = de->next; + de->next = de->next->next; + tmpde->next = NULL; + debugf(DBG_LEVEL_EXT, "dir_decache: free_dir3()", cw->path); + cloudfs_free_dir_list(tmpde); + break; + } + } + } + } + pthread_mutex_unlock(&dcachemut); +} + +dir_entry* init_dir_entry() +{ + dir_entry* de = (dir_entry*)malloc(sizeof(dir_entry)); + de->metadata_downloaded = false; + de->size = 0; + de->next = NULL; + de->md5sum = NULL; + de->accessed_in_cache = time(NULL); + de->last_modified = time(NULL); + de->mtime.tv_sec = time(NULL); + de->atime.tv_sec = time(NULL); + de->ctime.tv_sec = time(NULL); + de->mtime.tv_nsec = 0; + de->atime.tv_nsec = 0; + de->ctime.tv_nsec = 0; + de->chmod = 0; + de->gid = 0; + de->uid = 0; + return de; +} + +void copy_dir_entry(dir_entry* src, dir_entry* dst) +{ + dst->atime.tv_sec = src->atime.tv_sec; + dst->atime.tv_nsec = src->atime.tv_nsec; + dst->mtime.tv_sec = src->mtime.tv_sec; + dst->mtime.tv_nsec = src->mtime.tv_nsec; + dst->ctime.tv_sec = src->ctime.tv_sec; + dst->ctime.tv_nsec = src->ctime.tv_nsec; + dst->chmod = src->chmod; + //todo: copy md5sum as well +} + +//check for file in cache, if found size will be updated, if not found +//and this is a dir, a new dir cache entry is created +void update_dir_cache(const char* path, off_t size, int isdir, int islink) +{ + debugf(DBG_LEVEL_EXTALL, KCYN "update_dir_cache(%s)", path); + pthread_mutex_lock(&dcachemut); + dir_cache* cw; + dir_entry* de; + char dir[MAX_PATH_SIZE]; + dir_for(path, dir); + for (cw = dcache; cw; cw = cw->next) + { + if (!strcmp(cw->path, dir)) + { + for (de = cw->entries; de; de = de->next) + { + if (!strcmp(de->full_name, path)) + { + de->size = size; + pthread_mutex_unlock(&dcachemut); + debugf(DBG_LEVEL_EXTALL, "exit 0: update_dir_cache(%s)", path); + return; + } + } + de = init_dir_entry(); + de->size = size; + de->isdir = isdir; + de->islink = islink; + de->name = strdup(&path[strlen(cw->path) + 1]); + de->full_name = strdup(path); + //fixed: the conditions below were mixed up dir -> link? + if (islink) + de->content_type = strdup("application/link"); + if (isdir) + de->content_type = strdup("application/directory"); + else + de->content_type = strdup("application/octet-stream"); + de->next = cw->entries; + cw->entries = de; + if (isdir) + new_cache(path); + break; + } + } + debugf(DBG_LEVEL_EXTALL, "exit 1: update_dir_cache(%s)", path); + pthread_mutex_unlock(&dcachemut); +} + +//returns first file entry in linked list. if not in cache will be downloaded. +int caching_list_directory(const char* path, dir_entry** list) +{ + debugf(DBG_LEVEL_EXT, "caching_list_directory(%s)", path); + pthread_mutex_lock(&dcachemut); + bool new_entry = false; + if (!strcmp(path, "/")) + path = ""; + dir_cache* cw; + for (cw = dcache; cw; cw = cw->next) + { + if (cw->was_deleted == true) + { + debugf(DBG_LEVEL_EXT, + KMAG"caching_list_directory status: dir(%s) is empty as cached expired, reload from cloud", + cw->path); + if (!cloudfs_list_directory(cw->path, list)) + debugf(DBG_LEVEL_EXT, + KMAG"caching_list_directory status: cannot reload dir(%s)", cw->path); + else + { + debugf(DBG_LEVEL_EXT, KMAG"caching_list_directory status: reloaded dir(%s)", + cw->path); + //cw->entries = *list; + cw->was_deleted = false; + cw->cached = time(NULL); + } + } + if (cw->was_deleted == false) + { + if (!strcmp(cw->path, path)) + break; + } + } + if (!cw) + { + //trying to download this entry from cloud, list will point to cached or downloaded entries + if (!cloudfs_list_directory(path, list)) + { + //download was not ok + pthread_mutex_unlock(&dcachemut); + debugf(DBG_LEVEL_EXT, + "exit 0: caching_list_directory(%s) "KYEL"[CACHE-DIR-MISS]", path); + return 0; + } + debugf(DBG_LEVEL_EXT, + "caching_list_directory: new_cache(%s) "KYEL"[CACHE-CREATE]", path); + cw = new_cache(path); + new_entry = true; + } + else if (cache_timeout > 0 && (time(NULL) - cw->cached > cache_timeout)) + { + if (!cloudfs_list_directory(path, list)) + { + //mutex unlock was forgotten + pthread_mutex_unlock(&dcachemut); + debugf(DBG_LEVEL_EXT, "exit 1: caching_list_directory(%s)", path); + return 0; + } + //fixme: this frees dir subentries but leaves the dir parent entry, this confuses path_info + //which believes this dir has no entries + if (cw->entries != NULL) + { + cloudfs_free_dir_list(cw->entries); + cw->was_deleted = true; + cw->cached = time(NULL); + debugf(DBG_LEVEL_EXT, "caching_list_directory(%s) "KYEL"[CACHE-EXPIRED]", + path); + } + else + { + debugf(DBG_LEVEL_EXT, + "got NULL on caching_list_directory(%s) "KYEL"[CACHE-EXPIRED w NULL]", path); + pthread_mutex_unlock(&dcachemut); + return 0; + } + } + else + { + debugf(DBG_LEVEL_EXT, "caching_list_directory(%s) "KGRN"[CACHE-DIR-HIT]", + path); + *list = cw->entries; + } + //adding new dir file list to global cache, now this dir becomes visible in cache + cw->entries = *list; + pthread_mutex_unlock(&dcachemut); + debugf(DBG_LEVEL_EXT, "exit 2: caching_list_directory(%s)", path); + return 1; +} + +dir_entry* path_info(const char* path) +{ + debugf(DBG_LEVEL_EXT, "path_info(%s)", path); + char dir[MAX_PATH_SIZE]; + dir_for(path, dir); + dir_entry* tmp; + if (!caching_list_directory(dir, &tmp)) + { + debugf(DBG_LEVEL_EXT, "exit 0: path_info(%s) "KYEL"[CACHE-DIR-MISS]", dir); + return NULL; + } + else + debugf(DBG_LEVEL_EXT, "path_info(%s) "KGRN"[CACHE-DIR-HIT]", dir); + //iterate in file list obtained from cache or downloaded + for (; tmp; tmp = tmp->next) + { + if (!strcmp(tmp->full_name, path)) + { + debugf(DBG_LEVEL_EXT, "exit 1: path_info(%s) "KGRN"[CACHE-FILE-HIT]", path); + return tmp; + } + } + //miss in case the file is not found on a cached folder + debugf(DBG_LEVEL_EXT, "exit 2: path_info(%s) "KYEL"[CACHE-MISS]", path); + return NULL; +} + + +//retrieve folder from local cache if exists, return null if does not exist (rather than download) +int check_caching_list_directory(const char* path, dir_entry** list) +{ + debugf(DBG_LEVEL_EXT, "check_caching_list_directory(%s)", path); + pthread_mutex_lock(&dcachemut); + if (!strcmp(path, "/")) + path = ""; + dir_cache* cw; + for (cw = dcache; cw; cw = cw->next) + if (!strcmp(cw->path, path)) + { + *list = cw->entries; + pthread_mutex_unlock(&dcachemut); + debugf(DBG_LEVEL_EXT, + "exit 0: check_caching_list_directory(%s) "KGRN"[CACHE-DIR-HIT]", path); + return 1; + } + pthread_mutex_unlock(&dcachemut); + debugf(DBG_LEVEL_EXT, + "exit 1: check_caching_list_directory(%s) "KYEL"[CACHE-DIR-MISS]", path); + return 0; +} + +dir_entry* check_parent_folder_for_file(const char* path) +{ + char dir[MAX_PATH_SIZE]; + dir_for(path, dir); + dir_entry* tmp; + if (!check_caching_list_directory(dir, &tmp)) + return NULL; + else + return tmp; +} + +//check if local path is in cache, without downloading from cloud if not in cache +dir_entry* check_path_info(const char* path) +{ + debugf(DBG_LEVEL_EXT, "check_path_info(%s)", path); + char dir[MAX_PATH_SIZE]; + dir_for(path, dir); + dir_entry* tmp; + + //get parent folder cache entry + if (!check_caching_list_directory(dir, &tmp)) + { + debugf(DBG_LEVEL_EXT, "exit 0: check_path_info(%s) "KYEL"[CACHE-MISS]", path); + return NULL; + } + for (; tmp; tmp = tmp->next) + { + if (!strcmp(tmp->full_name, path)) + { + debugf(DBG_LEVEL_EXT, "exit 1: check_path_info(%s) "KGRN"[CACHE-HIT]", path); + return tmp; + } + } + if (!strcmp(path, "/")) + debugf(DBG_LEVEL_EXT, + "exit 2: check_path_info(%s) "KYEL"ignoring root [CACHE-MISS]", path); + else + debugf(DBG_LEVEL_EXT, "exit 3: check_path_info(%s) "KYEL"[CACHE-MISS]", path); + return NULL; +} + + +char* get_home_dir() +{ + char* home; + if ((home = getenv("HOME")) && !access(home, R_OK)) + return home; + struct passwd* pwd = getpwuid(geteuid()); + if ((home = pwd->pw_dir) && !access(home, R_OK)) + return home; + return "~"; +} + +void cloudfs_debug(int dbg) +{ + debug = dbg; +} + +void debugf(int level, char* fmt, ...) +{ + if (debug) + { + if (level <= option_debug_level) + { +#ifdef SYS_gettid + pid_t thread_id = syscall(SYS_gettid); +#else + int thread_id = 0; +#error "SYS_gettid unavailable on this system" +#endif + va_list args; + char prefix[] = "==DBG %d [%s]:%d=="; + char line[4096]; + char time_str[TIME_CHARS]; + get_time_now_as_str(time_str, sizeof(time_str)); + sprintf(line, prefix, level, time_str, thread_id); + fputs(line, stderr); + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); + fputs(KNRM, stderr); + putc('\n', stderr); + putc('\r', stderr); + } + } +} diff --git a/commonfs.h b/commonfs.h new file mode 100644 index 0000000..2a3ee3c --- /dev/null +++ b/commonfs.h @@ -0,0 +1,113 @@ +#ifndef _COMMONFS_H +#define _COMMONFS_H +#include + +typedef enum { false, true } bool; +#define MAX_PATH_SIZE (1024 + 256 + 3) +#define THREAD_NAMELEN 16 +// 64 bit time + nanoseconds +#define TIME_CHARS 32 +#define DBG_LEVEL_NORM 0 +#define DBG_LEVEL_EXT 1 +#define DBG_LEVEL_EXTALL 2 +#define INT_CHAR_LEN 16 + +// utimens support +#define HEADER_TEXT_MTIME "X-Object-Meta-Mtime" +#define HEADER_TEXT_ATIME "X-Object-Meta-Atime" +#define HEADER_TEXT_CTIME "X-Object-Meta-Ctime" +#define HEADER_TEXT_MTIME_DISPLAY "X-Object-Meta-Mtime-Display" +#define HEADER_TEXT_ATIME_DISPLAY "X-Object-Meta-Atime-Display" +#define HEADER_TEXT_CTIME_DISPLAY "X-Object-Meta-Ctime-Display" +#define HEADER_TEXT_CHMOD "X-Object-Meta-Chmod" +#define HEADER_TEXT_UID "X-Object-Meta-Uid" +#define HEADER_TEXT_GID "X-Object-Meta-Gid" +#define HEADER_TEXT_FILEPATH "X-Object-Meta-FilePath" +#define TEMP_FILE_NAME_FORMAT "%s/.cloudfuse_%s" +#define HUBIC_DATE_FORMAT "%Y-%m-%d %T." + +#define KNRM "\x1B[0m" +#define KRED "\x1B[31m" +#define KGRN "\x1B[32m" +#define KYEL "\x1B[33m" +#define KBLU "\x1B[34m" +#define KMAG "\x1B[35m" +#define KCYN "\x1B[36m" +#define KWHT "\x1B[37m" + +#define min(x, y) ({ \ + typeof(x) _min1 = (x); \ + typeof(y) _min2 = (y); \ + (void)(&_min1 == &_min2); \ + _min1 < _min2 ? _min1 : _min2; }) + +//linked list with files in a directory +typedef struct dir_entry +{ + char* name; + char* full_name; + char* content_type; + off_t size; + time_t last_modified; + // implement utimens + struct timespec mtime; + struct timespec ctime; + struct timespec atime; + char* md5sum; //interesting capability for rsync/scrub + mode_t chmod; + uid_t uid; + gid_t gid; + bool issegmented; + time_t accessed_in_cache;//todo: cache support based on access time + bool metadata_downloaded; + // end change + int isdir; + int islink; + struct dir_entry* next; +} dir_entry; + +// linked list with cached folder names +typedef struct dir_cache +{ + char* path; + dir_entry* entries; + time_t cached; + //added cache support based on access time + time_t accessed_in_cache; + bool was_deleted; + //end change + struct dir_cache* next, *prev; +} dir_cache; + +time_t my_timegm(struct tm* tm); +time_t get_time_from_str_as_gmt(char* time_str); +time_t get_time_as_local(time_t time_t_val, char time_str[], + int char_buf_size); +int get_time_as_string(time_t time_t_val, long nsec, char* time_str, + int time_str_len); +time_t get_time_now(); +int get_timespec_as_str(const struct timespec* times, char* time_str, + int time_str_len); +char* str2md5(const char* str, int length); +int file_md5(FILE* file_handle, char* md5_file_str); +void debug_print_descriptor(struct fuse_file_info* info); +int get_safe_cache_file_path(const char* file_path, char* file_path_safe, + char* temp_dir); +dir_entry* init_dir_entry(); +void copy_dir_entry(dir_entry* src, dir_entry* dst); +dir_cache* new_cache(const char* path); +void dir_for(const char* path, char* dir); +void debug_list_cache_content(); +void update_dir_cache(const char* path, off_t size, int isdir, int islink); +dir_entry* path_info(const char* path); +dir_entry* check_path_info(const char* path); +dir_entry* check_parent_folder_for_file(const char* path); +void dir_decache(const char* path); +void cloudfs_free_dir_list(dir_entry* dir_list); +extern int cloudfs_list_directory(const char* path, dir_entry**); +int caching_list_directory(const char* path, dir_entry** list); +char* get_home_dir(); +void cloudfs_debug(int dbg); +void debugf(int level, char* fmt, ...); + +#endif diff --git a/configure b/configure index 4b32952..0baa73d 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for hubicfuse 1.0.1. +# Generated by GNU Autoconf 2.69 for hubicfuse 3.0.0. # # Report bugs to >. # @@ -580,8 +580,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='hubicfuse' PACKAGE_TARNAME='hubicfuse' -PACKAGE_VERSION='1.0.1' -PACKAGE_STRING='hubicfuse 1.0.1' +PACKAGE_VERSION='3.0.0' +PACKAGE_STRING='hubicfuse 3.0.0' PACKAGE_BUGREPORT='Pascal Obry ' PACKAGE_URL='' @@ -673,6 +673,7 @@ infodir docdir oldincludedir includedir +runstatedir localstatedir sharedstatedir sysconfdir @@ -756,6 +757,7 @@ datadir='${datarootdir}' sysconfdir='${prefix}/etc' sharedstatedir='${prefix}/com' localstatedir='${prefix}/var' +runstatedir='${localstatedir}/run' includedir='${prefix}/include' oldincludedir='/usr/include' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' @@ -1008,6 +1010,15 @@ do | -silent | --silent | --silen | --sile | --sil) silent=yes ;; + -runstatedir | --runstatedir | --runstatedi | --runstated \ + | --runstate | --runstat | --runsta | --runst | --runs \ + | --run | --ru | --r) + ac_prev=runstatedir ;; + -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \ + | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \ + | --run=* | --ru=* | --r=*) + runstatedir=$ac_optarg ;; + -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) ac_prev=sbindir ;; -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ @@ -1145,7 +1156,7 @@ fi for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ datadir sysconfdir sharedstatedir localstatedir includedir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ - libdir localedir mandir + libdir localedir mandir runstatedir do eval ac_val=\$$ac_var # Remove trailing slashes. @@ -1258,7 +1269,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures hubicfuse 1.0.1 to adapt to many kinds of systems. +\`configure' configures hubicfuse 3.0.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1298,6 +1309,7 @@ Fine tuning of the installation directories: --sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --localstatedir=DIR modifiable single-machine data [PREFIX/var] + --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] --libdir=DIR object code libraries [EPREFIX/lib] --includedir=DIR C header files [PREFIX/include] --oldincludedir=DIR C header files for non-gcc [/usr/include] @@ -1319,7 +1331,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of hubicfuse 1.0.1:";; + short | recursive ) echo "Configuration of hubicfuse 3.0.0:";; esac cat <<\_ACEOF @@ -1414,7 +1426,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -hubicfuse configure 1.0.1 +hubicfuse configure 3.0.0 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1894,7 +1906,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by hubicfuse $as_me 1.0.1, which was +It was created by hubicfuse $as_me 3.0.0, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -3609,8 +3621,8 @@ fi # Put the nasty error message in config.log where it belongs echo "$JSON_PKG_ERRORS" >&5 - #JSON_CFLAGS C compiler flags for JSON, overriding pkg-config - #JSON_LIBS linker flags for JSON, overriding pkg-config + JSON_CFLAGS C compiler flags for JSON, overriding pkg-config + JSON_LIBS linker flags for JSON, overriding pkg-config pkg_failed=no { $as_echo "$as_me:${as_lineno-$LINENO}: checking for JSON" >&5 @@ -5733,7 +5745,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by hubicfuse $as_me 1.0.1, which was +This file was extended by hubicfuse $as_me 3.0.0, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -5795,7 +5807,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -hubicfuse config.status 1.0.1 +hubicfuse config.status 3.0.0 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index e1f66aa..8aa1fa1 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([hubicfuse], [2.1.0], [Pascal Obry ]) +AC_INIT([hubicfuse], [3.0.0], [Pascal Obry ]) AC_SUBST(CPPFLAGS, "$CPPFLAGS -D_FILE_OFFSET_BITS=64 -I/usr/include/libxml2") AC_LANG([C]) AC_CONFIG_SRCDIR([cloudfuse.c])