From 056bb5e319227c0f705bf7e0308a663b3bba6249 Mon Sep 17 00:00:00 2001 From: dcristian Date: Thu, 26 Nov 2015 22:34:15 +0200 Subject: [PATCH 01/12] Support for atime, mtime, chmod, chown. * Large files (segmented) have correct size listed (was 0 before). * Multiple speed improvements, minimised the number of HTTP calls and added more caching features. * Fixed many segmentation faults. * Cached files are deleted on cache expiration when using a custom temp folder. * Files copied have attributes preserved. * Working well with rsync due to mtime support and proper copy operations. * Support for http progress (track upload / download speed etc.) * Major code refactoring, code documentation, extensive debugging, additional config options --- .gitignore | 2 + .hubicfuse.sample | 13 + Makefile.in | 7 +- README | 18 + cloudfsapi.c | 841 +++++++++++++++++++++++++++++++++++----------- cloudfsapi.h | 39 ++- cloudfuse.c | 790 +++++++++++++++++++++++++------------------ commonfs.c | 608 +++++++++++++++++++++++++++++++++ commonfs.h | 112 ++++++ 9 files changed, 1901 insertions(+), 529 deletions(-) create mode 100644 .hubicfuse.sample create mode 100644 commonfs.c create mode 100644 commonfs.h diff --git a/.gitignore b/.gitignore index 8186834..474bb95 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ config.status # Binary outputs cloudfuse hubicfuse + +project-vc/ \ No newline at end of file diff --git a/.hubicfuse.sample b/.hubicfuse.sample new file mode 100644 index 0000000..732cf91 --- /dev/null +++ b/.hubicfuse.sample @@ -0,0 +1,13 @@ +client_id = api_hubic_xxxxx +client_secret = xxxxxx +refresh_token = xxxx +cache_timeout = 600 +temp_dir = /tmp/hubicfuse + +get_extended_metadata = true +curl_verbose = false +curl_progress_state = 1 +cache_statfs_timeout = 15 +debug_level = 0 +enable_chmod = false +enable_chown = false \ No newline at end of file diff --git a/Makefile.in b/Makefile.in index 29c7354..450d95b 100644 --- a/Makefile.in +++ b/Makefile.in @@ -11,8 +11,8 @@ prefix = @prefix@ exec_prefix = @exec_prefix@ bindir = $(DESTDIR)$(exec_prefix)/bin -SOURCES=cloudfsapi.c cloudfuse.c -HEADERS=cloudfsapi.h +SOURCES=cloudfsapi.c cloudfuse.c commonfs.c +HEADERS=cloudfsapi.h commonfs.h all: hubicfuse @@ -39,6 +39,9 @@ mostlyclean: clean maintainer-clean: clean +debug: CFLAGS += -g +debug: hubicfuse + config.h.in: stamp-h.in stamp-h.in: configure.in autoheader diff --git a/README b/README index 083efc8..92f7d76 100644 --- a/README +++ b/README @@ -39,6 +39,12 @@ USE: client_secret=[Hubic client secret for the registered application] refresh_token=[A refresh token you got from the script] + Optional variables: + get_extended_metadata=[true/false, force download of additional file metadata like atime and mtime on first directory list] + curl_verbose=[true/false, enable verbose output for curl HTTP requests] + cache_statfs_timeout=[value in seconds, large timeout increases the file access speed] + debug_level=[0 default, 1 extremely verbose for debugging purposes] + client_id, client_secret can be retrieved from the hubic web interface: https://hubic.com/home/browser/developers/ @@ -83,7 +89,19 @@ BUGS/SHORTCOMINGS: "application/directory". * Cloud Files limits container and object listings to 10,000 items. cloudfuse won't list more than that many files in a single directory. + * File copy progress when uploading does not work, progress is shown when file + is copied in local cache, then upload operation happens at 100% +Recent additions/fixes (from https://github.com/dan-cristian): + * Support for atime, mtime, chmod, chown. + * Large files (segmented) have correct size listed (was 0 before). + * Multiple speed improvements, minimised the number of HTTP calls and added more caching features. + * Fixed many segmentation faults. + * Cached files are deleted on cache expiration when using a custom temp folder. + * Files copied have attributes preserved. + * Working well with rsync due to mtime support and proper copy operations. + * Support for http progress (track upload / download speed etc.) + * Major code refactoring, code documentation, extensive debugging, additional config options AWESOME CONTRIBUTORS: diff --git a/cloudfsapi.c b/cloudfsapi.c index 18be34e..fc5d42b 100644 --- a/cloudfsapi.c +++ b/cloudfsapi.c @@ -20,21 +20,18 @@ #include #include #include +#include "commonfs.h" #include "cloudfsapi.h" #include "config.h" #include - #define RHEL5_LIBCURL_VERSION 462597 #define RHEL5_CERTIFICATE_FILE "/etc/pki/tls/certs/ca-bundle.crt" -#define REQUEST_RETRIES 4 +#define REQUEST_RETRIES 3 #define MAX_FILES 10000 -// 64 bit time + nanoseconds -#define TIME_CHARS 32 - // size of buffer for writing to disk look at ioblksize.h in coreutils // and try some values on your own system if you want the best performance #define DISK_BUFF_SIZE 32768 @@ -44,8 +41,16 @@ static char storage_token[MAX_HEADER_SIZE]; static pthread_mutex_t pool_mut; static CURL *curl_pool[1024]; static int curl_pool_count = 0; -static int debug = 0; -static int verify_ssl = 2; +extern int debug; +extern int verify_ssl; +extern bool option_get_extended_metadata; +extern bool option_curl_verbose; +extern int option_curl_progress_state; +extern int option_cache_statfs_timeout; +extern bool option_extensive_debug; +extern bool option_enable_chown; +extern bool option_enable_chmod; + static int rhel5_mode = 0; static struct statvfs statcache = { .f_bsize = 4096, @@ -58,9 +63,14 @@ static struct statvfs statcache = { .f_favail = 0, .f_namemax = INT_MAX }; - +static time_t last_stat_read_time = 0;//used to compute cache interval extern FuseOptions options; +struct MemoryStruct { + char *memory; + size_t size; +}; + #ifdef HAVE_OPENSSL #include static pthread_mutex_t *ssl_lockarray; @@ -78,6 +88,7 @@ static unsigned long thread_id() } #endif + static size_t xml_dispatch(void *ptr, size_t size, size_t nmemb, void *stream) { xmlParseChunk((xmlParserCtxtPtr)stream, (char *)ptr, size * nmemb, 0); @@ -90,7 +101,7 @@ static CURL *get_connection(const char *path) CURL *curl = curl_pool_count ? curl_pool[--curl_pool_count] : curl_easy_init(); if (!curl) { - debugf("curl alloc failed"); + debugf(DBG_LEVEL_NORM, KRED"curl alloc failed"); abort(); } pthread_mutex_unlock(&pool_mut); @@ -108,13 +119,33 @@ static void add_header(curl_slist **headers, const char *name, const char *value) { char x_header[MAX_HEADER_SIZE]; - snprintf(x_header, sizeof(x_header), "%s: %s", name, value); + char safe_value[256]; + const char *value_ptr; + + debugf(DBG_LEVEL_EXT, "add_header(%s:%s)", name, value); + if (strlen(value) > 256) { + debugf(DBG_LEVEL_NORM, KRED"add_header: warning, value size > 256 (%s:%s) ", name, value); + //hubic will throw an HTTP 400 error on X-Copy-To operation if X-Object-Meta-FilePath header value is larger than 256 chars + //fix for issue #95 https://github.com/TurboGit/hubicfuse/issues/95 + if (!strcasecmp(name, "X-Object-Meta-FilePath")) { + debugf(DBG_LEVEL_NORM, KRED"add_header: trimming header (%s) value to max allowed", name); + //trim header size to max allowed + strncpy(safe_value, value, 256 - 1); + safe_value[255] = '\0'; + value_ptr = safe_value; + } + else + value_ptr = value; + } + else + value_ptr = value; + + snprintf(x_header, sizeof(x_header), "%s: %s", name, value_ptr); *headers = curl_slist_append(*headers, x_header); } -static size_t header_dispatch(void *ptr, size_t size, size_t nmemb, void *stream) +static size_t header_dispatch(void *ptr, size_t size, size_t nmemb, void *dir_entry) { - debugf("Dispatching response headers"); char *header = (char *)alloca(size * nmemb + 1); char *head = (char *)alloca(size * nmemb + 1); char *value = (char *)alloca(size * nmemb + 1); @@ -139,6 +170,64 @@ static size_t header_dispatch(void *ptr, size_t size, size_t nmemb, void *stream return size * nmemb; } +static void header_set_time_from_str(char *time_str, struct timespec *time_entry){ + char sec_value[TIME_CHARS]; + char nsec_value[TIME_CHARS]; + time_t sec; + long nsec; + sscanf(time_str, "%[^.].%[^\n]", sec_value, nsec_value); + sec = strtoll(sec_value, NULL, 10);//to allow for larger numbers + nsec = atol(nsec_value); + debugf(DBG_LEVEL_EXT, "Received time=%s.%s / %li.%li, existing=%li.%li", + sec_value, nsec_value, sec, nsec, time_entry->tv_sec, time_entry->tv_nsec); + if (sec != time_entry->tv_sec || nsec != time_entry->tv_nsec){ + debugf(DBG_LEVEL_EXT, "Time changed, setting new time=%li.%li, existing was=%li.%li", + sec, nsec, time_entry->tv_sec, time_entry->tv_nsec); + time_entry->tv_sec = sec; + time_entry->tv_nsec = nsec; + + char time_str_local[TIME_CHARS] = ""; + get_time_as_string((time_t)sec, nsec, time_str_local, sizeof(time_str_local)); + debugf(DBG_LEVEL_EXT, "header_set_time_from_str received time=[%s]", time_str_local); + + get_timespec_as_str(time_entry, time_str_local, sizeof(time_str_local)); + debugf(DBG_LEVEL_EXT, "header_set_time_from_str set time=[%s]", time_str_local); + } +} + +static size_t header_get_utimens_dispatch(void *ptr, size_t size, size_t nmemb, void *userdata) +{ + char *header = (char *)alloca(size * nmemb + 1); + char *head = (char *)alloca(size * nmemb + 1); + char *value = (char *)alloca(size * nmemb + 1); + memcpy(header, (char *)ptr, size * nmemb); + header[size * nmemb] = '\0'; + static char storage[MAX_HEADER_SIZE]; + if (sscanf(header, "%[^:]: %[^\r\n]", head, value) == 2) + { + strncpy(storage, head, sizeof(storage)); + dir_entry *de = (dir_entry*)userdata; + if (de != NULL) { + if (!strncasecmp(head, HEADER_TEXT_ATIME, size * nmemb)) { + header_set_time_from_str(value, &de->atime); + } + if (!strncasecmp(head, HEADER_TEXT_CTIME, size * nmemb)) { + header_set_time_from_str(value, &de->ctime); + } + if (!strncasecmp(head, HEADER_TEXT_MTIME, size * nmemb)) { + header_set_time_from_str(value, &de->mtime); + } + } + else { + debugf(DBG_LEVEL_EXT, "Unexpected NULL dir_entry on header(%s), file should be in cache already", storage); + } + } + else { + //debugf(DBG_LEVEL_NORM, "Received unexpected header line"); + } + return size * nmemb; +} + static size_t rw_callback(size_t (*rw)(void*, size_t, size_t, FILE*), void *ptr, size_t size, size_t nmemb, void *userp) { @@ -169,18 +258,87 @@ static size_t write_callback(void *ptr, size_t size, size_t nmemb, void *userp) return rw_callback(fwrite2, ptr, size, nmemb, userp); } +//http://curl.haxx.se/libcurl/c/CURLOPT_XFERINFOFUNCTION.html +int progress_callback_xfer(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow){ + struct curl_progress *myp = (struct curl_progress *)clientp; + CURL *curl = myp->curl; + double curtime = 0; + double dspeed = 0, uspeed=0; + + curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &curtime); + curl_easy_getinfo(curl, CURLINFO_SPEED_DOWNLOAD, &dspeed); + curl_easy_getinfo(curl, CURLINFO_SPEED_UPLOAD, &uspeed); + + /* under certain circumstances it may be desirable for certain functionality + to only run every N seconds, in order to do this the transaction time can + be used */ + //http://curl.haxx.se/cvssource/src/tool_cb_prg.c + if ((curtime - myp->lastruntime) >= MINIMAL_PROGRESS_FUNCTIONALITY_INTERVAL) { + myp->lastruntime = curtime; + curl_off_t total; + curl_off_t point; + double frac, percent; + total = dltotal + ultotal; + point = dlnow + ulnow; + frac = (double)point / (double)total; + percent = frac * 100.0f; + debugf(DBG_LEVEL_EXT, "TOTAL TIME: %.0f sec Down=%.0f Kbps UP=%.0f Kbps", curtime, dspeed/1024, uspeed/1024); + debugf(DBG_LEVEL_EXT, "UP: %lld of %lld DOWN: %lld/%lld Completion %.1f %%", ulnow, ultotal, dlnow, dltotal, percent); + } + return 0; +} + +//http://curl.haxx.se/libcurl/c/CURLOPT_PROGRESSFUNCTION.html +int progress_callback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow){ + return progress_callback_xfer(clientp, (curl_off_t)dltotal, (curl_off_t)dlnow, (curl_off_t)ultotal, (curl_off_t)ulnow); +} + + +//get the response from HTTP requests, mostly for debug purposes +// http://stackoverflow.com/questions/2329571/c-libcurl-get-output-into-a-string +// http://curl.haxx.se/libcurl/c/getinmemory.html +size_t writefunc_callback(void *contents, size_t size, size_t nmemb, void *userp) +{ + size_t realsize = size * nmemb; + struct MemoryStruct *mem = (struct MemoryStruct *)userp; + + mem->memory = realloc(mem->memory, mem->size + realsize + 1); + if (mem->memory == NULL) { + /* out of memory! */ + debugf(DBG_LEVEL_NORM, KRED"writefunc_callback: realloc() failed"); + return 0; + } + + memcpy(&(mem->memory[mem->size]), contents, realsize); + mem->size += realsize; + mem->memory[mem->size] = 0; + + return realsize; +} + +// de_cached_entry must be NULL when the file is already in global cache +// otherwise point to a new dir_entry that will be added to the cache (usually happens on first dir load) static int send_request_size(const char *method, const char *path, void *fp, xmlParserCtxtPtr xmlctx, curl_slist *extra_headers, - off_t file_size, int is_segment) + off_t file_size, int is_segment, + dir_entry *de_cached_entry) { + debugf(DBG_LEVEL_EXT, "send_request_size(%s) (%s)", method, path); char url[MAX_URL_SIZE]; + char orig_path[MAX_URL_SIZE]; + char header_data[MAX_HEADER_SIZE]; + char *slash; long response = -1; int tries = 0; + + //needed to keep the response data, for debug purposes + struct MemoryStruct chunk; + if (!storage_url[0]) { - debugf("send_request with no storage_url?"); + debugf(DBG_LEVEL_NORM, KRED"send_request with no storage_url?"); abort(); } @@ -192,10 +350,13 @@ static int send_request_size(const char *method, const char *path, void *fp, while (*path == '/') path++; snprintf(url, sizeof(url), "%s/%s", storage_url, path); + snprintf(orig_path, sizeof(orig_path), "/%s", path); // retry on failures for (tries = 0; tries < REQUEST_RETRIES; tries++) { + chunk.memory = malloc(1); /* will be grown as needed by the realloc above */ + chunk.size = 0; /* no data at this point */ CURL *curl = get_connection(path); if (rhel5_mode) curl_easy_setopt(curl, CURLOPT_CAINFO, RHEL5_CERTIFICATE_FILE); @@ -203,13 +364,67 @@ static int send_request_size(const char *method, const char *path, void *fp, curl_easy_setopt(curl, CURLOPT_URL, url); curl_easy_setopt(curl, CURLOPT_HEADER, 0); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); - curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1); + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, option_curl_progress_state);//0=to enable progress curl_easy_setopt(curl, CURLOPT_USERAGENT, USER_AGENT); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, verify_ssl ? 1 : 0); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, verify_ssl); curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 10); - curl_easy_setopt(curl, CURLOPT_VERBOSE, debug); + curl_easy_setopt(curl, CURLOPT_VERBOSE, option_curl_verbose ? 1 : 0); add_header(&headers, "X-Auth-Token", storage_token); + dir_entry *de; + if (de_cached_entry == NULL) { + de = check_path_info(orig_path); + } + else { + // updating metadata on a file to be added to cache + de = de_cached_entry; + debugf(DBG_LEVEL_EXTALL, "send_request_size: using param dir_entry(%s)", orig_path); + } + if (!de) { + debugf(DBG_LEVEL_EXTALL, "send_request_size: file not in cache (%s)", orig_path); + } + else { + // add headers to save utimens attribs only on upload + if (!strcasecmp(method, "PUT") || !strcasecmp(method, "MKDIR")) { + debugf(DBG_LEVEL_EXT, "send_request_size: Saving utimens for file %s", orig_path); + //debugf(DBG_LEVEL_NORM, "File found in cache, path=%s", de->full_name); + debugf(DBG_LEVEL_EXT, "send_request_size: Cached utime for path=%s ctime=%li.%li mtime=%li.%li atime=%li.%li", orig_path, + de->ctime.tv_sec, de->ctime.tv_nsec, de->mtime.tv_sec, de->mtime.tv_nsec, de->atime.tv_sec, de->atime.tv_nsec); + + char atime_str_nice[TIME_CHARS] = "", mtime_str_nice[TIME_CHARS] = "", ctime_str_nice[TIME_CHARS] = ""; + get_timespec_as_str(&(de->atime), atime_str_nice, sizeof(atime_str_nice)); + debugf(DBG_LEVEL_EXT, KCYN"send_request_size: atime=[%s]", atime_str_nice); + get_timespec_as_str(&(de->mtime), mtime_str_nice, sizeof(mtime_str_nice)); + debugf(DBG_LEVEL_EXT, KCYN"send_request_size: mtime=[%s]", mtime_str_nice); + get_timespec_as_str(&(de->ctime), ctime_str_nice, sizeof(ctime_str_nice)); + debugf(DBG_LEVEL_EXT, KCYN"send_request_size: ctime=[%s]", ctime_str_nice); + + char mtime_str[TIME_CHARS], atime_str[TIME_CHARS], ctime_str[TIME_CHARS]; + char string_float[TIME_CHARS]; + snprintf(mtime_str, TIME_CHARS, "%lu.%lu", de->mtime.tv_sec, de->mtime.tv_nsec); + snprintf(atime_str, TIME_CHARS, "%lu.%lu", de->atime.tv_sec, de->atime.tv_nsec); + snprintf(ctime_str, TIME_CHARS, "%lu.%lu", de->ctime.tv_sec, de->ctime.tv_nsec); + add_header(&headers, HEADER_TEXT_FILEPATH, orig_path); + add_header(&headers, HEADER_TEXT_MTIME, mtime_str); + add_header(&headers, HEADER_TEXT_ATIME, atime_str); + add_header(&headers, HEADER_TEXT_CTIME, ctime_str); + add_header(&headers, HEADER_TEXT_MTIME_DISPLAY, mtime_str_nice); + add_header(&headers, HEADER_TEXT_ATIME_DISPLAY, atime_str_nice); + add_header(&headers, HEADER_TEXT_CTIME_DISPLAY, ctime_str_nice); + + char gid_str[INT_CHAR_LEN], uid_str[INT_CHAR_LEN], chmod_str[INT_CHAR_LEN]; + snprintf(gid_str, INT_CHAR_LEN, "%d", de->gid); + snprintf(uid_str, INT_CHAR_LEN, "%d", de->uid); + snprintf(chmod_str, INT_CHAR_LEN, "%d", de->chmod); + add_header(&headers, HEADER_TEXT_GID, gid_str); + add_header(&headers, HEADER_TEXT_UID, uid_str); + add_header(&headers, HEADER_TEXT_CHMOD, chmod_str); + } + else { + debugf(DBG_LEVEL_EXTALL, "send_request_size: not setting utimes (%s)", orig_path); + } + } + /**/ if (!strcasecmp(method, "MKDIR")) { curl_easy_setopt(curl, CURLOPT_UPLOAD, 1); @@ -225,40 +440,84 @@ static int send_request_size(const char *method, const char *path, void *fp, curl_easy_setopt(curl, CURLOPT_READDATA, fp); add_header(&headers, "Content-Type", "application/link"); } - else if (!strcasecmp(method, "PUT") && fp) + //this condition does not include copy-to PUT action as it does not have a FP + //else if (!strcasecmp(method, "PUT") && fp) + else if (!strcasecmp(method, "PUT")) { + //http://blog.chmouel.com/2012/02/06/anatomy-of-a-swift-put-query-to-object-server/ + debugf(DBG_LEVEL_EXT, "send_request_size: PUT w/o FP(%s)", orig_path); curl_easy_setopt(curl, CURLOPT_UPLOAD, 1); - curl_easy_setopt(curl, CURLOPT_INFILESIZE, file_size); - curl_easy_setopt(curl, CURLOPT_READDATA, fp); + if (fp) { + curl_easy_setopt(curl, CURLOPT_INFILESIZE, file_size); + curl_easy_setopt(curl, CURLOPT_READDATA, fp); + } + else { + curl_easy_setopt(curl, CURLOPT_INFILESIZE, 0); + } if (is_segment) curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_callback); + //enable progress reporting + //http://curl.haxx.se/libcurl/c/progressfunc.html + struct curl_progress prog; + prog.lastruntime = 0; + prog.curl = curl; + curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progress_callback); + /* pass the struct pointer into the progress function */ + curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, &prog); + //get the response for debug purposes + /* send all data to this function */ + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writefunc_callback); + /* we pass our 'chunk' struct to the callback function */ + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&chunk); } else if (!strcasecmp(method, "GET")) { if (is_segment) { - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); + debugf(DBG_LEVEL_EXT, "GET SEGMENT (%s)", orig_path); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); } else if (fp) { + debugf(DBG_LEVEL_EXT, "send_request_size: GET FP (%s)", orig_path); rewind(fp); // make sure the file is ready for a-writin' fflush(fp); if (ftruncate(fileno(fp), 0) < 0) { - debugf("ftruncate failed. I don't know what to do about that."); + debugf(DBG_LEVEL_NORM, KRED"ftruncate failed. I don't know what to do about that."); abort(); } curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, &header_get_utimens_dispatch); + // sample by UThreadCurl.cpp, https://bitbucket.org/pamungkas5/bcbcurl/src + // and http://www.codeproject.com/Articles/838366/BCBCurl-a-LibCurl-based-download-manager + curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void *)de); + + struct curl_progress prog; + prog.lastruntime = 0; + prog.curl = curl; + curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progress_callback); + curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, &prog); } else if (xmlctx) { + debugf(DBG_LEVEL_EXT, "send_request_size: GET XML (%s)", orig_path); curl_easy_setopt(curl, CURLOPT_WRITEDATA, xmlctx); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &xml_dispatch); } + else { + //asumming retrieval of headers only + debugf(DBG_LEVEL_EXT, "send_request_size: GET HEADERS only(%s)"); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, &header_get_utimens_dispatch); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void *)de); + curl_easy_setopt(curl, CURLOPT_NOBODY, 1); + } } else { + debugf(DBG_LEVEL_EXT, "send_request_size: catch_all (%s)"); + // this posts an HEAD request (e.g. for statfs) curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, method); curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, &header_dispatch); } @@ -266,28 +525,66 @@ static int send_request_size(const char *method, const char *path, void *fp, curl_slist *extra; for (extra = extra_headers; extra; extra = extra->next) { - debugf("adding header: %s", extra->data); + debugf(DBG_LEVEL_EXT, "adding header: %s", extra->data); headers = curl_slist_append(headers, extra->data); } curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + debugf(DBG_LEVEL_EXT, "status: send_request_size(%s) started HTTP REQ:%s", orig_path, url); curl_easy_perform(curl); + double total_time; + char *effective_url; curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response); - curl_slist_free_all(headers); + curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &effective_url); + curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &total_time); + debugf(DBG_LEVEL_EXT, "status: send_request_size(%s) completed HTTP REQ:%s total_time=%.1f seconds", + orig_path, effective_url, total_time); + curl_slist_free_all(headers); curl_easy_reset(curl); return_connection(curl); - if ((response >= 200 && response < 400) || (!strcasecmp(method, "DELETE") && response == 409)) - return response; - sleep(8 << tries); // backoff - if (response == 401 && !cloudfs_connect()) // re-authenticate on 401s + + if (response != 404 && (response >= 400 || response < 200)) { + /* + * Now, our chunk.memory points to a memory block that is chunk.size + * bytes big and contains the remote file. + */ + + //printf("%lu bytes retrieved\n", (long)chunk.size); + debugf(DBG_LEVEL_NORM, KRED"send_request_size: error message, size=%lu, [HTTP %d] (%s)(%s)", + (long)chunk.size, response, method, path); + debugf(DBG_LEVEL_NORM, KRED"send_request_size: error message=[%s]", chunk.memory); + } + + free(chunk.memory); + + if ((response >= 200 && response < 400) || (!strcasecmp(method, "DELETE") && response == 409)) { + debugf(DBG_LEVEL_NORM, "exit 0: send_request_size(%s) speed=%.1f sec "KCYN"(%s) "KGRN"[HTTP OK]", + orig_path, total_time, method); + return response; + } + //handle cases when file is not found, no point in retrying, should exit + if (response == 404){ + debugf(DBG_LEVEL_NORM, "send_request_size: not found error for (%s)(%s), ignored "KYEL"[HTTP 404].", method, path); return response; + } + else { + debugf(DBG_LEVEL_NORM, "send_request_size: httpcode=%d (%s)(%s), retrying "KRED"[HTTP ERR]", response, method, path); + //todo: try to list response content for debug purposes + + sleep(8 << tries); // backoff + } + if (response == 401 && !cloudfs_connect()) { // re-authenticate on 401s + debugf(DBG_LEVEL_NORM, KYEL"exit 1: send_request_size(%s) (%s) [HTTP REAUTH]", path, method); + return response; + } if (xmlctx) xmlCtxtResetPush(xmlctx, NULL, 0, NULL, NULL); } + debugf(DBG_LEVEL_NORM, "exit 2: send_request_size(%s)"KCYN"(%s) response=%d", path, method, response); return response; } static int send_request(char *method, const char *path, FILE *fp, - xmlParserCtxtPtr xmlctx, curl_slist *extra_headers) + xmlParserCtxtPtr xmlctx, curl_slist *extra_headers, dir_entry *de_cached_entry) { long flen = 0; @@ -297,23 +594,29 @@ static int send_request(char *method, const char *path, FILE *fp, flen = cloudfs_file_size(fileno(fp)); } - return send_request_size(method, path, fp, xmlctx, extra_headers, flen, 0); + return send_request_size(method, path, fp, xmlctx, extra_headers, flen, 0, de_cached_entry); } +//thread that downloads or uploads large file segments void *upload_segment(void *seginfo) { struct segment_info *info = (struct segment_info *)seginfo; + char seg_path[MAX_URL_SIZE] = { 0 }; - + //set pointer to the segment start index in the complete large file (several threads will write to same large file) fseek(info->fp, info->part * info->segment_size, SEEK_SET); setvbuf(info->fp, NULL, _IOFBF, DISK_BUFF_SIZE); snprintf(seg_path, MAX_URL_SIZE, "%s%08i", info->seg_base, info->part); char *encoded = curl_escape(seg_path, 0); + debugf(DBG_LEVEL_EXT, KCYN"upload_segment(%s) part=%d size=%d seg_size=%d %s", + info->method, info->part, info->size, info->segment_size, seg_path); + + // int response = send_request_size(info->method, encoded, info, NULL, NULL, - info->size, 1); + info->size, 1, NULL); if (!(response >= 200 && response < 300)) fprintf(stderr, "Segment upload %s failed with response %d", seg_path, @@ -329,98 +632,125 @@ void *upload_segment(void *seginfo) void run_segment_threads(const char *method, int segments, int full_segments, int remaining, FILE *fp, char *seg_base, int size_of_segments) { - char file_path[PATH_MAX] = { 0 }; - struct segment_info *info = (struct segment_info *) - malloc(segments * sizeof(struct segment_info)); + debugf(DBG_LEVEL_EXT, "run_segment_threads(%s)", method); + char file_path[PATH_MAX] = { 0 }; + struct segment_info *info = (struct segment_info *) + malloc(segments * sizeof(struct segment_info)); - pthread_t *threads = (pthread_t *)malloc(segments * sizeof(pthread_t)); + pthread_t *threads = (pthread_t *)malloc(segments * sizeof(pthread_t)); #ifdef __linux__ snprintf(file_path, PATH_MAX, "/proc/self/fd/%d", fileno(fp)); + debugf(DBG_LEVEL_NORM, "On run segment filepath=%s", file_path); #else //TODO: I haven't actually tested this if (fcntl(fileno(fp), F_GETPATH, file_path) == -1) fprintf(stderr, "couldn't get the path name\n"); #endif - int i, ret; - for (i = 0; i < segments; i++) { - info[i].method = method; - info[i].fp = fopen(file_path, method[0] == 'G' ? "r+" : "r"); - info[i].part = i; - info[i].segment_size = size_of_segments; - info[i].size = i < full_segments ? size_of_segments : remaining; - info[i].seg_base = seg_base; - pthread_create(&threads[i], NULL, upload_segment, (void *)&(info[i])); - } + int i, ret; + for (i = 0; i < segments; i++) { + info[i].method = method; - for (i = 0; i < segments; i++) { - if ((ret = pthread_join(threads[i], NULL)) != 0) - fprintf(stderr, "error waiting for thread %d, status = %d\n", i, ret); - } - free(info); - free(threads); + info[i].fp = fopen(file_path, method[0] == 'G' ? "r+" : "r"); + info[i].part = i; + info[i].segment_size = size_of_segments; + info[i].size = i < full_segments ? size_of_segments : remaining; + info[i].seg_base = seg_base; + pthread_create(&threads[i], NULL, upload_segment, (void *)&(info[i])); + //set a thread name for debug purposes + //pthread_setname_np(threads[i], "run_segment"); + } + + for (i = 0; i < segments; i++) { + if ((ret = pthread_join(threads[i], NULL)) != 0) + fprintf(stderr, "error waiting for thread %d, status = %d\n", i, ret); + } + free(info); + free(threads); + debugf(DBG_LEVEL_EXT, "exit: run_segment_threads(%s)", method); } void split_path(const char *path, char *seg_base, char *container, char *object) { char *string = strdup(path); - snprintf(seg_base, MAX_URL_SIZE, "%s", strsep(&string, "/")); - strncat(container, strsep(&string, "/"), MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE)); - char *_object = strsep(&string, "/"); - + char *remstr; while (remstr = strsep(&string, "/")) { - strncat(container, "/", + strncat(container, "/", MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE)); strncat(container, _object, MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE)); _object = remstr; } - - strncpy(object, _object, MAX_URL_SIZE); - free(string); + //fixme: when removing root folders this will generate a segfault, issue #83, https://github.com/TurboGit/hubicfuse/issues/83 + if (_object == NULL) + _object = object; + else + strncpy(object, _object, MAX_URL_SIZE); + free(string); } -int internal_is_segmented(const char *seg_path, const char *object) +//checks on the cloud if this file (seg_path) have an associated segment folder +int internal_is_segmented(const char *seg_path, const char *object, const char *parent_path) { + debugf(DBG_LEVEL_EXT, "internal_is_segmented(%s)", seg_path); + //try to avoid one additional http request for small files + bool potentially_segmented; + dir_entry *de = check_path_info(parent_path); + if (!de) { + //when files in folders are first loaded the path will not be yet in cache, so need + //to force segment meta download for segmented files + potentially_segmented = true; + } + else { + //potentially segmented, assumption is that 0 size files are potentially segmented + //while size>0 is for sure not segmented, so no point in making an expensive HTTP GET call + potentially_segmented = (de->size == 0 && !de->isdir) ? true : false; + } + debugf(DBG_LEVEL_EXT, "internal_is_segmented: potentially segmented=%d", potentially_segmented); + //end change dir_entry *seg_dir; - if (cloudfs_list_directory(seg_path, &seg_dir)) { + if (potentially_segmented && cloudfs_list_directory(seg_path, &seg_dir)) { if (seg_dir && seg_dir->isdir) { do { if (!strncmp(seg_dir->name, object, MAX_URL_SIZE)) { + debugf(DBG_LEVEL_EXT, "exit 0: internal_is_segmented(%s) "KGRN"TRUE", seg_path); return 1; } } while ((seg_dir = seg_dir->next)); } } + debugf(DBG_LEVEL_EXT, "exit 1: internal_is_segmented(%s) "KYEL"FALSE", seg_path); return 0; } int is_segmented(const char *path) { - char container[MAX_URL_SIZE] = ""; - char object[MAX_URL_SIZE] = ""; - char seg_base[MAX_URL_SIZE] = ""; + debugf(DBG_LEVEL_EXT, "is_segmented(%s)", path); + char container[MAX_URL_SIZE] = ""; + char object[MAX_URL_SIZE] = ""; + char seg_base[MAX_URL_SIZE] = ""; - split_path(path, seg_base, container, object); + split_path(path, seg_base, container, object); - char seg_path[MAX_URL_SIZE]; - snprintf(seg_path, MAX_URL_SIZE, "%s/%s_segments", seg_base, container); - - return internal_is_segmented(seg_path, object); + char seg_path[MAX_URL_SIZE]; + snprintf(seg_path, MAX_URL_SIZE, "%s/%s_segments", seg_base, container); + return internal_is_segmented(seg_path, object, path); } - +//returns segmented file properties by parsing and retrieving the folder structure on the cloud +//added totalsize as parameter to return the file size on list directory for segmented files +//old implementation returns file size=0 (issue #91) int format_segments(const char *path, char * seg_base, long *segments, - long *full_segments, long *remaining, long *size_of_segments) + long *full_segments, long *remaining, long *size_of_segments, long *total_size) { - + debugf(DBG_LEVEL_EXT, "format_segments(%s)", path); char container[MAX_URL_SIZE] = ""; char object[MAX_URL_SIZE] = ""; @@ -429,36 +759,50 @@ int format_segments(const char *path, char * seg_base, long *segments, char seg_path[MAX_URL_SIZE]; snprintf(seg_path, MAX_URL_SIZE, "%s/%s_segments", seg_base, container); - if (internal_is_segmented(seg_path, object)) { + if (internal_is_segmented(seg_path, object, path)) { char manifest[MAX_URL_SIZE]; dir_entry *seg_dir; snprintf(manifest, MAX_URL_SIZE, "%s/%s", seg_path, object); - if (!cloudfs_list_directory(manifest, &seg_dir)) - return 0; + debugf(DBG_LEVEL_EXT, KMAG"format_segments manifest(%s)", manifest); + if (!cloudfs_list_directory(manifest, &seg_dir)) { + debugf(DBG_LEVEL_EXT, "exit 0: format_segments(%s)", path); + return 0; + } // snprintf seesaw between manifest and seg_path to get // the total_size and the segment size as well as the actual objects char *timestamp = seg_dir->name; snprintf(seg_path, MAX_URL_SIZE, "%s/%s", manifest, timestamp); - if (!cloudfs_list_directory(seg_path, &seg_dir)) - return 0; + debugf(DBG_LEVEL_EXT, KMAG"format_segments seg_path(%s)", seg_path); + if (!cloudfs_list_directory(seg_path, &seg_dir)) { + debugf(DBG_LEVEL_EXT, "exit 1: format_segments(%s)", path); + return 0; + } char *str_size = seg_dir->name; snprintf(manifest, MAX_URL_SIZE, "%s/%s", seg_path, str_size); - if (!cloudfs_list_directory(manifest, &seg_dir)) - return 0; + debugf(DBG_LEVEL_EXT, KMAG"format_segments manifest2(%s) size=%s", manifest, str_size); + if (!cloudfs_list_directory(manifest, &seg_dir)) { + debugf(DBG_LEVEL_NORM, "exit 2: format_segments(%s)", path); + return 0; + } + //following folder name actually represents the parent file size char *str_segment = seg_dir->name; snprintf(seg_path, MAX_URL_SIZE, "%s/%s", manifest, str_segment); - if (!cloudfs_list_directory(seg_path, &seg_dir)) - return 0; + debugf(DBG_LEVEL_EXT, KMAG"format_segments seg_path2(%s)", seg_path); + //here is where we get a list with all segment files composing the parent large file + if (!cloudfs_list_directory(seg_path, &seg_dir)) { + debugf(DBG_LEVEL_EXT, "exit 3: format_segments(%s)", path); + return 0; + } - long total_size = strtoll(str_size, NULL, 10); + *total_size = strtoll(str_size, NULL, 10); *size_of_segments = strtoll(str_segment, NULL, 10); - *remaining = total_size % *size_of_segments; - *full_segments = total_size / *size_of_segments; + *remaining = *total_size % *size_of_segments; + *full_segments = *total_size / *size_of_segments; *segments = *full_segments + (*remaining > 0); snprintf(manifest, MAX_URL_SIZE, "%s_segments/%s/%s/%s/%s/", @@ -467,11 +811,13 @@ int format_segments(const char *path, char * seg_base, long *segments, char tmp[MAX_URL_SIZE]; strncpy(tmp, seg_base, MAX_URL_SIZE); snprintf(seg_base, MAX_URL_SIZE, "%s/%s", tmp, manifest); - + debugf(DBG_LEVEL_EXT, KMAG"format_segments seg_base(%s)", seg_base); + debugf(DBG_LEVEL_EXT, KMAG"exit 4: format_segments(%s) total=%d size_of_segments=%d remaining=%d, full_segments=%d segments=%d", + path, &total_size, &size_of_segments, &remaining, &full_segments, &segments); return 1; } - else { + debugf(DBG_LEVEL_EXT, KMAG"exit 5: format_segments(%s) not segmented?", path); return 0; } } @@ -491,7 +837,7 @@ void cloudfs_init() // CentOS/RHEL 5 get stupid mode, because they have a broken libcurl if (cvid->version_num == RHEL5_LIBCURL_VERSION) { - debugf("RHEL5 mode enabled."); + debugf(DBG_LEVEL_NORM, "RHEL5 mode enabled."); rhel5_mode = 1; } @@ -514,6 +860,18 @@ void cloudfs_init() } } +void cloudfs_free() +{ + debugf(DBG_LEVEL_EXT, "Destroy mutex"); + pthread_mutex_destroy(&pool_mut); + int n; + for (n = 0; n < curl_pool_count; ++n) { + debugf(DBG_LEVEL_EXT, "Cleaning curl conn %d", n); + curl_easy_cleanup(curl_pool[n]); + } +} + + int file_is_readable(const char *fname) { FILE *file; @@ -548,7 +906,7 @@ const char * get_file_mimetype ( const char *path ) int cloudfs_object_read_fp(const char *path, FILE *fp) { - + debugf(DBG_LEVEL_EXT, "cloudfs_object_read_fp(%s)", path); long flen; fflush(fp); const char *filemimetype = get_file_mimetype( path ); @@ -559,10 +917,15 @@ int cloudfs_object_read_fp(const char *path, FILE *fp) // delete the previously uploaded segments if (is_segmented(path)) { - if (!cloudfs_delete_object(path)) - debugf("Couldn't delete one of the existing files while uploading."); + if (!cloudfs_delete_object(path)) { + debugf(DBG_LEVEL_NORM, KRED"cloudfs_object_read_fp: couldn't delete existing file"); + } + else { + debugf(DBG_LEVEL_EXT, KYEL"cloudfs_object_read_fp: deleted existing file"); + } } - + + struct timespec now; if (flen >= segment_above) { int i; long remaining = flen % segment_size; @@ -571,7 +934,6 @@ int cloudfs_object_read_fp(const char *path, FILE *fp) // The best we can do here is to get the current time that way tools that // use the mtime can at least check if the file was changing after now - struct timespec now; clock_gettime(CLOCK_REALTIME, &now); char string_float[TIME_CHARS]; @@ -594,6 +956,7 @@ int cloudfs_object_read_fp(const char *path, FILE *fp) cloudfs_create_directory(manifest); // reusing manifest + // TODO: check how addition of meta_mtime in manifest impacts utimens implementation snprintf(manifest, MAX_URL_SIZE, "%s_segments/%s/%s/%ld/%ld/", container, object, meta_mtime, flen, segment_size); @@ -607,28 +970,44 @@ int cloudfs_object_read_fp(const char *path, FILE *fp) char *encoded = curl_escape(path, 0); curl_slist *headers = NULL; add_header(&headers, "x-object-manifest", manifest); - add_header(&headers, "x-object-meta-mtime", meta_mtime); + //due to utimens changes, not needed anymore + //add_header(&headers, "x-object-meta-mtime", meta_mtime); add_header(&headers, "Content-Length", "0"); add_header(&headers, "Content-Type", filemimetype); - int response = send_request_size("PUT", encoded, NULL, NULL, headers, 0, 0); + int response = send_request_size("PUT", encoded, NULL, NULL, headers, 0, 0, NULL); curl_slist_free_all(headers); curl_free(encoded); + debugf(DBG_LEVEL_EXT, "exit 0: cloudfs_object_read_fp(%s) uploaded ok, response=%d", path, response); return (response >= 200 && response < 300); - + } + else{ + // assume enters here when file is composed of only one segment (small files) + debugf(DBG_LEVEL_EXT, KRED"cloudfs_object_read_fp(%s) unknown state", path); } rewind(fp); char *encoded = curl_escape(path, 0); - int response = send_request("PUT", encoded, fp, NULL, NULL); + + dir_entry *de = path_info(path); + if (!de) + debugf(DBG_LEVEL_EXT, "cloudfs_object_read_fp(%s) not in cache", path); + else { + debugf(DBG_LEVEL_EXT, "cloudfs_object_read_fp(%s) found in cache", path); + + } + // end changes + int response = send_request("PUT", encoded, fp, NULL, NULL, NULL); curl_free(encoded); + debugf(DBG_LEVEL_EXT, "exit 1: cloudfs_object_read_fp(%s)", path); return (response >= 200 && response < 300); } - +//write file downloaded from cloud to local file int cloudfs_object_write_fp(const char *path, FILE *fp) { + debugf(DBG_LEVEL_EXT, "cloudfs_object_write_fp(%s)", path); char *encoded = curl_escape(path, 0); char seg_base[MAX_URL_SIZE] = ""; @@ -636,30 +1015,35 @@ int cloudfs_object_write_fp(const char *path, FILE *fp) long full_segments; long remaining; long size_of_segments; + long total_size; + //checks if this file is a segmented one if (format_segments(path, seg_base, &segments, &full_segments, &remaining, - &size_of_segments)) { + &size_of_segments, &total_size)) { rewind(fp); fflush(fp); if (ftruncate(fileno(fp), 0) < 0) { - debugf("ftruncate failed. I don't know what to do about that."); + debugf(DBG_LEVEL_NORM, KRED"ftruncate failed. I don't know what to do about that."); abort(); } - - run_segment_threads("GET", segments, full_segments, remaining, fp, + run_segment_threads("GET", segments, full_segments, remaining, fp, seg_base, size_of_segments); + debugf(DBG_LEVEL_EXT, "exit 0: cloudfs_object_write_fp(%s)", path); return 1; } - int response = send_request("GET", encoded, fp, NULL, NULL); + int response = send_request("GET", encoded, fp, NULL, NULL, NULL); curl_free(encoded); fflush(fp); - if ((response >= 200 && response < 300) || ftruncate(fileno(fp), 0)) - return 1; + if ((response >= 200 && response < 300) || ftruncate(fileno(fp), 0)) { + debugf(DBG_LEVEL_EXT, "exit 1: cloudfs_object_write_fp(%s)", path); + return 1; + } rewind(fp); + debugf(DBG_LEVEL_EXT, "exit 2: cloudfs_object_write_fp(%s)", path); return 0; } @@ -670,19 +1054,51 @@ int cloudfs_object_truncate(const char *path, off_t size) if (size == 0) { FILE *fp = fopen("/dev/null", "r"); - response = send_request("PUT", encoded, fp, NULL, NULL); + response = send_request("PUT", encoded, fp, NULL, NULL, NULL); fclose(fp); } else {//TODO: this is busted - response = send_request("GET", encoded, NULL, NULL, NULL); + response = send_request("GET", encoded, NULL, NULL, NULL, NULL); } curl_free(encoded); return (response >= 200 && response < 300); } +//get metadata from cloud, like time attribs. create new entry if not cached yet. +void get_file_metadata(dir_entry *de){ + if (de->size == 0 && !de->isdir && !de->metadata_downloaded){ + //this can be a potential segmented file, try to read segments size + debugf(DBG_LEVEL_EXT, KMAG"ZERO size file=%s", de->full_name); + char seg_base[MAX_URL_SIZE] = ""; + long segments; + long full_segments; + long remaining; + long size_of_segments; + long total_size; + + if (format_segments(de->full_name, seg_base, &segments, &full_segments, &remaining, + &size_of_segments, &total_size)) { + de->size = total_size; + } + } + if (option_get_extended_metadata) { + debugf(DBG_LEVEL_EXT, KCYN "get_file_metadata(%s)", de->full_name); + //retrieve additional file metadata with a quick HEAD query + char *encoded = curl_escape(de->full_name, 0); + de->metadata_downloaded = true; + int response = send_request("GET", encoded, NULL, NULL, NULL, de); + curl_free(encoded); + debugf(DBG_LEVEL_EXT, KCYN "exit: get_file_metadata(%s)", de->full_name); + } + return; +} + +//get list of folders from cloud +// return 1 for OK, 0 for error int cloudfs_list_directory(const char *path, dir_entry **dir_list) { + debugf(DBG_LEVEL_EXT, "cloudfs_list_directory(%s)", path); char container[MAX_PATH_SIZE * 3] = ""; char object[MAX_PATH_SIZE] = ""; char last_subdir[MAX_PATH_SIZE] = ""; @@ -724,8 +1140,10 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) if ((!strcmp(path, "") || !strcmp(path, "/")) && *override_storage_url) response = 404; - else - response = send_request("GET", container, NULL, xmlctx, NULL); + else{ + // this was generating 404 err on non segmented files (small files) + response = send_request("GET", container, NULL, xmlctx, NULL, NULL); + } if (response >= 200 && response < 300) xmlParseChunk(xmlctx, "", 0, 1); @@ -743,19 +1161,22 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) if (is_object || is_container || is_subdir) { entry_count++; - - dir_entry *de = (dir_entry *)malloc(sizeof(dir_entry)); - de->next = NULL; - de->size = 0; - de->last_modified = time(NULL); + dir_entry *de = init_dir_entry(); + // useful docs on nodes here: http://developer.openstack.org/api-ref-objectstorage-v1.html if (is_container || is_subdir) de->content_type = strdup("application/directory"); for (anode = onode->children; anode; anode = anode->next) { char *content = ""; - for (text_node = anode->children; text_node; text_node = text_node->next) - if (text_node->type == XML_TEXT_NODE) + for (text_node = anode->children; text_node; text_node = text_node->next){ + if (text_node->type == XML_TEXT_NODE){ content = (char *)text_node->content; + //debugf(DBG_LEVEL_NORM, "List dir anode=%s content=%s", (const char *)anode->name, content); + } + else { + //debugf(DBG_LEVEL_NORM, "List dir anode=%s", (const char *)anode->name); + } + } if (!strcasecmp((const char *)anode->name, "name")) { de->name = strdup(content + prefix_length); @@ -765,11 +1186,14 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) if (slash && (0 == *(slash + 1))) *slash = 0; - if (asprintf(&(de->full_name), "%s/%s", path, de->name) < 0) + if (asprintf(&(de->full_name), "%s/%s", path, de->name) < 0){ de->full_name = NULL; + } + } - if (!strcasecmp((const char *)anode->name, "bytes")) - de->size = strtoll(content, NULL, 10); + if (!strcasecmp((const char *)anode->name, "bytes")) { + de->size = strtoll(content, NULL, 10); + } if (!strcasecmp((const char *)anode->name, "content_type")) { de->content_type = strdup(content); @@ -777,11 +1201,24 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) if (semicolon) *semicolon = '\0'; } + if (!strcasecmp((const char *)anode->name, "hash")) + { + de->md5sum = strdup(content); + } if (!strcasecmp((const char *)anode->name, "last_modified")) { - struct tm last_modified; - strptime(content, "%FT%T", &last_modified); - de->last_modified = mktime(&last_modified); + time_t last_modified_t = get_time_from_str_as_gmt(content); + char local_time_str[64]; + time_t local_time_t = get_time_as_local(last_modified_t, local_time_str, sizeof(local_time_str)); + de->last_modified = local_time_t; + de->ctime.tv_sec = local_time_t; + de->ctime.tv_nsec = 0; + //initialise all fields with hubic last modified date in case the file does not have extended attributes set + de->mtime.tv_sec = local_time_t; + de->mtime.tv_nsec = 0; + de->atime.tv_sec = local_time_t; + de->atime.tv_nsec = 0; + // TODO check if I can retrieve nano seconds? } } de->isdir = de->content_type && @@ -791,35 +1228,44 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) ((strstr(de->content_type, "application/link") != NULL)); if (de->isdir) { + //i guess this will remove a dir_entry from cache if is there already if (!strncasecmp(de->name, last_subdir, sizeof(last_subdir))) { - cloudfs_free_dir_list(de); + //not sure when / why this is called, seems to generate many missed delete ops. + //cloudfs_free_dir_list(de); + debugf(DBG_LEVEL_EXT, "cloudfs_list_directory: "KYEL"ignore "KNRM"cloudfs_free_dir_list(%s) command", de->name); continue; } strncpy(last_subdir, de->name, sizeof(last_subdir)); } de->next = *dir_list; *dir_list = de; + char time_str[TIME_CHARS] = ""; + get_timespec_as_str(&(de->mtime), time_str, sizeof(time_str)); + debugf(DBG_LEVEL_EXT, KCYN"new dir_entry %s size=%d %s dir=%d lnk=%d mod=[%s]", + de->full_name, de->size, de->content_type, de->isdir, de->islink, time_str); + //attempt to read extended attributes on each dir entry + //commented out as lazzy metadata read is implemented in cfs_getattr() + //get_file_metadata(de); } - else - { - debugf("unknown element: %s", onode->name); + else { + debugf(DBG_LEVEL_EXT, "unknown element: %s", onode->name); } } retval = 1; } else if ((!strcmp(path, "") || !strcmp(path, "/")) && *override_storage_url) { entry_count = 1; - - dir_entry *de = (dir_entry *)malloc(sizeof(dir_entry)); + debugf(DBG_LEVEL_NORM, "Init cache entry container=[%s]", public_container); + dir_entry *de = init_dir_entry(); de->name = strdup(public_container); struct tm last_modified; + // TODO check what this default time means? strptime("1388434648.01238", "%FT%T", &last_modified); de->last_modified = mktime(&last_modified); de->content_type = strdup("application/directory"); if (asprintf(&(de->full_name), "%s/%s", path, de->name) < 0) de->full_name = NULL; - de->isdir = 1; de->islink = 0; de->size = 4096; @@ -827,79 +1273,105 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) *dir_list = de; retval = 1; } - - debugf("entry count: %d", entry_count); - xmlFreeDoc(xmlctx->myDoc); xmlFreeParserCtxt(xmlctx); + debugf(DBG_LEVEL_EXT, "exit: cloudfs_list_directory(%s)", path); return retval; } -void cloudfs_free_dir_list(dir_entry *dir_list) -{ - while (dir_list) - { - dir_entry *de = dir_list; - dir_list = dir_list->next; - free(de->name); - free(de->full_name); - free(de->content_type); - free(de); - } -} int cloudfs_delete_object(const char *path) { - + debugf(DBG_LEVEL_EXT, "cloudfs_delete_object(%s)", path); char seg_base[MAX_URL_SIZE] = ""; long segments; long full_segments; long remaining; long size_of_segments; + long total_size; if (format_segments(path, seg_base, &segments, &full_segments, &remaining, - &size_of_segments)) { + &size_of_segments, &total_size)) { int response; int i; char seg_path[MAX_URL_SIZE] = ""; for (i = 0; i < segments; i++) { snprintf(seg_path, MAX_URL_SIZE, "%s%08i", seg_base, i); char *encoded = curl_escape(seg_path, 0); - response = send_request("DELETE", encoded, NULL, NULL, NULL); - if (response < 200 || response >= 300) - return 0; + response = send_request("DELETE", encoded, NULL, NULL, NULL, NULL); + if (response < 200 || response >= 300) { + debugf(DBG_LEVEL_EXT, "exit 1: cloudfs_delete_object(%s) response=%d", path, response); + return 0; + } } } char *encoded = curl_escape(path, 0); - int response = send_request("DELETE", encoded, NULL, NULL, NULL); + int response = send_request("DELETE", encoded, NULL, NULL, NULL, NULL); curl_free(encoded); int ret = (response >= 200 && response < 300); - if (response == 409) - ret = -1; + debugf(DBG_LEVEL_EXT, "status: cloudfs_delete_object(%s) response=%d", path, response); + if (response == 409) { + debugf(DBG_LEVEL_EXT, "status: cloudfs_delete_object(%s) NOT EMPTY", path); + ret = -1; + } return ret; } +//fixme: this op does not preserve src attributes (e.g. will make rsync not work well) +// https://ask.openstack.org/en/question/14307/is-there-a-way-to-moverename-an-object/ +// this operation also causes an HTTP 400 error if X-Object-Meta-FilePath value is larger than 256 chars int cloudfs_copy_object(const char *src, const char *dst) { + debugf(DBG_LEVEL_EXT, "cloudfs_copy_object(%s, %s) lensrc=%d, lendst=%d", src, dst, strlen(src), strlen(dst)); + char *dst_encoded = curl_escape(dst, 0); + char *src_encoded = curl_escape(src, 0); + + //convert encoded string (slashes are encoded as well) to encoded string with slashes + char *slash; + while ((slash = strstr(src_encoded, "%2F")) || (slash = strstr(src_encoded, "%2f"))) { + *slash = '/'; + memmove(slash + 1, slash + 3, strlen(slash + 3) + 1); + } + curl_slist *headers = NULL; - add_header(&headers, "X-Copy-From", src); + add_header(&headers, "X-Copy-From", src_encoded); add_header(&headers, "Content-Length", "0"); - int response = send_request("PUT", dst_encoded, NULL, NULL, headers); + //get source file entry + dir_entry *de_src = check_path_info(src); + if (de_src) { + debugf(DBG_LEVEL_EXT, "status cloudfs_copy_object(%s, %s): src file found", src, dst); + } + else { + debugf(DBG_LEVEL_NORM, KRED"status cloudfs_copy_object(%s, %s): src file NOT found", src, dst); + } + //pass src metadata so that PUT will set time attributes of the src file + int response = send_request("PUT", dst_encoded, NULL, NULL, headers, de_src); curl_free(dst_encoded); + curl_free(src_encoded); curl_slist_free_all(headers); - return (response >= 200 && response < 300); + debugf(DBG_LEVEL_EXT, "exit: cloudfs_copy_object(%s,%s) response=%d", src, dst, response); + return (response >= 200 && response < 300); } +//optimised with cache int cloudfs_statfs(const char *path, struct statvfs *stat) { - int response = send_request("HEAD", "/", NULL, NULL, NULL); - - debugf("Assigning statvfs values from cache."); - *stat = statcache; - return (response >= 200 && response < 300); + time_t now = get_time_now(); + int lapsed = now - last_stat_read_time; + if (lapsed > option_cache_statfs_timeout) { + int response = send_request("HEAD", "/", NULL, NULL, NULL, NULL); + *stat = statcache; + debugf(DBG_LEVEL_EXT, "exit: cloudfs_statfs (new recent values, was cached since %d seconds)", lapsed); + last_stat_read_time = now; + return (response >= 200 && response < 300); + } + else { + debugf(DBG_LEVEL_EXT, "exit: cloudfs_statfs (old values, cached since %d seconds)", lapsed); + return 1; + } } int cloudfs_create_symlink(const char *src, const char *dst) @@ -910,7 +1382,7 @@ int cloudfs_create_symlink(const char *src, const char *dst) fwrite(src, 1, strlen(src), lnk); fwrite("\0", 1, 1, lnk); - int response = send_request("MKLINK", dst_encoded, lnk, NULL, NULL); + int response = send_request("MKLINK", dst_encoded, lnk, NULL, NULL, NULL); curl_free(dst_encoded); fclose(lnk); return (response >= 200 && response < 300); @@ -918,9 +1390,11 @@ int cloudfs_create_symlink(const char *src, const char *dst) int cloudfs_create_directory(const char *path) { + debugf(DBG_LEVEL_EXT, "cloudfs_create_directory(%s)", path); char *encoded = curl_escape(path, 0); - int response = send_request("MKDIR", encoded, NULL, NULL, NULL); + int response = send_request("MKDIR", encoded, NULL, NULL, NULL, NULL); curl_free(encoded); + debugf(DBG_LEVEL_EXT, "cloudfs_create_directory(%s) response=%d", path, response); return (response >= 200 && response < 300); } @@ -931,16 +1405,21 @@ off_t cloudfs_file_size(int fd) return buf.st_size; } -void cloudfs_debug(int dbg) -{ - debug = dbg; -} - void cloudfs_verify_ssl(int vrfy) { verify_ssl = vrfy ? 2 : 0; } +void cloudfs_option_get_extended_metadata(int option) +{ + option_get_extended_metadata = option ? true : false; +} + +void cloudfs_option_curl_verbose(int option) +{ + option_curl_verbose = option ? true : false; +} + static struct { char client_id [MAX_HEADER_SIZE]; char client_secret[MAX_HEADER_SIZE]; @@ -979,6 +1458,7 @@ char* htmlStringGet(CURL *curl) struct htmlString chunk; chunk.text = malloc(sizeof(char)); chunk.size = 0; + chunk.text[0] = '\0';//added to avoid valgrind unitialised warning curl_easy_setopt(curl, CURLOPT_WRITEDATA, &chunk); do { @@ -1026,7 +1506,7 @@ int safe_json_string(json_object *jobj, char *buffer, char *name) } if (!result) - debugf("HUBIC cannot get json field '%s'\n", name); + debugf(DBG_LEVEL_NORM, KRED"HUBIC cannot get json field '%s'\n", name); return result; } @@ -1047,7 +1527,7 @@ int cloudfs_connect() pthread_mutex_lock(&pool_mut); - debugf("Authenticating... (client_id = '%s')", HUBIC_CLIENT_ID); + debugf(DBG_LEVEL_NORM, "Authenticating... (client_id = '%s')", HUBIC_CLIENT_ID); storage_token[0] = storage_url[0] = '\0'; @@ -1090,7 +1570,7 @@ int cloudfs_connect() char *json_str = htmlStringGet(curl); json_obj = json_tokener_parse(json_str); - debugf ("HUBIC TOKEN_URL result: '%s'\n", json_str); + debugf(DBG_LEVEL_NORM, "HUBIC TOKEN_URL result: '%s'\n", json_str); free(json_str); char access_token[HUBIC_OPTIONS_SIZE]; @@ -1107,9 +1587,9 @@ int cloudfs_connect() found = json_object_object_get_ex(json_obj, "expires_in", &o); expire_sec = json_object_get_int(o); - debugf ("HUBIC Access token: %s\n", access_token); - debugf ("HUBIC Token type : %s\n", token_type); - debugf ("HUBIC Expire in : %d\n", expire_sec); + debugf(DBG_LEVEL_NORM, "HUBIC Access token: %s\n", access_token); + debugf(DBG_LEVEL_NORM, "HUBIC Token type : %s\n", token_type); + debugf(DBG_LEVEL_NORM, "HUBIC Expire in : %d\n", expire_sec); /* Step 4 : request OpenStack storage URL */ @@ -1130,7 +1610,7 @@ int cloudfs_connect() json_str = htmlStringGet(curl); json_obj = json_tokener_parse(json_str); - debugf ("CRED_URL result: '%s'\n", json_str); + debugf(DBG_LEVEL_NORM, "CRED_URL result: '%s'\n", json_str); free(json_str); if (!safe_json_string(json_obj, token, "token")) @@ -1152,16 +1632,3 @@ int cloudfs_connect() return (response >= 200 && response < 300 && storage_token[0] && storage_url[0]); } - -void debugf(char *fmt, ...) -{ - if (debug) - { - va_list args; - va_start(args, fmt); - fputs("!!! ", stderr); - vfprintf(stderr, fmt, args); - va_end(args); - putc('\n', stderr); - } -} diff --git a/cloudfsapi.h b/cloudfsapi.h index 14f83ce..00db6ac 100644 --- a/cloudfsapi.h +++ b/cloudfsapi.h @@ -4,27 +4,22 @@ #include #include #include +#include #define BUFFER_INITIAL_SIZE 4096 #define MAX_HEADER_SIZE 8192 -#define MAX_PATH_SIZE (1024 + 256 + 3) + #define MAX_URL_SIZE (MAX_PATH_SIZE * 3) #define USER_AGENT "CloudFuse" #define OPTION_SIZE 1024 typedef struct curl_slist curl_slist; -typedef struct dir_entry -{ - char *name; - char *full_name; - char *content_type; - off_t size; - time_t last_modified; - int isdir; - int islink; - struct dir_entry *next; -} dir_entry; +#define MINIMAL_PROGRESS_FUNCTIONALITY_INTERVAL 3 +struct curl_progress { + double lastruntime; + CURL *curl; +}; typedef struct options { char cache_timeout[OPTION_SIZE]; @@ -39,7 +34,18 @@ typedef struct options { char refresh_token[OPTION_SIZE]; } FuseOptions; +typedef struct extra_options { + char get_extended_metadata[OPTION_SIZE]; + char curl_verbose[OPTION_SIZE]; + char cache_statfs_timeout[OPTION_SIZE]; + char debug_level[OPTION_SIZE]; + char curl_progress_state[OPTION_SIZE]; + char enable_chmod[OPTION_SIZE]; + char enable_chown[OPTION_SIZE]; +} ExtraFuseOptions; + void cloudfs_init(void); +void cloudfs_free(void); void cloudfs_set_credentials(char *client_id, char *client_secret, char *refresh_token); int cloudfs_connect(void); @@ -71,10 +77,9 @@ int cloudfs_create_symlink(const char *src, const char *dst); int cloudfs_create_directory(const char *label); int cloudfs_object_truncate(const char *path, off_t size); off_t cloudfs_file_size(int fd); -void cloudfs_debug(int dbg); -void cloudfs_verify_ssl(int dbg); -void cloudfs_free_dir_list(dir_entry *dir_list); int cloudfs_statfs(const char *path, struct statvfs *stat); - -void debugf(char *fmt, ...); +void cloudfs_verify_ssl(int dbg); +void cloudfs_option_get_extended_metadata(int option); +void cloudfs_option_curl_verbose(int option); +void get_file_metadata(dir_entry *de); #endif diff --git a/cloudfuse.c b/cloudfuse.c index 90b6680..132af26 100644 --- a/cloudfuse.c +++ b/cloudfuse.c @@ -14,21 +14,20 @@ #include #include #include +#include "commonfs.h" #include "cloudfsapi.h" #include "config.h" -static int cache_timeout; -static char *temp_dir; - -typedef struct dir_cache -{ - char *path; - dir_entry *entries; - time_t cached; - struct dir_cache *next, *prev; -} dir_cache; -static dir_cache *dcache; -static pthread_mutex_t dmut; +extern char *temp_dir; +extern pthread_mutex_t dcachemut; +extern pthread_mutexattr_t mutex_attr; +extern int cache_timeout; +extern int option_cache_statfs_timeout; +extern int option_debug_level; +extern bool option_get_extended_metadata; +extern int option_curl_progress_state; +extern bool option_enable_chown; +extern bool option_enable_chmod; typedef struct { @@ -36,266 +35,179 @@ typedef struct int flags; } openfile; - -static void dir_for(const char *path, char *dir) -{ - strncpy(dir, path, MAX_PATH_SIZE); - char *slash = strrchr(dir, '/'); - if (slash) - *slash = '\0'; -} - -static dir_cache *new_cache(const char *path) -{ - dir_cache *cw = (dir_cache *)calloc(sizeof(dir_cache), 1); - cw->path = strdup(path); - cw->prev = NULL; - cw->entries = NULL; - cw->cached = time(NULL); - if (dcache) - dcache->prev = cw; - cw->next = dcache; - return (dcache = cw); -} - -static int caching_list_directory(const char *path, dir_entry **list) -{ - pthread_mutex_lock(&dmut); - if (!strcmp(path, "/")) - path = ""; - dir_cache *cw; - for (cw = dcache; cw; cw = cw->next) - if (!strcmp(cw->path, path)) - break; - if (!cw) - { - if (!cloudfs_list_directory(path, list)) - return 0; - cw = new_cache(path); - } - else if (cache_timeout > 0 && (time(NULL) - cw->cached > cache_timeout)) - { - if (!cloudfs_list_directory(path, list)) - return 0; - cloudfs_free_dir_list(cw->entries); - cw->cached = time(NULL); - } - else - *list = cw->entries; - cw->entries = *list; - pthread_mutex_unlock(&dmut); - return 1; -} - -static void update_dir_cache(const char *path, off_t size, int isdir, int islink) -{ - pthread_mutex_lock(&dmut); - dir_cache *cw; - dir_entry *de; - char dir[MAX_PATH_SIZE]; - dir_for(path, dir); - for (cw = dcache; cw; cw = cw->next) - { - if (!strcmp(cw->path, dir)) - { - for (de = cw->entries; de; de = de->next) - { - if (!strcmp(de->full_name, path)) - { - de->size = size; - pthread_mutex_unlock(&dmut); - return; - } - } - de = (dir_entry *)malloc(sizeof(dir_entry)); - de->size = size; - de->isdir = isdir; - de->islink = islink; - de->name = strdup(&path[strlen(cw->path)+1]); - de->full_name = strdup(path); - - if (isdir) - { - de->content_type = strdup("application/link"); - } - if(islink) - { - de->content_type = strdup("application/directory"); - } - else - { - - de->content_type = strdup("application/octet-stream"); - } - de->last_modified = time(NULL); - de->next = cw->entries; - cw->entries = de; - if (isdir) - new_cache(path); - break; - } - } - pthread_mutex_unlock(&dmut); -} - -static void dir_decache(const char *path) -{ - dir_cache *cw; - pthread_mutex_lock(&dmut); - dir_entry *de, *tmpde; - char dir[MAX_PATH_SIZE]; - dir_for(path, dir); - for (cw = dcache; cw; cw = cw->next) - { - if (!strcmp(cw->path, path)) - { - if (cw == dcache) - dcache = cw->next; - if (cw->prev) - cw->prev->next = cw->next; - if (cw->next) - cw->next->prev = cw->prev; - cloudfs_free_dir_list(cw->entries); - free(cw->path); - free(cw); - } - else if (cw->entries && !strcmp(dir, cw->path)) - { - if (!strcmp(cw->entries->full_name, path)) - { - de = cw->entries; - cw->entries = de->next; - de->next = NULL; - cloudfs_free_dir_list(de); - } - else for (de = cw->entries; de->next; de = de->next) - { - if (!strcmp(de->next->full_name, path)) - { - tmpde = de->next; - de->next = de->next->next; - tmpde->next = NULL; - cloudfs_free_dir_list(tmpde); - break; - } - } - } - } - pthread_mutex_unlock(&dmut); -} - -static dir_entry *path_info(const char *path) -{ - char dir[MAX_PATH_SIZE]; - dir_for(path, dir); - dir_entry *tmp; - if (!caching_list_directory(dir, &tmp)) - return NULL; - for (; tmp; tmp = tmp->next) - { - if (!strcmp(tmp->full_name, path)) - return tmp; - } - return NULL; -} - static int cfs_getattr(const char *path, struct stat *stbuf) { - stbuf->st_uid = geteuid(); - stbuf->st_gid = getegid(); + debugf(DBG_LEVEL_NORM, KBLU "cfs_getattr(%s)", path); + + + //return standard values for root folder if (!strcmp(path, "/")) { + stbuf->st_uid = geteuid(); + stbuf->st_gid = getegid(); stbuf->st_mode = S_IFDIR | 0755; stbuf->st_nlink = 2; - return 0; + debug_list_cache_content(); + debugf(DBG_LEVEL_NORM, KBLU "exit 0: cfs_getattr(%s)", path); + return 0; } + //get file. if not in cache will be downloaded. dir_entry *de = path_info(path); - if (!de) - return -ENOENT; - stbuf->st_ctime = stbuf->st_mtime = de->last_modified; - if (de->isdir) - { + if (!de) { + debug_list_cache_content(); + debugf(DBG_LEVEL_NORM, KBLU"exit 1: cfs_getattr(%s) "KYEL"not-in-cache/cloud", path); + return -ENOENT; + } + + //lazzy download of file metadata, only when really needed + if (option_get_extended_metadata && !de->metadata_downloaded) { + get_file_metadata(de); + } + if (option_enable_chown) { + stbuf->st_uid = de->uid; + stbuf->st_gid = de->gid; + } + else { + stbuf->st_uid = geteuid(); + stbuf->st_gid = getegid(); + } + // change needed due to utimens + stbuf->st_atime = de->atime.tv_sec; + stbuf->st_atim.tv_nsec = de->atime.tv_nsec; + stbuf->st_mtime = de->mtime.tv_sec; + stbuf->st_mtim.tv_nsec = de->mtime.tv_nsec; + stbuf->st_ctime = de->ctime.tv_sec; + stbuf->st_ctim.tv_nsec = de->ctime.tv_nsec; + char time_str[TIME_CHARS] = ""; + get_timespec_as_str(&(de->atime), time_str, sizeof(time_str)); + debugf(DBG_LEVEL_EXT, KCYN"cfs_getattr: atime=[%s]", time_str); + get_timespec_as_str(&(de->mtime), time_str, sizeof(time_str)); + debugf(DBG_LEVEL_EXT, KCYN"cfs_getattr: mtime=[%s]", time_str); + get_timespec_as_str(&(de->ctime), time_str, sizeof(time_str)); + debugf(DBG_LEVEL_EXT, KCYN"cfs_getattr: ctime=[%s]", time_str); + + int default_mode_dir, default_mode_file; + + if (option_enable_chmod) { + default_mode_dir = de->chmod; + default_mode_file = de->chmod; + } + else { + default_mode_dir = 0755; + default_mode_file = 0666; + } + + if (de->isdir) { stbuf->st_size = 0; - stbuf->st_mode = S_IFDIR | 0755; + stbuf->st_mode = S_IFDIR | default_mode_dir; stbuf->st_nlink = 2; } - else if (de->islink) - { + else if (de->islink) { stbuf->st_size = 1; - stbuf->st_mode = S_IFLNK | 0755; + stbuf->st_mode = S_IFLNK | default_mode_dir; stbuf->st_nlink = 1; stbuf->st_size = de->size; /* calc. blocks as if 4K blocksize filesystem; stat uses units of 512B */ stbuf->st_blocks = ((4095 + de->size) / 4096) * 8; } - else - { + else { stbuf->st_size = de->size; /* calc. blocks as if 4K blocksize filesystem; stat uses units of 512B */ stbuf->st_blocks = ((4095 + de->size) / 4096) * 8; - stbuf->st_mode = S_IFREG | 0666; + stbuf->st_mode = S_IFREG | default_mode_file; stbuf->st_nlink = 1; } - return 0; + debugf(DBG_LEVEL_NORM, KBLU "exit 2: cfs_getattr(%s)", path); + return 0; } static int cfs_fgetattr(const char *path, struct stat *stbuf, struct fuse_file_info *info) { + debugf(DBG_LEVEL_NORM, KBLU "cfs_fgetattr(%s)", path); openfile *of = (openfile *)(uintptr_t)info->fh; if (of) { + //get file. if not in cache will be downloaded. + dir_entry *de = path_info(path); + if (!de) { + debug_list_cache_content(); + debugf(DBG_LEVEL_NORM, KBLU"exit 1: cfs_fgetattr(%s) "KYEL"not-in-cache/cloud", path); + return -ENOENT; + } + int default_mode_file; + if (option_enable_chmod) { + default_mode_file = de->chmod; + } + else { + default_mode_file = 0666; + } + stbuf->st_size = cloudfs_file_size(of->fd); - stbuf->st_mode = S_IFREG | 0666; + stbuf->st_mode = S_IFREG | default_mode_file; stbuf->st_nlink = 1; + debugf(DBG_LEVEL_NORM, KBLU "exit 0: cfs_fgetattr(%s)", path); return 0; } + debugf(DBG_LEVEL_NORM, KRED "exit 1: cfs_fgetattr(%s)", path); return -ENOENT; } static int cfs_readdir(const char *path, void *buf, fuse_fill_dir_t filldir, off_t offset, struct fuse_file_info *info) { + debugf(DBG_LEVEL_NORM, KBLU "cfs_readdir(%s)", path); dir_entry *de; - if (!caching_list_directory(path, &de)) - return -ENOLINK; + if (!caching_list_directory(path, &de)) { + debug_list_cache_content(); + debugf(DBG_LEVEL_NORM, KRED "exit 0: cfs_readdir(%s)", path); + return -ENOLINK; + } filldir(buf, ".", NULL, 0); filldir(buf, "..", NULL, 0); for (; de; de = de->next) filldir(buf, de->name, NULL, 0); - return 0; + debug_list_cache_content(); + debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_readdir(%s)", path); + return 0; } static int cfs_mkdir(const char *path, mode_t mode) { - if (cloudfs_create_directory(path)) - { + debugf(DBG_LEVEL_NORM, KBLU "cfs_mkdir(%s)", path); + int response = cloudfs_create_directory(path); + if (response){ update_dir_cache(path, 0, 1, 0); + debug_list_cache_content(); + debugf(DBG_LEVEL_NORM, KBLU "exit 0: cfs_mkdir(%s)", path); return 0; } - return -ENOENT; + debugf(DBG_LEVEL_NORM, KRED "exit 1: cfs_mkdir(%s) response=%d", path, response); + return -ENOENT; } static int cfs_create(const char *path, mode_t mode, struct fuse_file_info *info) { + debugf(DBG_LEVEL_NORM, KBLU "cfs_create(%s)", path); FILE *temp_file; + int errsv; - if (*temp_dir) - { - char tmp_path[PATH_MAX]; - strncpy(tmp_path, path, PATH_MAX); - - char *pch; - while((pch = strchr(tmp_path, '/'))) { - *pch = '.'; - } - - char file_path[PATH_MAX]; - snprintf(file_path, PATH_MAX, "%s/.cloudfuse%ld-%s", temp_dir, - (long)getpid(), tmp_path); - temp_file = fopen(file_path, "w+b"); + if (*temp_dir) { + char file_path_safe[NAME_MAX] = ""; + get_safe_cache_file_path(path, file_path_safe, temp_dir); + temp_file = fopen(file_path_safe, "w+b"); + errsv = errno; + if (temp_file == NULL){ + debugf(DBG_LEVEL_NORM, KRED "exit 0: cfs_create cannot open temp file %s.error %s\n", file_path_safe, strerror(errsv)); + return -EIO; } - else + } + else { temp_file = tmpfile(); - + errsv = errno; + if (temp_file == NULL){ + debugf(DBG_LEVEL_NORM, KRED "exit 1: cfs_create cannot open tmp file for path %s.error %s\n", path, strerror(errsv)); + return -EIO; + } + } openfile *of = (openfile *)malloc(sizeof(openfile)); of->fd = dup(fileno(temp_file)); fclose(temp_file); @@ -303,235 +215,362 @@ static int cfs_create(const char *path, mode_t mode, struct fuse_file_info *info info->fh = (uintptr_t)of; update_dir_cache(path, 0, 0, 0); info->direct_io = 1; + dir_entry *de = check_path_info(path); + if (de) { + debugf(DBG_LEVEL_EXT, KCYN"cfs_create(%s): found in cache", path); + struct timespec now; + clock_gettime(CLOCK_REALTIME, &now); + debugf(DBG_LEVEL_EXT, KCYN"cfs_create(%s) set utimes as now", path); + de->atime.tv_sec = now.tv_sec; + de->atime.tv_nsec = now.tv_nsec; + de->mtime.tv_sec = now.tv_sec; + de->mtime.tv_nsec = now.tv_nsec; + de->ctime.tv_sec = now.tv_sec; + de->ctime.tv_nsec = now.tv_nsec; + + char time_str[TIME_CHARS] = ""; + get_timespec_as_str(&(de->atime), time_str, sizeof(time_str)); + debugf(DBG_LEVEL_EXT, KCYN"cfs_create: atime=[%s]", time_str); + get_timespec_as_str(&(de->mtime), time_str, sizeof(time_str)); + debugf(DBG_LEVEL_EXT, KCYN"cfs_create: mtime=[%s]", time_str); + get_timespec_as_str(&(de->ctime), time_str, sizeof(time_str)); + debugf(DBG_LEVEL_EXT, KCYN"cfs_create: ctime=[%s]", time_str); + } + debugf(DBG_LEVEL_NORM, KBLU "exit 2: cfs_create(%s) result=%d:%s", path, errsv, strerror(errsv)); return 0; } + +// open (download) file from cloud static int cfs_open(const char *path, struct fuse_file_info *info) { - FILE *temp_file; + debugf(DBG_LEVEL_NORM, KBLU "cfs_open(%s)", path); + FILE *temp_file = NULL; + int errsv; dir_entry *de = path_info(path); - if (*temp_dir) - { - char tmp_path[PATH_MAX]; - strncpy(tmp_path, path, PATH_MAX); + if (*temp_dir) { + char file_path_safe[NAME_MAX]; + get_safe_cache_file_path(path, file_path_safe, temp_dir); - char *pch; - while((pch = strchr(tmp_path, '/'))) { - *pch = '.'; - } - - char file_path[PATH_MAX]; - snprintf(file_path, PATH_MAX, "%s/.cloudfuse%ld-%s", temp_dir, - (long)getpid(), tmp_path); - - if(access(file_path, F_OK) != -1) - { + debugf(DBG_LEVEL_EXT, "cfs_open: try open (%s)", file_path_safe); + if (access(file_path_safe, F_OK) != -1){ // file exists - temp_file = fopen(file_path, "r"); + temp_file = fopen(file_path_safe, "r"); + errsv = errno; + if (temp_file == NULL) { + debugf(DBG_LEVEL_NORM, KRED"exit 0: cfs_open can't open temp_file=[%s] err=%d:%s", file_path_safe, errsv, strerror(errsv)); + return -ENOENT; + } + else + debugf(DBG_LEVEL_EXT, "cfs_open: file exists"); } - else if (!(info->flags & O_WRONLY)) - { - // we need to lock on the filename another process could open the file - // while we are writing to it and then only read part of the file + else { + errsv = errno; + debugf(DBG_LEVEL_EXT, "cfs_open: file not in cache, err=%s", strerror(errsv)); + //FIXME: commented out as this condition will not be meet in some odd cases and program will crash + //if (!(info->flags & O_WRONLY)) { + debugf(DBG_LEVEL_EXT, "cfs_open: opening for write"); - // duplicate the directory caching datastructure to make the code easier - // to understand. + // we need to lock on the filename another process could open the file + // while we are writing to it and then only read part of the file - // each file in the cache needs: - // filename, is_writing, last_closed, is_removing - // the first time a file is opened a new entry is created in the cache - // setting the filename and is_writing to true. This check needs to be - // wrapped with a lock. - // - // each time a file is closed we set the last_closed for the file to now - // and we check the cache for files whose last - // closed is greater than cache_timeout, then start a new thread rming - // that file. + // duplicate the directory caching datastructure to make the code easier + // to understand. - // TODO: just to prevent this craziness for now - temp_file = fopen(file_path, "w+b"); + // each file in the cache needs: + // filename, is_writing, last_closed, is_removing + // the first time a file is opened a new entry is created in the cache + // setting the filename and is_writing to true. This check needs to be + // wrapped with a lock. + // + // each time a file is closed we set the last_closed for the file to now + // and we check the cache for files whose last + // closed is greater than cache_timeout, then start a new thread rming + // that file. - if (!cloudfs_object_write_fp(path, temp_file)) - { - fclose(temp_file); - return -ENOENT; - } - } + // TODO: just to prevent this craziness for now + temp_file = fopen(file_path_safe, "w+b"); + errsv = errno; + if (temp_file == NULL) { + debugf(DBG_LEVEL_NORM, KRED"exit 1: cfs_open cannot open temp_file=[%s] err=%d:%s", file_path_safe, errsv, strerror(errsv)); + return -ENOENT; + } + + if (!cloudfs_object_write_fp(path, temp_file)) { + fclose(temp_file); + debugf(DBG_LEVEL_NORM, KRED "exit 2: cfs_open(%s) cannot download/write", path); + return -ENOENT; + } + } } else { temp_file = tmpfile(); - if (!(info->flags & O_TRUNC)) - { - if (!cloudfs_object_write_fp(path, temp_file) && !(info->flags & O_CREAT)) - { + if (temp_file == NULL) { + debugf(DBG_LEVEL_NORM, KRED"exit 3: cfs_open cannot create temp_file err=%s", strerror(errno)); + return -ENOENT; + } + + if (!(info->flags & O_TRUNC)) { + if (!cloudfs_object_write_fp(path, temp_file) && !(info->flags & O_CREAT)) { fclose(temp_file); + debugf(DBG_LEVEL_NORM, KRED"exit 4: cfs_open(%s) cannot download/write", path); return -ENOENT; } } } + update_dir_cache(path, (de ? de->size : 0), 0, 0); - openfile *of = (openfile *)malloc(sizeof(openfile)); of->fd = dup(fileno(temp_file)); - if (of->fd == -1) + if (of->fd == -1){ + //FIXME: potential leak if free not used? + free(of); + debugf(DBG_LEVEL_NORM, KRED "exit 5: cfs_open(%s) of->fd", path); return -ENOENT; + } fclose(temp_file); + //TODO: why this allocation to of? of->flags = info->flags; info->fh = (uintptr_t)of; info->direct_io = 1; info->nonseekable = 1; + //FIXME: potential leak if free(of) not used? although if free(of) is used will generate bad descriptor errors + debugf(DBG_LEVEL_NORM, KBLU "exit 6: cfs_open(%s)", path); return 0; } static int cfs_read(const char *path, char *buf, size_t size, off_t offset, struct fuse_file_info *info) { - return pread(((openfile *)(uintptr_t)info->fh)->fd, buf, size, offset); + debugf(DBG_LEVEL_EXTALL, KBLU "cfs_read(%s)", path); + debug_print_descriptor(info); + int result = pread(((openfile *)(uintptr_t)info->fh)->fd, buf, size, offset); + debugf(DBG_LEVEL_EXTALL, KBLU "exit: cfs_read(%s) result=%s", path, strerror(errno)); + return result; } static int cfs_flush(const char *path, struct fuse_file_info *info) { + debugf(DBG_LEVEL_NORM, KBLU "cfs_flush(%s)", path); + debug_print_descriptor(info); openfile *of = (openfile *)(uintptr_t)info->fh; - if (of) - { + int errsv = 0; + + if (of) { update_dir_cache(path, cloudfs_file_size(of->fd), 0, 0); - if (of->flags & O_RDWR || of->flags & O_WRONLY) - { + if (of->flags & O_RDWR || of->flags & O_WRONLY) { FILE *fp = fdopen(dup(of->fd), "r"); - rewind(fp); - if (!cloudfs_object_read_fp(path, fp)) - { - fclose(fp); - return -ENOENT; - } - fclose(fp); + errsv = errno; + if (fp != NULL) { + rewind(fp); + if (!cloudfs_object_read_fp(path, fp)) { + fclose(fp); + errsv = errno; + debugf(DBG_LEVEL_NORM, KRED"exit 0: cfs_flush(%s) result=%d:%s", path, errsv, strerror(errno)); + return -ENOENT; + } + fclose(fp); + errsv = errno; + } + else { + debugf(DBG_LEVEL_EXT, KRED "status: cfs_flush, err=%d:%s", errsv, strerror(errno)); + } } } - return 0; + debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_flush(%s) result=%d:%s", path, errsv, strerror(errno)); + return 0; } -static int cfs_release(const char *path, struct fuse_file_info *info) -{ +static int cfs_release(const char *path, struct fuse_file_info *info) { + debugf(DBG_LEVEL_NORM, KBLU "cfs_release(%s)", path); close(((openfile *)(uintptr_t)info->fh)->fd); - return 0; + debugf(DBG_LEVEL_NORM, KBLU "exit: cfs_release(%s)", path); + return 0; } static int cfs_rmdir(const char *path) { + debugf(DBG_LEVEL_NORM, KBLU "cfs_rmdir(%s)", path); int success = cloudfs_delete_object(path); - if (success == -1) - return -ENOTEMPTY; - if (success) - { + if (success == -1) { + debugf(DBG_LEVEL_NORM, KBLU "exit 0: cfs_rmdir(%s)", path); + return -ENOTEMPTY; + } + if (success){ dir_decache(path); + debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_rmdir(%s)", path); return 0; } + debugf(DBG_LEVEL_NORM, KBLU "exit 2: cfs_rmdir(%s)", path); return -ENOENT; } static int cfs_ftruncate(const char *path, off_t size, struct fuse_file_info *info) { + debugf(DBG_LEVEL_NORM, KBLU "cfs_ftruncate(%s)", path); openfile *of = (openfile *)(uintptr_t)info->fh; if (ftruncate(of->fd, size)) return -errno; lseek(of->fd, 0, SEEK_SET); update_dir_cache(path, size, 0, 0); + debugf(DBG_LEVEL_NORM, KBLU "exit: cfs_ftruncate(%s)", path); return 0; } static int cfs_write(const char *path, const char *buf, size_t length, off_t offset, struct fuse_file_info *info) { + debugf(DBG_LEVEL_EXTALL, KBLU "cfs_write(%s)", path); + // FIXME: Potential inconsistent cache update if pwrite fails? update_dir_cache(path, offset + length, 0, 0); - return pwrite(((openfile *)(uintptr_t)info->fh)->fd, buf, length, offset); + //int result = pwrite(info->fh, buf, length, offset); + int result = pwrite(((openfile *)(uintptr_t)info->fh)->fd, buf, length, offset); + int errsv = errno; + debugf(DBG_LEVEL_EXTALL, KBLU "exit: cfs_write(%s) result=%d:%s", path, errsv, strerror(errsv)); + return result; } static int cfs_unlink(const char *path) { + debugf(DBG_LEVEL_NORM, KBLU "cfs_unlink(%s)", path); int success = cloudfs_delete_object(path); - if (success == -1) - return -EACCES; + if (success == -1) { + debugf(DBG_LEVEL_NORM, KRED "exit 0: cfs_unlink(%s)", path); + return -EACCES; + } if (success) { dir_decache(path); + debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_unlink(%s)", path); return 0; } + debugf(DBG_LEVEL_NORM, KRED "exit 2: cfs_unlink(%s)", path); return -ENOENT; } static int cfs_fsync(const char *path, int idunno, struct fuse_file_info *info) { + debugf(DBG_LEVEL_NORM, "cfs_fsync(%s)", path); return 0; } static int cfs_truncate(const char *path, off_t size) { + debugf(DBG_LEVEL_NORM, "cfs_truncate(%s)", path); cloudfs_object_truncate(path, size); + debugf(DBG_LEVEL_NORM, "exit: cfs_truncate(%s)", path); return 0; } +//this is called regularly on copy (via mc), is optimised (cached) static int cfs_statfs(const char *path, struct statvfs *stat) { + debugf(DBG_LEVEL_NORM, KBLU "cfs_statfs(%s)", path); if (cloudfs_statfs(path, stat)){ + debugf(DBG_LEVEL_NORM, KBLU "exit 0: cfs_statfs(%s)", path); return 0; } - else - return -EIO; + else { + debugf(DBG_LEVEL_NORM, KRED"exit 1: cfs_statfs(%s) not-found", path); + return -EIO; + } } static int cfs_chown(const char *path, uid_t uid, gid_t gid) { + debugf(DBG_LEVEL_NORM, KBLU "cfs_chown(%s,%d,%d)", path, uid, gid); + dir_entry *de = check_path_info(path); + if (de) { + if (de->uid != uid || de->gid != gid) { + debugf(DBG_LEVEL_NORM, "cfs_chown(%s): change from uid:gid %d:%d to %d:%d", path, de->uid, de->gid, uid, gid); + de->uid = uid; + de->gid = gid; + } + } return 0; } static int cfs_chmod(const char *path, mode_t mode) { + debugf(DBG_LEVEL_NORM, KBLU"cfs_chmod(%s,%d)", path, mode); + dir_entry *de = check_path_info(path); + if (de) { + if (de->chmod != mode) { + debugf(DBG_LEVEL_NORM, "cfs_chmod(%s): change mode from %d to %d", path, de->chmod, mode); + de->chmod = mode; + } + } return 0; } static int cfs_rename(const char *src, const char *dst) { + debugf(DBG_LEVEL_NORM, KBLU"cfs_rename(%s, %s)", src, dst); dir_entry *src_de = path_info(src); - if (!src_de) - return -ENOENT; - if (src_de->isdir) - return -EISDIR; - if (cloudfs_copy_object(src, dst)) - { + if (!src_de) { + debugf(DBG_LEVEL_NORM, KRED"exit 0: cfs_rename(%s,%s) not-found", src, dst); + return -ENOENT; + } + if (src_de->isdir) { + debugf(DBG_LEVEL_NORM, KRED"exit 1: cfs_rename(%s,%s) cannot rename dirs!", src, dst); + return -EISDIR; + } + if (cloudfs_copy_object(src, dst)) { /* FIXME this isn't quite right as doesn't preserve last modified */ + //fix done in cloudfs_copy_object() update_dir_cache(dst, src_de->size, 0, 0); - return cfs_unlink(src); + int result = cfs_unlink(src); + + dir_entry *dst_de = path_info(dst); + if (!dst_de) { + debugf(DBG_LEVEL_NORM, KRED"cfs_rename(%s,%s) dest-not-found-in-cache", src, dst); + } + else { + debugf(DBG_LEVEL_NORM, KBLU"cfs_rename(%s,%s) upload ok", src, dst); + //copy attributes, shortcut, rather than forcing a download from cloud + copy_dir_entry(src_de, dst_de); + } + + debugf(DBG_LEVEL_NORM, KBLU"exit 3: cfs_rename(%s,%s)", src, dst); + return result; } + debugf(DBG_LEVEL_NORM, KRED"exit 4: cfs_rename(%s,%s) io error", src, dst); return -EIO; } static int cfs_symlink(const char *src, const char *dst) { + debugf(DBG_LEVEL_NORM, KBLU"cfs_symlink(%s, %s)", src, dst); if(cloudfs_create_symlink(src, dst)) { update_dir_cache(dst, 1, 0, 1); + debugf(DBG_LEVEL_NORM, KBLU"exit0: cfs_symlink(%s, %s)", src, dst); return 0; } + debugf(DBG_LEVEL_NORM, KRED"exit1: cfs_symlink(%s, %s) io error", src, dst); return -EIO; } static int cfs_readlink(const char* path, char* buf, size_t size) { - FILE *temp_file = tmpfile(); + debugf(DBG_LEVEL_NORM, KBLU"cfs_readlink(%s)", path); + //fixme: use temp file specified in config + FILE *temp_file = tmpfile(); int ret = 0; - if (!cloudfs_object_write_fp(path, temp_file)) - { - ret = -ENOENT; + if (!cloudfs_object_write_fp(path, temp_file)) { + debugf(DBG_LEVEL_NORM, KRED"exit 1: cfs_readlink(%s) not found", path); + ret = -ENOENT; } - if (!pread(fileno(temp_file), buf, size, 0)) - { - ret = -ENOENT; + if (!pread(fileno(temp_file), buf, size, 0)) { + debugf(DBG_LEVEL_NORM, KRED"exit 2: cfs_readlink(%s) not found", path); + ret = -ENOENT; } fclose(temp_file); + debugf(DBG_LEVEL_NORM, KBLU"exit 3: cfs_readlink(%s)", path); return ret; } @@ -541,17 +580,52 @@ static void *cfs_init(struct fuse_conn_info *conn) return NULL; } -char *get_home_dir() -{ - char *home; - if ((home = getenv("HOME")) && !access(home, R_OK)) - return home; - struct passwd *pwd = getpwuid(geteuid()); - if ((home = pwd->pw_dir) && !access(home, R_OK)) - return home; - return "~"; + +//http://man7.org/linux/man-pages/man2/utimensat.2.html +static int cfs_utimens(const char *path, const struct timespec times[2]){ + debugf(DBG_LEVEL_NORM, KBLU "cfs_utimens(%s)", path); + // looking for file entry in cache + dir_entry *path_de = path_info(path); + if (!path_de) { + debugf(DBG_LEVEL_NORM, KRED"exit 0: cfs_utimens(%s) file not in cache", path); + return -ENOENT; + } + struct timespec now; + clock_gettime(CLOCK_REALTIME, &now); + + if (path_de->atime.tv_sec != times[0].tv_sec || path_de->atime.tv_nsec != times[0].tv_nsec || + path_de->mtime.tv_sec != times[1].tv_sec || path_de->mtime.tv_nsec != times[1].tv_nsec) { + debugf(DBG_LEVEL_EXT, KCYN "cfs_utimens: change for %s, prev: atime=%li.%li mtime=%li.%li, new: atime=%li.%li mtime=%li.%li", path, + path_de->atime.tv_sec, path_de->atime.tv_nsec, path_de->mtime.tv_sec, path_de->mtime.tv_nsec, + times[0].tv_sec, times[0].tv_nsec, times[1].tv_sec, times[1].tv_nsec); + char time_str[TIME_CHARS] = ""; + get_timespec_as_str(×[1], time_str, sizeof(time_str)); + debugf(DBG_LEVEL_EXT, KCYN"cfs_utimens: set mtime=[%s]", time_str); + get_timespec_as_str(×[0], time_str, sizeof(time_str)); + debugf(DBG_LEVEL_EXT, KCYN"cfs_utimens: set atime=[%s]", time_str); + path_de->atime = times[0]; + path_de->mtime = times[1]; + // not sure how to best obtain ctime from fuse. just record current date. + path_de->ctime = now; + } + else { + debugf(DBG_LEVEL_EXT, KCYN"cfs_utimens: a/m/time not changed"); + } + + debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_utimens(%s)", path); + return 0; } + +int cfs_setxattr(const char *path, const char *name, const char *value, size_t size, int flags){ + return 0; +} + +int cfs_getxattr(const char *path, const char *name, char *value, size_t size){ + return 0; +} + + FuseOptions options = { .cache_timeout = "600", .verify_ssl = "true", @@ -566,26 +640,80 @@ FuseOptions options = { .refresh_token = "" }; +ExtraFuseOptions extra_options = { + .get_extended_metadata = "false", + .curl_verbose = "false", + .cache_statfs_timeout = 0, + .debug_level = 0, + .curl_progress_state = 1, + .enable_chown = "false", + .enable_chmod = "false" +}; + int parse_option(void *data, const char *arg, int key, struct fuse_args *outargs) { if (sscanf(arg, " cache_timeout = %[^\r\n ]", options.cache_timeout) || - sscanf(arg, " verify_ssl = %[^\r\n ]", options.verify_ssl) || - sscanf(arg, " segment_above = %[^\r\n ]", options.segment_above) || - sscanf(arg, " segment_size = %[^\r\n ]", options.segment_size) || - sscanf(arg, " storage_url = %[^\r\n ]", options.storage_url) || - sscanf(arg, " container = %[^\r\n ]", options.container) || - sscanf(arg, " temp_dir = %[^\r\n ]", options.temp_dir) || - sscanf(arg, " client_id = %[^\r\n ]", options.client_id) || - sscanf(arg, " client_secret = %[^\r\n ]", options.client_secret) || - sscanf(arg, " refresh_token = %[^\r\n ]", options.refresh_token)) + sscanf(arg, " verify_ssl = %[^\r\n ]", options.verify_ssl) || + sscanf(arg, " segment_above = %[^\r\n ]", options.segment_above) || + sscanf(arg, " segment_size = %[^\r\n ]", options.segment_size) || + sscanf(arg, " storage_url = %[^\r\n ]", options.storage_url) || + sscanf(arg, " container = %[^\r\n ]", options.container) || + sscanf(arg, " temp_dir = %[^\r\n ]", options.temp_dir) || + sscanf(arg, " client_id = %[^\r\n ]", options.client_id) || + sscanf(arg, " client_secret = %[^\r\n ]", options.client_secret) || + sscanf(arg, " refresh_token = %[^\r\n ]", options.refresh_token) || + + sscanf(arg, " get_extended_metadata = %[^\r\n ]", extra_options.get_extended_metadata) || + sscanf(arg, " curl_verbose = %[^\r\n ]", extra_options.curl_verbose) || + sscanf(arg, " cache_statfs_timeout = %[^\r\n ]", extra_options.cache_statfs_timeout) || + sscanf(arg, " debug_level = %[^\r\n ]", extra_options.debug_level) || + sscanf(arg, " curl_progress_state = %[^\r\n ]", extra_options.curl_progress_state) || + sscanf(arg, " enable_chmod = %[^\r\n ]", extra_options.enable_chmod) || + sscanf(arg, " enable_chown = %[^\r\n ]", extra_options.enable_chown) + ) return 0; if (!strcmp(arg, "-f") || !strcmp(arg, "-d") || !strcmp(arg, "debug")) cloudfs_debug(1); return 1; } +//allows memory leaks inspections +void interrupt_handler(int sig) { + debugf(DBG_LEVEL_NORM, "Got interrupt signal %d, cleaning memory", sig); + //TODO: clean memory allocations + //http://www.cprogramming.com/debugging/valgrind.html + cloudfs_free(); + //TODO: clear dir cache + pthread_mutex_destroy(&dcachemut); + exit(0); +} + +void initialise_options() { + cloudfs_verify_ssl(!strcasecmp(options.verify_ssl, "true")); + cloudfs_option_get_extended_metadata(!strcasecmp(extra_options.get_extended_metadata, "true")); + cloudfs_option_curl_verbose(!strcasecmp(extra_options.curl_verbose, "true")); + if (*extra_options.debug_level) { + option_debug_level = atoi(extra_options.debug_level); + } + if (*extra_options.cache_statfs_timeout) { + option_cache_statfs_timeout = atoi(extra_options.cache_statfs_timeout); + } + if (*extra_options.curl_progress_state) { + option_curl_progress_state = atoi(extra_options.curl_progress_state); + } + if (*extra_options.enable_chmod) { + option_enable_chmod = !strcasecmp(extra_options.enable_chmod, "true"); + } + if (*extra_options.enable_chown) { + option_enable_chown = !strcasecmp(extra_options.enable_chown, "true"); + } +} + int main(int argc, char **argv) { + fprintf(stderr, "Starting hubicfuse on homedir %s!\n", get_home_dir()); + signal(SIGINT, interrupt_handler); + char settings_filename[MAX_PATH_SIZE] = ""; FILE *settings; struct fuse_args args = FUSE_ARGS_INIT(argc, argv); @@ -619,20 +747,31 @@ int main(int argc, char **argv) fprintf(stderr, " refresh_token=[Get it running hubic_token]\n"); fprintf(stderr, "The following settings are optional:\n\n"); fprintf(stderr, " cache_timeout=[Seconds for directory caching, default 600]\n"); - fprintf(stderr, " verify_ssl=[False to disable SSL cert verification]\n"); + fprintf(stderr, " verify_ssl=[false to disable SSL cert verification]\n"); fprintf(stderr, " segment_size=[Size to use when creating DLOs, default 1073741824]\n"); fprintf(stderr, " segment_above=[File size at which to use segments, defult 2147483648]\n"); fprintf(stderr, " storage_url=[Storage URL for other tenant to view container]\n"); fprintf(stderr, " container=[Public container to view of tenant specified by storage_url]\n"); fprintf(stderr, " temp_dir=[Directory to store temp files]\n"); + fprintf(stderr, " get_extended_metadata=[true to enable download of utime, chmod, chown file attributes (but slower)]\n"); + fprintf(stderr, " curl_verbose=[true to debug info on curl requests (lots of output)]\n"); + fprintf(stderr, " curl_progress_state=[0 or 1, 0 for progress callback enabled, 1 for disabled. Mostly used for debugging]\n"); + fprintf(stderr, " cache_statfs_timeout=[number of seconds to cache requests to statfs (cloud statistics), 0 for no cache]\n"); + fprintf(stderr, " debug_level=[0 to n, 0 for minimal verbose debugging. No debug if -d or -f option is not provided.]\n"); + fprintf(stderr, " enable_chmod=[true to enable chmod support on fuse]\n"); + fprintf(stderr, " enable_chown=[true to enable chown support on fuse]\n"); + return 1; } cloudfs_init(); - - cloudfs_verify_ssl(!strcasecmp(options.verify_ssl, "true")); - + initialise_options(); + fprintf(stderr, "debug_level = %d\n", option_debug_level); + fprintf(stderr, "get_extended_metadata = %d\n", option_get_extended_metadata); + fprintf(stderr, "curl_progress_state = %d\n", option_curl_progress_state); + fprintf(stderr, "enable_chmod = %d\n", option_enable_chmod); + fprintf(stderr, "enable_chown = %d\n", option_enable_chown); cloudfs_set_credentials(options.client_id, options.client_secret, options.refresh_token); if (!cloudfs_connect()) @@ -669,8 +808,13 @@ int main(int argc, char **argv) .symlink = cfs_symlink, .readlink = cfs_readlink, .init = cfs_init, + .utimens = cfs_utimens, + .setxattr = cfs_setxattr, + .getxattr = cfs_getxattr, }; - pthread_mutex_init(&dmut, NULL); + pthread_mutexattr_init(&mutex_attr); + pthread_mutexattr_settype(&mutex_attr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&dcachemut, &mutex_attr); return fuse_main(args.argc, args.argv, &cfs_oper, &options); } diff --git a/commonfs.c b/commonfs.c new file mode 100644 index 0000000..ddfb156 --- /dev/null +++ b/commonfs.c @@ -0,0 +1,608 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#ifdef __linux__ +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "commonfs.h" +#include "config.h" + +pthread_mutex_t dcachemut; +pthread_mutexattr_t mutex_attr; +dir_cache *dcache; +char *temp_dir; +int cache_timeout; +int debug = 0; +int verify_ssl = 2; +bool option_get_extended_metadata = false; +bool option_curl_verbose = false; +int option_cache_statfs_timeout = 0; +int option_debug_level = 0; +int option_curl_progress_state = 1;//1 to disable curl progress +bool option_enable_chown = false; +bool option_enable_chmod = false; + +// needed to get correct GMT / local time, as it does not work +// http://zhu-qy.blogspot.ro/2012/11/ref-how-to-convert-from-utc-to-local.html +time_t my_timegm(struct tm *tm) { + time_t epoch = 0; + time_t offset = mktime(gmtime(&epoch)); + time_t utc = mktime(tm); + return difftime(utc, offset); +} + +// hubic stores time as GMT so we have to do conversions + +/*void time_t set_now_time_to_gmt(){ + struct timespec now; + clock_gettime(CLOCK_REALTIME, &now); + http://stackoverflow.com/questions/1764710/converting-string-containing-localtime-into-utc-in-c +} +*/ +//expect time_str as a friendly string format +time_t get_time_from_str_as_gmt(char *time_str){ + struct tm val_time_tm; + time_t val_time_t; + strptime(time_str, "%FT%T", &val_time_tm); + val_time_tm.tm_isdst = -1; + val_time_t = my_timegm(&val_time_tm); + return val_time_t; +} + +time_t get_time_as_local(time_t time_t_val, char time_str[], int char_buf_size){ + struct tm loc_time_tm; + loc_time_tm = *localtime(&time_t_val); + if (time_str != NULL) { + //debugf(DBG_LEVEL_NORM, 0,"Local len=%d size=%d pass=%d", strlen(time_str), sizeof(time_str), char_buf_size); + strftime(time_str, char_buf_size, "%c", &loc_time_tm); + //debugf(DBG_LEVEL_NORM, 0,"Local timestr=[%s] size=%d", time_str, strlen(time_str)); + } + //debugf(DBG_LEVEL_NORM, 0,"Local time_t %li", mktime(&loc_time_tm)); + return mktime(&loc_time_tm); +} + +int get_time_as_string(time_t time_t_val, long nsec, char *time_str, int time_str_len){ + struct tm time_val_tm; + time_t safe_input_time; + //if time is incorrect (too long) you get segfault, need to check length and trim + if (time_t_val > INT_MAX) { + debugf(DBG_LEVEL_NORM, KRED"get_time_as_string: input time length too long, %lu > max=%lu, trimming!", time_t_val, INT_MAX); + safe_input_time = 0;//(int)time_t_val; + } + else + safe_input_time = time_t_val; + time_val_tm = *gmtime(&safe_input_time); + int str_len = strftime(time_str, time_str_len, HUBIC_DATE_FORMAT, &time_val_tm); + char nsec_str[TIME_CHARS]; + sprintf(nsec_str, "%d", nsec); + strcat(time_str, nsec_str); + return str_len + strlen(nsec_str); +} + +time_t get_time_now() { + struct timespec now; + clock_gettime(CLOCK_REALTIME, &now); + return now.tv_sec; +} + +size_t get_time_now_as_str(char *time_str, int time_str_len) { + time_t now = time(0); + struct tm tstruct; + tstruct = *localtime(&now); + // Visit http://en.cppreference.com/w/cpp/chrono/c/strftime + // for more information about date/time format + size_t result = strftime(time_str, time_str_len, HUBIC_DATE_FORMAT, &tstruct); + return result; +} + +int get_timespec_as_str(const struct timespec *times, char *time_str, int time_str_len) { + return get_time_as_string(times->tv_sec, times->tv_nsec, time_str, time_str_len); +} + +char *str2md5(const char *str, int length) { + int n; + MD5_CTX c; + unsigned char digest[16]; + char *out = (char*)malloc(33); + + MD5_Init(&c); + + while (length > 0) { + if (length > 512) { + MD5_Update(&c, str, 512); + } + else { + MD5_Update(&c, str, length); + } + length -= 512; + str += 512; + } + + MD5_Final(digest, &c); + + for (n = 0; n < 16; ++n) { + snprintf(&(out[n * 2]), 16 * 2, "%02x", (unsigned int)digest[n]); + } + + return out; +} + + +int get_safe_cache_file_path(const char *path, char *file_path_safe, char *temp_dir) { + char tmp_path[PATH_MAX]; + strncpy(tmp_path, path, PATH_MAX); + char *pch; + while ((pch = strchr(tmp_path, '/'))) { + *pch = '.'; + } + char file_path[PATH_MAX] = ""; + //temp file name had process pid in it, removed as on restart files are left in cache (pid changes) + snprintf(file_path, PATH_MAX, TEMP_FILE_NAME_FORMAT, temp_dir, tmp_path); + //fixme check if sizeof or strlen is suitable + int file_path_len = sizeof(file_path); + //the file path name using this format can go beyond NAME_MAX size and will generate error on fopen + //solution: cap file length to NAME_MAX, use a prefix from original path for debug purposes and add md5 id + char *md5_path = str2md5(file_path, file_path_len); + int md5len = strlen(md5_path); + size_t safe_len_prefix = min(NAME_MAX - md5len, file_path_len); + strncpy(file_path_safe, file_path, safe_len_prefix); + strncpy(file_path_safe + safe_len_prefix, md5_path, md5len); + //sometimes above copy process produces longer strings that NAME_MAX, force a null terminated string + file_path_safe[safe_len_prefix + md5len - 1] = '\0'; + free(md5_path); + return strlen(file_path_safe); +} + +void get_file_path_from_fd(int fd, char *path, int size_path) { + char proc_path[MAX_PATH_SIZE]; + /* Read out the link to our file descriptor. */ + sprintf(proc_path, "/proc/self/fd/%d", fd); + memset(path, 0, size_path); + readlink(proc_path, path, size_path - 1); +} + +void debug_print_flags(int flags) { + int accmode, val; + accmode = flags & O_ACCMODE; + if (accmode == O_RDONLY) debugf(DBG_LEVEL_EXT, KRED "read only"); + else if (accmode == O_WRONLY) debugf(DBG_LEVEL_EXT, KRED "write only"); + else if (accmode == O_RDWR) debugf(DBG_LEVEL_EXT, KRED "read write"); + else debugf(DBG_LEVEL_EXT, KRED "unknown access mode"); + + if (val & O_APPEND) debugf(DBG_LEVEL_EXT, KRED ", append"); + if (val & O_NONBLOCK) debugf(DBG_LEVEL_EXT, KRED ", nonblocking"); +#if !defined(_POSIX_SOURCE) && defined(O_SYNC) + if (val & O_SYNC) debugf(DBG_LEVEL_EXT, 0,KRED ", synchronous writes"); +#endif + +} + +void debug_print_descriptor(struct fuse_file_info *info) { + char file_path[MAX_PATH_SIZE]; + get_file_path_from_fd(info->fh, file_path, sizeof(file_path)); + debugf(DBG_LEVEL_EXT, KCYN "descriptor localfile=[%s] fd=%d", file_path, info->fh); + debug_print_flags(info->flags); +} + +void dir_for(const char *path, char *dir) +{ + strncpy(dir, path, MAX_PATH_SIZE); + char *slash = strrchr(dir, '/'); + if (slash) + *slash = '\0'; +} + +//prints cache content for debug purposes +void debug_list_cache_content() { + return;//disabled + dir_cache *cw; + dir_entry *de; + for (cw = dcache; cw; cw = cw->next) { + debugf(DBG_LEVEL_EXT, "LIST-CACHE: DIR[%s]", cw->path); + for (de = cw->entries; de; de = de->next) { + debugf(DBG_LEVEL_EXT, "LIST-CACHE: FOLDER[%s]", de->full_name); + } + } +} + +int delete_file(char *path) { + debugf(DBG_LEVEL_NORM, KYEL"delete_file(%s)", path); + char file_path_safe[NAME_MAX] = ""; + get_safe_cache_file_path(path, file_path_safe, temp_dir); + int result = unlink(file_path_safe); + debugf(DBG_LEVEL_EXT, KYEL"delete_file(%s) (%s) result=%s", path, file_path_safe, strerror(result)); + return result; +} + +//adding a directory in cache +dir_cache *new_cache(const char *path) +{ + debugf(DBG_LEVEL_NORM, KCYN"new_cache(%s)", path); + dir_cache *cw = (dir_cache *)calloc(sizeof(dir_cache), 1); + cw->path = strdup(path); + cw->prev = NULL; + cw->entries = NULL; + cw->cached = time(NULL); + //added cache by access + cw->accessed_in_cache = time(NULL); + cw->was_deleted = false; + if (dcache) + dcache->prev = cw; + cw->next = dcache; + dir_cache *result; + result = (dcache = cw); + debugf(DBG_LEVEL_EXT, "exit: new_cache(%s)", path); + return result; +} + + +//todo: check if the program behaves ok when free_dir +//is made on a folder that has an operation in progress +void cloudfs_free_dir_list(dir_entry *dir_list) +{ + //check for NULL as dir might be already removed from cache by other thread + debugf(DBG_LEVEL_NORM, "cloudfs_free_dir_list(%s)", dir_list->full_name); + while (dir_list) { + dir_entry *de = dir_list; + dir_list = dir_list->next; + //remove file from disk cache, fix for issue #89, https://github.com/TurboGit/hubicfuse/issues/89 + delete_file(de->full_name); + free(de->name); + free(de->full_name); + free(de->content_type); + //TODO free all added fields + free(de->md5sum); + free(de); + } +} + + +void dir_decache(const char *path) +{ + dir_cache *cw; + debugf(DBG_LEVEL_NORM, "dir_decache(%s)", path); + pthread_mutex_lock(&dcachemut); + dir_entry *de, *tmpde; + char dir[MAX_PATH_SIZE]; + dir_for(path, dir); + for (cw = dcache; cw; cw = cw->next) { + debugf(DBG_LEVEL_EXT, "dir_decache: parse(%s)", cw->path); + if (!strcmp(cw->path, path)) { + if (cw == dcache) + dcache = cw->next; + if (cw->prev) + cw->prev->next = cw->next; + if (cw->next) + cw->next->prev = cw->prev; + debugf(DBG_LEVEL_EXT, "dir_decache: free_dir1(%s)", cw->path); + //fixme: this sometimes is NULL and generates segfaults, checking first + if (cw->entries != NULL) + cloudfs_free_dir_list(cw->entries); + free(cw->path); + free(cw); + } + else if (cw->entries && !strcmp(dir, cw->path)) + { + if (!strcmp(cw->entries->full_name, path)) { + de = cw->entries; + cw->entries = de->next; + de->next = NULL; + debugf(DBG_LEVEL_EXT, "dir_decache: free_dir2()"); + cloudfs_free_dir_list(de); + } + else for (de = cw->entries; de->next; de = de->next) + { + if (!strcmp(de->next->full_name, path)) + { + tmpde = de->next; + de->next = de->next->next; + tmpde->next = NULL; + debugf(DBG_LEVEL_EXT, "dir_decache: free_dir3()", cw->path); + cloudfs_free_dir_list(tmpde); + break; + } + } + } + } + pthread_mutex_unlock(&dcachemut); +} + +dir_entry* init_dir_entry() { + dir_entry *de = (dir_entry *)malloc(sizeof(dir_entry)); + de->metadata_downloaded = false; + de->size = 0; + de->next = NULL; + de->md5sum = NULL; + de->accessed_in_cache = time(NULL); + de->last_modified = time(NULL); + de->mtime.tv_sec = time(NULL); + de->atime.tv_sec = time(NULL); + de->ctime.tv_sec = time(NULL); + de->mtime.tv_nsec = 0; + de->atime.tv_nsec = 0; + de->ctime.tv_nsec = 0; + de->chmod = 0; + de->gid = 0; + de->uid = 0; + return de; +} + +void copy_dir_entry(dir_entry *src, dir_entry *dst) { + dst->atime.tv_sec = src->atime.tv_sec; + dst->atime.tv_nsec = src->atime.tv_nsec; + dst->mtime.tv_sec = src->mtime.tv_sec; + dst->mtime.tv_nsec = src->mtime.tv_nsec; + dst->ctime.tv_sec = src->ctime.tv_sec; + dst->ctime.tv_nsec = src->ctime.tv_nsec; + dst->chmod = src->chmod; + //dst->md5sum = src->md5sum; +} + +//check for file in cache, if found size will be updated, if not found +//and this is a dir, a new dir cache entry is created +void update_dir_cache(const char *path, off_t size, int isdir, int islink) +{ + debugf(DBG_LEVEL_EXTALL, KCYN "update_dir_cache(%s)", path); + pthread_mutex_lock(&dcachemut); + dir_cache *cw; + dir_entry *de; + char dir[MAX_PATH_SIZE]; + dir_for(path, dir); + for (cw = dcache; cw; cw = cw->next) + { + if (!strcmp(cw->path, dir)) + { + for (de = cw->entries; de; de = de->next) + { + if (!strcmp(de->full_name, path)) + { + de->size = size; + pthread_mutex_unlock(&dcachemut); + debugf(DBG_LEVEL_EXTALL, "exit 0: update_dir_cache(%s)", path); + return; + } + } + de = init_dir_entry(); + de->size = size; + de->isdir = isdir; + de->islink = islink; + de->name = strdup(&path[strlen(cw->path) + 1]); + de->full_name = strdup(path); + //fix: the conditions below were mixed up dir -> link? + if (islink) { + de->content_type = strdup("application/link"); + } + if (isdir) { + de->content_type = strdup("application/directory"); + } + else { + de->content_type = strdup("application/octet-stream"); + } + de->next = cw->entries; + cw->entries = de; + if (isdir) + new_cache(path); + break; + } + } + debugf(DBG_LEVEL_EXTALL, "exit 1: update_dir_cache(%s)", path); + pthread_mutex_unlock(&dcachemut); +} + +//returns first file entry in linked list. if not in cache will be downloaded. +int caching_list_directory(const char *path, dir_entry **list) +{ + debugf(DBG_LEVEL_EXT, "caching_list_directory(%s)", path); + pthread_mutex_lock(&dcachemut); + bool new_entry = false; + if (!strcmp(path, "/")) + path = ""; + + dir_cache *cw; + for (cw = dcache; cw; cw = cw->next) { + if (cw->was_deleted == true) { + debugf(DBG_LEVEL_EXT, KMAG"caching_list_directory status: dir(%s) is empty as cached expired, reload from cloud", cw->path); + if (!cloudfs_list_directory(cw->path, list)) { + debugf(DBG_LEVEL_EXT, KMAG"caching_list_directory status: cannot reload dir(%s)", cw->path); + } + else { + debugf(DBG_LEVEL_EXT, KMAG"caching_list_directory status: reloaded dir(%s)", cw->path); + //cw->entries = *list; + cw->was_deleted = false; + cw->cached = time(NULL); + } + } + if (cw->was_deleted == false) { + if (!strcmp(cw->path, path)) { + break; + } + } + } + if (!cw) { + //trying to download this entry from cloud, list will point to cached or downloaded entries + if (!cloudfs_list_directory(path, list)){ + //download was not ok + pthread_mutex_unlock(&dcachemut); + debugf(DBG_LEVEL_EXT, "exit 0: caching_list_directory(%s) "KYEL"[CACHE-DIR-MISS]", path); + return 0; + } + debugf(DBG_LEVEL_EXT, "caching_list_directory: new_cache(%s) "KYEL"[CACHE-CREATE]", path); + cw = new_cache(path); + new_entry = true; + } + else if (cache_timeout > 0 && (time(NULL) - cw->cached > cache_timeout)) + { + if (!cloudfs_list_directory(path, list)){ + //mutex unlock was forgotten + pthread_mutex_unlock(&dcachemut); + debugf(DBG_LEVEL_EXT, "exit 1: caching_list_directory(%s)", path); + return 0; + } + //fixme: this frees dir subentries but leaves the dir parent entry, this confuses path_info + //which believes this dir has no entries + if (cw->entries != NULL) { + cloudfs_free_dir_list(cw->entries); + cw->was_deleted = true; + cw->cached = time(NULL); + debugf(DBG_LEVEL_EXT, "caching_list_directory(%s) "KYEL"[CACHE-EXPIRED]", path); + } + else { + debugf(DBG_LEVEL_EXT, "got NULL on caching_list_directory(%s) "KYEL"[CACHE-EXPIRED w NULL]", path); + pthread_mutex_unlock(&dcachemut); + return 0; + } + } + else { + debugf(DBG_LEVEL_EXT, "caching_list_directory(%s) "KGRN"[CACHE-DIR-HIT]", path); + *list = cw->entries; + } + //adding new dir file list to global cache, now this dir becomes visible in cache + cw->entries = *list; + pthread_mutex_unlock(&dcachemut); + debugf(DBG_LEVEL_EXT, "exit 2: caching_list_directory(%s)", path); + return 1; +} + +dir_entry *path_info(const char *path) +{ + debugf(DBG_LEVEL_EXT, "path_info(%s)", path); + char dir[MAX_PATH_SIZE]; + dir_for(path, dir); + dir_entry *tmp; + if (!caching_list_directory(dir, &tmp)) { + debugf(DBG_LEVEL_EXT, "exit 0: path_info(%s) "KYEL"[CACHE-DIR-MISS]", dir); + return NULL; + } + else { + debugf(DBG_LEVEL_EXT, "path_info(%s) "KGRN"[CACHE-DIR-HIT]", dir); + } + //iterate in file list obtained from cache or downloaded + for (; tmp; tmp = tmp->next) { + if (!strcmp(tmp->full_name, path)) { + debugf(DBG_LEVEL_EXT, "exit 1: path_info(%s) "KGRN"[CACHE-FILE-HIT]", path); + return tmp; + } + } + //miss in case the file is not found on a cached folder + debugf(DBG_LEVEL_EXT, "exit 2: path_info(%s) "KYEL"[CACHE-MISS]", path); + return NULL; +} + + +//retrieve folder from local cache if exists, return null if does not exist (rather than download) +int check_caching_list_directory(const char *path, dir_entry **list) +{ + debugf(DBG_LEVEL_EXT, "check_caching_list_directory(%s)", path); + pthread_mutex_lock(&dcachemut); + if (!strcmp(path, "/")) + path = ""; + dir_cache *cw; + for (cw = dcache; cw; cw = cw->next) + if (!strcmp(cw->path, path)) { + *list = cw->entries; + pthread_mutex_unlock(&dcachemut); + debugf(DBG_LEVEL_EXT, "exit 0: check_caching_list_directory(%s) "KGRN"[CACHE-DIR-HIT]", path); + return 1; + } + pthread_mutex_unlock(&dcachemut); + debugf(DBG_LEVEL_EXT, "exit 1: check_caching_list_directory(%s) "KYEL"[CACHE-DIR-MISS]", path); + return 0; +} + +dir_entry * check_parent_folder_for_file(const char *path) { + char dir[MAX_PATH_SIZE]; + dir_for(path, dir); + dir_entry *tmp; + if (!check_caching_list_directory(dir, &tmp)) + return NULL; + else + return tmp; +} + +//check if local path is in cache, without downloading from cloud if not in cache +dir_entry *check_path_info(const char *path) +{ + debugf(DBG_LEVEL_EXT, "check_path_info(%s)", path); + char dir[MAX_PATH_SIZE]; + dir_for(path, dir); + dir_entry *tmp; + + //get parent folder cache entry + if (!check_caching_list_directory(dir, &tmp)) { + debugf(DBG_LEVEL_EXT, "exit 0: check_path_info(%s) "KYEL"[CACHE-MISS]", path); + return NULL; + } + for (; tmp; tmp = tmp->next) + { + if (!strcmp(tmp->full_name, path)) { + debugf(DBG_LEVEL_EXT, "exit 1: check_path_info(%s) "KGRN"[CACHE-HIT]", path); + return tmp; + } + } + if (!strcmp(path, "/")) { + debugf(DBG_LEVEL_EXT, "exit 2: check_path_info(%s) "KYEL "ignoring root [CACHE-MISS]", path); + } + else { + debugf(DBG_LEVEL_EXT, "exit 3: check_path_info(%s) "KYEL"[CACHE-MISS]", path); + } + return NULL; +} + + +char *get_home_dir() +{ + char *home; + if ((home = getenv("HOME")) && !access(home, R_OK)) + return home; + struct passwd *pwd = getpwuid(geteuid()); + if ((home = pwd->pw_dir) && !access(home, R_OK)) + return home; + return "~"; +} + +void cloudfs_debug(int dbg) +{ + debug = dbg; +} + +void debugf(int level, char *fmt, ...) +{ + if (debug) { + if (level <= option_debug_level) { +#ifdef SYS_gettid + pid_t thread_id = syscall(SYS_gettid); +#else + int thread_id = 0; +#error "SYS_gettid unavailable on this system" +#endif + va_list args; + char prefix[] = "==DBG %d [%s]:%d=="; + char line[4096]; + char time_str[TIME_CHARS]; + get_time_now_as_str(time_str, sizeof(time_str)); + sprintf(line, prefix, level, time_str, thread_id); + fputs(line, stderr); + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); + fputs(KNRM, stderr); + putc('\n', stderr); + putc('\r', stderr); + } + } +} diff --git a/commonfs.h b/commonfs.h new file mode 100644 index 0000000..ca5b47d --- /dev/null +++ b/commonfs.h @@ -0,0 +1,112 @@ +#ifndef _COMMONFS_H +#define _COMMONFS_H +#include + +typedef enum { false, true } bool; +#define MAX_PATH_SIZE (1024 + 256 + 3) +#define THREAD_NAMELEN 16 +// 64 bit time + nanoseconds +#define TIME_CHARS 32 +#define DBG_LEVEL_NORM 0 +#define DBG_LEVEL_EXT 1 +#define DBG_LEVEL_EXTALL 2 +#define INT_CHAR_LEN 16 + +// utimens support +#define HEADER_TEXT_MTIME "X-Object-Meta-Mtime" +#define HEADER_TEXT_ATIME "X-Object-Meta-Atime" +#define HEADER_TEXT_CTIME "X-Object-Meta-Ctime" +#define HEADER_TEXT_MTIME_DISPLAY "X-Object-Meta-Mtime-Display" +#define HEADER_TEXT_ATIME_DISPLAY "X-Object-Meta-Atime-Display" +#define HEADER_TEXT_CTIME_DISPLAY "X-Object-Meta-Ctime-Display" +#define HEADER_TEXT_CHMOD "X-Object-Meta-Chmod" +#define HEADER_TEXT_UID "X-Object-Meta-Uid" +#define HEADER_TEXT_GID "X-Object-Meta-Gid" +#define HEADER_TEXT_FILEPATH "X-Object-Meta-FilePath" +#define TEMP_FILE_NAME_FORMAT "%s/.cloudfuse_%s" +#define HUBIC_DATE_FORMAT "%Y-%m-%d %T." + +#define KNRM "\x1B[0m" +#define KRED "\x1B[31m" +#define KGRN "\x1B[32m" +#define KYEL "\x1B[33m" +#define KBLU "\x1B[34m" +#define KMAG "\x1B[35m" +#define KCYN "\x1B[36m" +#define KWHT "\x1B[37m" + +#define min(x, y) ({ \ + typeof(x) _min1 = (x); \ + typeof(y) _min2 = (y); \ + (void)(&_min1 == &_min2); \ + _min1 < _min2 ? _min1 : _min2; }) + +//linked list with files in a directory +typedef struct dir_entry +{ + char *name; + char *full_name; + char *content_type; + off_t size; + time_t last_modified; + // implement utimens + struct timespec mtime; + struct timespec ctime; + struct timespec atime; + char *md5sum; //interesting capability for rsync/scrub + mode_t chmod; + uid_t uid; + gid_t gid; + bool issegmented; + time_t accessed_in_cache;//todo: cache support based on access time + bool metadata_downloaded; + // end change + int isdir; + int islink; + struct dir_entry *next; +} dir_entry; + +// linked list with cached folder names +typedef struct dir_cache +{ + char *path; + dir_entry *entries; + time_t cached; + //added cache support based on access time + time_t accessed_in_cache; + bool was_deleted; + //end change + struct dir_cache *next, *prev; +} dir_cache; + +time_t my_timegm(struct tm *tm); +time_t get_time_from_str_as_gmt(char *time_str); +time_t get_time_as_local(time_t time_t_val, char time_str[], int char_buf_size); +int get_time_as_string(time_t time_t_val, long nsec, char *time_str, int time_str_len); +time_t get_time_now(); +int get_timespec_as_str(const struct timespec *times, char *time_str, int time_str_len); + +char *str2md5(const char *str, int length); +void debug_print_descriptor(struct fuse_file_info *info); +int get_safe_cache_file_path(const char *file_path, char *file_path_safe, char *temp_dir); + +dir_entry *init_dir_entry(); +void copy_dir_entry(dir_entry *src, dir_entry *dst); +dir_cache *new_cache(const char *path); +void dir_for(const char *path, char *dir); +void debug_list_cache_content(); +void update_dir_cache(const char *path, off_t size, int isdir, int islink); +dir_entry *path_info(const char *path); +dir_entry *check_path_info(const char *path); +dir_entry * check_parent_folder_for_file(const char *path); +void dir_decache(const char *path); +void cloudfs_free_dir_list(dir_entry *dir_list); +extern int cloudfs_list_directory(const char *path, dir_entry **); +int caching_list_directory(const char *path, dir_entry **list); + +char *get_home_dir(); +void cloudfs_debug(int dbg); +void debugf(int level, char *fmt, ...); + + +#endif From 1d520faf353c9978a51f5145b161c70a92a85041 Mon Sep 17 00:00:00 2001 From: dcristian Date: Sat, 28 Nov 2015 01:24:10 +0200 Subject: [PATCH 02/12] completed chmod / chown attrib. init fix: proper init of extended attributes on file create. --- README | 2 ++ cloudfsapi.c | 83 +++++++++++++++++++++++++++++++--------------------- cloudfsapi.h | 1 + cloudfuse.c | 13 +++++++- commonfs.c | 9 ++++++ 5 files changed, 73 insertions(+), 35 deletions(-) diff --git a/README b/README index 92f7d76..dcdd2e5 100644 --- a/README +++ b/README @@ -44,6 +44,8 @@ USE: curl_verbose=[true/false, enable verbose output for curl HTTP requests] cache_statfs_timeout=[value in seconds, large timeout increases the file access speed] debug_level=[0 default, 1 extremely verbose for debugging purposes] + enable_chmod=[true/false, false by default, still experimental feature] + enable_chown=[true/false, false by default, still experimental feature] client_id, client_secret can be retrieved from the hubic web interface: https://hubic.com/home/browser/developers/ diff --git a/cloudfsapi.c b/cloudfsapi.c index fc5d42b..59ca763 100644 --- a/cloudfsapi.c +++ b/cloudfsapi.c @@ -170,7 +170,7 @@ static size_t header_dispatch(void *ptr, size_t size, size_t nmemb, void *dir_en return size * nmemb; } -static void header_set_time_from_str(char *time_str, struct timespec *time_entry){ +static void header_set_time_from_str(char *time_str, struct timespec *time_entry) { char sec_value[TIME_CHARS]; char nsec_value[TIME_CHARS]; time_t sec; @@ -195,7 +195,7 @@ static void header_set_time_from_str(char *time_str, struct timespec *time_entry } } -static size_t header_get_utimens_dispatch(void *ptr, size_t size, size_t nmemb, void *userdata) +static size_t header_get_meta_dispatch(void *ptr, size_t size, size_t nmemb, void *userdata) { char *header = (char *)alloca(size * nmemb + 1); char *head = (char *)alloca(size * nmemb + 1); @@ -217,6 +217,15 @@ static size_t header_get_utimens_dispatch(void *ptr, size_t size, size_t nmemb, if (!strncasecmp(head, HEADER_TEXT_MTIME, size * nmemb)) { header_set_time_from_str(value, &de->mtime); } + if (!strncasecmp(head, HEADER_TEXT_CHMOD, size * nmemb)) { + de->chmod = atoi(value); + } + if (!strncasecmp(head, HEADER_TEXT_GID, size * nmemb)) { + de->gid = atoi(value); + } + if (!strncasecmp(head, HEADER_TEXT_UID, size * nmemb)) { + de->uid = atoi(value); + } } else { debugf(DBG_LEVEL_EXT, "Unexpected NULL dir_entry on header(%s), file should be in cache already", storage); @@ -321,7 +330,7 @@ size_t writefunc_callback(void *contents, size_t size, size_t nmemb, void *userp static int send_request_size(const char *method, const char *path, void *fp, xmlParserCtxtPtr xmlctx, curl_slist *extra_headers, off_t file_size, int is_segment, - dir_entry *de_cached_entry) + dir_entry *de_cached_entry, const char *unencoded_path) { debugf(DBG_LEVEL_EXT, "send_request_size(%s) (%s)", method, path); char url[MAX_URL_SIZE]; @@ -334,16 +343,14 @@ static int send_request_size(const char *method, const char *path, void *fp, //needed to keep the response data, for debug purposes struct MemoryStruct chunk; - - if (!storage_url[0]) - { + if (!storage_url[0]) { debugf(DBG_LEVEL_NORM, KRED"send_request with no storage_url?"); abort(); } + //char *encoded_path = curl_escape(path, 0); - while ((slash = strstr(path, "%2F")) || (slash = strstr(path, "%2f"))) - { + while ((slash = strstr(path, "%2F")) || (slash = strstr(path, "%2f"))) { *slash = '/'; memmove(slash+1, slash+3, strlen(slash+3)+1); } @@ -353,8 +360,7 @@ static int send_request_size(const char *method, const char *path, void *fp, snprintf(orig_path, sizeof(orig_path), "/%s", path); // retry on failures - for (tries = 0; tries < REQUEST_RETRIES; tries++) - { + for (tries = 0; tries < REQUEST_RETRIES; tries++) { chunk.memory = malloc(1); /* will be grown as needed by the realloc above */ chunk.size = 0; /* no data at this point */ CURL *curl = get_connection(path); @@ -373,7 +379,7 @@ static int send_request_size(const char *method, const char *path, void *fp, add_header(&headers, "X-Auth-Token", storage_token); dir_entry *de; if (de_cached_entry == NULL) { - de = check_path_info(orig_path); + de = check_path_info(unencoded_path); } else { // updating metadata on a file to be added to cache @@ -381,7 +387,7 @@ static int send_request_size(const char *method, const char *path, void *fp, debugf(DBG_LEVEL_EXTALL, "send_request_size: using param dir_entry(%s)", orig_path); } if (!de) { - debugf(DBG_LEVEL_EXTALL, "send_request_size: file not in cache (%s)", orig_path); + debugf(DBG_LEVEL_EXTALL, "send_request_size: "KYEL"file not in cache (%s)(%s)(%s)", orig_path, path, unencoded_path); } else { // add headers to save utimens attribs only on upload @@ -445,7 +451,7 @@ static int send_request_size(const char *method, const char *path, void *fp, else if (!strcasecmp(method, "PUT")) { //http://blog.chmouel.com/2012/02/06/anatomy-of-a-swift-put-query-to-object-server/ - debugf(DBG_LEVEL_EXT, "send_request_size: PUT w/o FP(%s)", orig_path); + debugf(DBG_LEVEL_EXT, "send_request_size: PUT (%s)", orig_path); curl_easy_setopt(curl, CURLOPT_UPLOAD, 1); if (fp) { curl_easy_setopt(curl, CURLOPT_INFILESIZE, file_size); @@ -474,7 +480,7 @@ static int send_request_size(const char *method, const char *path, void *fp, { if (is_segment) { - debugf(DBG_LEVEL_EXT, "GET SEGMENT (%s)", orig_path); + debugf(DBG_LEVEL_EXT, "send_request_size: GET SEGMENT (%s)", orig_path); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); } @@ -489,7 +495,7 @@ static int send_request_size(const char *method, const char *path, void *fp, abort(); } curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); - curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, &header_get_utimens_dispatch); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, &header_get_meta_dispatch); // sample by UThreadCurl.cpp, https://bitbucket.org/pamungkas5/bcbcurl/src // and http://www.codeproject.com/Articles/838366/BCBCurl-a-LibCurl-based-download-manager curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void *)de); @@ -509,7 +515,7 @@ static int send_request_size(const char *method, const char *path, void *fp, else { //asumming retrieval of headers only debugf(DBG_LEVEL_EXT, "send_request_size: GET HEADERS only(%s)"); - curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, &header_get_utimens_dispatch); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, &header_get_meta_dispatch); curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void *)de); curl_easy_setopt(curl, CURLOPT_NOBODY, 1); } @@ -583,8 +589,8 @@ static int send_request_size(const char *method, const char *path, void *fp, return response; } -static int send_request(char *method, const char *path, FILE *fp, - xmlParserCtxtPtr xmlctx, curl_slist *extra_headers, dir_entry *de_cached_entry) +int send_request(char *method, const char *path, FILE *fp, + xmlParserCtxtPtr xmlctx, curl_slist *extra_headers, dir_entry *de_cached_entry, const char *unencoded_path) { long flen = 0; @@ -594,7 +600,7 @@ static int send_request(char *method, const char *path, FILE *fp, flen = cloudfs_file_size(fileno(fp)); } - return send_request_size(method, path, fp, xmlctx, extra_headers, flen, 0, de_cached_entry); + return send_request_size(method, path, fp, xmlctx, extra_headers, flen, 0, de_cached_entry, unencoded_path); } @@ -616,7 +622,7 @@ void *upload_segment(void *seginfo) // int response = send_request_size(info->method, encoded, info, NULL, NULL, - info->size, 1, NULL); + info->size, 1, NULL, seg_path); if (!(response >= 200 && response < 300)) fprintf(stderr, "Segment upload %s failed with response %d", seg_path, @@ -975,7 +981,8 @@ int cloudfs_object_read_fp(const char *path, FILE *fp) add_header(&headers, "Content-Length", "0"); add_header(&headers, "Content-Type", filemimetype); - int response = send_request_size("PUT", encoded, NULL, NULL, headers, 0, 0, NULL); + //if path is encoded cache entry will not be found + int response = send_request_size("PUT", encoded, NULL, NULL, headers, 0, 0, NULL, path); curl_slist_free_all(headers); curl_free(encoded); @@ -997,8 +1004,8 @@ int cloudfs_object_read_fp(const char *path, FILE *fp) debugf(DBG_LEVEL_EXT, "cloudfs_object_read_fp(%s) found in cache", path); } - // end changes - int response = send_request("PUT", encoded, fp, NULL, NULL, NULL); + //if path is encoded cache entry will not be found + int response = send_request("PUT", encoded, fp, NULL, NULL, NULL, path); curl_free(encoded); debugf(DBG_LEVEL_EXT, "exit 1: cloudfs_object_read_fp(%s)", path); return (response >= 200 && response < 300); @@ -1035,7 +1042,7 @@ int cloudfs_object_write_fp(const char *path, FILE *fp) return 1; } - int response = send_request("GET", encoded, fp, NULL, NULL, NULL); + int response = send_request("GET", encoded, fp, NULL, NULL, NULL, path); curl_free(encoded); fflush(fp); if ((response >= 200 && response < 300) || ftruncate(fileno(fp), 0)) { @@ -1054,18 +1061,19 @@ int cloudfs_object_truncate(const char *path, off_t size) if (size == 0) { FILE *fp = fopen("/dev/null", "r"); - response = send_request("PUT", encoded, fp, NULL, NULL, NULL); + response = send_request("PUT", encoded, fp, NULL, NULL, NULL, path); fclose(fp); } else {//TODO: this is busted - response = send_request("GET", encoded, NULL, NULL, NULL, NULL); + response = send_request("GET", encoded, NULL, NULL, NULL, NULL, path); } curl_free(encoded); return (response >= 200 && response < 300); } //get metadata from cloud, like time attribs. create new entry if not cached yet. +//todo: not thread-safe? void get_file_metadata(dir_entry *de){ if (de->size == 0 && !de->isdir && !de->metadata_downloaded){ //this can be a potential segmented file, try to read segments size @@ -1087,7 +1095,7 @@ void get_file_metadata(dir_entry *de){ //retrieve additional file metadata with a quick HEAD query char *encoded = curl_escape(de->full_name, 0); de->metadata_downloaded = true; - int response = send_request("GET", encoded, NULL, NULL, NULL, de); + int response = send_request("GET", encoded, NULL, NULL, NULL, de, de->full_name); curl_free(encoded); debugf(DBG_LEVEL_EXT, KCYN "exit: get_file_metadata(%s)", de->full_name); } @@ -1142,7 +1150,7 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) response = 404; else{ // this was generating 404 err on non segmented files (small files) - response = send_request("GET", container, NULL, xmlctx, NULL, NULL); + response = send_request("GET", container, NULL, xmlctx, NULL, NULL, path); } if (response >= 200 && response < 300) @@ -1299,7 +1307,7 @@ int cloudfs_delete_object(const char *path) for (i = 0; i < segments; i++) { snprintf(seg_path, MAX_URL_SIZE, "%s%08i", seg_base, i); char *encoded = curl_escape(seg_path, 0); - response = send_request("DELETE", encoded, NULL, NULL, NULL, NULL); + response = send_request("DELETE", encoded, NULL, NULL, NULL, NULL, seg_path); if (response < 200 || response >= 300) { debugf(DBG_LEVEL_EXT, "exit 1: cloudfs_delete_object(%s) response=%d", path, response); return 0; @@ -1308,7 +1316,7 @@ int cloudfs_delete_object(const char *path) } char *encoded = curl_escape(path, 0); - int response = send_request("DELETE", encoded, NULL, NULL, NULL, NULL); + int response = send_request("DELETE", encoded, NULL, NULL, NULL, NULL, path); curl_free(encoded); int ret = (response >= 200 && response < 300); debugf(DBG_LEVEL_EXT, "status: cloudfs_delete_object(%s) response=%d", path, response); @@ -1348,7 +1356,7 @@ int cloudfs_copy_object(const char *src, const char *dst) debugf(DBG_LEVEL_NORM, KRED"status cloudfs_copy_object(%s, %s): src file NOT found", src, dst); } //pass src metadata so that PUT will set time attributes of the src file - int response = send_request("PUT", dst_encoded, NULL, NULL, headers, de_src); + int response = send_request("PUT", dst_encoded, NULL, NULL, headers, de_src, dst); curl_free(dst_encoded); curl_free(src_encoded); curl_slist_free_all(headers); @@ -1356,13 +1364,20 @@ int cloudfs_copy_object(const char *src, const char *dst) return (response >= 200 && response < 300); } +// http://developer.openstack.org/api-ref-objectstorage-v1.html#updateObjectMeta +int cloudfs_update_meta(dir_entry *de) { + int response = cloudfs_copy_object(de->full_name, de->full_name); + return response; +} + //optimised with cache int cloudfs_statfs(const char *path, struct statvfs *stat) { time_t now = get_time_now(); int lapsed = now - last_stat_read_time; if (lapsed > option_cache_statfs_timeout) { - int response = send_request("HEAD", "/", NULL, NULL, NULL, NULL); + //todo: check why stat head request is always set to /, why not path? + int response = send_request("HEAD", "/", NULL, NULL, NULL, NULL, "/"); *stat = statcache; debugf(DBG_LEVEL_EXT, "exit: cloudfs_statfs (new recent values, was cached since %d seconds)", lapsed); last_stat_read_time = now; @@ -1382,7 +1397,7 @@ int cloudfs_create_symlink(const char *src, const char *dst) fwrite(src, 1, strlen(src), lnk); fwrite("\0", 1, 1, lnk); - int response = send_request("MKLINK", dst_encoded, lnk, NULL, NULL, NULL); + int response = send_request("MKLINK", dst_encoded, lnk, NULL, NULL, NULL, dst); curl_free(dst_encoded); fclose(lnk); return (response >= 200 && response < 300); @@ -1392,7 +1407,7 @@ int cloudfs_create_directory(const char *path) { debugf(DBG_LEVEL_EXT, "cloudfs_create_directory(%s)", path); char *encoded = curl_escape(path, 0); - int response = send_request("MKDIR", encoded, NULL, NULL, NULL, NULL); + int response = send_request("MKDIR", encoded, NULL, NULL, NULL, NULL, path); curl_free(encoded); debugf(DBG_LEVEL_EXT, "cloudfs_create_directory(%s) response=%d", path, response); return (response >= 200 && response < 300); diff --git a/cloudfsapi.h b/cloudfsapi.h index 00db6ac..01cabe5 100644 --- a/cloudfsapi.h +++ b/cloudfsapi.h @@ -82,4 +82,5 @@ void cloudfs_verify_ssl(int dbg); void cloudfs_option_get_extended_metadata(int option); void cloudfs_option_curl_verbose(int option); void get_file_metadata(dir_entry *de); +int cloudfs_update_meta(dir_entry *de); #endif diff --git a/cloudfuse.c b/cloudfuse.c index 132af26..3860c1b 100644 --- a/cloudfuse.c +++ b/cloudfuse.c @@ -235,7 +235,15 @@ static int cfs_create(const char *path, mode_t mode, struct fuse_file_info *info debugf(DBG_LEVEL_EXT, KCYN"cfs_create: mtime=[%s]", time_str); get_timespec_as_str(&(de->ctime), time_str, sizeof(time_str)); debugf(DBG_LEVEL_EXT, KCYN"cfs_create: ctime=[%s]", time_str); + + //set chmod & chown + de->chmod = mode; + de->uid = geteuid(); + de->gid = getegid(); } + else { + debugf(DBG_LEVEL_EXT, KBLU "cfs_create(%s) "KYEL"dir-entry not found", path); + } debugf(DBG_LEVEL_NORM, KBLU "exit 2: cfs_create(%s) result=%d:%s", path, errsv, strerror(errsv)); return 0; } @@ -321,7 +329,6 @@ static int cfs_open(const char *path, struct fuse_file_info *info) } } - update_dir_cache(path, (de ? de->size : 0), 0, 0); openfile *of = (openfile *)malloc(sizeof(openfile)); of->fd = dup(fileno(temp_file)); @@ -486,6 +493,8 @@ static int cfs_chown(const char *path, uid_t uid, gid_t gid) debugf(DBG_LEVEL_NORM, "cfs_chown(%s): change from uid:gid %d:%d to %d:%d", path, de->uid, de->gid, uid, gid); de->uid = uid; de->gid = gid; + //todo: issue a PUT request to update metadata (empty request just to update headers?) + int response = cloudfs_update_meta(de); } } return 0; @@ -499,6 +508,8 @@ static int cfs_chmod(const char *path, mode_t mode) if (de->chmod != mode) { debugf(DBG_LEVEL_NORM, "cfs_chmod(%s): change mode from %d to %d", path, de->chmod, mode); de->chmod = mode; + //todo: issue a PUT request to update metadata (empty request just to update headers?) + int response = cloudfs_update_meta(de); } } return 0; diff --git a/commonfs.c b/commonfs.c index ddfb156..dcfe9f0 100644 --- a/commonfs.c +++ b/commonfs.c @@ -19,6 +19,8 @@ #include #include #include +#include +#include #include "commonfs.h" #include "config.h" @@ -553,6 +555,13 @@ dir_entry *check_path_info(const char *path) debugf(DBG_LEVEL_EXT, "exit 1: check_path_info(%s) "KGRN"[CACHE-HIT]", path); return tmp; } + + /*char *encoded = curl_escape(tmp->full_name, 0); + debugf(DBG_LEVEL_EXT, "check_path_info(%s) check[%s]=[%s]", path, encoded, path); + if (!strcmp(encoded, path)) { + debugf(DBG_LEVEL_EXT, "exit 1: check_path_info(%s) "KGRN"[CACHE-HIT-"KRED"ENCODED]", path); + return tmp; + }*/ } if (!strcmp(path, "/")) { debugf(DBG_LEVEL_EXT, "exit 2: check_path_info(%s) "KYEL "ignoring root [CACHE-MISS]", path); From aaf0f2478b8622d54e41b0714d0c91d1d71b275f Mon Sep 17 00:00:00 2001 From: dcristian Date: Sat, 28 Nov 2015 15:50:27 +0200 Subject: [PATCH 03/12] optimised upload to cloud. File is not uploaded on flush if content did not changed (compute md5 for file content), only metadata is saved. --- cloudfsapi.c | 22 +++++++++---------- cloudfsapi.h | 2 +- cloudfuse.c | 61 ++++++++++++++++++++++++++++++++++++++-------------- commonfs.c | 50 ++++++++++++++++++++++++++++++------------ commonfs.h | 1 + 5 files changed, 94 insertions(+), 42 deletions(-) diff --git a/cloudfsapi.c b/cloudfsapi.c index 59ca763..3463e9c 100644 --- a/cloudfsapi.c +++ b/cloudfsapi.c @@ -178,20 +178,20 @@ static void header_set_time_from_str(char *time_str, struct timespec *time_entry sscanf(time_str, "%[^.].%[^\n]", sec_value, nsec_value); sec = strtoll(sec_value, NULL, 10);//to allow for larger numbers nsec = atol(nsec_value); - debugf(DBG_LEVEL_EXT, "Received time=%s.%s / %li.%li, existing=%li.%li", + debugf(DBG_LEVEL_EXTALL, "Received time=%s.%s / %li.%li, existing=%li.%li", sec_value, nsec_value, sec, nsec, time_entry->tv_sec, time_entry->tv_nsec); if (sec != time_entry->tv_sec || nsec != time_entry->tv_nsec){ - debugf(DBG_LEVEL_EXT, "Time changed, setting new time=%li.%li, existing was=%li.%li", + debugf(DBG_LEVEL_EXTALL, "Time changed, setting new time=%li.%li, existing was=%li.%li", sec, nsec, time_entry->tv_sec, time_entry->tv_nsec); time_entry->tv_sec = sec; time_entry->tv_nsec = nsec; char time_str_local[TIME_CHARS] = ""; get_time_as_string((time_t)sec, nsec, time_str_local, sizeof(time_str_local)); - debugf(DBG_LEVEL_EXT, "header_set_time_from_str received time=[%s]", time_str_local); + debugf(DBG_LEVEL_EXTALL, "header_set_time_from_str received time=[%s]", time_str_local); get_timespec_as_str(time_entry, time_str_local, sizeof(time_str_local)); - debugf(DBG_LEVEL_EXT, "header_set_time_from_str set time=[%s]", time_str_local); + debugf(DBG_LEVEL_EXTALL, "header_set_time_from_str set time=[%s]", time_str_local); } } @@ -370,7 +370,7 @@ static int send_request_size(const char *method, const char *path, void *fp, curl_easy_setopt(curl, CURLOPT_URL, url); curl_easy_setopt(curl, CURLOPT_HEADER, 0); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); - curl_easy_setopt(curl, CURLOPT_NOPROGRESS, option_curl_progress_state);//0=to enable progress + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, option_curl_progress_state ? 0 : 1);//reversed logic, 0=to enable progress curl_easy_setopt(curl, CURLOPT_USERAGENT, USER_AGENT); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, verify_ssl ? 1 : 0); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, verify_ssl); @@ -392,18 +392,18 @@ static int send_request_size(const char *method, const char *path, void *fp, else { // add headers to save utimens attribs only on upload if (!strcasecmp(method, "PUT") || !strcasecmp(method, "MKDIR")) { - debugf(DBG_LEVEL_EXT, "send_request_size: Saving utimens for file %s", orig_path); + debugf(DBG_LEVEL_EXTALL, "send_request_size: Saving utimens for file %s", orig_path); //debugf(DBG_LEVEL_NORM, "File found in cache, path=%s", de->full_name); - debugf(DBG_LEVEL_EXT, "send_request_size: Cached utime for path=%s ctime=%li.%li mtime=%li.%li atime=%li.%li", orig_path, + debugf(DBG_LEVEL_EXTALL, "send_request_size: Cached utime for path=%s ctime=%li.%li mtime=%li.%li atime=%li.%li", orig_path, de->ctime.tv_sec, de->ctime.tv_nsec, de->mtime.tv_sec, de->mtime.tv_nsec, de->atime.tv_sec, de->atime.tv_nsec); char atime_str_nice[TIME_CHARS] = "", mtime_str_nice[TIME_CHARS] = "", ctime_str_nice[TIME_CHARS] = ""; get_timespec_as_str(&(de->atime), atime_str_nice, sizeof(atime_str_nice)); - debugf(DBG_LEVEL_EXT, KCYN"send_request_size: atime=[%s]", atime_str_nice); + debugf(DBG_LEVEL_EXTALL, KCYN"send_request_size: atime=[%s]", atime_str_nice); get_timespec_as_str(&(de->mtime), mtime_str_nice, sizeof(mtime_str_nice)); - debugf(DBG_LEVEL_EXT, KCYN"send_request_size: mtime=[%s]", mtime_str_nice); + debugf(DBG_LEVEL_EXTALL, KCYN"send_request_size: mtime=[%s]", mtime_str_nice); get_timespec_as_str(&(de->ctime), ctime_str_nice, sizeof(ctime_str_nice)); - debugf(DBG_LEVEL_EXT, KCYN"send_request_size: ctime=[%s]", ctime_str_nice); + debugf(DBG_LEVEL_EXTALL, KCYN"send_request_size: ctime=[%s]", ctime_str_nice); char mtime_str[TIME_CHARS], atime_str[TIME_CHARS], ctime_str[TIME_CHARS]; char string_float[TIME_CHARS]; @@ -991,7 +991,7 @@ int cloudfs_object_read_fp(const char *path, FILE *fp) } else{ // assume enters here when file is composed of only one segment (small files) - debugf(DBG_LEVEL_EXT, KRED"cloudfs_object_read_fp(%s) unknown state", path); + debugf(DBG_LEVEL_EXT, "cloudfs_object_read_fp(%s) "KYEL"unknown state", path); } rewind(fp); diff --git a/cloudfsapi.h b/cloudfsapi.h index 01cabe5..11f3bd3 100644 --- a/cloudfsapi.h +++ b/cloudfsapi.h @@ -15,7 +15,7 @@ typedef struct curl_slist curl_slist; -#define MINIMAL_PROGRESS_FUNCTIONALITY_INTERVAL 3 +#define MINIMAL_PROGRESS_FUNCTIONALITY_INTERVAL 5 struct curl_progress { double lastruntime; CURL *curl; diff --git a/cloudfuse.c b/cloudfuse.c index 3860c1b..ad1b224 100644 --- a/cloudfuse.c +++ b/cloudfuse.c @@ -14,6 +14,7 @@ #include #include #include +#include #include "commonfs.h" #include "cloudfsapi.h" #include "config.h" @@ -25,9 +26,10 @@ extern int cache_timeout; extern int option_cache_statfs_timeout; extern int option_debug_level; extern bool option_get_extended_metadata; -extern int option_curl_progress_state; +extern bool option_curl_progress_state; extern bool option_enable_chown; extern bool option_enable_chmod; +extern size_t file_buffer_size; typedef struct { @@ -351,13 +353,16 @@ static int cfs_open(const char *path, struct fuse_file_info *info) static int cfs_read(const char *path, char *buf, size_t size, off_t offset, struct fuse_file_info *info) { - debugf(DBG_LEVEL_EXTALL, KBLU "cfs_read(%s)", path); + debugf(DBG_LEVEL_EXTALL, KBLU "cfs_read(%s) buffsize=%lu offset=%lu", path, size, offset); + file_buffer_size = size; debug_print_descriptor(info); int result = pread(((openfile *)(uintptr_t)info->fh)->fd, buf, size, offset); debugf(DBG_LEVEL_EXTALL, KBLU "exit: cfs_read(%s) result=%s", path, strerror(errno)); return result; } +//todo: flush will upload a file again even if just file attributes are changed. +//optimisation needed to detect if content is changed and to only save meta when just attribs are modified. static int cfs_flush(const char *path, struct fuse_file_info *info) { debugf(DBG_LEVEL_NORM, KBLU "cfs_flush(%s)", path); @@ -372,12 +377,25 @@ static int cfs_flush(const char *path, struct fuse_file_info *info) errsv = errno; if (fp != NULL) { rewind(fp); - if (!cloudfs_object_read_fp(path, fp)) { - fclose(fp); - errsv = errno; - debugf(DBG_LEVEL_NORM, KRED"exit 0: cfs_flush(%s) result=%d:%s", path, errsv, strerror(errno)); - return -ENOENT; - } + //todo: calculate md5 hash, compare with cloud hash to determine if file content is changed + char md5_file_hash_str[MD5_DIGEST_LENGTH + 1] = "\0"; + file_md5(fp, md5_file_hash_str); + dir_entry *de = check_path_info(path); + if (de && !strcasecmp(md5_file_hash_str, de->md5sum)) { + //file content is identical, no need to upload entire file, just update metadata + debugf(DBG_LEVEL_NORM, KBLU "cfs_flush(%s): skip full upload as content did not change", path); + cloudfs_update_meta(de); + } + else { + rewind(fp); + debugf(DBG_LEVEL_NORM, KBLU "cfs_flush(%s): perform full upload as content changed (or no file found in cache)", path); + if (!cloudfs_object_read_fp(path, fp)) { + fclose(fp); + errsv = errno; + debugf(DBG_LEVEL_NORM, KRED"exit 0: cfs_flush(%s) result=%d:%s", path, errsv, strerror(errno)); + return -ENOENT; + } + } fclose(fp); errsv = errno; } @@ -417,6 +435,7 @@ static int cfs_rmdir(const char *path) static int cfs_ftruncate(const char *path, off_t size, struct fuse_file_info *info) { debugf(DBG_LEVEL_NORM, KBLU "cfs_ftruncate(%s)", path); + file_buffer_size = size; openfile *of = (openfile *)(uintptr_t)info->fh; if (ftruncate(of->fd, size)) return -errno; @@ -428,13 +447,18 @@ static int cfs_ftruncate(const char *path, off_t size, struct fuse_file_info *in static int cfs_write(const char *path, const char *buf, size_t length, off_t offset, struct fuse_file_info *info) { - debugf(DBG_LEVEL_EXTALL, KBLU "cfs_write(%s)", path); + debugf(DBG_LEVEL_EXTALL, KBLU "cfs_write(%s) bufflength=%lu offset=%lu", path, length, offset); // FIXME: Potential inconsistent cache update if pwrite fails? update_dir_cache(path, offset + length, 0, 0); //int result = pwrite(info->fh, buf, length, offset); int result = pwrite(((openfile *)(uintptr_t)info->fh)->fd, buf, length, offset); int errsv = errno; - debugf(DBG_LEVEL_EXTALL, KBLU "exit: cfs_write(%s) result=%d:%s", path, errsv, strerror(errsv)); + if (result == 0) { + debugf(DBG_LEVEL_EXTALL, KBLU "exit 0: cfs_write(%s) result=%d:%s", path, errsv, strerror(errsv)); + } + else { + debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_write(%s) "KRED"result=%d:%s", path, errsv, strerror(errsv)); + } return result; } @@ -493,7 +517,7 @@ static int cfs_chown(const char *path, uid_t uid, gid_t gid) debugf(DBG_LEVEL_NORM, "cfs_chown(%s): change from uid:gid %d:%d to %d:%d", path, de->uid, de->gid, uid, gid); de->uid = uid; de->gid = gid; - //todo: issue a PUT request to update metadata (empty request just to update headers?) + //issue a PUT request to update metadata (quick request just to update headers) int response = cloudfs_update_meta(de); } } @@ -616,8 +640,13 @@ static int cfs_utimens(const char *path, const struct timespec times[2]){ debugf(DBG_LEVEL_EXT, KCYN"cfs_utimens: set atime=[%s]", time_str); path_de->atime = times[0]; path_de->mtime = times[1]; - // not sure how to best obtain ctime from fuse. just record current date. + // not sure how to best obtain ctime from fuse source file. just record current date. path_de->ctime = now; + //calling a meta cloud update here is not always needed. + //touch for example opens and closes/flush the file. + //worth implementing a meta cache functionality to avoid multiple uploads on meta changes + //when changing timestamps on very large files, touch command will trigger 2 x long and useless file uploads on cfs_flush() + //cloudfs_update_meta(path_de); } else { debugf(DBG_LEVEL_EXT, KCYN"cfs_utimens: a/m/time not changed"); @@ -656,7 +685,7 @@ ExtraFuseOptions extra_options = { .curl_verbose = "false", .cache_statfs_timeout = 0, .debug_level = 0, - .curl_progress_state = 1, + .curl_progress_state = "false", .enable_chown = "false", .enable_chmod = "false" }; @@ -710,7 +739,7 @@ void initialise_options() { option_cache_statfs_timeout = atoi(extra_options.cache_statfs_timeout); } if (*extra_options.curl_progress_state) { - option_curl_progress_state = atoi(extra_options.curl_progress_state); + option_curl_progress_state = !strcasecmp(extra_options.curl_progress_state, "true"); } if (*extra_options.enable_chmod) { option_enable_chmod = !strcasecmp(extra_options.enable_chmod, "true"); @@ -767,9 +796,9 @@ int main(int argc, char **argv) fprintf(stderr, " get_extended_metadata=[true to enable download of utime, chmod, chown file attributes (but slower)]\n"); fprintf(stderr, " curl_verbose=[true to debug info on curl requests (lots of output)]\n"); - fprintf(stderr, " curl_progress_state=[0 or 1, 0 for progress callback enabled, 1 for disabled. Mostly used for debugging]\n"); + fprintf(stderr, " curl_progress_state=[true to enable progress callback enabled. Mostly used for debugging]\n"); fprintf(stderr, " cache_statfs_timeout=[number of seconds to cache requests to statfs (cloud statistics), 0 for no cache]\n"); - fprintf(stderr, " debug_level=[0 to n, 0 for minimal verbose debugging. No debug if -d or -f option is not provided.]\n"); + fprintf(stderr, " debug_level=[0 to 3, 0 for minimal verbose debugging. No debug if -d or -f option is not provided.]\n"); fprintf(stderr, " enable_chmod=[true to enable chmod support on fuse]\n"); fprintf(stderr, " enable_chown=[true to enable chown support on fuse]\n"); diff --git a/commonfs.c b/commonfs.c index dcfe9f0..27d6323 100644 --- a/commonfs.c +++ b/commonfs.c @@ -38,6 +38,7 @@ int option_debug_level = 0; int option_curl_progress_state = 1;//1 to disable curl progress bool option_enable_chown = false; bool option_enable_chmod = false; +size_t file_buffer_size = 0; // needed to get correct GMT / local time, as it does not work // http://zhu-qy.blogspot.ro/2012/11/ref-how-to-convert-from-utc-to-local.html @@ -144,6 +145,34 @@ char *str2md5(const char *str, int length) { return out; } +// http://stackoverflow.com/questions/10324611/how-to-calculate-the-md5-hash-of-a-large-file-in-c +int file_md5(FILE *file_handle, char *md5_file_str) { + if (file_handle == NULL) { + debugf(DBG_LEVEL_NORM, KRED"file_md5: NULL file handle"); + return 0; + } + if (file_buffer_size == 0) { + debugf(DBG_LEVEL_NORM, KYEL"file_md5: 0 file buffer size, using default"); + file_buffer_size = 1024; + } + unsigned char c[MD5_DIGEST_LENGTH]; + int i; + MD5_CTX mdContext; + int bytes; + char mdchar[3];//2 chars for md5 + null string terminator + unsigned char *data_buf = malloc(file_buffer_size * sizeof(unsigned char)); + MD5_Init(&mdContext); + while ((bytes = fread(data_buf, 1, file_buffer_size, file_handle)) != 0) + MD5_Update(&mdContext, data_buf, bytes); + MD5_Final(c, &mdContext); + for (i = 0; i < MD5_DIGEST_LENGTH; i++) { + snprintf(mdchar, 3, "%02x", c[i]); + strcat(md5_file_str, mdchar); + //fprintf(stderr, "%02x", c[i]); + } + free(data_buf); + return 0; +} int get_safe_cache_file_path(const char *path, char *file_path_safe, char *temp_dir) { char tmp_path[PATH_MAX]; @@ -181,13 +210,13 @@ void get_file_path_from_fd(int fd, char *path, int size_path) { void debug_print_flags(int flags) { int accmode, val; accmode = flags & O_ACCMODE; - if (accmode == O_RDONLY) debugf(DBG_LEVEL_EXT, KRED "read only"); - else if (accmode == O_WRONLY) debugf(DBG_LEVEL_EXT, KRED "write only"); - else if (accmode == O_RDWR) debugf(DBG_LEVEL_EXT, KRED "read write"); - else debugf(DBG_LEVEL_EXT, KRED "unknown access mode"); + if (accmode == O_RDONLY) debugf(DBG_LEVEL_EXTALL, KYEL"read only"); + else if (accmode == O_WRONLY) debugf(DBG_LEVEL_EXTALL, KYEL"write only"); + else if (accmode == O_RDWR) debugf(DBG_LEVEL_EXTALL, KYEL"read write"); + else debugf(DBG_LEVEL_EXT, KYEL"unknown access mode"); - if (val & O_APPEND) debugf(DBG_LEVEL_EXT, KRED ", append"); - if (val & O_NONBLOCK) debugf(DBG_LEVEL_EXT, KRED ", nonblocking"); + if (val & O_APPEND) debugf(DBG_LEVEL_EXTALL, KYEL", append"); + if (val & O_NONBLOCK) debugf(DBG_LEVEL_EXTALL, KYEL", nonblocking"); #if !defined(_POSIX_SOURCE) && defined(O_SYNC) if (val & O_SYNC) debugf(DBG_LEVEL_EXT, 0,KRED ", synchronous writes"); #endif @@ -555,16 +584,9 @@ dir_entry *check_path_info(const char *path) debugf(DBG_LEVEL_EXT, "exit 1: check_path_info(%s) "KGRN"[CACHE-HIT]", path); return tmp; } - - /*char *encoded = curl_escape(tmp->full_name, 0); - debugf(DBG_LEVEL_EXT, "check_path_info(%s) check[%s]=[%s]", path, encoded, path); - if (!strcmp(encoded, path)) { - debugf(DBG_LEVEL_EXT, "exit 1: check_path_info(%s) "KGRN"[CACHE-HIT-"KRED"ENCODED]", path); - return tmp; - }*/ } if (!strcmp(path, "/")) { - debugf(DBG_LEVEL_EXT, "exit 2: check_path_info(%s) "KYEL "ignoring root [CACHE-MISS]", path); + debugf(DBG_LEVEL_EXT, "exit 2: check_path_info(%s) "KYEL"ignoring root [CACHE-MISS]", path); } else { debugf(DBG_LEVEL_EXT, "exit 3: check_path_info(%s) "KYEL"[CACHE-MISS]", path); diff --git a/commonfs.h b/commonfs.h index ca5b47d..db0729d 100644 --- a/commonfs.h +++ b/commonfs.h @@ -87,6 +87,7 @@ time_t get_time_now(); int get_timespec_as_str(const struct timespec *times, char *time_str, int time_str_len); char *str2md5(const char *str, int length); +int file_md5(FILE *file_handle, char *md5_file_str); void debug_print_descriptor(struct fuse_file_info *info); int get_safe_cache_file_path(const char *file_path, char *file_path_safe, char *temp_dir); From 0c6100469a786cb6b91975329cebd32510833fb9 Mon Sep 17 00:00:00 2001 From: dcristian Date: Sat, 28 Nov 2015 16:26:52 +0200 Subject: [PATCH 04/12] segfault fix for md5sum --- cloudfuse.c | 2 +- commonfs.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudfuse.c b/cloudfuse.c index ad1b224..cef2c17 100644 --- a/cloudfuse.c +++ b/cloudfuse.c @@ -381,7 +381,7 @@ static int cfs_flush(const char *path, struct fuse_file_info *info) char md5_file_hash_str[MD5_DIGEST_LENGTH + 1] = "\0"; file_md5(fp, md5_file_hash_str); dir_entry *de = check_path_info(path); - if (de && !strcasecmp(md5_file_hash_str, de->md5sum)) { + if (de && (!strcasecmp(md5_file_hash_str, de->md5sum))) { //file content is identical, no need to upload entire file, just update metadata debugf(DBG_LEVEL_NORM, KBLU "cfs_flush(%s): skip full upload as content did not change", path); cloudfs_update_meta(de); diff --git a/commonfs.c b/commonfs.c index 27d6323..5818393 100644 --- a/commonfs.c +++ b/commonfs.c @@ -358,7 +358,7 @@ dir_entry* init_dir_entry() { de->metadata_downloaded = false; de->size = 0; de->next = NULL; - de->md5sum = NULL; + de->md5sum = "\0"; de->accessed_in_cache = time(NULL); de->last_modified = time(NULL); de->mtime.tv_sec = time(NULL); From 0c074844e7b200e8518b0a415714391e86665787 Mon Sep 17 00:00:00 2001 From: dcristian Date: Sat, 28 Nov 2015 16:59:25 +0200 Subject: [PATCH 05/12] proper md5sum segfault fix --- cloudfuse.c | 16 ++++++++++++++-- commonfs.c | 2 +- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/cloudfuse.c b/cloudfuse.c index cef2c17..e4edbf8 100644 --- a/cloudfuse.c +++ b/cloudfuse.c @@ -252,6 +252,7 @@ static int cfs_create(const char *path, mode_t mode, struct fuse_file_info *info // open (download) file from cloud +// todo: implement etag optimisation, download only if content changed, http://www.17od.com/2012/12/19/ten-useful-openstack-swift-features/ static int cfs_open(const char *path, struct fuse_file_info *info) { debugf(DBG_LEVEL_NORM, KBLU "cfs_open(%s)", path); @@ -381,7 +382,7 @@ static int cfs_flush(const char *path, struct fuse_file_info *info) char md5_file_hash_str[MD5_DIGEST_LENGTH + 1] = "\0"; file_md5(fp, md5_file_hash_str); dir_entry *de = check_path_info(path); - if (de && (!strcasecmp(md5_file_hash_str, de->md5sum))) { + if (de && de->md5sum != NULL && (!strcasecmp(md5_file_hash_str, de->md5sum))) { //file content is identical, no need to upload entire file, just update metadata debugf(DBG_LEVEL_NORM, KBLU "cfs_flush(%s): skip full upload as content did not change", path); cloudfs_update_meta(de); @@ -453,7 +454,7 @@ static int cfs_write(const char *path, const char *buf, size_t length, off_t off //int result = pwrite(info->fh, buf, length, offset); int result = pwrite(((openfile *)(uintptr_t)info->fh)->fd, buf, length, offset); int errsv = errno; - if (result == 0) { + if (errsv == 0) { debugf(DBG_LEVEL_EXTALL, KBLU "exit 0: cfs_write(%s) result=%d:%s", path, errsv, strerror(errsv)); } else { @@ -665,6 +666,13 @@ int cfs_getxattr(const char *path, const char *name, char *value, size_t size){ return 0; } +int cfs_removexattr(const char *path, const char *name) { + return 0; +} + +int cfs_listxattr(const char *path, char *list, size_t size) { + return 0; +} FuseOptions options = { .cache_timeout = "600", @@ -849,8 +857,12 @@ int main(int argc, char **argv) .readlink = cfs_readlink, .init = cfs_init, .utimens = cfs_utimens, +#ifdef HAVE_SETXATTR .setxattr = cfs_setxattr, .getxattr = cfs_getxattr, + .listxattr = cfs_listxattr, + .removexattr = cfs_removexattr, +#endif }; pthread_mutexattr_init(&mutex_attr); diff --git a/commonfs.c b/commonfs.c index 5818393..27d6323 100644 --- a/commonfs.c +++ b/commonfs.c @@ -358,7 +358,7 @@ dir_entry* init_dir_entry() { de->metadata_downloaded = false; de->size = 0; de->next = NULL; - de->md5sum = "\0"; + de->md5sum = NULL; de->accessed_in_cache = time(NULL); de->last_modified = time(NULL); de->mtime.tv_sec = time(NULL); From 4207c97123ab438cc5438f626f2b33e92c593fc7 Mon Sep 17 00:00:00 2001 From: dcristian Date: Sat, 28 Nov 2015 17:58:36 +0200 Subject: [PATCH 06/12] small tweaks on debugging messages --- README | 4 +++- cloudfuse.c | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/README b/README index dcdd2e5..8c16f8a 100644 --- a/README +++ b/README @@ -42,6 +42,7 @@ USE: Optional variables: get_extended_metadata=[true/false, force download of additional file metadata like atime and mtime on first directory list] curl_verbose=[true/false, enable verbose output for curl HTTP requests] + curl_progress_state=[true/false, enable verbose progress output for curl HTTP requests. Used for debugging.] cache_statfs_timeout=[value in seconds, large timeout increases the file access speed] debug_level=[0 default, 1 extremely verbose for debugging purposes] enable_chmod=[true/false, false by default, still experimental feature] @@ -102,7 +103,8 @@ Recent additions/fixes (from https://github.com/dan-cristian): * Cached files are deleted on cache expiration when using a custom temp folder. * Files copied have attributes preserved. * Working well with rsync due to mtime support and proper copy operations. - * Support for http progress (track upload / download speed etc.) + * Debugging support for http progress (track upload / download speed etc.) + * Reduced traffic, skips file uploads to cloud if content does not change (using md5sum compare) * Major code refactoring, code documentation, extensive debugging, additional config options AWESOME CONTRIBUTORS: diff --git a/cloudfuse.c b/cloudfuse.c index e4edbf8..1f85553 100644 --- a/cloudfuse.c +++ b/cloudfuse.c @@ -191,9 +191,9 @@ static int cfs_create(const char *path, mode_t mode, struct fuse_file_info *info debugf(DBG_LEVEL_NORM, KBLU "cfs_create(%s)", path); FILE *temp_file; int errsv; + char file_path_safe[NAME_MAX] = ""; - if (*temp_dir) { - char file_path_safe[NAME_MAX] = ""; + if (*temp_dir) { get_safe_cache_file_path(path, file_path_safe, temp_dir); temp_file = fopen(file_path_safe, "w+b"); errsv = errno; @@ -246,7 +246,7 @@ static int cfs_create(const char *path, mode_t mode, struct fuse_file_info *info else { debugf(DBG_LEVEL_EXT, KBLU "cfs_create(%s) "KYEL"dir-entry not found", path); } - debugf(DBG_LEVEL_NORM, KBLU "exit 2: cfs_create(%s) result=%d:%s", path, errsv, strerror(errsv)); + debugf(DBG_LEVEL_NORM, KBLU "exit 2: cfs_create(%s)=(%s) result=%d:%s", path, file_path_safe, errsv, strerror(errsv)); return 0; } @@ -458,7 +458,7 @@ static int cfs_write(const char *path, const char *buf, size_t length, off_t off debugf(DBG_LEVEL_EXTALL, KBLU "exit 0: cfs_write(%s) result=%d:%s", path, errsv, strerror(errsv)); } else { - debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_write(%s) "KRED"result=%d:%s", path, errsv, strerror(errsv)); + debugf(DBG_LEVEL_EXTALL, KBLU "exit 1: cfs_write(%s) "KRED"result=%d:%s", path, errsv, strerror(errsv)); } return result; } From 668e0ac24284dfdf30b3ab662dfb3fff89acc6e1 Mon Sep 17 00:00:00 2001 From: dcristian Date: Mon, 30 Nov 2015 03:54:41 +0200 Subject: [PATCH 07/12] consistency / style changes as per merge comments --- .hubicfuse.sample | 4 +- README | 27 +- cloudfsapi.c | 817 ++++++++++++++++++++++------------------------ cloudfsapi.h | 41 ++- cloudfuse.c | 588 +++++++++++++++++---------------- commonfs.c | 623 ++++++++++++++++++----------------- commonfs.h | 16 +- 7 files changed, 1062 insertions(+), 1054 deletions(-) diff --git a/.hubicfuse.sample b/.hubicfuse.sample index 732cf91..ee8dc5b 100644 --- a/.hubicfuse.sample +++ b/.hubicfuse.sample @@ -6,8 +6,8 @@ temp_dir = /tmp/hubicfuse get_extended_metadata = true curl_verbose = false -curl_progress_state = 1 +curl_progress_state = true cache_statfs_timeout = 15 debug_level = 0 enable_chmod = false -enable_chown = false \ No newline at end of file +enable_chown = false diff --git a/README b/README index 8c16f8a..e345d16 100644 --- a/README +++ b/README @@ -92,20 +92,20 @@ BUGS/SHORTCOMINGS: "application/directory". * Cloud Files limits container and object listings to 10,000 items. cloudfuse won't list more than that many files in a single directory. - * File copy progress when uploading does not work, progress is shown when file - is copied in local cache, then upload operation happens at 100% + * File copy progress when uploading does not work, progress is shown when file + is copied in local cache, then upload operation happens at 100% -Recent additions/fixes (from https://github.com/dan-cristian): - * Support for atime, mtime, chmod, chown. - * Large files (segmented) have correct size listed (was 0 before). - * Multiple speed improvements, minimised the number of HTTP calls and added more caching features. - * Fixed many segmentation faults. - * Cached files are deleted on cache expiration when using a custom temp folder. - * Files copied have attributes preserved. - * Working well with rsync due to mtime support and proper copy operations. - * Debugging support for http progress (track upload / download speed etc.) - * Reduced traffic, skips file uploads to cloud if content does not change (using md5sum compare) - * Major code refactoring, code documentation, extensive debugging, additional config options +Recent additions/fixes: + * Support for atime, mtime, chmod, chown. + * Large files (segmented) have correct size listed (was 0 before). + * Multiple speed improvements, minimised the number of HTTP calls and added more caching features. + * Fixed many segmentation faults. + * Cached files are deleted on cache expiration when using a custom temp folder. + * Files copied have attributes preserved. + * Working well with rsync due to mtime support and proper copy operations. + * Debugging support for http progress (track upload / download speed etc.) + * Reduced traffic, skips file uploads to cloud if content does not change (using md5sum compare) + * Major code refactoring, code documentation, extensive debugging, additional config options AWESOME CONTRIBUTORS: @@ -119,6 +119,7 @@ AWESOME CONTRIBUTORS: * Mike Lundy https://github.com/novas0x2a * justinb https://github.com/justinsb * Matt Greenway https://github.com/LabAdvComp + * Dan Cristian https://github.com/dan-cristian Thanks, and I hope you find it useful. diff --git a/cloudfsapi.c b/cloudfsapi.c index 3463e9c..0032db8 100644 --- a/cloudfsapi.c +++ b/cloudfsapi.c @@ -27,11 +27,8 @@ #define RHEL5_LIBCURL_VERSION 462597 #define RHEL5_CERTIFICATE_FILE "/etc/pki/tls/certs/ca-bundle.crt" - #define REQUEST_RETRIES 3 - #define MAX_FILES 10000 - // size of buffer for writing to disk look at ioblksize.h in coreutils // and try some values on your own system if you want the best performance #define DISK_BUFF_SIZE 32768 @@ -50,7 +47,6 @@ extern int option_cache_statfs_timeout; extern bool option_extensive_debug; extern bool option_enable_chown; extern bool option_enable_chmod; - static int rhel5_mode = 0; static struct statvfs statcache = { .f_bsize = 4096, @@ -63,9 +59,9 @@ static struct statvfs statcache = { .f_favail = 0, .f_namemax = INT_MAX }; -static time_t last_stat_read_time = 0;//used to compute cache interval +//used to compute statfs cache interval +static time_t last_stat_read_time = 0; extern FuseOptions options; - struct MemoryStruct { char *memory; size_t size; @@ -88,7 +84,6 @@ static unsigned long thread_id() } #endif - static size_t xml_dispatch(void *ptr, size_t size, size_t nmemb, void *stream) { xmlParseChunk((xmlParserCtxtPtr)stream, (char *)ptr, size * nmemb, 0); @@ -161,7 +156,8 @@ static size_t header_dispatch(void *ptr, size_t size, size_t nmemb, void *dir_en statcache.f_blocks = (unsigned long) (strtoull(value, NULL, 10)/statcache.f_frsize); if (!strncasecmp(head, "x-account-bytes-used", size * nmemb)) statcache.f_bfree = statcache.f_bavail = statcache.f_blocks - (unsigned long) (strtoull(value, NULL, 10)/statcache.f_frsize); - if (!strncasecmp(head, "x-account-object-count", size * nmemb)) { + if (!strncasecmp(head, "x-account-object-count", size * nmemb)) + { unsigned long object_count = strtoul(value, NULL, 10); statcache.f_ffree = MAX_FILES - object_count; statcache.f_favail = MAX_FILES - object_count; @@ -171,27 +167,28 @@ static size_t header_dispatch(void *ptr, size_t size, size_t nmemb, void *dir_en } static void header_set_time_from_str(char *time_str, struct timespec *time_entry) { - char sec_value[TIME_CHARS]; - char nsec_value[TIME_CHARS]; + char sec_value[TIME_CHARS] = { 0 }; + char nsec_value[TIME_CHARS] = { 0 }; time_t sec; long nsec; sscanf(time_str, "%[^.].%[^\n]", sec_value, nsec_value); sec = strtoll(sec_value, NULL, 10);//to allow for larger numbers nsec = atol(nsec_value); debugf(DBG_LEVEL_EXTALL, "Received time=%s.%s / %li.%li, existing=%li.%li", - sec_value, nsec_value, sec, nsec, time_entry->tv_sec, time_entry->tv_nsec); - if (sec != time_entry->tv_sec || nsec != time_entry->tv_nsec){ + sec_value, nsec_value, sec, nsec, time_entry->tv_sec, time_entry->tv_nsec); + if (sec != time_entry->tv_sec || nsec != time_entry->tv_nsec) + { debugf(DBG_LEVEL_EXTALL, "Time changed, setting new time=%li.%li, existing was=%li.%li", - sec, nsec, time_entry->tv_sec, time_entry->tv_nsec); - time_entry->tv_sec = sec; - time_entry->tv_nsec = nsec; + sec, nsec, time_entry->tv_sec, time_entry->tv_nsec); + time_entry->tv_sec = sec; + time_entry->tv_nsec = nsec; - char time_str_local[TIME_CHARS] = ""; - get_time_as_string((time_t)sec, nsec, time_str_local, sizeof(time_str_local)); - debugf(DBG_LEVEL_EXTALL, "header_set_time_from_str received time=[%s]", time_str_local); + char time_str_local[TIME_CHARS] = ""; + get_time_as_string((time_t)sec, nsec, time_str_local, sizeof(time_str_local)); + debugf(DBG_LEVEL_EXTALL, "header_set_time_from_str received time=[%s]", time_str_local); - get_timespec_as_str(time_entry, time_str_local, sizeof(time_str_local)); - debugf(DBG_LEVEL_EXTALL, "header_set_time_from_str set time=[%s]", time_str_local); + get_timespec_as_str(time_entry, time_str_local, sizeof(time_str_local)); + debugf(DBG_LEVEL_EXTALL, "header_set_time_from_str set time=[%s]", time_str_local); } } @@ -207,29 +204,36 @@ static size_t header_get_meta_dispatch(void *ptr, size_t size, size_t nmemb, voi { strncpy(storage, head, sizeof(storage)); dir_entry *de = (dir_entry*)userdata; - if (de != NULL) { - if (!strncasecmp(head, HEADER_TEXT_ATIME, size * nmemb)) { - header_set_time_from_str(value, &de->atime); - } - if (!strncasecmp(head, HEADER_TEXT_CTIME, size * nmemb)) { - header_set_time_from_str(value, &de->ctime); - } - if (!strncasecmp(head, HEADER_TEXT_MTIME, size * nmemb)) { - header_set_time_from_str(value, &de->mtime); - } - if (!strncasecmp(head, HEADER_TEXT_CHMOD, size * nmemb)) { + if (de != NULL) + { + if (!strncasecmp(head, HEADER_TEXT_ATIME, size * nmemb)) + { + header_set_time_from_str(value, &de->atime); + } + if (!strncasecmp(head, HEADER_TEXT_CTIME, size * nmemb)) + { + header_set_time_from_str(value, &de->ctime); + } + if (!strncasecmp(head, HEADER_TEXT_MTIME, size * nmemb)) + { + header_set_time_from_str(value, &de->mtime); + } + if (!strncasecmp(head, HEADER_TEXT_CHMOD, size * nmemb)) + { de->chmod = atoi(value); } - if (!strncasecmp(head, HEADER_TEXT_GID, size * nmemb)) { + if (!strncasecmp(head, HEADER_TEXT_GID, size * nmemb)) + { de->gid = atoi(value); } - if (!strncasecmp(head, HEADER_TEXT_UID, size * nmemb)) { + if (!strncasecmp(head, HEADER_TEXT_UID, size * nmemb)) + { de->uid = atoi(value); } - } - else { - debugf(DBG_LEVEL_EXT, "Unexpected NULL dir_entry on header(%s), file should be in cache already", storage); - } + } + else { + debugf(DBG_LEVEL_EXT, "Unexpected NULL dir_entry on header(%s), file should be in cache already", storage); + } } else { //debugf(DBG_LEVEL_NORM, "Received unexpected header line"); @@ -242,33 +246,32 @@ static size_t rw_callback(size_t (*rw)(void*, size_t, size_t, FILE*), void *ptr, { struct segment_info *info = (struct segment_info *)userp; size_t mem = size * nmemb; - if (mem < 1 || info->size < 1) return 0; size_t amt_read = rw(ptr, 1, info->size < mem ? info->size : mem, info->fp); info->size -= amt_read; - return amt_read; } size_t fwrite2(void *ptr, size_t size, size_t nmemb, FILE *filep) { - return fwrite((const void*)ptr, size, nmemb, filep); + return fwrite((const void*)ptr, size, nmemb, filep); } static size_t read_callback(void *ptr, size_t size, size_t nmemb, void *userp) { - return rw_callback(fread, ptr, size, nmemb, userp); + return rw_callback(fread, ptr, size, nmemb, userp); } static size_t write_callback(void *ptr, size_t size, size_t nmemb, void *userp) { - return rw_callback(fwrite2, ptr, size, nmemb, userp); + return rw_callback(fwrite2, ptr, size, nmemb, userp); } //http://curl.haxx.se/libcurl/c/CURLOPT_XFERINFOFUNCTION.html -int progress_callback_xfer(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow){ +int progress_callback_xfer(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) +{ struct curl_progress *myp = (struct curl_progress *)clientp; CURL *curl = myp->curl; double curtime = 0; @@ -282,7 +285,8 @@ int progress_callback_xfer(void *clientp, curl_off_t dltotal, curl_off_t dlnow, to only run every N seconds, in order to do this the transaction time can be used */ //http://curl.haxx.se/cvssource/src/tool_cb_prg.c - if ((curtime - myp->lastruntime) >= MINIMAL_PROGRESS_FUNCTIONALITY_INTERVAL) { + if ((curtime - myp->lastruntime) >= MINIMAL_PROGRESS_FUNCTIONALITY_INTERVAL) + { myp->lastruntime = curtime; curl_off_t total; curl_off_t point; @@ -298,7 +302,8 @@ int progress_callback_xfer(void *clientp, curl_off_t dltotal, curl_off_t dlnow, } //http://curl.haxx.se/libcurl/c/CURLOPT_PROGRESSFUNCTION.html -int progress_callback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow){ +int progress_callback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow) +{ return progress_callback_xfer(clientp, (curl_off_t)dltotal, (curl_off_t)dlnow, (curl_off_t)ultotal, (curl_off_t)ulnow); } @@ -312,16 +317,15 @@ size_t writefunc_callback(void *contents, size_t size, size_t nmemb, void *userp struct MemoryStruct *mem = (struct MemoryStruct *)userp; mem->memory = realloc(mem->memory, mem->size + realsize + 1); - if (mem->memory == NULL) { + if (mem->memory == NULL) + { /* out of memory! */ debugf(DBG_LEVEL_NORM, KRED"writefunc_callback: realloc() failed"); return 0; } - memcpy(&(mem->memory[mem->size]), contents, realsize); mem->size += realsize; mem->memory[mem->size] = 0; - return realsize; } @@ -330,7 +334,7 @@ size_t writefunc_callback(void *contents, size_t size, size_t nmemb, void *userp static int send_request_size(const char *method, const char *path, void *fp, xmlParserCtxtPtr xmlctx, curl_slist *extra_headers, off_t file_size, int is_segment, - dir_entry *de_cached_entry, const char *unencoded_path) + dir_entry *de_cached_entry, const char *unencoded_path) { debugf(DBG_LEVEL_EXT, "send_request_size(%s) (%s)", method, path); char url[MAX_URL_SIZE]; @@ -360,7 +364,8 @@ static int send_request_size(const char *method, const char *path, void *fp, snprintf(orig_path, sizeof(orig_path), "/%s", path); // retry on failures - for (tries = 0; tries < REQUEST_RETRIES; tries++) { + for (tries = 0; tries < REQUEST_RETRIES; tries++) + { chunk.memory = malloc(1); /* will be grown as needed by the realloc above */ chunk.size = 0; /* no data at this point */ CURL *curl = get_connection(path); @@ -370,42 +375,45 @@ static int send_request_size(const char *method, const char *path, void *fp, curl_easy_setopt(curl, CURLOPT_URL, url); curl_easy_setopt(curl, CURLOPT_HEADER, 0); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); - curl_easy_setopt(curl, CURLOPT_NOPROGRESS, option_curl_progress_state ? 0 : 1);//reversed logic, 0=to enable progress + //reversed logic, 0=to enable curl progress + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, option_curl_progress_state ? 0 : 1); curl_easy_setopt(curl, CURLOPT_USERAGENT, USER_AGENT); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, verify_ssl ? 1 : 0); curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, verify_ssl); curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 10); - curl_easy_setopt(curl, CURLOPT_VERBOSE, option_curl_verbose ? 1 : 0); + curl_easy_setopt(curl, CURLOPT_VERBOSE, option_curl_verbose ? 1 : 0); add_header(&headers, "X-Auth-Token", storage_token); - dir_entry *de; - if (de_cached_entry == NULL) { - de = check_path_info(unencoded_path); - } - else { - // updating metadata on a file to be added to cache - de = de_cached_entry; - debugf(DBG_LEVEL_EXTALL, "send_request_size: using param dir_entry(%s)", orig_path); - } - if (!de) { - debugf(DBG_LEVEL_EXTALL, "send_request_size: "KYEL"file not in cache (%s)(%s)(%s)", orig_path, path, unencoded_path); - } + dir_entry *de; + if (de_cached_entry == NULL) + { + de = check_path_info(unencoded_path); + } + else { + // updating metadata on a file about to be added to cache (for x-copy, dest meta = src meta) + de = de_cached_entry; + debugf(DBG_LEVEL_EXTALL, "send_request_size: using param dir_entry(%s)", orig_path); + } + if (!de) + { + debugf(DBG_LEVEL_EXTALL, "send_request_size: "KYEL"file not in cache (%s)(%s)(%s)", orig_path, path, unencoded_path); + } else { // add headers to save utimens attribs only on upload - if (!strcasecmp(method, "PUT") || !strcasecmp(method, "MKDIR")) { + if (!strcasecmp(method, "PUT") || !strcasecmp(method, "MKDIR")) + { debugf(DBG_LEVEL_EXTALL, "send_request_size: Saving utimens for file %s", orig_path); - //debugf(DBG_LEVEL_NORM, "File found in cache, path=%s", de->full_name); - debugf(DBG_LEVEL_EXTALL, "send_request_size: Cached utime for path=%s ctime=%li.%li mtime=%li.%li atime=%li.%li", orig_path, - de->ctime.tv_sec, de->ctime.tv_nsec, de->mtime.tv_sec, de->mtime.tv_nsec, de->atime.tv_sec, de->atime.tv_nsec); + debugf(DBG_LEVEL_EXTALL, "send_request_size: Cached utime for path=%s ctime=%li.%li mtime=%li.%li atime=%li.%li", orig_path, + de->ctime.tv_sec, de->ctime.tv_nsec, de->mtime.tv_sec, de->mtime.tv_nsec, de->atime.tv_sec, de->atime.tv_nsec); - char atime_str_nice[TIME_CHARS] = "", mtime_str_nice[TIME_CHARS] = "", ctime_str_nice[TIME_CHARS] = ""; - get_timespec_as_str(&(de->atime), atime_str_nice, sizeof(atime_str_nice)); - debugf(DBG_LEVEL_EXTALL, KCYN"send_request_size: atime=[%s]", atime_str_nice); - get_timespec_as_str(&(de->mtime), mtime_str_nice, sizeof(mtime_str_nice)); - debugf(DBG_LEVEL_EXTALL, KCYN"send_request_size: mtime=[%s]", mtime_str_nice); - get_timespec_as_str(&(de->ctime), ctime_str_nice, sizeof(ctime_str_nice)); - debugf(DBG_LEVEL_EXTALL, KCYN"send_request_size: ctime=[%s]", ctime_str_nice); + char atime_str_nice[TIME_CHARS] = "", mtime_str_nice[TIME_CHARS] = "", ctime_str_nice[TIME_CHARS] = ""; + get_timespec_as_str(&(de->atime), atime_str_nice, sizeof(atime_str_nice)); + debugf(DBG_LEVEL_EXTALL, KCYN"send_request_size: atime=[%s]", atime_str_nice); + get_timespec_as_str(&(de->mtime), mtime_str_nice, sizeof(mtime_str_nice)); + debugf(DBG_LEVEL_EXTALL, KCYN"send_request_size: mtime=[%s]", mtime_str_nice); + get_timespec_as_str(&(de->ctime), ctime_str_nice, sizeof(ctime_str_nice)); + debugf(DBG_LEVEL_EXTALL, KCYN"send_request_size: ctime=[%s]", ctime_str_nice); - char mtime_str[TIME_CHARS], atime_str[TIME_CHARS], ctime_str[TIME_CHARS]; + char mtime_str[TIME_CHARS], atime_str[TIME_CHARS], ctime_str[TIME_CHARS]; char string_float[TIME_CHARS]; snprintf(mtime_str, TIME_CHARS, "%lu.%lu", de->mtime.tv_sec, de->mtime.tv_nsec); snprintf(atime_str, TIME_CHARS, "%lu.%lu", de->atime.tv_sec, de->atime.tv_nsec); @@ -414,9 +422,9 @@ static int send_request_size(const char *method, const char *path, void *fp, add_header(&headers, HEADER_TEXT_MTIME, mtime_str); add_header(&headers, HEADER_TEXT_ATIME, atime_str); add_header(&headers, HEADER_TEXT_CTIME, ctime_str); - add_header(&headers, HEADER_TEXT_MTIME_DISPLAY, mtime_str_nice); - add_header(&headers, HEADER_TEXT_ATIME_DISPLAY, atime_str_nice); - add_header(&headers, HEADER_TEXT_CTIME_DISPLAY, ctime_str_nice); + add_header(&headers, HEADER_TEXT_MTIME_DISPLAY, mtime_str_nice); + add_header(&headers, HEADER_TEXT_ATIME_DISPLAY, atime_str_nice); + add_header(&headers, HEADER_TEXT_CTIME_DISPLAY, ctime_str_nice); char gid_str[INT_CHAR_LEN], uid_str[INT_CHAR_LEN], chmod_str[INT_CHAR_LEN]; snprintf(gid_str, INT_CHAR_LEN, "%d", de->gid); @@ -426,11 +434,10 @@ static int send_request_size(const char *method, const char *path, void *fp, add_header(&headers, HEADER_TEXT_UID, uid_str); add_header(&headers, HEADER_TEXT_CHMOD, chmod_str); } - else { - debugf(DBG_LEVEL_EXTALL, "send_request_size: not setting utimes (%s)", orig_path); - } + else { + debugf(DBG_LEVEL_EXTALL, "send_request_size: not setting utimes (%s)", orig_path); + } } - /**/ if (!strcasecmp(method, "MKDIR")) { curl_easy_setopt(curl, CURLOPT_UPLOAD, 1); @@ -441,19 +448,17 @@ static int send_request_size(const char *method, const char *path, void *fp, { rewind(fp); curl_easy_setopt(curl, CURLOPT_UPLOAD, 1); - //curl_easy_setopt(curl, CURLOPT_INFILESIZE, 0); curl_easy_setopt(curl, CURLOPT_INFILESIZE, file_size); curl_easy_setopt(curl, CURLOPT_READDATA, fp); add_header(&headers, "Content-Type", "application/link"); } - //this condition does not include copy-to PUT action as it does not have a FP - //else if (!strcasecmp(method, "PUT") && fp) else if (!strcasecmp(method, "PUT")) { //http://blog.chmouel.com/2012/02/06/anatomy-of-a-swift-put-query-to-object-server/ debugf(DBG_LEVEL_EXT, "send_request_size: PUT (%s)", orig_path); curl_easy_setopt(curl, CURLOPT_UPLOAD, 1); - if (fp) { + if (fp) + { curl_easy_setopt(curl, CURLOPT_INFILESIZE, file_size); curl_easy_setopt(curl, CURLOPT_READDATA, fp); } @@ -481,8 +486,8 @@ static int send_request_size(const char *method, const char *path, void *fp, if (is_segment) { debugf(DBG_LEVEL_EXT, "send_request_size: GET SEGMENT (%s)", orig_path); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); } else if (fp) { @@ -520,10 +525,9 @@ static int send_request_size(const char *method, const char *path, void *fp, curl_easy_setopt(curl, CURLOPT_NOBODY, 1); } } - else - { + else { debugf(DBG_LEVEL_EXT, "send_request_size: catch_all (%s)"); - // this posts an HEAD request (e.g. for statfs) + // this posts an HEAD request (e.g. for statfs) curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, method); curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, &header_dispatch); } @@ -535,73 +539,71 @@ static int send_request_size(const char *method, const char *path, void *fp, headers = curl_slist_append(headers, extra->data); } curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - debugf(DBG_LEVEL_EXT, "status: send_request_size(%s) started HTTP REQ:%s", orig_path, url); + debugf(DBG_LEVEL_EXT, "status: send_request_size(%s) started HTTP REQ:%s", orig_path, url); curl_easy_perform(curl); - double total_time; - char *effective_url; + double total_time; + char *effective_url; curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response); - curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &effective_url); - curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &total_time); - debugf(DBG_LEVEL_EXT, "status: send_request_size(%s) completed HTTP REQ:%s total_time=%.1f seconds", - orig_path, effective_url, total_time); - curl_slist_free_all(headers); + curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &effective_url); + curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &total_time); + debugf(DBG_LEVEL_EXT, "status: send_request_size(%s) completed HTTP REQ:%s total_time=%.1f seconds", + orig_path, effective_url, total_time); + curl_slist_free_all(headers); curl_easy_reset(curl); return_connection(curl); - if (response != 404 && (response >= 400 || response < 200)) { + if (response != 404 && (response >= 400 || response < 200)) + { /* * Now, our chunk.memory points to a memory block that is chunk.size * bytes big and contains the remote file. */ - - //printf("%lu bytes retrieved\n", (long)chunk.size); debugf(DBG_LEVEL_NORM, KRED"send_request_size: error message, size=%lu, [HTTP %d] (%s)(%s)", (long)chunk.size, response, method, path); debugf(DBG_LEVEL_NORM, KRED"send_request_size: error message=[%s]", chunk.memory); } - free(chunk.memory); - if ((response >= 200 && response < 400) || (!strcasecmp(method, "DELETE") && response == 409)) { - debugf(DBG_LEVEL_NORM, "exit 0: send_request_size(%s) speed=%.1f sec "KCYN"(%s) "KGRN"[HTTP OK]", - orig_path, total_time, method); - return response; + if ((response >= 200 && response < 400) || (!strcasecmp(method, "DELETE") && response == 409)) + { + debugf(DBG_LEVEL_NORM, "exit 0: send_request_size(%s) speed=%.1f sec "KCYN"(%s) "KGRN"[HTTP OK]", + orig_path, total_time, method); + return response; } - //handle cases when file is not found, no point in retrying, should exit - if (response == 404){ + //handle cases when file is not found, no point in retrying, will exit + if (response == 404) + { debugf(DBG_LEVEL_NORM, "send_request_size: not found error for (%s)(%s), ignored "KYEL"[HTTP 404].", method, path); return response; } else { - debugf(DBG_LEVEL_NORM, "send_request_size: httpcode=%d (%s)(%s), retrying "KRED"[HTTP ERR]", response, method, path); + debugf(DBG_LEVEL_NORM, "send_request_size: httpcode=%d (%s)(%s), retrying "KRED"[HTTP ERR]", response, method, path); //todo: try to list response content for debug purposes - sleep(8 << tries); // backoff } - if (response == 401 && !cloudfs_connect()) { // re-authenticate on 401s - debugf(DBG_LEVEL_NORM, KYEL"exit 1: send_request_size(%s) (%s) [HTTP REAUTH]", path, method); - return response; - } + if (response == 401 && !cloudfs_connect()) + { // re-authenticate on 401s + debugf(DBG_LEVEL_NORM, KYEL"exit 1: send_request_size(%s) (%s) [HTTP REAUTH]", path, method); + return response; + } if (xmlctx) xmlCtxtResetPush(xmlctx, NULL, 0, NULL, NULL); } - debugf(DBG_LEVEL_NORM, "exit 2: send_request_size(%s)"KCYN"(%s) response=%d", path, method, response); + debugf(DBG_LEVEL_NORM, "exit 2: send_request_size(%s)"KCYN"(%s) response=%d", path, method, response); return response; } int send_request(char *method, const char *path, FILE *fp, xmlParserCtxtPtr xmlctx, curl_slist *extra_headers, dir_entry *de_cached_entry, const char *unencoded_path) { - - long flen = 0; - if (fp) { - // if we don't flush the size will probably be zero - fflush(fp); - flen = cloudfs_file_size(fileno(fp)); - } - - return send_request_size(method, path, fp, xmlctx, extra_headers, flen, 0, de_cached_entry, unencoded_path); - + long flen = 0; + if (fp) + { + // if we don't flush the size will probably be zero + fflush(fp); + flen = cloudfs_file_size(fileno(fp)); + } + return send_request_size(method, path, fp, xmlctx, extra_headers, flen, 0, de_cached_entry, unencoded_path); } //thread that downloads or uploads large file segments @@ -610,7 +612,7 @@ void *upload_segment(void *seginfo) struct segment_info *info = (struct segment_info *)seginfo; char seg_path[MAX_URL_SIZE] = { 0 }; - //set pointer to the segment start index in the complete large file (several threads will write to same large file) + //set pointer to the segment start index in the complete large file (several threads will write to same large file) fseek(info->fp, info->part * info->segment_size, SEEK_SET); setvbuf(info->fp, NULL, _IOFBF, DISK_BUFF_SIZE); @@ -618,15 +620,13 @@ void *upload_segment(void *seginfo) char *encoded = curl_escape(seg_path, 0); debugf(DBG_LEVEL_EXT, KCYN"upload_segment(%s) part=%d size=%d seg_size=%d %s", - info->method, info->part, info->size, info->segment_size, seg_path); + info->method, info->part, info->size, info->segment_size, seg_path); - // int response = send_request_size(info->method, encoded, info, NULL, NULL, - info->size, 1, NULL, seg_path); + info->size, 1, NULL, seg_path); if (!(response >= 200 && response < 300)) - fprintf(stderr, "Segment upload %s failed with response %d", seg_path, - response); + fprintf(stderr, "Segment upload %s failed with response %d", seg_path, response); curl_free(encoded); fclose(info->fp); @@ -638,33 +638,31 @@ void *upload_segment(void *seginfo) void run_segment_threads(const char *method, int segments, int full_segments, int remaining, FILE *fp, char *seg_base, int size_of_segments) { - debugf(DBG_LEVEL_EXT, "run_segment_threads(%s)", method); + debugf(DBG_LEVEL_EXT, "run_segment_threads(%s)", method); char file_path[PATH_MAX] = { 0 }; struct segment_info *info = (struct segment_info *) - malloc(segments * sizeof(struct segment_info)); + malloc(segments * sizeof(struct segment_info)); pthread_t *threads = (pthread_t *)malloc(segments * sizeof(pthread_t)); #ifdef __linux__ - snprintf(file_path, PATH_MAX, "/proc/self/fd/%d", fileno(fp)); - debugf(DBG_LEVEL_NORM, "On run segment filepath=%s", file_path); + snprintf(file_path, PATH_MAX, "/proc/self/fd/%d", fileno(fp)); + debugf(DBG_LEVEL_NORM, "On run segment filepath=%s", file_path); #else - //TODO: I haven't actually tested this - if (fcntl(fileno(fp), F_GETPATH, file_path) == -1) - fprintf(stderr, "couldn't get the path name\n"); + //TODO: I haven't actually tested this + if (fcntl(fileno(fp), F_GETPATH, file_path) == -1) + fprintf(stderr, "couldn't get the path name\n"); #endif int i, ret; - for (i = 0; i < segments; i++) { + for (i = 0; i < segments; i++) + { info[i].method = method; - info[i].fp = fopen(file_path, method[0] == 'G' ? "r+" : "r"); info[i].part = i; info[i].segment_size = size_of_segments; info[i].size = i < full_segments ? size_of_segments : remaining; info[i].seg_base = seg_base; pthread_create(&threads[i], NULL, upload_segment, (void *)&(info[i])); - //set a thread name for debug purposes - //pthread_setname_np(threads[i], "run_segment"); } for (i = 0; i < segments; i++) { @@ -673,7 +671,7 @@ void run_segment_threads(const char *method, int segments, int full_segments, in } free(info); free(threads); - debugf(DBG_LEVEL_EXT, "exit: run_segment_threads(%s)", method); + debugf(DBG_LEVEL_EXT, "exit: run_segment_threads(%s)", method); } void split_path(const char *path, char *seg_base, char *container, @@ -684,77 +682,78 @@ void split_path(const char *path, char *seg_base, char *container, strncat(container, strsep(&string, "/"), MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE)); char *_object = strsep(&string, "/"); - char *remstr; while (remstr = strsep(&string, "/")) { - strncat(container, "/", - MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE)); - strncat(container, _object, - MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE)); - _object = remstr; + strncat(container, "/", + MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE)); + strncat(container, _object, + MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE)); + _object = remstr; } - //fixme: when removing root folders this will generate a segfault, issue #83, https://github.com/TurboGit/hubicfuse/issues/83 - if (_object == NULL) - _object = object; - else - strncpy(object, _object, MAX_URL_SIZE); - free(string); + //fixed: when removing root folders this will generate a segfault + //issue #83, https://github.com/TurboGit/hubicfuse/issues/83 + if (_object == NULL) + _object = object; + else + strncpy(object, _object, MAX_URL_SIZE); + free(string); } //checks on the cloud if this file (seg_path) have an associated segment folder int internal_is_segmented(const char *seg_path, const char *object, const char *parent_path) { debugf(DBG_LEVEL_EXT, "internal_is_segmented(%s)", seg_path); - //try to avoid one additional http request for small files - bool potentially_segmented; - dir_entry *de = check_path_info(parent_path); - if (!de) { - //when files in folders are first loaded the path will not be yet in cache, so need - //to force segment meta download for segmented files - potentially_segmented = true; - } - else { - //potentially segmented, assumption is that 0 size files are potentially segmented - //while size>0 is for sure not segmented, so no point in making an expensive HTTP GET call - potentially_segmented = (de->size == 0 && !de->isdir) ? true : false; - } - debugf(DBG_LEVEL_EXT, "internal_is_segmented: potentially segmented=%d", potentially_segmented); - //end change + //try to avoid an additional http request for small files + bool potentially_segmented; + dir_entry *de = check_path_info(parent_path); + if (!de) + { + //when files in folders are first loaded the path will not be yet in cache, so need + //to force segment meta download for segmented files + potentially_segmented = true; + } + else { + //potentially segmented, assumption is that 0 size files are potentially segmented + //while size>0 is for sure not segmented, so no point in making an expensive HTTP GET call + potentially_segmented = (de->size == 0 && !de->isdir) ? true : false; + } + debugf(DBG_LEVEL_EXT, "internal_is_segmented: potentially segmented=%d", potentially_segmented); dir_entry *seg_dir; - if (potentially_segmented && cloudfs_list_directory(seg_path, &seg_dir)) { - if (seg_dir && seg_dir->isdir) { - do { - if (!strncmp(seg_dir->name, object, MAX_URL_SIZE)) { - debugf(DBG_LEVEL_EXT, "exit 0: internal_is_segmented(%s) "KGRN"TRUE", seg_path); - return 1; - } - } while ((seg_dir = seg_dir->next)); + if (potentially_segmented && cloudfs_list_directory(seg_path, &seg_dir)) + { + if (seg_dir && seg_dir->isdir) + { + do { + if (!strncmp(seg_dir->name, object, MAX_URL_SIZE)) + { + debugf(DBG_LEVEL_EXT, "exit 0: internal_is_segmented(%s) "KGRN"TRUE", seg_path); + return 1; + } + } while ((seg_dir = seg_dir->next)); } } - debugf(DBG_LEVEL_EXT, "exit 1: internal_is_segmented(%s) "KYEL"FALSE", seg_path); + debugf(DBG_LEVEL_EXT, "exit 1: internal_is_segmented(%s) "KYEL"FALSE", seg_path); return 0; } int is_segmented(const char *path) { - debugf(DBG_LEVEL_EXT, "is_segmented(%s)", path); - char container[MAX_URL_SIZE] = ""; - char object[MAX_URL_SIZE] = ""; - char seg_base[MAX_URL_SIZE] = ""; - - split_path(path, seg_base, container, object); - - char seg_path[MAX_URL_SIZE]; - snprintf(seg_path, MAX_URL_SIZE, "%s/%s_segments", seg_base, container); - return internal_is_segmented(seg_path, object, path); + debugf(DBG_LEVEL_EXT, "is_segmented(%s)", path); + char container[MAX_URL_SIZE] = { 0 }; + char object[MAX_URL_SIZE] = { 0 }; + char seg_base[MAX_URL_SIZE] = { 0 }; + split_path(path, seg_base, container, object); + char seg_path[MAX_URL_SIZE]; + snprintf(seg_path, MAX_URL_SIZE, "%s/%s_segments", seg_base, container); + return internal_is_segmented(seg_path, object, path); } //returns segmented file properties by parsing and retrieving the folder structure on the cloud //added totalsize as parameter to return the file size on list directory for segmented files //old implementation returns file size=0 (issue #91) int format_segments(const char *path, char * seg_base, long *segments, - long *full_segments, long *remaining, long *size_of_segments, long *total_size) + long *full_segments, long *remaining, long *size_of_segments, long *total_size) { debugf(DBG_LEVEL_EXT, "format_segments(%s)", path); char container[MAX_URL_SIZE] = ""; @@ -765,48 +764,52 @@ int format_segments(const char *path, char * seg_base, long *segments, char seg_path[MAX_URL_SIZE]; snprintf(seg_path, MAX_URL_SIZE, "%s/%s_segments", seg_base, container); - if (internal_is_segmented(seg_path, object, path)) { + if (internal_is_segmented(seg_path, object, path)) + { char manifest[MAX_URL_SIZE]; dir_entry *seg_dir; snprintf(manifest, MAX_URL_SIZE, "%s/%s", seg_path, object); - debugf(DBG_LEVEL_EXT, KMAG"format_segments manifest(%s)", manifest); - if (!cloudfs_list_directory(manifest, &seg_dir)) { - debugf(DBG_LEVEL_EXT, "exit 0: format_segments(%s)", path); - return 0; - } + debugf(DBG_LEVEL_EXT, KMAG"format_segments manifest(%s)", manifest); + if (!cloudfs_list_directory(manifest, &seg_dir)) + { + debugf(DBG_LEVEL_EXT, "exit 0: format_segments(%s)", path); + return 0; + } // snprintf seesaw between manifest and seg_path to get // the total_size and the segment size as well as the actual objects char *timestamp = seg_dir->name; snprintf(seg_path, MAX_URL_SIZE, "%s/%s", manifest, timestamp); - debugf(DBG_LEVEL_EXT, KMAG"format_segments seg_path(%s)", seg_path); - if (!cloudfs_list_directory(seg_path, &seg_dir)) { - debugf(DBG_LEVEL_EXT, "exit 1: format_segments(%s)", path); - return 0; - } + debugf(DBG_LEVEL_EXT, KMAG"format_segments seg_path(%s)", seg_path); + if (!cloudfs_list_directory(seg_path, &seg_dir)) + { + debugf(DBG_LEVEL_EXT, "exit 1: format_segments(%s)", path); + return 0; + } char *str_size = seg_dir->name; snprintf(manifest, MAX_URL_SIZE, "%s/%s", seg_path, str_size); - debugf(DBG_LEVEL_EXT, KMAG"format_segments manifest2(%s) size=%s", manifest, str_size); - if (!cloudfs_list_directory(manifest, &seg_dir)) { - debugf(DBG_LEVEL_NORM, "exit 2: format_segments(%s)", path); - return 0; - } + debugf(DBG_LEVEL_EXT, KMAG"format_segments manifest2(%s) size=%s", manifest, str_size); + if (!cloudfs_list_directory(manifest, &seg_dir)) + { + debugf(DBG_LEVEL_EXT, "exit 2: format_segments(%s)", path); + return 0; + } - //following folder name actually represents the parent file size + //following folder name actually represents the parent file size char *str_segment = seg_dir->name; snprintf(seg_path, MAX_URL_SIZE, "%s/%s", manifest, str_segment); - debugf(DBG_LEVEL_EXT, KMAG"format_segments seg_path2(%s)", seg_path); - //here is where we get a list with all segment files composing the parent large file - if (!cloudfs_list_directory(seg_path, &seg_dir)) { - debugf(DBG_LEVEL_EXT, "exit 3: format_segments(%s)", path); - return 0; - } + debugf(DBG_LEVEL_EXT, KMAG"format_segments seg_path2(%s)", seg_path); + //here is where we get a list with all segment files composing the parent large file + if (!cloudfs_list_directory(seg_path, &seg_dir)) + { + debugf(DBG_LEVEL_EXT, "exit 3: format_segments(%s)", path); + return 0; + } *total_size = strtoll(str_size, NULL, 10); *size_of_segments = strtoll(str_segment, NULL, 10); - *remaining = *total_size % *size_of_segments; *full_segments = *total_size / *size_of_segments; *segments = *full_segments + (*remaining > 0); @@ -817,13 +820,13 @@ int format_segments(const char *path, char * seg_base, long *segments, char tmp[MAX_URL_SIZE]; strncpy(tmp, seg_base, MAX_URL_SIZE); snprintf(seg_base, MAX_URL_SIZE, "%s/%s", tmp, manifest); - debugf(DBG_LEVEL_EXT, KMAG"format_segments seg_base(%s)", seg_base); - debugf(DBG_LEVEL_EXT, KMAG"exit 4: format_segments(%s) total=%d size_of_segments=%d remaining=%d, full_segments=%d segments=%d", - path, &total_size, &size_of_segments, &remaining, &full_segments, &segments); + debugf(DBG_LEVEL_EXT, KMAG"format_segments seg_base(%s)", seg_base); + debugf(DBG_LEVEL_EXT, KMAG"exit 4: format_segments(%s) total=%d size_of_segments=%d remaining=%d, full_segments=%d segments=%d", + path, &total_size, &size_of_segments, &remaining, &full_segments, &segments); return 1; } else { - debugf(DBG_LEVEL_EXT, KMAG"exit 5: format_segments(%s) not segmented?", path); + debugf(DBG_LEVEL_EXT, KMAG"exit 5: format_segments(%s) not segmented?", path); return 0; } } @@ -877,7 +880,6 @@ void cloudfs_free() } } - int file_is_readable(const char *fname) { FILE *file; @@ -891,20 +893,19 @@ int file_is_readable(const char *fname) const char * get_file_mimetype ( const char *path ) { - if( file_is_readable( path ) == 1 ) - { - magic_t magic; - const char *mime; + if( file_is_readable( path ) == 1 ) + { + magic_t magic; + const char *mime; - magic = magic_open( MAGIC_MIME_TYPE ); - magic_load( magic, NULL ); - magic_compile( magic, NULL ); - mime = magic_file( magic, path ); - magic_close( magic ); + magic = magic_open( MAGIC_MIME_TYPE ); + magic_load( magic, NULL ); + magic_compile( magic, NULL ); + mime = magic_file( magic, path ); + magic_close( magic ); - return mime; + return mime; } - const char *error = "application/octet-stream"; return error; } @@ -922,17 +923,21 @@ int cloudfs_object_read_fp(const char *path, FILE *fp) flen = ftell(fp); // delete the previously uploaded segments - if (is_segmented(path)) { - if (!cloudfs_delete_object(path)) { - debugf(DBG_LEVEL_NORM, KRED"cloudfs_object_read_fp: couldn't delete existing file"); - } - else { - debugf(DBG_LEVEL_EXT, KYEL"cloudfs_object_read_fp: deleted existing file"); - } + if (is_segmented(path)) + { + if (!cloudfs_delete_object(path)) + { + debugf(DBG_LEVEL_NORM, KRED"cloudfs_object_read_fp: couldn't delete existing file"); + } + else + { + debugf(DBG_LEVEL_EXT, KYEL"cloudfs_object_read_fp: deleted existing file"); + } } struct timespec now; - if (flen >= segment_above) { + if (flen >= segment_above) + { int i; long remaining = flen % segment_size; int full_segments = flen / segment_size; @@ -941,31 +946,22 @@ int cloudfs_object_read_fp(const char *path, FILE *fp) // The best we can do here is to get the current time that way tools that // use the mtime can at least check if the file was changing after now clock_gettime(CLOCK_REALTIME, &now); - char string_float[TIME_CHARS]; snprintf(string_float, TIME_CHARS, "%lu.%lu", now.tv_sec, now.tv_nsec); - char meta_mtime[TIME_CHARS]; snprintf(meta_mtime, TIME_CHARS, "%f", atof(string_float)); - char seg_base[MAX_URL_SIZE] = ""; - char container[MAX_URL_SIZE] = ""; char object[MAX_URL_SIZE] = ""; - split_path(path, seg_base, container, object); - char manifest[MAX_URL_SIZE]; snprintf(manifest, MAX_URL_SIZE, "%s_segments", container); - // create the segments container cloudfs_create_directory(manifest); - // reusing manifest // TODO: check how addition of meta_mtime in manifest impacts utimens implementation snprintf(manifest, MAX_URL_SIZE, "%s_segments/%s/%s/%ld/%ld/", container, object, meta_mtime, flen, segment_size); - char tmp[MAX_URL_SIZE]; strncpy(tmp, seg_base, MAX_URL_SIZE); snprintf(seg_base, MAX_URL_SIZE, "%s/%s", tmp, manifest); @@ -976,35 +972,26 @@ int cloudfs_object_read_fp(const char *path, FILE *fp) char *encoded = curl_escape(path, 0); curl_slist *headers = NULL; add_header(&headers, "x-object-manifest", manifest); - //due to utimens changes, not needed anymore - //add_header(&headers, "x-object-meta-mtime", meta_mtime); add_header(&headers, "Content-Length", "0"); add_header(&headers, "Content-Type", filemimetype); - - //if path is encoded cache entry will not be found int response = send_request_size("PUT", encoded, NULL, NULL, headers, 0, 0, NULL, path); curl_slist_free_all(headers); - curl_free(encoded); - debugf(DBG_LEVEL_EXT, "exit 0: cloudfs_object_read_fp(%s) uploaded ok, response=%d", path, response); + debugf(DBG_LEVEL_EXT, "exit 0: cloudfs_object_read_fp(%s) uploaded ok, response=%d", path, response); return (response >= 200 && response < 300); } else{ // assume enters here when file is composed of only one segment (small files) debugf(DBG_LEVEL_EXT, "cloudfs_object_read_fp(%s) "KYEL"unknown state", path); } - rewind(fp); char *encoded = curl_escape(path, 0); - dir_entry *de = path_info(path); if (!de) debugf(DBG_LEVEL_EXT, "cloudfs_object_read_fp(%s) not in cache", path); else { - debugf(DBG_LEVEL_EXT, "cloudfs_object_read_fp(%s) found in cache", path); - + debugf(DBG_LEVEL_EXT, "cloudfs_object_read_fp(%s) found in cache", path); } - //if path is encoded cache entry will not be found int response = send_request("PUT", encoded, fp, NULL, NULL, NULL, path); curl_free(encoded); debugf(DBG_LEVEL_EXT, "exit 1: cloudfs_object_read_fp(%s)", path); @@ -1022,35 +1009,35 @@ int cloudfs_object_write_fp(const char *path, FILE *fp) long full_segments; long remaining; long size_of_segments; - long total_size; + long total_size; - //checks if this file is a segmented one + //checks if this file is a segmented one if (format_segments(path, seg_base, &segments, &full_segments, &remaining, - &size_of_segments, &total_size)) { - + &size_of_segments, &total_size)) + { rewind(fp); fflush(fp); - if (ftruncate(fileno(fp), 0) < 0) { debugf(DBG_LEVEL_NORM, KRED"ftruncate failed. I don't know what to do about that."); abort(); } - run_segment_threads("GET", segments, full_segments, remaining, fp, - seg_base, size_of_segments); - debugf(DBG_LEVEL_EXT, "exit 0: cloudfs_object_write_fp(%s)", path); + run_segment_threads("GET", segments, full_segments, remaining, fp, + seg_base, size_of_segments); + debugf(DBG_LEVEL_EXT, "exit 0: cloudfs_object_write_fp(%s)", path); return 1; } int response = send_request("GET", encoded, fp, NULL, NULL, NULL, path); curl_free(encoded); fflush(fp); - if ((response >= 200 && response < 300) || ftruncate(fileno(fp), 0)) { - debugf(DBG_LEVEL_EXT, "exit 1: cloudfs_object_write_fp(%s)", path); - return 1; - } + if ((response >= 200 && response < 300) || ftruncate(fileno(fp), 0)) + { + debugf(DBG_LEVEL_EXT, "exit 1: cloudfs_object_write_fp(%s)", path); + return 1; + } rewind(fp); - debugf(DBG_LEVEL_EXT, "exit 2: cloudfs_object_write_fp(%s)", path); + debugf(DBG_LEVEL_EXT, "exit 2: cloudfs_object_write_fp(%s)", path); return 0; } @@ -1073,33 +1060,35 @@ int cloudfs_object_truncate(const char *path, off_t size) } //get metadata from cloud, like time attribs. create new entry if not cached yet. -//todo: not thread-safe? -void get_file_metadata(dir_entry *de){ - if (de->size == 0 && !de->isdir && !de->metadata_downloaded){ - //this can be a potential segmented file, try to read segments size - debugf(DBG_LEVEL_EXT, KMAG"ZERO size file=%s", de->full_name); - char seg_base[MAX_URL_SIZE] = ""; - long segments; - long full_segments; - long remaining; - long size_of_segments; - long total_size; - - if (format_segments(de->full_name, seg_base, &segments, &full_segments, &remaining, - &size_of_segments, &total_size)) { - de->size = total_size; - } - } - if (option_get_extended_metadata) { - debugf(DBG_LEVEL_EXT, KCYN "get_file_metadata(%s)", de->full_name); - //retrieve additional file metadata with a quick HEAD query - char *encoded = curl_escape(de->full_name, 0); +void get_file_metadata(dir_entry *de) +{ + if (de->size == 0 && !de->isdir && !de->metadata_downloaded) + { + //this can be a potential segmented file, try to read segments size + debugf(DBG_LEVEL_EXT, KMAG"ZERO size file=%s", de->full_name); + char seg_base[MAX_URL_SIZE] = ""; + long segments; + long full_segments; + long remaining; + long size_of_segments; + long total_size; + if (format_segments(de->full_name, seg_base, &segments, &full_segments, &remaining, + &size_of_segments, &total_size)) + { + de->size = total_size; + } + } + if (option_get_extended_metadata) + { + debugf(DBG_LEVEL_EXT, KCYN "get_file_metadata(%s)", de->full_name); + //retrieve additional file metadata with a quick HEAD query + char *encoded = curl_escape(de->full_name, 0); de->metadata_downloaded = true; - int response = send_request("GET", encoded, NULL, NULL, NULL, de, de->full_name); - curl_free(encoded); - debugf(DBG_LEVEL_EXT, KCYN "exit: get_file_metadata(%s)", de->full_name); - } - return; + int response = send_request("GET", encoded, NULL, NULL, NULL, de, de->full_name); + curl_free(encoded); + debugf(DBG_LEVEL_EXT, KCYN "exit: get_file_metadata(%s)", de->full_name); + } + return; } //get list of folders from cloud @@ -1139,16 +1128,15 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) trailing_slash = "/"; prefix_length++; } - snprintf(container, sizeof(container), "%s?format=xml&delimiter=/&prefix=%s%s", - encoded_container, encoded_object, trailing_slash); + encoded_container, encoded_object, trailing_slash); curl_free(encoded_container); curl_free(encoded_object); } if ((!strcmp(path, "") || !strcmp(path, "/")) && *override_storage_url) response = 404; - else{ + else { // this was generating 404 err on non segmented files (small files) response = send_request("GET", container, NULL, xmlctx, NULL, NULL, path); } @@ -1161,7 +1149,6 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) for (onode = root_element->children; onode; onode = onode->next) { if (onode->type != XML_ELEMENT_NODE) continue; - char is_object = !strcasecmp((const char *)onode->name, "object"); char is_container = !strcasecmp((const char *)onode->name, "container"); char is_subdir = !strcasecmp((const char *)onode->name, "subdir"); @@ -1169,14 +1156,15 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) if (is_object || is_container || is_subdir) { entry_count++; - dir_entry *de = init_dir_entry(); + dir_entry *de = init_dir_entry(); // useful docs on nodes here: http://developer.openstack.org/api-ref-objectstorage-v1.html if (is_container || is_subdir) de->content_type = strdup("application/directory"); for (anode = onode->children; anode; anode = anode->next) { char *content = ""; - for (text_node = anode->children; text_node; text_node = text_node->next){ + for (text_node = anode->children; text_node; text_node = text_node->next) + { if (text_node->type == XML_TEXT_NODE){ content = (char *)text_node->content; //debugf(DBG_LEVEL_NORM, "List dir anode=%s content=%s", (const char *)anode->name, content); @@ -1188,20 +1176,19 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) if (!strcasecmp((const char *)anode->name, "name")) { de->name = strdup(content + prefix_length); - // Remove trailing slash char *slash = strrchr(de->name, '/'); if (slash && (0 == *(slash + 1))) *slash = 0; - - if (asprintf(&(de->full_name), "%s/%s", path, de->name) < 0){ + if (asprintf(&(de->full_name), "%s/%s", path, de->name) < 0) + { de->full_name = NULL; } - } - if (!strcasecmp((const char *)anode->name, "bytes")) { - de->size = strtoll(content, NULL, 10); - } + if (!strcasecmp((const char *)anode->name, "bytes")) + { + de->size = strtoll(content, NULL, 10); + } if (!strcasecmp((const char *)anode->name, "content_type")) { de->content_type = strdup(content); @@ -1226,7 +1213,7 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) de->mtime.tv_nsec = 0; de->atime.tv_sec = local_time_t; de->atime.tv_nsec = 0; - // TODO check if I can retrieve nano seconds? + //todo: how can we retrieve and set nanoseconds, are stored by hubic? } } de->isdir = de->content_type && @@ -1236,10 +1223,10 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) ((strstr(de->content_type, "application/link") != NULL)); if (de->isdir) { - //i guess this will remove a dir_entry from cache if is there already + //i guess this will remove a dir_entry from cache if is there already if (!strncasecmp(de->name, last_subdir, sizeof(last_subdir))) { - //not sure when / why this is called, seems to generate many missed delete ops. + //todo: check why is needed and if memory is freed properly, seems to generate many missed delete operations //cloudfs_free_dir_list(de); debugf(DBG_LEVEL_EXT, "cloudfs_list_directory: "KYEL"ignore "KNRM"cloudfs_free_dir_list(%s) command", de->name); continue; @@ -1248,13 +1235,10 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) } de->next = *dir_list; *dir_list = de; - char time_str[TIME_CHARS] = ""; - get_timespec_as_str(&(de->mtime), time_str, sizeof(time_str)); + char time_str[TIME_CHARS] = { 0 }; + get_timespec_as_str(&(de->mtime), time_str, sizeof(time_str)); debugf(DBG_LEVEL_EXT, KCYN"new dir_entry %s size=%d %s dir=%d lnk=%d mod=[%s]", - de->full_name, de->size, de->content_type, de->isdir, de->islink, time_str); - //attempt to read extended attributes on each dir entry - //commented out as lazzy metadata read is implemented in cfs_getattr() - //get_file_metadata(de); + de->full_name, de->size, de->content_type, de->isdir, de->islink, time_str); } else { debugf(DBG_LEVEL_EXT, "unknown element: %s", onode->name); @@ -1262,13 +1246,14 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) } retval = 1; } - else if ((!strcmp(path, "") || !strcmp(path, "/")) && *override_storage_url) { + else if ((!strcmp(path, "") || !strcmp(path, "/")) && *override_storage_url) + { entry_count = 1; debugf(DBG_LEVEL_NORM, "Init cache entry container=[%s]", public_container); - dir_entry *de = init_dir_entry(); + dir_entry *de = init_dir_entry(); de->name = strdup(public_container); struct tm last_modified; - // TODO check what this default time means? + //todo: check what this default time means? strptime("1388434648.01238", "%FT%T", &last_modified); de->last_modified = mktime(&last_modified); de->content_type = strdup("application/directory"); @@ -1287,31 +1272,33 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) return retval; } - int cloudfs_delete_object(const char *path) { - debugf(DBG_LEVEL_EXT, "cloudfs_delete_object(%s)", path); + debugf(DBG_LEVEL_EXT, "cloudfs_delete_object(%s)", path); char seg_base[MAX_URL_SIZE] = ""; long segments; long full_segments; long remaining; long size_of_segments; - long total_size; + long total_size; if (format_segments(path, seg_base, &segments, &full_segments, &remaining, - &size_of_segments, &total_size)) { + &size_of_segments, &total_size)) + { int response; int i; char seg_path[MAX_URL_SIZE] = ""; - for (i = 0; i < segments; i++) { + for (i = 0; i < segments; i++) + { snprintf(seg_path, MAX_URL_SIZE, "%s%08i", seg_base, i); char *encoded = curl_escape(seg_path, 0); response = send_request("DELETE", encoded, NULL, NULL, NULL, NULL, seg_path); - if (response < 200 || response >= 300) { - debugf(DBG_LEVEL_EXT, "exit 1: cloudfs_delete_object(%s) response=%d", path, response); - return 0; - } + if (response < 200 || response >= 300) + { + debugf(DBG_LEVEL_EXT, "exit 1: cloudfs_delete_object(%s) response=%d", path, response); + return 0; + } } } @@ -1319,11 +1306,12 @@ int cloudfs_delete_object(const char *path) int response = send_request("DELETE", encoded, NULL, NULL, NULL, NULL, path); curl_free(encoded); int ret = (response >= 200 && response < 300); - debugf(DBG_LEVEL_EXT, "status: cloudfs_delete_object(%s) response=%d", path, response); - if (response == 409) { - debugf(DBG_LEVEL_EXT, "status: cloudfs_delete_object(%s) NOT EMPTY", path); - ret = -1; - } + debugf(DBG_LEVEL_EXT, "status: cloudfs_delete_object(%s) response=%d", path, response); + if (response == 409) + { + debugf(DBG_LEVEL_EXT, "status: cloudfs_delete_object(%s) NOT EMPTY", path); + ret = -1; + } return ret; } @@ -1332,7 +1320,7 @@ int cloudfs_delete_object(const char *path) // this operation also causes an HTTP 400 error if X-Object-Meta-FilePath value is larger than 256 chars int cloudfs_copy_object(const char *src, const char *dst) { - debugf(DBG_LEVEL_EXT, "cloudfs_copy_object(%s, %s) lensrc=%d, lendst=%d", src, dst, strlen(src), strlen(dst)); + debugf(DBG_LEVEL_EXT, "cloudfs_copy_object(%s, %s) lensrc=%d, lendst=%d", src, dst, strlen(src), strlen(dst)); char *dst_encoded = curl_escape(dst, 0); char *src_encoded = curl_escape(src, 0); @@ -1347,25 +1335,26 @@ int cloudfs_copy_object(const char *src, const char *dst) curl_slist *headers = NULL; add_header(&headers, "X-Copy-From", src_encoded); add_header(&headers, "Content-Length", "0"); - //get source file entry - dir_entry *de_src = check_path_info(src); - if (de_src) { - debugf(DBG_LEVEL_EXT, "status cloudfs_copy_object(%s, %s): src file found", src, dst); - } + //get source file entry + dir_entry *de_src = check_path_info(src); + if (de_src) { + debugf(DBG_LEVEL_EXT, "status cloudfs_copy_object(%s, %s): src file found", src, dst); + } else { debugf(DBG_LEVEL_NORM, KRED"status cloudfs_copy_object(%s, %s): src file NOT found", src, dst); } - //pass src metadata so that PUT will set time attributes of the src file + //pass src metadata so that PUT will set time attributes of the src file int response = send_request("PUT", dst_encoded, NULL, NULL, headers, de_src, dst); curl_free(dst_encoded); curl_free(src_encoded); curl_slist_free_all(headers); - debugf(DBG_LEVEL_EXT, "exit: cloudfs_copy_object(%s,%s) response=%d", src, dst, response); - return (response >= 200 && response < 300); + debugf(DBG_LEVEL_EXT, "exit: cloudfs_copy_object(%s,%s) response=%d", src, dst, response); + return (response >= 200 && response < 300); } // http://developer.openstack.org/api-ref-objectstorage-v1.html#updateObjectMeta -int cloudfs_update_meta(dir_entry *de) { +int cloudfs_update_meta(dir_entry *de) +{ int response = cloudfs_copy_object(de->full_name, de->full_name); return response; } @@ -1373,28 +1362,27 @@ int cloudfs_update_meta(dir_entry *de) { //optimised with cache int cloudfs_statfs(const char *path, struct statvfs *stat) { - time_t now = get_time_now(); - int lapsed = now - last_stat_read_time; - if (lapsed > option_cache_statfs_timeout) { + time_t now = get_time_now(); + int lapsed = now - last_stat_read_time; + if (lapsed > option_cache_statfs_timeout) + { //todo: check why stat head request is always set to /, why not path? - int response = send_request("HEAD", "/", NULL, NULL, NULL, NULL, "/"); - *stat = statcache; - debugf(DBG_LEVEL_EXT, "exit: cloudfs_statfs (new recent values, was cached since %d seconds)", lapsed); - last_stat_read_time = now; - return (response >= 200 && response < 300); - } - else { - debugf(DBG_LEVEL_EXT, "exit: cloudfs_statfs (old values, cached since %d seconds)", lapsed); + int response = send_request("HEAD", "/", NULL, NULL, NULL, NULL, "/"); + *stat = statcache; + debugf(DBG_LEVEL_EXT, "exit: cloudfs_statfs (new recent values, was cached since %d seconds)", lapsed); + last_stat_read_time = now; + return (response >= 200 && response < 300); + } + else { + debugf(DBG_LEVEL_EXT, "exit: cloudfs_statfs (old values, cached since %d seconds)", lapsed); return 1; - } + } } int cloudfs_create_symlink(const char *src, const char *dst) { char *dst_encoded = curl_escape(dst, 0); - FILE *lnk = tmpfile(); - fwrite(src, 1, strlen(src), lnk); fwrite("\0", 1, 1, lnk); int response = send_request("MKLINK", dst_encoded, lnk, NULL, NULL, NULL, dst); @@ -1405,11 +1393,11 @@ int cloudfs_create_symlink(const char *src, const char *dst) int cloudfs_create_directory(const char *path) { - debugf(DBG_LEVEL_EXT, "cloudfs_create_directory(%s)", path); + debugf(DBG_LEVEL_EXT, "cloudfs_create_directory(%s)", path); char *encoded = curl_escape(path, 0); int response = send_request("MKDIR", encoded, NULL, NULL, NULL, NULL, path); curl_free(encoded); - debugf(DBG_LEVEL_EXT, "cloudfs_create_directory(%s) response=%d", path, response); + debugf(DBG_LEVEL_EXT, "cloudfs_create_directory(%s) response=%d", path, response); return (response >= 200 && response < 300); } @@ -1427,12 +1415,12 @@ void cloudfs_verify_ssl(int vrfy) void cloudfs_option_get_extended_metadata(int option) { - option_get_extended_metadata = option ? true : false; + option_get_extended_metadata = option ? true : false; } void cloudfs_option_curl_verbose(int option) { - option_curl_verbose = option ? true : false; + option_curl_verbose = option ? true : false; } static struct { @@ -1449,59 +1437,59 @@ void cloudfs_set_credentials(char *client_id, char *client_secret, char *refresh } struct htmlString { - char *text; - size_t size; + char *text; + size_t size; }; static size_t writefunc_string(void *contents, size_t size, size_t nmemb, void *data) { - struct htmlString *mem = (struct htmlString *) data; - size_t realsize = size * nmemb; - mem->text = realloc(mem->text, mem->size + realsize + 1); - if (mem->text == NULL) { /* out of memory! */ - perror(__FILE__); - exit(EXIT_FAILURE); - } + struct htmlString *mem = (struct htmlString *) data; + size_t realsize = size * nmemb; + mem->text = realloc(mem->text, mem->size + realsize + 1); + if (mem->text == NULL) { /* out of memory! */ + perror(__FILE__); + exit(EXIT_FAILURE); + } - memcpy(&(mem->text[mem->size]), contents, realsize); - mem->size += realsize; - return realsize; + memcpy(&(mem->text[mem->size]), contents, realsize); + mem->size += realsize; + return realsize; } char* htmlStringGet(CURL *curl) { - struct htmlString chunk; - chunk.text = malloc(sizeof(char)); - chunk.size = 0; - chunk.text[0] = '\0';//added to avoid valgrind unitialised warning + struct htmlString chunk; + chunk.text = malloc(sizeof(char)); + chunk.size = 0; + chunk.text[0] = '\0';//added to avoid valgrind unitialised warning - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &chunk); - do { - curl_easy_perform(curl); - } while (chunk.size == 0); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &chunk); + do { + curl_easy_perform(curl); + } while (chunk.size == 0); - chunk.text[chunk.size] = '\0'; - return chunk.text; + chunk.text[chunk.size] = '\0'; + return chunk.text; } /* thanks to http://devenix.wordpress.com */ char *unbase64(unsigned char *input, int length) { - BIO *b64, *bmem; + BIO *b64, *bmem; - char *buffer = (char *)malloc(length); - memset(buffer, 0, length); + char *buffer = (char *)malloc(length); + memset(buffer, 0, length); - b64 = BIO_new(BIO_f_base64()); - bmem = BIO_new_mem_buf(input, length); - bmem = BIO_push(b64, bmem); - BIO_set_flags(bmem, BIO_FLAGS_BASE64_NO_NL); + b64 = BIO_new(BIO_f_base64()); + bmem = BIO_new_mem_buf(input, length); + bmem = BIO_push(b64, bmem); + BIO_set_flags(bmem, BIO_FLAGS_BASE64_NO_NL); - BIO_read(bmem, buffer, length); + BIO_read(bmem, buffer, length); - BIO_free_all(bmem); + BIO_free_all(bmem); - return buffer; + return buffer; } int safe_json_string(json_object *jobj, char *buffer, char *name) @@ -1509,16 +1497,16 @@ int safe_json_string(json_object *jobj, char *buffer, char *name) int result = 0; if (jobj) + { + json_object *o; + int found; + found = json_object_object_get_ex(jobj, name, &o); + if (found) { - json_object *o; - int found; - found = json_object_object_get_ex(jobj, name, &o); - if (found) - { - strcpy (buffer, json_object_get_string(o)); - result = 1; - } + strcpy (buffer, json_object_get_string(o)); + result = 1; } + } if (!result) debugf(DBG_LEVEL_NORM, KRED"HUBIC cannot get json field '%s'\n", name); @@ -1541,11 +1529,8 @@ int cloudfs_connect() struct json_object *json_obj; pthread_mutex_lock(&pool_mut); - debugf(DBG_LEVEL_NORM, "Authenticating... (client_id = '%s')", HUBIC_CLIENT_ID); - storage_token[0] = storage_url[0] = '\0'; - CURL *curl = curl_easy_init(); /* curl default options */ @@ -1563,22 +1548,15 @@ int cloudfs_connect() curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writefunc_string); /* Step 1 : request a token - Not needed anymore with refresh_token */ - - /* Step 2 : get request code - Not needed anymore with refresh_token */ - - /* Step 3 : get access token */ sprintf(payload, "refresh_token=%s&grant_type=refresh_token", HUBIC_REFRESH_TOKEN); - curl_easy_setopt(curl, CURLOPT_URL, HUBIC_TOKEN_URL); curl_easy_setopt(curl, CURLOPT_POST, 1L); curl_easy_setopt(curl, CURLOPT_HEADER, 0); - curl_easy_setopt(curl, CURLOPT_POSTFIELDS, payload); curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, strlen(payload)); - curl_easy_setopt(curl, CURLOPT_USERNAME, HUBIC_CLIENT_ID); curl_easy_setopt(curl, CURLOPT_PASSWORD, HUBIC_CLIENT_SECRET); curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); @@ -1600,14 +1578,12 @@ int cloudfs_connect() return 0; found = json_object_object_get_ex(json_obj, "expires_in", &o); - expire_sec = json_object_get_int(o); debugf(DBG_LEVEL_NORM, "HUBIC Access token: %s\n", access_token); debugf(DBG_LEVEL_NORM, "HUBIC Token type : %s\n", token_type); debugf(DBG_LEVEL_NORM, "HUBIC Expire in : %d\n", expire_sec); /* Step 4 : request OpenStack storage URL */ - curl_easy_setopt(curl, CURLOPT_URL, HUBIC_CRED_URL); curl_easy_setopt(curl, CURLOPT_POST, 0L); curl_easy_setopt(curl, CURLOPT_HEADER, 0); @@ -1622,7 +1598,6 @@ int cloudfs_connect() char token[HUBIC_OPTIONS_SIZE]; char endpoint[HUBIC_OPTIONS_SIZE]; char expires[HUBIC_OPTIONS_SIZE]; - json_str = htmlStringGet(curl); json_obj = json_tokener_parse(json_str); debugf(DBG_LEVEL_NORM, "CRED_URL result: '%s'\n", json_str); @@ -1638,12 +1613,8 @@ int cloudfs_connect() /* set the global storage_url and storage_token, the only parameters needed for swift */ strcpy (storage_url, endpoint); strcpy (storage_token, token); - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response); - curl_easy_cleanup(curl); - pthread_mutex_unlock(&pool_mut); - return (response >= 200 && response < 300 && storage_token[0] && storage_url[0]); } diff --git a/cloudfsapi.h b/cloudfsapi.h index 11f3bd3..b9438c4 100644 --- a/cloudfsapi.h +++ b/cloudfsapi.h @@ -22,23 +22,23 @@ struct curl_progress { }; typedef struct options { - char cache_timeout[OPTION_SIZE]; - char verify_ssl[OPTION_SIZE]; - char segment_size[OPTION_SIZE]; - char segment_above[OPTION_SIZE]; - char storage_url[OPTION_SIZE]; - char container[OPTION_SIZE]; - char temp_dir[OPTION_SIZE]; - char client_id[OPTION_SIZE]; - char client_secret[OPTION_SIZE]; - char refresh_token[OPTION_SIZE]; + char cache_timeout[OPTION_SIZE]; + char verify_ssl[OPTION_SIZE]; + char segment_size[OPTION_SIZE]; + char segment_above[OPTION_SIZE]; + char storage_url[OPTION_SIZE]; + char container[OPTION_SIZE]; + char temp_dir[OPTION_SIZE]; + char client_id[OPTION_SIZE]; + char client_secret[OPTION_SIZE]; + char refresh_token[OPTION_SIZE]; } FuseOptions; typedef struct extra_options { - char get_extended_metadata[OPTION_SIZE]; - char curl_verbose[OPTION_SIZE]; - char cache_statfs_timeout[OPTION_SIZE]; - char debug_level[OPTION_SIZE]; + char get_extended_metadata[OPTION_SIZE]; + char curl_verbose[OPTION_SIZE]; + char cache_statfs_timeout[OPTION_SIZE]; + char debug_level[OPTION_SIZE]; char curl_progress_state[OPTION_SIZE]; char enable_chmod[OPTION_SIZE]; char enable_chown[OPTION_SIZE]; @@ -51,12 +51,12 @@ int cloudfs_connect(void); struct segment_info { - FILE *fp; - int part; - long size; - long segment_size; - char *seg_base; - const char *method; + FILE *fp; + int part; + long size; + long segment_size; + char *seg_base; + const char *method; }; long segment_size; @@ -67,7 +67,6 @@ char *public_container; int file_is_readable(const char *fname); const char * get_file_mimetype ( const char *filename ); - int cloudfs_object_read_fp(const char *path, FILE *fp); int cloudfs_object_write_fp(const char *path, FILE *fp); int cloudfs_list_directory(const char *path, dir_entry **); diff --git a/cloudfuse.c b/cloudfuse.c index 1f85553..34a68ea 100644 --- a/cloudfuse.c +++ b/cloudfuse.c @@ -22,6 +22,7 @@ extern char *temp_dir; extern pthread_mutex_t dcachemut; extern pthread_mutexattr_t mutex_attr; +extern int debug; extern int cache_timeout; extern int option_cache_statfs_timeout; extern int option_debug_level; @@ -49,16 +50,16 @@ static int cfs_getattr(const char *path, struct stat *stbuf) stbuf->st_gid = getegid(); stbuf->st_mode = S_IFDIR | 0755; stbuf->st_nlink = 2; - debug_list_cache_content(); - debugf(DBG_LEVEL_NORM, KBLU "exit 0: cfs_getattr(%s)", path); - return 0; + debug_list_cache_content(); + debugf(DBG_LEVEL_NORM, KBLU "exit 0: cfs_getattr(%s)", path); + return 0; } - //get file. if not in cache will be downloaded. + //get file. if not in cache will be downloaded. dir_entry *de = path_info(path); if (!de) { - debug_list_cache_content(); - debugf(DBG_LEVEL_NORM, KBLU"exit 1: cfs_getattr(%s) "KYEL"not-in-cache/cloud", path); - return -ENOENT; + debug_list_cache_content(); + debugf(DBG_LEVEL_NORM, KBLU"exit 1: cfs_getattr(%s) "KYEL"not-in-cache/cloud", path); + return -ENOENT; } //lazzy download of file metadata, only when really needed @@ -75,18 +76,18 @@ static int cfs_getattr(const char *path, struct stat *stbuf) } // change needed due to utimens stbuf->st_atime = de->atime.tv_sec; - stbuf->st_atim.tv_nsec = de->atime.tv_nsec; + stbuf->st_atim.tv_nsec = de->atime.tv_nsec; stbuf->st_mtime = de->mtime.tv_sec; - stbuf->st_mtim.tv_nsec = de->mtime.tv_nsec; - stbuf->st_ctime = de->ctime.tv_sec; - stbuf->st_ctim.tv_nsec = de->ctime.tv_nsec; - char time_str[TIME_CHARS] = ""; - get_timespec_as_str(&(de->atime), time_str, sizeof(time_str)); - debugf(DBG_LEVEL_EXT, KCYN"cfs_getattr: atime=[%s]", time_str); - get_timespec_as_str(&(de->mtime), time_str, sizeof(time_str)); - debugf(DBG_LEVEL_EXT, KCYN"cfs_getattr: mtime=[%s]", time_str); - get_timespec_as_str(&(de->ctime), time_str, sizeof(time_str)); - debugf(DBG_LEVEL_EXT, KCYN"cfs_getattr: ctime=[%s]", time_str); + stbuf->st_mtim.tv_nsec = de->mtime.tv_nsec; + stbuf->st_ctime = de->ctime.tv_sec; + stbuf->st_ctim.tv_nsec = de->ctime.tv_nsec; + char time_str[TIME_CHARS] = ""; + get_timespec_as_str(&(de->atime), time_str, sizeof(time_str)); + debugf(DBG_LEVEL_EXT, KCYN"cfs_getattr: atime=[%s]", time_str); + get_timespec_as_str(&(de->mtime), time_str, sizeof(time_str)); + debugf(DBG_LEVEL_EXT, KCYN"cfs_getattr: mtime=[%s]", time_str); + get_timespec_as_str(&(de->ctime), time_str, sizeof(time_str)); + debugf(DBG_LEVEL_EXT, KCYN"cfs_getattr: ctime=[%s]", time_str); int default_mode_dir, default_mode_file; @@ -119,8 +120,8 @@ static int cfs_getattr(const char *path, struct stat *stbuf) stbuf->st_mode = S_IFREG | default_mode_file; stbuf->st_nlink = 1; } - debugf(DBG_LEVEL_NORM, KBLU "exit 2: cfs_getattr(%s)", path); - return 0; + debugf(DBG_LEVEL_NORM, KBLU "exit 2: cfs_getattr(%s)", path); + return 0; } static int cfs_fgetattr(const char *path, struct stat *stbuf, struct fuse_file_info *info) @@ -147,10 +148,10 @@ static int cfs_fgetattr(const char *path, struct stat *stbuf, struct fuse_file_i stbuf->st_size = cloudfs_file_size(of->fd); stbuf->st_mode = S_IFREG | default_mode_file; stbuf->st_nlink = 1; - debugf(DBG_LEVEL_NORM, KBLU "exit 0: cfs_fgetattr(%s)", path); + debugf(DBG_LEVEL_NORM, KBLU "exit 0: cfs_fgetattr(%s)", path); return 0; } - debugf(DBG_LEVEL_NORM, KRED "exit 1: cfs_fgetattr(%s)", path); + debugf(DBG_LEVEL_NORM, KRED "exit 1: cfs_fgetattr(%s)", path); return -ENOENT; } @@ -158,45 +159,45 @@ static int cfs_readdir(const char *path, void *buf, fuse_fill_dir_t filldir, off { debugf(DBG_LEVEL_NORM, KBLU "cfs_readdir(%s)", path); dir_entry *de; - if (!caching_list_directory(path, &de)) { - debug_list_cache_content(); - debugf(DBG_LEVEL_NORM, KRED "exit 0: cfs_readdir(%s)", path); - return -ENOLINK; - } + if (!caching_list_directory(path, &de)) { + debug_list_cache_content(); + debugf(DBG_LEVEL_NORM, KRED "exit 0: cfs_readdir(%s)", path); + return -ENOLINK; + } filldir(buf, ".", NULL, 0); filldir(buf, "..", NULL, 0); for (; de; de = de->next) filldir(buf, de->name, NULL, 0); - debug_list_cache_content(); - debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_readdir(%s)", path); - return 0; + debug_list_cache_content(); + debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_readdir(%s)", path); + return 0; } static int cfs_mkdir(const char *path, mode_t mode) { - debugf(DBG_LEVEL_NORM, KBLU "cfs_mkdir(%s)", path); - int response = cloudfs_create_directory(path); + debugf(DBG_LEVEL_NORM, KBLU "cfs_mkdir(%s)", path); + int response = cloudfs_create_directory(path); if (response){ update_dir_cache(path, 0, 1, 0); - debug_list_cache_content(); - debugf(DBG_LEVEL_NORM, KBLU "exit 0: cfs_mkdir(%s)", path); + debug_list_cache_content(); + debugf(DBG_LEVEL_NORM, KBLU "exit 0: cfs_mkdir(%s)", path); return 0; } - debugf(DBG_LEVEL_NORM, KRED "exit 1: cfs_mkdir(%s) response=%d", path, response); - return -ENOENT; + debugf(DBG_LEVEL_NORM, KRED "exit 1: cfs_mkdir(%s) response=%d", path, response); + return -ENOENT; } static int cfs_create(const char *path, mode_t mode, struct fuse_file_info *info) { debugf(DBG_LEVEL_NORM, KBLU "cfs_create(%s)", path); FILE *temp_file; - int errsv; + int errsv; char file_path_safe[NAME_MAX] = ""; if (*temp_dir) { - get_safe_cache_file_path(path, file_path_safe, temp_dir); + get_safe_cache_file_path(path, file_path_safe, temp_dir); temp_file = fopen(file_path_safe, "w+b"); - errsv = errno; + errsv = errno; if (temp_file == NULL){ debugf(DBG_LEVEL_NORM, KRED "exit 0: cfs_create cannot open temp file %s.error %s\n", file_path_safe, strerror(errsv)); return -EIO; @@ -204,7 +205,7 @@ static int cfs_create(const char *path, mode_t mode, struct fuse_file_info *info } else { temp_file = tmpfile(); - errsv = errno; + errsv = errno; if (temp_file == NULL){ debugf(DBG_LEVEL_NORM, KRED "exit 1: cfs_create cannot open tmp file for path %s.error %s\n", path, strerror(errsv)); return -EIO; @@ -217,116 +218,115 @@ static int cfs_create(const char *path, mode_t mode, struct fuse_file_info *info info->fh = (uintptr_t)of; update_dir_cache(path, 0, 0, 0); info->direct_io = 1; - dir_entry *de = check_path_info(path); - if (de) { - debugf(DBG_LEVEL_EXT, KCYN"cfs_create(%s): found in cache", path); - struct timespec now; - clock_gettime(CLOCK_REALTIME, &now); - debugf(DBG_LEVEL_EXT, KCYN"cfs_create(%s) set utimes as now", path); - de->atime.tv_sec = now.tv_sec; - de->atime.tv_nsec = now.tv_nsec; - de->mtime.tv_sec = now.tv_sec; - de->mtime.tv_nsec = now.tv_nsec; - de->ctime.tv_sec = now.tv_sec; - de->ctime.tv_nsec = now.tv_nsec; + dir_entry *de = check_path_info(path); + if (de) { + debugf(DBG_LEVEL_EXT, KCYN"cfs_create(%s): found in cache", path); + struct timespec now; + clock_gettime(CLOCK_REALTIME, &now); + debugf(DBG_LEVEL_EXT, KCYN"cfs_create(%s) set utimes as now", path); + de->atime.tv_sec = now.tv_sec; + de->atime.tv_nsec = now.tv_nsec; + de->mtime.tv_sec = now.tv_sec; + de->mtime.tv_nsec = now.tv_nsec; + de->ctime.tv_sec = now.tv_sec; + de->ctime.tv_nsec = now.tv_nsec; - char time_str[TIME_CHARS] = ""; - get_timespec_as_str(&(de->atime), time_str, sizeof(time_str)); - debugf(DBG_LEVEL_EXT, KCYN"cfs_create: atime=[%s]", time_str); - get_timespec_as_str(&(de->mtime), time_str, sizeof(time_str)); - debugf(DBG_LEVEL_EXT, KCYN"cfs_create: mtime=[%s]", time_str); - get_timespec_as_str(&(de->ctime), time_str, sizeof(time_str)); - debugf(DBG_LEVEL_EXT, KCYN"cfs_create: ctime=[%s]", time_str); + char time_str[TIME_CHARS] = ""; + get_timespec_as_str(&(de->atime), time_str, sizeof(time_str)); + debugf(DBG_LEVEL_EXT, KCYN"cfs_create: atime=[%s]", time_str); + get_timespec_as_str(&(de->mtime), time_str, sizeof(time_str)); + debugf(DBG_LEVEL_EXT, KCYN"cfs_create: mtime=[%s]", time_str); + get_timespec_as_str(&(de->ctime), time_str, sizeof(time_str)); + debugf(DBG_LEVEL_EXT, KCYN"cfs_create: ctime=[%s]", time_str); //set chmod & chown de->chmod = mode; de->uid = geteuid(); de->gid = getegid(); - } + } else { debugf(DBG_LEVEL_EXT, KBLU "cfs_create(%s) "KYEL"dir-entry not found", path); } - debugf(DBG_LEVEL_NORM, KBLU "exit 2: cfs_create(%s)=(%s) result=%d:%s", path, file_path_safe, errsv, strerror(errsv)); + debugf(DBG_LEVEL_NORM, KBLU "exit 2: cfs_create(%s)=(%s) result=%d:%s", path, file_path_safe, errsv, strerror(errsv)); return 0; } - // open (download) file from cloud // todo: implement etag optimisation, download only if content changed, http://www.17od.com/2012/12/19/ten-useful-openstack-swift-features/ static int cfs_open(const char *path, struct fuse_file_info *info) { debugf(DBG_LEVEL_NORM, KBLU "cfs_open(%s)", path); FILE *temp_file = NULL; - int errsv; + int errsv; dir_entry *de = path_info(path); if (*temp_dir) { char file_path_safe[NAME_MAX]; - get_safe_cache_file_path(path, file_path_safe, temp_dir); + get_safe_cache_file_path(path, file_path_safe, temp_dir); - debugf(DBG_LEVEL_EXT, "cfs_open: try open (%s)", file_path_safe); + debugf(DBG_LEVEL_EXT, "cfs_open: try open (%s)", file_path_safe); if (access(file_path_safe, F_OK) != -1){ // file exists temp_file = fopen(file_path_safe, "r"); - errsv = errno; - if (temp_file == NULL) { - debugf(DBG_LEVEL_NORM, KRED"exit 0: cfs_open can't open temp_file=[%s] err=%d:%s", file_path_safe, errsv, strerror(errsv)); - return -ENOENT; - } - else - debugf(DBG_LEVEL_EXT, "cfs_open: file exists"); + errsv = errno; + if (temp_file == NULL) { + debugf(DBG_LEVEL_NORM, KRED"exit 0: cfs_open can't open temp_file=[%s] err=%d:%s", file_path_safe, errsv, strerror(errsv)); + return -ENOENT; + } + else + debugf(DBG_LEVEL_EXT, "cfs_open: file exists"); } - else { - errsv = errno; - debugf(DBG_LEVEL_EXT, "cfs_open: file not in cache, err=%s", strerror(errsv)); - //FIXME: commented out as this condition will not be meet in some odd cases and program will crash - //if (!(info->flags & O_WRONLY)) { - debugf(DBG_LEVEL_EXT, "cfs_open: opening for write"); + else { + errsv = errno; + debugf(DBG_LEVEL_EXT, "cfs_open: file not in cache, err=%s", strerror(errsv)); + //FIXME: commented out as this condition will not be meet in some odd cases and program will crash + //if (!(info->flags & O_WRONLY)) { + debugf(DBG_LEVEL_EXT, "cfs_open: opening for write"); - // we need to lock on the filename another process could open the file - // while we are writing to it and then only read part of the file + // we need to lock on the filename another process could open the file + // while we are writing to it and then only read part of the file - // duplicate the directory caching datastructure to make the code easier - // to understand. + // duplicate the directory caching datastructure to make the code easier + // to understand. - // each file in the cache needs: - // filename, is_writing, last_closed, is_removing - // the first time a file is opened a new entry is created in the cache - // setting the filename and is_writing to true. This check needs to be - // wrapped with a lock. - // - // each time a file is closed we set the last_closed for the file to now - // and we check the cache for files whose last - // closed is greater than cache_timeout, then start a new thread rming - // that file. + // each file in the cache needs: + // filename, is_writing, last_closed, is_removing + // the first time a file is opened a new entry is created in the cache + // setting the filename and is_writing to true. This check needs to be + // wrapped with a lock. + // + // each time a file is closed we set the last_closed for the file to now + // and we check the cache for files whose last + // closed is greater than cache_timeout, then start a new thread rming + // that file. - // TODO: just to prevent this craziness for now - temp_file = fopen(file_path_safe, "w+b"); - errsv = errno; - if (temp_file == NULL) { - debugf(DBG_LEVEL_NORM, KRED"exit 1: cfs_open cannot open temp_file=[%s] err=%d:%s", file_path_safe, errsv, strerror(errsv)); - return -ENOENT; - } + // TODO: just to prevent this craziness for now + temp_file = fopen(file_path_safe, "w+b"); + errsv = errno; + if (temp_file == NULL) { + debugf(DBG_LEVEL_NORM, KRED"exit 1: cfs_open cannot open temp_file=[%s] err=%d:%s", file_path_safe, errsv, strerror(errsv)); + return -ENOENT; + } - if (!cloudfs_object_write_fp(path, temp_file)) { - fclose(temp_file); - debugf(DBG_LEVEL_NORM, KRED "exit 2: cfs_open(%s) cannot download/write", path); - return -ENOENT; - } - } + if (!cloudfs_object_write_fp(path, temp_file)) { + fclose(temp_file); + debugf(DBG_LEVEL_NORM, KRED "exit 2: cfs_open(%s) cannot download/write", path); + return -ENOENT; + } + } } else { temp_file = tmpfile(); if (temp_file == NULL) { debugf(DBG_LEVEL_NORM, KRED"exit 3: cfs_open cannot create temp_file err=%s", strerror(errno)); - return -ENOENT; + return -ENOENT; } if (!(info->flags & O_TRUNC)) { if (!cloudfs_object_write_fp(path, temp_file) && !(info->flags & O_CREAT)) { fclose(temp_file); - debugf(DBG_LEVEL_NORM, KRED"exit 4: cfs_open(%s) cannot download/write", path); + debugf(DBG_LEVEL_NORM, KRED"exit 4: cfs_open(%s) cannot download/write", path); return -ENOENT; } } @@ -335,10 +335,11 @@ static int cfs_open(const char *path, struct fuse_file_info *info) update_dir_cache(path, (de ? de->size : 0), 0, 0); openfile *of = (openfile *)malloc(sizeof(openfile)); of->fd = dup(fileno(temp_file)); - if (of->fd == -1){ + if (of->fd == -1) + { //FIXME: potential leak if free not used? free(of); - debugf(DBG_LEVEL_NORM, KRED "exit 5: cfs_open(%s) of->fd", path); + debugf(DBG_LEVEL_NORM, KRED "exit 5: cfs_open(%s) of->fd", path); return -ENOENT; } fclose(temp_file); @@ -348,7 +349,7 @@ static int cfs_open(const char *path, struct fuse_file_info *info) info->direct_io = 1; info->nonseekable = 1; //FIXME: potential leak if free(of) not used? although if free(of) is used will generate bad descriptor errors - debugf(DBG_LEVEL_NORM, KBLU "exit 6: cfs_open(%s)", path); + debugf(DBG_LEVEL_NORM, KBLU "exit 6: cfs_open(%s)", path); return 0; } @@ -356,10 +357,10 @@ static int cfs_read(const char *path, char *buf, size_t size, off_t offset, stru { debugf(DBG_LEVEL_EXTALL, KBLU "cfs_read(%s) buffsize=%lu offset=%lu", path, size, offset); file_buffer_size = size; - debug_print_descriptor(info); - int result = pread(((openfile *)(uintptr_t)info->fh)->fd, buf, size, offset); - debugf(DBG_LEVEL_EXTALL, KBLU "exit: cfs_read(%s) result=%s", path, strerror(errno)); - return result; + debug_print_descriptor(info); + int result = pread(((openfile *)(uintptr_t)info->fh)->fd, buf, size, offset); + debugf(DBG_LEVEL_EXTALL, KBLU "exit: cfs_read(%s) result=%s", path, strerror(errno)); + return result; } //todo: flush will upload a file again even if just file attributes are changed. @@ -367,18 +368,19 @@ static int cfs_read(const char *path, char *buf, size_t size, off_t offset, stru static int cfs_flush(const char *path, struct fuse_file_info *info) { debugf(DBG_LEVEL_NORM, KBLU "cfs_flush(%s)", path); - debug_print_descriptor(info); + debug_print_descriptor(info); openfile *of = (openfile *)(uintptr_t)info->fh; - int errsv = 0; + int errsv = 0; if (of) { update_dir_cache(path, cloudfs_file_size(of->fd), 0, 0); if (of->flags & O_RDWR || of->flags & O_WRONLY) { FILE *fp = fdopen(dup(of->fd), "r"); - errsv = errno; - if (fp != NULL) { - rewind(fp); - //todo: calculate md5 hash, compare with cloud hash to determine if file content is changed + errsv = errno; + if (fp != NULL) + { + rewind(fp); + //calculate md5 hash, compare with cloud hash to determine if file content is changed char md5_file_hash_str[MD5_DIGEST_LENGTH + 1] = "\0"; file_md5(fp, md5_file_hash_str); dir_entry *de = check_path_info(path); @@ -397,39 +399,39 @@ static int cfs_flush(const char *path, struct fuse_file_info *info) return -ENOENT; } } - fclose(fp); - errsv = errno; - } - else { - debugf(DBG_LEVEL_EXT, KRED "status: cfs_flush, err=%d:%s", errsv, strerror(errno)); - } + fclose(fp); + errsv = errno; + } + else { + debugf(DBG_LEVEL_EXT, KRED "status: cfs_flush, err=%d:%s", errsv, strerror(errno)); + } } } - debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_flush(%s) result=%d:%s", path, errsv, strerror(errno)); - return 0; + debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_flush(%s) result=%d:%s", path, errsv, strerror(errno)); + return 0; } static int cfs_release(const char *path, struct fuse_file_info *info) { debugf(DBG_LEVEL_NORM, KBLU "cfs_release(%s)", path); close(((openfile *)(uintptr_t)info->fh)->fd); - debugf(DBG_LEVEL_NORM, KBLU "exit: cfs_release(%s)", path); - return 0; + debugf(DBG_LEVEL_NORM, KBLU "exit: cfs_release(%s)", path); + return 0; } static int cfs_rmdir(const char *path) { debugf(DBG_LEVEL_NORM, KBLU "cfs_rmdir(%s)", path); int success = cloudfs_delete_object(path); - if (success == -1) { - debugf(DBG_LEVEL_NORM, KBLU "exit 0: cfs_rmdir(%s)", path); - return -ENOTEMPTY; - } + if (success == -1) { + debugf(DBG_LEVEL_NORM, KBLU "exit 0: cfs_rmdir(%s)", path); + return -ENOTEMPTY; + } if (success){ dir_decache(path); - debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_rmdir(%s)", path); + debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_rmdir(%s)", path); return 0; } - debugf(DBG_LEVEL_NORM, KBLU "exit 2: cfs_rmdir(%s)", path); + debugf(DBG_LEVEL_NORM, KBLU "exit 2: cfs_rmdir(%s)", path); return -ENOENT; } @@ -442,7 +444,7 @@ static int cfs_ftruncate(const char *path, off_t size, struct fuse_file_info *in return -errno; lseek(of->fd, 0, SEEK_SET); update_dir_cache(path, size, 0, 0); - debugf(DBG_LEVEL_NORM, KBLU "exit: cfs_ftruncate(%s)", path); + debugf(DBG_LEVEL_NORM, KBLU "exit: cfs_ftruncate(%s)", path); return 0; } @@ -451,33 +453,33 @@ static int cfs_write(const char *path, const char *buf, size_t length, off_t off debugf(DBG_LEVEL_EXTALL, KBLU "cfs_write(%s) bufflength=%lu offset=%lu", path, length, offset); // FIXME: Potential inconsistent cache update if pwrite fails? update_dir_cache(path, offset + length, 0, 0); - //int result = pwrite(info->fh, buf, length, offset); - int result = pwrite(((openfile *)(uintptr_t)info->fh)->fd, buf, length, offset); - int errsv = errno; + //int result = pwrite(info->fh, buf, length, offset); + int result = pwrite(((openfile *)(uintptr_t)info->fh)->fd, buf, length, offset); + int errsv = errno; if (errsv == 0) { debugf(DBG_LEVEL_EXTALL, KBLU "exit 0: cfs_write(%s) result=%d:%s", path, errsv, strerror(errsv)); } else { debugf(DBG_LEVEL_EXTALL, KBLU "exit 1: cfs_write(%s) "KRED"result=%d:%s", path, errsv, strerror(errsv)); } - return result; + return result; } static int cfs_unlink(const char *path) { - debugf(DBG_LEVEL_NORM, KBLU "cfs_unlink(%s)", path); + debugf(DBG_LEVEL_NORM, KBLU "cfs_unlink(%s)", path); int success = cloudfs_delete_object(path); - if (success == -1) { - debugf(DBG_LEVEL_NORM, KRED "exit 0: cfs_unlink(%s)", path); - return -EACCES; - } + if (success == -1) { + debugf(DBG_LEVEL_NORM, KRED "exit 0: cfs_unlink(%s)", path); + return -EACCES; + } if (success) { dir_decache(path); - debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_unlink(%s)", path); + debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_unlink(%s)", path); return 0; } - debugf(DBG_LEVEL_NORM, KRED "exit 2: cfs_unlink(%s)", path); + debugf(DBG_LEVEL_NORM, KRED "exit 2: cfs_unlink(%s)", path); return -ENOENT; } @@ -491,7 +493,7 @@ static int cfs_truncate(const char *path, off_t size) { debugf(DBG_LEVEL_NORM, "cfs_truncate(%s)", path); cloudfs_object_truncate(path, size); - debugf(DBG_LEVEL_NORM, "exit: cfs_truncate(%s)", path); + debugf(DBG_LEVEL_NORM, "exit: cfs_truncate(%s)", path); return 0; } @@ -500,13 +502,13 @@ static int cfs_statfs(const char *path, struct statvfs *stat) { debugf(DBG_LEVEL_NORM, KBLU "cfs_statfs(%s)", path); if (cloudfs_statfs(path, stat)){ - debugf(DBG_LEVEL_NORM, KBLU "exit 0: cfs_statfs(%s)", path); + debugf(DBG_LEVEL_NORM, KBLU "exit 0: cfs_statfs(%s)", path); return 0; } - else { - debugf(DBG_LEVEL_NORM, KRED"exit 1: cfs_statfs(%s) not-found", path); - return -EIO; - } + else { + debugf(DBG_LEVEL_NORM, KRED"exit 1: cfs_statfs(%s) not-found", path); + return -EIO; + } } static int cfs_chown(const char *path, uid_t uid, gid_t gid) @@ -544,34 +546,34 @@ static int cfs_rename(const char *src, const char *dst) { debugf(DBG_LEVEL_NORM, KBLU"cfs_rename(%s, %s)", src, dst); dir_entry *src_de = path_info(src); - if (!src_de) { - debugf(DBG_LEVEL_NORM, KRED"exit 0: cfs_rename(%s,%s) not-found", src, dst); - return -ENOENT; - } - if (src_de->isdir) { - debugf(DBG_LEVEL_NORM, KRED"exit 1: cfs_rename(%s,%s) cannot rename dirs!", src, dst); - return -EISDIR; - } + if (!src_de) { + debugf(DBG_LEVEL_NORM, KRED"exit 0: cfs_rename(%s,%s) not-found", src, dst); + return -ENOENT; + } + if (src_de->isdir) { + debugf(DBG_LEVEL_NORM, KRED"exit 1: cfs_rename(%s,%s) cannot rename dirs!", src, dst); + return -EISDIR; + } if (cloudfs_copy_object(src, dst)) { /* FIXME this isn't quite right as doesn't preserve last modified */ - //fix done in cloudfs_copy_object() + //fix done in cloudfs_copy_object() update_dir_cache(dst, src_de->size, 0, 0); - int result = cfs_unlink(src); + int result = cfs_unlink(src); - dir_entry *dst_de = path_info(dst); - if (!dst_de) { - debugf(DBG_LEVEL_NORM, KRED"cfs_rename(%s,%s) dest-not-found-in-cache", src, dst); - } - else { + dir_entry *dst_de = path_info(dst); + if (!dst_de) { + debugf(DBG_LEVEL_NORM, KRED"cfs_rename(%s,%s) dest-not-found-in-cache", src, dst); + } + else { debugf(DBG_LEVEL_NORM, KBLU"cfs_rename(%s,%s) upload ok", src, dst); - //copy attributes, shortcut, rather than forcing a download from cloud - copy_dir_entry(src_de, dst_de); - } + //copy attributes, shortcut, rather than forcing a download from cloud + copy_dir_entry(src_de, dst_de); + } - debugf(DBG_LEVEL_NORM, KBLU"exit 3: cfs_rename(%s,%s)", src, dst); - return result; + debugf(DBG_LEVEL_NORM, KBLU"exit 3: cfs_rename(%s,%s)", src, dst); + return result; } - debugf(DBG_LEVEL_NORM, KRED"exit 4: cfs_rename(%s,%s) io error", src, dst); + debugf(DBG_LEVEL_NORM, KRED"exit 4: cfs_rename(%s,%s) io error", src, dst); return -EIO; } @@ -581,32 +583,32 @@ static int cfs_symlink(const char *src, const char *dst) if(cloudfs_create_symlink(src, dst)) { update_dir_cache(dst, 1, 0, 1); - debugf(DBG_LEVEL_NORM, KBLU"exit0: cfs_symlink(%s, %s)", src, dst); + debugf(DBG_LEVEL_NORM, KBLU"exit0: cfs_symlink(%s, %s)", src, dst); return 0; } - debugf(DBG_LEVEL_NORM, KRED"exit1: cfs_symlink(%s, %s) io error", src, dst); + debugf(DBG_LEVEL_NORM, KRED"exit1: cfs_symlink(%s, %s) io error", src, dst); return -EIO; } static int cfs_readlink(const char* path, char* buf, size_t size) { - debugf(DBG_LEVEL_NORM, KBLU"cfs_readlink(%s)", path); + debugf(DBG_LEVEL_NORM, KBLU"cfs_readlink(%s)", path); //fixme: use temp file specified in config - FILE *temp_file = tmpfile(); + FILE *temp_file = tmpfile(); int ret = 0; if (!cloudfs_object_write_fp(path, temp_file)) { - debugf(DBG_LEVEL_NORM, KRED"exit 1: cfs_readlink(%s) not found", path); + debugf(DBG_LEVEL_NORM, KRED"exit 1: cfs_readlink(%s) not found", path); ret = -ENOENT; } if (!pread(fileno(temp_file), buf, size, 0)) { - debugf(DBG_LEVEL_NORM, KRED"exit 2: cfs_readlink(%s) not found", path); + debugf(DBG_LEVEL_NORM, KRED"exit 2: cfs_readlink(%s) not found", path); ret = -ENOENT; } fclose(temp_file); - debugf(DBG_LEVEL_NORM, KBLU"exit 3: cfs_readlink(%s)", path); + debugf(DBG_LEVEL_NORM, KBLU"exit 3: cfs_readlink(%s)", path); return ret; } @@ -616,82 +618,85 @@ static void *cfs_init(struct fuse_conn_info *conn) return NULL; } - //http://man7.org/linux/man-pages/man2/utimensat.2.html -static int cfs_utimens(const char *path, const struct timespec times[2]){ - debugf(DBG_LEVEL_NORM, KBLU "cfs_utimens(%s)", path); - // looking for file entry in cache +static int cfs_utimens(const char *path, const struct timespec times[2]) +{ + debugf(DBG_LEVEL_NORM, KBLU "cfs_utimens(%s)", path); dir_entry *path_de = path_info(path); - if (!path_de) { - debugf(DBG_LEVEL_NORM, KRED"exit 0: cfs_utimens(%s) file not in cache", path); + if (!path_de) + { + debugf(DBG_LEVEL_NORM, KRED"exit 0: cfs_utimens(%s) file not in cache", path); return -ENOENT; } struct timespec now; clock_gettime(CLOCK_REALTIME, &now); - if (path_de->atime.tv_sec != times[0].tv_sec || path_de->atime.tv_nsec != times[0].tv_nsec || - path_de->mtime.tv_sec != times[1].tv_sec || path_de->mtime.tv_nsec != times[1].tv_nsec) { - debugf(DBG_LEVEL_EXT, KCYN "cfs_utimens: change for %s, prev: atime=%li.%li mtime=%li.%li, new: atime=%li.%li mtime=%li.%li", path, - path_de->atime.tv_sec, path_de->atime.tv_nsec, path_de->mtime.tv_sec, path_de->mtime.tv_nsec, - times[0].tv_sec, times[0].tv_nsec, times[1].tv_sec, times[1].tv_nsec); - char time_str[TIME_CHARS] = ""; - get_timespec_as_str(×[1], time_str, sizeof(time_str)); - debugf(DBG_LEVEL_EXT, KCYN"cfs_utimens: set mtime=[%s]", time_str); - get_timespec_as_str(×[0], time_str, sizeof(time_str)); - debugf(DBG_LEVEL_EXT, KCYN"cfs_utimens: set atime=[%s]", time_str); - path_de->atime = times[0]; - path_de->mtime = times[1]; - // not sure how to best obtain ctime from fuse source file. just record current date. - path_de->ctime = now; + if (path_de->atime.tv_sec != times[0].tv_sec || path_de->atime.tv_nsec != times[0].tv_nsec || + path_de->mtime.tv_sec != times[1].tv_sec || path_de->mtime.tv_nsec != times[1].tv_nsec) + { + debugf(DBG_LEVEL_EXT, KCYN "cfs_utimens: change for %s, prev: atime=%li.%li mtime=%li.%li, new: atime=%li.%li mtime=%li.%li", path, + path_de->atime.tv_sec, path_de->atime.tv_nsec, path_de->mtime.tv_sec, path_de->mtime.tv_nsec, + times[0].tv_sec, times[0].tv_nsec, times[1].tv_sec, times[1].tv_nsec); + char time_str[TIME_CHARS] = ""; + get_timespec_as_str(×[1], time_str, sizeof(time_str)); + debugf(DBG_LEVEL_EXT, KCYN"cfs_utimens: set mtime=[%s]", time_str); + get_timespec_as_str(×[0], time_str, sizeof(time_str)); + debugf(DBG_LEVEL_EXT, KCYN"cfs_utimens: set atime=[%s]", time_str); + path_de->atime = times[0]; + path_de->mtime = times[1]; + // not sure how to best obtain ctime from fuse source file. just record current date. + path_de->ctime = now; //calling a meta cloud update here is not always needed. //touch for example opens and closes/flush the file. //worth implementing a meta cache functionality to avoid multiple uploads on meta changes //when changing timestamps on very large files, touch command will trigger 2 x long and useless file uploads on cfs_flush() - //cloudfs_update_meta(path_de); - } - else { - debugf(DBG_LEVEL_EXT, KCYN"cfs_utimens: a/m/time not changed"); - } - - debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_utimens(%s)", path); + } + else { + debugf(DBG_LEVEL_EXT, KCYN"cfs_utimens: a/m/time not changed"); + } + debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_utimens(%s)", path); return 0; } -int cfs_setxattr(const char *path, const char *name, const char *value, size_t size, int flags){ +int cfs_setxattr(const char *path, const char *name, const char *value, size_t size, int flags) +{ return 0; } -int cfs_getxattr(const char *path, const char *name, char *value, size_t size){ +int cfs_getxattr(const char *path, const char *name, char *value, size_t size) +{ return 0; } -int cfs_removexattr(const char *path, const char *name) { +int cfs_removexattr(const char *path, const char *name) +{ return 0; } -int cfs_listxattr(const char *path, char *list, size_t size) { +int cfs_listxattr(const char *path, char *list, size_t size) +{ return 0; } FuseOptions options = { - .cache_timeout = "600", - .verify_ssl = "true", - .segment_size = "1073741824", - .segment_above = "2147483647", - .storage_url = "", - .container = "", - //.temp_dir = "/tmp/", - .temp_dir = "", - .client_id = "", - .client_secret = "", - .refresh_token = "" + .cache_timeout = "600", + .verify_ssl = "true", + .segment_size = "1073741824", + .segment_above = "2147483647", + .storage_url = "", + .container = "", + //.temp_dir = "/tmp/", + .temp_dir = "", + .client_id = "", + .client_secret = "", + .refresh_token = "" }; ExtraFuseOptions extra_options = { - .get_extended_metadata = "false", - .curl_verbose = "false", - .cache_statfs_timeout = 0, + .get_extended_metadata = "false", + .curl_verbose = "false", + .cache_statfs_timeout = 0, .debug_level = 0, .curl_progress_state = "false", .enable_chown = "false", @@ -701,7 +706,7 @@ ExtraFuseOptions extra_options = { int parse_option(void *data, const char *arg, int key, struct fuse_args *outargs) { if (sscanf(arg, " cache_timeout = %[^\r\n ]", options.cache_timeout) || - sscanf(arg, " verify_ssl = %[^\r\n ]", options.verify_ssl) || + sscanf(arg, " verify_ssl = %[^\r\n ]", options.verify_ssl) || sscanf(arg, " segment_above = %[^\r\n ]", options.segment_above) || sscanf(arg, " segment_size = %[^\r\n ]", options.segment_size) || sscanf(arg, " storage_url = %[^\r\n ]", options.storage_url) || @@ -711,14 +716,14 @@ int parse_option(void *data, const char *arg, int key, struct fuse_args *outargs sscanf(arg, " client_secret = %[^\r\n ]", options.client_secret) || sscanf(arg, " refresh_token = %[^\r\n ]", options.refresh_token) || - sscanf(arg, " get_extended_metadata = %[^\r\n ]", extra_options.get_extended_metadata) || - sscanf(arg, " curl_verbose = %[^\r\n ]", extra_options.curl_verbose) || - sscanf(arg, " cache_statfs_timeout = %[^\r\n ]", extra_options.cache_statfs_timeout) || + sscanf(arg, " get_extended_metadata = %[^\r\n ]", extra_options.get_extended_metadata) || + sscanf(arg, " curl_verbose = %[^\r\n ]", extra_options.curl_verbose) || + sscanf(arg, " cache_statfs_timeout = %[^\r\n ]", extra_options.cache_statfs_timeout) || sscanf(arg, " debug_level = %[^\r\n ]", extra_options.debug_level) || sscanf(arg, " curl_progress_state = %[^\r\n ]", extra_options.curl_progress_state) || sscanf(arg, " enable_chmod = %[^\r\n ]", extra_options.enable_chmod) || sscanf(arg, " enable_chown = %[^\r\n ]", extra_options.enable_chown) - ) + ) return 0; if (!strcmp(arg, "-f") || !strcmp(arg, "-d") || !strcmp(arg, "debug")) cloudfs_debug(1); @@ -736,23 +741,31 @@ void interrupt_handler(int sig) { exit(0); } -void initialise_options() { - cloudfs_verify_ssl(!strcasecmp(options.verify_ssl, "true")); - cloudfs_option_get_extended_metadata(!strcasecmp(extra_options.get_extended_metadata, "true")); - cloudfs_option_curl_verbose(!strcasecmp(extra_options.curl_verbose, "true")); - if (*extra_options.debug_level) { - option_debug_level = atoi(extra_options.debug_level); - } - if (*extra_options.cache_statfs_timeout) { - option_cache_statfs_timeout = atoi(extra_options.cache_statfs_timeout); - } - if (*extra_options.curl_progress_state) { +void initialise_options() +{ + //todo: handle param init consistently, quite heavy implementation + cloudfs_verify_ssl(!strcasecmp(options.verify_ssl, "true")); + cloudfs_option_get_extended_metadata(!strcasecmp(extra_options.get_extended_metadata, "true")); + cloudfs_option_curl_verbose(!strcasecmp(extra_options.curl_verbose, "true")); + //lean way to init params, to be used as reference + if (*extra_options.debug_level) + { + option_debug_level = atoi(extra_options.debug_level); + } + if (*extra_options.cache_statfs_timeout) + { + option_cache_statfs_timeout = atoi(extra_options.cache_statfs_timeout); + } + if (*extra_options.curl_progress_state) + { option_curl_progress_state = !strcasecmp(extra_options.curl_progress_state, "true"); } - if (*extra_options.enable_chmod) { + if (*extra_options.enable_chmod) + { option_enable_chmod = !strcasecmp(extra_options.enable_chmod, "true"); } - if (*extra_options.enable_chown) { + if (*extra_options.enable_chown) + { option_enable_chown = !strcasecmp(extra_options.enable_chown, "true"); } } @@ -776,7 +789,6 @@ int main(int argc, char **argv) } fuse_opt_parse(&args, &options, NULL, parse_option); - cache_timeout = atoi(options.cache_timeout); segment_size = atoll(options.segment_size); segment_above = atoll(options.segment_above); @@ -801,25 +813,25 @@ int main(int argc, char **argv) fprintf(stderr, " storage_url=[Storage URL for other tenant to view container]\n"); fprintf(stderr, " container=[Public container to view of tenant specified by storage_url]\n"); fprintf(stderr, " temp_dir=[Directory to store temp files]\n"); - - fprintf(stderr, " get_extended_metadata=[true to enable download of utime, chmod, chown file attributes (but slower)]\n"); - fprintf(stderr, " curl_verbose=[true to debug info on curl requests (lots of output)]\n"); + fprintf(stderr, " get_extended_metadata=[true to enable download of utime, chmod, chown file attributes (but slower)]\n"); + fprintf(stderr, " curl_verbose=[true to debug info on curl requests (lots of output)]\n"); fprintf(stderr, " curl_progress_state=[true to enable progress callback enabled. Mostly used for debugging]\n"); fprintf(stderr, " cache_statfs_timeout=[number of seconds to cache requests to statfs (cloud statistics), 0 for no cache]\n"); - fprintf(stderr, " debug_level=[0 to 3, 0 for minimal verbose debugging. No debug if -d or -f option is not provided.]\n"); + fprintf(stderr, " debug_level=[0 to 2, 0 for minimal verbose debugging. No debug if -d or -f option is not provided.]\n"); fprintf(stderr, " enable_chmod=[true to enable chmod support on fuse]\n"); fprintf(stderr, " enable_chown=[true to enable chown support on fuse]\n"); - return 1; } - cloudfs_init(); - initialise_options(); - fprintf(stderr, "debug_level = %d\n", option_debug_level); - fprintf(stderr, "get_extended_metadata = %d\n", option_get_extended_metadata); - fprintf(stderr, "curl_progress_state = %d\n", option_curl_progress_state); - fprintf(stderr, "enable_chmod = %d\n", option_enable_chmod); - fprintf(stderr, "enable_chown = %d\n", option_enable_chown); + initialise_options(); + if (debug) + { + fprintf(stderr, "debug_level = %d\n", option_debug_level); + fprintf(stderr, "get_extended_metadata = %d\n", option_get_extended_metadata); + fprintf(stderr, "curl_progress_state = %d\n", option_curl_progress_state); + fprintf(stderr, "enable_chmod = %d\n", option_enable_chmod); + fprintf(stderr, "enable_chown = %d\n", option_enable_chown); + } cloudfs_set_credentials(options.client_id, options.client_secret, options.refresh_token); if (!cloudfs_connect()) @@ -827,46 +839,46 @@ int main(int argc, char **argv) fprintf(stderr, "Failed to authenticate.\n"); return 1; } - + //todo: check why in some cases the define below is not available (when running the binary on symbolic linked folders) #ifndef HAVE_OPENSSL #warning Compiling without libssl, will run single-threaded. fuse_opt_add_arg(&args, "-s"); #endif struct fuse_operations cfs_oper = { - .readdir = cfs_readdir, - .mkdir = cfs_mkdir, - .read = cfs_read, - .create = cfs_create, - .open = cfs_open, - .fgetattr = cfs_fgetattr, - .getattr = cfs_getattr, - .flush = cfs_flush, - .release = cfs_release, - .rmdir = cfs_rmdir, - .ftruncate = cfs_ftruncate, - .truncate = cfs_truncate, - .write = cfs_write, - .unlink = cfs_unlink, - .fsync = cfs_fsync, - .statfs = cfs_statfs, - .chmod = cfs_chmod, - .chown = cfs_chown, - .rename = cfs_rename, - .symlink = cfs_symlink, - .readlink = cfs_readlink, - .init = cfs_init, - .utimens = cfs_utimens, + .readdir = cfs_readdir, + .mkdir = cfs_mkdir, + .read = cfs_read, + .create = cfs_create, + .open = cfs_open, + .fgetattr = cfs_fgetattr, + .getattr = cfs_getattr, + .flush = cfs_flush, + .release = cfs_release, + .rmdir = cfs_rmdir, + .ftruncate = cfs_ftruncate, + .truncate = cfs_truncate, + .write = cfs_write, + .unlink = cfs_unlink, + .fsync = cfs_fsync, + .statfs = cfs_statfs, + .chmod = cfs_chmod, + .chown = cfs_chown, + .rename = cfs_rename, + .symlink = cfs_symlink, + .readlink = cfs_readlink, + .init = cfs_init, + .utimens = cfs_utimens, #ifdef HAVE_SETXATTR - .setxattr = cfs_setxattr, - .getxattr = cfs_getxattr, - .listxattr = cfs_listxattr, - .removexattr = cfs_removexattr, + .setxattr = cfs_setxattr, + .getxattr = cfs_getxattr, + .listxattr = cfs_listxattr, + .removexattr = cfs_removexattr, #endif }; - pthread_mutexattr_init(&mutex_attr); - pthread_mutexattr_settype(&mutex_attr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutexattr_init(&mutex_attr); + pthread_mutexattr_settype(&mutex_attr, PTHREAD_MUTEX_RECURSIVE); pthread_mutex_init(&dcachemut, &mutex_attr); return fuse_main(args.argc, args.argv, &cfs_oper, &options); } diff --git a/commonfs.c b/commonfs.c index 27d6323..54ab189 100644 --- a/commonfs.c +++ b/commonfs.c @@ -38,27 +38,24 @@ int option_debug_level = 0; int option_curl_progress_state = 1;//1 to disable curl progress bool option_enable_chown = false; bool option_enable_chmod = false; +bool option_enable_progressive_upload = false; +bool option_enable_progressive_download = false; size_t file_buffer_size = 0; -// needed to get correct GMT / local time, as it does not work +// needed to get correct GMT / local time +// hubic stores time as GMT so we have to do conversions // http://zhu-qy.blogspot.ro/2012/11/ref-how-to-convert-from-utc-to-local.html -time_t my_timegm(struct tm *tm) { +time_t my_timegm(struct tm *tm) +{ time_t epoch = 0; time_t offset = mktime(gmtime(&epoch)); time_t utc = mktime(tm); return difftime(utc, offset); } -// hubic stores time as GMT so we have to do conversions - -/*void time_t set_now_time_to_gmt(){ - struct timespec now; - clock_gettime(CLOCK_REALTIME, &now); - http://stackoverflow.com/questions/1764710/converting-string-containing-localtime-into-utc-in-c -} -*/ //expect time_str as a friendly string format -time_t get_time_from_str_as_gmt(char *time_str){ +time_t get_time_from_str_as_gmt(char *time_str) +{ struct tm val_time_tm; time_t val_time_t; strptime(time_str, "%FT%T", &val_time_tm); @@ -67,7 +64,8 @@ time_t get_time_from_str_as_gmt(char *time_str){ return val_time_t; } -time_t get_time_as_local(time_t time_t_val, char time_str[], int char_buf_size){ +time_t get_time_as_local(time_t time_t_val, char time_str[], int char_buf_size) +{ struct tm loc_time_tm; loc_time_tm = *localtime(&time_t_val); if (time_str != NULL) { @@ -79,7 +77,8 @@ time_t get_time_as_local(time_t time_t_val, char time_str[], int char_buf_size){ return mktime(&loc_time_tm); } -int get_time_as_string(time_t time_t_val, long nsec, char *time_str, int time_str_len){ +int get_time_as_string(time_t time_t_val, long nsec, char *time_str, int time_str_len) +{ struct tm time_val_tm; time_t safe_input_time; //if time is incorrect (too long) you get segfault, need to check length and trim @@ -91,40 +90,43 @@ int get_time_as_string(time_t time_t_val, long nsec, char *time_str, int time_st safe_input_time = time_t_val; time_val_tm = *gmtime(&safe_input_time); int str_len = strftime(time_str, time_str_len, HUBIC_DATE_FORMAT, &time_val_tm); - char nsec_str[TIME_CHARS]; - sprintf(nsec_str, "%d", nsec); - strcat(time_str, nsec_str); + char nsec_str[TIME_CHARS]; + sprintf(nsec_str, "%d", nsec); + strcat(time_str, nsec_str); return str_len + strlen(nsec_str); } -time_t get_time_now() { - struct timespec now; - clock_gettime(CLOCK_REALTIME, &now); - return now.tv_sec; +time_t get_time_now() +{ + struct timespec now; + clock_gettime(CLOCK_REALTIME, &now); + return now.tv_sec; } -size_t get_time_now_as_str(char *time_str, int time_str_len) { - time_t now = time(0); - struct tm tstruct; - tstruct = *localtime(&now); - // Visit http://en.cppreference.com/w/cpp/chrono/c/strftime - // for more information about date/time format - size_t result = strftime(time_str, time_str_len, HUBIC_DATE_FORMAT, &tstruct); - return result; +size_t get_time_now_as_str(char *time_str, int time_str_len) +{ + time_t now = time(0); + struct tm tstruct; + tstruct = *localtime(&now); + // Visit http://en.cppreference.com/w/cpp/chrono/c/strftime + // for more information about date/time format + size_t result = strftime(time_str, time_str_len, HUBIC_DATE_FORMAT, &tstruct); + return result; } -int get_timespec_as_str(const struct timespec *times, char *time_str, int time_str_len) { - return get_time_as_string(times->tv_sec, times->tv_nsec, time_str, time_str_len); +int get_timespec_as_str(const struct timespec *times, char *time_str, int time_str_len) +{ + return get_time_as_string(times->tv_sec, times->tv_nsec, time_str, time_str_len); } -char *str2md5(const char *str, int length) { +char *str2md5(const char *str, int length) +{ int n; MD5_CTX c; unsigned char digest[16]; char *out = (char*)malloc(33); MD5_Init(&c); - while (length > 0) { if (length > 512) { MD5_Update(&c, str, 512); @@ -135,18 +137,16 @@ char *str2md5(const char *str, int length) { length -= 512; str += 512; } - MD5_Final(digest, &c); - for (n = 0; n < 16; ++n) { snprintf(&(out[n * 2]), 16 * 2, "%02x", (unsigned int)digest[n]); } - return out; } // http://stackoverflow.com/questions/10324611/how-to-calculate-the-md5-hash-of-a-large-file-in-c -int file_md5(FILE *file_handle, char *md5_file_str) { +int file_md5(FILE *file_handle, char *md5_file_str) +{ if (file_handle == NULL) { debugf(DBG_LEVEL_NORM, KRED"file_md5: NULL file handle"); return 0; @@ -168,66 +168,71 @@ int file_md5(FILE *file_handle, char *md5_file_str) { for (i = 0; i < MD5_DIGEST_LENGTH; i++) { snprintf(mdchar, 3, "%02x", c[i]); strcat(md5_file_str, mdchar); - //fprintf(stderr, "%02x", c[i]); } free(data_buf); return 0; } -int get_safe_cache_file_path(const char *path, char *file_path_safe, char *temp_dir) { - char tmp_path[PATH_MAX]; - strncpy(tmp_path, path, PATH_MAX); - char *pch; - while ((pch = strchr(tmp_path, '/'))) { - *pch = '.'; - } - char file_path[PATH_MAX] = ""; +int get_safe_cache_file_path(const char *path, char *file_path_safe, char *temp_dir) +{ + char tmp_path[PATH_MAX]; + strncpy(tmp_path, path, PATH_MAX); + char *pch; + while ((pch = strchr(tmp_path, '/'))) { + *pch = '.'; + } + char file_path[PATH_MAX] = ""; //temp file name had process pid in it, removed as on restart files are left in cache (pid changes) - snprintf(file_path, PATH_MAX, TEMP_FILE_NAME_FORMAT, temp_dir, tmp_path); - //fixme check if sizeof or strlen is suitable - int file_path_len = sizeof(file_path); - //the file path name using this format can go beyond NAME_MAX size and will generate error on fopen - //solution: cap file length to NAME_MAX, use a prefix from original path for debug purposes and add md5 id - char *md5_path = str2md5(file_path, file_path_len); - int md5len = strlen(md5_path); - size_t safe_len_prefix = min(NAME_MAX - md5len, file_path_len); - strncpy(file_path_safe, file_path, safe_len_prefix); - strncpy(file_path_safe + safe_len_prefix, md5_path, md5len); - //sometimes above copy process produces longer strings that NAME_MAX, force a null terminated string - file_path_safe[safe_len_prefix + md5len - 1] = '\0'; - free(md5_path); - return strlen(file_path_safe); + snprintf(file_path, PATH_MAX, TEMP_FILE_NAME_FORMAT, temp_dir, tmp_path); + //fixme check if sizeof or strlen is suitable + int file_path_len = sizeof(file_path); + //the file path name using this format can go beyond NAME_MAX size and will generate error on fopen + //solution: cap file length to NAME_MAX, use a prefix from original path for debug purposes and add md5 id + char *md5_path = str2md5(file_path, file_path_len); + int md5len = strlen(md5_path); + size_t safe_len_prefix = min(NAME_MAX - md5len, file_path_len); + strncpy(file_path_safe, file_path, safe_len_prefix); + strncpy(file_path_safe + safe_len_prefix, md5_path, md5len); + //sometimes above copy process produces longer strings that NAME_MAX, force a null terminated string + file_path_safe[safe_len_prefix + md5len - 1] = '\0'; + free(md5_path); + return strlen(file_path_safe); } -void get_file_path_from_fd(int fd, char *path, int size_path) { - char proc_path[MAX_PATH_SIZE]; - /* Read out the link to our file descriptor. */ - sprintf(proc_path, "/proc/self/fd/%d", fd); - memset(path, 0, size_path); - readlink(proc_path, path, size_path - 1); +void get_file_path_from_fd(int fd, char *path, int size_path) +{ + char proc_path[MAX_PATH_SIZE]; + /* Read out the link to our file descriptor. */ + sprintf(proc_path, "/proc/self/fd/%d", fd); + memset(path, 0, size_path); + readlink(proc_path, path, size_path - 1); } -void debug_print_flags(int flags) { - int accmode, val; - accmode = flags & O_ACCMODE; - if (accmode == O_RDONLY) debugf(DBG_LEVEL_EXTALL, KYEL"read only"); - else if (accmode == O_WRONLY) debugf(DBG_LEVEL_EXTALL, KYEL"write only"); - else if (accmode == O_RDWR) debugf(DBG_LEVEL_EXTALL, KYEL"read write"); - else debugf(DBG_LEVEL_EXT, KYEL"unknown access mode"); +//for file descriptor debugging +void debug_print_flags(int flags) +{ + int accmode, val; + accmode = flags & O_ACCMODE; + if (accmode == O_RDONLY) debugf(DBG_LEVEL_EXTALL, KYEL"read only"); + else if (accmode == O_WRONLY) debugf(DBG_LEVEL_EXTALL, KYEL"write only"); + else if (accmode == O_RDWR) debugf(DBG_LEVEL_EXTALL, KYEL"read write"); + else debugf(DBG_LEVEL_EXT, KYEL"unknown access mode"); - if (val & O_APPEND) debugf(DBG_LEVEL_EXTALL, KYEL", append"); - if (val & O_NONBLOCK) debugf(DBG_LEVEL_EXTALL, KYEL", nonblocking"); + if (val & O_APPEND) debugf(DBG_LEVEL_EXTALL, KYEL", append"); + if (val & O_NONBLOCK) debugf(DBG_LEVEL_EXTALL, KYEL", nonblocking"); #if !defined(_POSIX_SOURCE) && defined(O_SYNC) - if (val & O_SYNC) debugf(DBG_LEVEL_EXT, 0,KRED ", synchronous writes"); + if (val & O_SYNC) debugf(DBG_LEVEL_EXT, 0,KRED ", synchronous writes"); #endif } -void debug_print_descriptor(struct fuse_file_info *info) { - char file_path[MAX_PATH_SIZE]; - get_file_path_from_fd(info->fh, file_path, sizeof(file_path)); - debugf(DBG_LEVEL_EXT, KCYN "descriptor localfile=[%s] fd=%d", file_path, info->fh); - debug_print_flags(info->flags); +//for file descriptor debugging +void debug_print_descriptor(struct fuse_file_info *info) +{ + char file_path[MAX_PATH_SIZE]; + get_file_path_from_fd(info->fh, file_path, sizeof(file_path)); + debugf(DBG_LEVEL_EXT, KCYN "descriptor localfile=[%s] fd=%d", file_path, info->fh); + debug_print_flags(info->flags); } void dir_for(const char *path, char *dir) @@ -240,24 +245,26 @@ void dir_for(const char *path, char *dir) //prints cache content for debug purposes void debug_list_cache_content() { - return;//disabled - dir_cache *cw; - dir_entry *de; - for (cw = dcache; cw; cw = cw->next) { - debugf(DBG_LEVEL_EXT, "LIST-CACHE: DIR[%s]", cw->path); - for (de = cw->entries; de; de = de->next) { - debugf(DBG_LEVEL_EXT, "LIST-CACHE: FOLDER[%s]", de->full_name); - } - } + return;//disabled + dir_cache *cw; + dir_entry *de; + for (cw = dcache; cw; cw = cw->next) + { + debugf(DBG_LEVEL_EXT, "LIST-CACHE: DIR[%s]", cw->path); + for (de = cw->entries; de; de = de->next) { + debugf(DBG_LEVEL_EXT, "LIST-CACHE: FOLDER[%s]", de->full_name); + } + } } -int delete_file(char *path) { - debugf(DBG_LEVEL_NORM, KYEL"delete_file(%s)", path); - char file_path_safe[NAME_MAX] = ""; - get_safe_cache_file_path(path, file_path_safe, temp_dir); - int result = unlink(file_path_safe); - debugf(DBG_LEVEL_EXT, KYEL"delete_file(%s) (%s) result=%s", path, file_path_safe, strerror(result)); - return result; +int delete_file(char *path) +{ + debugf(DBG_LEVEL_NORM, KYEL"delete_file(%s)", path); + char file_path_safe[NAME_MAX] = ""; + get_safe_cache_file_path(path, file_path_safe, temp_dir); + int result = unlink(file_path_safe); + debugf(DBG_LEVEL_EXT, KYEL"delete_file(%s) (%s) result=%s", path, file_path_safe, strerror(result)); + return result; } //adding a directory in cache @@ -269,71 +276,71 @@ dir_cache *new_cache(const char *path) cw->prev = NULL; cw->entries = NULL; cw->cached = time(NULL); - //added cache by access - cw->accessed_in_cache = time(NULL); - cw->was_deleted = false; + //added cache by access + cw->accessed_in_cache = time(NULL); + cw->was_deleted = false; if (dcache) dcache->prev = cw; cw->next = dcache; - dir_cache *result; - result = (dcache = cw); - debugf(DBG_LEVEL_EXT, "exit: new_cache(%s)", path); + dir_cache *result; + result = (dcache = cw); + debugf(DBG_LEVEL_EXT, "exit: new_cache(%s)", path); return result; } - //todo: check if the program behaves ok when free_dir //is made on a folder that has an operation in progress void cloudfs_free_dir_list(dir_entry *dir_list) { - //check for NULL as dir might be already removed from cache by other thread - debugf(DBG_LEVEL_NORM, "cloudfs_free_dir_list(%s)", dir_list->full_name); - while (dir_list) { - dir_entry *de = dir_list; - dir_list = dir_list->next; - //remove file from disk cache, fix for issue #89, https://github.com/TurboGit/hubicfuse/issues/89 - delete_file(de->full_name); - free(de->name); - free(de->full_name); - free(de->content_type); - //TODO free all added fields - free(de->md5sum); - free(de); - } + //check for NULL as dir might be already removed from cache by other thread + debugf(DBG_LEVEL_NORM, "cloudfs_free_dir_list(%s)", dir_list->full_name); + while (dir_list) { + dir_entry *de = dir_list; + dir_list = dir_list->next; + //remove file from disk cache, fix for issue #89, https://github.com/TurboGit/hubicfuse/issues/89 + delete_file(de->full_name); + free(de->name); + free(de->full_name); + free(de->content_type); + //TODO free all added fields + free(de->md5sum); + free(de); + } } - void dir_decache(const char *path) { dir_cache *cw; - debugf(DBG_LEVEL_NORM, "dir_decache(%s)", path); + debugf(DBG_LEVEL_NORM, "dir_decache(%s)", path); pthread_mutex_lock(&dcachemut); dir_entry *de, *tmpde; char dir[MAX_PATH_SIZE]; dir_for(path, dir); for (cw = dcache; cw; cw = cw->next) { - debugf(DBG_LEVEL_EXT, "dir_decache: parse(%s)", cw->path); - if (!strcmp(cw->path, path)) { + debugf(DBG_LEVEL_EXT, "dir_decache: parse(%s)", cw->path); + if (!strcmp(cw->path, path)) + { if (cw == dcache) dcache = cw->next; if (cw->prev) cw->prev->next = cw->next; if (cw->next) cw->next->prev = cw->prev; - debugf(DBG_LEVEL_EXT, "dir_decache: free_dir1(%s)", cw->path); - //fixme: this sometimes is NULL and generates segfaults, checking first - if (cw->entries != NULL) - cloudfs_free_dir_list(cw->entries); + debugf(DBG_LEVEL_EXT, "dir_decache: free_dir1(%s)", cw->path); + //fixme: this sometimes is NULL and generates segfaults, checking first + if (cw->entries != NULL) + cloudfs_free_dir_list(cw->entries); free(cw->path); free(cw); } else if (cw->entries && !strcmp(dir, cw->path)) { - if (!strcmp(cw->entries->full_name, path)) { + if (!strcmp(cw->entries->full_name, path)) + { de = cw->entries; cw->entries = de->next; de->next = NULL; - debugf(DBG_LEVEL_EXT, "dir_decache: free_dir2()"); + debugf(DBG_LEVEL_EXT, "dir_decache: free_dir2()"); cloudfs_free_dir_list(de); } else for (de = cw->entries; de->next; de = de->next) @@ -343,8 +350,8 @@ void dir_decache(const char *path) tmpde = de->next; de->next = de->next->next; tmpde->next = NULL; - debugf(DBG_LEVEL_EXT, "dir_decache: free_dir3()", cw->path); - cloudfs_free_dir_list(tmpde); + debugf(DBG_LEVEL_EXT, "dir_decache: free_dir3()", cw->path); + cloudfs_free_dir_list(tmpde); break; } } @@ -353,35 +360,37 @@ void dir_decache(const char *path) pthread_mutex_unlock(&dcachemut); } -dir_entry* init_dir_entry() { - dir_entry *de = (dir_entry *)malloc(sizeof(dir_entry)); +dir_entry* init_dir_entry() +{ + dir_entry *de = (dir_entry *)malloc(sizeof(dir_entry)); de->metadata_downloaded = false; - de->size = 0; - de->next = NULL; - de->md5sum = NULL; - de->accessed_in_cache = time(NULL); - de->last_modified = time(NULL); - de->mtime.tv_sec = time(NULL); - de->atime.tv_sec = time(NULL); - de->ctime.tv_sec = time(NULL); - de->mtime.tv_nsec = 0; - de->atime.tv_nsec = 0; - de->ctime.tv_nsec = 0; + de->size = 0; + de->next = NULL; + de->md5sum = NULL; + de->accessed_in_cache = time(NULL); + de->last_modified = time(NULL); + de->mtime.tv_sec = time(NULL); + de->atime.tv_sec = time(NULL); + de->ctime.tv_sec = time(NULL); + de->mtime.tv_nsec = 0; + de->atime.tv_nsec = 0; + de->ctime.tv_nsec = 0; de->chmod = 0; de->gid = 0; de->uid = 0; - return de; + return de; } -void copy_dir_entry(dir_entry *src, dir_entry *dst) { - dst->atime.tv_sec = src->atime.tv_sec; - dst->atime.tv_nsec = src->atime.tv_nsec; - dst->mtime.tv_sec = src->mtime.tv_sec; - dst->mtime.tv_nsec = src->mtime.tv_nsec; - dst->ctime.tv_sec = src->ctime.tv_sec; - dst->ctime.tv_nsec = src->ctime.tv_nsec; - dst->chmod = src->chmod; - //dst->md5sum = src->md5sum; +void copy_dir_entry(dir_entry *src, dir_entry *dst) +{ + dst->atime.tv_sec = src->atime.tv_sec; + dst->atime.tv_nsec = src->atime.tv_nsec; + dst->mtime.tv_sec = src->mtime.tv_sec; + dst->mtime.tv_nsec = src->mtime.tv_nsec; + dst->ctime.tv_sec = src->ctime.tv_sec; + dst->ctime.tv_nsec = src->ctime.tv_nsec; + dst->chmod = src->chmod; + //todo: copy md5sum as well } //check for file in cache, if found size will be updated, if not found @@ -404,21 +413,23 @@ void update_dir_cache(const char *path, off_t size, int isdir, int islink) { de->size = size; pthread_mutex_unlock(&dcachemut); - debugf(DBG_LEVEL_EXTALL, "exit 0: update_dir_cache(%s)", path); + debugf(DBG_LEVEL_EXTALL, "exit 0: update_dir_cache(%s)", path); return; } } - de = init_dir_entry(); + de = init_dir_entry(); de->size = size; de->isdir = isdir; de->islink = islink; de->name = strdup(&path[strlen(cw->path) + 1]); de->full_name = strdup(path); - //fix: the conditions below were mixed up dir -> link? - if (islink) { + //fixed: the conditions below were mixed up dir -> link? + if (islink) + { de->content_type = strdup("application/link"); } - if (isdir) { + if (isdir) + { de->content_type = strdup("application/directory"); } else { @@ -431,167 +442,183 @@ void update_dir_cache(const char *path, off_t size, int isdir, int islink) break; } } - debugf(DBG_LEVEL_EXTALL, "exit 1: update_dir_cache(%s)", path); + debugf(DBG_LEVEL_EXTALL, "exit 1: update_dir_cache(%s)", path); pthread_mutex_unlock(&dcachemut); } //returns first file entry in linked list. if not in cache will be downloaded. int caching_list_directory(const char *path, dir_entry **list) { - debugf(DBG_LEVEL_EXT, "caching_list_directory(%s)", path); + debugf(DBG_LEVEL_EXT, "caching_list_directory(%s)", path); pthread_mutex_lock(&dcachemut); - bool new_entry = false; - if (!strcmp(path, "/")) + bool new_entry = false; + if (!strcmp(path, "/")) path = ""; - - dir_cache *cw; - for (cw = dcache; cw; cw = cw->next) { - if (cw->was_deleted == true) { - debugf(DBG_LEVEL_EXT, KMAG"caching_list_directory status: dir(%s) is empty as cached expired, reload from cloud", cw->path); - if (!cloudfs_list_directory(cw->path, list)) { - debugf(DBG_LEVEL_EXT, KMAG"caching_list_directory status: cannot reload dir(%s)", cw->path); - } - else { - debugf(DBG_LEVEL_EXT, KMAG"caching_list_directory status: reloaded dir(%s)", cw->path); - //cw->entries = *list; - cw->was_deleted = false; - cw->cached = time(NULL); - } - } - if (cw->was_deleted == false) { - if (!strcmp(cw->path, path)) { - break; - } - } - } - if (!cw) { - //trying to download this entry from cloud, list will point to cached or downloaded entries - if (!cloudfs_list_directory(path, list)){ - //download was not ok + dir_cache *cw; + for (cw = dcache; cw; cw = cw->next) + { + if (cw->was_deleted == true) + { + debugf(DBG_LEVEL_EXT, KMAG"caching_list_directory status: dir(%s) is empty as cached expired, reload from cloud", cw->path); + if (!cloudfs_list_directory(cw->path, list)) + { + debugf(DBG_LEVEL_EXT, KMAG"caching_list_directory status: cannot reload dir(%s)", cw->path); + } + else { + debugf(DBG_LEVEL_EXT, KMAG"caching_list_directory status: reloaded dir(%s)", cw->path); + //cw->entries = *list; + cw->was_deleted = false; + cw->cached = time(NULL); + } + } + if (cw->was_deleted == false) + { + if (!strcmp(cw->path, path)) + { + break; + } + } + } + if (!cw) + { + //trying to download this entry from cloud, list will point to cached or downloaded entries + if (!cloudfs_list_directory(path, list)) + { + //download was not ok pthread_mutex_unlock(&dcachemut); - debugf(DBG_LEVEL_EXT, "exit 0: caching_list_directory(%s) "KYEL"[CACHE-DIR-MISS]", path); + debugf(DBG_LEVEL_EXT, "exit 0: caching_list_directory(%s) "KYEL"[CACHE-DIR-MISS]", path); return 0; } - debugf(DBG_LEVEL_EXT, "caching_list_directory: new_cache(%s) "KYEL"[CACHE-CREATE]", path); + debugf(DBG_LEVEL_EXT, "caching_list_directory: new_cache(%s) "KYEL"[CACHE-CREATE]", path); cw = new_cache(path); - new_entry = true; + new_entry = true; } else if (cache_timeout > 0 && (time(NULL) - cw->cached > cache_timeout)) { - if (!cloudfs_list_directory(path, list)){ + if (!cloudfs_list_directory(path, list)) + { //mutex unlock was forgotten pthread_mutex_unlock(&dcachemut); - debugf(DBG_LEVEL_EXT, "exit 1: caching_list_directory(%s)", path); + debugf(DBG_LEVEL_EXT, "exit 1: caching_list_directory(%s)", path); return 0; } - //fixme: this frees dir subentries but leaves the dir parent entry, this confuses path_info - //which believes this dir has no entries - if (cw->entries != NULL) { - cloudfs_free_dir_list(cw->entries); - cw->was_deleted = true; - cw->cached = time(NULL); - debugf(DBG_LEVEL_EXT, "caching_list_directory(%s) "KYEL"[CACHE-EXPIRED]", path); - } - else { - debugf(DBG_LEVEL_EXT, "got NULL on caching_list_directory(%s) "KYEL"[CACHE-EXPIRED w NULL]", path); - pthread_mutex_unlock(&dcachemut); - return 0; - } + //fixme: this frees dir subentries but leaves the dir parent entry, this confuses path_info + //which believes this dir has no entries + if (cw->entries != NULL) + { + cloudfs_free_dir_list(cw->entries); + cw->was_deleted = true; + cw->cached = time(NULL); + debugf(DBG_LEVEL_EXT, "caching_list_directory(%s) "KYEL"[CACHE-EXPIRED]", path); + } + else { + debugf(DBG_LEVEL_EXT, "got NULL on caching_list_directory(%s) "KYEL"[CACHE-EXPIRED w NULL]", path); + pthread_mutex_unlock(&dcachemut); + return 0; + } } - else { - debugf(DBG_LEVEL_EXT, "caching_list_directory(%s) "KGRN"[CACHE-DIR-HIT]", path); - *list = cw->entries; - } - //adding new dir file list to global cache, now this dir becomes visible in cache + else { + debugf(DBG_LEVEL_EXT, "caching_list_directory(%s) "KGRN"[CACHE-DIR-HIT]", path); + *list = cw->entries; + } + //adding new dir file list to global cache, now this dir becomes visible in cache cw->entries = *list; pthread_mutex_unlock(&dcachemut); - debugf(DBG_LEVEL_EXT, "exit 2: caching_list_directory(%s)", path); + debugf(DBG_LEVEL_EXT, "exit 2: caching_list_directory(%s)", path); return 1; } dir_entry *path_info(const char *path) { - debugf(DBG_LEVEL_EXT, "path_info(%s)", path); - char dir[MAX_PATH_SIZE]; - dir_for(path, dir); - dir_entry *tmp; - if (!caching_list_directory(dir, &tmp)) { - debugf(DBG_LEVEL_EXT, "exit 0: path_info(%s) "KYEL"[CACHE-DIR-MISS]", dir); - return NULL; - } - else { - debugf(DBG_LEVEL_EXT, "path_info(%s) "KGRN"[CACHE-DIR-HIT]", dir); - } - //iterate in file list obtained from cache or downloaded - for (; tmp; tmp = tmp->next) { - if (!strcmp(tmp->full_name, path)) { - debugf(DBG_LEVEL_EXT, "exit 1: path_info(%s) "KGRN"[CACHE-FILE-HIT]", path); - return tmp; - } - } - //miss in case the file is not found on a cached folder - debugf(DBG_LEVEL_EXT, "exit 2: path_info(%s) "KYEL"[CACHE-MISS]", path); - return NULL; + debugf(DBG_LEVEL_EXT, "path_info(%s)", path); + char dir[MAX_PATH_SIZE]; + dir_for(path, dir); + dir_entry *tmp; + if (!caching_list_directory(dir, &tmp)) + { + debugf(DBG_LEVEL_EXT, "exit 0: path_info(%s) "KYEL"[CACHE-DIR-MISS]", dir); + return NULL; + } + else { + debugf(DBG_LEVEL_EXT, "path_info(%s) "KGRN"[CACHE-DIR-HIT]", dir); + } + //iterate in file list obtained from cache or downloaded + for (; tmp; tmp = tmp->next) + { + if (!strcmp(tmp->full_name, path)) + { + debugf(DBG_LEVEL_EXT, "exit 1: path_info(%s) "KGRN"[CACHE-FILE-HIT]", path); + return tmp; + } + } + //miss in case the file is not found on a cached folder + debugf(DBG_LEVEL_EXT, "exit 2: path_info(%s) "KYEL"[CACHE-MISS]", path); + return NULL; } //retrieve folder from local cache if exists, return null if does not exist (rather than download) int check_caching_list_directory(const char *path, dir_entry **list) { - debugf(DBG_LEVEL_EXT, "check_caching_list_directory(%s)", path); - pthread_mutex_lock(&dcachemut); - if (!strcmp(path, "/")) - path = ""; - dir_cache *cw; - for (cw = dcache; cw; cw = cw->next) - if (!strcmp(cw->path, path)) { - *list = cw->entries; - pthread_mutex_unlock(&dcachemut); - debugf(DBG_LEVEL_EXT, "exit 0: check_caching_list_directory(%s) "KGRN"[CACHE-DIR-HIT]", path); - return 1; - } - pthread_mutex_unlock(&dcachemut); - debugf(DBG_LEVEL_EXT, "exit 1: check_caching_list_directory(%s) "KYEL"[CACHE-DIR-MISS]", path); - return 0; + debugf(DBG_LEVEL_EXT, "check_caching_list_directory(%s)", path); + pthread_mutex_lock(&dcachemut); + if (!strcmp(path, "/")) + path = ""; + dir_cache *cw; + for (cw = dcache; cw; cw = cw->next) + if (!strcmp(cw->path, path)) + { + *list = cw->entries; + pthread_mutex_unlock(&dcachemut); + debugf(DBG_LEVEL_EXT, "exit 0: check_caching_list_directory(%s) "KGRN"[CACHE-DIR-HIT]", path); + return 1; + } + pthread_mutex_unlock(&dcachemut); + debugf(DBG_LEVEL_EXT, "exit 1: check_caching_list_directory(%s) "KYEL"[CACHE-DIR-MISS]", path); + return 0; } -dir_entry * check_parent_folder_for_file(const char *path) { - char dir[MAX_PATH_SIZE]; - dir_for(path, dir); - dir_entry *tmp; - if (!check_caching_list_directory(dir, &tmp)) - return NULL; - else - return tmp; +dir_entry * check_parent_folder_for_file(const char *path) +{ + char dir[MAX_PATH_SIZE]; + dir_for(path, dir); + dir_entry *tmp; + if (!check_caching_list_directory(dir, &tmp)) + return NULL; + else + return tmp; } //check if local path is in cache, without downloading from cloud if not in cache dir_entry *check_path_info(const char *path) { - debugf(DBG_LEVEL_EXT, "check_path_info(%s)", path); - char dir[MAX_PATH_SIZE]; - dir_for(path, dir); - dir_entry *tmp; + debugf(DBG_LEVEL_EXT, "check_path_info(%s)", path); + char dir[MAX_PATH_SIZE]; + dir_for(path, dir); + dir_entry *tmp; - //get parent folder cache entry - if (!check_caching_list_directory(dir, &tmp)) { - debugf(DBG_LEVEL_EXT, "exit 0: check_path_info(%s) "KYEL"[CACHE-MISS]", path); - return NULL; - } - for (; tmp; tmp = tmp->next) - { - if (!strcmp(tmp->full_name, path)) { - debugf(DBG_LEVEL_EXT, "exit 1: check_path_info(%s) "KGRN"[CACHE-HIT]", path); - return tmp; - } - } - if (!strcmp(path, "/")) { - debugf(DBG_LEVEL_EXT, "exit 2: check_path_info(%s) "KYEL"ignoring root [CACHE-MISS]", path); - } - else { - debugf(DBG_LEVEL_EXT, "exit 3: check_path_info(%s) "KYEL"[CACHE-MISS]", path); - } - return NULL; + //get parent folder cache entry + if (!check_caching_list_directory(dir, &tmp)) + { + debugf(DBG_LEVEL_EXT, "exit 0: check_path_info(%s) "KYEL"[CACHE-MISS]", path); + return NULL; + } + for (; tmp; tmp = tmp->next) + { + if (!strcmp(tmp->full_name, path)) + { + debugf(DBG_LEVEL_EXT, "exit 1: check_path_info(%s) "KGRN"[CACHE-HIT]", path); + return tmp; + } + } + if (!strcmp(path, "/")) + { + debugf(DBG_LEVEL_EXT, "exit 2: check_path_info(%s) "KYEL"ignoring root [CACHE-MISS]", path); + } + else { + debugf(DBG_LEVEL_EXT, "exit 3: check_path_info(%s) "KYEL"[CACHE-MISS]", path); + } + return NULL; } @@ -608,32 +635,34 @@ char *get_home_dir() void cloudfs_debug(int dbg) { - debug = dbg; + debug = dbg; } void debugf(int level, char *fmt, ...) { - if (debug) { - if (level <= option_debug_level) { + if (debug) + { + if (level <= option_debug_level) + { #ifdef SYS_gettid - pid_t thread_id = syscall(SYS_gettid); -#else - int thread_id = 0; + pid_t thread_id = syscall(SYS_gettid); + #else + int thread_id = 0; #error "SYS_gettid unavailable on this system" #endif - va_list args; - char prefix[] = "==DBG %d [%s]:%d=="; - char line[4096]; - char time_str[TIME_CHARS]; - get_time_now_as_str(time_str, sizeof(time_str)); - sprintf(line, prefix, level, time_str, thread_id); - fputs(line, stderr); - va_start(args, fmt); - vfprintf(stderr, fmt, args); - va_end(args); - fputs(KNRM, stderr); - putc('\n', stderr); - putc('\r', stderr); - } + va_list args; + char prefix[] = "==DBG %d [%s]:%d=="; + char line[4096]; + char time_str[TIME_CHARS]; + get_time_now_as_str(time_str, sizeof(time_str)); + sprintf(line, prefix, level, time_str, thread_id); + fputs(line, stderr); + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); + fputs(KNRM, stderr); + putc('\n', stderr); + putc('\r', stderr); + } } } diff --git a/commonfs.h b/commonfs.h index db0729d..eb31b9a 100644 --- a/commonfs.h +++ b/commonfs.h @@ -57,8 +57,8 @@ typedef struct dir_entry mode_t chmod; uid_t uid; gid_t gid; - bool issegmented; - time_t accessed_in_cache;//todo: cache support based on access time + bool issegmented; + time_t accessed_in_cache;//todo: cache support based on access time bool metadata_downloaded; // end change int isdir; @@ -72,10 +72,10 @@ typedef struct dir_cache char *path; dir_entry *entries; time_t cached; - //added cache support based on access time - time_t accessed_in_cache; - bool was_deleted; - //end change + //added cache support based on access time + time_t accessed_in_cache; + bool was_deleted; + //end change struct dir_cache *next, *prev; } dir_cache; @@ -85,12 +85,10 @@ time_t get_time_as_local(time_t time_t_val, char time_str[], int char_buf_size); int get_time_as_string(time_t time_t_val, long nsec, char *time_str, int time_str_len); time_t get_time_now(); int get_timespec_as_str(const struct timespec *times, char *time_str, int time_str_len); - char *str2md5(const char *str, int length); int file_md5(FILE *file_handle, char *md5_file_str); void debug_print_descriptor(struct fuse_file_info *info); int get_safe_cache_file_path(const char *file_path, char *file_path_safe, char *temp_dir); - dir_entry *init_dir_entry(); void copy_dir_entry(dir_entry *src, dir_entry *dst); dir_cache *new_cache(const char *path); @@ -104,10 +102,8 @@ void dir_decache(const char *path); void cloudfs_free_dir_list(dir_entry *dir_list); extern int cloudfs_list_directory(const char *path, dir_entry **); int caching_list_directory(const char *path, dir_entry **list); - char *get_home_dir(); void cloudfs_debug(int dbg); void debugf(int level, char *fmt, ...); - #endif From a2e4b8132981889c48b05f1f3d52c0b58265ff36 Mon Sep 17 00:00:00 2001 From: Pascal Obry Date: Mon, 30 Nov 2015 21:49:11 +0100 Subject: [PATCH 08/12] Message at startup only in debug mode. --- cloudfuse.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cloudfuse.c b/cloudfuse.c index 34a68ea..8eae2a7 100644 --- a/cloudfuse.c +++ b/cloudfuse.c @@ -772,7 +772,9 @@ void initialise_options() int main(int argc, char **argv) { - fprintf(stderr, "Starting hubicfuse on homedir %s!\n", get_home_dir()); + if (debug) + fprintf(stderr, "Starting hubicfuse on homedir %s!\n", get_home_dir()); + signal(SIGINT, interrupt_handler); char settings_filename[MAX_PATH_SIZE] = ""; From ba9d9fc988eddd1bc4819199f302fd6d6d7f1207 Mon Sep 17 00:00:00 2001 From: Pascal Obry Date: Mon, 30 Nov 2015 22:00:15 +0100 Subject: [PATCH 09/12] Add code reformatting rule. --- Makefile.in | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Makefile.in b/Makefile.in index 450d95b..d5e5d79 100644 --- a/Makefile.in +++ b/Makefile.in @@ -55,3 +55,18 @@ Makefile: Makefile.in config.status config.status: configure ./config.status --recheck +reformat: + astyle --style=allman \ + --convert-tabs \ + --indent=spaces=2 \ + --lineend=linux \ + --max-code-length=79 \ + --pad-oper \ + --pad-header \ + --align-pointer=type \ + --align-reference=name \ + --break-closing-brackets \ + --min-conditional-indent=0 \ + --remove-brackets \ + --remove-comment-prefix \ + *.c *.h From 66c934c8a0fa305eb9a31c86bcbd050d71e52123 Mon Sep 17 00:00:00 2001 From: Pascal Obry Date: Mon, 30 Nov 2015 22:04:56 +0100 Subject: [PATCH 10/12] Reformat code using astyle. --- cloudfsapi.c | 817 ++++++++++++++++++++++++++++----------------------- cloudfsapi.h | 50 ++-- cloudfuse.c | 593 +++++++++++++++++++++---------------- commonfs.c | 260 ++++++++-------- commonfs.h | 66 +++-- 5 files changed, 991 insertions(+), 795 deletions(-) diff --git a/cloudfsapi.c b/cloudfsapi.c index 0032db8..c52ce17 100644 --- a/cloudfsapi.c +++ b/cloudfsapi.c @@ -36,7 +36,7 @@ static char storage_url[MAX_URL_SIZE]; static char storage_token[MAX_HEADER_SIZE]; static pthread_mutex_t pool_mut; -static CURL *curl_pool[1024]; +static CURL* curl_pool[1024]; static int curl_pool_count = 0; extern int debug; extern int verify_ssl; @@ -48,7 +48,8 @@ extern bool option_extensive_debug; extern bool option_enable_chown; extern bool option_enable_chmod; static int rhel5_mode = 0; -static struct statvfs statcache = { +static struct statvfs statcache = +{ .f_bsize = 4096, .f_frsize = 4096, .f_blocks = INT_MAX, @@ -62,15 +63,16 @@ static struct statvfs statcache = { //used to compute statfs cache interval static time_t last_stat_read_time = 0; extern FuseOptions options; -struct MemoryStruct { - char *memory; +struct MemoryStruct +{ + char* memory; size_t size; }; #ifdef HAVE_OPENSSL #include -static pthread_mutex_t *ssl_lockarray; -static void lock_callback(int mode, int type, char *file, int line) +static pthread_mutex_t* ssl_lockarray; +static void lock_callback(int mode, int type, char* file, int line) { if (mode & CRYPTO_LOCK) pthread_mutex_lock(&(ssl_lockarray[type])); @@ -84,16 +86,16 @@ static unsigned long thread_id() } #endif -static size_t xml_dispatch(void *ptr, size_t size, size_t nmemb, void *stream) +static size_t xml_dispatch(void* ptr, size_t size, size_t nmemb, void* stream) { - xmlParseChunk((xmlParserCtxtPtr)stream, (char *)ptr, size * nmemb, 0); + xmlParseChunk((xmlParserCtxtPtr)stream, (char*)ptr, size * nmemb, 0); return size * nmemb; } -static CURL *get_connection(const char *path) +static CURL* get_connection(const char* path) { pthread_mutex_lock(&pool_mut); - CURL *curl = curl_pool_count ? curl_pool[--curl_pool_count] : curl_easy_init(); + CURL* curl = curl_pool_count ? curl_pool[--curl_pool_count] : curl_easy_init(); if (!curl) { debugf(DBG_LEVEL_NORM, KRED"curl alloc failed"); @@ -103,27 +105,31 @@ static CURL *get_connection(const char *path) return curl; } -static void return_connection(CURL *curl) +static void return_connection(CURL* curl) { pthread_mutex_lock(&pool_mut); curl_pool[curl_pool_count++] = curl; pthread_mutex_unlock(&pool_mut); } -static void add_header(curl_slist **headers, const char *name, - const char *value) +static void add_header(curl_slist** headers, const char* name, + const char* value) { char x_header[MAX_HEADER_SIZE]; char safe_value[256]; - const char *value_ptr; + const char* value_ptr; debugf(DBG_LEVEL_EXT, "add_header(%s:%s)", name, value); - if (strlen(value) > 256) { - debugf(DBG_LEVEL_NORM, KRED"add_header: warning, value size > 256 (%s:%s) ", name, value); + if (strlen(value) > 256) + { + debugf(DBG_LEVEL_NORM, KRED"add_header: warning, value size > 256 (%s:%s) ", + name, value); //hubic will throw an HTTP 400 error on X-Copy-To operation if X-Object-Meta-FilePath header value is larger than 256 chars //fix for issue #95 https://github.com/TurboGit/hubicfuse/issues/95 - if (!strcasecmp(name, "X-Object-Meta-FilePath")) { - debugf(DBG_LEVEL_NORM, KRED"add_header: trimming header (%s) value to max allowed", name); + if (!strcasecmp(name, "X-Object-Meta-FilePath")) + { + debugf(DBG_LEVEL_NORM, + KRED"add_header: trimming header (%s) value to max allowed", name); //trim header size to max allowed strncpy(safe_value, value, 256 - 1); safe_value[255] = '\0'; @@ -139,12 +145,13 @@ static void add_header(curl_slist **headers, const char *name, *headers = curl_slist_append(*headers, x_header); } -static size_t header_dispatch(void *ptr, size_t size, size_t nmemb, void *dir_entry) +static size_t header_dispatch(void* ptr, size_t size, size_t nmemb, + void* dir_entry) { - char *header = (char *)alloca(size * nmemb + 1); - char *head = (char *)alloca(size * nmemb + 1); - char *value = (char *)alloca(size * nmemb + 1); - memcpy(header, (char *)ptr, size * nmemb); + char* header = (char*)alloca(size * nmemb + 1); + char* head = (char*)alloca(size * nmemb + 1); + char* value = (char*)alloca(size * nmemb + 1); + memcpy(header, (char*)ptr, size * nmemb); header[size * nmemb] = '\0'; if (sscanf(header, "%[^:]: %[^\r\n]", head, value) == 2) { @@ -153,9 +160,11 @@ static size_t header_dispatch(void *ptr, size_t size, size_t nmemb, void *dir_en if (!strncasecmp(head, "x-storage-url", size * nmemb)) strncpy(storage_url, value, sizeof(storage_url)); if (!strncasecmp(head, "x-account-meta-quota", size * nmemb)) - statcache.f_blocks = (unsigned long) (strtoull(value, NULL, 10)/statcache.f_frsize); + statcache.f_blocks = (unsigned long) (strtoull(value, NULL, + 10) / statcache.f_frsize); if (!strncasecmp(head, "x-account-bytes-used", size * nmemb)) - statcache.f_bfree = statcache.f_bavail = statcache.f_blocks - (unsigned long) (strtoull(value, NULL, 10)/statcache.f_frsize); + statcache.f_bfree = statcache.f_bavail = statcache.f_blocks - (unsigned long) ( + strtoull(value, NULL, 10) / statcache.f_frsize); if (!strncasecmp(head, "x-account-object-count", size * nmemb)) { unsigned long object_count = strtoul(value, NULL, 10); @@ -166,7 +175,9 @@ static size_t header_dispatch(void *ptr, size_t size, size_t nmemb, void *dir_en return size * nmemb; } -static void header_set_time_from_str(char *time_str, struct timespec *time_entry) { +static void header_set_time_from_str(char* time_str, + struct timespec* time_entry) +{ char sec_value[TIME_CHARS] = { 0 }; char nsec_value[TIME_CHARS] = { 0 }; time_t sec; @@ -174,116 +185,112 @@ static void header_set_time_from_str(char *time_str, struct timespec *time_entry sscanf(time_str, "%[^.].%[^\n]", sec_value, nsec_value); sec = strtoll(sec_value, NULL, 10);//to allow for larger numbers nsec = atol(nsec_value); - debugf(DBG_LEVEL_EXTALL, "Received time=%s.%s / %li.%li, existing=%li.%li", - sec_value, nsec_value, sec, nsec, time_entry->tv_sec, time_entry->tv_nsec); + debugf(DBG_LEVEL_EXTALL, "Received time=%s.%s / %li.%li, existing=%li.%li", + sec_value, nsec_value, sec, nsec, time_entry->tv_sec, time_entry->tv_nsec); if (sec != time_entry->tv_sec || nsec != time_entry->tv_nsec) { - debugf(DBG_LEVEL_EXTALL, "Time changed, setting new time=%li.%li, existing was=%li.%li", - sec, nsec, time_entry->tv_sec, time_entry->tv_nsec); + debugf(DBG_LEVEL_EXTALL, + "Time changed, setting new time=%li.%li, existing was=%li.%li", + sec, nsec, time_entry->tv_sec, time_entry->tv_nsec); time_entry->tv_sec = sec; time_entry->tv_nsec = nsec; char time_str_local[TIME_CHARS] = ""; get_time_as_string((time_t)sec, nsec, time_str_local, sizeof(time_str_local)); - debugf(DBG_LEVEL_EXTALL, "header_set_time_from_str received time=[%s]", time_str_local); + debugf(DBG_LEVEL_EXTALL, "header_set_time_from_str received time=[%s]", + time_str_local); get_timespec_as_str(time_entry, time_str_local, sizeof(time_str_local)); - debugf(DBG_LEVEL_EXTALL, "header_set_time_from_str set time=[%s]", time_str_local); + debugf(DBG_LEVEL_EXTALL, "header_set_time_from_str set time=[%s]", + time_str_local); } } -static size_t header_get_meta_dispatch(void *ptr, size_t size, size_t nmemb, void *userdata) +static size_t header_get_meta_dispatch(void* ptr, size_t size, size_t nmemb, + void* userdata) { - char *header = (char *)alloca(size * nmemb + 1); - char *head = (char *)alloca(size * nmemb + 1); - char *value = (char *)alloca(size * nmemb + 1); - memcpy(header, (char *)ptr, size * nmemb); + char* header = (char*)alloca(size * nmemb + 1); + char* head = (char*)alloca(size * nmemb + 1); + char* value = (char*)alloca(size * nmemb + 1); + memcpy(header, (char*)ptr, size * nmemb); header[size * nmemb] = '\0'; static char storage[MAX_HEADER_SIZE]; if (sscanf(header, "%[^:]: %[^\r\n]", head, value) == 2) { strncpy(storage, head, sizeof(storage)); - dir_entry *de = (dir_entry*)userdata; - if (de != NULL) + dir_entry* de = (dir_entry*)userdata; + if (de != NULL) { if (!strncasecmp(head, HEADER_TEXT_ATIME, size * nmemb)) - { header_set_time_from_str(value, &de->atime); - } if (!strncasecmp(head, HEADER_TEXT_CTIME, size * nmemb)) - { header_set_time_from_str(value, &de->ctime); - } if (!strncasecmp(head, HEADER_TEXT_MTIME, size * nmemb)) - { header_set_time_from_str(value, &de->mtime); - } if (!strncasecmp(head, HEADER_TEXT_CHMOD, size * nmemb)) - { de->chmod = atoi(value); - } if (!strncasecmp(head, HEADER_TEXT_GID, size * nmemb)) - { de->gid = atoi(value); - } if (!strncasecmp(head, HEADER_TEXT_UID, size * nmemb)) - { de->uid = atoi(value); - } - } - else { - debugf(DBG_LEVEL_EXT, "Unexpected NULL dir_entry on header(%s), file should be in cache already", storage); } + else + debugf(DBG_LEVEL_EXT, + "Unexpected NULL dir_entry on header(%s), file should be in cache already", + storage); } - else { + else + { //debugf(DBG_LEVEL_NORM, "Received unexpected header line"); } return size * nmemb; } -static size_t rw_callback(size_t (*rw)(void*, size_t, size_t, FILE*), void *ptr, - size_t size, size_t nmemb, void *userp) +static size_t rw_callback(size_t (*rw)(void*, size_t, size_t, FILE*), + void* ptr, + size_t size, size_t nmemb, void* userp) { - struct segment_info *info = (struct segment_info *)userp; - size_t mem = size * nmemb; - if (mem < 1 || info->size < 1) - return 0; + struct segment_info* info = (struct segment_info*)userp; + size_t mem = size * nmemb; + if (mem < 1 || info->size < 1) + return 0; - size_t amt_read = rw(ptr, 1, info->size < mem ? info->size : mem, info->fp); - info->size -= amt_read; - return amt_read; + size_t amt_read = rw(ptr, 1, info->size < mem ? info->size : mem, info->fp); + info->size -= amt_read; + return amt_read; } -size_t fwrite2(void *ptr, size_t size, size_t nmemb, FILE *filep) +size_t fwrite2(void* ptr, size_t size, size_t nmemb, FILE* filep) { return fwrite((const void*)ptr, size, nmemb, filep); } -static size_t read_callback(void *ptr, size_t size, size_t nmemb, void *userp) +static size_t read_callback(void* ptr, size_t size, size_t nmemb, void* userp) { return rw_callback(fread, ptr, size, nmemb, userp); } -static size_t write_callback(void *ptr, size_t size, size_t nmemb, void *userp) +static size_t write_callback(void* ptr, size_t size, size_t nmemb, void* userp) { return rw_callback(fwrite2, ptr, size, nmemb, userp); } //http://curl.haxx.se/libcurl/c/CURLOPT_XFERINFOFUNCTION.html -int progress_callback_xfer(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) +int progress_callback_xfer(void* clientp, curl_off_t dltotal, curl_off_t dlnow, + curl_off_t ultotal, curl_off_t ulnow) { - struct curl_progress *myp = (struct curl_progress *)clientp; - CURL *curl = myp->curl; + struct curl_progress* myp = (struct curl_progress*)clientp; + CURL* curl = myp->curl; double curtime = 0; - double dspeed = 0, uspeed=0; + double dspeed = 0, uspeed = 0; curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &curtime); curl_easy_getinfo(curl, CURLINFO_SPEED_DOWNLOAD, &dspeed); curl_easy_getinfo(curl, CURLINFO_SPEED_UPLOAD, &uspeed); /* under certain circumstances it may be desirable for certain functionality - to only run every N seconds, in order to do this the transaction time can - be used */ + to only run every N seconds, in order to do this the transaction time can + be used */ //http://curl.haxx.se/cvssource/src/tool_cb_prg.c if ((curtime - myp->lastruntime) >= MINIMAL_PROGRESS_FUNCTIONALITY_INTERVAL) { @@ -295,29 +302,34 @@ int progress_callback_xfer(void *clientp, curl_off_t dltotal, curl_off_t dlnow, point = dlnow + ulnow; frac = (double)point / (double)total; percent = frac * 100.0f; - debugf(DBG_LEVEL_EXT, "TOTAL TIME: %.0f sec Down=%.0f Kbps UP=%.0f Kbps", curtime, dspeed/1024, uspeed/1024); - debugf(DBG_LEVEL_EXT, "UP: %lld of %lld DOWN: %lld/%lld Completion %.1f %%", ulnow, ultotal, dlnow, dltotal, percent); + debugf(DBG_LEVEL_EXT, "TOTAL TIME: %.0f sec Down=%.0f Kbps UP=%.0f Kbps", + curtime, dspeed / 1024, uspeed / 1024); + debugf(DBG_LEVEL_EXT, "UP: %lld of %lld DOWN: %lld/%lld Completion %.1f %%", + ulnow, ultotal, dlnow, dltotal, percent); } return 0; } //http://curl.haxx.se/libcurl/c/CURLOPT_PROGRESSFUNCTION.html -int progress_callback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow) +int progress_callback(void* clientp, double dltotal, double dlnow, + double ultotal, double ulnow) { - return progress_callback_xfer(clientp, (curl_off_t)dltotal, (curl_off_t)dlnow, (curl_off_t)ultotal, (curl_off_t)ulnow); + return progress_callback_xfer(clientp, (curl_off_t)dltotal, (curl_off_t)dlnow, + (curl_off_t)ultotal, (curl_off_t)ulnow); } //get the response from HTTP requests, mostly for debug purposes // http://stackoverflow.com/questions/2329571/c-libcurl-get-output-into-a-string // http://curl.haxx.se/libcurl/c/getinmemory.html -size_t writefunc_callback(void *contents, size_t size, size_t nmemb, void *userp) +size_t writefunc_callback(void* contents, size_t size, size_t nmemb, + void* userp) { size_t realsize = size * nmemb; - struct MemoryStruct *mem = (struct MemoryStruct *)userp; + struct MemoryStruct* mem = (struct MemoryStruct*)userp; mem->memory = realloc(mem->memory, mem->size + realsize + 1); - if (mem->memory == NULL) + if (mem->memory == NULL) { /* out of memory! */ debugf(DBG_LEVEL_NORM, KRED"writefunc_callback: realloc() failed"); @@ -331,32 +343,34 @@ size_t writefunc_callback(void *contents, size_t size, size_t nmemb, void *userp // de_cached_entry must be NULL when the file is already in global cache // otherwise point to a new dir_entry that will be added to the cache (usually happens on first dir load) -static int send_request_size(const char *method, const char *path, void *fp, - xmlParserCtxtPtr xmlctx, curl_slist *extra_headers, - off_t file_size, int is_segment, - dir_entry *de_cached_entry, const char *unencoded_path) +static int send_request_size(const char* method, const char* path, void* fp, + xmlParserCtxtPtr xmlctx, curl_slist* extra_headers, + off_t file_size, int is_segment, + dir_entry* de_cached_entry, const char* unencoded_path) { debugf(DBG_LEVEL_EXT, "send_request_size(%s) (%s)", method, path); char url[MAX_URL_SIZE]; char orig_path[MAX_URL_SIZE]; char header_data[MAX_HEADER_SIZE]; - char *slash; + char* slash; long response = -1; int tries = 0; - + //needed to keep the response data, for debug purposes struct MemoryStruct chunk; - if (!storage_url[0]) { + if (!storage_url[0]) + { debugf(DBG_LEVEL_NORM, KRED"send_request with no storage_url?"); abort(); } //char *encoded_path = curl_escape(path, 0); - while ((slash = strstr(path, "%2F")) || (slash = strstr(path, "%2f"))) { + while ((slash = strstr(path, "%2F")) || (slash = strstr(path, "%2f"))) + { *slash = '/'; - memmove(slash+1, slash+3, strlen(slash+3)+1); + memmove(slash + 1, slash + 3, strlen(slash + 3) + 1); } while (*path == '/') path++; @@ -368,10 +382,10 @@ static int send_request_size(const char *method, const char *path, void *fp, { chunk.memory = malloc(1); /* will be grown as needed by the realloc above */ chunk.size = 0; /* no data at this point */ - CURL *curl = get_connection(path); + CURL* curl = get_connection(path); if (rhel5_mode) curl_easy_setopt(curl, CURLOPT_CAINFO, RHEL5_CERTIFICATE_FILE); - curl_slist *headers = NULL; + curl_slist* headers = NULL; curl_easy_setopt(curl, CURLOPT_URL, url); curl_easy_setopt(curl, CURLOPT_HEADER, 0); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); @@ -383,29 +397,35 @@ static int send_request_size(const char *method, const char *path, void *fp, curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 10); curl_easy_setopt(curl, CURLOPT_VERBOSE, option_curl_verbose ? 1 : 0); add_header(&headers, "X-Auth-Token", storage_token); - dir_entry *de; + dir_entry* de; if (de_cached_entry == NULL) - { de = check_path_info(unencoded_path); - } - else { + else + { // updating metadata on a file about to be added to cache (for x-copy, dest meta = src meta) de = de_cached_entry; - debugf(DBG_LEVEL_EXTALL, "send_request_size: using param dir_entry(%s)", orig_path); + debugf(DBG_LEVEL_EXTALL, "send_request_size: using param dir_entry(%s)", + orig_path); } if (!de) + debugf(DBG_LEVEL_EXTALL, + "send_request_size: "KYEL"file not in cache (%s)(%s)(%s)", orig_path, path, + unencoded_path); + else { - debugf(DBG_LEVEL_EXTALL, "send_request_size: "KYEL"file not in cache (%s)(%s)(%s)", orig_path, path, unencoded_path); - } - else { // add headers to save utimens attribs only on upload if (!strcasecmp(method, "PUT") || !strcasecmp(method, "MKDIR")) { - debugf(DBG_LEVEL_EXTALL, "send_request_size: Saving utimens for file %s", orig_path); - debugf(DBG_LEVEL_EXTALL, "send_request_size: Cached utime for path=%s ctime=%li.%li mtime=%li.%li atime=%li.%li", orig_path, - de->ctime.tv_sec, de->ctime.tv_nsec, de->mtime.tv_sec, de->mtime.tv_nsec, de->atime.tv_sec, de->atime.tv_nsec); + debugf(DBG_LEVEL_EXTALL, "send_request_size: Saving utimens for file %s", + orig_path); + debugf(DBG_LEVEL_EXTALL, + "send_request_size: Cached utime for path=%s ctime=%li.%li mtime=%li.%li atime=%li.%li", + orig_path, + de->ctime.tv_sec, de->ctime.tv_nsec, de->mtime.tv_sec, de->mtime.tv_nsec, + de->atime.tv_sec, de->atime.tv_nsec); - char atime_str_nice[TIME_CHARS] = "", mtime_str_nice[TIME_CHARS] = "", ctime_str_nice[TIME_CHARS] = ""; + char atime_str_nice[TIME_CHARS] = "", mtime_str_nice[TIME_CHARS] = "", + ctime_str_nice[TIME_CHARS] = ""; get_timespec_as_str(&(de->atime), atime_str_nice, sizeof(atime_str_nice)); debugf(DBG_LEVEL_EXTALL, KCYN"send_request_size: atime=[%s]", atime_str_nice); get_timespec_as_str(&(de->mtime), mtime_str_nice, sizeof(mtime_str_nice)); @@ -415,9 +435,12 @@ static int send_request_size(const char *method, const char *path, void *fp, char mtime_str[TIME_CHARS], atime_str[TIME_CHARS], ctime_str[TIME_CHARS]; char string_float[TIME_CHARS]; - snprintf(mtime_str, TIME_CHARS, "%lu.%lu", de->mtime.tv_sec, de->mtime.tv_nsec); - snprintf(atime_str, TIME_CHARS, "%lu.%lu", de->atime.tv_sec, de->atime.tv_nsec); - snprintf(ctime_str, TIME_CHARS, "%lu.%lu", de->ctime.tv_sec, de->ctime.tv_nsec); + snprintf(mtime_str, TIME_CHARS, "%lu.%lu", de->mtime.tv_sec, + de->mtime.tv_nsec); + snprintf(atime_str, TIME_CHARS, "%lu.%lu", de->atime.tv_sec, + de->atime.tv_nsec); + snprintf(ctime_str, TIME_CHARS, "%lu.%lu", de->ctime.tv_sec, + de->ctime.tv_nsec); add_header(&headers, HEADER_TEXT_FILEPATH, orig_path); add_header(&headers, HEADER_TEXT_MTIME, mtime_str); add_header(&headers, HEADER_TEXT_ATIME, atime_str); @@ -434,9 +457,9 @@ static int send_request_size(const char *method, const char *path, void *fp, add_header(&headers, HEADER_TEXT_UID, uid_str); add_header(&headers, HEADER_TEXT_CHMOD, chmod_str); } - else { - debugf(DBG_LEVEL_EXTALL, "send_request_size: not setting utimes (%s)", orig_path); - } + else + debugf(DBG_LEVEL_EXTALL, "send_request_size: not setting utimes (%s)", + orig_path); } if (!strcasecmp(method, "MKDIR")) { @@ -457,14 +480,13 @@ static int send_request_size(const char *method, const char *path, void *fp, //http://blog.chmouel.com/2012/02/06/anatomy-of-a-swift-put-query-to-object-server/ debugf(DBG_LEVEL_EXT, "send_request_size: PUT (%s)", orig_path); curl_easy_setopt(curl, CURLOPT_UPLOAD, 1); - if (fp) + if (fp) { curl_easy_setopt(curl, CURLOPT_INFILESIZE, file_size); curl_easy_setopt(curl, CURLOPT_READDATA, fp); } - else { + else curl_easy_setopt(curl, CURLOPT_INFILESIZE, 0); - } if (is_segment) curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_callback); //enable progress reporting @@ -479,15 +501,15 @@ static int send_request_size(const char *method, const char *path, void *fp, /* send all data to this function */ curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writefunc_callback); /* we pass our 'chunk' struct to the callback function */ - curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)&chunk); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&chunk); } else if (!strcasecmp(method, "GET")) { if (is_segment) { debugf(DBG_LEVEL_EXT, "send_request_size: GET SEGMENT (%s)", orig_path); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); } else if (fp) { @@ -496,14 +518,15 @@ static int send_request_size(const char *method, const char *path, void *fp, fflush(fp); if (ftruncate(fileno(fp), 0) < 0) { - debugf(DBG_LEVEL_NORM, KRED"ftruncate failed. I don't know what to do about that."); + debugf(DBG_LEVEL_NORM, + KRED"ftruncate failed. I don't know what to do about that."); abort(); } curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, &header_get_meta_dispatch); // sample by UThreadCurl.cpp, https://bitbucket.org/pamungkas5/bcbcurl/src // and http://www.codeproject.com/Articles/838366/BCBCurl-a-LibCurl-based-download-manager - curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void *)de); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void*)de); struct curl_progress prog; prog.lastruntime = 0; @@ -517,116 +540,135 @@ static int send_request_size(const char *method, const char *path, void *fp, curl_easy_setopt(curl, CURLOPT_WRITEDATA, xmlctx); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &xml_dispatch); } - else { + else + { //asumming retrieval of headers only debugf(DBG_LEVEL_EXT, "send_request_size: GET HEADERS only(%s)"); curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, &header_get_meta_dispatch); - curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void *)de); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, (void*)de); curl_easy_setopt(curl, CURLOPT_NOBODY, 1); } } - else { + else + { debugf(DBG_LEVEL_EXT, "send_request_size: catch_all (%s)"); // this posts an HEAD request (e.g. for statfs) curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, method); curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, &header_dispatch); } /* add the headers from extra_headers if any */ - curl_slist *extra; + curl_slist* extra; for (extra = extra_headers; extra; extra = extra->next) { debugf(DBG_LEVEL_EXT, "adding header: %s", extra->data); headers = curl_slist_append(headers, extra->data); } curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - debugf(DBG_LEVEL_EXT, "status: send_request_size(%s) started HTTP REQ:%s", orig_path, url); + debugf(DBG_LEVEL_EXT, "status: send_request_size(%s) started HTTP REQ:%s", + orig_path, url); curl_easy_perform(curl); double total_time; - char *effective_url; + char* effective_url; curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response); curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &effective_url); curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &total_time); - debugf(DBG_LEVEL_EXT, "status: send_request_size(%s) completed HTTP REQ:%s total_time=%.1f seconds", - orig_path, effective_url, total_time); + debugf(DBG_LEVEL_EXT, + "status: send_request_size(%s) completed HTTP REQ:%s total_time=%.1f seconds", + orig_path, effective_url, total_time); curl_slist_free_all(headers); curl_easy_reset(curl); return_connection(curl); - if (response != 404 && (response >= 400 || response < 200)) + if (response != 404 && (response >= 400 || response < 200)) { /* - * Now, our chunk.memory points to a memory block that is chunk.size - * bytes big and contains the remote file. + Now, our chunk.memory points to a memory block that is chunk.size + bytes big and contains the remote file. */ - debugf(DBG_LEVEL_NORM, KRED"send_request_size: error message, size=%lu, [HTTP %d] (%s)(%s)", - (long)chunk.size, response, method, path); - debugf(DBG_LEVEL_NORM, KRED"send_request_size: error message=[%s]", chunk.memory); + debugf(DBG_LEVEL_NORM, + KRED"send_request_size: error message, size=%lu, [HTTP %d] (%s)(%s)", + (long)chunk.size, response, method, path); + debugf(DBG_LEVEL_NORM, KRED"send_request_size: error message=[%s]", + chunk.memory); } free(chunk.memory); - if ((response >= 200 && response < 400) || (!strcasecmp(method, "DELETE") && response == 409)) + if ((response >= 200 && response < 400) || (!strcasecmp(method, "DELETE") + && response == 409)) { - debugf(DBG_LEVEL_NORM, "exit 0: send_request_size(%s) speed=%.1f sec "KCYN"(%s) "KGRN"[HTTP OK]", - orig_path, total_time, method); + debugf(DBG_LEVEL_NORM, + "exit 0: send_request_size(%s) speed=%.1f sec "KCYN"(%s) "KGRN"[HTTP OK]", + orig_path, total_time, method); return response; } //handle cases when file is not found, no point in retrying, will exit if (response == 404) { - debugf(DBG_LEVEL_NORM, "send_request_size: not found error for (%s)(%s), ignored "KYEL"[HTTP 404].", method, path); + debugf(DBG_LEVEL_NORM, + "send_request_size: not found error for (%s)(%s), ignored "KYEL"[HTTP 404].", + method, path); return response; } - else { - debugf(DBG_LEVEL_NORM, "send_request_size: httpcode=%d (%s)(%s), retrying "KRED"[HTTP ERR]", response, method, path); + else + { + debugf(DBG_LEVEL_NORM, + "send_request_size: httpcode=%d (%s)(%s), retrying "KRED"[HTTP ERR]", response, + method, path); //todo: try to list response content for debug purposes sleep(8 << tries); // backoff } - if (response == 401 && !cloudfs_connect()) - { // re-authenticate on 401s - debugf(DBG_LEVEL_NORM, KYEL"exit 1: send_request_size(%s) (%s) [HTTP REAUTH]", path, method); + if (response == 401 && !cloudfs_connect()) + { + // re-authenticate on 401s + debugf(DBG_LEVEL_NORM, KYEL"exit 1: send_request_size(%s) (%s) [HTTP REAUTH]", + path, method); return response; } if (xmlctx) xmlCtxtResetPush(xmlctx, NULL, 0, NULL, NULL); } - debugf(DBG_LEVEL_NORM, "exit 2: send_request_size(%s)"KCYN"(%s) response=%d", path, method, response); + debugf(DBG_LEVEL_NORM, "exit 2: send_request_size(%s)"KCYN"(%s) response=%d", + path, method, response); return response; } -int send_request(char *method, const char *path, FILE *fp, - xmlParserCtxtPtr xmlctx, curl_slist *extra_headers, dir_entry *de_cached_entry, const char *unencoded_path) +int send_request(char* method, const char* path, FILE* fp, + xmlParserCtxtPtr xmlctx, curl_slist* extra_headers, dir_entry* de_cached_entry, + const char* unencoded_path) { long flen = 0; - if (fp) + if (fp) { // if we don't flush the size will probably be zero fflush(fp); flen = cloudfs_file_size(fileno(fp)); } - return send_request_size(method, path, fp, xmlctx, extra_headers, flen, 0, de_cached_entry, unencoded_path); + return send_request_size(method, path, fp, xmlctx, extra_headers, flen, 0, + de_cached_entry, unencoded_path); } //thread that downloads or uploads large file segments -void *upload_segment(void *seginfo) +void* upload_segment(void* seginfo) { - struct segment_info *info = (struct segment_info *)seginfo; - + struct segment_info* info = (struct segment_info*)seginfo; + char seg_path[MAX_URL_SIZE] = { 0 }; //set pointer to the segment start index in the complete large file (several threads will write to same large file) fseek(info->fp, info->part * info->segment_size, SEEK_SET); setvbuf(info->fp, NULL, _IOFBF, DISK_BUFF_SIZE); snprintf(seg_path, MAX_URL_SIZE, "%s%08i", info->seg_base, info->part); - char *encoded = curl_escape(seg_path, 0); + char* encoded = curl_escape(seg_path, 0); - debugf(DBG_LEVEL_EXT, KCYN"upload_segment(%s) part=%d size=%d seg_size=%d %s", - info->method, info->part, info->size, info->segment_size, seg_path); + debugf(DBG_LEVEL_EXT, KCYN"upload_segment(%s) part=%d size=%d seg_size=%d %s", + info->method, info->part, info->size, info->segment_size, seg_path); int response = send_request_size(info->method, encoded, info, NULL, NULL, - info->size, 1, NULL, seg_path); + info->size, 1, NULL, seg_path); if (!(response >= 200 && response < 300)) - fprintf(stderr, "Segment upload %s failed with response %d", seg_path, response); + fprintf(stderr, "Segment upload %s failed with response %d", seg_path, + response); curl_free(encoded); fclose(info->fp); @@ -635,15 +677,16 @@ void *upload_segment(void *seginfo) // segment_size is the globabl config variable and size_of_segment is local //TODO: return whether the upload/download failed or not -void run_segment_threads(const char *method, int segments, int full_segments, int remaining, - FILE *fp, char *seg_base, int size_of_segments) +void run_segment_threads(const char* method, int segments, int full_segments, + int remaining, + FILE* fp, char* seg_base, int size_of_segments) { debugf(DBG_LEVEL_EXT, "run_segment_threads(%s)", method); char file_path[PATH_MAX] = { 0 }; - struct segment_info *info = (struct segment_info *) - malloc(segments * sizeof(struct segment_info)); + struct segment_info* info = (struct segment_info*) + malloc(segments * sizeof(struct segment_info)); - pthread_t *threads = (pthread_t *)malloc(segments * sizeof(pthread_t)); + pthread_t* threads = (pthread_t*)malloc(segments * sizeof(pthread_t)); #ifdef __linux__ snprintf(file_path, PATH_MAX, "/proc/self/fd/%d", fileno(fp)); debugf(DBG_LEVEL_NORM, "On run segment filepath=%s", file_path); @@ -654,7 +697,7 @@ void run_segment_threads(const char *method, int segments, int full_segments, in #endif int i, ret; - for (i = 0; i < segments; i++) + for (i = 0; i < segments; i++) { info[i].method = method; info[i].fp = fopen(file_path, method[0] == 'G' ? "r+" : "r"); @@ -662,10 +705,11 @@ void run_segment_threads(const char *method, int segments, int full_segments, in info[i].segment_size = size_of_segments; info[i].size = i < full_segments ? size_of_segments : remaining; info[i].seg_base = seg_base; - pthread_create(&threads[i], NULL, upload_segment, (void *)&(info[i])); + pthread_create(&threads[i], NULL, upload_segment, (void*) & (info[i])); } - for (i = 0; i < segments; i++) { + for (i = 0; i < segments; i++) + { if ((ret = pthread_join(threads[i], NULL)) != 0) fprintf(stderr, "error waiting for thread %d, status = %d\n", i, ret); } @@ -674,21 +718,22 @@ void run_segment_threads(const char *method, int segments, int full_segments, in debugf(DBG_LEVEL_EXT, "exit: run_segment_threads(%s)", method); } -void split_path(const char *path, char *seg_base, char *container, - char *object) +void split_path(const char* path, char* seg_base, char* container, + char* object) { - char *string = strdup(path); + char* string = strdup(path); snprintf(seg_base, MAX_URL_SIZE, "%s", strsep(&string, "/")); strncat(container, strsep(&string, "/"), - MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE)); - char *_object = strsep(&string, "/"); - char *remstr; + MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE)); + char* _object = strsep(&string, "/"); + char* remstr; - while (remstr = strsep(&string, "/")) { + while (remstr = strsep(&string, "/")) + { strncat(container, "/", - MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE)); + MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE)); strncat(container, _object, - MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE)); + MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE)); _object = remstr; } //fixed: when removing root folders this will generate a segfault @@ -701,45 +746,52 @@ void split_path(const char *path, char *seg_base, char *container, } //checks on the cloud if this file (seg_path) have an associated segment folder -int internal_is_segmented(const char *seg_path, const char *object, const char *parent_path) +int internal_is_segmented(const char* seg_path, const char* object, + const char* parent_path) { debugf(DBG_LEVEL_EXT, "internal_is_segmented(%s)", seg_path); //try to avoid an additional http request for small files bool potentially_segmented; - dir_entry *de = check_path_info(parent_path); - if (!de) + dir_entry* de = check_path_info(parent_path); + if (!de) { - //when files in folders are first loaded the path will not be yet in cache, so need - //to force segment meta download for segmented files + //when files in folders are first loaded the path will not be yet in cache, so need + //to force segment meta download for segmented files potentially_segmented = true; } - else { + else + { //potentially segmented, assumption is that 0 size files are potentially segmented //while size>0 is for sure not segmented, so no point in making an expensive HTTP GET call potentially_segmented = (de->size == 0 && !de->isdir) ? true : false; } - debugf(DBG_LEVEL_EXT, "internal_is_segmented: potentially segmented=%d", potentially_segmented); - dir_entry *seg_dir; - if (potentially_segmented && cloudfs_list_directory(seg_path, &seg_dir)) + debugf(DBG_LEVEL_EXT, "internal_is_segmented: potentially segmented=%d", + potentially_segmented); + dir_entry* seg_dir; + if (potentially_segmented && cloudfs_list_directory(seg_path, &seg_dir)) { - if (seg_dir && seg_dir->isdir) + if (seg_dir && seg_dir->isdir) { - do { + do + { if (!strncmp(seg_dir->name, object, MAX_URL_SIZE)) { - debugf(DBG_LEVEL_EXT, "exit 0: internal_is_segmented(%s) "KGRN"TRUE", seg_path); - return 1; + debugf(DBG_LEVEL_EXT, "exit 0: internal_is_segmented(%s) "KGRN"TRUE", + seg_path); + return 1; } - } while ((seg_dir = seg_dir->next)); + } + while ((seg_dir = seg_dir->next)); } } - debugf(DBG_LEVEL_EXT, "exit 1: internal_is_segmented(%s) "KYEL"FALSE", seg_path); + debugf(DBG_LEVEL_EXT, "exit 1: internal_is_segmented(%s) "KYEL"FALSE", + seg_path); return 0; } -int is_segmented(const char *path) +int is_segmented(const char* path) { - debugf(DBG_LEVEL_EXT, "is_segmented(%s)", path); + debugf(DBG_LEVEL_EXT, "is_segmented(%s)", path); char container[MAX_URL_SIZE] = { 0 }; char object[MAX_URL_SIZE] = { 0 }; char seg_base[MAX_URL_SIZE] = { 0 }; @@ -752,8 +804,8 @@ int is_segmented(const char *path) //returns segmented file properties by parsing and retrieving the folder structure on the cloud //added totalsize as parameter to return the file size on list directory for segmented files //old implementation returns file size=0 (issue #91) -int format_segments(const char *path, char * seg_base, long *segments, - long *full_segments, long *remaining, long *size_of_segments, long *total_size) +int format_segments(const char* path, char* seg_base, long* segments, + long* full_segments, long* remaining, long* size_of_segments, long* total_size) { debugf(DBG_LEVEL_EXT, "format_segments(%s)", path); char container[MAX_URL_SIZE] = ""; @@ -764,10 +816,10 @@ int format_segments(const char *path, char * seg_base, long *segments, char seg_path[MAX_URL_SIZE]; snprintf(seg_path, MAX_URL_SIZE, "%s/%s_segments", seg_base, container); - if (internal_is_segmented(seg_path, object, path)) + if (internal_is_segmented(seg_path, object, path)) { char manifest[MAX_URL_SIZE]; - dir_entry *seg_dir; + dir_entry* seg_dir; snprintf(manifest, MAX_URL_SIZE, "%s/%s", seg_path, object); debugf(DBG_LEVEL_EXT, KMAG"format_segments manifest(%s)", manifest); @@ -779,7 +831,7 @@ int format_segments(const char *path, char * seg_base, long *segments, // snprintf seesaw between manifest and seg_path to get // the total_size and the segment size as well as the actual objects - char *timestamp = seg_dir->name; + char* timestamp = seg_dir->name; snprintf(seg_path, MAX_URL_SIZE, "%s/%s", manifest, timestamp); debugf(DBG_LEVEL_EXT, KMAG"format_segments seg_path(%s)", seg_path); if (!cloudfs_list_directory(seg_path, &seg_dir)) @@ -788,9 +840,10 @@ int format_segments(const char *path, char * seg_base, long *segments, return 0; } - char *str_size = seg_dir->name; + char* str_size = seg_dir->name; snprintf(manifest, MAX_URL_SIZE, "%s/%s", seg_path, str_size); - debugf(DBG_LEVEL_EXT, KMAG"format_segments manifest2(%s) size=%s", manifest, str_size); + debugf(DBG_LEVEL_EXT, KMAG"format_segments manifest2(%s) size=%s", manifest, + str_size); if (!cloudfs_list_directory(manifest, &seg_dir)) { debugf(DBG_LEVEL_EXT, "exit 2: format_segments(%s)", path); @@ -798,7 +851,7 @@ int format_segments(const char *path, char * seg_base, long *segments, } //following folder name actually represents the parent file size - char *str_segment = seg_dir->name; + char* str_segment = seg_dir->name; snprintf(seg_path, MAX_URL_SIZE, "%s/%s", manifest, str_segment); debugf(DBG_LEVEL_EXT, KMAG"format_segments seg_path2(%s)", seg_path); //here is where we get a list with all segment files composing the parent large file @@ -815,25 +868,27 @@ int format_segments(const char *path, char * seg_base, long *segments, *segments = *full_segments + (*remaining > 0); snprintf(manifest, MAX_URL_SIZE, "%s_segments/%s/%s/%s/%s/", - container, object, timestamp, str_size, str_segment); + container, object, timestamp, str_size, str_segment); char tmp[MAX_URL_SIZE]; strncpy(tmp, seg_base, MAX_URL_SIZE); snprintf(seg_base, MAX_URL_SIZE, "%s/%s", tmp, manifest); debugf(DBG_LEVEL_EXT, KMAG"format_segments seg_base(%s)", seg_base); - debugf(DBG_LEVEL_EXT, KMAG"exit 4: format_segments(%s) total=%d size_of_segments=%d remaining=%d, full_segments=%d segments=%d", - path, &total_size, &size_of_segments, &remaining, &full_segments, &segments); + debugf(DBG_LEVEL_EXT, + KMAG"exit 4: format_segments(%s) total=%d size_of_segments=%d remaining=%d, full_segments=%d segments=%d", + path, &total_size, &size_of_segments, &remaining, &full_segments, &segments); return 1; } - else { + else + { debugf(DBG_LEVEL_EXT, KMAG"exit 5: format_segments(%s) not segmented?", path); return 0; } } /* - * Public interface - */ + Public interface +*/ void cloudfs_init() { @@ -841,7 +896,7 @@ void cloudfs_init() xmlXPathInit(); curl_global_init(CURL_GLOBAL_ALL); pthread_mutex_init(&pool_mut, NULL); - curl_version_info_data *cvid = curl_version_info(CURLVERSION_NOW); + curl_version_info_data* cvid = curl_version_info(CURLVERSION_NOW); // CentOS/RHEL 5 get stupid mode, because they have a broken libcurl if (cvid->version_num == RHEL5_LIBCURL_VERSION) @@ -852,15 +907,15 @@ void cloudfs_init() if (!strncasecmp(cvid->ssl_version, "openssl", 7)) { - #ifdef HAVE_OPENSSL +#ifdef HAVE_OPENSSL int i; - ssl_lockarray = (pthread_mutex_t *)OPENSSL_malloc(CRYPTO_num_locks() * - sizeof(pthread_mutex_t)); + ssl_lockarray = (pthread_mutex_t*)OPENSSL_malloc(CRYPTO_num_locks() * + sizeof(pthread_mutex_t)); for (i = 0; i < CRYPTO_num_locks(); i++) pthread_mutex_init(&(ssl_lockarray[i]), NULL); CRYPTO_set_id_callback((unsigned long (*)())thread_id); CRYPTO_set_locking_callback((void (*)())lock_callback); - #endif +#endif } else if (!strncasecmp(cvid->ssl_version, "nss", 3)) { @@ -874,29 +929,30 @@ void cloudfs_free() debugf(DBG_LEVEL_EXT, "Destroy mutex"); pthread_mutex_destroy(&pool_mut); int n; - for (n = 0; n < curl_pool_count; ++n) { + for (n = 0; n < curl_pool_count; ++n) + { debugf(DBG_LEVEL_EXT, "Cleaning curl conn %d", n); curl_easy_cleanup(curl_pool[n]); } } -int file_is_readable(const char *fname) +int file_is_readable(const char* fname) { - FILE *file; - if ( file = fopen( fname, "r" ) ) - { - fclose( file ); - return 1; - } - return 0; + FILE* file; + if ( file = fopen( fname, "r" ) ) + { + fclose( file ); + return 1; + } + return 0; } -const char * get_file_mimetype ( const char *path ) +const char* get_file_mimetype ( const char* path ) { - if( file_is_readable( path ) == 1 ) + if ( file_is_readable( path ) == 1 ) { magic_t magic; - const char *mime; + const char* mime; magic = magic_open( MAGIC_MIME_TYPE ); magic_load( magic, NULL ); @@ -906,35 +962,32 @@ const char * get_file_mimetype ( const char *path ) return mime; } - const char *error = "application/octet-stream"; + const char* error = "application/octet-stream"; return error; } -int cloudfs_object_read_fp(const char *path, FILE *fp) +int cloudfs_object_read_fp(const char* path, FILE* fp) { debugf(DBG_LEVEL_EXT, "cloudfs_object_read_fp(%s)", path); long flen; fflush(fp); - const char *filemimetype = get_file_mimetype( path ); + const char* filemimetype = get_file_mimetype( path ); // determine the size of the file and segment if it is above the threshhold fseek(fp, 0, SEEK_END); flen = ftell(fp); // delete the previously uploaded segments - if (is_segmented(path)) + if (is_segmented(path)) { - if (!cloudfs_delete_object(path)) - { - debugf(DBG_LEVEL_NORM, KRED"cloudfs_object_read_fp: couldn't delete existing file"); - } - else - { + if (!cloudfs_delete_object(path)) + debugf(DBG_LEVEL_NORM, + KRED"cloudfs_object_read_fp: couldn't delete existing file"); + else debugf(DBG_LEVEL_EXT, KYEL"cloudfs_object_read_fp: deleted existing file"); - } } - + struct timespec now; if (flen >= segment_above) { @@ -961,37 +1014,39 @@ int cloudfs_object_read_fp(const char *path, FILE *fp) // reusing manifest // TODO: check how addition of meta_mtime in manifest impacts utimens implementation snprintf(manifest, MAX_URL_SIZE, "%s_segments/%s/%s/%ld/%ld/", - container, object, meta_mtime, flen, segment_size); + container, object, meta_mtime, flen, segment_size); char tmp[MAX_URL_SIZE]; strncpy(tmp, seg_base, MAX_URL_SIZE); snprintf(seg_base, MAX_URL_SIZE, "%s/%s", tmp, manifest); run_segment_threads("PUT", segments, full_segments, remaining, fp, - seg_base, segment_size); + seg_base, segment_size); - char *encoded = curl_escape(path, 0); - curl_slist *headers = NULL; + char* encoded = curl_escape(path, 0); + curl_slist* headers = NULL; add_header(&headers, "x-object-manifest", manifest); add_header(&headers, "Content-Length", "0"); add_header(&headers, "Content-Type", filemimetype); - int response = send_request_size("PUT", encoded, NULL, NULL, headers, 0, 0, NULL, path); + int response = send_request_size("PUT", encoded, NULL, NULL, headers, 0, 0, + NULL, path); curl_slist_free_all(headers); curl_free(encoded); - debugf(DBG_LEVEL_EXT, "exit 0: cloudfs_object_read_fp(%s) uploaded ok, response=%d", path, response); + debugf(DBG_LEVEL_EXT, + "exit 0: cloudfs_object_read_fp(%s) uploaded ok, response=%d", path, response); return (response >= 200 && response < 300); } - else{ + else + { // assume enters here when file is composed of only one segment (small files) debugf(DBG_LEVEL_EXT, "cloudfs_object_read_fp(%s) "KYEL"unknown state", path); } rewind(fp); - char *encoded = curl_escape(path, 0); - dir_entry *de = path_info(path); + char* encoded = curl_escape(path, 0); + dir_entry* de = path_info(path); if (!de) debugf(DBG_LEVEL_EXT, "cloudfs_object_read_fp(%s) not in cache", path); - else { + else debugf(DBG_LEVEL_EXT, "cloudfs_object_read_fp(%s) found in cache", path); - } int response = send_request("PUT", encoded, fp, NULL, NULL, NULL, path); curl_free(encoded); debugf(DBG_LEVEL_EXT, "exit 1: cloudfs_object_read_fp(%s)", path); @@ -999,10 +1054,10 @@ int cloudfs_object_read_fp(const char *path, FILE *fp) } //write file downloaded from cloud to local file -int cloudfs_object_write_fp(const char *path, FILE *fp) +int cloudfs_object_write_fp(const char* path, FILE* fp) { debugf(DBG_LEVEL_EXT, "cloudfs_object_write_fp(%s)", path); - char *encoded = curl_escape(path, 0); + char* encoded = curl_escape(path, 0); char seg_base[MAX_URL_SIZE] = ""; long segments; @@ -1013,17 +1068,18 @@ int cloudfs_object_write_fp(const char *path, FILE *fp) //checks if this file is a segmented one if (format_segments(path, seg_base, &segments, &full_segments, &remaining, - &size_of_segments, &total_size)) + &size_of_segments, &total_size)) { rewind(fp); fflush(fp); if (ftruncate(fileno(fp), 0) < 0) { - debugf(DBG_LEVEL_NORM, KRED"ftruncate failed. I don't know what to do about that."); + debugf(DBG_LEVEL_NORM, + KRED"ftruncate failed. I don't know what to do about that."); abort(); } run_segment_threads("GET", segments, full_segments, remaining, fp, - seg_base, size_of_segments); + seg_base, size_of_segments); debugf(DBG_LEVEL_EXT, "exit 0: cloudfs_object_write_fp(%s)", path); return 1; } @@ -1031,7 +1087,7 @@ int cloudfs_object_write_fp(const char *path, FILE *fp) int response = send_request("GET", encoded, fp, NULL, NULL, NULL, path); curl_free(encoded); fflush(fp); - if ((response >= 200 && response < 300) || ftruncate(fileno(fp), 0)) + if ((response >= 200 && response < 300) || ftruncate(fileno(fp), 0)) { debugf(DBG_LEVEL_EXT, "exit 1: cloudfs_object_write_fp(%s)", path); return 1; @@ -1041,18 +1097,19 @@ int cloudfs_object_write_fp(const char *path, FILE *fp) return 0; } -int cloudfs_object_truncate(const char *path, off_t size) +int cloudfs_object_truncate(const char* path, off_t size) { - char *encoded = curl_escape(path, 0); + char* encoded = curl_escape(path, 0); int response; if (size == 0) { - FILE *fp = fopen("/dev/null", "r"); + FILE* fp = fopen("/dev/null", "r"); response = send_request("PUT", encoded, fp, NULL, NULL, NULL, path); fclose(fp); } else - {//TODO: this is busted + { + //TODO: this is busted response = send_request("GET", encoded, NULL, NULL, NULL, NULL, path); } curl_free(encoded); @@ -1060,7 +1117,7 @@ int cloudfs_object_truncate(const char *path, off_t size) } //get metadata from cloud, like time attribs. create new entry if not cached yet. -void get_file_metadata(dir_entry *de) +void get_file_metadata(dir_entry* de) { if (de->size == 0 && !de->isdir && !de->metadata_downloaded) { @@ -1072,19 +1129,19 @@ void get_file_metadata(dir_entry *de) long remaining; long size_of_segments; long total_size; - if (format_segments(de->full_name, seg_base, &segments, &full_segments, &remaining, - &size_of_segments, &total_size)) - { + if (format_segments(de->full_name, seg_base, &segments, &full_segments, + &remaining, + &size_of_segments, &total_size)) de->size = total_size; - } } if (option_get_extended_metadata) { debugf(DBG_LEVEL_EXT, KCYN "get_file_metadata(%s)", de->full_name); //retrieve additional file metadata with a quick HEAD query - char *encoded = curl_escape(de->full_name, 0); + char* encoded = curl_escape(de->full_name, 0); de->metadata_downloaded = true; - int response = send_request("GET", encoded, NULL, NULL, NULL, de, de->full_name); + int response = send_request("GET", encoded, NULL, NULL, NULL, de, + de->full_name); curl_free(encoded); debugf(DBG_LEVEL_EXT, KCYN "exit: get_file_metadata(%s)", de->full_name); } @@ -1093,7 +1150,7 @@ void get_file_metadata(dir_entry *de) //get list of folders from cloud // return 1 for OK, 0 for error -int cloudfs_list_directory(const char *path, dir_entry **dir_list) +int cloudfs_list_directory(const char* path, dir_entry** dir_list) { debugf(DBG_LEVEL_EXT, "cloudfs_list_directory(%s)", path); char container[MAX_PATH_SIZE * 3] = ""; @@ -1105,7 +1162,7 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) int entry_count = 0; *dir_list = NULL; - xmlNode *onode = NULL, *anode = NULL, *text_node = NULL; + xmlNode* onode = NULL, *anode = NULL, *text_node = NULL; xmlParserCtxtPtr xmlctx = xmlCreatePushParserCtxt(NULL, NULL, "", 0, NULL); if (!strcmp(path, "") || !strcmp(path, "/")) { @@ -1115,11 +1172,11 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) else { sscanf(path, "/%[^/]/%[^\n]", container, object); - char *encoded_container = curl_escape(container, 0); - char *encoded_object = curl_escape(object, 0); + char* encoded_container = curl_escape(container, 0); + char* encoded_object = curl_escape(object, 0); // The empty path doesn't get a trailing slash, everything else does - char *trailing_slash; + char* trailing_slash; prefix_length = strlen(object); if (object[0] == 0) trailing_slash = ""; @@ -1129,14 +1186,15 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) prefix_length++; } snprintf(container, sizeof(container), "%s?format=xml&delimiter=/&prefix=%s%s", - encoded_container, encoded_object, trailing_slash); + encoded_container, encoded_object, trailing_slash); curl_free(encoded_container); curl_free(encoded_object); } if ((!strcmp(path, "") || !strcmp(path, "/")) && *override_storage_url) response = 404; - else { + else + { // this was generating 404 err on non segmented files (small files) response = send_request("GET", container, NULL, xmlctx, NULL, NULL, path); } @@ -1145,66 +1203,63 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) xmlParseChunk(xmlctx, "", 0, 1); if (response >= 200 && response < 300 && xmlctx->wellFormed ) { - xmlNode *root_element = xmlDocGetRootElement(xmlctx->myDoc); + xmlNode* root_element = xmlDocGetRootElement(xmlctx->myDoc); for (onode = root_element->children; onode; onode = onode->next) { if (onode->type != XML_ELEMENT_NODE) continue; - char is_object = !strcasecmp((const char *)onode->name, "object"); - char is_container = !strcasecmp((const char *)onode->name, "container"); - char is_subdir = !strcasecmp((const char *)onode->name, "subdir"); + char is_object = !strcasecmp((const char*)onode->name, "object"); + char is_container = !strcasecmp((const char*)onode->name, "container"); + char is_subdir = !strcasecmp((const char*)onode->name, "subdir"); if (is_object || is_container || is_subdir) { entry_count++; - dir_entry *de = init_dir_entry(); + dir_entry* de = init_dir_entry(); // useful docs on nodes here: http://developer.openstack.org/api-ref-objectstorage-v1.html if (is_container || is_subdir) de->content_type = strdup("application/directory"); for (anode = onode->children; anode; anode = anode->next) { - char *content = ""; + char* content = ""; for (text_node = anode->children; text_node; text_node = text_node->next) { - if (text_node->type == XML_TEXT_NODE){ - content = (char *)text_node->content; + if (text_node->type == XML_TEXT_NODE) + { + content = (char*)text_node->content; //debugf(DBG_LEVEL_NORM, "List dir anode=%s content=%s", (const char *)anode->name, content); } - else { + else + { //debugf(DBG_LEVEL_NORM, "List dir anode=%s", (const char *)anode->name); } } - if (!strcasecmp((const char *)anode->name, "name")) + if (!strcasecmp((const char*)anode->name, "name")) { de->name = strdup(content + prefix_length); // Remove trailing slash - char *slash = strrchr(de->name, '/'); + char* slash = strrchr(de->name, '/'); if (slash && (0 == *(slash + 1))) *slash = 0; if (asprintf(&(de->full_name), "%s/%s", path, de->name) < 0) - { de->full_name = NULL; - } } - if (!strcasecmp((const char *)anode->name, "bytes")) - { - de->size = strtoll(content, NULL, 10); - } - if (!strcasecmp((const char *)anode->name, "content_type")) + if (!strcasecmp((const char*)anode->name, "bytes")) + de->size = strtoll(content, NULL, 10); + if (!strcasecmp((const char*)anode->name, "content_type")) { de->content_type = strdup(content); - char *semicolon = strchr(de->content_type, ';'); + char* semicolon = strchr(de->content_type, ';'); if (semicolon) *semicolon = '\0'; } - if (!strcasecmp((const char *)anode->name, "hash")) - { + if (!strcasecmp((const char*)anode->name, "hash")) de->md5sum = strdup(content); - } - if (!strcasecmp((const char *)anode->name, "last_modified")) + if (!strcasecmp((const char*)anode->name, "last_modified")) { time_t last_modified_t = get_time_from_str_as_gmt(content); char local_time_str[64]; - time_t local_time_t = get_time_as_local(last_modified_t, local_time_str, sizeof(local_time_str)); + time_t local_time_t = get_time_as_local(last_modified_t, local_time_str, + sizeof(local_time_str)); de->last_modified = local_time_t; de->ctime.tv_sec = local_time_t; de->ctime.tv_nsec = 0; @@ -1217,18 +1272,20 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) } } de->isdir = de->content_type && - ((strstr(de->content_type, "application/folder") != NULL) || - (strstr(de->content_type, "application/directory") != NULL)); + ((strstr(de->content_type, "application/folder") != NULL) || + (strstr(de->content_type, "application/directory") != NULL)); de->islink = de->content_type && - ((strstr(de->content_type, "application/link") != NULL)); + ((strstr(de->content_type, "application/link") != NULL)); if (de->isdir) { - //i guess this will remove a dir_entry from cache if is there already + //i guess this will remove a dir_entry from cache if is there already if (!strncasecmp(de->name, last_subdir, sizeof(last_subdir))) { //todo: check why is needed and if memory is freed properly, seems to generate many missed delete operations //cloudfs_free_dir_list(de); - debugf(DBG_LEVEL_EXT, "cloudfs_list_directory: "KYEL"ignore "KNRM"cloudfs_free_dir_list(%s) command", de->name); + debugf(DBG_LEVEL_EXT, + "cloudfs_list_directory: "KYEL"ignore "KNRM"cloudfs_free_dir_list(%s) command", + de->name); continue; } strncpy(last_subdir, de->name, sizeof(last_subdir)); @@ -1237,20 +1294,19 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) *dir_list = de; char time_str[TIME_CHARS] = { 0 }; get_timespec_as_str(&(de->mtime), time_str, sizeof(time_str)); - debugf(DBG_LEVEL_EXT, KCYN"new dir_entry %s size=%d %s dir=%d lnk=%d mod=[%s]", - de->full_name, de->size, de->content_type, de->isdir, de->islink, time_str); + debugf(DBG_LEVEL_EXT, KCYN"new dir_entry %s size=%d %s dir=%d lnk=%d mod=[%s]", + de->full_name, de->size, de->content_type, de->isdir, de->islink, time_str); } - else { + else debugf(DBG_LEVEL_EXT, "unknown element: %s", onode->name); - } } retval = 1; } - else if ((!strcmp(path, "") || !strcmp(path, "/")) && *override_storage_url) + else if ((!strcmp(path, "") || !strcmp(path, "/")) && *override_storage_url) { entry_count = 1; debugf(DBG_LEVEL_NORM, "Init cache entry container=[%s]", public_container); - dir_entry *de = init_dir_entry(); + dir_entry* de = init_dir_entry(); de->name = strdup(public_container); struct tm last_modified; //todo: check what this default time means? @@ -1272,7 +1328,7 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) return retval; } -int cloudfs_delete_object(const char *path) +int cloudfs_delete_object(const char* path) { debugf(DBG_LEVEL_EXT, "cloudfs_delete_object(%s)", path); char seg_base[MAX_URL_SIZE] = ""; @@ -1284,7 +1340,7 @@ int cloudfs_delete_object(const char *path) long total_size; if (format_segments(path, seg_base, &segments, &full_segments, &remaining, - &size_of_segments, &total_size)) + &size_of_segments, &total_size)) { int response; int i; @@ -1292,21 +1348,23 @@ int cloudfs_delete_object(const char *path) for (i = 0; i < segments; i++) { snprintf(seg_path, MAX_URL_SIZE, "%s%08i", seg_base, i); - char *encoded = curl_escape(seg_path, 0); + char* encoded = curl_escape(seg_path, 0); response = send_request("DELETE", encoded, NULL, NULL, NULL, NULL, seg_path); if (response < 200 || response >= 300) { - debugf(DBG_LEVEL_EXT, "exit 1: cloudfs_delete_object(%s) response=%d", path, response); + debugf(DBG_LEVEL_EXT, "exit 1: cloudfs_delete_object(%s) response=%d", path, + response); return 0; } } } - char *encoded = curl_escape(path, 0); + char* encoded = curl_escape(path, 0); int response = send_request("DELETE", encoded, NULL, NULL, NULL, NULL, path); curl_free(encoded); int ret = (response >= 200 && response < 300); - debugf(DBG_LEVEL_EXT, "status: cloudfs_delete_object(%s) response=%d", path, response); + debugf(DBG_LEVEL_EXT, "status: cloudfs_delete_object(%s) response=%d", path, + response); if (response == 409) { debugf(DBG_LEVEL_EXT, "status: cloudfs_delete_object(%s) NOT EMPTY", path); @@ -1318,49 +1376,54 @@ int cloudfs_delete_object(const char *path) //fixme: this op does not preserve src attributes (e.g. will make rsync not work well) // https://ask.openstack.org/en/question/14307/is-there-a-way-to-moverename-an-object/ // this operation also causes an HTTP 400 error if X-Object-Meta-FilePath value is larger than 256 chars -int cloudfs_copy_object(const char *src, const char *dst) +int cloudfs_copy_object(const char* src, const char* dst) { - debugf(DBG_LEVEL_EXT, "cloudfs_copy_object(%s, %s) lensrc=%d, lendst=%d", src, dst, strlen(src), strlen(dst)); + debugf(DBG_LEVEL_EXT, "cloudfs_copy_object(%s, %s) lensrc=%d, lendst=%d", src, + dst, strlen(src), strlen(dst)); + + char* dst_encoded = curl_escape(dst, 0); + char* src_encoded = curl_escape(src, 0); - char *dst_encoded = curl_escape(dst, 0); - char *src_encoded = curl_escape(src, 0); - //convert encoded string (slashes are encoded as well) to encoded string with slashes - char *slash; - while ((slash = strstr(src_encoded, "%2F")) || (slash = strstr(src_encoded, "%2f"))) { + char* slash; + while ((slash = strstr(src_encoded, "%2F")) + || (slash = strstr(src_encoded, "%2f"))) + { *slash = '/'; memmove(slash + 1, slash + 3, strlen(slash + 3) + 1); } - curl_slist *headers = NULL; + curl_slist* headers = NULL; add_header(&headers, "X-Copy-From", src_encoded); add_header(&headers, "Content-Length", "0"); //get source file entry - dir_entry *de_src = check_path_info(src); - if (de_src) { - debugf(DBG_LEVEL_EXT, "status cloudfs_copy_object(%s, %s): src file found", src, dst); - } - else { - debugf(DBG_LEVEL_NORM, KRED"status cloudfs_copy_object(%s, %s): src file NOT found", src, dst); - } + dir_entry* de_src = check_path_info(src); + if (de_src) + debugf(DBG_LEVEL_EXT, "status cloudfs_copy_object(%s, %s): src file found", + src, dst); + else + debugf(DBG_LEVEL_NORM, + KRED"status cloudfs_copy_object(%s, %s): src file NOT found", src, dst); //pass src metadata so that PUT will set time attributes of the src file - int response = send_request("PUT", dst_encoded, NULL, NULL, headers, de_src, dst); + int response = send_request("PUT", dst_encoded, NULL, NULL, headers, de_src, + dst); curl_free(dst_encoded); curl_free(src_encoded); curl_slist_free_all(headers); - debugf(DBG_LEVEL_EXT, "exit: cloudfs_copy_object(%s,%s) response=%d", src, dst, response); + debugf(DBG_LEVEL_EXT, "exit: cloudfs_copy_object(%s,%s) response=%d", src, dst, + response); return (response >= 200 && response < 300); } // http://developer.openstack.org/api-ref-objectstorage-v1.html#updateObjectMeta -int cloudfs_update_meta(dir_entry *de) +int cloudfs_update_meta(dir_entry* de) { int response = cloudfs_copy_object(de->full_name, de->full_name); return response; } //optimised with cache -int cloudfs_statfs(const char *path, struct statvfs *stat) +int cloudfs_statfs(const char* path, struct statvfs* stat) { time_t now = get_time_now(); int lapsed = now - last_stat_read_time; @@ -1369,20 +1432,24 @@ int cloudfs_statfs(const char *path, struct statvfs *stat) //todo: check why stat head request is always set to /, why not path? int response = send_request("HEAD", "/", NULL, NULL, NULL, NULL, "/"); *stat = statcache; - debugf(DBG_LEVEL_EXT, "exit: cloudfs_statfs (new recent values, was cached since %d seconds)", lapsed); + debugf(DBG_LEVEL_EXT, + "exit: cloudfs_statfs (new recent values, was cached since %d seconds)", + lapsed); last_stat_read_time = now; return (response >= 200 && response < 300); } - else { - debugf(DBG_LEVEL_EXT, "exit: cloudfs_statfs (old values, cached since %d seconds)", lapsed); + else + { + debugf(DBG_LEVEL_EXT, + "exit: cloudfs_statfs (old values, cached since %d seconds)", lapsed); return 1; } } -int cloudfs_create_symlink(const char *src, const char *dst) +int cloudfs_create_symlink(const char* src, const char* dst) { - char *dst_encoded = curl_escape(dst, 0); - FILE *lnk = tmpfile(); + char* dst_encoded = curl_escape(dst, 0); + FILE* lnk = tmpfile(); fwrite(src, 1, strlen(src), lnk); fwrite("\0", 1, 1, lnk); int response = send_request("MKLINK", dst_encoded, lnk, NULL, NULL, NULL, dst); @@ -1391,13 +1458,14 @@ int cloudfs_create_symlink(const char *src, const char *dst) return (response >= 200 && response < 300); } -int cloudfs_create_directory(const char *path) +int cloudfs_create_directory(const char* path) { debugf(DBG_LEVEL_EXT, "cloudfs_create_directory(%s)", path); - char *encoded = curl_escape(path, 0); + char* encoded = curl_escape(path, 0); int response = send_request("MKDIR", encoded, NULL, NULL, NULL, NULL, path); curl_free(encoded); - debugf(DBG_LEVEL_EXT, "cloudfs_create_directory(%s) response=%d", path, response); + debugf(DBG_LEVEL_EXT, "cloudfs_create_directory(%s) response=%d", path, + response); return (response >= 200 && response < 300); } @@ -1423,32 +1491,40 @@ void cloudfs_option_curl_verbose(int option) option_curl_verbose = option ? true : false; } -static struct { +static struct +{ char client_id [MAX_HEADER_SIZE]; char client_secret[MAX_HEADER_SIZE]; char refresh_token[MAX_HEADER_SIZE]; } reconnect_args; -void cloudfs_set_credentials(char *client_id, char *client_secret, char *refresh_token) +void cloudfs_set_credentials(char* client_id, char* client_secret, + char* refresh_token) { - strncpy(reconnect_args.client_id , client_id , sizeof(reconnect_args.client_id )); - strncpy(reconnect_args.client_secret, client_secret, sizeof(reconnect_args.client_secret)); - strncpy(reconnect_args.refresh_token, refresh_token, sizeof(reconnect_args.refresh_token)); + strncpy(reconnect_args.client_id , client_id , + sizeof(reconnect_args.client_id )); + strncpy(reconnect_args.client_secret, client_secret, + sizeof(reconnect_args.client_secret)); + strncpy(reconnect_args.refresh_token, refresh_token, + sizeof(reconnect_args.refresh_token)); } -struct htmlString { - char *text; +struct htmlString +{ + char* text; size_t size; }; -static size_t writefunc_string(void *contents, size_t size, size_t nmemb, void *data) +static size_t writefunc_string(void* contents, size_t size, size_t nmemb, + void* data) { - struct htmlString *mem = (struct htmlString *) data; + struct htmlString* mem = (struct htmlString*) data; size_t realsize = size * nmemb; mem->text = realloc(mem->text, mem->size + realsize + 1); - if (mem->text == NULL) { /* out of memory! */ - perror(__FILE__); - exit(EXIT_FAILURE); + if (mem->text == NULL) /* out of memory! */ + { + perror(__FILE__); + exit(EXIT_FAILURE); } memcpy(&(mem->text[mem->size]), contents, realsize); @@ -1456,7 +1532,7 @@ static size_t writefunc_string(void *contents, size_t size, size_t nmemb, void * return realsize; } -char* htmlStringGet(CURL *curl) +char* htmlStringGet(CURL* curl) { struct htmlString chunk; chunk.text = malloc(sizeof(char)); @@ -1464,20 +1540,22 @@ char* htmlStringGet(CURL *curl) chunk.text[0] = '\0';//added to avoid valgrind unitialised warning curl_easy_setopt(curl, CURLOPT_WRITEDATA, &chunk); - do { + do + { curl_easy_perform(curl); - } while (chunk.size == 0); + } + while (chunk.size == 0); chunk.text[chunk.size] = '\0'; return chunk.text; } /* thanks to http://devenix.wordpress.com */ -char *unbase64(unsigned char *input, int length) +char* unbase64(unsigned char* input, int length) { - BIO *b64, *bmem; + BIO* b64, *bmem; - char *buffer = (char *)malloc(length); + char* buffer = (char*)malloc(length); memset(buffer, 0, length); b64 = BIO_new(BIO_f_base64()); @@ -1492,13 +1570,13 @@ char *unbase64(unsigned char *input, int length) return buffer; } -int safe_json_string(json_object *jobj, char *buffer, char *name) +int safe_json_string(json_object* jobj, char* buffer, char* name) { int result = 0; if (jobj) { - json_object *o; + json_object* o; int found; found = json_object_object_get_ex(jobj, name, &o); if (found) @@ -1516,22 +1594,23 @@ int safe_json_string(json_object *jobj, char *buffer, char *name) int cloudfs_connect() { - #define HUBIC_TOKEN_URL "https://api.hubic.com/oauth/token" - #define HUBIC_CRED_URL "https://api.hubic.com/1.0/account/credentials" - #define HUBIC_CLIENT_ID (reconnect_args.client_id) - #define HUBIC_CLIENT_SECRET (reconnect_args.client_secret) - #define HUBIC_REFRESH_TOKEN (reconnect_args.refresh_token) - #define HUBIC_OPTIONS_SIZE 2048 +#define HUBIC_TOKEN_URL "https://api.hubic.com/oauth/token" +#define HUBIC_CRED_URL "https://api.hubic.com/1.0/account/credentials" +#define HUBIC_CLIENT_ID (reconnect_args.client_id) +#define HUBIC_CLIENT_SECRET (reconnect_args.client_secret) +#define HUBIC_REFRESH_TOKEN (reconnect_args.refresh_token) +#define HUBIC_OPTIONS_SIZE 2048 long response = -1; char url[HUBIC_OPTIONS_SIZE]; char payload[HUBIC_OPTIONS_SIZE]; - struct json_object *json_obj; + struct json_object* json_obj; pthread_mutex_lock(&pool_mut); - debugf(DBG_LEVEL_NORM, "Authenticating... (client_id = '%s')", HUBIC_CLIENT_ID); + debugf(DBG_LEVEL_NORM, "Authenticating... (client_id = '%s')", + HUBIC_CLIENT_ID); storage_token[0] = storage_url[0] = '\0'; - CURL *curl = curl_easy_init(); + CURL* curl = curl_easy_init(); /* curl default options */ curl_easy_setopt(curl, CURLOPT_VERBOSE, debug); @@ -1551,7 +1630,8 @@ int cloudfs_connect() /* Step 2 : get request code - Not needed anymore with refresh_token */ /* Step 3 : get access token */ - sprintf(payload, "refresh_token=%s&grant_type=refresh_token", HUBIC_REFRESH_TOKEN); + sprintf(payload, "refresh_token=%s&grant_type=refresh_token", + HUBIC_REFRESH_TOKEN); curl_easy_setopt(curl, CURLOPT_URL, HUBIC_TOKEN_URL); curl_easy_setopt(curl, CURLOPT_POST, 1L); curl_easy_setopt(curl, CURLOPT_HEADER, 0); @@ -1561,7 +1641,7 @@ int cloudfs_connect() curl_easy_setopt(curl, CURLOPT_PASSWORD, HUBIC_CLIENT_SECRET); curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); - char *json_str = htmlStringGet(curl); + char* json_str = htmlStringGet(curl); json_obj = json_tokener_parse(json_str); debugf(DBG_LEVEL_NORM, "HUBIC TOKEN_URL result: '%s'\n", json_str); free(json_str); @@ -1570,7 +1650,7 @@ int cloudfs_connect() char token_type[HUBIC_OPTIONS_SIZE]; int expire_sec; int found; - json_object *o; + json_object* o; if (!safe_json_string(json_obj, access_token, "access_token")) return 0; @@ -1590,7 +1670,7 @@ int cloudfs_connect() curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_NONE); /* create the Bearer authentication header */ - curl_slist *headers = NULL; + curl_slist* headers = NULL; sprintf (payload, "Bearer %s", access_token); add_header(&headers, "Authorization", payload); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); @@ -1616,5 +1696,6 @@ int cloudfs_connect() curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response); curl_easy_cleanup(curl); pthread_mutex_unlock(&pool_mut); - return (response >= 200 && response < 300 && storage_token[0] && storage_url[0]); + return (response >= 200 && response < 300 && storage_token[0] + && storage_url[0]); } diff --git a/cloudfsapi.h b/cloudfsapi.h index b9438c4..2dfbc76 100644 --- a/cloudfsapi.h +++ b/cloudfsapi.h @@ -16,12 +16,14 @@ typedef struct curl_slist curl_slist; #define MINIMAL_PROGRESS_FUNCTIONALITY_INTERVAL 5 -struct curl_progress { +struct curl_progress +{ double lastruntime; - CURL *curl; + CURL* curl; }; -typedef struct options { +typedef struct options +{ char cache_timeout[OPTION_SIZE]; char verify_ssl[OPTION_SIZE]; char segment_size[OPTION_SIZE]; @@ -34,7 +36,8 @@ typedef struct options { char refresh_token[OPTION_SIZE]; } FuseOptions; -typedef struct extra_options { +typedef struct extra_options +{ char get_extended_metadata[OPTION_SIZE]; char curl_verbose[OPTION_SIZE]; char cache_statfs_timeout[OPTION_SIZE]; @@ -46,40 +49,41 @@ typedef struct extra_options { void cloudfs_init(void); void cloudfs_free(void); -void cloudfs_set_credentials(char *client_id, char *client_secret, char *refresh_token); +void cloudfs_set_credentials(char* client_id, char* client_secret, + char* refresh_token); int cloudfs_connect(void); struct segment_info { - FILE *fp; + FILE* fp; int part; long size; long segment_size; - char *seg_base; - const char *method; + char* seg_base; + const char* method; }; long segment_size; long segment_above; -char *override_storage_url; -char *public_container; +char* override_storage_url; +char* public_container; -int file_is_readable(const char *fname); -const char * get_file_mimetype ( const char *filename ); -int cloudfs_object_read_fp(const char *path, FILE *fp); -int cloudfs_object_write_fp(const char *path, FILE *fp); -int cloudfs_list_directory(const char *path, dir_entry **); -int cloudfs_delete_object(const char *path); -int cloudfs_copy_object(const char *src, const char *dst); -int cloudfs_create_symlink(const char *src, const char *dst); -int cloudfs_create_directory(const char *label); -int cloudfs_object_truncate(const char *path, off_t size); +int file_is_readable(const char* fname); +const char* get_file_mimetype ( const char* filename ); +int cloudfs_object_read_fp(const char* path, FILE* fp); +int cloudfs_object_write_fp(const char* path, FILE* fp); +int cloudfs_list_directory(const char* path, dir_entry**); +int cloudfs_delete_object(const char* path); +int cloudfs_copy_object(const char* src, const char* dst); +int cloudfs_create_symlink(const char* src, const char* dst); +int cloudfs_create_directory(const char* label); +int cloudfs_object_truncate(const char* path, off_t size); off_t cloudfs_file_size(int fd); -int cloudfs_statfs(const char *path, struct statvfs *stat); +int cloudfs_statfs(const char* path, struct statvfs* stat); void cloudfs_verify_ssl(int dbg); void cloudfs_option_get_extended_metadata(int option); void cloudfs_option_curl_verbose(int option); -void get_file_metadata(dir_entry *de); -int cloudfs_update_meta(dir_entry *de); +void get_file_metadata(dir_entry* de); +int cloudfs_update_meta(dir_entry* de); #endif diff --git a/cloudfuse.c b/cloudfuse.c index 8eae2a7..1900a32 100644 --- a/cloudfuse.c +++ b/cloudfuse.c @@ -19,7 +19,7 @@ #include "cloudfsapi.h" #include "config.h" -extern char *temp_dir; +extern char* temp_dir; extern pthread_mutex_t dcachemut; extern pthread_mutexattr_t mutex_attr; extern int debug; @@ -38,10 +38,10 @@ typedef struct int flags; } openfile; -static int cfs_getattr(const char *path, struct stat *stbuf) +static int cfs_getattr(const char* path, struct stat* stbuf) { debugf(DBG_LEVEL_NORM, KBLU "cfs_getattr(%s)", path); - + //return standard values for root folder if (!strcmp(path, "/")) @@ -55,22 +55,25 @@ static int cfs_getattr(const char *path, struct stat *stbuf) return 0; } //get file. if not in cache will be downloaded. - dir_entry *de = path_info(path); - if (!de) { + dir_entry* de = path_info(path); + if (!de) + { debug_list_cache_content(); - debugf(DBG_LEVEL_NORM, KBLU"exit 1: cfs_getattr(%s) "KYEL"not-in-cache/cloud", path); + debugf(DBG_LEVEL_NORM, KBLU"exit 1: cfs_getattr(%s) "KYEL"not-in-cache/cloud", + path); return -ENOENT; } - + //lazzy download of file metadata, only when really needed - if (option_get_extended_metadata && !de->metadata_downloaded) { + if (option_get_extended_metadata && !de->metadata_downloaded) get_file_metadata(de); - } - if (option_enable_chown) { + if (option_enable_chown) + { stbuf->st_uid = de->uid; stbuf->st_gid = de->gid; } - else { + else + { stbuf->st_uid = geteuid(); stbuf->st_gid = getegid(); } @@ -91,21 +94,25 @@ static int cfs_getattr(const char *path, struct stat *stbuf) int default_mode_dir, default_mode_file; - if (option_enable_chmod) { + if (option_enable_chmod) + { default_mode_dir = de->chmod; default_mode_file = de->chmod; } - else { + else + { default_mode_dir = 0755; default_mode_file = 0666; } - if (de->isdir) { + if (de->isdir) + { stbuf->st_size = 0; stbuf->st_mode = S_IFDIR | default_mode_dir; stbuf->st_nlink = 2; } - else if (de->islink) { + else if (de->islink) + { stbuf->st_size = 1; stbuf->st_mode = S_IFLNK | default_mode_dir; stbuf->st_nlink = 1; @@ -113,7 +120,8 @@ static int cfs_getattr(const char *path, struct stat *stbuf) /* calc. blocks as if 4K blocksize filesystem; stat uses units of 512B */ stbuf->st_blocks = ((4095 + de->size) / 4096) * 8; } - else { + else + { stbuf->st_size = de->size; /* calc. blocks as if 4K blocksize filesystem; stat uses units of 512B */ stbuf->st_blocks = ((4095 + de->size) / 4096) * 8; @@ -124,26 +132,27 @@ static int cfs_getattr(const char *path, struct stat *stbuf) return 0; } -static int cfs_fgetattr(const char *path, struct stat *stbuf, struct fuse_file_info *info) +static int cfs_fgetattr(const char* path, struct stat* stbuf, + struct fuse_file_info* info) { debugf(DBG_LEVEL_NORM, KBLU "cfs_fgetattr(%s)", path); - openfile *of = (openfile *)(uintptr_t)info->fh; + openfile* of = (openfile*)(uintptr_t)info->fh; if (of) { //get file. if not in cache will be downloaded. - dir_entry *de = path_info(path); - if (!de) { + dir_entry* de = path_info(path); + if (!de) + { debug_list_cache_content(); - debugf(DBG_LEVEL_NORM, KBLU"exit 1: cfs_fgetattr(%s) "KYEL"not-in-cache/cloud", path); + debugf(DBG_LEVEL_NORM, KBLU"exit 1: cfs_fgetattr(%s) "KYEL"not-in-cache/cloud", + path); return -ENOENT; } int default_mode_file; - if (option_enable_chmod) { + if (option_enable_chmod) default_mode_file = de->chmod; - } - else { + else default_mode_file = 0666; - } stbuf->st_size = cloudfs_file_size(of->fd); stbuf->st_mode = S_IFREG | default_mode_file; @@ -155,11 +164,13 @@ static int cfs_fgetattr(const char *path, struct stat *stbuf, struct fuse_file_i return -ENOENT; } -static int cfs_readdir(const char *path, void *buf, fuse_fill_dir_t filldir, off_t offset, struct fuse_file_info *info) +static int cfs_readdir(const char* path, void* buf, fuse_fill_dir_t filldir, + off_t offset, struct fuse_file_info* info) { debugf(DBG_LEVEL_NORM, KBLU "cfs_readdir(%s)", path); - dir_entry *de; - if (!caching_list_directory(path, &de)) { + dir_entry* de; + if (!caching_list_directory(path, &de)) + { debug_list_cache_content(); debugf(DBG_LEVEL_NORM, KRED "exit 0: cfs_readdir(%s)", path); return -ENOLINK; @@ -173,53 +184,65 @@ static int cfs_readdir(const char *path, void *buf, fuse_fill_dir_t filldir, off return 0; } -static int cfs_mkdir(const char *path, mode_t mode) +static int cfs_mkdir(const char* path, mode_t mode) { debugf(DBG_LEVEL_NORM, KBLU "cfs_mkdir(%s)", path); int response = cloudfs_create_directory(path); - if (response){ + if (response) + { update_dir_cache(path, 0, 1, 0); debug_list_cache_content(); debugf(DBG_LEVEL_NORM, KBLU "exit 0: cfs_mkdir(%s)", path); return 0; } - debugf(DBG_LEVEL_NORM, KRED "exit 1: cfs_mkdir(%s) response=%d", path, response); + debugf(DBG_LEVEL_NORM, KRED "exit 1: cfs_mkdir(%s) response=%d", path, + response); return -ENOENT; } -static int cfs_create(const char *path, mode_t mode, struct fuse_file_info *info) +static int cfs_create(const char* path, mode_t mode, + struct fuse_file_info* info) { debugf(DBG_LEVEL_NORM, KBLU "cfs_create(%s)", path); - FILE *temp_file; + FILE* temp_file; int errsv; char file_path_safe[NAME_MAX] = ""; - if (*temp_dir) { + if (*temp_dir) + { get_safe_cache_file_path(path, file_path_safe, temp_dir); temp_file = fopen(file_path_safe, "w+b"); errsv = errno; - if (temp_file == NULL){ - debugf(DBG_LEVEL_NORM, KRED "exit 0: cfs_create cannot open temp file %s.error %s\n", file_path_safe, strerror(errsv)); + if (temp_file == NULL) + { + debugf(DBG_LEVEL_NORM, KRED + "exit 0: cfs_create cannot open temp file %s.error %s\n", file_path_safe, + strerror(errsv)); return -EIO; } } - else { + else + { temp_file = tmpfile(); errsv = errno; - if (temp_file == NULL){ - debugf(DBG_LEVEL_NORM, KRED "exit 1: cfs_create cannot open tmp file for path %s.error %s\n", path, strerror(errsv)); + if (temp_file == NULL) + { + debugf(DBG_LEVEL_NORM, KRED + "exit 1: cfs_create cannot open tmp file for path %s.error %s\n", path, + strerror(errsv)); return -EIO; } } - openfile *of = (openfile *)malloc(sizeof(openfile)); + openfile* of = (openfile*)malloc(sizeof(openfile)); of->fd = dup(fileno(temp_file)); fclose(temp_file); of->flags = info->flags; info->fh = (uintptr_t)of; update_dir_cache(path, 0, 0, 0); info->direct_io = 1; - dir_entry *de = check_path_info(path); - if (de) { + dir_entry* de = check_path_info(path); + if (de) + { debugf(DBG_LEVEL_EXT, KCYN"cfs_create(%s): found in cache", path); struct timespec now; clock_gettime(CLOCK_REALTIME, &now); @@ -244,87 +267,102 @@ static int cfs_create(const char *path, mode_t mode, struct fuse_file_info *info de->uid = geteuid(); de->gid = getegid(); } - else { + else debugf(DBG_LEVEL_EXT, KBLU "cfs_create(%s) "KYEL"dir-entry not found", path); - } - debugf(DBG_LEVEL_NORM, KBLU "exit 2: cfs_create(%s)=(%s) result=%d:%s", path, file_path_safe, errsv, strerror(errsv)); + debugf(DBG_LEVEL_NORM, KBLU "exit 2: cfs_create(%s)=(%s) result=%d:%s", path, + file_path_safe, errsv, strerror(errsv)); return 0; } // open (download) file from cloud // todo: implement etag optimisation, download only if content changed, http://www.17od.com/2012/12/19/ten-useful-openstack-swift-features/ -static int cfs_open(const char *path, struct fuse_file_info *info) +static int cfs_open(const char* path, struct fuse_file_info* info) { debugf(DBG_LEVEL_NORM, KBLU "cfs_open(%s)", path); - FILE *temp_file = NULL; + FILE* temp_file = NULL; int errsv; - dir_entry *de = path_info(path); + dir_entry* de = path_info(path); - if (*temp_dir) { + if (*temp_dir) + { char file_path_safe[NAME_MAX]; get_safe_cache_file_path(path, file_path_safe, temp_dir); debugf(DBG_LEVEL_EXT, "cfs_open: try open (%s)", file_path_safe); - if (access(file_path_safe, F_OK) != -1){ + if (access(file_path_safe, F_OK) != -1) + { // file exists temp_file = fopen(file_path_safe, "r"); errsv = errno; - if (temp_file == NULL) { - debugf(DBG_LEVEL_NORM, KRED"exit 0: cfs_open can't open temp_file=[%s] err=%d:%s", file_path_safe, errsv, strerror(errsv)); + if (temp_file == NULL) + { + debugf(DBG_LEVEL_NORM, + KRED"exit 0: cfs_open can't open temp_file=[%s] err=%d:%s", file_path_safe, + errsv, strerror(errsv)); return -ENOENT; } else debugf(DBG_LEVEL_EXT, "cfs_open: file exists"); } - else { + else + { errsv = errno; debugf(DBG_LEVEL_EXT, "cfs_open: file not in cache, err=%s", strerror(errsv)); //FIXME: commented out as this condition will not be meet in some odd cases and program will crash //if (!(info->flags & O_WRONLY)) { - debugf(DBG_LEVEL_EXT, "cfs_open: opening for write"); + debugf(DBG_LEVEL_EXT, "cfs_open: opening for write"); - // we need to lock on the filename another process could open the file - // while we are writing to it and then only read part of the file + // we need to lock on the filename another process could open the file + // while we are writing to it and then only read part of the file - // duplicate the directory caching datastructure to make the code easier - // to understand. + // duplicate the directory caching datastructure to make the code easier + // to understand. - // each file in the cache needs: - // filename, is_writing, last_closed, is_removing - // the first time a file is opened a new entry is created in the cache - // setting the filename and is_writing to true. This check needs to be - // wrapped with a lock. - // - // each time a file is closed we set the last_closed for the file to now - // and we check the cache for files whose last - // closed is greater than cache_timeout, then start a new thread rming - // that file. + // each file in the cache needs: + // filename, is_writing, last_closed, is_removing + // the first time a file is opened a new entry is created in the cache + // setting the filename and is_writing to true. This check needs to be + // wrapped with a lock. + // + // each time a file is closed we set the last_closed for the file to now + // and we check the cache for files whose last + // closed is greater than cache_timeout, then start a new thread rming + // that file. - // TODO: just to prevent this craziness for now - temp_file = fopen(file_path_safe, "w+b"); - errsv = errno; - if (temp_file == NULL) { - debugf(DBG_LEVEL_NORM, KRED"exit 1: cfs_open cannot open temp_file=[%s] err=%d:%s", file_path_safe, errsv, strerror(errsv)); - return -ENOENT; - } + // TODO: just to prevent this craziness for now + temp_file = fopen(file_path_safe, "w+b"); + errsv = errno; + if (temp_file == NULL) + { + debugf(DBG_LEVEL_NORM, + KRED"exit 1: cfs_open cannot open temp_file=[%s] err=%d:%s", file_path_safe, + errsv, strerror(errsv)); + return -ENOENT; + } - if (!cloudfs_object_write_fp(path, temp_file)) { - fclose(temp_file); - debugf(DBG_LEVEL_NORM, KRED "exit 2: cfs_open(%s) cannot download/write", path); - return -ENOENT; - } + if (!cloudfs_object_write_fp(path, temp_file)) + { + fclose(temp_file); + debugf(DBG_LEVEL_NORM, KRED "exit 2: cfs_open(%s) cannot download/write", + path); + return -ENOENT; + } } } else { temp_file = tmpfile(); - if (temp_file == NULL) { - debugf(DBG_LEVEL_NORM, KRED"exit 3: cfs_open cannot create temp_file err=%s", strerror(errno)); + if (temp_file == NULL) + { + debugf(DBG_LEVEL_NORM, KRED"exit 3: cfs_open cannot create temp_file err=%s", + strerror(errno)); return -ENOENT; } - if (!(info->flags & O_TRUNC)) { - if (!cloudfs_object_write_fp(path, temp_file) && !(info->flags & O_CREAT)) { + if (!(info->flags & O_TRUNC)) + { + if (!cloudfs_object_write_fp(path, temp_file) && !(info->flags & O_CREAT)) + { fclose(temp_file); debugf(DBG_LEVEL_NORM, KRED"exit 4: cfs_open(%s) cannot download/write", path); return -ENOENT; @@ -333,7 +371,7 @@ static int cfs_open(const char *path, struct fuse_file_info *info) } update_dir_cache(path, (de ? de->size : 0), 0, 0); - openfile *of = (openfile *)malloc(sizeof(openfile)); + openfile* of = (openfile*)malloc(sizeof(openfile)); of->fd = dup(fileno(temp_file)); if (of->fd == -1) { @@ -353,80 +391,96 @@ static int cfs_open(const char *path, struct fuse_file_info *info) return 0; } -static int cfs_read(const char *path, char *buf, size_t size, off_t offset, struct fuse_file_info *info) +static int cfs_read(const char* path, char* buf, size_t size, off_t offset, + struct fuse_file_info* info) { - debugf(DBG_LEVEL_EXTALL, KBLU "cfs_read(%s) buffsize=%lu offset=%lu", path, size, offset); + debugf(DBG_LEVEL_EXTALL, KBLU "cfs_read(%s) buffsize=%lu offset=%lu", path, + size, offset); file_buffer_size = size; debug_print_descriptor(info); - int result = pread(((openfile *)(uintptr_t)info->fh)->fd, buf, size, offset); - debugf(DBG_LEVEL_EXTALL, KBLU "exit: cfs_read(%s) result=%s", path, strerror(errno)); + int result = pread(((openfile*)(uintptr_t)info->fh)->fd, buf, size, offset); + debugf(DBG_LEVEL_EXTALL, KBLU "exit: cfs_read(%s) result=%s", path, + strerror(errno)); return result; } -//todo: flush will upload a file again even if just file attributes are changed. +//todo: flush will upload a file again even if just file attributes are changed. //optimisation needed to detect if content is changed and to only save meta when just attribs are modified. -static int cfs_flush(const char *path, struct fuse_file_info *info) +static int cfs_flush(const char* path, struct fuse_file_info* info) { debugf(DBG_LEVEL_NORM, KBLU "cfs_flush(%s)", path); debug_print_descriptor(info); - openfile *of = (openfile *)(uintptr_t)info->fh; + openfile* of = (openfile*)(uintptr_t)info->fh; int errsv = 0; - if (of) { + if (of) + { update_dir_cache(path, cloudfs_file_size(of->fd), 0, 0); - if (of->flags & O_RDWR || of->flags & O_WRONLY) { - FILE *fp = fdopen(dup(of->fd), "r"); + if (of->flags & O_RDWR || of->flags & O_WRONLY) + { + FILE* fp = fdopen(dup(of->fd), "r"); errsv = errno; - if (fp != NULL) + if (fp != NULL) { rewind(fp); //calculate md5 hash, compare with cloud hash to determine if file content is changed char md5_file_hash_str[MD5_DIGEST_LENGTH + 1] = "\0"; file_md5(fp, md5_file_hash_str); - dir_entry *de = check_path_info(path); - if (de && de->md5sum != NULL && (!strcasecmp(md5_file_hash_str, de->md5sum))) { + dir_entry* de = check_path_info(path); + if (de && de->md5sum != NULL && (!strcasecmp(md5_file_hash_str, de->md5sum))) + { //file content is identical, no need to upload entire file, just update metadata - debugf(DBG_LEVEL_NORM, KBLU "cfs_flush(%s): skip full upload as content did not change", path); + debugf(DBG_LEVEL_NORM, KBLU + "cfs_flush(%s): skip full upload as content did not change", path); cloudfs_update_meta(de); } - else { + else + { rewind(fp); - debugf(DBG_LEVEL_NORM, KBLU "cfs_flush(%s): perform full upload as content changed (or no file found in cache)", path); - if (!cloudfs_object_read_fp(path, fp)) { + debugf(DBG_LEVEL_NORM, KBLU + "cfs_flush(%s): perform full upload as content changed (or no file found in cache)", + path); + if (!cloudfs_object_read_fp(path, fp)) + { fclose(fp); errsv = errno; - debugf(DBG_LEVEL_NORM, KRED"exit 0: cfs_flush(%s) result=%d:%s", path, errsv, strerror(errno)); + debugf(DBG_LEVEL_NORM, KRED"exit 0: cfs_flush(%s) result=%d:%s", path, errsv, + strerror(errno)); return -ENOENT; } } fclose(fp); errsv = errno; } - else { - debugf(DBG_LEVEL_EXT, KRED "status: cfs_flush, err=%d:%s", errsv, strerror(errno)); - } + else + debugf(DBG_LEVEL_EXT, KRED "status: cfs_flush, err=%d:%s", errsv, + strerror(errno)); } } - debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_flush(%s) result=%d:%s", path, errsv, strerror(errno)); + debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_flush(%s) result=%d:%s", path, errsv, + strerror(errno)); return 0; } -static int cfs_release(const char *path, struct fuse_file_info *info) { +static int cfs_release(const char* path, struct fuse_file_info* info) +{ debugf(DBG_LEVEL_NORM, KBLU "cfs_release(%s)", path); - close(((openfile *)(uintptr_t)info->fh)->fd); + close(((openfile*)(uintptr_t)info->fh)->fd); debugf(DBG_LEVEL_NORM, KBLU "exit: cfs_release(%s)", path); return 0; } -static int cfs_rmdir(const char *path) +static int cfs_rmdir(const char* path) { debugf(DBG_LEVEL_NORM, KBLU "cfs_rmdir(%s)", path); int success = cloudfs_delete_object(path); - if (success == -1) { + if (success == -1) + { debugf(DBG_LEVEL_NORM, KBLU "exit 0: cfs_rmdir(%s)", path); return -ENOTEMPTY; } - if (success){ + if (success) + { dir_decache(path); debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_rmdir(%s)", path); return 0; @@ -435,11 +489,12 @@ static int cfs_rmdir(const char *path) return -ENOENT; } -static int cfs_ftruncate(const char *path, off_t size, struct fuse_file_info *info) +static int cfs_ftruncate(const char* path, off_t size, + struct fuse_file_info* info) { debugf(DBG_LEVEL_NORM, KBLU "cfs_ftruncate(%s)", path); file_buffer_size = size; - openfile *of = (openfile *)(uintptr_t)info->fh; + openfile* of = (openfile*)(uintptr_t)info->fh; if (ftruncate(of->fd, size)) return -errno; lseek(of->fd, 0, SEEK_SET); @@ -448,28 +503,31 @@ static int cfs_ftruncate(const char *path, off_t size, struct fuse_file_info *in return 0; } -static int cfs_write(const char *path, const char *buf, size_t length, off_t offset, struct fuse_file_info *info) +static int cfs_write(const char* path, const char* buf, size_t length, + off_t offset, struct fuse_file_info* info) { - debugf(DBG_LEVEL_EXTALL, KBLU "cfs_write(%s) bufflength=%lu offset=%lu", path, length, offset); + debugf(DBG_LEVEL_EXTALL, KBLU "cfs_write(%s) bufflength=%lu offset=%lu", path, + length, offset); // FIXME: Potential inconsistent cache update if pwrite fails? update_dir_cache(path, offset + length, 0, 0); //int result = pwrite(info->fh, buf, length, offset); - int result = pwrite(((openfile *)(uintptr_t)info->fh)->fd, buf, length, offset); + int result = pwrite(((openfile*)(uintptr_t)info->fh)->fd, buf, length, offset); int errsv = errno; - if (errsv == 0) { - debugf(DBG_LEVEL_EXTALL, KBLU "exit 0: cfs_write(%s) result=%d:%s", path, errsv, strerror(errsv)); - } - else { - debugf(DBG_LEVEL_EXTALL, KBLU "exit 1: cfs_write(%s) "KRED"result=%d:%s", path, errsv, strerror(errsv)); - } + if (errsv == 0) + debugf(DBG_LEVEL_EXTALL, KBLU "exit 0: cfs_write(%s) result=%d:%s", path, + errsv, strerror(errsv)); + else + debugf(DBG_LEVEL_EXTALL, KBLU "exit 1: cfs_write(%s) "KRED"result=%d:%s", path, + errsv, strerror(errsv)); return result; } -static int cfs_unlink(const char *path) +static int cfs_unlink(const char* path) { debugf(DBG_LEVEL_NORM, KBLU "cfs_unlink(%s)", path); int success = cloudfs_delete_object(path); - if (success == -1) { + if (success == -1) + { debugf(DBG_LEVEL_NORM, KRED "exit 0: cfs_unlink(%s)", path); return -EACCES; } @@ -483,13 +541,13 @@ static int cfs_unlink(const char *path) return -ENOENT; } -static int cfs_fsync(const char *path, int idunno, struct fuse_file_info *info) +static int cfs_fsync(const char* path, int idunno, struct fuse_file_info* info) { debugf(DBG_LEVEL_NORM, "cfs_fsync(%s)", path); return 0; } -static int cfs_truncate(const char *path, off_t size) +static int cfs_truncate(const char* path, off_t size) { debugf(DBG_LEVEL_NORM, "cfs_truncate(%s)", path); cloudfs_object_truncate(path, size); @@ -498,26 +556,31 @@ static int cfs_truncate(const char *path, off_t size) } //this is called regularly on copy (via mc), is optimised (cached) -static int cfs_statfs(const char *path, struct statvfs *stat) +static int cfs_statfs(const char* path, struct statvfs* stat) { debugf(DBG_LEVEL_NORM, KBLU "cfs_statfs(%s)", path); - if (cloudfs_statfs(path, stat)){ + if (cloudfs_statfs(path, stat)) + { debugf(DBG_LEVEL_NORM, KBLU "exit 0: cfs_statfs(%s)", path); return 0; } - else { + else + { debugf(DBG_LEVEL_NORM, KRED"exit 1: cfs_statfs(%s) not-found", path); return -EIO; } } -static int cfs_chown(const char *path, uid_t uid, gid_t gid) +static int cfs_chown(const char* path, uid_t uid, gid_t gid) { debugf(DBG_LEVEL_NORM, KBLU "cfs_chown(%s,%d,%d)", path, uid, gid); - dir_entry *de = check_path_info(path); - if (de) { - if (de->uid != uid || de->gid != gid) { - debugf(DBG_LEVEL_NORM, "cfs_chown(%s): change from uid:gid %d:%d to %d:%d", path, de->uid, de->gid, uid, gid); + dir_entry* de = check_path_info(path); + if (de) + { + if (de->uid != uid || de->gid != gid) + { + debugf(DBG_LEVEL_NORM, "cfs_chown(%s): change from uid:gid %d:%d to %d:%d", + path, de->uid, de->gid, uid, gid); de->uid = uid; de->gid = gid; //issue a PUT request to update metadata (quick request just to update headers) @@ -527,13 +590,16 @@ static int cfs_chown(const char *path, uid_t uid, gid_t gid) return 0; } -static int cfs_chmod(const char *path, mode_t mode) +static int cfs_chmod(const char* path, mode_t mode) { debugf(DBG_LEVEL_NORM, KBLU"cfs_chmod(%s,%d)", path, mode); - dir_entry *de = check_path_info(path); - if (de) { - if (de->chmod != mode) { - debugf(DBG_LEVEL_NORM, "cfs_chmod(%s): change mode from %d to %d", path, de->chmod, mode); + dir_entry* de = check_path_info(path); + if (de) + { + if (de->chmod != mode) + { + debugf(DBG_LEVEL_NORM, "cfs_chmod(%s): change mode from %d to %d", path, + de->chmod, mode); de->chmod = mode; //todo: issue a PUT request to update metadata (empty request just to update headers?) int response = cloudfs_update_meta(de); @@ -542,29 +608,34 @@ static int cfs_chmod(const char *path, mode_t mode) return 0; } -static int cfs_rename(const char *src, const char *dst) +static int cfs_rename(const char* src, const char* dst) { debugf(DBG_LEVEL_NORM, KBLU"cfs_rename(%s, %s)", src, dst); - dir_entry *src_de = path_info(src); - if (!src_de) { + dir_entry* src_de = path_info(src); + if (!src_de) + { debugf(DBG_LEVEL_NORM, KRED"exit 0: cfs_rename(%s,%s) not-found", src, dst); return -ENOENT; } - if (src_de->isdir) { - debugf(DBG_LEVEL_NORM, KRED"exit 1: cfs_rename(%s,%s) cannot rename dirs!", src, dst); + if (src_de->isdir) + { + debugf(DBG_LEVEL_NORM, KRED"exit 1: cfs_rename(%s,%s) cannot rename dirs!", + src, dst); return -EISDIR; } - if (cloudfs_copy_object(src, dst)) { + if (cloudfs_copy_object(src, dst)) + { /* FIXME this isn't quite right as doesn't preserve last modified */ //fix done in cloudfs_copy_object() update_dir_cache(dst, src_de->size, 0, 0); int result = cfs_unlink(src); - dir_entry *dst_de = path_info(dst); - if (!dst_de) { - debugf(DBG_LEVEL_NORM, KRED"cfs_rename(%s,%s) dest-not-found-in-cache", src, dst); - } - else { + dir_entry* dst_de = path_info(dst); + if (!dst_de) + debugf(DBG_LEVEL_NORM, KRED"cfs_rename(%s,%s) dest-not-found-in-cache", src, + dst); + else + { debugf(DBG_LEVEL_NORM, KBLU"cfs_rename(%s,%s) upload ok", src, dst); //copy attributes, shortcut, rather than forcing a download from cloud copy_dir_entry(src_de, dst_de); @@ -577,10 +648,10 @@ static int cfs_rename(const char *src, const char *dst) return -EIO; } -static int cfs_symlink(const char *src, const char *dst) +static int cfs_symlink(const char* src, const char* dst) { debugf(DBG_LEVEL_NORM, KBLU"cfs_symlink(%s, %s)", src, dst); - if(cloudfs_create_symlink(src, dst)) + if (cloudfs_create_symlink(src, dst)) { update_dir_cache(dst, 1, 0, 1); debugf(DBG_LEVEL_NORM, KBLU"exit0: cfs_symlink(%s, %s)", src, dst); @@ -594,15 +665,17 @@ static int cfs_readlink(const char* path, char* buf, size_t size) { debugf(DBG_LEVEL_NORM, KBLU"cfs_readlink(%s)", path); //fixme: use temp file specified in config - FILE *temp_file = tmpfile(); + FILE* temp_file = tmpfile(); int ret = 0; - if (!cloudfs_object_write_fp(path, temp_file)) { + if (!cloudfs_object_write_fp(path, temp_file)) + { debugf(DBG_LEVEL_NORM, KRED"exit 1: cfs_readlink(%s) not found", path); ret = -ENOENT; } - if (!pread(fileno(temp_file), buf, size, 0)) { + if (!pread(fileno(temp_file), buf, size, 0)) + { debugf(DBG_LEVEL_NORM, KRED"exit 2: cfs_readlink(%s) not found", path); ret = -ENOENT; } @@ -612,17 +685,17 @@ static int cfs_readlink(const char* path, char* buf, size_t size) return ret; } -static void *cfs_init(struct fuse_conn_info *conn) +static void* cfs_init(struct fuse_conn_info* conn) { signal(SIGPIPE, SIG_IGN); return NULL; } //http://man7.org/linux/man-pages/man2/utimensat.2.html -static int cfs_utimens(const char *path, const struct timespec times[2]) +static int cfs_utimens(const char* path, const struct timespec times[2]) { debugf(DBG_LEVEL_NORM, KBLU "cfs_utimens(%s)", path); - dir_entry *path_de = path_info(path); + dir_entry* path_de = path_info(path); if (!path_de) { debugf(DBG_LEVEL_NORM, KRED"exit 0: cfs_utimens(%s) file not in cache", path); @@ -631,12 +704,17 @@ static int cfs_utimens(const char *path, const struct timespec times[2]) struct timespec now; clock_gettime(CLOCK_REALTIME, &now); - if (path_de->atime.tv_sec != times[0].tv_sec || path_de->atime.tv_nsec != times[0].tv_nsec || - path_de->mtime.tv_sec != times[1].tv_sec || path_de->mtime.tv_nsec != times[1].tv_nsec) + if (path_de->atime.tv_sec != times[0].tv_sec + || path_de->atime.tv_nsec != times[0].tv_nsec || + path_de->mtime.tv_sec != times[1].tv_sec + || path_de->mtime.tv_nsec != times[1].tv_nsec) { - debugf(DBG_LEVEL_EXT, KCYN "cfs_utimens: change for %s, prev: atime=%li.%li mtime=%li.%li, new: atime=%li.%li mtime=%li.%li", path, - path_de->atime.tv_sec, path_de->atime.tv_nsec, path_de->mtime.tv_sec, path_de->mtime.tv_nsec, - times[0].tv_sec, times[0].tv_nsec, times[1].tv_sec, times[1].tv_nsec); + debugf(DBG_LEVEL_EXT, KCYN + "cfs_utimens: change for %s, prev: atime=%li.%li mtime=%li.%li, new: atime=%li.%li mtime=%li.%li", + path, + path_de->atime.tv_sec, path_de->atime.tv_nsec, path_de->mtime.tv_sec, + path_de->mtime.tv_nsec, + times[0].tv_sec, times[0].tv_nsec, times[1].tv_sec, times[1].tv_nsec); char time_str[TIME_CHARS] = ""; get_timespec_as_str(×[1], time_str, sizeof(time_str)); debugf(DBG_LEVEL_EXT, KCYN"cfs_utimens: set mtime=[%s]", time_str); @@ -646,40 +724,41 @@ static int cfs_utimens(const char *path, const struct timespec times[2]) path_de->mtime = times[1]; // not sure how to best obtain ctime from fuse source file. just record current date. path_de->ctime = now; - //calling a meta cloud update here is not always needed. + //calling a meta cloud update here is not always needed. //touch for example opens and closes/flush the file. //worth implementing a meta cache functionality to avoid multiple uploads on meta changes //when changing timestamps on very large files, touch command will trigger 2 x long and useless file uploads on cfs_flush() } - else { + else debugf(DBG_LEVEL_EXT, KCYN"cfs_utimens: a/m/time not changed"); - } debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_utimens(%s)", path); return 0; } -int cfs_setxattr(const char *path, const char *name, const char *value, size_t size, int flags) +int cfs_setxattr(const char* path, const char* name, const char* value, + size_t size, int flags) { return 0; } -int cfs_getxattr(const char *path, const char *name, char *value, size_t size) +int cfs_getxattr(const char* path, const char* name, char* value, size_t size) { return 0; } -int cfs_removexattr(const char *path, const char *name) +int cfs_removexattr(const char* path, const char* name) { return 0; } -int cfs_listxattr(const char *path, char *list, size_t size) +int cfs_listxattr(const char* path, char* list, size_t size) { return 0; } -FuseOptions options = { +FuseOptions options = +{ .cache_timeout = "600", .verify_ssl = "true", .segment_size = "1073741824", @@ -693,7 +772,8 @@ FuseOptions options = { .refresh_token = "" }; -ExtraFuseOptions extra_options = { +ExtraFuseOptions extra_options = +{ .get_extended_metadata = "false", .curl_verbose = "false", .cache_statfs_timeout = 0, @@ -703,27 +783,31 @@ ExtraFuseOptions extra_options = { .enable_chmod = "false" }; -int parse_option(void *data, const char *arg, int key, struct fuse_args *outargs) +int parse_option(void* data, const char* arg, int key, + struct fuse_args* outargs) { if (sscanf(arg, " cache_timeout = %[^\r\n ]", options.cache_timeout) || - sscanf(arg, " verify_ssl = %[^\r\n ]", options.verify_ssl) || - sscanf(arg, " segment_above = %[^\r\n ]", options.segment_above) || - sscanf(arg, " segment_size = %[^\r\n ]", options.segment_size) || - sscanf(arg, " storage_url = %[^\r\n ]", options.storage_url) || - sscanf(arg, " container = %[^\r\n ]", options.container) || - sscanf(arg, " temp_dir = %[^\r\n ]", options.temp_dir) || - sscanf(arg, " client_id = %[^\r\n ]", options.client_id) || - sscanf(arg, " client_secret = %[^\r\n ]", options.client_secret) || - sscanf(arg, " refresh_token = %[^\r\n ]", options.refresh_token) || + sscanf(arg, " verify_ssl = %[^\r\n ]", options.verify_ssl) || + sscanf(arg, " segment_above = %[^\r\n ]", options.segment_above) || + sscanf(arg, " segment_size = %[^\r\n ]", options.segment_size) || + sscanf(arg, " storage_url = %[^\r\n ]", options.storage_url) || + sscanf(arg, " container = %[^\r\n ]", options.container) || + sscanf(arg, " temp_dir = %[^\r\n ]", options.temp_dir) || + sscanf(arg, " client_id = %[^\r\n ]", options.client_id) || + sscanf(arg, " client_secret = %[^\r\n ]", options.client_secret) || + sscanf(arg, " refresh_token = %[^\r\n ]", options.refresh_token) || - sscanf(arg, " get_extended_metadata = %[^\r\n ]", extra_options.get_extended_metadata) || - sscanf(arg, " curl_verbose = %[^\r\n ]", extra_options.curl_verbose) || - sscanf(arg, " cache_statfs_timeout = %[^\r\n ]", extra_options.cache_statfs_timeout) || - sscanf(arg, " debug_level = %[^\r\n ]", extra_options.debug_level) || - sscanf(arg, " curl_progress_state = %[^\r\n ]", extra_options.curl_progress_state) || - sscanf(arg, " enable_chmod = %[^\r\n ]", extra_options.enable_chmod) || - sscanf(arg, " enable_chown = %[^\r\n ]", extra_options.enable_chown) - ) + sscanf(arg, " get_extended_metadata = %[^\r\n ]", + extra_options.get_extended_metadata) || + sscanf(arg, " curl_verbose = %[^\r\n ]", extra_options.curl_verbose) || + sscanf(arg, " cache_statfs_timeout = %[^\r\n ]", + extra_options.cache_statfs_timeout) || + sscanf(arg, " debug_level = %[^\r\n ]", extra_options.debug_level) || + sscanf(arg, " curl_progress_state = %[^\r\n ]", + extra_options.curl_progress_state) || + sscanf(arg, " enable_chmod = %[^\r\n ]", extra_options.enable_chmod) || + sscanf(arg, " enable_chown = %[^\r\n ]", extra_options.enable_chown) + ) return 0; if (!strcmp(arg, "-f") || !strcmp(arg, "-d") || !strcmp(arg, "debug")) cloudfs_debug(1); @@ -731,7 +815,8 @@ int parse_option(void *data, const char *arg, int key, struct fuse_args *outargs } //allows memory leaks inspections -void interrupt_handler(int sig) { +void interrupt_handler(int sig) +{ debugf(DBG_LEVEL_NORM, "Got interrupt signal %d, cleaning memory", sig); //TODO: clean memory allocations //http://www.cprogramming.com/debugging/valgrind.html @@ -745,32 +830,24 @@ void initialise_options() { //todo: handle param init consistently, quite heavy implementation cloudfs_verify_ssl(!strcasecmp(options.verify_ssl, "true")); - cloudfs_option_get_extended_metadata(!strcasecmp(extra_options.get_extended_metadata, "true")); + cloudfs_option_get_extended_metadata(!strcasecmp( + extra_options.get_extended_metadata, "true")); cloudfs_option_curl_verbose(!strcasecmp(extra_options.curl_verbose, "true")); //lean way to init params, to be used as reference if (*extra_options.debug_level) - { option_debug_level = atoi(extra_options.debug_level); - } if (*extra_options.cache_statfs_timeout) - { option_cache_statfs_timeout = atoi(extra_options.cache_statfs_timeout); - } if (*extra_options.curl_progress_state) - { - option_curl_progress_state = !strcasecmp(extra_options.curl_progress_state, "true"); - } + option_curl_progress_state = !strcasecmp(extra_options.curl_progress_state, + "true"); if (*extra_options.enable_chmod) - { option_enable_chmod = !strcasecmp(extra_options.enable_chmod, "true"); - } if (*extra_options.enable_chown) - { option_enable_chown = !strcasecmp(extra_options.enable_chown, "true"); - } } -int main(int argc, char **argv) +int main(int argc, char** argv) { if (debug) fprintf(stderr, "Starting hubicfuse on homedir %s!\n", get_home_dir()); @@ -778,10 +855,11 @@ int main(int argc, char **argv) signal(SIGINT, interrupt_handler); char settings_filename[MAX_PATH_SIZE] = ""; - FILE *settings; + FILE* settings; struct fuse_args args = FUSE_ARGS_INIT(argc, argv); - snprintf(settings_filename, sizeof(settings_filename), "%s/.hubicfuse", get_home_dir()); + snprintf(settings_filename, sizeof(settings_filename), "%s/.hubicfuse", + get_home_dir()); if ((settings = fopen(settings_filename, "r"))) { char line[OPTION_SIZE]; @@ -801,25 +879,36 @@ int main(int argc, char **argv) if (!*options.client_id || !*options.client_secret || !*options.refresh_token) { - fprintf(stderr, "Unable to determine client_id, client_secret or refresh_token.\n\n"); + fprintf(stderr, + "Unable to determine client_id, client_secret or refresh_token.\n\n"); fprintf(stderr, "These can be set either as mount options or in " - "a file named %s\n\n", settings_filename); + "a file named %s\n\n", settings_filename); fprintf(stderr, " client_id=[App's id]\n"); fprintf(stderr, " client_secret=[App's secret]\n"); fprintf(stderr, " refresh_token=[Get it running hubic_token]\n"); fprintf(stderr, "The following settings are optional:\n\n"); - fprintf(stderr, " cache_timeout=[Seconds for directory caching, default 600]\n"); + fprintf(stderr, + " cache_timeout=[Seconds for directory caching, default 600]\n"); fprintf(stderr, " verify_ssl=[false to disable SSL cert verification]\n"); - fprintf(stderr, " segment_size=[Size to use when creating DLOs, default 1073741824]\n"); - fprintf(stderr, " segment_above=[File size at which to use segments, defult 2147483648]\n"); - fprintf(stderr, " storage_url=[Storage URL for other tenant to view container]\n"); - fprintf(stderr, " container=[Public container to view of tenant specified by storage_url]\n"); + fprintf(stderr, + " segment_size=[Size to use when creating DLOs, default 1073741824]\n"); + fprintf(stderr, + " segment_above=[File size at which to use segments, defult 2147483648]\n"); + fprintf(stderr, + " storage_url=[Storage URL for other tenant to view container]\n"); + fprintf(stderr, + " container=[Public container to view of tenant specified by storage_url]\n"); fprintf(stderr, " temp_dir=[Directory to store temp files]\n"); - fprintf(stderr, " get_extended_metadata=[true to enable download of utime, chmod, chown file attributes (but slower)]\n"); - fprintf(stderr, " curl_verbose=[true to debug info on curl requests (lots of output)]\n"); - fprintf(stderr, " curl_progress_state=[true to enable progress callback enabled. Mostly used for debugging]\n"); - fprintf(stderr, " cache_statfs_timeout=[number of seconds to cache requests to statfs (cloud statistics), 0 for no cache]\n"); - fprintf(stderr, " debug_level=[0 to 2, 0 for minimal verbose debugging. No debug if -d or -f option is not provided.]\n"); + fprintf(stderr, + " get_extended_metadata=[true to enable download of utime, chmod, chown file attributes (but slower)]\n"); + fprintf(stderr, + " curl_verbose=[true to debug info on curl requests (lots of output)]\n"); + fprintf(stderr, + " curl_progress_state=[true to enable progress callback enabled. Mostly used for debugging]\n"); + fprintf(stderr, + " cache_statfs_timeout=[number of seconds to cache requests to statfs (cloud statistics), 0 for no cache]\n"); + fprintf(stderr, + " debug_level=[0 to 2, 0 for minimal verbose debugging. No debug if -d or -f option is not provided.]\n"); fprintf(stderr, " enable_chmod=[true to enable chmod support on fuse]\n"); fprintf(stderr, " enable_chown=[true to enable chown support on fuse]\n"); return 1; @@ -834,7 +923,8 @@ int main(int argc, char **argv) fprintf(stderr, "enable_chmod = %d\n", option_enable_chmod); fprintf(stderr, "enable_chown = %d\n", option_enable_chown); } - cloudfs_set_credentials(options.client_id, options.client_secret, options.refresh_token); + cloudfs_set_credentials(options.client_id, options.client_secret, + options.refresh_token); if (!cloudfs_connect()) { @@ -842,40 +932,41 @@ int main(int argc, char **argv) return 1; } //todo: check why in some cases the define below is not available (when running the binary on symbolic linked folders) - #ifndef HAVE_OPENSSL - #warning Compiling without libssl, will run single-threaded. +#ifndef HAVE_OPENSSL +#warning Compiling without libssl, will run single-threaded. fuse_opt_add_arg(&args, "-s"); - #endif +#endif - struct fuse_operations cfs_oper = { - .readdir = cfs_readdir, - .mkdir = cfs_mkdir, - .read = cfs_read, - .create = cfs_create, - .open = cfs_open, - .fgetattr = cfs_fgetattr, - .getattr = cfs_getattr, - .flush = cfs_flush, - .release = cfs_release, - .rmdir = cfs_rmdir, - .ftruncate = cfs_ftruncate, - .truncate = cfs_truncate, - .write = cfs_write, - .unlink = cfs_unlink, - .fsync = cfs_fsync, - .statfs = cfs_statfs, - .chmod = cfs_chmod, - .chown = cfs_chown, - .rename = cfs_rename, - .symlink = cfs_symlink, - .readlink = cfs_readlink, - .init = cfs_init, - .utimens = cfs_utimens, + struct fuse_operations cfs_oper = + { + .readdir = cfs_readdir, + .mkdir = cfs_mkdir, + .read = cfs_read, + .create = cfs_create, + .open = cfs_open, + .fgetattr = cfs_fgetattr, + .getattr = cfs_getattr, + .flush = cfs_flush, + .release = cfs_release, + .rmdir = cfs_rmdir, + .ftruncate = cfs_ftruncate, + .truncate = cfs_truncate, + .write = cfs_write, + .unlink = cfs_unlink, + .fsync = cfs_fsync, + .statfs = cfs_statfs, + .chmod = cfs_chmod, + .chown = cfs_chown, + .rename = cfs_rename, + .symlink = cfs_symlink, + .readlink = cfs_readlink, + .init = cfs_init, + .utimens = cfs_utimens, #ifdef HAVE_SETXATTR - .setxattr = cfs_setxattr, - .getxattr = cfs_getxattr, - .listxattr = cfs_listxattr, - .removexattr = cfs_removexattr, + .setxattr = cfs_setxattr, + .getxattr = cfs_getxattr, + .listxattr = cfs_listxattr, + .removexattr = cfs_removexattr, #endif }; diff --git a/commonfs.c b/commonfs.c index 54ab189..5d54e1b 100644 --- a/commonfs.c +++ b/commonfs.c @@ -26,8 +26,8 @@ pthread_mutex_t dcachemut; pthread_mutexattr_t mutex_attr; -dir_cache *dcache; -char *temp_dir; +dir_cache* dcache; +char* temp_dir; int cache_timeout; int debug = 0; int verify_ssl = 2; @@ -45,7 +45,7 @@ size_t file_buffer_size = 0; // needed to get correct GMT / local time // hubic stores time as GMT so we have to do conversions // http://zhu-qy.blogspot.ro/2012/11/ref-how-to-convert-from-utc-to-local.html -time_t my_timegm(struct tm *tm) +time_t my_timegm(struct tm* tm) { time_t epoch = 0; time_t offset = mktime(gmtime(&epoch)); @@ -54,7 +54,7 @@ time_t my_timegm(struct tm *tm) } //expect time_str as a friendly string format -time_t get_time_from_str_as_gmt(char *time_str) +time_t get_time_from_str_as_gmt(char* time_str) { struct tm val_time_tm; time_t val_time_t; @@ -68,7 +68,8 @@ time_t get_time_as_local(time_t time_t_val, char time_str[], int char_buf_size) { struct tm loc_time_tm; loc_time_tm = *localtime(&time_t_val); - if (time_str != NULL) { + if (time_str != NULL) + { //debugf(DBG_LEVEL_NORM, 0,"Local len=%d size=%d pass=%d", strlen(time_str), sizeof(time_str), char_buf_size); strftime(time_str, char_buf_size, "%c", &loc_time_tm); //debugf(DBG_LEVEL_NORM, 0,"Local timestr=[%s] size=%d", time_str, strlen(time_str)); @@ -77,19 +78,24 @@ time_t get_time_as_local(time_t time_t_val, char time_str[], int char_buf_size) return mktime(&loc_time_tm); } -int get_time_as_string(time_t time_t_val, long nsec, char *time_str, int time_str_len) +int get_time_as_string(time_t time_t_val, long nsec, char* time_str, + int time_str_len) { struct tm time_val_tm; time_t safe_input_time; //if time is incorrect (too long) you get segfault, need to check length and trim - if (time_t_val > INT_MAX) { - debugf(DBG_LEVEL_NORM, KRED"get_time_as_string: input time length too long, %lu > max=%lu, trimming!", time_t_val, INT_MAX); + if (time_t_val > INT_MAX) + { + debugf(DBG_LEVEL_NORM, + KRED"get_time_as_string: input time length too long, %lu > max=%lu, trimming!", + time_t_val, INT_MAX); safe_input_time = 0;//(int)time_t_val; } - else + else safe_input_time = time_t_val; time_val_tm = *gmtime(&safe_input_time); - int str_len = strftime(time_str, time_str_len, HUBIC_DATE_FORMAT, &time_val_tm); + int str_len = strftime(time_str, time_str_len, HUBIC_DATE_FORMAT, + &time_val_tm); char nsec_str[TIME_CHARS]; sprintf(nsec_str, "%d", nsec); strcat(time_str, nsec_str); @@ -103,7 +109,7 @@ time_t get_time_now() return now.tv_sec; } -size_t get_time_now_as_str(char *time_str, int time_str_len) +size_t get_time_now_as_str(char* time_str, int time_str_len) { time_t now = time(0); struct tm tstruct; @@ -114,44 +120,46 @@ size_t get_time_now_as_str(char *time_str, int time_str_len) return result; } -int get_timespec_as_str(const struct timespec *times, char *time_str, int time_str_len) +int get_timespec_as_str(const struct timespec* times, char* time_str, + int time_str_len) { - return get_time_as_string(times->tv_sec, times->tv_nsec, time_str, time_str_len); + return get_time_as_string(times->tv_sec, times->tv_nsec, time_str, + time_str_len); } -char *str2md5(const char *str, int length) +char* str2md5(const char* str, int length) { int n; MD5_CTX c; unsigned char digest[16]; - char *out = (char*)malloc(33); + char* out = (char*)malloc(33); MD5_Init(&c); - while (length > 0) { - if (length > 512) { + while (length > 0) + { + if (length > 512) MD5_Update(&c, str, 512); - } - else { + else MD5_Update(&c, str, length); - } length -= 512; str += 512; } MD5_Final(digest, &c); - for (n = 0; n < 16; ++n) { + for (n = 0; n < 16; ++n) snprintf(&(out[n * 2]), 16 * 2, "%02x", (unsigned int)digest[n]); - } return out; } // http://stackoverflow.com/questions/10324611/how-to-calculate-the-md5-hash-of-a-large-file-in-c -int file_md5(FILE *file_handle, char *md5_file_str) +int file_md5(FILE* file_handle, char* md5_file_str) { - if (file_handle == NULL) { + if (file_handle == NULL) + { debugf(DBG_LEVEL_NORM, KRED"file_md5: NULL file handle"); return 0; } - if (file_buffer_size == 0) { + if (file_buffer_size == 0) + { debugf(DBG_LEVEL_NORM, KYEL"file_md5: 0 file buffer size, using default"); file_buffer_size = 1024; } @@ -160,12 +168,13 @@ int file_md5(FILE *file_handle, char *md5_file_str) MD5_CTX mdContext; int bytes; char mdchar[3];//2 chars for md5 + null string terminator - unsigned char *data_buf = malloc(file_buffer_size * sizeof(unsigned char)); + unsigned char* data_buf = malloc(file_buffer_size * sizeof(unsigned char)); MD5_Init(&mdContext); while ((bytes = fread(data_buf, 1, file_buffer_size, file_handle)) != 0) MD5_Update(&mdContext, data_buf, bytes); MD5_Final(c, &mdContext); - for (i = 0; i < MD5_DIGEST_LENGTH; i++) { + for (i = 0; i < MD5_DIGEST_LENGTH; i++) + { snprintf(mdchar, 3, "%02x", c[i]); strcat(md5_file_str, mdchar); } @@ -173,14 +182,14 @@ int file_md5(FILE *file_handle, char *md5_file_str) return 0; } -int get_safe_cache_file_path(const char *path, char *file_path_safe, char *temp_dir) +int get_safe_cache_file_path(const char* path, char* file_path_safe, + char* temp_dir) { char tmp_path[PATH_MAX]; strncpy(tmp_path, path, PATH_MAX); - char *pch; - while ((pch = strchr(tmp_path, '/'))) { - *pch = '.'; - } + char* pch; + while ((pch = strchr(tmp_path, '/'))) + * pch = '.'; char file_path[PATH_MAX] = ""; //temp file name had process pid in it, removed as on restart files are left in cache (pid changes) snprintf(file_path, PATH_MAX, TEMP_FILE_NAME_FORMAT, temp_dir, tmp_path); @@ -188,7 +197,7 @@ int get_safe_cache_file_path(const char *path, char *file_path_safe, char *temp_ int file_path_len = sizeof(file_path); //the file path name using this format can go beyond NAME_MAX size and will generate error on fopen //solution: cap file length to NAME_MAX, use a prefix from original path for debug purposes and add md5 id - char *md5_path = str2md5(file_path, file_path_len); + char* md5_path = str2md5(file_path, file_path_len); int md5len = strlen(md5_path); size_t safe_len_prefix = min(NAME_MAX - md5len, file_path_len); strncpy(file_path_safe, file_path, safe_len_prefix); @@ -199,7 +208,7 @@ int get_safe_cache_file_path(const char *path, char *file_path_safe, char *temp_ return strlen(file_path_safe); } -void get_file_path_from_fd(int fd, char *path, int size_path) +void get_file_path_from_fd(int fd, char* path, int size_path) { char proc_path[MAX_PATH_SIZE]; /* Read out the link to our file descriptor. */ @@ -221,57 +230,60 @@ void debug_print_flags(int flags) if (val & O_APPEND) debugf(DBG_LEVEL_EXTALL, KYEL", append"); if (val & O_NONBLOCK) debugf(DBG_LEVEL_EXTALL, KYEL", nonblocking"); #if !defined(_POSIX_SOURCE) && defined(O_SYNC) - if (val & O_SYNC) debugf(DBG_LEVEL_EXT, 0,KRED ", synchronous writes"); + if (val & O_SYNC) debugf(DBG_LEVEL_EXT, 0, + KRED ", synchronous writes"); #endif } //for file descriptor debugging -void debug_print_descriptor(struct fuse_file_info *info) +void debug_print_descriptor(struct fuse_file_info* info) { char file_path[MAX_PATH_SIZE]; get_file_path_from_fd(info->fh, file_path, sizeof(file_path)); - debugf(DBG_LEVEL_EXT, KCYN "descriptor localfile=[%s] fd=%d", file_path, info->fh); + debugf(DBG_LEVEL_EXT, KCYN "descriptor localfile=[%s] fd=%d", file_path, + info->fh); debug_print_flags(info->flags); } -void dir_for(const char *path, char *dir) +void dir_for(const char* path, char* dir) { strncpy(dir, path, MAX_PATH_SIZE); - char *slash = strrchr(dir, '/'); + char* slash = strrchr(dir, '/'); if (slash) *slash = '\0'; } //prints cache content for debug purposes -void debug_list_cache_content() { +void debug_list_cache_content() +{ return;//disabled - dir_cache *cw; - dir_entry *de; + dir_cache* cw; + dir_entry* de; for (cw = dcache; cw; cw = cw->next) { debugf(DBG_LEVEL_EXT, "LIST-CACHE: DIR[%s]", cw->path); - for (de = cw->entries; de; de = de->next) { + for (de = cw->entries; de; de = de->next) debugf(DBG_LEVEL_EXT, "LIST-CACHE: FOLDER[%s]", de->full_name); - } } } -int delete_file(char *path) +int delete_file(char* path) { debugf(DBG_LEVEL_NORM, KYEL"delete_file(%s)", path); char file_path_safe[NAME_MAX] = ""; get_safe_cache_file_path(path, file_path_safe, temp_dir); int result = unlink(file_path_safe); - debugf(DBG_LEVEL_EXT, KYEL"delete_file(%s) (%s) result=%s", path, file_path_safe, strerror(result)); + debugf(DBG_LEVEL_EXT, KYEL"delete_file(%s) (%s) result=%s", path, + file_path_safe, strerror(result)); return result; } //adding a directory in cache -dir_cache *new_cache(const char *path) +dir_cache* new_cache(const char* path) { debugf(DBG_LEVEL_NORM, KCYN"new_cache(%s)", path); - dir_cache *cw = (dir_cache *)calloc(sizeof(dir_cache), 1); + dir_cache* cw = (dir_cache*)calloc(sizeof(dir_cache), 1); cw->path = strdup(path); cw->prev = NULL; cw->entries = NULL; @@ -282,20 +294,21 @@ dir_cache *new_cache(const char *path) if (dcache) dcache->prev = cw; cw->next = dcache; - dir_cache *result; + dir_cache* result; result = (dcache = cw); debugf(DBG_LEVEL_EXT, "exit: new_cache(%s)", path); return result; } -//todo: check if the program behaves ok when free_dir +//todo: check if the program behaves ok when free_dir //is made on a folder that has an operation in progress -void cloudfs_free_dir_list(dir_entry *dir_list) +void cloudfs_free_dir_list(dir_entry* dir_list) { //check for NULL as dir might be already removed from cache by other thread debugf(DBG_LEVEL_NORM, "cloudfs_free_dir_list(%s)", dir_list->full_name); - while (dir_list) { - dir_entry *de = dir_list; + while (dir_list) + { + dir_entry* de = dir_list; dir_list = dir_list->next; //remove file from disk cache, fix for issue #89, https://github.com/TurboGit/hubicfuse/issues/89 delete_file(de->full_name); @@ -308,15 +321,16 @@ void cloudfs_free_dir_list(dir_entry *dir_list) } } -void dir_decache(const char *path) +void dir_decache(const char* path) { - dir_cache *cw; + dir_cache* cw; debugf(DBG_LEVEL_NORM, "dir_decache(%s)", path); pthread_mutex_lock(&dcachemut); - dir_entry *de, *tmpde; + dir_entry* de, *tmpde; char dir[MAX_PATH_SIZE]; dir_for(path, dir); - for (cw = dcache; cw; cw = cw->next) { + for (cw = dcache; cw; cw = cw->next) + { debugf(DBG_LEVEL_EXT, "dir_decache: parse(%s)", cw->path); if (!strcmp(cw->path, path)) { @@ -344,17 +358,17 @@ void dir_decache(const char *path) cloudfs_free_dir_list(de); } else for (de = cw->entries; de->next; de = de->next) - { - if (!strcmp(de->next->full_name, path)) { - tmpde = de->next; - de->next = de->next->next; - tmpde->next = NULL; - debugf(DBG_LEVEL_EXT, "dir_decache: free_dir3()", cw->path); - cloudfs_free_dir_list(tmpde); - break; + if (!strcmp(de->next->full_name, path)) + { + tmpde = de->next; + de->next = de->next->next; + tmpde->next = NULL; + debugf(DBG_LEVEL_EXT, "dir_decache: free_dir3()", cw->path); + cloudfs_free_dir_list(tmpde); + break; + } } - } } } pthread_mutex_unlock(&dcachemut); @@ -362,7 +376,7 @@ void dir_decache(const char *path) dir_entry* init_dir_entry() { - dir_entry *de = (dir_entry *)malloc(sizeof(dir_entry)); + dir_entry* de = (dir_entry*)malloc(sizeof(dir_entry)); de->metadata_downloaded = false; de->size = 0; de->next = NULL; @@ -381,7 +395,7 @@ dir_entry* init_dir_entry() return de; } -void copy_dir_entry(dir_entry *src, dir_entry *dst) +void copy_dir_entry(dir_entry* src, dir_entry* dst) { dst->atime.tv_sec = src->atime.tv_sec; dst->atime.tv_nsec = src->atime.tv_nsec; @@ -392,15 +406,15 @@ void copy_dir_entry(dir_entry *src, dir_entry *dst) dst->chmod = src->chmod; //todo: copy md5sum as well } - -//check for file in cache, if found size will be updated, if not found + +//check for file in cache, if found size will be updated, if not found //and this is a dir, a new dir cache entry is created -void update_dir_cache(const char *path, off_t size, int isdir, int islink) +void update_dir_cache(const char* path, off_t size, int isdir, int islink) { debugf(DBG_LEVEL_EXTALL, KCYN "update_dir_cache(%s)", path); pthread_mutex_lock(&dcachemut); - dir_cache *cw; - dir_entry *de; + dir_cache* cw; + dir_entry* de; char dir[MAX_PATH_SIZE]; dir_for(path, dir); for (cw = dcache; cw; cw = cw->next) @@ -413,7 +427,7 @@ void update_dir_cache(const char *path, off_t size, int isdir, int islink) { de->size = size; pthread_mutex_unlock(&dcachemut); - debugf(DBG_LEVEL_EXTALL, "exit 0: update_dir_cache(%s)", path); + debugf(DBG_LEVEL_EXTALL, "exit 0: update_dir_cache(%s)", path); return; } } @@ -424,17 +438,12 @@ void update_dir_cache(const char *path, off_t size, int isdir, int islink) de->name = strdup(&path[strlen(cw->path) + 1]); de->full_name = strdup(path); //fixed: the conditions below were mixed up dir -> link? - if (islink) - { + if (islink) de->content_type = strdup("application/link"); - } - if (isdir) - { + if (isdir) de->content_type = strdup("application/directory"); - } - else { + else de->content_type = strdup("application/octet-stream"); - } de->next = cw->entries; cw->entries = de; if (isdir) @@ -447,25 +456,28 @@ void update_dir_cache(const char *path, off_t size, int isdir, int islink) } //returns first file entry in linked list. if not in cache will be downloaded. -int caching_list_directory(const char *path, dir_entry **list) +int caching_list_directory(const char* path, dir_entry** list) { debugf(DBG_LEVEL_EXT, "caching_list_directory(%s)", path); pthread_mutex_lock(&dcachemut); bool new_entry = false; if (!strcmp(path, "/")) path = ""; - dir_cache *cw; - for (cw = dcache; cw; cw = cw->next) + dir_cache* cw; + for (cw = dcache; cw; cw = cw->next) { if (cw->was_deleted == true) { - debugf(DBG_LEVEL_EXT, KMAG"caching_list_directory status: dir(%s) is empty as cached expired, reload from cloud", cw->path); + debugf(DBG_LEVEL_EXT, + KMAG"caching_list_directory status: dir(%s) is empty as cached expired, reload from cloud", + cw->path); if (!cloudfs_list_directory(cw->path, list)) + debugf(DBG_LEVEL_EXT, + KMAG"caching_list_directory status: cannot reload dir(%s)", cw->path); + else { - debugf(DBG_LEVEL_EXT, KMAG"caching_list_directory status: cannot reload dir(%s)", cw->path); - } - else { - debugf(DBG_LEVEL_EXT, KMAG"caching_list_directory status: reloaded dir(%s)", cw->path); + debugf(DBG_LEVEL_EXT, KMAG"caching_list_directory status: reloaded dir(%s)", + cw->path); //cw->entries = *list; cw->was_deleted = false; cw->cached = time(NULL); @@ -474,9 +486,7 @@ int caching_list_directory(const char *path, dir_entry **list) if (cw->was_deleted == false) { if (!strcmp(cw->path, path)) - { break; - } } } if (!cw) @@ -486,10 +496,12 @@ int caching_list_directory(const char *path, dir_entry **list) { //download was not ok pthread_mutex_unlock(&dcachemut); - debugf(DBG_LEVEL_EXT, "exit 0: caching_list_directory(%s) "KYEL"[CACHE-DIR-MISS]", path); + debugf(DBG_LEVEL_EXT, + "exit 0: caching_list_directory(%s) "KYEL"[CACHE-DIR-MISS]", path); return 0; } - debugf(DBG_LEVEL_EXT, "caching_list_directory: new_cache(%s) "KYEL"[CACHE-CREATE]", path); + debugf(DBG_LEVEL_EXT, + "caching_list_directory: new_cache(%s) "KYEL"[CACHE-CREATE]", path); cw = new_cache(path); new_entry = true; } @@ -509,16 +521,21 @@ int caching_list_directory(const char *path, dir_entry **list) cloudfs_free_dir_list(cw->entries); cw->was_deleted = true; cw->cached = time(NULL); - debugf(DBG_LEVEL_EXT, "caching_list_directory(%s) "KYEL"[CACHE-EXPIRED]", path); + debugf(DBG_LEVEL_EXT, "caching_list_directory(%s) "KYEL"[CACHE-EXPIRED]", + path); } - else { - debugf(DBG_LEVEL_EXT, "got NULL on caching_list_directory(%s) "KYEL"[CACHE-EXPIRED w NULL]", path); + else + { + debugf(DBG_LEVEL_EXT, + "got NULL on caching_list_directory(%s) "KYEL"[CACHE-EXPIRED w NULL]", path); pthread_mutex_unlock(&dcachemut); return 0; } } - else { - debugf(DBG_LEVEL_EXT, "caching_list_directory(%s) "KGRN"[CACHE-DIR-HIT]", path); + else + { + debugf(DBG_LEVEL_EXT, "caching_list_directory(%s) "KGRN"[CACHE-DIR-HIT]", + path); *list = cw->entries; } //adding new dir file list to global cache, now this dir becomes visible in cache @@ -528,20 +545,19 @@ int caching_list_directory(const char *path, dir_entry **list) return 1; } -dir_entry *path_info(const char *path) +dir_entry* path_info(const char* path) { debugf(DBG_LEVEL_EXT, "path_info(%s)", path); char dir[MAX_PATH_SIZE]; dir_for(path, dir); - dir_entry *tmp; + dir_entry* tmp; if (!caching_list_directory(dir, &tmp)) { debugf(DBG_LEVEL_EXT, "exit 0: path_info(%s) "KYEL"[CACHE-DIR-MISS]", dir); return NULL; } - else { + else debugf(DBG_LEVEL_EXT, "path_info(%s) "KGRN"[CACHE-DIR-HIT]", dir); - } //iterate in file list obtained from cache or downloaded for (; tmp; tmp = tmp->next) { @@ -558,31 +574,33 @@ dir_entry *path_info(const char *path) //retrieve folder from local cache if exists, return null if does not exist (rather than download) -int check_caching_list_directory(const char *path, dir_entry **list) +int check_caching_list_directory(const char* path, dir_entry** list) { debugf(DBG_LEVEL_EXT, "check_caching_list_directory(%s)", path); pthread_mutex_lock(&dcachemut); if (!strcmp(path, "/")) path = ""; - dir_cache *cw; + dir_cache* cw; for (cw = dcache; cw; cw = cw->next) if (!strcmp(cw->path, path)) { *list = cw->entries; pthread_mutex_unlock(&dcachemut); - debugf(DBG_LEVEL_EXT, "exit 0: check_caching_list_directory(%s) "KGRN"[CACHE-DIR-HIT]", path); + debugf(DBG_LEVEL_EXT, + "exit 0: check_caching_list_directory(%s) "KGRN"[CACHE-DIR-HIT]", path); return 1; } pthread_mutex_unlock(&dcachemut); - debugf(DBG_LEVEL_EXT, "exit 1: check_caching_list_directory(%s) "KYEL"[CACHE-DIR-MISS]", path); + debugf(DBG_LEVEL_EXT, + "exit 1: check_caching_list_directory(%s) "KYEL"[CACHE-DIR-MISS]", path); return 0; } -dir_entry * check_parent_folder_for_file(const char *path) +dir_entry* check_parent_folder_for_file(const char* path) { char dir[MAX_PATH_SIZE]; dir_for(path, dir); - dir_entry *tmp; + dir_entry* tmp; if (!check_caching_list_directory(dir, &tmp)) return NULL; else @@ -590,12 +608,12 @@ dir_entry * check_parent_folder_for_file(const char *path) } //check if local path is in cache, without downloading from cloud if not in cache -dir_entry *check_path_info(const char *path) +dir_entry* check_path_info(const char* path) { debugf(DBG_LEVEL_EXT, "check_path_info(%s)", path); char dir[MAX_PATH_SIZE]; dir_for(path, dir); - dir_entry *tmp; + dir_entry* tmp; //get parent folder cache entry if (!check_caching_list_directory(dir, &tmp)) @@ -612,22 +630,20 @@ dir_entry *check_path_info(const char *path) } } if (!strcmp(path, "/")) - { - debugf(DBG_LEVEL_EXT, "exit 2: check_path_info(%s) "KYEL"ignoring root [CACHE-MISS]", path); - } - else { + debugf(DBG_LEVEL_EXT, + "exit 2: check_path_info(%s) "KYEL"ignoring root [CACHE-MISS]", path); + else debugf(DBG_LEVEL_EXT, "exit 3: check_path_info(%s) "KYEL"[CACHE-MISS]", path); - } return NULL; } -char *get_home_dir() +char* get_home_dir() { - char *home; + char* home; if ((home = getenv("HOME")) && !access(home, R_OK)) return home; - struct passwd *pwd = getpwuid(geteuid()); + struct passwd* pwd = getpwuid(geteuid()); if ((home = pwd->pw_dir) && !access(home, R_OK)) return home; return "~"; @@ -638,15 +654,15 @@ void cloudfs_debug(int dbg) debug = dbg; } -void debugf(int level, char *fmt, ...) +void debugf(int level, char* fmt, ...) { - if (debug) + if (debug) { if (level <= option_debug_level) { #ifdef SYS_gettid pid_t thread_id = syscall(SYS_gettid); - #else +#else int thread_id = 0; #error "SYS_gettid unavailable on this system" #endif diff --git a/commonfs.h b/commonfs.h index eb31b9a..2a3ee3c 100644 --- a/commonfs.h +++ b/commonfs.h @@ -44,16 +44,16 @@ typedef enum { false, true } bool; //linked list with files in a directory typedef struct dir_entry { - char *name; - char *full_name; - char *content_type; + char* name; + char* full_name; + char* content_type; off_t size; time_t last_modified; // implement utimens struct timespec mtime; struct timespec ctime; struct timespec atime; - char *md5sum; //interesting capability for rsync/scrub + char* md5sum; //interesting capability for rsync/scrub mode_t chmod; uid_t uid; gid_t gid; @@ -63,47 +63,51 @@ typedef struct dir_entry // end change int isdir; int islink; - struct dir_entry *next; + struct dir_entry* next; } dir_entry; // linked list with cached folder names typedef struct dir_cache { - char *path; - dir_entry *entries; + char* path; + dir_entry* entries; time_t cached; //added cache support based on access time time_t accessed_in_cache; bool was_deleted; //end change - struct dir_cache *next, *prev; + struct dir_cache* next, *prev; } dir_cache; -time_t my_timegm(struct tm *tm); -time_t get_time_from_str_as_gmt(char *time_str); -time_t get_time_as_local(time_t time_t_val, char time_str[], int char_buf_size); -int get_time_as_string(time_t time_t_val, long nsec, char *time_str, int time_str_len); +time_t my_timegm(struct tm* tm); +time_t get_time_from_str_as_gmt(char* time_str); +time_t get_time_as_local(time_t time_t_val, char time_str[], + int char_buf_size); +int get_time_as_string(time_t time_t_val, long nsec, char* time_str, + int time_str_len); time_t get_time_now(); -int get_timespec_as_str(const struct timespec *times, char *time_str, int time_str_len); -char *str2md5(const char *str, int length); -int file_md5(FILE *file_handle, char *md5_file_str); -void debug_print_descriptor(struct fuse_file_info *info); -int get_safe_cache_file_path(const char *file_path, char *file_path_safe, char *temp_dir); -dir_entry *init_dir_entry(); -void copy_dir_entry(dir_entry *src, dir_entry *dst); -dir_cache *new_cache(const char *path); -void dir_for(const char *path, char *dir); +int get_timespec_as_str(const struct timespec* times, char* time_str, + int time_str_len); +char* str2md5(const char* str, int length); +int file_md5(FILE* file_handle, char* md5_file_str); +void debug_print_descriptor(struct fuse_file_info* info); +int get_safe_cache_file_path(const char* file_path, char* file_path_safe, + char* temp_dir); +dir_entry* init_dir_entry(); +void copy_dir_entry(dir_entry* src, dir_entry* dst); +dir_cache* new_cache(const char* path); +void dir_for(const char* path, char* dir); void debug_list_cache_content(); -void update_dir_cache(const char *path, off_t size, int isdir, int islink); -dir_entry *path_info(const char *path); -dir_entry *check_path_info(const char *path); -dir_entry * check_parent_folder_for_file(const char *path); -void dir_decache(const char *path); -void cloudfs_free_dir_list(dir_entry *dir_list); -extern int cloudfs_list_directory(const char *path, dir_entry **); -int caching_list_directory(const char *path, dir_entry **list); -char *get_home_dir(); +void update_dir_cache(const char* path, off_t size, int isdir, int islink); +dir_entry* path_info(const char* path); +dir_entry* check_path_info(const char* path); +dir_entry* check_parent_folder_for_file(const char* path); +void dir_decache(const char* path); +void cloudfs_free_dir_list(dir_entry* dir_list); +extern int cloudfs_list_directory(const char* path, dir_entry**); +int caching_list_directory(const char* path, dir_entry** list); +char* get_home_dir(); void cloudfs_debug(int dbg); -void debugf(int level, char *fmt, ...); +void debugf(int level, char* fmt, ...); #endif From a1a0e4f353b338fce89f255b1d4cc8cb1207cab8 Mon Sep 17 00:00:00 2001 From: Pascal Obry Date: Mon, 30 Nov 2015 22:10:12 +0100 Subject: [PATCH 11/12] Do some more clean-up. --- Makefile.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.in b/Makefile.in index d5e5d79..54f2548 100644 --- a/Makefile.in +++ b/Makefile.in @@ -29,7 +29,7 @@ hubicfuse: $(SOURCES) $(HEADERS) $(CC) $(CFLAGS) -o hubicfuse $(SOURCES) $(LIBS) $(LDFLAGS) clean: - /bin/rm -f hubicfuse + /bin/rm -fr hubicfuse *.orig config.h *~ aclocal.m4 autom4te.cache distclean: clean /bin/rm -f Makefile config.h config.status config.cache config.log \ From 2fde7267f50fcce80ddb679142bd29ebf6e96b0b Mon Sep 17 00:00:00 2001 From: Pascal Obry Date: Mon, 30 Nov 2015 22:12:29 +0100 Subject: [PATCH 12/12] Bump version to 3.0.0 --- configure | 36 ++++++++++++++++++++++++------------ configure.ac | 2 +- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/configure b/configure index 4b32952..0baa73d 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for hubicfuse 1.0.1. +# Generated by GNU Autoconf 2.69 for hubicfuse 3.0.0. # # Report bugs to >. # @@ -580,8 +580,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='hubicfuse' PACKAGE_TARNAME='hubicfuse' -PACKAGE_VERSION='1.0.1' -PACKAGE_STRING='hubicfuse 1.0.1' +PACKAGE_VERSION='3.0.0' +PACKAGE_STRING='hubicfuse 3.0.0' PACKAGE_BUGREPORT='Pascal Obry ' PACKAGE_URL='' @@ -673,6 +673,7 @@ infodir docdir oldincludedir includedir +runstatedir localstatedir sharedstatedir sysconfdir @@ -756,6 +757,7 @@ datadir='${datarootdir}' sysconfdir='${prefix}/etc' sharedstatedir='${prefix}/com' localstatedir='${prefix}/var' +runstatedir='${localstatedir}/run' includedir='${prefix}/include' oldincludedir='/usr/include' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' @@ -1008,6 +1010,15 @@ do | -silent | --silent | --silen | --sile | --sil) silent=yes ;; + -runstatedir | --runstatedir | --runstatedi | --runstated \ + | --runstate | --runstat | --runsta | --runst | --runs \ + | --run | --ru | --r) + ac_prev=runstatedir ;; + -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \ + | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \ + | --run=* | --ru=* | --r=*) + runstatedir=$ac_optarg ;; + -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) ac_prev=sbindir ;; -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ @@ -1145,7 +1156,7 @@ fi for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ datadir sysconfdir sharedstatedir localstatedir includedir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ - libdir localedir mandir + libdir localedir mandir runstatedir do eval ac_val=\$$ac_var # Remove trailing slashes. @@ -1258,7 +1269,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures hubicfuse 1.0.1 to adapt to many kinds of systems. +\`configure' configures hubicfuse 3.0.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1298,6 +1309,7 @@ Fine tuning of the installation directories: --sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --localstatedir=DIR modifiable single-machine data [PREFIX/var] + --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] --libdir=DIR object code libraries [EPREFIX/lib] --includedir=DIR C header files [PREFIX/include] --oldincludedir=DIR C header files for non-gcc [/usr/include] @@ -1319,7 +1331,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of hubicfuse 1.0.1:";; + short | recursive ) echo "Configuration of hubicfuse 3.0.0:";; esac cat <<\_ACEOF @@ -1414,7 +1426,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -hubicfuse configure 1.0.1 +hubicfuse configure 3.0.0 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1894,7 +1906,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by hubicfuse $as_me 1.0.1, which was +It was created by hubicfuse $as_me 3.0.0, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -3609,8 +3621,8 @@ fi # Put the nasty error message in config.log where it belongs echo "$JSON_PKG_ERRORS" >&5 - #JSON_CFLAGS C compiler flags for JSON, overriding pkg-config - #JSON_LIBS linker flags for JSON, overriding pkg-config + JSON_CFLAGS C compiler flags for JSON, overriding pkg-config + JSON_LIBS linker flags for JSON, overriding pkg-config pkg_failed=no { $as_echo "$as_me:${as_lineno-$LINENO}: checking for JSON" >&5 @@ -5733,7 +5745,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by hubicfuse $as_me 1.0.1, which was +This file was extended by hubicfuse $as_me 3.0.0, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -5795,7 +5807,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -hubicfuse config.status 1.0.1 +hubicfuse config.status 3.0.0 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index e1f66aa..8aa1fa1 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ(2.61) -AC_INIT([hubicfuse], [2.1.0], [Pascal Obry ]) +AC_INIT([hubicfuse], [3.0.0], [Pascal Obry ]) AC_SUBST(CPPFLAGS, "$CPPFLAGS -D_FILE_OFFSET_BITS=64 -I/usr/include/libxml2") AC_LANG([C]) AC_CONFIG_SRCDIR([cloudfuse.c])