starred/hubicfuse
1
0
Fork 0
mirror of https://github.com/TurboGit/hubicfuse.git synced 2026-08-03 03:11:54 +02:00
Code Issues Packages Projects Releases Wiki Activity

Reformat code using astyle.

Browse Source
This commit is contained in:
Pascal Obry
2015-11-30 22:08:13 +01:00
parent ba9d9fc988
commit 66c934c8a0
5 changed files with 991 additions and 795 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cloudfsapi.c
+232 -151
View File
@@ -48,7 +48,8 @@ extern bool option_extensive_debug;
extern bool option_enable_chown;
extern bool option_enable_chmod;
static int rhel5_mode = 0;
static struct statvfs statcache = {
static struct statvfs statcache =
{
.f_bsize = 4096,
.f_frsize = 4096,
.f_blocks = INT_MAX,
@@ -62,7 +63,8 @@ static struct statvfs statcache = {
//used to compute statfs cache interval
static time_t last_stat_read_time = 0;
extern FuseOptions options;
struct MemoryStruct {
struct MemoryStruct
{
char* memory;
size_t size;
};
@@ -118,12 +120,16 @@ static void add_header(curl_slist **headers, const char *name,
const char* value_ptr;
debugf(DBG_LEVEL_EXT, "add_header(%s:%s)", name, value);
if (strlen(value) > 256) {
debugf(DBG_LEVEL_NORM, KRED"add_header: warning, value size > 256 (%s:%s) ", name, value);
if (strlen(value) > 256)
{
debugf(DBG_LEVEL_NORM, KRED"add_header: warning, value size > 256 (%s:%s) ",
name, value);
//hubic will throw an HTTP 400 error on X-Copy-To operation if X-Object-Meta-FilePath header value is larger than 256 chars
//fix for issue #95 https://github.com/TurboGit/hubicfuse/issues/95
if (!strcasecmp(name, "X-Object-Meta-FilePath")) {
debugf(DBG_LEVEL_NORM, KRED"add_header: trimming header (%s) value to max allowed", name);
if (!strcasecmp(name, "X-Object-Meta-FilePath"))
{
debugf(DBG_LEVEL_NORM,
KRED"add_header: trimming header (%s) value to max allowed", name);
//trim header size to max allowed
strncpy(safe_value, value, 256 - 1);
safe_value[255] = '\0';
@@ -139,7 +145,8 @@ static void add_header(curl_slist **headers, const char *name,
*headers = curl_slist_append(*headers, x_header);
}
static size_t header_dispatch(void *ptr, size_t size, size_t nmemb, void *dir_entry)
static size_t header_dispatch(void* ptr, size_t size, size_t nmemb,
void* dir_entry)
{
char* header = (char*)alloca(size * nmemb + 1);
char* head = (char*)alloca(size * nmemb + 1);
@@ -153,9 +160,11 @@ static size_t header_dispatch(void *ptr, size_t size, size_t nmemb, void *dir_en
if (!strncasecmp(head, "x-storage-url", size * nmemb))
strncpy(storage_url, value, sizeof(storage_url));
if (!strncasecmp(head, "x-account-meta-quota", size * nmemb))
statcache.f_blocks = (unsigned long) (strtoull(value, NULL, 10)/statcache.f_frsize);
statcache.f_blocks = (unsigned long) (strtoull(value, NULL,
10) / statcache.f_frsize);
if (!strncasecmp(head, "x-account-bytes-used", size * nmemb))
statcache.f_bfree = statcache.f_bavail = statcache.f_blocks - (unsigned long) (strtoull(value, NULL, 10)/statcache.f_frsize);
statcache.f_bfree = statcache.f_bavail = statcache.f_blocks - (unsigned long) (
strtoull(value, NULL, 10) / statcache.f_frsize);
if (!strncasecmp(head, "x-account-object-count", size * nmemb))
{
unsigned long object_count = strtoul(value, NULL, 10);
@@ -166,7 +175,9 @@ static size_t header_dispatch(void *ptr, size_t size, size_t nmemb, void *dir_en
return size * nmemb;
}
static void header_set_time_from_str(char *time_str, struct timespec *time_entry) {
static void header_set_time_from_str(char* time_str,
struct timespec* time_entry)
{
char sec_value[TIME_CHARS] = { 0 };
char nsec_value[TIME_CHARS] = { 0 };
time_t sec;
@@ -178,21 +189,25 @@ static void header_set_time_from_str(char *time_str, struct timespec *time_entry
sec_value, nsec_value, sec, nsec, time_entry->tv_sec, time_entry->tv_nsec);
if (sec != time_entry->tv_sec || nsec != time_entry->tv_nsec)
{
debugf(DBG_LEVEL_EXTALL, "Time changed, setting new time=%li.%li, existing was=%li.%li",
debugf(DBG_LEVEL_EXTALL,
"Time changed, setting new time=%li.%li, existing was=%li.%li",
sec, nsec, time_entry->tv_sec, time_entry->tv_nsec);
time_entry->tv_sec = sec;
time_entry->tv_nsec = nsec;
char time_str_local[TIME_CHARS] = "";
get_time_as_string((time_t)sec, nsec, time_str_local, sizeof(time_str_local));
debugf(DBG_LEVEL_EXTALL, "header_set_time_from_str received time=[%s]", time_str_local);
debugf(DBG_LEVEL_EXTALL, "header_set_time_from_str received time=[%s]",
time_str_local);
get_timespec_as_str(time_entry, time_str_local, sizeof(time_str_local));
debugf(DBG_LEVEL_EXTALL, "header_set_time_from_str set time=[%s]", time_str_local);
debugf(DBG_LEVEL_EXTALL, "header_set_time_from_str set time=[%s]",
time_str_local);
}
}
static size_t header_get_meta_dispatch(void *ptr, size_t size, size_t nmemb, void *userdata)
static size_t header_get_meta_dispatch(void* ptr, size_t size, size_t nmemb,
void* userdata)
{
char* header = (char*)alloca(size * nmemb + 1);
char* head = (char*)alloca(size * nmemb + 1);
@@ -207,41 +222,32 @@ static size_t header_get_meta_dispatch(void *ptr, size_t size, size_t nmemb, voi
if (de != NULL)
{
if (!strncasecmp(head, HEADER_TEXT_ATIME, size * nmemb))
{
header_set_time_from_str(value, &de->atime);
}
if (!strncasecmp(head, HEADER_TEXT_CTIME, size * nmemb))
{
header_set_time_from_str(value, &de->ctime);
}
if (!strncasecmp(head, HEADER_TEXT_MTIME, size * nmemb))
{
header_set_time_from_str(value, &de->mtime);
}
if (!strncasecmp(head, HEADER_TEXT_CHMOD, size * nmemb))
{
de->chmod = atoi(value);
}
if (!strncasecmp(head, HEADER_TEXT_GID, size * nmemb))
{
de->gid = atoi(value);
}
if (!strncasecmp(head, HEADER_TEXT_UID, size * nmemb))
{
de->uid = atoi(value);
}
else
debugf(DBG_LEVEL_EXT,
"Unexpected NULL dir_entry on header(%s), file should be in cache already",
storage);
}
else {
debugf(DBG_LEVEL_EXT, "Unexpected NULL dir_entry on header(%s), file should be in cache already", storage);
}
}
else {
else
{
//debugf(DBG_LEVEL_NORM, "Received unexpected header line");
}
return size * nmemb;
}
static size_t rw_callback(size_t (*rw)(void*, size_t, size_t, FILE*), void *ptr,
static size_t rw_callback(size_t (*rw)(void*, size_t, size_t, FILE*),
void* ptr,
size_t size, size_t nmemb, void* userp)
{
struct segment_info* info = (struct segment_info*)userp;
@@ -270,7 +276,8 @@ static size_t write_callback(void *ptr, size_t size, size_t nmemb, void *userp)
}
//http://curl.haxx.se/libcurl/c/CURLOPT_XFERINFOFUNCTION.html
int progress_callback_xfer(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
int progress_callback_xfer(void* clientp, curl_off_t dltotal, curl_off_t dlnow,
curl_off_t ultotal, curl_off_t ulnow)
{
struct curl_progress* myp = (struct curl_progress*)clientp;
CURL* curl = myp->curl;
@@ -295,23 +302,28 @@ int progress_callback_xfer(void *clientp, curl_off_t dltotal, curl_off_t dlnow,
point = dlnow + ulnow;
frac = (double)point / (double)total;
percent = frac * 100.0f;
debugf(DBG_LEVEL_EXT, "TOTAL TIME: %.0f sec Down=%.0f Kbps UP=%.0f Kbps", curtime, dspeed/1024, uspeed/1024);
debugf(DBG_LEVEL_EXT, "UP: %lld of %lld DOWN: %lld/%lld Completion %.1f %%", ulnow, ultotal, dlnow, dltotal, percent);
debugf(DBG_LEVEL_EXT, "TOTAL TIME: %.0f sec Down=%.0f Kbps UP=%.0f Kbps",
curtime, dspeed / 1024, uspeed / 1024);
debugf(DBG_LEVEL_EXT, "UP: %lld of %lld DOWN: %lld/%lld Completion %.1f %%",
ulnow, ultotal, dlnow, dltotal, percent);
}
return 0;
}
//http://curl.haxx.se/libcurl/c/CURLOPT_PROGRESSFUNCTION.html
int progress_callback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
int progress_callback(void* clientp, double dltotal, double dlnow,
double ultotal, double ulnow)
{
return progress_callback_xfer(clientp, (curl_off_t)dltotal, (curl_off_t)dlnow, (curl_off_t)ultotal, (curl_off_t)ulnow);
return progress_callback_xfer(clientp, (curl_off_t)dltotal, (curl_off_t)dlnow,
(curl_off_t)ultotal, (curl_off_t)ulnow);
}
//get the response from HTTP requests, mostly for debug purposes
// http://stackoverflow.com/questions/2329571/c-libcurl-get-output-into-a-string
// http://curl.haxx.se/libcurl/c/getinmemory.html
size_t writefunc_callback(void *contents, size_t size, size_t nmemb, void *userp)
size_t writefunc_callback(void* contents, size_t size, size_t nmemb,
void* userp)
{
size_t realsize = size * nmemb;
struct MemoryStruct* mem = (struct MemoryStruct*)userp;
@@ -348,13 +360,15 @@ static int send_request_size(const char *method, const char *path, void *fp,
//needed to keep the response data, for debug purposes
struct MemoryStruct chunk;
if (!storage_url[0]) {
if (!storage_url[0])
{
debugf(DBG_LEVEL_NORM, KRED"send_request with no storage_url?");
abort();
}
//char *encoded_path = curl_escape(path, 0);
while ((slash = strstr(path, "%2F")) || (slash = strstr(path, "%2f"))) {
while ((slash = strstr(path, "%2F")) || (slash = strstr(path, "%2f")))
{
*slash = '/';
memmove(slash + 1, slash + 3, strlen(slash + 3) + 1);
}
@@ -385,27 +399,33 @@ static int send_request_size(const char *method, const char *path, void *fp,
add_header(&headers, "X-Auth-Token", storage_token);
dir_entry* de;
if (de_cached_entry == NULL)
{
de = check_path_info(unencoded_path);
}
else {
else
{
// updating metadata on a file about to be added to cache (for x-copy, dest meta = src meta)
de = de_cached_entry;
debugf(DBG_LEVEL_EXTALL, "send_request_size: using param dir_entry(%s)", orig_path);
debugf(DBG_LEVEL_EXTALL, "send_request_size: using param dir_entry(%s)",
orig_path);
}
if (!de)
debugf(DBG_LEVEL_EXTALL,
"send_request_size: "KYEL"file not in cache (%s)(%s)(%s)", orig_path, path,
unencoded_path);
else
{
debugf(DBG_LEVEL_EXTALL, "send_request_size: "KYEL"file not in cache (%s)(%s)(%s)", orig_path, path, unencoded_path);
}
else {
// add headers to save utimens attribs only on upload
if (!strcasecmp(method, "PUT") || !strcasecmp(method, "MKDIR"))
{
debugf(DBG_LEVEL_EXTALL, "send_request_size: Saving utimens for file %s", orig_path);
debugf(DBG_LEVEL_EXTALL, "send_request_size: Cached utime for path=%s ctime=%li.%li mtime=%li.%li atime=%li.%li", orig_path,
de->ctime.tv_sec, de->ctime.tv_nsec, de->mtime.tv_sec, de->mtime.tv_nsec, de->atime.tv_sec, de->atime.tv_nsec);
debugf(DBG_LEVEL_EXTALL, "send_request_size: Saving utimens for file %s",
orig_path);
debugf(DBG_LEVEL_EXTALL,
"send_request_size: Cached utime for path=%s ctime=%li.%li mtime=%li.%li atime=%li.%li",
orig_path,
de->ctime.tv_sec, de->ctime.tv_nsec, de->mtime.tv_sec, de->mtime.tv_nsec,
de->atime.tv_sec, de->atime.tv_nsec);
char atime_str_nice[TIME_CHARS] = "", mtime_str_nice[TIME_CHARS] = "", ctime_str_nice[TIME_CHARS] = "";
char atime_str_nice[TIME_CHARS] = "", mtime_str_nice[TIME_CHARS] = "",
ctime_str_nice[TIME_CHARS] = "";
get_timespec_as_str(&(de->atime), atime_str_nice, sizeof(atime_str_nice));
debugf(DBG_LEVEL_EXTALL, KCYN"send_request_size: atime=[%s]", atime_str_nice);
get_timespec_as_str(&(de->mtime), mtime_str_nice, sizeof(mtime_str_nice));
@@ -415,9 +435,12 @@ static int send_request_size(const char *method, const char *path, void *fp,
char mtime_str[TIME_CHARS], atime_str[TIME_CHARS], ctime_str[TIME_CHARS];
char string_float[TIME_CHARS];
snprintf(mtime_str, TIME_CHARS, "%lu.%lu", de->mtime.tv_sec, de->mtime.tv_nsec);
snprintf(atime_str, TIME_CHARS, "%lu.%lu", de->atime.tv_sec, de->atime.tv_nsec);
snprintf(ctime_str, TIME_CHARS, "%lu.%lu", de->ctime.tv_sec, de->ctime.tv_nsec);
snprintf(mtime_str, TIME_CHARS, "%lu.%lu", de->mtime.tv_sec,
de->mtime.tv_nsec);
snprintf(atime_str, TIME_CHARS, "%lu.%lu", de->atime.tv_sec,
de->atime.tv_nsec);
snprintf(ctime_str, TIME_CHARS, "%lu.%lu", de->ctime.tv_sec,
de->ctime.tv_nsec);
add_header(&headers, HEADER_TEXT_FILEPATH, orig_path);
add_header(&headers, HEADER_TEXT_MTIME, mtime_str);
add_header(&headers, HEADER_TEXT_ATIME, atime_str);
@@ -434,9 +457,9 @@ static int send_request_size(const char *method, const char *path, void *fp,
add_header(&headers, HEADER_TEXT_UID, uid_str);
add_header(&headers, HEADER_TEXT_CHMOD, chmod_str);
}
else {
debugf(DBG_LEVEL_EXTALL, "send_request_size: not setting utimes (%s)", orig_path);
}
else
debugf(DBG_LEVEL_EXTALL, "send_request_size: not setting utimes (%s)",
orig_path);
}
if (!strcasecmp(method, "MKDIR"))
{
@@ -462,9 +485,8 @@ static int send_request_size(const char *method, const char *path, void *fp,
curl_easy_setopt(curl, CURLOPT_INFILESIZE, file_size);
curl_easy_setopt(curl, CURLOPT_READDATA, fp);
}
else {
else
curl_easy_setopt(curl, CURLOPT_INFILESIZE, 0);
}
if (is_segment)
curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_callback);
//enable progress reporting
@@ -496,7 +518,8 @@ static int send_request_size(const char *method, const char *path, void *fp,
fflush(fp);
if (ftruncate(fileno(fp), 0) < 0)
{
debugf(DBG_LEVEL_NORM, KRED"ftruncate failed. I don't know what to do about that.");
debugf(DBG_LEVEL_NORM,
KRED"ftruncate failed. I don't know what to do about that.");
abort();
}
curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
@@ -517,7 +540,8 @@ static int send_request_size(const char *method, const char *path, void *fp,
curl_easy_setopt(curl, CURLOPT_WRITEDATA, xmlctx);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &xml_dispatch);
}
else {
else
{
//asumming retrieval of headers only
debugf(DBG_LEVEL_EXT, "send_request_size: GET HEADERS only(%s)");
curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, &header_get_meta_dispatch);
@@ -525,7 +549,8 @@ static int send_request_size(const char *method, const char *path, void *fp,
curl_easy_setopt(curl, CURLOPT_NOBODY, 1);
}
}
else {
else
{
debugf(DBG_LEVEL_EXT, "send_request_size: catch_all (%s)");
// this posts an HEAD request (e.g. for statfs)
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, method);
@@ -539,14 +564,16 @@ static int send_request_size(const char *method, const char *path, void *fp,
headers = curl_slist_append(headers, extra->data);
}
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
debugf(DBG_LEVEL_EXT, "status: send_request_size(%s) started HTTP REQ:%s", orig_path, url);
debugf(DBG_LEVEL_EXT, "status: send_request_size(%s) started HTTP REQ:%s",
orig_path, url);
curl_easy_perform(curl);
double total_time;
char* effective_url;
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &effective_url);
curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &total_time);
debugf(DBG_LEVEL_EXT, "status: send_request_size(%s) completed HTTP REQ:%s total_time=%.1f seconds",
debugf(DBG_LEVEL_EXT,
"status: send_request_size(%s) completed HTTP REQ:%s total_time=%.1f seconds",
orig_path, effective_url, total_time);
curl_slist_free_all(headers);
curl_easy_reset(curl);
@@ -555,46 +582,59 @@ static int send_request_size(const char *method, const char *path, void *fp,
if (response != 404 && (response >= 400 || response < 200))
{
/*
* Now, our chunk.memory points to a memory block that is chunk.size
* bytes big and contains the remote file.
Now, our chunk.memory points to a memory block that is chunk.size
bytes big and contains the remote file.
*/
debugf(DBG_LEVEL_NORM, KRED"send_request_size: error message, size=%lu, [HTTP %d] (%s)(%s)",
debugf(DBG_LEVEL_NORM,
KRED"send_request_size: error message, size=%lu, [HTTP %d] (%s)(%s)",
(long)chunk.size, response, method, path);
debugf(DBG_LEVEL_NORM, KRED"send_request_size: error message=[%s]", chunk.memory);
debugf(DBG_LEVEL_NORM, KRED"send_request_size: error message=[%s]",
chunk.memory);
}
free(chunk.memory);
if ((response >= 200 && response < 400) || (!strcasecmp(method, "DELETE") && response == 409))
if ((response >= 200 && response < 400) || (!strcasecmp(method, "DELETE")
&& response == 409))
{
debugf(DBG_LEVEL_NORM, "exit 0: send_request_size(%s) speed=%.1f sec "KCYN"(%s) "KGRN"[HTTP OK]",
debugf(DBG_LEVEL_NORM,
"exit 0: send_request_size(%s) speed=%.1f sec "KCYN"(%s) "KGRN"[HTTP OK]",
orig_path, total_time, method);
return response;
}
//handle cases when file is not found, no point in retrying, will exit
if (response == 404)
{
debugf(DBG_LEVEL_NORM, "send_request_size: not found error for (%s)(%s), ignored "KYEL"[HTTP 404].", method, path);
debugf(DBG_LEVEL_NORM,
"send_request_size: not found error for (%s)(%s), ignored "KYEL"[HTTP 404].",
method, path);
return response;
}
else {
debugf(DBG_LEVEL_NORM, "send_request_size: httpcode=%d (%s)(%s), retrying "KRED"[HTTP ERR]", response, method, path);
else
{
debugf(DBG_LEVEL_NORM,
"send_request_size: httpcode=%d (%s)(%s), retrying "KRED"[HTTP ERR]", response,
method, path);
//todo: try to list response content for debug purposes
sleep(8 << tries); // backoff
}
if (response == 401 && !cloudfs_connect())
{ // re-authenticate on 401s
debugf(DBG_LEVEL_NORM, KYEL"exit 1: send_request_size(%s) (%s) [HTTP REAUTH]", path, method);
{
// re-authenticate on 401s
debugf(DBG_LEVEL_NORM, KYEL"exit 1: send_request_size(%s) (%s) [HTTP REAUTH]",
path, method);
return response;
}
if (xmlctx)
xmlCtxtResetPush(xmlctx, NULL, 0, NULL, NULL);
}
debugf(DBG_LEVEL_NORM, "exit 2: send_request_size(%s)"KCYN"(%s) response=%d", path, method, response);
debugf(DBG_LEVEL_NORM, "exit 2: send_request_size(%s)"KCYN"(%s) response=%d",
path, method, response);
return response;
}
int send_request(char* method, const char* path, FILE* fp,
xmlParserCtxtPtr xmlctx, curl_slist *extra_headers, dir_entry *de_cached_entry, const char *unencoded_path)
xmlParserCtxtPtr xmlctx, curl_slist* extra_headers, dir_entry* de_cached_entry,
const char* unencoded_path)
{
long flen = 0;
if (fp)
@@ -603,7 +643,8 @@ int send_request(char *method, const char *path, FILE *fp,
fflush(fp);
flen = cloudfs_file_size(fileno(fp));
}
return send_request_size(method, path, fp, xmlctx, extra_headers, flen, 0, de_cached_entry, unencoded_path);
return send_request_size(method, path, fp, xmlctx, extra_headers, flen, 0,
de_cached_entry, unencoded_path);
}
//thread that downloads or uploads large file segments
@@ -626,7 +667,8 @@ void *upload_segment(void *seginfo)
info->size, 1, NULL, seg_path);
if (!(response >= 200 && response < 300))
fprintf(stderr, "Segment upload %s failed with response %d", seg_path, response);
fprintf(stderr, "Segment upload %s failed with response %d", seg_path,
response);
curl_free(encoded);
fclose(info->fp);
@@ -635,7 +677,8 @@ void *upload_segment(void *seginfo)
// segment_size is the globabl config variable and size_of_segment is local
//TODO: return whether the upload/download failed or not
void run_segment_threads(const char *method, int segments, int full_segments, int remaining,
void run_segment_threads(const char* method, int segments, int full_segments,
int remaining,
FILE* fp, char* seg_base, int size_of_segments)
{
debugf(DBG_LEVEL_EXT, "run_segment_threads(%s)", method);
@@ -665,7 +708,8 @@ void run_segment_threads(const char *method, int segments, int full_segments, in
pthread_create(&threads[i], NULL, upload_segment, (void*) & (info[i]));
}
for (i = 0; i < segments; i++) {
for (i = 0; i < segments; i++)
{
if ((ret = pthread_join(threads[i], NULL)) != 0)
fprintf(stderr, "error waiting for thread %d, status = %d\n", i, ret);
}
@@ -684,7 +728,8 @@ void split_path(const char *path, char *seg_base, char *container,
char* _object = strsep(&string, "/");
char* remstr;
while (remstr = strsep(&string, "/")) {
while (remstr = strsep(&string, "/"))
{
strncat(container, "/",
MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE));
strncat(container, _object,
@@ -701,7 +746,8 @@ void split_path(const char *path, char *seg_base, char *container,
}
//checks on the cloud if this file (seg_path) have an associated segment folder
int internal_is_segmented(const char *seg_path, const char *object, const char *parent_path)
int internal_is_segmented(const char* seg_path, const char* object,
const char* parent_path)
{
debugf(DBG_LEVEL_EXT, "internal_is_segmented(%s)", seg_path);
//try to avoid an additional http request for small files
@@ -713,27 +759,33 @@ int internal_is_segmented(const char *seg_path, const char *object, const char *
//to force segment meta download for segmented files
potentially_segmented = true;
}
else {
else
{
//potentially segmented, assumption is that 0 size files are potentially segmented
//while size>0 is for sure not segmented, so no point in making an expensive HTTP GET call
potentially_segmented = (de->size == 0 && !de->isdir) ? true : false;
}
debugf(DBG_LEVEL_EXT, "internal_is_segmented: potentially segmented=%d", potentially_segmented);
debugf(DBG_LEVEL_EXT, "internal_is_segmented: potentially segmented=%d",
potentially_segmented);
dir_entry* seg_dir;
if (potentially_segmented && cloudfs_list_directory(seg_path, &seg_dir))
{
if (seg_dir && seg_dir->isdir)
{
do {
do
{
if (!strncmp(seg_dir->name, object, MAX_URL_SIZE))
{
debugf(DBG_LEVEL_EXT, "exit 0: internal_is_segmented(%s) "KGRN"TRUE", seg_path);
debugf(DBG_LEVEL_EXT, "exit 0: internal_is_segmented(%s) "KGRN"TRUE",
seg_path);
return 1;
}
} while ((seg_dir = seg_dir->next));
}
while ((seg_dir = seg_dir->next));
}
}
debugf(DBG_LEVEL_EXT, "exit 1: internal_is_segmented(%s) "KYEL"FALSE", seg_path);
debugf(DBG_LEVEL_EXT, "exit 1: internal_is_segmented(%s) "KYEL"FALSE",
seg_path);
return 0;
}
@@ -790,7 +842,8 @@ int format_segments(const char *path, char * seg_base, long *segments,
char* str_size = seg_dir->name;
snprintf(manifest, MAX_URL_SIZE, "%s/%s", seg_path, str_size);
debugf(DBG_LEVEL_EXT, KMAG"format_segments manifest2(%s) size=%s", manifest, str_size);
debugf(DBG_LEVEL_EXT, KMAG"format_segments manifest2(%s) size=%s", manifest,
str_size);
if (!cloudfs_list_directory(manifest, &seg_dir))
{
debugf(DBG_LEVEL_EXT, "exit 2: format_segments(%s)", path);
@@ -821,18 +874,20 @@ int format_segments(const char *path, char * seg_base, long *segments,
strncpy(tmp, seg_base, MAX_URL_SIZE);
snprintf(seg_base, MAX_URL_SIZE, "%s/%s", tmp, manifest);
debugf(DBG_LEVEL_EXT, KMAG"format_segments seg_base(%s)", seg_base);
debugf(DBG_LEVEL_EXT, KMAG"exit 4: format_segments(%s) total=%d size_of_segments=%d remaining=%d, full_segments=%d segments=%d",
debugf(DBG_LEVEL_EXT,
KMAG"exit 4: format_segments(%s) total=%d size_of_segments=%d remaining=%d, full_segments=%d segments=%d",
path, &total_size, &size_of_segments, &remaining, &full_segments, &segments);
return 1;
}
else {
else
{
debugf(DBG_LEVEL_EXT, KMAG"exit 5: format_segments(%s) not segmented?", path);
return 0;
}
}
/*
* Public interface
Public interface
*/
void cloudfs_init()
@@ -874,7 +929,8 @@ void cloudfs_free()
debugf(DBG_LEVEL_EXT, "Destroy mutex");
pthread_mutex_destroy(&pool_mut);
int n;
for (n = 0; n < curl_pool_count; ++n) {
for (n = 0; n < curl_pool_count; ++n)
{
debugf(DBG_LEVEL_EXT, "Cleaning curl conn %d", n);
curl_easy_cleanup(curl_pool[n]);
}
@@ -926,14 +982,11 @@ int cloudfs_object_read_fp(const char *path, FILE *fp)
if (is_segmented(path))
{
if (!cloudfs_delete_object(path))
{
debugf(DBG_LEVEL_NORM, KRED"cloudfs_object_read_fp: couldn't delete existing file");
}
debugf(DBG_LEVEL_NORM,
KRED"cloudfs_object_read_fp: couldn't delete existing file");
else
{
debugf(DBG_LEVEL_EXT, KYEL"cloudfs_object_read_fp: deleted existing file");
}
}
struct timespec now;
if (flen >= segment_above)
@@ -974,13 +1027,16 @@ int cloudfs_object_read_fp(const char *path, FILE *fp)
add_header(&headers, "x-object-manifest", manifest);
add_header(&headers, "Content-Length", "0");
add_header(&headers, "Content-Type", filemimetype);
int response = send_request_size("PUT", encoded, NULL, NULL, headers, 0, 0, NULL, path);
int response = send_request_size("PUT", encoded, NULL, NULL, headers, 0, 0,
NULL, path);
curl_slist_free_all(headers);
curl_free(encoded);
debugf(DBG_LEVEL_EXT, "exit 0: cloudfs_object_read_fp(%s) uploaded ok, response=%d", path, response);
debugf(DBG_LEVEL_EXT,
"exit 0: cloudfs_object_read_fp(%s) uploaded ok, response=%d", path, response);
return (response >= 200 && response < 300);
}
else{
else
{
// assume enters here when file is composed of only one segment (small files)
debugf(DBG_LEVEL_EXT, "cloudfs_object_read_fp(%s) "KYEL"unknown state", path);
}
@@ -989,9 +1045,8 @@ int cloudfs_object_read_fp(const char *path, FILE *fp)
dir_entry* de = path_info(path);
if (!de)
debugf(DBG_LEVEL_EXT, "cloudfs_object_read_fp(%s) not in cache", path);
else {
else
debugf(DBG_LEVEL_EXT, "cloudfs_object_read_fp(%s) found in cache", path);
}
int response = send_request("PUT", encoded, fp, NULL, NULL, NULL, path);
curl_free(encoded);
debugf(DBG_LEVEL_EXT, "exit 1: cloudfs_object_read_fp(%s)", path);
@@ -1019,7 +1074,8 @@ int cloudfs_object_write_fp(const char *path, FILE *fp)
fflush(fp);
if (ftruncate(fileno(fp), 0) < 0)
{
debugf(DBG_LEVEL_NORM, KRED"ftruncate failed. I don't know what to do about that.");
debugf(DBG_LEVEL_NORM,
KRED"ftruncate failed. I don't know what to do about that.");
abort();
}
run_segment_threads("GET", segments, full_segments, remaining, fp,
@@ -1052,7 +1108,8 @@ int cloudfs_object_truncate(const char *path, off_t size)
fclose(fp);
}
else
{//TODO: this is busted
{
//TODO: this is busted
response = send_request("GET", encoded, NULL, NULL, NULL, NULL, path);
}
curl_free(encoded);
@@ -1072,19 +1129,19 @@ void get_file_metadata(dir_entry *de)
long remaining;
long size_of_segments;
long total_size;
if (format_segments(de->full_name, seg_base, &segments, &full_segments, &remaining,
if (format_segments(de->full_name, seg_base, &segments, &full_segments,
&remaining,
&size_of_segments, &total_size))
{
de->size = total_size;
}
}
if (option_get_extended_metadata)
{
debugf(DBG_LEVEL_EXT, KCYN "get_file_metadata(%s)", de->full_name);
//retrieve additional file metadata with a quick HEAD query
char* encoded = curl_escape(de->full_name, 0);
de->metadata_downloaded = true;
int response = send_request("GET", encoded, NULL, NULL, NULL, de, de->full_name);
int response = send_request("GET", encoded, NULL, NULL, NULL, de,
de->full_name);
curl_free(encoded);
debugf(DBG_LEVEL_EXT, KCYN "exit: get_file_metadata(%s)", de->full_name);
}
@@ -1136,7 +1193,8 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list)
if ((!strcmp(path, "") || !strcmp(path, "/")) && *override_storage_url)
response = 404;
else {
else
{
// this was generating 404 err on non segmented files (small files)
response = send_request("GET", container, NULL, xmlctx, NULL, NULL, path);
}
@@ -1165,11 +1223,13 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list)
char* content = "<?!?>";
for (text_node = anode->children; text_node; text_node = text_node->next)
{
if (text_node->type == XML_TEXT_NODE){
if (text_node->type == XML_TEXT_NODE)
{
content = (char*)text_node->content;
//debugf(DBG_LEVEL_NORM, "List dir anode=%s content=%s", (const char *)anode->name, content);
}
else {
else
{
//debugf(DBG_LEVEL_NORM, "List dir anode=%s", (const char *)anode->name);
}
}
@@ -1181,14 +1241,10 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list)
if (slash && (0 == *(slash + 1)))
*slash = 0;
if (asprintf(&(de->full_name), "%s/%s", path, de->name) < 0)
{
de->full_name = NULL;
}
}
if (!strcasecmp((const char*)anode->name, "bytes"))
{
de->size = strtoll(content, NULL, 10);
}
if (!strcasecmp((const char*)anode->name, "content_type"))
{
de->content_type = strdup(content);
@@ -1197,14 +1253,13 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list)
*semicolon = '\0';
}
if (!strcasecmp((const char*)anode->name, "hash"))
{
de->md5sum = strdup(content);
}
if (!strcasecmp((const char*)anode->name, "last_modified"))
{
time_t last_modified_t = get_time_from_str_as_gmt(content);
char local_time_str[64];
time_t local_time_t = get_time_as_local(last_modified_t, local_time_str, sizeof(local_time_str));
time_t local_time_t = get_time_as_local(last_modified_t, local_time_str,
sizeof(local_time_str));
de->last_modified = local_time_t;
de->ctime.tv_sec = local_time_t;
de->ctime.tv_nsec = 0;
@@ -1228,7 +1283,9 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list)
{
//todo: check why is needed and if memory is freed properly, seems to generate many missed delete operations
//cloudfs_free_dir_list(de);
debugf(DBG_LEVEL_EXT, "cloudfs_list_directory: "KYEL"ignore "KNRM"cloudfs_free_dir_list(%s) command", de->name);
debugf(DBG_LEVEL_EXT,
"cloudfs_list_directory: "KYEL"ignore "KNRM"cloudfs_free_dir_list(%s) command",
de->name);
continue;
}
strncpy(last_subdir, de->name, sizeof(last_subdir));
@@ -1240,10 +1297,9 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list)
debugf(DBG_LEVEL_EXT, KCYN"new dir_entry %s size=%d %s dir=%d lnk=%d mod=[%s]",
de->full_name, de->size, de->content_type, de->isdir, de->islink, time_str);
}
else {
else
debugf(DBG_LEVEL_EXT, "unknown element: %s", onode->name);
}
}
retval = 1;
}
else if ((!strcmp(path, "") || !strcmp(path, "/")) && *override_storage_url)
@@ -1296,7 +1352,8 @@ int cloudfs_delete_object(const char *path)
response = send_request("DELETE", encoded, NULL, NULL, NULL, NULL, seg_path);
if (response < 200 || response >= 300)
{
debugf(DBG_LEVEL_EXT, "exit 1: cloudfs_delete_object(%s) response=%d", path, response);
debugf(DBG_LEVEL_EXT, "exit 1: cloudfs_delete_object(%s) response=%d", path,
response);
return 0;
}
}
@@ -1306,7 +1363,8 @@ int cloudfs_delete_object(const char *path)
int response = send_request("DELETE", encoded, NULL, NULL, NULL, NULL, path);
curl_free(encoded);
int ret = (response >= 200 && response < 300);
debugf(DBG_LEVEL_EXT, "status: cloudfs_delete_object(%s) response=%d", path, response);
debugf(DBG_LEVEL_EXT, "status: cloudfs_delete_object(%s) response=%d", path,
response);
if (response == 409)
{
debugf(DBG_LEVEL_EXT, "status: cloudfs_delete_object(%s) NOT EMPTY", path);
@@ -1320,14 +1378,17 @@ int cloudfs_delete_object(const char *path)
// this operation also causes an HTTP 400 error if X-Object-Meta-FilePath value is larger than 256 chars
int cloudfs_copy_object(const char* src, const char* dst)
{
debugf(DBG_LEVEL_EXT, "cloudfs_copy_object(%s, %s) lensrc=%d, lendst=%d", src, dst, strlen(src), strlen(dst));
debugf(DBG_LEVEL_EXT, "cloudfs_copy_object(%s, %s) lensrc=%d, lendst=%d", src,
dst, strlen(src), strlen(dst));
char* dst_encoded = curl_escape(dst, 0);
char* src_encoded = curl_escape(src, 0);
//convert encoded string (slashes are encoded as well) to encoded string with slashes
char* slash;
while ((slash = strstr(src_encoded, "%2F")) || (slash = strstr(src_encoded, "%2f"))) {
while ((slash = strstr(src_encoded, "%2F"))
|| (slash = strstr(src_encoded, "%2f")))
{
*slash = '/';
memmove(slash + 1, slash + 3, strlen(slash + 3) + 1);
}
@@ -1337,18 +1398,20 @@ int cloudfs_copy_object(const char *src, const char *dst)
add_header(&headers, "Content-Length", "0");
//get source file entry
dir_entry* de_src = check_path_info(src);
if (de_src) {
debugf(DBG_LEVEL_EXT, "status cloudfs_copy_object(%s, %s): src file found", src, dst);
}
else {
debugf(DBG_LEVEL_NORM, KRED"status cloudfs_copy_object(%s, %s): src file NOT found", src, dst);
}
if (de_src)
debugf(DBG_LEVEL_EXT, "status cloudfs_copy_object(%s, %s): src file found",
src, dst);
else
debugf(DBG_LEVEL_NORM,
KRED"status cloudfs_copy_object(%s, %s): src file NOT found", src, dst);
//pass src metadata so that PUT will set time attributes of the src file
int response = send_request("PUT", dst_encoded, NULL, NULL, headers, de_src, dst);
int response = send_request("PUT", dst_encoded, NULL, NULL, headers, de_src,
dst);
curl_free(dst_encoded);
curl_free(src_encoded);
curl_slist_free_all(headers);
debugf(DBG_LEVEL_EXT, "exit: cloudfs_copy_object(%s,%s) response=%d", src, dst, response);
debugf(DBG_LEVEL_EXT, "exit: cloudfs_copy_object(%s,%s) response=%d", src, dst,
response);
return (response >= 200 && response < 300);
}
@@ -1369,12 +1432,16 @@ int cloudfs_statfs(const char *path, struct statvfs *stat)
//todo: check why stat head request is always set to /, why not path?
int response = send_request("HEAD", "/", NULL, NULL, NULL, NULL, "/");
*stat = statcache;
debugf(DBG_LEVEL_EXT, "exit: cloudfs_statfs (new recent values, was cached since %d seconds)", lapsed);
debugf(DBG_LEVEL_EXT,
"exit: cloudfs_statfs (new recent values, was cached since %d seconds)",
lapsed);
last_stat_read_time = now;
return (response >= 200 && response < 300);
}
else {
debugf(DBG_LEVEL_EXT, "exit: cloudfs_statfs (old values, cached since %d seconds)", lapsed);
else
{
debugf(DBG_LEVEL_EXT,
"exit: cloudfs_statfs (old values, cached since %d seconds)", lapsed);
return 1;
}
}
@@ -1397,7 +1464,8 @@ int cloudfs_create_directory(const char *path)
char* encoded = curl_escape(path, 0);
int response = send_request("MKDIR", encoded, NULL, NULL, NULL, NULL, path);
curl_free(encoded);
debugf(DBG_LEVEL_EXT, "cloudfs_create_directory(%s) response=%d", path, response);
debugf(DBG_LEVEL_EXT, "cloudfs_create_directory(%s) response=%d", path,
response);
return (response >= 200 && response < 300);
}
@@ -1423,30 +1491,38 @@ void cloudfs_option_curl_verbose(int option)
option_curl_verbose = option ? true : false;
}
static struct {
static struct
{
char client_id [MAX_HEADER_SIZE];
char client_secret[MAX_HEADER_SIZE];
char refresh_token[MAX_HEADER_SIZE];
} reconnect_args;
void cloudfs_set_credentials(char *client_id, char *client_secret, char *refresh_token)
void cloudfs_set_credentials(char* client_id, char* client_secret,
char* refresh_token)
{
strncpy(reconnect_args.client_id , client_id , sizeof(reconnect_args.client_id ));
strncpy(reconnect_args.client_secret, client_secret, sizeof(reconnect_args.client_secret));
strncpy(reconnect_args.refresh_token, refresh_token, sizeof(reconnect_args.refresh_token));
strncpy(reconnect_args.client_id , client_id ,
sizeof(reconnect_args.client_id ));
strncpy(reconnect_args.client_secret, client_secret,
sizeof(reconnect_args.client_secret));
strncpy(reconnect_args.refresh_token, refresh_token,
sizeof(reconnect_args.refresh_token));
}
struct htmlString {
struct htmlString
{
char* text;
size_t size;
};
static size_t writefunc_string(void *contents, size_t size, size_t nmemb, void *data)
static size_t writefunc_string(void* contents, size_t size, size_t nmemb,
void* data)
{
struct htmlString* mem = (struct htmlString*) data;
size_t realsize = size * nmemb;
mem->text = realloc(mem->text, mem->size + realsize + 1);
if (mem->text == NULL) { /* out of memory! */
if (mem->text == NULL) /* out of memory! */
{
perror(__FILE__);
exit(EXIT_FAILURE);
}
@@ -1464,9 +1540,11 @@ char* htmlStringGet(CURL *curl)
chunk.text[0] = '\0';//added to avoid valgrind unitialised warning
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &chunk);
do {
do
{
curl_easy_perform(curl);
} while (chunk.size == 0);
}
while (chunk.size == 0);
chunk.text[chunk.size] = '\0';
return chunk.text;
@@ -1529,7 +1607,8 @@ int cloudfs_connect()
struct json_object* json_obj;
pthread_mutex_lock(&pool_mut);
debugf(DBG_LEVEL_NORM, "Authenticating... (client_id = '%s')", HUBIC_CLIENT_ID);
debugf(DBG_LEVEL_NORM, "Authenticating... (client_id = '%s')",
HUBIC_CLIENT_ID);
storage_token[0] = storage_url[0] = '\0';
CURL* curl = curl_easy_init();
@@ -1551,7 +1630,8 @@ int cloudfs_connect()
/* Step 2 : get request code - Not needed anymore with refresh_token */
/* Step 3 : get access token */
sprintf(payload, "refresh_token=%s&grant_type=refresh_token", HUBIC_REFRESH_TOKEN);
sprintf(payload, "refresh_token=%s&grant_type=refresh_token",
HUBIC_REFRESH_TOKEN);
curl_easy_setopt(curl, CURLOPT_URL, HUBIC_TOKEN_URL);
curl_easy_setopt(curl, CURLOPT_POST, 1L);
curl_easy_setopt(curl, CURLOPT_HEADER, 0);
@@ -1616,5 +1696,6 @@ int cloudfs_connect()
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
curl_easy_cleanup(curl);
pthread_mutex_unlock(&pool_mut);
return (response >= 200 && response < 300 && storage_token[0] && storage_url[0]);
return (response >= 200 && response < 300 && storage_token[0]
&& storage_url[0]);
}
cloudfsapi.h
+8 -4
View File
@@ -16,12 +16,14 @@
typedef struct curl_slist curl_slist;
#define MINIMAL_PROGRESS_FUNCTIONALITY_INTERVAL 5
struct curl_progress {
struct curl_progress
{
double lastruntime;
CURL* curl;
};
typedef struct options {
typedef struct options
{
char cache_timeout[OPTION_SIZE];
char verify_ssl[OPTION_SIZE];
char segment_size[OPTION_SIZE];
@@ -34,7 +36,8 @@ typedef struct options {
char refresh_token[OPTION_SIZE];
} FuseOptions;
typedef struct extra_options {
typedef struct extra_options
{
char get_extended_metadata[OPTION_SIZE];
char curl_verbose[OPTION_SIZE];
char cache_statfs_timeout[OPTION_SIZE];
@@ -46,7 +49,8 @@ typedef struct extra_options {
void cloudfs_init(void);
void cloudfs_free(void);
void cloudfs_set_credentials(char *client_id, char *client_secret, char *refresh_token);
void cloudfs_set_credentials(char* client_id, char* client_secret,
char* refresh_token);
int cloudfs_connect(void);
struct segment_info
cloudfuse.c
+223 -132
View File
@@ -56,21 +56,24 @@ static int cfs_getattr(const char *path, struct stat *stbuf)
}
//get file. if not in cache will be downloaded.
dir_entry* de = path_info(path);
if (!de) {
if (!de)
{
debug_list_cache_content();
debugf(DBG_LEVEL_NORM, KBLU"exit 1: cfs_getattr(%s) "KYEL"not-in-cache/cloud", path);
debugf(DBG_LEVEL_NORM, KBLU"exit 1: cfs_getattr(%s) "KYEL"not-in-cache/cloud",
path);
return -ENOENT;
}
//lazzy download of file metadata, only when really needed
if (option_get_extended_metadata && !de->metadata_downloaded) {
if (option_get_extended_metadata && !de->metadata_downloaded)
get_file_metadata(de);
}
if (option_enable_chown) {
if (option_enable_chown)
{
stbuf->st_uid = de->uid;
stbuf->st_gid = de->gid;
}
else {
else
{
stbuf->st_uid = geteuid();
stbuf->st_gid = getegid();
}
@@ -91,21 +94,25 @@ static int cfs_getattr(const char *path, struct stat *stbuf)
int default_mode_dir, default_mode_file;
if (option_enable_chmod) {
if (option_enable_chmod)
{
default_mode_dir = de->chmod;
default_mode_file = de->chmod;
}
else {
else
{
default_mode_dir = 0755;
default_mode_file = 0666;
}
if (de->isdir) {
if (de->isdir)
{
stbuf->st_size = 0;
stbuf->st_mode = S_IFDIR | default_mode_dir;
stbuf->st_nlink = 2;
}
else if (de->islink) {
else if (de->islink)
{
stbuf->st_size = 1;
stbuf->st_mode = S_IFLNK | default_mode_dir;
stbuf->st_nlink = 1;
@@ -113,7 +120,8 @@ static int cfs_getattr(const char *path, struct stat *stbuf)
/* calc. blocks as if 4K blocksize filesystem; stat uses units of 512B */
stbuf->st_blocks = ((4095 + de->size) / 4096) * 8;
}
else {
else
{
stbuf->st_size = de->size;
/* calc. blocks as if 4K blocksize filesystem; stat uses units of 512B */
stbuf->st_blocks = ((4095 + de->size) / 4096) * 8;
@@ -124,7 +132,8 @@ static int cfs_getattr(const char *path, struct stat *stbuf)
return 0;
}
static int cfs_fgetattr(const char *path, struct stat *stbuf, struct fuse_file_info *info)
static int cfs_fgetattr(const char* path, struct stat* stbuf,
struct fuse_file_info* info)
{
debugf(DBG_LEVEL_NORM, KBLU "cfs_fgetattr(%s)", path);
openfile* of = (openfile*)(uintptr_t)info->fh;
@@ -132,18 +141,18 @@ static int cfs_fgetattr(const char *path, struct stat *stbuf, struct fuse_file_i
{
//get file. if not in cache will be downloaded.
dir_entry* de = path_info(path);
if (!de) {
if (!de)
{
debug_list_cache_content();
debugf(DBG_LEVEL_NORM, KBLU"exit 1: cfs_fgetattr(%s) "KYEL"not-in-cache/cloud", path);
debugf(DBG_LEVEL_NORM, KBLU"exit 1: cfs_fgetattr(%s) "KYEL"not-in-cache/cloud",
path);
return -ENOENT;
}
int default_mode_file;
if (option_enable_chmod) {
if (option_enable_chmod)
default_mode_file = de->chmod;
}
else {
else
default_mode_file = 0666;
}
stbuf->st_size = cloudfs_file_size(of->fd);
stbuf->st_mode = S_IFREG | default_mode_file;
@@ -155,11 +164,13 @@ static int cfs_fgetattr(const char *path, struct stat *stbuf, struct fuse_file_i
return -ENOENT;
}
static int cfs_readdir(const char *path, void *buf, fuse_fill_dir_t filldir, off_t offset, struct fuse_file_info *info)
static int cfs_readdir(const char* path, void* buf, fuse_fill_dir_t filldir,
off_t offset, struct fuse_file_info* info)
{
debugf(DBG_LEVEL_NORM, KBLU "cfs_readdir(%s)", path);
dir_entry* de;
if (!caching_list_directory(path, &de)) {
if (!caching_list_directory(path, &de))
{
debug_list_cache_content();
debugf(DBG_LEVEL_NORM, KRED "exit 0: cfs_readdir(%s)", path);
return -ENOLINK;
@@ -177,37 +188,48 @@ static int cfs_mkdir(const char *path, mode_t mode)
{
debugf(DBG_LEVEL_NORM, KBLU "cfs_mkdir(%s)", path);
int response = cloudfs_create_directory(path);
if (response){
if (response)
{
update_dir_cache(path, 0, 1, 0);
debug_list_cache_content();
debugf(DBG_LEVEL_NORM, KBLU "exit 0: cfs_mkdir(%s)", path);
return 0;
}
debugf(DBG_LEVEL_NORM, KRED "exit 1: cfs_mkdir(%s) response=%d", path, response);
debugf(DBG_LEVEL_NORM, KRED "exit 1: cfs_mkdir(%s) response=%d", path,
response);
return -ENOENT;
}
static int cfs_create(const char *path, mode_t mode, struct fuse_file_info *info)
static int cfs_create(const char* path, mode_t mode,
struct fuse_file_info* info)
{
debugf(DBG_LEVEL_NORM, KBLU "cfs_create(%s)", path);
FILE* temp_file;
int errsv;
char file_path_safe[NAME_MAX] = "";
if (*temp_dir) {
if (*temp_dir)
{
get_safe_cache_file_path(path, file_path_safe, temp_dir);
temp_file = fopen(file_path_safe, "w+b");
errsv = errno;
if (temp_file == NULL){
debugf(DBG_LEVEL_NORM, KRED "exit 0: cfs_create cannot open temp file %s.error %s\n", file_path_safe, strerror(errsv));
if (temp_file == NULL)
{
debugf(DBG_LEVEL_NORM, KRED
"exit 0: cfs_create cannot open temp file %s.error %s\n", file_path_safe,
strerror(errsv));
return -EIO;
}
}
else {
else
{
temp_file = tmpfile();
errsv = errno;
if (temp_file == NULL){
debugf(DBG_LEVEL_NORM, KRED "exit 1: cfs_create cannot open tmp file for path %s.error %s\n", path, strerror(errsv));
if (temp_file == NULL)
{
debugf(DBG_LEVEL_NORM, KRED
"exit 1: cfs_create cannot open tmp file for path %s.error %s\n", path,
strerror(errsv));
return -EIO;
}
}
@@ -219,7 +241,8 @@ static int cfs_create(const char *path, mode_t mode, struct fuse_file_info *info
update_dir_cache(path, 0, 0, 0);
info->direct_io = 1;
dir_entry* de = check_path_info(path);
if (de) {
if (de)
{
debugf(DBG_LEVEL_EXT, KCYN"cfs_create(%s): found in cache", path);
struct timespec now;
clock_gettime(CLOCK_REALTIME, &now);
@@ -244,10 +267,10 @@ static int cfs_create(const char *path, mode_t mode, struct fuse_file_info *info
de->uid = geteuid();
de->gid = getegid();
}
else {
else
debugf(DBG_LEVEL_EXT, KBLU "cfs_create(%s) "KYEL"dir-entry not found", path);
}
debugf(DBG_LEVEL_NORM, KBLU "exit 2: cfs_create(%s)=(%s) result=%d:%s", path, file_path_safe, errsv, strerror(errsv));
debugf(DBG_LEVEL_NORM, KBLU "exit 2: cfs_create(%s)=(%s) result=%d:%s", path,
file_path_safe, errsv, strerror(errsv));
return 0;
}
@@ -260,23 +283,29 @@ static int cfs_open(const char *path, struct fuse_file_info *info)
int errsv;
dir_entry* de = path_info(path);
if (*temp_dir) {
if (*temp_dir)
{
char file_path_safe[NAME_MAX];
get_safe_cache_file_path(path, file_path_safe, temp_dir);
debugf(DBG_LEVEL_EXT, "cfs_open: try open (%s)", file_path_safe);
if (access(file_path_safe, F_OK) != -1){
if (access(file_path_safe, F_OK) != -1)
{
// file exists
temp_file = fopen(file_path_safe, "r");
errsv = errno;
if (temp_file == NULL) {
debugf(DBG_LEVEL_NORM, KRED"exit 0: cfs_open can't open temp_file=[%s] err=%d:%s", file_path_safe, errsv, strerror(errsv));
if (temp_file == NULL)
{
debugf(DBG_LEVEL_NORM,
KRED"exit 0: cfs_open can't open temp_file=[%s] err=%d:%s", file_path_safe,
errsv, strerror(errsv));
return -ENOENT;
}
else
debugf(DBG_LEVEL_EXT, "cfs_open: file exists");
}
else {
else
{
errsv = errno;
debugf(DBG_LEVEL_EXT, "cfs_open: file not in cache, err=%s", strerror(errsv));
//FIXME: commented out as this condition will not be meet in some odd cases and program will crash
@@ -303,14 +332,19 @@ static int cfs_open(const char *path, struct fuse_file_info *info)
// TODO: just to prevent this craziness for now
temp_file = fopen(file_path_safe, "w+b");
errsv = errno;
if (temp_file == NULL) {
debugf(DBG_LEVEL_NORM, KRED"exit 1: cfs_open cannot open temp_file=[%s] err=%d:%s", file_path_safe, errsv, strerror(errsv));
if (temp_file == NULL)
{
debugf(DBG_LEVEL_NORM,
KRED"exit 1: cfs_open cannot open temp_file=[%s] err=%d:%s", file_path_safe,
errsv, strerror(errsv));
return -ENOENT;
}
if (!cloudfs_object_write_fp(path, temp_file)) {
if (!cloudfs_object_write_fp(path, temp_file))
{
fclose(temp_file);
debugf(DBG_LEVEL_NORM, KRED "exit 2: cfs_open(%s) cannot download/write", path);
debugf(DBG_LEVEL_NORM, KRED "exit 2: cfs_open(%s) cannot download/write",
path);
return -ENOENT;
}
}
@@ -318,13 +352,17 @@ static int cfs_open(const char *path, struct fuse_file_info *info)
else
{
temp_file = tmpfile();
if (temp_file == NULL) {
debugf(DBG_LEVEL_NORM, KRED"exit 3: cfs_open cannot create temp_file err=%s", strerror(errno));
if (temp_file == NULL)
{
debugf(DBG_LEVEL_NORM, KRED"exit 3: cfs_open cannot create temp_file err=%s",
strerror(errno));
return -ENOENT;
}
if (!(info->flags & O_TRUNC)) {
if (!cloudfs_object_write_fp(path, temp_file) && !(info->flags & O_CREAT)) {
if (!(info->flags & O_TRUNC))
{
if (!cloudfs_object_write_fp(path, temp_file) && !(info->flags & O_CREAT))
{
fclose(temp_file);
debugf(DBG_LEVEL_NORM, KRED"exit 4: cfs_open(%s) cannot download/write", path);
return -ENOENT;
@@ -353,13 +391,16 @@ static int cfs_open(const char *path, struct fuse_file_info *info)
return 0;
}
static int cfs_read(const char *path, char *buf, size_t size, off_t offset, struct fuse_file_info *info)
static int cfs_read(const char* path, char* buf, size_t size, off_t offset,
struct fuse_file_info* info)
{
debugf(DBG_LEVEL_EXTALL, KBLU "cfs_read(%s) buffsize=%lu offset=%lu", path, size, offset);
debugf(DBG_LEVEL_EXTALL, KBLU "cfs_read(%s) buffsize=%lu offset=%lu", path,
size, offset);
file_buffer_size = size;
debug_print_descriptor(info);
int result = pread(((openfile*)(uintptr_t)info->fh)->fd, buf, size, offset);
debugf(DBG_LEVEL_EXTALL, KBLU "exit: cfs_read(%s) result=%s", path, strerror(errno));
debugf(DBG_LEVEL_EXTALL, KBLU "exit: cfs_read(%s) result=%s", path,
strerror(errno));
return result;
}
@@ -372,9 +413,11 @@ static int cfs_flush(const char *path, struct fuse_file_info *info)
openfile* of = (openfile*)(uintptr_t)info->fh;
int errsv = 0;
if (of) {
if (of)
{
update_dir_cache(path, cloudfs_file_size(of->fd), 0, 0);
if (of->flags & O_RDWR || of->flags & O_WRONLY) {
if (of->flags & O_RDWR || of->flags & O_WRONLY)
{
FILE* fp = fdopen(dup(of->fd), "r");
errsv = errno;
if (fp != NULL)
@@ -384,34 +427,43 @@ static int cfs_flush(const char *path, struct fuse_file_info *info)
char md5_file_hash_str[MD5_DIGEST_LENGTH + 1] = "\0";
file_md5(fp, md5_file_hash_str);
dir_entry* de = check_path_info(path);
if (de && de->md5sum != NULL && (!strcasecmp(md5_file_hash_str, de->md5sum))) {
if (de && de->md5sum != NULL && (!strcasecmp(md5_file_hash_str, de->md5sum)))
{
//file content is identical, no need to upload entire file, just update metadata
debugf(DBG_LEVEL_NORM, KBLU "cfs_flush(%s): skip full upload as content did not change", path);
debugf(DBG_LEVEL_NORM, KBLU
"cfs_flush(%s): skip full upload as content did not change", path);
cloudfs_update_meta(de);
}
else {
else
{
rewind(fp);
debugf(DBG_LEVEL_NORM, KBLU "cfs_flush(%s): perform full upload as content changed (or no file found in cache)", path);
if (!cloudfs_object_read_fp(path, fp)) {
debugf(DBG_LEVEL_NORM, KBLU
"cfs_flush(%s): perform full upload as content changed (or no file found in cache)",
path);
if (!cloudfs_object_read_fp(path, fp))
{
fclose(fp);
errsv = errno;
debugf(DBG_LEVEL_NORM, KRED"exit 0: cfs_flush(%s) result=%d:%s", path, errsv, strerror(errno));
debugf(DBG_LEVEL_NORM, KRED"exit 0: cfs_flush(%s) result=%d:%s", path, errsv,
strerror(errno));
return -ENOENT;
}
}
fclose(fp);
errsv = errno;
}
else {
debugf(DBG_LEVEL_EXT, KRED "status: cfs_flush, err=%d:%s", errsv, strerror(errno));
else
debugf(DBG_LEVEL_EXT, KRED "status: cfs_flush, err=%d:%s", errsv,
strerror(errno));
}
}
}
debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_flush(%s) result=%d:%s", path, errsv, strerror(errno));
debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_flush(%s) result=%d:%s", path, errsv,
strerror(errno));
return 0;
}
static int cfs_release(const char *path, struct fuse_file_info *info) {
static int cfs_release(const char* path, struct fuse_file_info* info)
{
debugf(DBG_LEVEL_NORM, KBLU "cfs_release(%s)", path);
close(((openfile*)(uintptr_t)info->fh)->fd);
debugf(DBG_LEVEL_NORM, KBLU "exit: cfs_release(%s)", path);
@@ -422,11 +474,13 @@ static int cfs_rmdir(const char *path)
{
debugf(DBG_LEVEL_NORM, KBLU "cfs_rmdir(%s)", path);
int success = cloudfs_delete_object(path);
if (success == -1) {
if (success == -1)
{
debugf(DBG_LEVEL_NORM, KBLU "exit 0: cfs_rmdir(%s)", path);
return -ENOTEMPTY;
}
if (success){
if (success)
{
dir_decache(path);
debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_rmdir(%s)", path);
return 0;
@@ -435,7 +489,8 @@ static int cfs_rmdir(const char *path)
return -ENOENT;
}
static int cfs_ftruncate(const char *path, off_t size, struct fuse_file_info *info)
static int cfs_ftruncate(const char* path, off_t size,
struct fuse_file_info* info)
{
debugf(DBG_LEVEL_NORM, KBLU "cfs_ftruncate(%s)", path);
file_buffer_size = size;
@@ -448,20 +503,22 @@ static int cfs_ftruncate(const char *path, off_t size, struct fuse_file_info *in
return 0;
}
static int cfs_write(const char *path, const char *buf, size_t length, off_t offset, struct fuse_file_info *info)
static int cfs_write(const char* path, const char* buf, size_t length,
off_t offset, struct fuse_file_info* info)
{
debugf(DBG_LEVEL_EXTALL, KBLU "cfs_write(%s) bufflength=%lu offset=%lu", path, length, offset);
debugf(DBG_LEVEL_EXTALL, KBLU "cfs_write(%s) bufflength=%lu offset=%lu", path,
length, offset);
// FIXME: Potential inconsistent cache update if pwrite fails?
update_dir_cache(path, offset + length, 0, 0);
//int result = pwrite(info->fh, buf, length, offset);
int result = pwrite(((openfile*)(uintptr_t)info->fh)->fd, buf, length, offset);
int errsv = errno;
if (errsv == 0) {
debugf(DBG_LEVEL_EXTALL, KBLU "exit 0: cfs_write(%s) result=%d:%s", path, errsv, strerror(errsv));
}
else {
debugf(DBG_LEVEL_EXTALL, KBLU "exit 1: cfs_write(%s) "KRED"result=%d:%s", path, errsv, strerror(errsv));
}
if (errsv == 0)
debugf(DBG_LEVEL_EXTALL, KBLU "exit 0: cfs_write(%s) result=%d:%s", path,
errsv, strerror(errsv));
else
debugf(DBG_LEVEL_EXTALL, KBLU "exit 1: cfs_write(%s) "KRED"result=%d:%s", path,
errsv, strerror(errsv));
return result;
}
@@ -469,7 +526,8 @@ static int cfs_unlink(const char *path)
{
debugf(DBG_LEVEL_NORM, KBLU "cfs_unlink(%s)", path);
int success = cloudfs_delete_object(path);
if (success == -1) {
if (success == -1)
{
debugf(DBG_LEVEL_NORM, KRED "exit 0: cfs_unlink(%s)", path);
return -EACCES;
}
@@ -501,11 +559,13 @@ static int cfs_truncate(const char *path, off_t size)
static int cfs_statfs(const char* path, struct statvfs* stat)
{
debugf(DBG_LEVEL_NORM, KBLU "cfs_statfs(%s)", path);
if (cloudfs_statfs(path, stat)){
if (cloudfs_statfs(path, stat))
{
debugf(DBG_LEVEL_NORM, KBLU "exit 0: cfs_statfs(%s)", path);
return 0;
}
else {
else
{
debugf(DBG_LEVEL_NORM, KRED"exit 1: cfs_statfs(%s) not-found", path);
return -EIO;
}
@@ -515,9 +575,12 @@ static int cfs_chown(const char *path, uid_t uid, gid_t gid)
{
debugf(DBG_LEVEL_NORM, KBLU "cfs_chown(%s,%d,%d)", path, uid, gid);
dir_entry* de = check_path_info(path);
if (de) {
if (de->uid != uid || de->gid != gid) {
debugf(DBG_LEVEL_NORM, "cfs_chown(%s): change from uid:gid %d:%d to %d:%d", path, de->uid, de->gid, uid, gid);
if (de)
{
if (de->uid != uid || de->gid != gid)
{
debugf(DBG_LEVEL_NORM, "cfs_chown(%s): change from uid:gid %d:%d to %d:%d",
path, de->uid, de->gid, uid, gid);
de->uid = uid;
de->gid = gid;
//issue a PUT request to update metadata (quick request just to update headers)
@@ -531,9 +594,12 @@ static int cfs_chmod(const char *path, mode_t mode)
{
debugf(DBG_LEVEL_NORM, KBLU"cfs_chmod(%s,%d)", path, mode);
dir_entry* de = check_path_info(path);
if (de) {
if (de->chmod != mode) {
debugf(DBG_LEVEL_NORM, "cfs_chmod(%s): change mode from %d to %d", path, de->chmod, mode);
if (de)
{
if (de->chmod != mode)
{
debugf(DBG_LEVEL_NORM, "cfs_chmod(%s): change mode from %d to %d", path,
de->chmod, mode);
de->chmod = mode;
//todo: issue a PUT request to update metadata (empty request just to update headers?)
int response = cloudfs_update_meta(de);
@@ -546,25 +612,30 @@ static int cfs_rename(const char *src, const char *dst)
{
debugf(DBG_LEVEL_NORM, KBLU"cfs_rename(%s, %s)", src, dst);
dir_entry* src_de = path_info(src);
if (!src_de) {
if (!src_de)
{
debugf(DBG_LEVEL_NORM, KRED"exit 0: cfs_rename(%s,%s) not-found", src, dst);
return -ENOENT;
}
if (src_de->isdir) {
debugf(DBG_LEVEL_NORM, KRED"exit 1: cfs_rename(%s,%s) cannot rename dirs!", src, dst);
if (src_de->isdir)
{
debugf(DBG_LEVEL_NORM, KRED"exit 1: cfs_rename(%s,%s) cannot rename dirs!",
src, dst);
return -EISDIR;
}
if (cloudfs_copy_object(src, dst)) {
if (cloudfs_copy_object(src, dst))
{
/* FIXME this isn't quite right as doesn't preserve last modified */
//fix done in cloudfs_copy_object()
update_dir_cache(dst, src_de->size, 0, 0);
int result = cfs_unlink(src);
dir_entry* dst_de = path_info(dst);
if (!dst_de) {
debugf(DBG_LEVEL_NORM, KRED"cfs_rename(%s,%s) dest-not-found-in-cache", src, dst);
}
else {
if (!dst_de)
debugf(DBG_LEVEL_NORM, KRED"cfs_rename(%s,%s) dest-not-found-in-cache", src,
dst);
else
{
debugf(DBG_LEVEL_NORM, KBLU"cfs_rename(%s,%s) upload ok", src, dst);
//copy attributes, shortcut, rather than forcing a download from cloud
copy_dir_entry(src_de, dst_de);
@@ -597,12 +668,14 @@ static int cfs_readlink(const char* path, char* buf, size_t size)
FILE* temp_file = tmpfile();
int ret = 0;
if (!cloudfs_object_write_fp(path, temp_file)) {
if (!cloudfs_object_write_fp(path, temp_file))
{
debugf(DBG_LEVEL_NORM, KRED"exit 1: cfs_readlink(%s) not found", path);
ret = -ENOENT;
}
if (!pread(fileno(temp_file), buf, size, 0)) {
if (!pread(fileno(temp_file), buf, size, 0))
{
debugf(DBG_LEVEL_NORM, KRED"exit 2: cfs_readlink(%s) not found", path);
ret = -ENOENT;
}
@@ -631,11 +704,16 @@ static int cfs_utimens(const char *path, const struct timespec times[2])
struct timespec now;
clock_gettime(CLOCK_REALTIME, &now);
if (path_de->atime.tv_sec != times[0].tv_sec || path_de->atime.tv_nsec != times[0].tv_nsec ||
path_de->mtime.tv_sec != times[1].tv_sec || path_de->mtime.tv_nsec != times[1].tv_nsec)
if (path_de->atime.tv_sec != times[0].tv_sec
|| path_de->atime.tv_nsec != times[0].tv_nsec ||
path_de->mtime.tv_sec != times[1].tv_sec
|| path_de->mtime.tv_nsec != times[1].tv_nsec)
{
debugf(DBG_LEVEL_EXT, KCYN "cfs_utimens: change for %s, prev: atime=%li.%li mtime=%li.%li, new: atime=%li.%li mtime=%li.%li", path,
path_de->atime.tv_sec, path_de->atime.tv_nsec, path_de->mtime.tv_sec, path_de->mtime.tv_nsec,
debugf(DBG_LEVEL_EXT, KCYN
"cfs_utimens: change for %s, prev: atime=%li.%li mtime=%li.%li, new: atime=%li.%li mtime=%li.%li",
path,
path_de->atime.tv_sec, path_de->atime.tv_nsec, path_de->mtime.tv_sec,
path_de->mtime.tv_nsec,
times[0].tv_sec, times[0].tv_nsec, times[1].tv_sec, times[1].tv_nsec);
char time_str[TIME_CHARS] = "";
get_timespec_as_str(&times[1], time_str, sizeof(time_str));
@@ -651,15 +729,15 @@ static int cfs_utimens(const char *path, const struct timespec times[2])
//worth implementing a meta cache functionality to avoid multiple uploads on meta changes
//when changing timestamps on very large files, touch command will trigger 2 x long and useless file uploads on cfs_flush()
}
else {
else
debugf(DBG_LEVEL_EXT, KCYN"cfs_utimens: a/m/time not changed");
}
debugf(DBG_LEVEL_NORM, KBLU "exit 1: cfs_utimens(%s)", path);
return 0;
}
int cfs_setxattr(const char *path, const char *name, const char *value, size_t size, int flags)
int cfs_setxattr(const char* path, const char* name, const char* value,
size_t size, int flags)
{
return 0;
}
@@ -679,7 +757,8 @@ int cfs_listxattr(const char *path, char *list, size_t size)
return 0;
}
FuseOptions options = {
FuseOptions options =
{
.cache_timeout = "600",
.verify_ssl = "true",
.segment_size = "1073741824",
@@ -693,7 +772,8 @@ FuseOptions options = {
.refresh_token = ""
};
ExtraFuseOptions extra_options = {
ExtraFuseOptions extra_options =
{
.get_extended_metadata = "false",
.curl_verbose = "false",
.cache_statfs_timeout = 0,
@@ -703,7 +783,8 @@ ExtraFuseOptions extra_options = {
.enable_chmod = "false"
};
int parse_option(void *data, const char *arg, int key, struct fuse_args *outargs)
int parse_option(void* data, const char* arg, int key,
struct fuse_args* outargs)
{
if (sscanf(arg, " cache_timeout = %[^\r\n ]", options.cache_timeout) ||
sscanf(arg, " verify_ssl = %[^\r\n ]", options.verify_ssl) ||
@@ -716,11 +797,14 @@ int parse_option(void *data, const char *arg, int key, struct fuse_args *outargs
sscanf(arg, " client_secret = %[^\r\n ]", options.client_secret) ||
sscanf(arg, " refresh_token = %[^\r\n ]", options.refresh_token) ||
sscanf(arg, " get_extended_metadata = %[^\r\n ]", extra_options.get_extended_metadata) ||
sscanf(arg, " get_extended_metadata = %[^\r\n ]",
extra_options.get_extended_metadata) ||
sscanf(arg, " curl_verbose = %[^\r\n ]", extra_options.curl_verbose) ||
sscanf(arg, " cache_statfs_timeout = %[^\r\n ]", extra_options.cache_statfs_timeout) ||
sscanf(arg, " cache_statfs_timeout = %[^\r\n ]",
extra_options.cache_statfs_timeout) ||
sscanf(arg, " debug_level = %[^\r\n ]", extra_options.debug_level) ||
sscanf(arg, " curl_progress_state = %[^\r\n ]", extra_options.curl_progress_state) ||
sscanf(arg, " curl_progress_state = %[^\r\n ]",
extra_options.curl_progress_state) ||
sscanf(arg, " enable_chmod = %[^\r\n ]", extra_options.enable_chmod) ||
sscanf(arg, " enable_chown = %[^\r\n ]", extra_options.enable_chown)
)
@@ -731,7 +815,8 @@ int parse_option(void *data, const char *arg, int key, struct fuse_args *outargs
}
//allows memory leaks inspections
void interrupt_handler(int sig) {
void interrupt_handler(int sig)
{
debugf(DBG_LEVEL_NORM, "Got interrupt signal %d, cleaning memory", sig);
//TODO: clean memory allocations
//http://www.cprogramming.com/debugging/valgrind.html
@@ -745,30 +830,22 @@ void initialise_options()
{
//todo: handle param init consistently, quite heavy implementation
cloudfs_verify_ssl(!strcasecmp(options.verify_ssl, "true"));
cloudfs_option_get_extended_metadata(!strcasecmp(extra_options.get_extended_metadata, "true"));
cloudfs_option_get_extended_metadata(!strcasecmp(
extra_options.get_extended_metadata, "true"));
cloudfs_option_curl_verbose(!strcasecmp(extra_options.curl_verbose, "true"));
//lean way to init params, to be used as reference
if (*extra_options.debug_level)
{
option_debug_level = atoi(extra_options.debug_level);
}
if (*extra_options.cache_statfs_timeout)
{
option_cache_statfs_timeout = atoi(extra_options.cache_statfs_timeout);
}
if (*extra_options.curl_progress_state)
{
option_curl_progress_state = !strcasecmp(extra_options.curl_progress_state, "true");
}
option_curl_progress_state = !strcasecmp(extra_options.curl_progress_state,
"true");
if (*extra_options.enable_chmod)
{
option_enable_chmod = !strcasecmp(extra_options.enable_chmod, "true");
}
if (*extra_options.enable_chown)
{
option_enable_chown = !strcasecmp(extra_options.enable_chown, "true");
}
}
int main(int argc, char** argv)
{
@@ -781,7 +858,8 @@ int main(int argc, char **argv)
FILE* settings;
struct fuse_args args = FUSE_ARGS_INIT(argc, argv);
snprintf(settings_filename, sizeof(settings_filename), "%s/.hubicfuse", get_home_dir());
snprintf(settings_filename, sizeof(settings_filename), "%s/.hubicfuse",
get_home_dir());
if ((settings = fopen(settings_filename, "r")))
{
char line[OPTION_SIZE];
@@ -801,25 +879,36 @@ int main(int argc, char **argv)
if (!*options.client_id || !*options.client_secret || !*options.refresh_token)
{
fprintf(stderr, "Unable to determine client_id, client_secret or refresh_token.\n\n");
fprintf(stderr,
"Unable to determine client_id, client_secret or refresh_token.\n\n");
fprintf(stderr, "These can be set either as mount options or in "
"a file named %s\n\n", settings_filename);
fprintf(stderr, " client_id=[App's id]\n");
fprintf(stderr, " client_secret=[App's secret]\n");
fprintf(stderr, " refresh_token=[Get it running hubic_token]\n");
fprintf(stderr, "The following settings are optional:\n\n");
fprintf(stderr, " cache_timeout=[Seconds for directory caching, default 600]\n");
fprintf(stderr,
" cache_timeout=[Seconds for directory caching, default 600]\n");
fprintf(stderr, " verify_ssl=[false to disable SSL cert verification]\n");
fprintf(stderr, " segment_size=[Size to use when creating DLOs, default 1073741824]\n");
fprintf(stderr, " segment_above=[File size at which to use segments, defult 2147483648]\n");
fprintf(stderr, " storage_url=[Storage URL for other tenant to view container]\n");
fprintf(stderr, " container=[Public container to view of tenant specified by storage_url]\n");
fprintf(stderr,
" segment_size=[Size to use when creating DLOs, default 1073741824]\n");
fprintf(stderr,
" segment_above=[File size at which to use segments, defult 2147483648]\n");
fprintf(stderr,
" storage_url=[Storage URL for other tenant to view container]\n");
fprintf(stderr,
" container=[Public container to view of tenant specified by storage_url]\n");
fprintf(stderr, " temp_dir=[Directory to store temp files]\n");
fprintf(stderr, " get_extended_metadata=[true to enable download of utime, chmod, chown file attributes (but slower)]\n");
fprintf(stderr, " curl_verbose=[true to debug info on curl requests (lots of output)]\n");
fprintf(stderr, " curl_progress_state=[true to enable progress callback enabled. Mostly used for debugging]\n");
fprintf(stderr, " cache_statfs_timeout=[number of seconds to cache requests to statfs (cloud statistics), 0 for no cache]\n");
fprintf(stderr, " debug_level=[0 to 2, 0 for minimal verbose debugging. No debug if -d or -f option is not provided.]\n");
fprintf(stderr,
" get_extended_metadata=[true to enable download of utime, chmod, chown file attributes (but slower)]\n");
fprintf(stderr,
" curl_verbose=[true to debug info on curl requests (lots of output)]\n");
fprintf(stderr,
" curl_progress_state=[true to enable progress callback enabled. Mostly used for debugging]\n");
fprintf(stderr,
" cache_statfs_timeout=[number of seconds to cache requests to statfs (cloud statistics), 0 for no cache]\n");
fprintf(stderr,
" debug_level=[0 to 2, 0 for minimal verbose debugging. No debug if -d or -f option is not provided.]\n");
fprintf(stderr, " enable_chmod=[true to enable chmod support on fuse]\n");
fprintf(stderr, " enable_chown=[true to enable chown support on fuse]\n");
return 1;
@@ -834,7 +923,8 @@ int main(int argc, char **argv)
fprintf(stderr, "enable_chmod = %d\n", option_enable_chmod);
fprintf(stderr, "enable_chown = %d\n", option_enable_chown);
}
cloudfs_set_credentials(options.client_id, options.client_secret, options.refresh_token);
cloudfs_set_credentials(options.client_id, options.client_secret,
options.refresh_token);
if (!cloudfs_connect())
{
@@ -847,7 +937,8 @@ int main(int argc, char **argv)
fuse_opt_add_arg(&args, "-s");
#endif
struct fuse_operations cfs_oper = {
struct fuse_operations cfs_oper =
{
.readdir = cfs_readdir,
.mkdir = cfs_mkdir,
.read = cfs_read,
commonfs.c
+73 -57
View File
@@ -68,7 +68,8 @@ time_t get_time_as_local(time_t time_t_val, char time_str[], int char_buf_size)
{
struct tm loc_time_tm;
loc_time_tm = *localtime(&time_t_val);
if (time_str != NULL) {
if (time_str != NULL)
{
//debugf(DBG_LEVEL_NORM, 0,"Local len=%d size=%d pass=%d", strlen(time_str), sizeof(time_str), char_buf_size);
strftime(time_str, char_buf_size, "%c", &loc_time_tm);
//debugf(DBG_LEVEL_NORM, 0,"Local timestr=[%s] size=%d", time_str, strlen(time_str));
@@ -77,19 +78,24 @@ time_t get_time_as_local(time_t time_t_val, char time_str[], int char_buf_size)
return mktime(&loc_time_tm);
}
int get_time_as_string(time_t time_t_val, long nsec, char *time_str, int time_str_len)
int get_time_as_string(time_t time_t_val, long nsec, char* time_str,
int time_str_len)
{
struct tm time_val_tm;
time_t safe_input_time;
//if time is incorrect (too long) you get segfault, need to check length and trim
if (time_t_val > INT_MAX) {
debugf(DBG_LEVEL_NORM, KRED"get_time_as_string: input time length too long, %lu > max=%lu, trimming!", time_t_val, INT_MAX);
if (time_t_val > INT_MAX)
{
debugf(DBG_LEVEL_NORM,
KRED"get_time_as_string: input time length too long, %lu > max=%lu, trimming!",
time_t_val, INT_MAX);
safe_input_time = 0;//(int)time_t_val;
}
else
safe_input_time = time_t_val;
time_val_tm = *gmtime(&safe_input_time);
int str_len = strftime(time_str, time_str_len, HUBIC_DATE_FORMAT, &time_val_tm);
int str_len = strftime(time_str, time_str_len, HUBIC_DATE_FORMAT,
&time_val_tm);
char nsec_str[TIME_CHARS];
sprintf(nsec_str, "%d", nsec);
strcat(time_str, nsec_str);
@@ -114,9 +120,11 @@ size_t get_time_now_as_str(char *time_str, int time_str_len)
return result;
}
int get_timespec_as_str(const struct timespec *times, char *time_str, int time_str_len)
int get_timespec_as_str(const struct timespec* times, char* time_str,
int time_str_len)
{
return get_time_as_string(times->tv_sec, times->tv_nsec, time_str, time_str_len);
return get_time_as_string(times->tv_sec, times->tv_nsec, time_str,
time_str_len);
}
char* str2md5(const char* str, int length)
@@ -127,31 +135,31 @@ char *str2md5(const char *str, int length)
char* out = (char*)malloc(33);
MD5_Init(&c);
while (length > 0) {
if (length > 512) {
while (length > 0)
{
if (length > 512)
MD5_Update(&c, str, 512);
}
else {
else
MD5_Update(&c, str, length);
}
length -= 512;
str += 512;
}
MD5_Final(digest, &c);
for (n = 0; n < 16; ++n) {
for (n = 0; n < 16; ++n)
snprintf(&(out[n * 2]), 16 * 2, "%02x", (unsigned int)digest[n]);
}
return out;
}
// http://stackoverflow.com/questions/10324611/how-to-calculate-the-md5-hash-of-a-large-file-in-c
int file_md5(FILE* file_handle, char* md5_file_str)
{
if (file_handle == NULL) {
if (file_handle == NULL)
{
debugf(DBG_LEVEL_NORM, KRED"file_md5: NULL file handle");
return 0;
}
if (file_buffer_size == 0) {
if (file_buffer_size == 0)
{
debugf(DBG_LEVEL_NORM, KYEL"file_md5: 0 file buffer size, using default");
file_buffer_size = 1024;
}
@@ -165,7 +173,8 @@ int file_md5(FILE *file_handle, char *md5_file_str)
while ((bytes = fread(data_buf, 1, file_buffer_size, file_handle)) != 0)
MD5_Update(&mdContext, data_buf, bytes);
MD5_Final(c, &mdContext);
for (i = 0; i < MD5_DIGEST_LENGTH; i++) {
for (i = 0; i < MD5_DIGEST_LENGTH; i++)
{
snprintf(mdchar, 3, "%02x", c[i]);
strcat(md5_file_str, mdchar);
}
@@ -173,14 +182,14 @@ int file_md5(FILE *file_handle, char *md5_file_str)
return 0;
}
int get_safe_cache_file_path(const char *path, char *file_path_safe, char *temp_dir)
int get_safe_cache_file_path(const char* path, char* file_path_safe,
char* temp_dir)
{
char tmp_path[PATH_MAX];
strncpy(tmp_path, path, PATH_MAX);
char* pch;
while ((pch = strchr(tmp_path, '/'))) {
while ((pch = strchr(tmp_path, '/')))
* pch = '.';
}
char file_path[PATH_MAX] = "";
//temp file name had process pid in it, removed as on restart files are left in cache (pid changes)
snprintf(file_path, PATH_MAX, TEMP_FILE_NAME_FORMAT, temp_dir, tmp_path);
@@ -221,7 +230,8 @@ void debug_print_flags(int flags)
if (val & O_APPEND) debugf(DBG_LEVEL_EXTALL, KYEL", append");
if (val & O_NONBLOCK) debugf(DBG_LEVEL_EXTALL, KYEL", nonblocking");
#if !defined(_POSIX_SOURCE) && defined(O_SYNC)
if (val & O_SYNC) debugf(DBG_LEVEL_EXT, 0,KRED ", synchronous writes");
if (val & O_SYNC) debugf(DBG_LEVEL_EXT, 0,
KRED ", synchronous writes");
#endif
}
@@ -231,7 +241,8 @@ void debug_print_descriptor(struct fuse_file_info *info)
{
char file_path[MAX_PATH_SIZE];
get_file_path_from_fd(info->fh, file_path, sizeof(file_path));
debugf(DBG_LEVEL_EXT, KCYN "descriptor localfile=[%s] fd=%d", file_path, info->fh);
debugf(DBG_LEVEL_EXT, KCYN "descriptor localfile=[%s] fd=%d", file_path,
info->fh);
debug_print_flags(info->flags);
}
@@ -244,18 +255,18 @@ void dir_for(const char *path, char *dir)
}
//prints cache content for debug purposes
void debug_list_cache_content() {
void debug_list_cache_content()
{
return;//disabled
dir_cache* cw;
dir_entry* de;
for (cw = dcache; cw; cw = cw->next)
{
debugf(DBG_LEVEL_EXT, "LIST-CACHE: DIR[%s]", cw->path);
for (de = cw->entries; de; de = de->next) {
for (de = cw->entries; de; de = de->next)
debugf(DBG_LEVEL_EXT, "LIST-CACHE: FOLDER[%s]", de->full_name);
}
}
}
int delete_file(char* path)
{
@@ -263,7 +274,8 @@ int delete_file(char *path)
char file_path_safe[NAME_MAX] = "";
get_safe_cache_file_path(path, file_path_safe, temp_dir);
int result = unlink(file_path_safe);
debugf(DBG_LEVEL_EXT, KYEL"delete_file(%s) (%s) result=%s", path, file_path_safe, strerror(result));
debugf(DBG_LEVEL_EXT, KYEL"delete_file(%s) (%s) result=%s", path,
file_path_safe, strerror(result));
return result;
}
@@ -294,7 +306,8 @@ void cloudfs_free_dir_list(dir_entry *dir_list)
{
//check for NULL as dir might be already removed from cache by other thread
debugf(DBG_LEVEL_NORM, "cloudfs_free_dir_list(%s)", dir_list->full_name);
while (dir_list) {
while (dir_list)
{
dir_entry* de = dir_list;
dir_list = dir_list->next;
//remove file from disk cache, fix for issue #89, https://github.com/TurboGit/hubicfuse/issues/89
@@ -316,7 +329,8 @@ void dir_decache(const char *path)
dir_entry* de, *tmpde;
char dir[MAX_PATH_SIZE];
dir_for(path, dir);
for (cw = dcache; cw; cw = cw->next) {
for (cw = dcache; cw; cw = cw->next)
{
debugf(DBG_LEVEL_EXT, "dir_decache: parse(%s)", cw->path);
if (!strcmp(cw->path, path))
{
@@ -425,16 +439,11 @@ void update_dir_cache(const char *path, off_t size, int isdir, int islink)
de->full_name = strdup(path);
//fixed: the conditions below were mixed up dir -> link?
if (islink)
{
de->content_type = strdup("application/link");
}
if (isdir)
{
de->content_type = strdup("application/directory");
}
else {
else
de->content_type = strdup("application/octet-stream");
}
de->next = cw->entries;
cw->entries = de;
if (isdir)
@@ -459,13 +468,16 @@ int caching_list_directory(const char *path, dir_entry **list)
{
if (cw->was_deleted == true)
{
debugf(DBG_LEVEL_EXT, KMAG"caching_list_directory status: dir(%s) is empty as cached expired, reload from cloud", cw->path);
debugf(DBG_LEVEL_EXT,
KMAG"caching_list_directory status: dir(%s) is empty as cached expired, reload from cloud",
cw->path);
if (!cloudfs_list_directory(cw->path, list))
debugf(DBG_LEVEL_EXT,
KMAG"caching_list_directory status: cannot reload dir(%s)", cw->path);
else
{
debugf(DBG_LEVEL_EXT, KMAG"caching_list_directory status: cannot reload dir(%s)", cw->path);
}
else {
debugf(DBG_LEVEL_EXT, KMAG"caching_list_directory status: reloaded dir(%s)", cw->path);
debugf(DBG_LEVEL_EXT, KMAG"caching_list_directory status: reloaded dir(%s)",
cw->path);
//cw->entries = *list;
cw->was_deleted = false;
cw->cached = time(NULL);
@@ -474,11 +486,9 @@ int caching_list_directory(const char *path, dir_entry **list)
if (cw->was_deleted == false)
{
if (!strcmp(cw->path, path))
{
break;
}
}
}
if (!cw)
{
//trying to download this entry from cloud, list will point to cached or downloaded entries
@@ -486,10 +496,12 @@ int caching_list_directory(const char *path, dir_entry **list)
{
//download was not ok
pthread_mutex_unlock(&dcachemut);
debugf(DBG_LEVEL_EXT, "exit 0: caching_list_directory(%s) "KYEL"[CACHE-DIR-MISS]", path);
debugf(DBG_LEVEL_EXT,
"exit 0: caching_list_directory(%s) "KYEL"[CACHE-DIR-MISS]", path);
return 0;
}
debugf(DBG_LEVEL_EXT, "caching_list_directory: new_cache(%s) "KYEL"[CACHE-CREATE]", path);
debugf(DBG_LEVEL_EXT,
"caching_list_directory: new_cache(%s) "KYEL"[CACHE-CREATE]", path);
cw = new_cache(path);
new_entry = true;
}
@@ -509,16 +521,21 @@ int caching_list_directory(const char *path, dir_entry **list)
cloudfs_free_dir_list(cw->entries);
cw->was_deleted = true;
cw->cached = time(NULL);
debugf(DBG_LEVEL_EXT, "caching_list_directory(%s) "KYEL"[CACHE-EXPIRED]", path);
debugf(DBG_LEVEL_EXT, "caching_list_directory(%s) "KYEL"[CACHE-EXPIRED]",
path);
}
else {
debugf(DBG_LEVEL_EXT, "got NULL on caching_list_directory(%s) "KYEL"[CACHE-EXPIRED w NULL]", path);
else
{
debugf(DBG_LEVEL_EXT,
"got NULL on caching_list_directory(%s) "KYEL"[CACHE-EXPIRED w NULL]", path);
pthread_mutex_unlock(&dcachemut);
return 0;
}
}
else {
debugf(DBG_LEVEL_EXT, "caching_list_directory(%s) "KGRN"[CACHE-DIR-HIT]", path);
else
{
debugf(DBG_LEVEL_EXT, "caching_list_directory(%s) "KGRN"[CACHE-DIR-HIT]",
path);
*list = cw->entries;
}
//adding new dir file list to global cache, now this dir becomes visible in cache
@@ -539,9 +556,8 @@ dir_entry *path_info(const char *path)
debugf(DBG_LEVEL_EXT, "exit 0: path_info(%s) "KYEL"[CACHE-DIR-MISS]", dir);
return NULL;
}
else {
else
debugf(DBG_LEVEL_EXT, "path_info(%s) "KGRN"[CACHE-DIR-HIT]", dir);
}
//iterate in file list obtained from cache or downloaded
for (; tmp; tmp = tmp->next)
{
@@ -570,11 +586,13 @@ int check_caching_list_directory(const char *path, dir_entry **list)
{
*list = cw->entries;
pthread_mutex_unlock(&dcachemut);
debugf(DBG_LEVEL_EXT, "exit 0: check_caching_list_directory(%s) "KGRN"[CACHE-DIR-HIT]", path);
debugf(DBG_LEVEL_EXT,
"exit 0: check_caching_list_directory(%s) "KGRN"[CACHE-DIR-HIT]", path);
return 1;
}
pthread_mutex_unlock(&dcachemut);
debugf(DBG_LEVEL_EXT, "exit 1: check_caching_list_directory(%s) "KYEL"[CACHE-DIR-MISS]", path);
debugf(DBG_LEVEL_EXT,
"exit 1: check_caching_list_directory(%s) "KYEL"[CACHE-DIR-MISS]", path);
return 0;
}
@@ -612,12 +630,10 @@ dir_entry *check_path_info(const char *path)
}
}
if (!strcmp(path, "/"))
{
debugf(DBG_LEVEL_EXT, "exit 2: check_path_info(%s) "KYEL"ignoring root [CACHE-MISS]", path);
}
else {
debugf(DBG_LEVEL_EXT,
"exit 2: check_path_info(%s) "KYEL"ignoring root [CACHE-MISS]", path);
else
debugf(DBG_LEVEL_EXT, "exit 3: check_path_info(%s) "KYEL"[CACHE-MISS]", path);
}
return NULL;
}
commonfs.h
+8 -4
View File
@@ -81,14 +81,18 @@ typedef struct dir_cache
time_t my_timegm(struct tm* tm);
time_t get_time_from_str_as_gmt(char* time_str);
time_t get_time_as_local(time_t time_t_val, char time_str[], int char_buf_size);
int get_time_as_string(time_t time_t_val, long nsec, char *time_str, int time_str_len);
time_t get_time_as_local(time_t time_t_val, char time_str[],
int char_buf_size);
int get_time_as_string(time_t time_t_val, long nsec, char* time_str,
int time_str_len);
time_t get_time_now();
int get_timespec_as_str(const struct timespec *times, char *time_str, int time_str_len);
int get_timespec_as_str(const struct timespec* times, char* time_str,
int time_str_len);
char* str2md5(const char* str, int length);
int file_md5(FILE* file_handle, char* md5_file_str);
void debug_print_descriptor(struct fuse_file_info* info);
int get_safe_cache_file_path(const char *file_path, char *file_path_safe, char *temp_dir);
int get_safe_cache_file_path(const char* file_path, char* file_path_safe,
char* temp_dir);
dir_entry* init_dir_entry();
void copy_dir_entry(dir_entry* src, dir_entry* dst);
dir_cache* new_cache(const char* path);