diff --git a/README b/README index f6e5235..0491826 100644 --- a/README +++ b/README @@ -2,6 +2,11 @@ HubicFuse is a FUSE application which provides access to Hubic's cloud files via a mount-point. +This version contains support for DLO, symlinks and support to see other +tenant's containers. Those features are coming from: + + https://github.com/LabAdvComp/cloudfuse.git + BUILDING: You'll need libcurl, fuse, libssl, and libxml2 (and probably their dev @@ -79,16 +84,16 @@ BUGS/SHORTCOMINGS: AWESOME CONTRIBUTORS: - * Pascal Obry https://github.com/TurboGit - * Tim Dysinger https://github.com/dysinger - * Chris Wedgwood https://github.com/cwedgwood - * Nick Craig-Wood https://github.com/ncw - * Dillon Amburgey https://github.com/dillona - * Manfred Touron https://github.com/moul - * David Brownlee https://github.com/abs0 - * Mike Lundy https://github.com/novas0x2a - * justinb https://github.com/justinsb - + * Pascal Obry https://github.com/TurboGit + * Tim Dysinger https://github.com/dysinger + * Chris Wedgwood https://github.com/cwedgwood + * Nick Craig-Wood https://github.com/ncw + * Dillon Amburgey https://github.com/dillona + * Manfred Touron https://github.com/moul + * David Brownlee https://github.com/abs0 + * Mike Lundy https://github.com/novas0x2a + * justinb https://github.com/justinsb + * Matt Greenway https://github.com/LabAdvComp Thanks, and I hope you find it useful. diff --git a/cloudfsapi.c b/cloudfsapi.c index caec82e..126be0d 100644 --- a/cloudfsapi.c +++ b/cloudfsapi.c @@ -4,6 +4,7 @@ #include #include #include +#include #ifdef __linux__ #include #endif @@ -16,6 +17,8 @@ #include #include #include +#include +#include #include "cloudfsapi.h" #include "config.h" #include @@ -27,6 +30,13 @@ #define MAX_FILES 10000 +// 64 bit time + nanoseconds +#define TIME_CHARS 32 + +// size of buffer for writing to disk look at ioblksize.h in coreutils +// and try some values on your own system if you want the best performance +#define DISK_BUFF_SIZE 32768 + static char storage_url[MAX_URL_SIZE]; static char storage_token[MAX_HEADER_SIZE]; static pthread_mutex_t pool_mut; @@ -92,7 +102,8 @@ static void return_connection(CURL *curl) pthread_mutex_unlock(&pool_mut); } -static void add_header(curl_slist **headers, const char *name, const char *value) +static void add_header(curl_slist **headers, const char *name, + const char *value) { char x_header[MAX_HEADER_SIZE]; snprintf(x_header, sizeof(x_header), "%s: %s", name, value); @@ -126,8 +137,39 @@ static size_t header_dispatch(void *ptr, size_t size, size_t nmemb, void *stream return size * nmemb; } -static int send_request(char *method, const char *path, FILE *fp, - xmlParserCtxtPtr xmlctx, curl_slist *extra_headers) +static size_t rw_callback(size_t (*rw)(void*, size_t, size_t, FILE*), void *ptr, + size_t size, size_t nmemb, void *userp) +{ + struct segment_info *info = (struct segment_info *)userp; + size_t mem = size * nmemb; + + if (mem < 1 || info->size < 1) + return 0; + + size_t amt_read = rw(ptr, 1, info->size < mem ? info->size : mem, info->fp); + info->size -= amt_read; + + return amt_read; +} + +size_t fwrite2(void *ptr, size_t size, size_t nmemb, FILE *filep) +{ + return fwrite((const void*)ptr, size, nmemb, filep); +} + +static size_t read_callback(void *ptr, size_t size, size_t nmemb, void *userp) +{ + return rw_callback(fread, ptr, size, nmemb, userp); +} + +static size_t write_callback(void *ptr, size_t size, size_t nmemb, void *userp) +{ + return rw_callback(fwrite2, ptr, size, nmemb, userp); +} + +static int send_request_size(const char *method, const char *path, void *fp, + xmlParserCtxtPtr xmlctx, curl_slist *extra_headers, + off_t file_size, int is_segment) { char url[MAX_URL_SIZE]; char *slash; @@ -172,16 +214,31 @@ static int send_request(char *method, const char *path, FILE *fp, curl_easy_setopt(curl, CURLOPT_INFILESIZE, 0); add_header(&headers, "Content-Type", "application/directory"); } - else if (!strcasecmp(method, "PUT") && fp) + else if (!strcasecmp(method, "MKLINK") && fp) { rewind(fp); curl_easy_setopt(curl, CURLOPT_UPLOAD, 1); - curl_easy_setopt(curl, CURLOPT_INFILESIZE, cloudfs_file_size(fileno(fp))); + //curl_easy_setopt(curl, CURLOPT_INFILESIZE, 0); + curl_easy_setopt(curl, CURLOPT_INFILESIZE, file_size); curl_easy_setopt(curl, CURLOPT_READDATA, fp); + add_header(&headers, "Content-Type", "application/link"); + } + else if (!strcasecmp(method, "PUT") && fp) + { + curl_easy_setopt(curl, CURLOPT_UPLOAD, 1); + curl_easy_setopt(curl, CURLOPT_INFILESIZE, file_size); + curl_easy_setopt(curl, CURLOPT_READDATA, fp); + if (is_segment) + curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_callback); } else if (!strcasecmp(method, "GET")) { - if (fp) + if (is_segment) + { + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp); + } + else if (fp) { rewind(fp); // make sure the file is ready for a-writin' fflush(fp); @@ -227,6 +284,196 @@ static int send_request(char *method, const char *path, FILE *fp, return response; } +static int send_request(char *method, const char *path, FILE *fp, + xmlParserCtxtPtr xmlctx, curl_slist *extra_headers) +{ + + long flen = 0; + if (fp) { + // if we don't flush the size will probably be zero + fflush(fp); + flen = cloudfs_file_size(fileno(fp)); + } + + return send_request_size(method, path, fp, xmlctx, extra_headers, flen, 0); + +} + +void *upload_segment(void *seginfo) +{ + char seg_path[MAX_URL_SIZE]; + struct segment_info *info; + info = (struct segment_info *)seginfo; + + fseek(info->fp, info->part * info->segment_size, SEEK_SET); + setvbuf(info->fp, NULL, _IOFBF, DISK_BUFF_SIZE); + + snprintf(seg_path, MAX_URL_SIZE, "%s%08i", info->seg_base, info->part); + char *encoded = curl_escape(seg_path, 0); + + int response = send_request_size(info->method, encoded, info, NULL, NULL, + info->size, 1); + + if (!(response >= 200 && response < 300)) + fprintf(stderr, "Segment upload %s failed with response %d", seg_path, + response); + + curl_free(encoded); + fclose(info->fp); + pthread_exit(NULL); +} + +// segment_size is the globabl config variable and size_of_segment is local +//TODO: return whether the upload/download failed or not +void run_segment_threads(const char *method, int segments, int full_segments, int remaining, + FILE *fp, char *seg_base, int size_of_segments) +{ + char file_path[PATH_MAX] = ""; + struct segment_info *info = (struct segment_info *) + malloc(segments * sizeof(struct segment_info)); + + pthread_t *threads = (pthread_t *)malloc(segments * sizeof(pthread_t)); +#ifdef __linux__ + snprintf(file_path, PATH_MAX, "/proc/self/fd/%d", fileno(fp)); +#else + //TODO: I haven't actually tested this + if (fcntl(fileno(fp), F_GETPATH, file_path) == -1) + fprintf(stderr, "couldn't get the path name\n"); +#endif + + int i; + for (i = 0; i < segments; i++) { + info[i].method = method; + info[i].fp = fopen(file_path, method[0] == 'G' ? "r+" : "r"); + info[i].part = i; + info[i].segment_size = size_of_segments; + info[i].size = i < full_segments ? size_of_segments : remaining; + info[i].seg_base = seg_base; + pthread_create(&threads[i], NULL, upload_segment, (void *)&(info[i])); + } + + for (i = 0; i < segments; i++) { + pthread_join(threads[i], NULL); + } + free(info); + free(threads); +} + +void split_path(const char *path, char *seg_base, char *container, + char *object) +{ + char *string = strdup(path); + + snprintf(seg_base, MAX_URL_SIZE, "%s", strsep(&string, "/")); + + strncat(container, strsep(&string, "/"), + MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE)); + + char *_object = strsep(&string, "/"); + + char *remstr; + + while (remstr = strsep(&string, "/")) { + strncat(container, "/", + MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE)); + strncat(container, _object, + MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE)); + _object = remstr; + } + + strncpy(object, _object, MAX_URL_SIZE); + free(string); +} + +int internal_is_segmented(const char *seg_path, const char *object) +{ + dir_entry *seg_dir; + if (cloudfs_list_directory(seg_path, &seg_dir)) { + if (seg_dir && seg_dir->isdir) { + do { + if (!strncmp(seg_dir->name, object, MAX_URL_SIZE)) { + return 1; + } + } while ((seg_dir = seg_dir->next)); + } + } + return 0; +} + +int is_segmented(const char *path) +{ + char container[MAX_URL_SIZE] = ""; + char object[MAX_URL_SIZE] = ""; + char seg_base[MAX_URL_SIZE] = ""; + + split_path(path, seg_base, container, object); + + char seg_path[MAX_URL_SIZE]; + snprintf(seg_path, MAX_URL_SIZE, "%s/%s_segments", seg_base, container); + + return internal_is_segmented(seg_path, object); +} + + +int format_segments(const char *path, char * seg_base, long *segments, + long *full_segments, long *remaining, long *size_of_segments) +{ + + char container[MAX_URL_SIZE] = ""; + char object[MAX_URL_SIZE] = ""; + + split_path(path, seg_base, container, object); + + char seg_path[MAX_URL_SIZE]; + snprintf(seg_path, MAX_URL_SIZE, "%s/%s_segments", seg_base, container); + + if (internal_is_segmented(seg_path, object)) { + char manifest[MAX_URL_SIZE]; + dir_entry *seg_dir; + + snprintf(manifest, MAX_URL_SIZE, "%s/%s", seg_path, object); + if (!cloudfs_list_directory(manifest, &seg_dir)) + return 0; + + // snprintf seesaw between manifest and seg_path to get + // the total_size and the segment size as well as the actual objects + char *timestamp = seg_dir->name; + snprintf(seg_path, MAX_URL_SIZE, "%s/%s", manifest, timestamp); + if (!cloudfs_list_directory(seg_path, &seg_dir)) + return 0; + + char *str_size = seg_dir->name; + snprintf(manifest, MAX_URL_SIZE, "%s/%s", seg_path, str_size); + if (!cloudfs_list_directory(manifest, &seg_dir)) + return 0; + + char *str_segment = seg_dir->name; + snprintf(seg_path, MAX_URL_SIZE, "%s/%s", manifest, str_segment); + if (!cloudfs_list_directory(seg_path, &seg_dir)) + return 0; + + long total_size = strtoll(str_size, NULL, 10); + *size_of_segments = strtoll(str_segment, NULL, 10); + + *remaining = total_size % *size_of_segments; + *full_segments = total_size / *size_of_segments; + *segments = *full_segments + (*remaining > 0); + + snprintf(manifest, MAX_URL_SIZE, "%s_segments/%s/%s/%s/%s/", + container, object, timestamp, str_size, str_segment); + + char tmp[MAX_URL_SIZE]; + strncpy(tmp, seg_base, MAX_URL_SIZE); + snprintf(seg_base, MAX_URL_SIZE, "%s/%s", tmp, manifest); + + return 1; + } + + else { + return 0; + } +} + /* * Public interface */ @@ -234,6 +481,7 @@ static int send_request(char *method, const char *path, FILE *fp, void cloudfs_init() { LIBXML_TEST_VERSION + xmlXPathInit(); curl_global_init(CURL_GLOBAL_ALL); pthread_mutex_init(&pool_mut, NULL); curl_version_info_data *cvid = curl_version_info(CURLVERSION_NOW); @@ -266,7 +514,75 @@ void cloudfs_init() int cloudfs_object_read_fp(const char *path, FILE *fp) { + + long flen; fflush(fp); + + // determine the size of the file and segment if it is above the threshhold + fseek(fp, 0, SEEK_END); + flen = ftell(fp); + + // delete the previously uploaded segments + if (is_segmented(path)) { + if (!cloudfs_delete_object(path)) + debugf("Couldn't delete one of the existing files while uploading."); + } + + if (flen >= segment_above) { + int i; + long remaining = flen % segment_size; + int full_segments = flen / segment_size; + int segments = full_segments + (remaining > 0); + + // The best we can do here is to get the current time that way tools that + // use the mtime can at least check if the file was changing after now + struct timespec now; + clock_gettime(CLOCK_REALTIME, &now); + + char string_float[TIME_CHARS]; + snprintf(string_float, TIME_CHARS, "%lu.%lu", now.tv_sec, now.tv_nsec); + + char meta_mtime[TIME_CHARS]; + snprintf(meta_mtime, TIME_CHARS, "%f", atof(string_float)); + + char seg_base[MAX_URL_SIZE] = ""; + + char container[MAX_URL_SIZE] = ""; + char object[MAX_URL_SIZE] = ""; + + split_path(path, seg_base, container, object); + + char manifest[MAX_URL_SIZE]; + snprintf(manifest, MAX_URL_SIZE, "%s_segments", container); + + // create the segments container + cloudfs_create_directory(manifest); + + // reusing manifest + snprintf(manifest, MAX_URL_SIZE, "%s_segments/%s/%s/%ld/%ld/", + container, object, meta_mtime, flen, segment_size); + + char tmp[MAX_URL_SIZE]; + strncpy(tmp, seg_base, MAX_URL_SIZE); + snprintf(seg_base, MAX_URL_SIZE, "%s/%s", tmp, manifest); + + run_segment_threads("PUT", segments, full_segments, remaining, fp, + seg_base, segment_size); + + char *encoded = curl_escape(path, 0); + curl_slist *headers = NULL; + add_header(&headers, "x-object-manifest", manifest); + add_header(&headers, "x-object-meta-mtime", meta_mtime); + add_header(&headers, "Content-Length", "0"); + + int response = send_request_size("PUT", encoded, NULL, NULL, headers, 0, 0); + curl_slist_free_all(headers); + + curl_free(encoded); + return (response >= 200 && response < 300); + + } + rewind(fp); char *encoded = curl_escape(path, 0); int response = send_request("PUT", encoded, fp, NULL, NULL); @@ -274,9 +590,34 @@ int cloudfs_object_read_fp(const char *path, FILE *fp) return (response >= 200 && response < 300); } + int cloudfs_object_write_fp(const char *path, FILE *fp) { char *encoded = curl_escape(path, 0); + char seg_base[MAX_URL_SIZE] = ""; + + long segments; + long full_segments; + long remaining; + long size_of_segments; + + if (format_segments(path, seg_base, &segments, &full_segments, &remaining, + &size_of_segments)) { + + rewind(fp); + fflush(fp); + + if (ftruncate(fileno(fp), 0) < 0) + { + debugf("ftruncate failed. I don't know what to do about that."); + abort(); + } + + run_segment_threads("GET", segments, full_segments, remaining, fp, + seg_base, size_of_segments); + return 1; + } + int response = send_request("GET", encoded, fp, NULL, NULL); curl_free(encoded); fflush(fp); @@ -345,9 +686,14 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) curl_free(encoded_object); } - response = send_request("GET", container, NULL, xmlctx, NULL); - xmlParseChunk(xmlctx, "", 0, 1); - if (xmlctx->wellFormed && response >= 200 && response < 300) + if ((!strcmp(path, "") || !strcmp(path, "/")) && *override_storage_url) + response = 404; + else + response = send_request("GET", container, NULL, xmlctx, NULL); + + if (response >= 200 && response < 300) + xmlParseChunk(xmlctx, "", 0, 1); + if (response >= 200 && response < 300 && xmlctx->wellFormed ) { xmlNode *root_element = xmlDocGetRootElement(xmlctx->myDoc); for (onode = root_element->children; onode; onode = onode->next) @@ -405,6 +751,8 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) de->isdir = de->content_type && ((strstr(de->content_type, "application/folder") != NULL) || (strstr(de->content_type, "application/directory") != NULL)); + de->islink = de->content_type && + ((strstr(de->content_type, "application/link") != NULL)); if (de->isdir) { if (!strncasecmp(de->name, last_subdir, sizeof(last_subdir))) @@ -424,6 +772,25 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list) } retval = 1; } + else if ((!strcmp(path, "") || !strcmp(path, "/")) && *override_storage_url) { + entry_count = 1; + + dir_entry *de = (dir_entry *)malloc(sizeof(dir_entry)); + de->name = strdup(public_container); + struct tm last_modified; + strptime("1388434648.01238", "%FT%T", &last_modified); + de->last_modified = mktime(&last_modified); + de->content_type = strdup("application/directory"); + if (asprintf(&(de->full_name), "%s/%s", path, de->name) < 0) + de->full_name = NULL; + + de->isdir = 1; + de->islink = 0; + de->size = 4096; + de->next = *dir_list; + *dir_list = de; + retval = 1; + } debugf("entry count: %d", entry_count); @@ -447,6 +814,28 @@ void cloudfs_free_dir_list(dir_entry *dir_list) int cloudfs_delete_object(const char *path) { + + char seg_base[MAX_URL_SIZE] = ""; + + long segments; + long full_segments; + long remaining; + long size_of_segments; + + if (format_segments(path, seg_base, &segments, &full_segments, &remaining, + &size_of_segments)) { + int response; + int i; + char seg_path[MAX_URL_SIZE] = ""; + for (i = 0; i < segments; i++) { + snprintf(seg_path, MAX_URL_SIZE, "%s%08i", seg_base, i); + char *encoded = curl_escape(seg_path, 0); + response = send_request("DELETE", encoded, NULL, NULL, NULL); + if (response < 200 || response >= 300) + return 0; + } + } + char *encoded = curl_escape(path, 0); int response = send_request("DELETE", encoded, NULL, NULL, NULL); curl_free(encoded); @@ -474,7 +863,20 @@ int cloudfs_statfs(const char *path, struct statvfs *stat) debugf("Assigning statvfs values from cache."); *stat = statcache; + return (response >= 200 && response < 300); +} +int cloudfs_create_symlink(const char *src, const char *dst) +{ + char *dst_encoded = curl_escape(dst, 0); + + FILE *lnk = tmpfile(); + + fwrite(src, 1, strlen(src), lnk); + fwrite("\0", 1, 1, lnk); + int response = send_request("MKLINK", dst_encoded, lnk, NULL, NULL); + curl_free(dst_encoded); + fclose(lnk); return (response >= 200 && response < 300); } diff --git a/cloudfsapi.h b/cloudfsapi.h index cfa301a..8dfd97f 100644 --- a/cloudfsapi.h +++ b/cloudfsapi.h @@ -5,7 +5,7 @@ #include #define BUFFER_INITIAL_SIZE 4096 -#define MAX_HEADER_SIZE 4096 +#define MAX_HEADER_SIZE 8192 #define MAX_PATH_SIZE (1024 + 256 + 3) #define MAX_URL_SIZE (MAX_PATH_SIZE * 3) #define USER_AGENT "CloudFuse" @@ -21,12 +21,18 @@ typedef struct dir_entry off_t size; time_t last_modified; int isdir; + int islink; struct dir_entry *next; } dir_entry; typedef struct options { char cache_timeout[OPTION_SIZE]; char verify_ssl[OPTION_SIZE]; + char segment_size[OPTION_SIZE]; + char segment_above[OPTION_SIZE]; + char storage_url[OPTION_SIZE]; + char container[OPTION_SIZE]; + char temp_dir[OPTION_SIZE]; char client_id[OPTION_SIZE]; char client_secret[OPTION_SIZE]; char refresh_token[OPTION_SIZE]; @@ -35,11 +41,29 @@ typedef struct options { void cloudfs_init(void); void cloudfs_set_credentials(char *client_id, char *client_secret, char *refresh_token); int cloudfs_connect(void); + +struct segment_info +{ + FILE *fp; + int part; + long size; + long segment_size; + char *seg_base; + const char *method; +}; + +long segment_size; +long segment_above; + +char *override_storage_url; +char *public_container; + int cloudfs_object_read_fp(const char *path, FILE *fp); int cloudfs_object_write_fp(const char *path, FILE *fp); int cloudfs_list_directory(const char *path, dir_entry **); int cloudfs_delete_object(const char *path); int cloudfs_copy_object(const char *src, const char *dst); +int cloudfs_create_symlink(const char *src, const char *dst); int cloudfs_create_directory(const char *label); int cloudfs_object_truncate(const char *path, off_t size); off_t cloudfs_file_size(int fd); diff --git a/cloudfuse.c b/cloudfuse.c index 37e580e..1247a4c 100644 --- a/cloudfuse.c +++ b/cloudfuse.c @@ -18,6 +18,7 @@ #include "config.h" static int cache_timeout; +static char *temp_dir; typedef struct dir_cache { @@ -86,7 +87,7 @@ static int caching_list_directory(const char *path, dir_entry **list) return 1; } -static void update_dir_cache(const char *path, off_t size, int isdir) +static void update_dir_cache(const char *path, off_t size, int isdir, int islink) { pthread_mutex_lock(&dmut); dir_cache *cw; @@ -109,9 +110,23 @@ static void update_dir_cache(const char *path, off_t size, int isdir) de = (dir_entry *)malloc(sizeof(dir_entry)); de->size = size; de->isdir = isdir; + de->islink = islink; de->name = strdup(&path[strlen(cw->path)+1]); de->full_name = strdup(path); - de->content_type = strdup(isdir ? "application/directory" : "application/octet-stream"); + + if (isdir) + { + de->content_type = strdup("application/link"); + } + if(islink) + { + de->content_type = strdup("application/directory"); + } + else + { + + de->content_type = strdup("application/octet-stream"); + } de->last_modified = time(NULL); de->next = cw->entries; cw->entries = de; @@ -204,6 +219,15 @@ static int cfs_getattr(const char *path, struct stat *stbuf) stbuf->st_mode = S_IFDIR | 0755; stbuf->st_nlink = 2; } + else if (de->islink) + { + stbuf->st_size = 1; + stbuf->st_mode = S_IFLNK | 0755; + stbuf->st_nlink = 1; + stbuf->st_size = de->size; + /* calc. blocks as if 4K blocksize filesystem; stat uses units of 512B */ + stbuf->st_blocks = ((4095 + de->size) / 4096) * 8; + } else { stbuf->st_size = de->size; @@ -244,7 +268,7 @@ static int cfs_mkdir(const char *path, mode_t mode) { if (cloudfs_create_directory(path)) { - update_dir_cache(path, 0, 1); + update_dir_cache(path, 0, 1, 0); return 0; } return -ENOENT; @@ -258,24 +282,90 @@ static int cfs_create(const char *path, mode_t mode, struct fuse_file_info *info fclose(temp_file); of->flags = info->flags; info->fh = (uintptr_t)of; - update_dir_cache(path, 0, 0); + update_dir_cache(path, 0, 0, 0); info->direct_io = 1; return 0; } static int cfs_open(const char *path, struct fuse_file_info *info) { - FILE *temp_file = tmpfile(); + //TODO: need to clean this up and reduce the duplicated code + if (!*temp_dir) { + + FILE *temp_file = tmpfile(); + dir_entry *de = path_info(path); + if (!(info->flags & O_WRONLY)) + { + if (!cloudfs_object_write_fp(path, temp_file)) + { + fclose(temp_file); + return -ENOENT; + } + update_dir_cache(path, (de ? de->size : 0), 0, 0); + } + openfile *of = (openfile *)malloc(sizeof(openfile)); + of->fd = dup(fileno(temp_file)); + fclose(temp_file); + of->flags = info->flags; + info->fh = (uintptr_t)of; + info->direct_io = 1; + return 0; + } + + + FILE *temp_file; + + char tmp_path[PATH_MAX]; + strncpy(tmp_path, path, PATH_MAX); + + char *pch; + while((pch = strchr(tmp_path, '/'))) { + *pch = '.'; + } + + char file_path[PATH_MAX]; + snprintf(file_path, PATH_MAX, "%s/.cloudfuse%ld-%s", temp_dir, + (long)getpid(), tmp_path); + dir_entry *de = path_info(path); - if (!(info->flags & O_WRONLY)) - { + + if(access(file_path, F_OK) != -1) { + temp_file = fopen(file_path, "r"); + update_dir_cache(path, (de ? de->size : 0), 0, 0); + // file exists + } else if (!(info->flags & O_WRONLY)) { + // we need to lock on the filename another process could open the file + // while we are writing to it and then only read part of the file + + // duplicate the directory caching datastructure to make the code easier + // to understand. + + // each file in the cache needs: + // filename, is_writing, last_closed, is_removing + // the first time a file is opened a new entry is created in the cache + // setting the filename and is_writing to true. This check needs to be + // wrapped with a lock. + // + // each time a file is closed we set the last_closed for the file to now + // and we check the cache for files whose last + // closed is greater than cache_timeout, then start a new thread rming + // that file. + + // TODO: just to prevent this craziness for now + if (*temp_dir) + temp_file = fopen(file_path, "w+b"); + else + temp_file = tmpfile(); + + if (!cloudfs_object_write_fp(path, temp_file)) { fclose(temp_file); return -ENOENT; } - update_dir_cache(path, (de ? de->size : 0), 0); + update_dir_cache(path, (de ? de->size : 0), 0, 0); } + openfile *of = (openfile *)malloc(sizeof(openfile)); of->fd = dup(fileno(temp_file)); fclose(temp_file); @@ -295,7 +385,7 @@ static int cfs_flush(const char *path, struct fuse_file_info *info) openfile *of = (openfile *)(uintptr_t)info->fh; if (of) { - update_dir_cache(path, cloudfs_file_size(of->fd), 0); + update_dir_cache(path, cloudfs_file_size(of->fd), 0, 0); if (of->flags & O_RDWR || of->flags & O_WRONLY) { FILE *fp = fdopen(dup(of->fd), "r"); @@ -336,13 +426,13 @@ static int cfs_ftruncate(const char *path, off_t size, struct fuse_file_info *in if (ftruncate(of->fd, size)) return -errno; lseek(of->fd, 0, SEEK_SET); - update_dir_cache(path, size, 0); + update_dir_cache(path, size, 0, 0); return 0; } static int cfs_write(const char *path, const char *buf, size_t length, off_t offset, struct fuse_file_info *info) { - update_dir_cache(path, offset + length, 0); + update_dir_cache(path, offset + length, 0, 0); return pwrite(((openfile *)(uintptr_t)info->fh)->fd, buf, length, offset); } @@ -399,12 +489,41 @@ static int cfs_rename(const char *src, const char *dst) if (cloudfs_copy_object(src, dst)) { /* FIXME this isn't quite right as doesn't preserve last modified */ - update_dir_cache(dst, src_de->size, 0); + update_dir_cache(dst, src_de->size, 0, 0); return cfs_unlink(src); } return -EIO; } +static int cfs_symlink(const char *src, const char *dst) +{ + if(cloudfs_create_symlink(src, dst)) + { + update_dir_cache(dst, 1, 0, 1); + return 0; + } + return -EIO; +} + +static int cfs_readlink(const char* path, char* buf, size_t size) +{ + FILE *temp_file = tmpfile(); + int ret = 0; + + if (!cloudfs_object_write_fp(path, temp_file)) + { + ret = -ENOENT; + } + + if (!pread(fileno(temp_file), buf, size, 0)) + { + ret = -ENOENT; + } + + fclose(temp_file); + return ret; +} + static void *cfs_init(struct fuse_conn_info *conn) { signal(SIGPIPE, SIG_IGN); @@ -425,6 +544,12 @@ char *get_home_dir() FuseOptions options = { .cache_timeout = "600", .verify_ssl = "true", + .segment_size = "1073741824", + .segment_above = "2147483648", + .storage_url = "", + .container = "", + //.temp_dir = "/tmp/", + .temp_dir = "", .client_id = "", .client_secret = "", .refresh_token = "" @@ -434,6 +559,11 @@ int parse_option(void *data, const char *arg, int key, struct fuse_args *outargs { if (sscanf(arg, " cache_timeout = %[^\r\n ]", options.cache_timeout) || sscanf(arg, " verify_ssl = %[^\r\n ]", options.verify_ssl) || + sscanf(arg, " segment_above = %[^\r\n ]", options.segment_above) || + sscanf(arg, " segment_size = %[^\r\n ]", options.segment_size) || + sscanf(arg, " storage_url = %[^\r\n ]", options.storage_url) || + sscanf(arg, " container = %[^\r\n ]", options.container) || + sscanf(arg, " temp_dir = %[^\r\n ]", options.temp_dir) || sscanf(arg, " client_id = %[^\r\n ]", options.client_id) || sscanf(arg, " client_secret = %[^\r\n ]", options.client_secret) || sscanf(arg, " refresh_token = %[^\r\n ]", options.refresh_token)) @@ -461,6 +591,12 @@ int main(int argc, char **argv) fuse_opt_parse(&args, &options, NULL, parse_option); cache_timeout = atoi(options.cache_timeout); + segment_size = atoll(options.segment_size); + segment_above = atoll(options.segment_above); + // this is ok since main is on the stack during the entire execution + override_storage_url = options.storage_url; + public_container = options.container; + temp_dir = options.temp_dir; if (!*options.client_id || !*options.client_secret || !*options.refresh_token) { @@ -473,6 +609,11 @@ int main(int argc, char **argv) fprintf(stderr, "The following settings are optional:\n\n"); fprintf(stderr, " cache_timeout=[Seconds for directory caching, default 600]\n"); fprintf(stderr, " verify_ssl=[False to disable SSL cert verification]\n"); + fprintf(stderr, " segment_size=[Size to use when creating DLOs, default 1073741824]\n"); + fprintf(stderr, " segment_above=[File size at which to use segments, defult 2147483648]\n"); + fprintf(stderr, " storage_url=[Storage URL for other tenant to view container]\n"); + fprintf(stderr, " container=[Public container to view of tenant specified by storage_url]\n"); + fprintf(stderr, " temp_dir=[Directory to store temp files]\n"); return 1; } @@ -482,6 +623,7 @@ int main(int argc, char **argv) cloudfs_verify_ssl(!strcasecmp(options.verify_ssl, "true")); cloudfs_set_credentials(options.client_id, options.client_secret, options.refresh_token); + if (!cloudfs_connect()) { fprintf(stderr, "Failed to authenticate.\n"); @@ -513,6 +655,8 @@ int main(int argc, char **argv) .chmod = cfs_chmod, .chown = cfs_chown, .rename = cfs_rename, + .symlink = cfs_symlink, + .readlink = cfs_readlink, .init = cfs_init, }; diff --git a/debian/README.Debian b/debian/README.Debian new file mode 100644 index 0000000..c394db6 --- /dev/null +++ b/debian/README.Debian @@ -0,0 +1,16 @@ +cloudfuse for Debian +-------------------- + +Please note that the simplest way to allow mounting of Swift shares is to +add the user to the fuse group (which will be created by the fuse-utils +package). + +To add user 'foo' to the 'fuse' group type the following command (with root +privileges): + + adduser foo fuse + +If the user is currently logged in he/she should log out and log in +in order for this change to take effect. + + -- Mike Mestnik Mon, 04 Jun 2012 13:47:28 -0500 diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..78c036a --- /dev/null +++ b/debian/changelog @@ -0,0 +1,12 @@ +cloudfuse (0.9-1ubuntu1) precise; urgency=low + + * Imported Upstream version 0.9 + * Remove Debian patches already committed upstream + + -- pataquets Wed, 15 Jan 2014 21:07:18 +0100 + +cloudfuse (0.1-1) unstable; urgency=low + + * Initial release + + -- Mike Mestnik Mon, 04 Jun 2012 13:47:28 -0500 diff --git a/debian/compat b/debian/compat new file mode 100644 index 0000000..45a4fb7 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +8 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..ef94d79 --- /dev/null +++ b/debian/control @@ -0,0 +1,20 @@ +Source: cloudfuse +Section: utils +Priority: optional +Maintainer: Mike Mestnik +Build-Depends: debhelper (>= 8.0.0), autotools-dev, libcurl4-openssl-dev, libxml2-dev, libssl-dev, libfuse-dev +Standards-Version: 3.9.2 +#Vcs-Git: git://git.debian.org/collab-maint/cloudfuse.git +#Vcs-Browser: http://git.debian.org/?p=collab-maint/cloudfuse.git;a=summary + +Package: cloudfuse +Architecture: any +Depends: ${shlibs:Depends}, ${misc:Depends}, fuse (>= 2.7) [linux-any] | fuse4bsd [kfreebsd-any] +Description: filesystem client based on Swift as used by Rackspace. + cloudfuse is a FUSE application which provides access to Rackspace's + Cloud Files (or any installation of Swift). + . + cloudfuse is FUSE (Filesystem in USErspace). + . + cloudfuse may work with Openstack's Swift, but is not provided by Openstack. + diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000..bb5cf6a --- /dev/null +++ b/debian/copyright @@ -0,0 +1,76 @@ +Format: http://dep.debian.net/deps/dep5 +Upstream-Name: cloudfuse + +Files: * +Copyright: 2009 Michael Barton +License: MIT + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + . + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + . + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + +Files: debian/* +Copyright: 2012 Mike Mestnik +License: GPL-2+ + This package is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + . + This package is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + . + You should have received a copy of the GNU General Public License + along with this program. If not, see + . + On Debian systems, the complete text of the GNU General + Public License version 2 can be found in "/usr/share/common-licenses/GPL-2". + +Files: configure +Copyright: 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, + 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 + Free Software Foundation, Inc. +License: No problem Bugroff + configure script is free software; the Free Software Foundation + gives unlimited permission to copy, distribute and modify it. + +Files: install-sh +Copyright: 1994 X Consortium +License: MIT + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to + deal in the Software without restriction, including without limitation the + rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + sell copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + . + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + . + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + X CONSORTIUM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNEC- + TION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + . + Except as contained in this notice, the name of the X Consortium shall not + be used in advertising or otherwise to promote the sale, use or other deal- + ings in this Software without prior written authorization from the X Consor- + tium. + diff --git a/debian/docs b/debian/docs new file mode 100644 index 0000000..e845566 --- /dev/null +++ b/debian/docs @@ -0,0 +1 @@ +README diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000..b760bee --- /dev/null +++ b/debian/rules @@ -0,0 +1,13 @@ +#!/usr/bin/make -f +# -*- makefile -*- +# Sample debian/rules that uses debhelper. +# This file was originally written by Joey Hess and Craig Small. +# As a special exception, when this file is copied by dh-make into a +# dh-make output file, you may use that output file without restriction. +# This special exception was added by Craig Small in version 0.37 of dh-make. + +# Uncomment this to turn on verbose mode. +#export DH_VERBOSE=1 + +%: + dh $@ diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/debian/watch b/debian/watch new file mode 100644 index 0000000..d6b81b3 --- /dev/null +++ b/debian/watch @@ -0,0 +1,2 @@ +version=3 +http://githubredir.debian.net/github/redbo/cloudfuse (.*).tar.gz