Merge branch 'next'

* next: (42 commits) Update README Expand readme about tenant names & usage requirements rm on segments fixed segfault bug due to incorrectly handling persistent tempfiles removed unnecessary flush symlinks work again, needed to fflush for proper size rearanging functions w/out temp_dir uses tmpfile() from 14 to 130MB/s configure temp path not tested keep tmep files returned 0 removed threaded segment download ready for testing factored out the threads for segments and verified tip to remove segments put back the retry Add chengelog entry and bump package version. errors when public container not visible fast mounting of public container and checks not (options.storage_url ^ options.container)) updated README ...
2026-08-01 18:32:23 +02:00 · 2015-03-17 07:04:11 +01:00
parent 4e1bda3ed5 db0c086327
commit 6ab9d470fe
13 changed files with 749 additions and 32 deletions
@@ -2,6 +2,11 @@
 HubicFuse is a FUSE application which provides access to Hubic's
 cloud files via a mount-point.

+This version contains support for DLO, symlinks and support to see other
+tenant's containers. Those features are coming from:
+
+   https://github.com/LabAdvComp/cloudfuse.git
+
 BUILDING:

    You'll need libcurl, fuse, libssl, and libxml2 (and probably their dev
@@ -79,16 +84,16 @@ BUGS/SHORTCOMINGS:

 AWESOME CONTRIBUTORS:

-    * Pascal Obry                                  https://github.com/TurboGit
-    * Tim Dysinger                                 https://github.com/dysinger
-    * Chris Wedgwood                               https://github.com/cwedgwood
-    * Nick Craig-Wood                              https://github.com/ncw
-    * Dillon Amburgey                              https://github.com/dillona
-    * Manfred Touron                               https://github.com/moul
-    * David Brownlee                               https://github.com/abs0
-    * Mike Lundy                                   https://github.com/novas0x2a
-    * justinb                                      https://github.com/justinsb
-
+    * Pascal Obry                        https://github.com/TurboGit
+    * Tim Dysinger                       https://github.com/dysinger
+    * Chris Wedgwood                     https://github.com/cwedgwood
+    * Nick Craig-Wood                    https://github.com/ncw
+    * Dillon Amburgey                    https://github.com/dillona
+    * Manfred Touron                     https://github.com/moul
+    * David Brownlee                     https://github.com/abs0
+    * Mike Lundy                         https://github.com/novas0x2a
+    * justinb                            https://github.com/justinsb
+    * Matt Greenway                      https://github.com/LabAdvComp

 Thanks, and I hope you find it useful.

@@ -4,6 +4,7 @@
 #include <stdarg.h>
 #include <stdlib.h>
 #include <sys/stat.h>
+#include <fcntl.h>
 #ifdef __linux__
 #include <alloca.h>
 #endif
@@ -16,6 +17,8 @@
 #include <openssl/bio.h>
 #include <openssl/evp.h>
 #include <json.h>
+#include <libxml/xpath.h>
+#include <libxml/xpathInternals.h>
 #include "cloudfsapi.h"
 #include "config.h"
 #include <fuse.h>
@@ -27,6 +30,13 @@

 #define MAX_FILES 10000

+// 64 bit time + nanoseconds
+#define TIME_CHARS 32
+
+// size of buffer for writing to disk look at ioblksize.h in coreutils
+// and try some values on your own system if you want the best performance
+#define DISK_BUFF_SIZE 32768
+
 static char storage_url[MAX_URL_SIZE];
 static char storage_token[MAX_HEADER_SIZE];
 static pthread_mutex_t pool_mut;
@@ -92,7 +102,8 @@ static void return_connection(CURL *curl)
  pthread_mutex_unlock(&pool_mut);
 }

-static void add_header(curl_slist **headers, const char *name, const char *value)
+static void add_header(curl_slist **headers, const char *name,
+                       const char *value)
 {
  char x_header[MAX_HEADER_SIZE];
  snprintf(x_header, sizeof(x_header), "%s: %s", name, value);
@@ -126,8 +137,39 @@ static size_t header_dispatch(void *ptr, size_t size, size_t nmemb, void *stream
  return size * nmemb;
 }

-static int send_request(char *method, const char *path, FILE *fp,
-                        xmlParserCtxtPtr xmlctx, curl_slist *extra_headers)
+static size_t rw_callback(size_t (*rw)(void*, size_t, size_t, FILE*), void *ptr,
+        size_t size, size_t nmemb, void *userp)
+{
+    struct segment_info *info = (struct segment_info *)userp;
+    size_t mem = size * nmemb;
+
+    if (mem < 1 || info->size < 1)
+      return 0;
+
+    size_t amt_read = rw(ptr, 1, info->size < mem ? info->size : mem, info->fp);
+    info->size -= amt_read;
+
+    return amt_read;
+}
+
+size_t fwrite2(void *ptr, size_t size, size_t nmemb, FILE *filep)
+{
+    return fwrite((const void*)ptr, size, nmemb, filep);
+}
+
+static size_t read_callback(void *ptr, size_t size, size_t nmemb, void *userp)
+{
+    return rw_callback(fread, ptr, size, nmemb, userp);
+}
+
+static size_t write_callback(void *ptr, size_t size, size_t nmemb, void *userp)
+{
+   return rw_callback(fwrite2, ptr, size, nmemb, userp);
+}
+
+static int send_request_size(const char *method, const char *path, void *fp,
+                        xmlParserCtxtPtr xmlctx, curl_slist *extra_headers,
+                        off_t file_size, int is_segment)
 {
  char url[MAX_URL_SIZE];
  char *slash;
@@ -172,16 +214,31 @@ static int send_request(char *method, const char *path, FILE *fp,
      curl_easy_setopt(curl, CURLOPT_INFILESIZE, 0);
      add_header(&headers, "Content-Type", "application/directory");
    }
-    else if (!strcasecmp(method, "PUT") && fp)
+    else if (!strcasecmp(method, "MKLINK") && fp)
    {
      rewind(fp);
      curl_easy_setopt(curl, CURLOPT_UPLOAD, 1);
-      curl_easy_setopt(curl, CURLOPT_INFILESIZE, cloudfs_file_size(fileno(fp)));
+      //curl_easy_setopt(curl, CURLOPT_INFILESIZE, 0);
+      curl_easy_setopt(curl, CURLOPT_INFILESIZE, file_size);
      curl_easy_setopt(curl, CURLOPT_READDATA, fp);
+      add_header(&headers, "Content-Type", "application/link");
+    }
+    else if (!strcasecmp(method, "PUT") && fp)
+    {
+      curl_easy_setopt(curl, CURLOPT_UPLOAD, 1);
+      curl_easy_setopt(curl, CURLOPT_INFILESIZE, file_size);
+      curl_easy_setopt(curl, CURLOPT_READDATA, fp);
+      if (is_segment)
+        curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_callback);
    }
    else if (!strcasecmp(method, "GET"))
    {
-      if (fp)
+      if (is_segment)
+      {
+        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback);
+        curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
+      }
+      else if (fp)
      {
        rewind(fp); // make sure the file is ready for a-writin'
        fflush(fp);
@@ -227,6 +284,196 @@ static int send_request(char *method, const char *path, FILE *fp,
  return response;
 }

+static int send_request(char *method, const char *path, FILE *fp,
+                        xmlParserCtxtPtr xmlctx, curl_slist *extra_headers)
+{
+
+    long flen = 0;
+    if (fp) {
+      // if we don't flush the size will probably be zero
+      fflush(fp);
+      flen = cloudfs_file_size(fileno(fp));
+    }
+
+    return send_request_size(method, path, fp, xmlctx, extra_headers, flen, 0);
+
+}
+
+void *upload_segment(void *seginfo)
+{
+  char seg_path[MAX_URL_SIZE];
+  struct segment_info *info;
+  info = (struct segment_info *)seginfo;
+
+  fseek(info->fp, info->part * info->segment_size, SEEK_SET);
+  setvbuf(info->fp, NULL, _IOFBF, DISK_BUFF_SIZE);
+
+  snprintf(seg_path, MAX_URL_SIZE, "%s%08i", info->seg_base, info->part);
+  char *encoded = curl_escape(seg_path, 0);
+
+  int response = send_request_size(info->method, encoded, info, NULL, NULL,
+      info->size, 1);
+
+  if (!(response >= 200 && response < 300))
+    fprintf(stderr, "Segment upload %s failed with response %d", seg_path,
+         response);
+
+  curl_free(encoded);
+  fclose(info->fp);
+  pthread_exit(NULL);
+}
+
+// segment_size is the globabl config variable and size_of_segment is local
+//TODO: return whether the upload/download failed or not
+void run_segment_threads(const char *method, int segments, int full_segments, int remaining,
+        FILE *fp, char *seg_base, int size_of_segments)
+{
+    char file_path[PATH_MAX] = "";
+    struct segment_info *info = (struct segment_info *)
+            malloc(segments * sizeof(struct segment_info));
+
+    pthread_t *threads = (pthread_t *)malloc(segments * sizeof(pthread_t));
+#ifdef __linux__
+    snprintf(file_path, PATH_MAX, "/proc/self/fd/%d", fileno(fp));
+#else
+    //TODO: I haven't actually tested this
+    if (fcntl(fileno(fp), F_GETPATH, file_path) == -1)
+      fprintf(stderr, "couldn't get the path name\n");
+#endif
+
+    int i;
+    for (i = 0; i < segments; i++) {
+      info[i].method = method;
+      info[i].fp = fopen(file_path, method[0] == 'G' ? "r+" : "r");
+      info[i].part = i;
+      info[i].segment_size = size_of_segments;
+      info[i].size = i < full_segments ? size_of_segments : remaining;
+      info[i].seg_base = seg_base;
+      pthread_create(&threads[i], NULL, upload_segment, (void *)&(info[i]));
+    }
+
+    for (i = 0; i < segments; i++) {
+      pthread_join(threads[i], NULL);
+    }
+    free(info);
+    free(threads);
+}
+
+void split_path(const char *path, char *seg_base, char *container,
+        char *object)
+{
+  char *string = strdup(path);
+
+  snprintf(seg_base, MAX_URL_SIZE, "%s", strsep(&string, "/"));
+
+  strncat(container, strsep(&string, "/"),
+      MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE));
+
+  char *_object = strsep(&string, "/");
+
+  char *remstr;
+
+  while (remstr = strsep(&string, "/")) {
+      strncat(container, "/",
+          MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE));
+      strncat(container, _object,
+          MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE));
+      _object = remstr;
+  }
+
+  strncpy(object, _object, MAX_URL_SIZE);
+  free(string);
+}
+
+int internal_is_segmented(const char *seg_path, const char *object)
+{
+  dir_entry *seg_dir;
+  if (cloudfs_list_directory(seg_path, &seg_dir)) {
+    if (seg_dir && seg_dir->isdir) {
+        do {
+            if (!strncmp(seg_dir->name, object, MAX_URL_SIZE)) {
+                return 1;
+            }
+        } while ((seg_dir = seg_dir->next));
+    }
+  }
+  return 0;
+}
+
+int is_segmented(const char *path)
+{
+  char container[MAX_URL_SIZE] = "";
+  char object[MAX_URL_SIZE] = "";
+  char seg_base[MAX_URL_SIZE] = "";
+
+  split_path(path, seg_base, container, object);
+
+  char seg_path[MAX_URL_SIZE];
+  snprintf(seg_path, MAX_URL_SIZE, "%s/%s_segments", seg_base, container);
+
+  return internal_is_segmented(seg_path, object);
+}
+
+
+int format_segments(const char *path, char * seg_base,  long *segments,
+        long *full_segments, long *remaining, long *size_of_segments)
+{
+
+  char container[MAX_URL_SIZE] = "";
+  char object[MAX_URL_SIZE] = "";
+
+  split_path(path, seg_base, container, object);
+
+  char seg_path[MAX_URL_SIZE];
+  snprintf(seg_path, MAX_URL_SIZE, "%s/%s_segments", seg_base, container);
+
+  if (internal_is_segmented(seg_path, object)) {
+    char manifest[MAX_URL_SIZE];
+    dir_entry *seg_dir;
+
+    snprintf(manifest, MAX_URL_SIZE, "%s/%s", seg_path, object);
+    if (!cloudfs_list_directory(manifest, &seg_dir))
+      return 0;
+
+    // snprintf seesaw between manifest and seg_path to get
+    // the total_size and the segment size as well as the actual objects
+    char *timestamp = seg_dir->name;
+    snprintf(seg_path, MAX_URL_SIZE, "%s/%s", manifest, timestamp);
+    if (!cloudfs_list_directory(seg_path, &seg_dir))
+      return 0;
+
+    char *str_size = seg_dir->name;
+    snprintf(manifest, MAX_URL_SIZE, "%s/%s", seg_path, str_size);
+    if (!cloudfs_list_directory(manifest, &seg_dir))
+      return 0;
+
+    char *str_segment = seg_dir->name;
+    snprintf(seg_path, MAX_URL_SIZE, "%s/%s", manifest, str_segment);
+    if (!cloudfs_list_directory(seg_path, &seg_dir))
+      return 0;
+
+    long total_size = strtoll(str_size, NULL, 10);
+    *size_of_segments = strtoll(str_segment, NULL, 10);
+
+    *remaining = total_size % *size_of_segments;
+    *full_segments = total_size / *size_of_segments;
+    *segments = *full_segments + (*remaining > 0);
+
+    snprintf(manifest, MAX_URL_SIZE, "%s_segments/%s/%s/%s/%s/",
+        container, object, timestamp, str_size, str_segment);
+
+    char tmp[MAX_URL_SIZE];
+    strncpy(tmp, seg_base, MAX_URL_SIZE);
+    snprintf(seg_base, MAX_URL_SIZE, "%s/%s", tmp, manifest);
+
+    return 1;
+  }
+
+  else {
+    return 0;
+  }
+}
+
 /*
 * Public interface
 */
@@ -234,6 +481,7 @@ static int send_request(char *method, const char *path, FILE *fp,
 void cloudfs_init()
 {
  LIBXML_TEST_VERSION
+  xmlXPathInit();
  curl_global_init(CURL_GLOBAL_ALL);
  pthread_mutex_init(&pool_mut, NULL);
  curl_version_info_data *cvid = curl_version_info(CURLVERSION_NOW);
@@ -266,7 +514,75 @@ void cloudfs_init()

 int cloudfs_object_read_fp(const char *path, FILE *fp)
 {
+
+  long flen;
  fflush(fp);
+
+  // determine the size of the file and segment if it is above the threshhold
+  fseek(fp, 0, SEEK_END);
+  flen = ftell(fp);
+
+  // delete the previously uploaded segments
+  if (is_segmented(path)) {
+    if (!cloudfs_delete_object(path))
+      debugf("Couldn't delete one of the existing files while uploading.");
+  }
+
+  if (flen >= segment_above) {
+    int i;
+    long remaining = flen % segment_size;
+    int full_segments = flen / segment_size;
+    int segments = full_segments + (remaining > 0);
+
+    // The best we can do here is to get the current time that way tools that
+    // use the mtime can at least check if the file was changing after now
+    struct timespec now;
+    clock_gettime(CLOCK_REALTIME, &now);
+
+    char string_float[TIME_CHARS];
+    snprintf(string_float, TIME_CHARS, "%lu.%lu", now.tv_sec, now.tv_nsec);
+
+    char meta_mtime[TIME_CHARS];
+    snprintf(meta_mtime, TIME_CHARS, "%f", atof(string_float));
+
+    char seg_base[MAX_URL_SIZE] = "";
+
+    char container[MAX_URL_SIZE] = "";
+    char object[MAX_URL_SIZE] = "";
+
+    split_path(path, seg_base, container, object);
+
+    char manifest[MAX_URL_SIZE];
+    snprintf(manifest, MAX_URL_SIZE, "%s_segments", container);
+
+    // create the segments container
+    cloudfs_create_directory(manifest);
+
+    // reusing manifest
+    snprintf(manifest, MAX_URL_SIZE, "%s_segments/%s/%s/%ld/%ld/",
+        container, object, meta_mtime, flen, segment_size);
+
+    char tmp[MAX_URL_SIZE];
+    strncpy(tmp, seg_base, MAX_URL_SIZE);
+    snprintf(seg_base, MAX_URL_SIZE, "%s/%s", tmp, manifest);
+
+    run_segment_threads("PUT", segments, full_segments, remaining, fp,
+            seg_base, segment_size);
+
+    char *encoded = curl_escape(path, 0);
+    curl_slist *headers = NULL;
+    add_header(&headers, "x-object-manifest", manifest);
+    add_header(&headers, "x-object-meta-mtime", meta_mtime);
+    add_header(&headers, "Content-Length", "0");
+
+    int response = send_request_size("PUT", encoded, NULL, NULL, headers, 0, 0);
+    curl_slist_free_all(headers);
+
+    curl_free(encoded);
+    return (response >= 200 && response < 300);
+
+  }
+
  rewind(fp);
  char *encoded = curl_escape(path, 0);
  int response = send_request("PUT", encoded, fp, NULL, NULL);
@@ -274,9 +590,34 @@ int cloudfs_object_read_fp(const char *path, FILE *fp)
  return (response >= 200 && response < 300);
 }

+
 int cloudfs_object_write_fp(const char *path, FILE *fp)
 {
  char *encoded = curl_escape(path, 0);
+  char seg_base[MAX_URL_SIZE] = "";
+
+  long segments;
+  long full_segments;
+  long remaining;
+  long size_of_segments;
+
+  if (format_segments(path, seg_base, &segments, &full_segments, &remaining,
+        &size_of_segments)) {
+
+    rewind(fp);
+    fflush(fp);
+
+    if (ftruncate(fileno(fp), 0) < 0)
+    {
+      debugf("ftruncate failed.  I don't know what to do about that.");
+      abort();
+    }
+
+    run_segment_threads("GET", segments, full_segments, remaining, fp,
+            seg_base, size_of_segments);
+    return 1;
+  }
+
  int response = send_request("GET", encoded, fp, NULL, NULL);
  curl_free(encoded);
  fflush(fp);
@@ -345,9 +686,14 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list)
    curl_free(encoded_object);
  }

-  response = send_request("GET", container, NULL, xmlctx, NULL);
-  xmlParseChunk(xmlctx, "", 0, 1);
-  if (xmlctx->wellFormed && response >= 200 && response < 300)
+  if ((!strcmp(path, "") || !strcmp(path, "/")) && *override_storage_url)
+    response = 404;
+  else
+    response = send_request("GET", container, NULL, xmlctx, NULL);
+
+  if (response >= 200 && response < 300)
+    xmlParseChunk(xmlctx, "", 0, 1);
+  if (response >= 200 && response < 300 && xmlctx->wellFormed )
  {
    xmlNode *root_element = xmlDocGetRootElement(xmlctx->myDoc);
    for (onode = root_element->children; onode; onode = onode->next)
@@ -405,6 +751,8 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list)
        de->isdir = de->content_type &&
            ((strstr(de->content_type, "application/folder") != NULL) ||
             (strstr(de->content_type, "application/directory") != NULL));
+        de->islink = de->content_type &&
+            ((strstr(de->content_type, "application/link") != NULL));
        if (de->isdir)
        {
          if (!strncasecmp(de->name, last_subdir, sizeof(last_subdir)))
@@ -424,6 +772,25 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list)
    }
    retval = 1;
  }
+  else if ((!strcmp(path, "") || !strcmp(path, "/")) && *override_storage_url) {
+    entry_count = 1;
+
+    dir_entry *de = (dir_entry *)malloc(sizeof(dir_entry));
+    de->name = strdup(public_container);
+    struct tm last_modified;
+    strptime("1388434648.01238", "%FT%T", &last_modified);
+    de->last_modified = mktime(&last_modified);
+    de->content_type = strdup("application/directory");
+    if (asprintf(&(de->full_name), "%s/%s", path, de->name) < 0)
+      de->full_name = NULL;
+
+    de->isdir = 1;
+    de->islink = 0;
+    de->size = 4096;
+    de->next = *dir_list;
+    *dir_list = de;
+    retval = 1;
+  }

  debugf("entry count: %d", entry_count);

@@ -447,6 +814,28 @@ void cloudfs_free_dir_list(dir_entry *dir_list)

 int cloudfs_delete_object(const char *path)
 {
+
+  char seg_base[MAX_URL_SIZE] = "";
+
+  long segments;
+  long full_segments;
+  long remaining;
+  long size_of_segments;
+
+  if (format_segments(path, seg_base, &segments, &full_segments, &remaining,
+        &size_of_segments)) {
+    int response;
+    int i;
+    char seg_path[MAX_URL_SIZE] = "";
+    for (i = 0; i < segments; i++) {
+      snprintf(seg_path, MAX_URL_SIZE, "%s%08i", seg_base, i);
+      char *encoded = curl_escape(seg_path, 0);
+      response = send_request("DELETE", encoded, NULL, NULL, NULL);
+      if (response < 200 || response >= 300)
+        return 0;
+    }
+  }
+
  char *encoded = curl_escape(path, 0);
  int response = send_request("DELETE", encoded, NULL, NULL, NULL);
  curl_free(encoded);
@@ -474,7 +863,20 @@ int cloudfs_statfs(const char *path, struct statvfs *stat)

  debugf("Assigning statvfs values from cache.");
  *stat = statcache;
+  return (response >= 200 && response < 300);
+}

+int cloudfs_create_symlink(const char *src, const char *dst)
+{
+  char *dst_encoded = curl_escape(dst, 0);
+
+  FILE *lnk = tmpfile();
+
+  fwrite(src, 1, strlen(src), lnk);
+  fwrite("\0", 1, 1, lnk);
+  int response = send_request("MKLINK", dst_encoded, lnk, NULL, NULL);
+  curl_free(dst_encoded);
+  fclose(lnk);
  return (response >= 200 && response < 300);
 }

@@ -5,7 +5,7 @@
 #include <curl/easy.h>

 #define BUFFER_INITIAL_SIZE 4096
-#define MAX_HEADER_SIZE 4096
+#define MAX_HEADER_SIZE 8192
 #define MAX_PATH_SIZE (1024 + 256 + 3)
 #define MAX_URL_SIZE (MAX_PATH_SIZE * 3)
 #define USER_AGENT "CloudFuse"
@@ -21,12 +21,18 @@ typedef struct dir_entry
  off_t size;
  time_t last_modified;
  int isdir;
+  int islink;
  struct dir_entry *next;
 } dir_entry;

 typedef struct options {
    char cache_timeout[OPTION_SIZE];
    char verify_ssl[OPTION_SIZE];
+    char segment_size[OPTION_SIZE];
+    char segment_above[OPTION_SIZE];
+    char storage_url[OPTION_SIZE];
+    char container[OPTION_SIZE];
+    char temp_dir[OPTION_SIZE];
    char client_id[OPTION_SIZE];
    char client_secret[OPTION_SIZE];
    char refresh_token[OPTION_SIZE];
@@ -35,11 +41,29 @@ typedef struct options {
 void cloudfs_init(void);
 void cloudfs_set_credentials(char *client_id, char *client_secret, char *refresh_token);
 int cloudfs_connect(void);
+
+struct segment_info
+{
+    FILE *fp;
+    int part;
+    long size;
+    long segment_size;
+    char *seg_base;
+    const char *method;
+};
+
+long segment_size;
+long segment_above;
+
+char *override_storage_url;
+char *public_container;
+
 int cloudfs_object_read_fp(const char *path, FILE *fp);
 int cloudfs_object_write_fp(const char *path, FILE *fp);
 int cloudfs_list_directory(const char *path, dir_entry **);
 int cloudfs_delete_object(const char *path);
 int cloudfs_copy_object(const char *src, const char *dst);
+int cloudfs_create_symlink(const char *src, const char *dst);
 int cloudfs_create_directory(const char *label);
 int cloudfs_object_truncate(const char *path, off_t size);
 off_t cloudfs_file_size(int fd);
@@ -18,6 +18,7 @@
 #include "config.h"

 static int cache_timeout;
+static char *temp_dir;

 typedef struct dir_cache
 {
@@ -86,7 +87,7 @@ static int caching_list_directory(const char *path, dir_entry **list)
  return 1;
 }

-static void update_dir_cache(const char *path, off_t size, int isdir)
+static void update_dir_cache(const char *path, off_t size, int isdir, int islink)
 {
  pthread_mutex_lock(&dmut);
  dir_cache *cw;
@@ -109,9 +110,23 @@ static void update_dir_cache(const char *path, off_t size, int isdir)
      de = (dir_entry *)malloc(sizeof(dir_entry));
      de->size = size;
      de->isdir = isdir;
+      de->islink = islink;
      de->name = strdup(&path[strlen(cw->path)+1]);
      de->full_name = strdup(path);
-      de->content_type = strdup(isdir ? "application/directory" : "application/octet-stream");
+
+      if (isdir)
+      {
+        de->content_type = strdup("application/link");
+      }
+      if(islink)
+      {
+        de->content_type = strdup("application/directory");
+      }
+      else
+      {
+
+        de->content_type = strdup("application/octet-stream");
+      }
      de->last_modified = time(NULL);
      de->next = cw->entries;
      cw->entries = de;
@@ -204,6 +219,15 @@ static int cfs_getattr(const char *path, struct stat *stbuf)
    stbuf->st_mode = S_IFDIR | 0755;
    stbuf->st_nlink = 2;
  }
+  else if (de->islink)
+  {
+    stbuf->st_size = 1;
+    stbuf->st_mode = S_IFLNK | 0755;
+    stbuf->st_nlink = 1;
+    stbuf->st_size = de->size;
+    /* calc. blocks as if 4K blocksize filesystem; stat uses units of 512B */
+    stbuf->st_blocks = ((4095 + de->size) / 4096) * 8;
+  }
  else
  {
    stbuf->st_size = de->size;
@@ -244,7 +268,7 @@ static int cfs_mkdir(const char *path, mode_t mode)
 {
  if (cloudfs_create_directory(path))
  {
-    update_dir_cache(path, 0, 1);
+    update_dir_cache(path, 0, 1, 0);
    return 0;
  }
  return -ENOENT;
@@ -258,24 +282,90 @@ static int cfs_create(const char *path, mode_t mode, struct fuse_file_info *info
  fclose(temp_file);
  of->flags = info->flags;
  info->fh = (uintptr_t)of;
-  update_dir_cache(path, 0, 0);
+  update_dir_cache(path, 0, 0, 0);
  info->direct_io = 1;
  return 0;
 }

 static int cfs_open(const char *path, struct fuse_file_info *info)
 {
-  FILE *temp_file = tmpfile();
+  //TODO: need to clean this up and reduce the duplicated code
+  if (!*temp_dir) {
+
+    FILE *temp_file = tmpfile();
+    dir_entry *de = path_info(path);
+    if (!(info->flags & O_WRONLY))
+    {
+      if (!cloudfs_object_write_fp(path, temp_file))
+      {
+        fclose(temp_file);
+        return -ENOENT;
+      }
+      update_dir_cache(path, (de ? de->size : 0), 0, 0);
+    }
+    openfile *of = (openfile *)malloc(sizeof(openfile));
+    of->fd = dup(fileno(temp_file));
+    fclose(temp_file);
+    of->flags = info->flags;
+    info->fh = (uintptr_t)of;
+    info->direct_io = 1;
+    return 0;
+  }
+
+
+  FILE *temp_file;
+
+  char tmp_path[PATH_MAX];
+  strncpy(tmp_path, path, PATH_MAX);
+
+  char *pch;
+  while((pch = strchr(tmp_path, '/'))) {
+    *pch = '.';
+  }
+
+  char file_path[PATH_MAX];
+  snprintf(file_path, PATH_MAX, "%s/.cloudfuse%ld-%s", temp_dir,
+      (long)getpid(), tmp_path);
+
  dir_entry *de = path_info(path);
-  if (!(info->flags & O_WRONLY))
-  {
+
+  if(access(file_path, F_OK) != -1) {
+    temp_file = fopen(file_path, "r");
+    update_dir_cache(path, (de ? de->size : 0), 0, 0);
+    // file exists
+  } else if (!(info->flags & O_WRONLY)) {
+    // we need to lock on the filename another process could open the file
+    // while we are writing to it and then only read part of the file
+
+    // duplicate the directory caching datastructure to make the code easier
+    // to understand.
+
+    // each file in the cache needs:
+    //  filename, is_writing, last_closed, is_removing
+    // the first time a file is opened a new entry is created in the cache
+    // setting the filename and is_writing to true.  This check needs to be
+    // wrapped with a lock.
+    //
+    // each time a file is closed we set the last_closed for the file to now
+    // and we check the cache for files whose last
+    // closed is greater than cache_timeout, then start a new thread rming
+    // that file.
+
+    // TODO: just to prevent this craziness for now
+    if (*temp_dir)
+        temp_file = fopen(file_path, "w+b");
+    else
+        temp_file = tmpfile();
+
+
    if (!cloudfs_object_write_fp(path, temp_file))
    {
      fclose(temp_file);
      return -ENOENT;
    }
-    update_dir_cache(path, (de ? de->size : 0), 0);
+    update_dir_cache(path, (de ? de->size : 0), 0, 0);
  }
+
  openfile *of = (openfile *)malloc(sizeof(openfile));
  of->fd = dup(fileno(temp_file));
  fclose(temp_file);
@@ -295,7 +385,7 @@ static int cfs_flush(const char *path, struct fuse_file_info *info)
  openfile *of = (openfile *)(uintptr_t)info->fh;
  if (of)
  {
-    update_dir_cache(path, cloudfs_file_size(of->fd), 0);
+    update_dir_cache(path, cloudfs_file_size(of->fd), 0, 0);
    if (of->flags & O_RDWR || of->flags & O_WRONLY)
    {
      FILE *fp = fdopen(dup(of->fd), "r");
@@ -336,13 +426,13 @@ static int cfs_ftruncate(const char *path, off_t size, struct fuse_file_info *in
  if (ftruncate(of->fd, size))
    return -errno;
  lseek(of->fd, 0, SEEK_SET);
-  update_dir_cache(path, size, 0);
+  update_dir_cache(path, size, 0, 0);
  return 0;
 }

 static int cfs_write(const char *path, const char *buf, size_t length, off_t offset, struct fuse_file_info *info)
 {
-  update_dir_cache(path, offset + length, 0);
+  update_dir_cache(path, offset + length, 0, 0);
  return pwrite(((openfile *)(uintptr_t)info->fh)->fd, buf, length, offset);
 }

@@ -399,12 +489,41 @@ static int cfs_rename(const char *src, const char *dst)
  if (cloudfs_copy_object(src, dst))
  {
    /* FIXME this isn't quite right as doesn't preserve last modified */
-    update_dir_cache(dst, src_de->size, 0);
+    update_dir_cache(dst, src_de->size, 0, 0);
    return cfs_unlink(src);
  }
  return -EIO;
 }

+static int cfs_symlink(const char *src, const char *dst)
+{
+  if(cloudfs_create_symlink(src, dst))
+  {
+    update_dir_cache(dst, 1, 0, 1);
+    return 0;
+  }
+  return -EIO;
+}
+
+static int cfs_readlink(const char* path, char* buf, size_t size)
+{
+  FILE *temp_file = tmpfile();
+  int ret = 0;
+
+  if (!cloudfs_object_write_fp(path, temp_file))
+  {
+      ret = -ENOENT;
+  }
+
+  if (!pread(fileno(temp_file), buf, size, 0))
+  {
+      ret = -ENOENT;
+  }
+
+  fclose(temp_file);
+  return ret;
+}
+
 static void *cfs_init(struct fuse_conn_info *conn)
 {
  signal(SIGPIPE, SIG_IGN);
@@ -425,6 +544,12 @@ char *get_home_dir()
 FuseOptions options = {
    .cache_timeout = "600",
    .verify_ssl = "true",
+    .segment_size = "1073741824",
+    .segment_above = "2147483648",
+    .storage_url = "",
+    .container = "",
+    //.temp_dir = "/tmp/",
+    .temp_dir = "",
    .client_id = "",
    .client_secret = "",
    .refresh_token = ""
@@ -434,6 +559,11 @@ int parse_option(void *data, const char *arg, int key, struct fuse_args *outargs
 {
  if (sscanf(arg, " cache_timeout = %[^\r\n ]", options.cache_timeout) ||
      sscanf(arg, " verify_ssl = %[^\r\n ]", options.verify_ssl) ||
+      sscanf(arg, " segment_above = %[^\r\n ]", options.segment_above) ||
+      sscanf(arg, " segment_size = %[^\r\n ]", options.segment_size) ||
+      sscanf(arg, " storage_url = %[^\r\n ]", options.storage_url) ||
+      sscanf(arg, " container = %[^\r\n ]", options.container) ||
+      sscanf(arg, " temp_dir = %[^\r\n ]", options.temp_dir) ||
      sscanf(arg, " client_id = %[^\r\n ]", options.client_id) ||
      sscanf(arg, " client_secret = %[^\r\n ]", options.client_secret) ||
      sscanf(arg, " refresh_token = %[^\r\n ]", options.refresh_token))
@@ -461,6 +591,12 @@ int main(int argc, char **argv)
  fuse_opt_parse(&args, &options, NULL, parse_option);

  cache_timeout = atoi(options.cache_timeout);
+  segment_size = atoll(options.segment_size);
+  segment_above = atoll(options.segment_above);
+  // this is ok since main is on the stack during the entire execution
+  override_storage_url = options.storage_url;
+  public_container = options.container;
+  temp_dir = options.temp_dir;

  if (!*options.client_id || !*options.client_secret || !*options.refresh_token)
  {
@@ -473,6 +609,11 @@ int main(int argc, char **argv)
    fprintf(stderr, "The following settings are optional:\n\n");
    fprintf(stderr, "  cache_timeout=[Seconds for directory caching, default 600]\n");
    fprintf(stderr, "  verify_ssl=[False to disable SSL cert verification]\n");
+    fprintf(stderr, "  segment_size=[Size to use when creating DLOs, default 1073741824]\n");
+    fprintf(stderr, "  segment_above=[File size at which to use segments, defult 2147483648]\n");
+    fprintf(stderr, "  storage_url=[Storage URL for other tenant to view container]\n");
+    fprintf(stderr, "  container=[Public container to view of tenant specified by storage_url]\n");
+    fprintf(stderr, "  temp_dir=[Directory to store temp files]\n");

    return 1;
  }
@@ -482,6 +623,7 @@ int main(int argc, char **argv)
  cloudfs_verify_ssl(!strcasecmp(options.verify_ssl, "true"));

  cloudfs_set_credentials(options.client_id, options.client_secret, options.refresh_token);
+
  if (!cloudfs_connect())
  {
    fprintf(stderr, "Failed to authenticate.\n");
@@ -513,6 +655,8 @@ int main(int argc, char **argv)
    .chmod = cfs_chmod,
    .chown = cfs_chown,
    .rename = cfs_rename,
+    .symlink = cfs_symlink,
+    .readlink = cfs_readlink,
    .init = cfs_init,
  };

@@ -0,0 +1,16 @@
+cloudfuse for Debian
+--------------------
+
+Please note that the simplest way to allow mounting of Swift shares is to
+add the user to the fuse group (which will be created by the fuse-utils
+package).
+
+To add user 'foo' to the 'fuse' group type the following command (with root
+privileges):
+
+	adduser foo fuse
+
+If the user is currently logged in he/she should log out and log in 
+in order for this change to take effect.
+
+ -- Mike Mestnik <cheako+apt_repo@mikemestnik.net>  Mon, 04 Jun 2012 13:47:28 -0500
@@ -0,0 +1,12 @@
+cloudfuse (0.9-1ubuntu1) precise; urgency=low
+
+  * Imported Upstream version 0.9
+  * Remove Debian patches already committed upstream
+
+ -- pataquets <amontero@tinet.org>  Wed, 15 Jan 2014 21:07:18 +0100
+
+cloudfuse (0.1-1) unstable; urgency=low
+
+  * Initial release
+
+ -- Mike Mestnik <cheako+apt_repo@mikemestnik.net>  Mon, 04 Jun 2012 13:47:28 -0500
@@ -0,0 +1 @@
+8
@@ -0,0 +1,20 @@
+Source: cloudfuse
+Section: utils
+Priority: optional
+Maintainer: Mike Mestnik <cheako+apt_repo@mikemestnik.net>
+Build-Depends: debhelper (>= 8.0.0), autotools-dev, libcurl4-openssl-dev, libxml2-dev, libssl-dev, libfuse-dev
+Standards-Version: 3.9.2
+#Vcs-Git: git://git.debian.org/collab-maint/cloudfuse.git
+#Vcs-Browser: http://git.debian.org/?p=collab-maint/cloudfuse.git;a=summary
+
+Package: cloudfuse
+Architecture: any
+Depends: ${shlibs:Depends}, ${misc:Depends}, fuse (>= 2.7) [linux-any] | fuse4bsd [kfreebsd-any]
+Description: filesystem client based on Swift as used by Rackspace.
+ cloudfuse is a FUSE application which provides access to Rackspace's
+ Cloud Files (or any installation of Swift).
+ .
+ cloudfuse is FUSE (Filesystem in USErspace).
+ .
+ cloudfuse may work with Openstack's Swift, but is not provided by Openstack.
+
@@ -0,0 +1,76 @@
+Format: http://dep.debian.net/deps/dep5
+Upstream-Name: cloudfuse
+
+Files: *
+Copyright: 2009 Michael Barton <mike@weirdlooking.com>
+License: MIT
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+ .
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+ .
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+
+Files: debian/*
+Copyright: 2012 Mike Mestnik <cheako+apt_repo@mikemestnik.net>
+License: GPL-2+
+ This package is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+ .
+ This package is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>
+ .
+ On Debian systems, the complete text of the GNU General
+ Public License version 2 can be found in "/usr/share/common-licenses/GPL-2".
+
+Files: configure
+Copyright: 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001,
+ 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010
+ Free Software Foundation, Inc.
+License: No problem Bugroff
+ configure script is free software; the Free Software Foundation
+ gives unlimited permission to copy, distribute and modify it.
+
+Files: install-sh
+Copyright: 1994 X Consortium
+License: MIT
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to
+ deal in the Software without restriction, including without limitation the
+ rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ sell copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+ .
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+ .
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ X CONSORTIUM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNEC-
+ TION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ .
+ Except as contained in this notice, the name of the X Consortium shall not
+ be used in advertising or otherwise to promote the sale, use or other deal-
+ ings in this Software without prior written authorization from the X Consor-
+ tium.
+
@@ -0,0 +1 @@
+README
@@ -0,0 +1,13 @@
+#!/usr/bin/make -f
+# -*- makefile -*-
+# Sample debian/rules that uses debhelper.
+# This file was originally written by Joey Hess and Craig Small.
+# As a special exception, when this file is copied by dh-make into a
+# dh-make output file, you may use that output file without restriction.
+# This special exception was added by Craig Small in version 0.37 of dh-make.
+
+# Uncomment this to turn on verbose mode.
+#export DH_VERBOSE=1
+
+%:
+	dh $@ 
@@ -0,0 +1 @@
+3.0 (quilt)
@@ -0,0 +1,2 @@
+version=3
+http://githubredir.debian.net/github/redbo/cloudfuse (.*).tar.gz