Merge remote-tracking branch 'origin/dlo' into next

Conflicts: cloudfsapi.c cloudfsapi.h cloudfuse.c
2026-08-02 02:42:00 +02:00 · 2015-03-15 18:23:07 +01:00
parent 4e1bda3ed5 568a8f6dbe
commit db0c086327
13 changed files with 749 additions and 32 deletions
@@ -2,6 +2,11 @@
 HubicFuse is a FUSE application which provides access to Hubic's
 cloud files via a mount-point.

+This version contains support for DLO, symlinks and support to see other
+tenant's containers. Those features are coming from:
+
+   https://github.com/LabAdvComp/cloudfuse.git
+
 BUILDING:

    You'll need libcurl, fuse, libssl, and libxml2 (and probably their dev
@@ -79,16 +84,16 @@ BUGS/SHORTCOMINGS:

 AWESOME CONTRIBUTORS:

-    * Pascal Obry                                  https://github.com/TurboGit
-    * Tim Dysinger                                 https://github.com/dysinger
-    * Chris Wedgwood                               https://github.com/cwedgwood
-    * Nick Craig-Wood                              https://github.com/ncw
-    * Dillon Amburgey                              https://github.com/dillona
-    * Manfred Touron                               https://github.com/moul
-    * David Brownlee                               https://github.com/abs0
-    * Mike Lundy                                   https://github.com/novas0x2a
-    * justinb                                      https://github.com/justinsb
-
+    * Pascal Obry                        https://github.com/TurboGit
+    * Tim Dysinger                       https://github.com/dysinger
+    * Chris Wedgwood                     https://github.com/cwedgwood
+    * Nick Craig-Wood                    https://github.com/ncw
+    * Dillon Amburgey                    https://github.com/dillona
+    * Manfred Touron                     https://github.com/moul
+    * David Brownlee                     https://github.com/abs0
+    * Mike Lundy                         https://github.com/novas0x2a
+    * justinb                            https://github.com/justinsb
+    * Matt Greenway                      https://github.com/LabAdvComp

 Thanks, and I hope you find it useful.

@@ -4,6 +4,7 @@
 #include <stdarg.h>
 #include <stdlib.h>
 #include <sys/stat.h>
+#include <fcntl.h>
 #ifdef __linux__
 #include <alloca.h>
 #endif
@@ -16,6 +17,8 @@
 #include <openssl/bio.h>
 #include <openssl/evp.h>
 #include <json.h>
+#include <libxml/xpath.h>
+#include <libxml/xpathInternals.h>
 #include "cloudfsapi.h"
 #include "config.h"
 #include <fuse.h>
@@ -27,6 +30,13 @@

 #define MAX_FILES 10000

+// 64 bit time + nanoseconds
+#define TIME_CHARS 32
+
+// size of buffer for writing to disk look at ioblksize.h in coreutils
+// and try some values on your own system if you want the best performance
+#define DISK_BUFF_SIZE 32768
+
 static char storage_url[MAX_URL_SIZE];
 static char storage_token[MAX_HEADER_SIZE];
 static pthread_mutex_t pool_mut;
@@ -92,7 +102,8 @@ static void return_connection(CURL *curl)
  pthread_mutex_unlock(&pool_mut);
 }

-static void add_header(curl_slist **headers, const char *name, const char *value)
+static void add_header(curl_slist **headers, const char *name,
+                       const char *value)
 {
  char x_header[MAX_HEADER_SIZE];
  snprintf(x_header, sizeof(x_header), "%s: %s", name, value);
@@ -126,8 +137,39 @@ static size_t header_dispatch(void *ptr, size_t size, size_t nmemb, void *stream
  return size * nmemb;
 }

-static int send_request(char *method, const char *path, FILE *fp,
-                        xmlParserCtxtPtr xmlctx, curl_slist *extra_headers)
+static size_t rw_callback(size_t (*rw)(void*, size_t, size_t, FILE*), void *ptr,
+        size_t size, size_t nmemb, void *userp)
+{
+    struct segment_info *info = (struct segment_info *)userp;
+    size_t mem = size * nmemb;
+
+    if (mem < 1 || info->size < 1)
+      return 0;
+
+    size_t amt_read = rw(ptr, 1, info->size < mem ? info->size : mem, info->fp);
+    info->size -= amt_read;
+
+    return amt_read;
+}
+
+size_t fwrite2(void *ptr, size_t size, size_t nmemb, FILE *filep)
+{
+    return fwrite((const void*)ptr, size, nmemb, filep);
+}
+
+static size_t read_callback(void *ptr, size_t size, size_t nmemb, void *userp)
+{
+    return rw_callback(fread, ptr, size, nmemb, userp);
+}
+
+static size_t write_callback(void *ptr, size_t size, size_t nmemb, void *userp)
+{
+   return rw_callback(fwrite2, ptr, size, nmemb, userp);
+}
+
+static int send_request_size(const char *method, const char *path, void *fp,
+                        xmlParserCtxtPtr xmlctx, curl_slist *extra_headers,
+                        off_t file_size, int is_segment)
 {
  char url[MAX_URL_SIZE];
  char *slash;
@@ -172,16 +214,31 @@ static int send_request(char *method, const char *path, FILE *fp,
      curl_easy_setopt(curl, CURLOPT_INFILESIZE, 0);
      add_header(&headers, "Content-Type", "application/directory");
    }
-    else if (!strcasecmp(method, "PUT") && fp)
+    else if (!strcasecmp(method, "MKLINK") && fp)
    {
      rewind(fp);
      curl_easy_setopt(curl, CURLOPT_UPLOAD, 1);
-      curl_easy_setopt(curl, CURLOPT_INFILESIZE, cloudfs_file_size(fileno(fp)));
+      //curl_easy_setopt(curl, CURLOPT_INFILESIZE, 0);
+      curl_easy_setopt(curl, CURLOPT_INFILESIZE, file_size);
      curl_easy_setopt(curl, CURLOPT_READDATA, fp);
+      add_header(&headers, "Content-Type", "application/link");
+    }
+    else if (!strcasecmp(method, "PUT") && fp)
+    {
+      curl_easy_setopt(curl, CURLOPT_UPLOAD, 1);
+      curl_easy_setopt(curl, CURLOPT_INFILESIZE, file_size);
+      curl_easy_setopt(curl, CURLOPT_READDATA, fp);
+      if (is_segment)
+        curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_callback);
    }
    else if (!strcasecmp(method, "GET"))
    {
-      if (fp)
+      if (is_segment)
+      {
+        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback);
+        curl_easy_setopt(curl, CURLOPT_WRITEDATA, fp);
+      }
+      else if (fp)
      {
        rewind(fp); // make sure the file is ready for a-writin'
        fflush(fp);
@@ -227,6 +284,196 @@ static int send_request(char *method, const char *path, FILE *fp,
  return response;
 }

+static int send_request(char *method, const char *path, FILE *fp,
+                        xmlParserCtxtPtr xmlctx, curl_slist *extra_headers)
+{
+
+    long flen = 0;
+    if (fp) {
+      // if we don't flush the size will probably be zero
+      fflush(fp);
+      flen = cloudfs_file_size(fileno(fp));
+    }
+
+    return send_request_size(method, path, fp, xmlctx, extra_headers, flen, 0);
+
+}
+
+void *upload_segment(void *seginfo)
+{
+  char seg_path[MAX_URL_SIZE];
+  struct segment_info *info;
+  info = (struct segment_info *)seginfo;
+
+  fseek(info->fp, info->part * info->segment_size, SEEK_SET);
+  setvbuf(info->fp, NULL, _IOFBF, DISK_BUFF_SIZE);
+
+  snprintf(seg_path, MAX_URL_SIZE, "%s%08i", info->seg_base, info->part);
+  char *encoded = curl_escape(seg_path, 0);
+
+  int response = send_request_size(info->method, encoded, info, NULL, NULL,
+      info->size, 1);
+
+  if (!(response >= 200 && response < 300))
+    fprintf(stderr, "Segment upload %s failed with response %d", seg_path,
+         response);
+
+  curl_free(encoded);
+  fclose(info->fp);
+  pthread_exit(NULL);
+}
+
+// segment_size is the globabl config variable and size_of_segment is local
+//TODO: return whether the upload/download failed or not
+void run_segment_threads(const char *method, int segments, int full_segments, int remaining,
+        FILE *fp, char *seg_base, int size_of_segments)
+{
+    char file_path[PATH_MAX] = "";
+    struct segment_info *info = (struct segment_info *)
+            malloc(segments * sizeof(struct segment_info));
+
+    pthread_t *threads = (pthread_t *)malloc(segments * sizeof(pthread_t));
+#ifdef __linux__
+    snprintf(file_path, PATH_MAX, "/proc/self/fd/%d", fileno(fp));
+#else
+    //TODO: I haven't actually tested this
+    if (fcntl(fileno(fp), F_GETPATH, file_path) == -1)
+      fprintf(stderr, "couldn't get the path name\n");
+#endif
+
+    int i;
+    for (i = 0; i < segments; i++) {
+      info[i].method = method;
+      info[i].fp = fopen(file_path, method[0] == 'G' ? "r+" : "r");
+      info[i].part = i;
+      info[i].segment_size = size_of_segments;
+      info[i].size = i < full_segments ? size_of_segments : remaining;
+      info[i].seg_base = seg_base;
+      pthread_create(&threads[i], NULL, upload_segment, (void *)&(info[i]));
+    }
+
+    for (i = 0; i < segments; i++) {
+      pthread_join(threads[i], NULL);
+    }
+    free(info);
+    free(threads);
+}
+
+void split_path(const char *path, char *seg_base, char *container,
+        char *object)
+{
+  char *string = strdup(path);
+
+  snprintf(seg_base, MAX_URL_SIZE, "%s", strsep(&string, "/"));
+
+  strncat(container, strsep(&string, "/"),
+      MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE));
+
+  char *_object = strsep(&string, "/");
+
+  char *remstr;
+
+  while (remstr = strsep(&string, "/")) {
+      strncat(container, "/",
+          MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE));
+      strncat(container, _object,
+          MAX_URL_SIZE - strnlen(container, MAX_URL_SIZE));
+      _object = remstr;
+  }
+
+  strncpy(object, _object, MAX_URL_SIZE);
+  free(string);
+}
+
+int internal_is_segmented(const char *seg_path, const char *object)
+{
+  dir_entry *seg_dir;
+  if (cloudfs_list_directory(seg_path, &seg_dir)) {
+    if (seg_dir && seg_dir->isdir) {
+        do {
+            if (!strncmp(seg_dir->name, object, MAX_URL_SIZE)) {
+                return 1;
+            }
+        } while ((seg_dir = seg_dir->next));
+    }
+  }
+  return 0;
+}
+
+int is_segmented(const char *path)
+{
+  char container[MAX_URL_SIZE] = "";
+  char object[MAX_URL_SIZE] = "";
+  char seg_base[MAX_URL_SIZE] = "";
+
+  split_path(path, seg_base, container, object);
+
+  char seg_path[MAX_URL_SIZE];
+  snprintf(seg_path, MAX_URL_SIZE, "%s/%s_segments", seg_base, container);
+
+  return internal_is_segmented(seg_path, object);
+}
+
+
+int format_segments(const char *path, char * seg_base,  long *segments,
+        long *full_segments, long *remaining, long *size_of_segments)
+{
+
+  char container[MAX_URL_SIZE] = "";
+  char object[MAX_URL_SIZE] = "";
+
+  split_path(path, seg_base, container, object);
+
+  char seg_path[MAX_URL_SIZE];
+  snprintf(seg_path, MAX_URL_SIZE, "%s/%s_segments", seg_base, container);
+
+  if (internal_is_segmented(seg_path, object)) {
+    char manifest[MAX_URL_SIZE];
+    dir_entry *seg_dir;
+
+    snprintf(manifest, MAX_URL_SIZE, "%s/%s", seg_path, object);
+    if (!cloudfs_list_directory(manifest, &seg_dir))
+      return 0;
+
+    // snprintf seesaw between manifest and seg_path to get
+    // the total_size and the segment size as well as the actual objects
+    char *timestamp = seg_dir->name;
+    snprintf(seg_path, MAX_URL_SIZE, "%s/%s", manifest, timestamp);
+    if (!cloudfs_list_directory(seg_path, &seg_dir))
+      return 0;
+
+    char *str_size = seg_dir->name;
+    snprintf(manifest, MAX_URL_SIZE, "%s/%s", seg_path, str_size);
+    if (!cloudfs_list_directory(manifest, &seg_dir))
+      return 0;
+
+    char *str_segment = seg_dir->name;
+    snprintf(seg_path, MAX_URL_SIZE, "%s/%s", manifest, str_segment);
+    if (!cloudfs_list_directory(seg_path, &seg_dir))
+      return 0;
+
+    long total_size = strtoll(str_size, NULL, 10);
+    *size_of_segments = strtoll(str_segment, NULL, 10);
+
+    *remaining = total_size % *size_of_segments;
+    *full_segments = total_size / *size_of_segments;
+    *segments = *full_segments + (*remaining > 0);
+
+    snprintf(manifest, MAX_URL_SIZE, "%s_segments/%s/%s/%s/%s/",
+        container, object, timestamp, str_size, str_segment);
+
+    char tmp[MAX_URL_SIZE];
+    strncpy(tmp, seg_base, MAX_URL_SIZE);
+    snprintf(seg_base, MAX_URL_SIZE, "%s/%s", tmp, manifest);
+
+    return 1;
+  }
+
+  else {
+    return 0;
+  }
+}
+
 /*
 * Public interface
 */
@@ -234,6 +481,7 @@ static int send_request(char *method, const char *path, FILE *fp,
 void cloudfs_init()
 {
  LIBXML_TEST_VERSION
+  xmlXPathInit();
  curl_global_init(CURL_GLOBAL_ALL);
  pthread_mutex_init(&pool_mut, NULL);
  curl_version_info_data *cvid = curl_version_info(CURLVERSION_NOW);
@@ -266,7 +514,75 @@ void cloudfs_init()

 int cloudfs_object_read_fp(const char *path, FILE *fp)
 {
+
+  long flen;
  fflush(fp);
+
+  // determine the size of the file and segment if it is above the threshhold
+  fseek(fp, 0, SEEK_END);
+  flen = ftell(fp);
+
+  // delete the previously uploaded segments
+  if (is_segmented(path)) {
+    if (!cloudfs_delete_object(path))
+      debugf("Couldn't delete one of the existing files while uploading.");
+  }
+
+  if (flen >= segment_above) {
+    int i;
+    long remaining = flen % segment_size;
+    int full_segments = flen / segment_size;
+    int segments = full_segments + (remaining > 0);
+
+    // The best we can do here is to get the current time that way tools that
+    // use the mtime can at least check if the file was changing after now
+    struct timespec now;
+    clock_gettime(CLOCK_REALTIME, &now);
+
+    char string_float[TIME_CHARS];
+    snprintf(string_float, TIME_CHARS, "%lu.%lu", now.tv_sec, now.tv_nsec);
+
+    char meta_mtime[TIME_CHARS];
+    snprintf(meta_mtime, TIME_CHARS, "%f", atof(string_float));
+
+    char seg_base[MAX_URL_SIZE] = "";
+
+    char container[MAX_URL_SIZE] = "";
+    char object[MAX_URL_SIZE] = "";
+
+    split_path(path, seg_base, container, object);
+
+    char manifest[MAX_URL_SIZE];
+    snprintf(manifest, MAX_URL_SIZE, "%s_segments", container);
+
+    // create the segments container
+    cloudfs_create_directory(manifest);
+
+    // reusing manifest
+    snprintf(manifest, MAX_URL_SIZE, "%s_segments/%s/%s/%ld/%ld/",
+        container, object, meta_mtime, flen, segment_size);
+
+    char tmp[MAX_URL_SIZE];
+    strncpy(tmp, seg_base, MAX_URL_SIZE);
+    snprintf(seg_base, MAX_URL_SIZE, "%s/%s", tmp, manifest);
+
+    run_segment_threads("PUT", segments, full_segments, remaining, fp,
+            seg_base, segment_size);
+
+    char *encoded = curl_escape(path, 0);
+    curl_slist *headers = NULL;
+    add_header(&headers, "x-object-manifest", manifest);
+    add_header(&headers, "x-object-meta-mtime", meta_mtime);
+    add_header(&headers, "Content-Length", "0");
+
+    int response = send_request_size("PUT", encoded, NULL, NULL, headers, 0, 0);
+    curl_slist_free_all(headers);
+
+    curl_free(encoded);
+    return (response >= 200 && response < 300);
+
+  }
+
  rewind(fp);
  char *encoded = curl_escape(path, 0);
  int response = send_request("PUT", encoded, fp, NULL, NULL);
@@ -274,9 +590,34 @@ int cloudfs_object_read_fp(const char *path, FILE *fp)
  return (response >= 200 && response < 300);
 }

+
 int cloudfs_object_write_fp(const char *path, FILE *fp)
 {
  char *encoded = curl_escape(path, 0);
+  char seg_base[MAX_URL_SIZE] = "";
+
+  long segments;
+  long full_segments;
+  long remaining;
+  long size_of_segments;
+
+  if (format_segments(path, seg_base, &segments, &full_segments, &remaining,
+        &size_of_segments)) {
+
+    rewind(fp);
+    fflush(fp);
+
+    if (ftruncate(fileno(fp), 0) < 0)
+    {
+      debugf("ftruncate failed.  I don't know what to do about that.");
+      abort();
+    }
+
+    run_segment_threads("GET", segments, full_segments, remaining, fp,
+            seg_base, size_of_segments);
+    return 1;
+  }
+
  int response = send_request("GET", encoded, fp, NULL, NULL);
  curl_free(encoded);
  fflush(fp);
@@ -345,9 +686,14 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list)
    curl_free(encoded_object);
  }

-  response = send_request("GET", container, NULL, xmlctx, NULL);
-  xmlParseChunk(xmlctx, "", 0, 1);
-  if (xmlctx->wellFormed && response >= 200 && response < 300)
+  if ((!strcmp(path, "") || !strcmp(path, "/")) && *override_storage_url)
+    response = 404;
+  else
+    response = send_request("GET", container, NULL, xmlctx, NULL);
+
+  if (response >= 200 && response < 300)
+    xmlParseChunk(xmlctx, "", 0, 1);
+  if (response >= 200 && response < 300 && xmlctx->wellFormed )
  {
    xmlNode *root_element = xmlDocGetRootElement(xmlctx->myDoc);
    for (onode = root_element->children; onode; onode = onode->next)
@@ -405,6 +751,8 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list)
        de->isdir = de->content_type &&
            ((strstr(de->content_type, "application/folder") != NULL) ||
             (strstr(de->content_type, "application/directory") != NULL));
+        de->islink = de->content_type &&
+            ((strstr(de->content_type, "application/link") != NULL));
        if (de->isdir)
        {
          if (!strncasecmp(de->name, last_subdir, sizeof(last_subdir)))
@@ -424,6 +772,25 @@ int cloudfs_list_directory(const char *path, dir_entry **dir_list)
    }
    retval = 1;
  }
+  else if ((!strcmp(path, "") || !strcmp(path, "/")) && *override_storage_url) {
+    entry_count = 1;
+
+    dir_entry *de = (dir_entry *)malloc(sizeof(dir_entry));
+    de->name = strdup(public_container);
+    struct tm last_modified;
+    strptime("1388434648.01238", "%FT%T", &last_modified);
+    de->last_modified = mktime(&last_modified);
+    de->content_type = strdup("application/directory");
+    if (asprintf(&(de->full_name), "%s/%s", path, de->name) < 0)
+      de->full_name = NULL;
+
+    de->isdir = 1;
+    de->islink = 0;
+    de->size = 4096;
+    de->next = *dir_list;
+    *dir_list = de;
+    retval = 1;
+  }

  debugf("entry count: %d", entry_count);

@@ -447,6 +814,28 @@ void cloudfs_free_dir_list(dir_entry *dir_list)

 int cloudfs_delete_object(const char *path)
 {
+
+  char seg_base[MAX_URL_SIZE] = "";
+
+  long segments;
+  long full_segments;
+  long remaining;
+  long size_of_segments;
+
+  if (format_segments(path, seg_base, &segments, &full_segments, &remaining,
+        &size_of_segments)) {
+    int response;
+    int i;
+    char seg_path[MAX_URL_SIZE] = "";
+    for (i = 0; i < segments; i++) {
+      snprintf(seg_path, MAX_URL_SIZE, "%s%08i", seg_base, i);
+      char *encoded = curl_escape(seg_path, 0);
+      response = send_request("DELETE", encoded, NULL, NULL, NULL);
+      if (response < 200 || response >= 300)
+        return 0;
+    }
+  }
+
  char *encoded = curl_escape(path, 0);
  int response = send_request("DELETE", encoded, NULL, NULL, NULL);
  curl_free(encoded);
@@ -474,7 +863,20 @@ int cloudfs_statfs(const char *path, struct statvfs *stat)

  debugf("Assigning statvfs values from cache.");
  *stat = statcache;
+  return (response >= 200 && response < 300);
+}

+int cloudfs_create_symlink(const char *src, const char *dst)
+{
+  char *dst_encoded = curl_escape(dst, 0);
+
+  FILE *lnk = tmpfile();
+
+  fwrite(src, 1, strlen(src), lnk);
+  fwrite("\0", 1, 1, lnk);
+  int response = send_request("MKLINK", dst_encoded, lnk, NULL, NULL);
+  curl_free(dst_encoded);
+  fclose(lnk);
  return (response >= 200 && response < 300);
 }

@@ -5,7 +5,7 @@
 #include <curl/easy.h>

 #define BUFFER_INITIAL_SIZE 4096
-#define MAX_HEADER_SIZE 4096
+#define MAX_HEADER_SIZE 8192
 #define MAX_PATH_SIZE (1024 + 256 + 3)
 #define MAX_URL_SIZE (MAX_PATH_SIZE * 3)
 #define USER_AGENT "CloudFuse"
@@ -21,12 +21,18 @@ typedef struct dir_entry
  off_t size;
  time_t last_modified;
  int isdir;
+  int islink;
  struct dir_entry *next;
 } dir_entry;

 typedef struct options {
    char cache_timeout[OPTION_SIZE];
    char verify_ssl[OPTION_SIZE];
+    char segment_size[OPTION_SIZE];
+    char segment_above[OPTION_SIZE];
+    char storage_url[OPTION_SIZE];
+    char container[OPTION_SIZE];
+    char temp_dir[OPTION_SIZE];
    char client_id[OPTION_SIZE];
    char client_secret[OPTION_SIZE];
    char refresh_token[OPTION_SIZE];
@@ -35,11 +41,29 @@ typedef struct options {
 void cloudfs_init(void);
 void cloudfs_set_credentials(char *client_id, char *client_secret, char *refresh_token);
 int cloudfs_connect(void);
+
+struct segment_info
+{
+    FILE *fp;
+    int part;
+    long size;
+    long segment_size;
+    char *seg_base;
+    const char *method;
+};
+
+long segment_size;
+long segment_above;
+
+char *override_storage_url;
+char *public_container;
+
 int cloudfs_object_read_fp(const char *path, FILE *fp);
 int cloudfs_object_write_fp(const char *path, FILE *fp);
 int cloudfs_list_directory(const char *path, dir_entry **);
 int cloudfs_delete_object(const char *path);
 int cloudfs_copy_object(const char *src, const char *dst);
+int cloudfs_create_symlink(const char *src, const char *dst);
 int cloudfs_create_directory(const char *label);
 int cloudfs_object_truncate(const char *path, off_t size);
 off_t cloudfs_file_size(int fd);
@@ -18,6 +18,7 @@
 #include "config.h"

 static int cache_timeout;
+static char *temp_dir;

 typedef struct dir_cache
 {
@@ -86,7 +87,7 @@ static int caching_list_directory(const char *path, dir_entry **list)
  return 1;
 }

-static void update_dir_cache(const char *path, off_t size, int isdir)
+static void update_dir_cache(const char *path, off_t size, int isdir, int islink)
 {
  pthread_mutex_lock(&dmut);
  dir_cache *cw;
@@ -109,9 +110,23 @@ static void update_dir_cache(const char *path, off_t size, int isdir)
      de = (dir_entry *)malloc(sizeof(dir_entry));
      de->size = size;
      de->isdir = isdir;
+      de->islink = islink;
      de->name = strdup(&path[strlen(cw->path)+1]);
      de->full_name = strdup(path);
-      de->content_type = strdup(isdir ? "application/directory" : "application/octet-stream");
+
+      if (isdir)
+      {
+        de->content_type = strdup("application/link");
+      }
+      if(islink)
+      {
+        de->content_type = strdup("application/directory");
+      }
+      else
+      {
+
+        de->content_type = strdup("application/octet-stream");
+      }
      de->last_modified = time(NULL);
      de->next = cw->entries;
      cw->entries = de;
@@ -204,6 +219,15 @@ static int cfs_getattr(const char *path, struct stat *stbuf)
    stbuf->st_mode = S_IFDIR | 0755;
    stbuf->st_nlink = 2;
  }
+  else if (de->islink)
+  {
+    stbuf->st_size = 1;
+    stbuf->st_mode = S_IFLNK | 0755;
+    stbuf->st_nlink = 1;
+    stbuf->st_size = de->size;
+    /* calc. blocks as if 4K blocksize filesystem; stat uses units of 512B */
+    stbuf->st_blocks = ((4095 + de->size) / 4096) * 8;
+  }
  else
  {
    stbuf->st_size = de->size;
@@ -244,7 +268,7 @@ static int cfs_mkdir(const char *path, mode_t mode)
 {
  if (cloudfs_create_directory(path))
  {
-    update_dir_cache(path, 0, 1);
+    update_dir_cache(path, 0, 1, 0);
    return 0;
  }
  return -ENOENT;
@@ -258,24 +282,90 @@ static int cfs_create(const char *path, mode_t mode, struct fuse_file_info *info
  fclose(temp_file);
  of->flags = info->flags;
  info->fh = (uintptr_t)of;
-  update_dir_cache(path, 0, 0);
+  update_dir_cache(path, 0, 0, 0);
  info->direct_io = 1;
  return 0;
 }

 static int cfs_open(const char *path, struct fuse_file_info *info)
 {
-  FILE *temp_file = tmpfile();
+  //TODO: need to clean this up and reduce the duplicated code
+  if (!*temp_dir) {
+
+    FILE *temp_file = tmpfile();
+    dir_entry *de = path_info(path);
+    if (!(info->flags & O_WRONLY))
+    {
+      if (!cloudfs_object_write_fp(path, temp_file))
+      {
+        fclose(temp_file);
+        return -ENOENT;
+      }
+      update_dir_cache(path, (de ? de->size : 0), 0, 0);
+    }
+    openfile *of = (openfile *)malloc(sizeof(openfile));
+    of->fd = dup(fileno(temp_file));
+    fclose(temp_file);
+    of->flags = info->flags;
+    info->fh = (uintptr_t)of;
+    info->direct_io = 1;
+    return 0;
+  }
+
+
+  FILE *temp_file;
+
+  char tmp_path[PATH_MAX];
+  strncpy(tmp_path, path, PATH_MAX);
+
+  char *pch;
+  while((pch = strchr(tmp_path, '/'))) {
+    *pch = '.';
+  }
+
+  char file_path[PATH_MAX];
+  snprintf(file_path, PATH_MAX, "%s/.cloudfuse%ld-%s", temp_dir,
+      (long)getpid(), tmp_path);
+
  dir_entry *de = path_info(path);
-  if (!(info->flags & O_WRONLY))
-  {
+
+  if(access(file_path, F_OK) != -1) {
+    temp_file = fopen(file_path, "r");
+    update_dir_cache(path, (de ? de->size : 0), 0, 0);
+    // file exists
+  } else if (!(info->flags & O_WRONLY)) {
+    // we need to lock on the filename another process could open the file
+    // while we are writing to it and then only read part of the file
+
+    // duplicate the directory caching datastructure to make the code easier
+    // to understand.
+
+    // each file in the cache needs:
+    //  filename, is_writing, last_closed, is_removing
+    // the first time a file is opened a new entry is created in the cache
+    // setting the filename and is_writing to true.  This check needs to be
+    // wrapped with a lock.
+    //
+    // each time a file is closed we set the last_closed for the file to now
+    // and we check the cache for files whose last
+    // closed is greater than cache_timeout, then start a new thread rming
+    // that file.
+
+    // TODO: just to prevent this craziness for now
+    if (*temp_dir)
+        temp_file = fopen(file_path, "w+b");
+    else
+        temp_file = tmpfile();
+
+
    if (!cloudfs_object_write_fp(path, temp_file))
    {
      fclose(temp_file);
      return -ENOENT;
    }
-    update_dir_cache(path, (de ? de->size : 0), 0);
+    update_dir_cache(path, (de ? de->size : 0), 0, 0);
  }
+
  openfile *of = (openfile *)malloc(sizeof(openfile));
  of->fd = dup(fileno(temp_file));
  fclose(temp_file);
@@ -295,7 +385,7 @@ static int cfs_flush(const char *path, struct fuse_file_info *info)
  openfile *of = (openfile *)(uintptr_t)info->fh;
  if (of)
  {
-    update_dir_cache(path, cloudfs_file_size(of->fd), 0);
+    update_dir_cache(path, cloudfs_file_size(of->fd), 0, 0);
    if (of->flags & O_RDWR || of->flags & O_WRONLY)
    {
      FILE *fp = fdopen(dup(of->fd), "r");
@@ -336,13 +426,13 @@ static int cfs_ftruncate(const char *path, off_t size, struct fuse_file_info *in
  if (ftruncate(of->fd, size))
    return -errno;
  lseek(of->fd, 0, SEEK_SET);
-  update_dir_cache(path, size, 0);
+  update_dir_cache(path, size, 0, 0);
  return 0;
 }

 static int cfs_write(const char *path, const char *buf, size_t length, off_t offset, struct fuse_file_info *info)
 {
-  update_dir_cache(path, offset + length, 0);
+  update_dir_cache(path, offset + length, 0, 0);
  return pwrite(((openfile *)(uintptr_t)info->fh)->fd, buf, length, offset);
 }

@@ -399,12 +489,41 @@ static int cfs_rename(const char *src, const char *dst)
  if (cloudfs_copy_object(src, dst))
  {
    /* FIXME this isn't quite right as doesn't preserve last modified */
-    update_dir_cache(dst, src_de->size, 0);
+    update_dir_cache(dst, src_de->size, 0, 0);
    return cfs_unlink(src);
  }
  return -EIO;
 }

+static int cfs_symlink(const char *src, const char *dst)
+{
+  if(cloudfs_create_symlink(src, dst))
+  {
+    update_dir_cache(dst, 1, 0, 1);
+    return 0;
+  }
+  return -EIO;
+}
+
+static int cfs_readlink(const char* path, char* buf, size_t size)
+{
+  FILE *temp_file = tmpfile();
+  int ret = 0;
+
+  if (!cloudfs_object_write_fp(path, temp_file))
+  {
+      ret = -ENOENT;
+  }
+
+  if (!pread(fileno(temp_file), buf, size, 0))
+  {
+      ret = -ENOENT;
+  }
+
+  fclose(temp_file);
+  return ret;
+}
+
 static void *cfs_init(struct fuse_conn_info *conn)
 {
  signal(SIGPIPE, SIG_IGN);
@@ -425,6 +544,12 @@ char *get_home_dir()
 FuseOptions options = {
    .cache_timeout = "600",
    .verify_ssl = "true",
+    .segment_size = "1073741824",
+    .segment_above = "2147483648",
+    .storage_url = "",
+    .container = "",
+    //.temp_dir = "/tmp/",
+    .temp_dir = "",
    .client_id = "",
    .client_secret = "",
    .refresh_token = ""
@@ -434,6 +559,11 @@ int parse_option(void *data, const char *arg, int key, struct fuse_args *outargs
 {
  if (sscanf(arg, " cache_timeout = %[^\r\n ]", options.cache_timeout) ||
      sscanf(arg, " verify_ssl = %[^\r\n ]", options.verify_ssl) ||
+      sscanf(arg, " segment_above = %[^\r\n ]", options.segment_above) ||
+      sscanf(arg, " segment_size = %[^\r\n ]", options.segment_size) ||
+      sscanf(arg, " storage_url = %[^\r\n ]", options.storage_url) ||
+      sscanf(arg, " container = %[^\r\n ]", options.container) ||
+      sscanf(arg, " temp_dir = %[^\r\n ]", options.temp_dir) ||
      sscanf(arg, " client_id = %[^\r\n ]", options.client_id) ||
      sscanf(arg, " client_secret = %[^\r\n ]", options.client_secret) ||
      sscanf(arg, " refresh_token = %[^\r\n ]", options.refresh_token))
@@ -461,6 +591,12 @@ int main(int argc, char **argv)
  fuse_opt_parse(&args, &options, NULL, parse_option);

  cache_timeout = atoi(options.cache_timeout);
+  segment_size = atoll(options.segment_size);
+  segment_above = atoll(options.segment_above);
+  // this is ok since main is on the stack during the entire execution
+  override_storage_url = options.storage_url;
+  public_container = options.container;
+  temp_dir = options.temp_dir;

  if (!*options.client_id || !*options.client_secret || !*options.refresh_token)
  {
@@ -473,6 +609,11 @@ int main(int argc, char **argv)
    fprintf(stderr, "The following settings are optional:\n\n");
    fprintf(stderr, "  cache_timeout=[Seconds for directory caching, default 600]\n");
    fprintf(stderr, "  verify_ssl=[False to disable SSL cert verification]\n");
+    fprintf(stderr, "  segment_size=[Size to use when creating DLOs, default 1073741824]\n");
+    fprintf(stderr, "  segment_above=[File size at which to use segments, defult 2147483648]\n");
+    fprintf(stderr, "  storage_url=[Storage URL for other tenant to view container]\n");
+    fprintf(stderr, "  container=[Public container to view of tenant specified by storage_url]\n");
+    fprintf(stderr, "  temp_dir=[Directory to store temp files]\n");

    return 1;
  }
@@ -482,6 +623,7 @@ int main(int argc, char **argv)
  cloudfs_verify_ssl(!strcasecmp(options.verify_ssl, "true"));

  cloudfs_set_credentials(options.client_id, options.client_secret, options.refresh_token);
+
  if (!cloudfs_connect())
  {
    fprintf(stderr, "Failed to authenticate.\n");
@@ -513,6 +655,8 @@ int main(int argc, char **argv)
    .chmod = cfs_chmod,
    .chown = cfs_chown,
    .rename = cfs_rename,
+    .symlink = cfs_symlink,
+    .readlink = cfs_readlink,
    .init = cfs_init,
  };

@@ -0,0 +1,16 @@
+cloudfuse for Debian
+--------------------
+
+Please note that the simplest way to allow mounting of Swift shares is to
+add the user to the fuse group (which will be created by the fuse-utils
+package).
+
+To add user 'foo' to the 'fuse' group type the following command (with root
+privileges):
+
+	adduser foo fuse
+
+If the user is currently logged in he/she should log out and log in 
+in order for this change to take effect.
+
+ -- Mike Mestnik <cheako+apt_repo@mikemestnik.net>  Mon, 04 Jun 2012 13:47:28 -0500
@@ -0,0 +1,12 @@
+cloudfuse (0.9-1ubuntu1) precise; urgency=low
+
+  * Imported Upstream version 0.9
+  * Remove Debian patches already committed upstream
+
+ -- pataquets <amontero@tinet.org>  Wed, 15 Jan 2014 21:07:18 +0100
+
+cloudfuse (0.1-1) unstable; urgency=low
+
+  * Initial release
+
+ -- Mike Mestnik <cheako+apt_repo@mikemestnik.net>  Mon, 04 Jun 2012 13:47:28 -0500
@@ -0,0 +1 @@
+8
@@ -0,0 +1,20 @@
+Source: cloudfuse
+Section: utils
+Priority: optional
+Maintainer: Mike Mestnik <cheako+apt_repo@mikemestnik.net>
+Build-Depends: debhelper (>= 8.0.0), autotools-dev, libcurl4-openssl-dev, libxml2-dev, libssl-dev, libfuse-dev
+Standards-Version: 3.9.2
+#Vcs-Git: git://git.debian.org/collab-maint/cloudfuse.git
+#Vcs-Browser: http://git.debian.org/?p=collab-maint/cloudfuse.git;a=summary
+
+Package: cloudfuse
+Architecture: any
+Depends: ${shlibs:Depends}, ${misc:Depends}, fuse (>= 2.7) [linux-any] | fuse4bsd [kfreebsd-any]
+Description: filesystem client based on Swift as used by Rackspace.
+ cloudfuse is a FUSE application which provides access to Rackspace's
+ Cloud Files (or any installation of Swift).
+ .
+ cloudfuse is FUSE (Filesystem in USErspace).
+ .
+ cloudfuse may work with Openstack's Swift, but is not provided by Openstack.
+
@@ -0,0 +1,76 @@
+Format: http://dep.debian.net/deps/dep5
+Upstream-Name: cloudfuse
+
+Files: *
+Copyright: 2009 Michael Barton <mike@weirdlooking.com>
+License: MIT
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+ .
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+ .
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+
+Files: debian/*
+Copyright: 2012 Mike Mestnik <cheako+apt_repo@mikemestnik.net>
+License: GPL-2+
+ This package is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+ .
+ This package is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>
+ .
+ On Debian systems, the complete text of the GNU General
+ Public License version 2 can be found in "/usr/share/common-licenses/GPL-2".
+
+Files: configure
+Copyright: 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001,
+ 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010
+ Free Software Foundation, Inc.
+License: No problem Bugroff
+ configure script is free software; the Free Software Foundation
+ gives unlimited permission to copy, distribute and modify it.
+
+Files: install-sh
+Copyright: 1994 X Consortium
+License: MIT
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to
+ deal in the Software without restriction, including without limitation the
+ rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ sell copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+ .
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+ .
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ X CONSORTIUM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNEC-
+ TION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ .
+ Except as contained in this notice, the name of the X Consortium shall not
+ be used in advertising or otherwise to promote the sale, use or other deal-
+ ings in this Software without prior written authorization from the X Consor-
+ tium.
+
@@ -0,0 +1 @@
+README
@@ -0,0 +1,13 @@
+#!/usr/bin/make -f
+# -*- makefile -*-
+# Sample debian/rules that uses debhelper.
+# This file was originally written by Joey Hess and Craig Small.
+# As a special exception, when this file is copied by dh-make into a
+# dh-make output file, you may use that output file without restriction.
+# This special exception was added by Craig Small in version 0.37 of dh-make.
+
+# Uncomment this to turn on verbose mode.
+#export DH_VERBOSE=1
+
+%:
+	dh $@ 
@@ -0,0 +1 @@
+3.0 (quilt)
@@ -0,0 +1,2 @@
+version=3
+http://githubredir.debian.net/github/redbo/cloudfuse (.*).tar.gz