Merge branch 'utime'

* utime:
  Bump version to 3.0.0
  Do some more clean-up.
  Reformat code using astyle.
  Add code reformatting rule.
  Message at startup only in debug mode.
  consistency / style changes as per merge comments
  small tweaks on debugging messages
  proper md5sum segfault fix
  segfault fix for md5sum
  optimised upload to cloud.
  completed chmod / chown attrib. init
  Support for atime, mtime, chmod, chown.

.gitignore

@@ -7,3 +7,5 @@ config.status
 # Binary outputs
 cloudfuse
 hubicfuse
+
+project-vc/

.hubicfuse.sample

@@ -0,0 +1,13 @@
+client_id = api_hubic_xxxxx
+client_secret = xxxxxx
+refresh_token = xxxx
+cache_timeout = 600
+temp_dir = /tmp/hubicfuse
+
+get_extended_metadata = true
+curl_verbose = false
+curl_progress_state = true
+cache_statfs_timeout = 15
+debug_level = 0
+enable_chmod = false
+enable_chown = false

Makefile.in

@@ -11,8 +11,8 @@ prefix = @prefix@
 exec_prefix = @exec_prefix@
 bindir = $(DESTDIR)$(exec_prefix)/bin
 
-SOURCES=cloudfsapi.c cloudfuse.c
-HEADERS=cloudfsapi.h
+SOURCES=cloudfsapi.c cloudfuse.c commonfs.c
+HEADERS=cloudfsapi.h commonfs.h
 
 all: hubicfuse
 
@@ -29,7 +29,7 @@ hubicfuse: $(SOURCES) $(HEADERS)
 	$(CC) $(CFLAGS) -o hubicfuse $(SOURCES) $(LIBS) $(LDFLAGS)
 
 clean:
-	/bin/rm -f hubicfuse
+	/bin/rm -fr hubicfuse *.orig config.h *~ aclocal.m4 autom4te.cache
 
 distclean: clean
 	/bin/rm -f Makefile config.h config.status config.cache config.log \
@@ -39,6 +39,9 @@ mostlyclean: clean
 
 maintainer-clean: clean
 
+debug: CFLAGS += -g
+debug: hubicfuse
+
 config.h.in: stamp-h.in
 stamp-h.in: configure.in
 	autoheader
@@ -52,3 +55,18 @@ Makefile: Makefile.in config.status
 config.status: configure
 	./config.status --recheck
 
+reformat:
+	astyle --style=allman \
+		--convert-tabs \
+		--indent=spaces=2 \
+		--lineend=linux \
+		--max-code-length=79 \
+		--pad-oper \
+		--pad-header \
+		--align-pointer=type \
+		--align-reference=name \
+		--break-closing-brackets \
+		--min-conditional-indent=0 \
+		--remove-brackets \
+		--remove-comment-prefix \
+		*.c *.h

README

@@ -39,6 +39,15 @@ USE:
         client_secret=[Hubic client secret for the registered application]
         refresh_token=[A refresh token you got from the script]
 
+    Optional variables:
+        get_extended_metadata=[true/false, force download of additional file metadata like atime and mtime on first directory list]
+        curl_verbose=[true/false, enable verbose output for curl HTTP requests]
+		curl_progress_state=[true/false, enable verbose progress output for curl HTTP requests. Used for debugging.]
+        cache_statfs_timeout=[value in seconds, large timeout increases the file access speed]
+        debug_level=[0 default, 1 extremely verbose for debugging purposes]
+		enable_chmod=[true/false, false by default, still experimental feature]
+		enable_chown=[true/false, false by default, still experimental feature]
+
     client_id, client_secret can be retrieved from the hubic
     web interface: https://hubic.com/home/browser/developers/
 
@@ -83,7 +92,20 @@ BUGS/SHORTCOMINGS:
       "application/directory".
     * Cloud Files limits container and object listings to 10,000 items.
       cloudfuse won't list more than that many files in a single directory.
+	  * File copy progress when uploading does not work, progress is shown when file 
+	    is copied in local cache, then upload operation happens at 100%
 
+Recent additions/fixes:
+    * Support for atime, mtime, chmod, chown.
+    * Large files (segmented) have correct size listed (was 0 before).
+    * Multiple speed improvements, minimised the number of HTTP calls and added more caching features.
+    * Fixed many segmentation faults.
+    * Cached files are deleted on cache expiration when using a custom temp folder.
+    * Files copied have attributes preserved.
+    * Working well with rsync due to mtime support and proper copy operations.
+    * Debugging support for http progress (track upload / download speed etc.)
+    * Reduced traffic, skips file uploads to cloud if content does not change (using md5sum compare)
+    * Major code refactoring, code documentation, extensive debugging, additional config options
 
 AWESOME CONTRIBUTORS:
 
@@ -97,6 +119,7 @@ AWESOME CONTRIBUTORS:
     * Mike Lundy                         https://github.com/novas0x2a
     * justinb                            https://github.com/justinsb
     * Matt Greenway                      https://github.com/LabAdvComp
+    * Dan Cristian                       https://github.com/dan-cristian
 
 Thanks, and I hope you find it useful.
 

cloudfsapi.h

@@ -4,77 +4,86 @@
 #include <curl/curl.h>
 #include <curl/easy.h>
 #include <fuse.h>
+#include <time.h>
 
 #define BUFFER_INITIAL_SIZE 4096
 #define MAX_HEADER_SIZE 8192
-#define MAX_PATH_SIZE (1024 + 256 + 3)
+
 #define MAX_URL_SIZE (MAX_PATH_SIZE * 3)
 #define USER_AGENT "CloudFuse"
 #define OPTION_SIZE 1024
 
 typedef struct curl_slist curl_slist;
 
-typedef struct dir_entry
+#define MINIMAL_PROGRESS_FUNCTIONALITY_INTERVAL 5
+struct curl_progress
 {
-  char *name;
-  char *full_name;
-  char *content_type;
-  off_t size;
-  time_t last_modified;
-  int isdir;
-  int islink;
-  struct dir_entry *next;
-} dir_entry;
+  double lastruntime;
+  CURL* curl;
+};
 
-typedef struct options {
-    char cache_timeout[OPTION_SIZE];
-    char verify_ssl[OPTION_SIZE];
-    char segment_size[OPTION_SIZE];
-    char segment_above[OPTION_SIZE];
-    char storage_url[OPTION_SIZE];
-    char container[OPTION_SIZE];
-    char temp_dir[OPTION_SIZE];
-    char client_id[OPTION_SIZE];
-    char client_secret[OPTION_SIZE];
-    char refresh_token[OPTION_SIZE];
+typedef struct options
+{
+  char cache_timeout[OPTION_SIZE];
+  char verify_ssl[OPTION_SIZE];
+  char segment_size[OPTION_SIZE];
+  char segment_above[OPTION_SIZE];
+  char storage_url[OPTION_SIZE];
+  char container[OPTION_SIZE];
+  char temp_dir[OPTION_SIZE];
+  char client_id[OPTION_SIZE];
+  char client_secret[OPTION_SIZE];
+  char refresh_token[OPTION_SIZE];
 } FuseOptions;
 
+typedef struct extra_options
+{
+  char get_extended_metadata[OPTION_SIZE];
+  char curl_verbose[OPTION_SIZE];
+  char cache_statfs_timeout[OPTION_SIZE];
+  char debug_level[OPTION_SIZE];
+  char curl_progress_state[OPTION_SIZE];
+  char enable_chmod[OPTION_SIZE];
+  char enable_chown[OPTION_SIZE];
+} ExtraFuseOptions;
+
 void cloudfs_init(void);
-void cloudfs_set_credentials(char *client_id, char *client_secret, char *refresh_token);
+void cloudfs_free(void);
+void cloudfs_set_credentials(char* client_id, char* client_secret,
+                             char* refresh_token);
 int cloudfs_connect(void);
 
 struct segment_info
 {
-    FILE *fp;
-    int part;
-    long size;
-    long segment_size;
-    char *seg_base;
-    const char *method;
+  FILE* fp;
+  int part;
+  long size;
+  long segment_size;
+  char* seg_base;
+  const char* method;
 };
 
 long segment_size;
 long segment_above;
 
-char *override_storage_url;
-char *public_container;
+char* override_storage_url;
+char* public_container;
 
-int file_is_readable(const char *fname);
-const char * get_file_mimetype ( const char *filename );
-
-int cloudfs_object_read_fp(const char *path, FILE *fp);
-int cloudfs_object_write_fp(const char *path, FILE *fp);
-int cloudfs_list_directory(const char *path, dir_entry **);
-int cloudfs_delete_object(const char *path);
-int cloudfs_copy_object(const char *src, const char *dst);
-int cloudfs_create_symlink(const char *src, const char *dst);
-int cloudfs_create_directory(const char *label);
-int cloudfs_object_truncate(const char *path, off_t size);
+int file_is_readable(const char* fname);
+const char* get_file_mimetype ( const char* filename );
+int cloudfs_object_read_fp(const char* path, FILE* fp);
+int cloudfs_object_write_fp(const char* path, FILE* fp);
+int cloudfs_list_directory(const char* path, dir_entry**);
+int cloudfs_delete_object(const char* path);
+int cloudfs_copy_object(const char* src, const char* dst);
+int cloudfs_create_symlink(const char* src, const char* dst);
+int cloudfs_create_directory(const char* label);
+int cloudfs_object_truncate(const char* path, off_t size);
 off_t cloudfs_file_size(int fd);
-void cloudfs_debug(int dbg);
+int cloudfs_statfs(const char* path, struct statvfs* stat);
 void cloudfs_verify_ssl(int dbg);
-void cloudfs_free_dir_list(dir_entry *dir_list);
-int cloudfs_statfs(const char *path, struct statvfs *stat);
-
-void debugf(char *fmt, ...);
+void cloudfs_option_get_extended_metadata(int option);
+void cloudfs_option_curl_verbose(int option);
+void get_file_metadata(dir_entry* de);
+int cloudfs_update_meta(dir_entry* de);
 #endif

commonfs.c

@@ -0,0 +1,684 @@
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <magic.h>
+#include <string.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#ifdef __linux__
+#include <alloca.h>
+#endif
+#include <pthread.h>
+#include <time.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/time.h>
+#include <sys/syscall.h>
+#include <openssl/md5.h>
+#include <pwd.h>
+#include <fuse.h>
+#include <limits.h>
+#include <curl/curl.h>
+#include <curl/easy.h>
+#include "commonfs.h"
+#include "config.h"
+
+pthread_mutex_t dcachemut;
+pthread_mutexattr_t mutex_attr;
+dir_cache* dcache;
+char* temp_dir;
+int cache_timeout;
+int debug = 0;
+int verify_ssl = 2;
+bool option_get_extended_metadata = false;
+bool option_curl_verbose = false;
+int option_cache_statfs_timeout = 0;
+int option_debug_level = 0;
+int option_curl_progress_state = 1;//1 to disable curl progress
+bool option_enable_chown = false;
+bool option_enable_chmod = false;
+bool option_enable_progressive_upload = false;
+bool option_enable_progressive_download = false;
+size_t file_buffer_size = 0;
+
+// needed to get correct GMT / local time
+// hubic stores time as GMT so we have to do conversions
+// http://zhu-qy.blogspot.ro/2012/11/ref-how-to-convert-from-utc-to-local.html
+time_t my_timegm(struct tm* tm)
+{
+  time_t epoch = 0;
+  time_t offset = mktime(gmtime(&epoch));
+  time_t utc = mktime(tm);
+  return difftime(utc, offset);
+}
+
+//expect time_str as a friendly string format
+time_t get_time_from_str_as_gmt(char* time_str)
+{
+  struct tm val_time_tm;
+  time_t val_time_t;
+  strptime(time_str, "%FT%T", &val_time_tm);
+  val_time_tm.tm_isdst = -1;
+  val_time_t = my_timegm(&val_time_tm);
+  return val_time_t;
+}
+
+time_t get_time_as_local(time_t time_t_val, char time_str[], int char_buf_size)
+{
+  struct tm loc_time_tm;
+  loc_time_tm = *localtime(&time_t_val);
+  if (time_str != NULL)
+  {
+    //debugf(DBG_LEVEL_NORM, 0,"Local len=%d size=%d pass=%d", strlen(time_str), sizeof(time_str), char_buf_size);
+    strftime(time_str, char_buf_size, "%c", &loc_time_tm);
+    //debugf(DBG_LEVEL_NORM, 0,"Local timestr=[%s] size=%d", time_str, strlen(time_str));
+  }
+  //debugf(DBG_LEVEL_NORM, 0,"Local time_t %li", mktime(&loc_time_tm));
+  return mktime(&loc_time_tm);
+}
+
+int get_time_as_string(time_t time_t_val, long nsec, char* time_str,
+                       int time_str_len)
+{
+  struct tm time_val_tm;
+  time_t safe_input_time;
+  //if time is incorrect (too long) you get segfault, need to check length and trim
+  if (time_t_val > INT_MAX)
+  {
+    debugf(DBG_LEVEL_NORM,
+           KRED"get_time_as_string: input time length too long, %lu > max=%lu, trimming!",
+           time_t_val, INT_MAX);
+    safe_input_time = 0;//(int)time_t_val;
+  }
+  else
+    safe_input_time = time_t_val;
+  time_val_tm = *gmtime(&safe_input_time);
+  int str_len = strftime(time_str, time_str_len, HUBIC_DATE_FORMAT,
+                         &time_val_tm);
+  char nsec_str[TIME_CHARS];
+  sprintf(nsec_str, "%d", nsec);
+  strcat(time_str, nsec_str);
+  return str_len + strlen(nsec_str);
+}
+
+time_t get_time_now()
+{
+  struct timespec now;
+  clock_gettime(CLOCK_REALTIME, &now);
+  return now.tv_sec;
+}
+
+size_t get_time_now_as_str(char* time_str, int time_str_len)
+{
+  time_t     now = time(0);
+  struct tm  tstruct;
+  tstruct = *localtime(&now);
+  // Visit http://en.cppreference.com/w/cpp/chrono/c/strftime
+  // for more information about date/time format
+  size_t result = strftime(time_str, time_str_len, HUBIC_DATE_FORMAT, &tstruct);
+  return result;
+}
+
+int get_timespec_as_str(const struct timespec* times, char* time_str,
+                        int time_str_len)
+{
+  return get_time_as_string(times->tv_sec, times->tv_nsec, time_str,
+                            time_str_len);
+}
+
+char* str2md5(const char* str, int length)
+{
+  int n;
+  MD5_CTX c;
+  unsigned char digest[16];
+  char* out = (char*)malloc(33);
+
+  MD5_Init(&c);
+  while (length > 0)
+  {
+    if (length > 512)
+      MD5_Update(&c, str, 512);
+    else
+      MD5_Update(&c, str, length);
+    length -= 512;
+    str += 512;
+  }
+  MD5_Final(digest, &c);
+  for (n = 0; n < 16; ++n)
+    snprintf(&(out[n * 2]), 16 * 2, "%02x", (unsigned int)digest[n]);
+  return out;
+}
+
+// http://stackoverflow.com/questions/10324611/how-to-calculate-the-md5-hash-of-a-large-file-in-c
+int file_md5(FILE* file_handle, char* md5_file_str)
+{
+  if (file_handle == NULL)
+  {
+    debugf(DBG_LEVEL_NORM, KRED"file_md5: NULL file handle");
+    return 0;
+  }
+  if (file_buffer_size == 0)
+  {
+    debugf(DBG_LEVEL_NORM, KYEL"file_md5: 0 file buffer size, using default");
+    file_buffer_size = 1024;
+  }
+  unsigned char c[MD5_DIGEST_LENGTH];
+  int i;
+  MD5_CTX mdContext;
+  int bytes;
+  char mdchar[3];//2 chars for md5 + null string terminator
+  unsigned char* data_buf = malloc(file_buffer_size * sizeof(unsigned char));
+  MD5_Init(&mdContext);
+  while ((bytes = fread(data_buf, 1, file_buffer_size, file_handle)) != 0)
+    MD5_Update(&mdContext, data_buf, bytes);
+  MD5_Final(c, &mdContext);
+  for (i = 0; i < MD5_DIGEST_LENGTH; i++)
+  {
+    snprintf(mdchar, 3, "%02x", c[i]);
+    strcat(md5_file_str, mdchar);
+  }
+  free(data_buf);
+  return 0;
+}
+
+int get_safe_cache_file_path(const char* path, char* file_path_safe,
+                             char* temp_dir)
+{
+  char tmp_path[PATH_MAX];
+  strncpy(tmp_path, path, PATH_MAX);
+  char* pch;
+  while ((pch = strchr(tmp_path, '/')))
+    * pch = '.';
+  char file_path[PATH_MAX] = "";
+  //temp file name had process pid in it, removed as on restart files are left in cache (pid changes)
+  snprintf(file_path, PATH_MAX, TEMP_FILE_NAME_FORMAT, temp_dir, tmp_path);
+  //fixme check if sizeof or strlen is suitable
+  int file_path_len = sizeof(file_path);
+  //the file path name using this format can go beyond NAME_MAX size and will generate error on fopen
+  //solution: cap file length to NAME_MAX, use a prefix from original path for debug purposes and add md5 id
+  char* md5_path = str2md5(file_path, file_path_len);
+  int md5len = strlen(md5_path);
+  size_t safe_len_prefix = min(NAME_MAX - md5len, file_path_len);
+  strncpy(file_path_safe, file_path, safe_len_prefix);
+  strncpy(file_path_safe + safe_len_prefix, md5_path, md5len);
+  //sometimes above copy process produces longer strings that NAME_MAX, force a null terminated string
+  file_path_safe[safe_len_prefix + md5len - 1] = '\0';
+  free(md5_path);
+  return strlen(file_path_safe);
+}
+
+void get_file_path_from_fd(int fd, char* path, int size_path)
+{
+  char proc_path[MAX_PATH_SIZE];
+  /* Read out the link to our file descriptor. */
+  sprintf(proc_path, "/proc/self/fd/%d", fd);
+  memset(path, 0, size_path);
+  readlink(proc_path, path, size_path - 1);
+}
+
+//for file descriptor debugging
+void debug_print_flags(int flags)
+{
+  int accmode, val;
+  accmode = flags & O_ACCMODE;
+  if (accmode == O_RDONLY)        debugf(DBG_LEVEL_EXTALL, KYEL"read only");
+  else if (accmode == O_WRONLY)   debugf(DBG_LEVEL_EXTALL, KYEL"write only");
+  else if (accmode == O_RDWR)     debugf(DBG_LEVEL_EXTALL, KYEL"read write");
+  else debugf(DBG_LEVEL_EXT, KYEL"unknown access mode");
+
+  if (val & O_APPEND)         debugf(DBG_LEVEL_EXTALL, KYEL", append");
+  if (val & O_NONBLOCK)       debugf(DBG_LEVEL_EXTALL, KYEL", nonblocking");
+#if !defined(_POSIX_SOURCE) && defined(O_SYNC)
+  if (val & O_SYNC)           debugf(DBG_LEVEL_EXT, 0,
+                                       KRED ", synchronous writes");
+#endif
+
+}
+
+//for file descriptor debugging
+void debug_print_descriptor(struct fuse_file_info* info)
+{
+  char file_path[MAX_PATH_SIZE];
+  get_file_path_from_fd(info->fh, file_path, sizeof(file_path));
+  debugf(DBG_LEVEL_EXT, KCYN "descriptor localfile=[%s] fd=%d", file_path,
+         info->fh);
+  debug_print_flags(info->flags);
+}
+
+void dir_for(const char* path, char* dir)
+{
+  strncpy(dir, path, MAX_PATH_SIZE);
+  char* slash = strrchr(dir, '/');
+  if (slash)
+    *slash = '\0';
+}
+
+//prints cache content for debug purposes
+void debug_list_cache_content()
+{
+  return;//disabled
+  dir_cache* cw;
+  dir_entry* de;
+  for (cw = dcache; cw; cw = cw->next)
+  {
+    debugf(DBG_LEVEL_EXT, "LIST-CACHE: DIR[%s]", cw->path);
+    for (de = cw->entries; de; de = de->next)
+      debugf(DBG_LEVEL_EXT, "LIST-CACHE:   FOLDER[%s]", de->full_name);
+  }
+}
+
+int delete_file(char* path)
+{
+  debugf(DBG_LEVEL_NORM, KYEL"delete_file(%s)", path);
+  char file_path_safe[NAME_MAX] = "";
+  get_safe_cache_file_path(path, file_path_safe, temp_dir);
+  int result = unlink(file_path_safe);
+  debugf(DBG_LEVEL_EXT, KYEL"delete_file(%s) (%s) result=%s", path,
+         file_path_safe, strerror(result));
+  return result;
+}
+
+//adding a directory in cache
+dir_cache* new_cache(const char* path)
+{
+  debugf(DBG_LEVEL_NORM, KCYN"new_cache(%s)", path);
+  dir_cache* cw = (dir_cache*)calloc(sizeof(dir_cache), 1);
+  cw->path = strdup(path);
+  cw->prev = NULL;
+  cw->entries = NULL;
+  cw->cached = time(NULL);
+  //added cache by access
+  cw->accessed_in_cache = time(NULL);
+  cw->was_deleted = false;
+  if (dcache)
+    dcache->prev = cw;
+  cw->next = dcache;
+  dir_cache* result;
+  result = (dcache = cw);
+  debugf(DBG_LEVEL_EXT, "exit: new_cache(%s)", path);
+  return result;
+}
+
+//todo: check if the program behaves ok  when free_dir
+//is made on a folder that has an operation in progress
+void cloudfs_free_dir_list(dir_entry* dir_list)
+{
+  //check for NULL as dir might be already removed from cache by other thread
+  debugf(DBG_LEVEL_NORM, "cloudfs_free_dir_list(%s)", dir_list->full_name);
+  while (dir_list)
+  {
+    dir_entry* de = dir_list;
+    dir_list = dir_list->next;
+    //remove file from disk cache, fix for issue #89, https://github.com/TurboGit/hubicfuse/issues/89
+    delete_file(de->full_name);
+    free(de->name);
+    free(de->full_name);
+    free(de->content_type);
+    //TODO free all added fields
+    free(de->md5sum);
+    free(de);
+  }
+}
+
+void dir_decache(const char* path)
+{
+  dir_cache* cw;
+  debugf(DBG_LEVEL_NORM, "dir_decache(%s)", path);
+  pthread_mutex_lock(&dcachemut);
+  dir_entry* de, *tmpde;
+  char dir[MAX_PATH_SIZE];
+  dir_for(path, dir);
+  for (cw = dcache; cw; cw = cw->next)
+  {
+    debugf(DBG_LEVEL_EXT, "dir_decache: parse(%s)", cw->path);
+    if (!strcmp(cw->path, path))
+    {
+      if (cw == dcache)
+        dcache = cw->next;
+      if (cw->prev)
+        cw->prev->next = cw->next;
+      if (cw->next)
+        cw->next->prev = cw->prev;
+      debugf(DBG_LEVEL_EXT, "dir_decache: free_dir1(%s)", cw->path);
+      //fixme: this sometimes is NULL and generates segfaults, checking first
+      if (cw->entries != NULL)
+        cloudfs_free_dir_list(cw->entries);
+      free(cw->path);
+      free(cw);
+    }
+    else if (cw->entries && !strcmp(dir, cw->path))
+    {
+      if (!strcmp(cw->entries->full_name, path))
+      {
+        de = cw->entries;
+        cw->entries = de->next;
+        de->next = NULL;
+        debugf(DBG_LEVEL_EXT, "dir_decache: free_dir2()");
+        cloudfs_free_dir_list(de);
+      }
+      else for (de = cw->entries; de->next; de = de->next)
+        {
+          if (!strcmp(de->next->full_name, path))
+          {
+            tmpde = de->next;
+            de->next = de->next->next;
+            tmpde->next = NULL;
+            debugf(DBG_LEVEL_EXT, "dir_decache: free_dir3()", cw->path);
+            cloudfs_free_dir_list(tmpde);
+            break;
+          }
+        }
+    }
+  }
+  pthread_mutex_unlock(&dcachemut);
+}
+
+dir_entry* init_dir_entry()
+{
+  dir_entry* de = (dir_entry*)malloc(sizeof(dir_entry));
+  de->metadata_downloaded = false;
+  de->size = 0;
+  de->next = NULL;
+  de->md5sum = NULL;
+  de->accessed_in_cache = time(NULL);
+  de->last_modified = time(NULL);
+  de->mtime.tv_sec = time(NULL);
+  de->atime.tv_sec = time(NULL);
+  de->ctime.tv_sec = time(NULL);
+  de->mtime.tv_nsec = 0;
+  de->atime.tv_nsec = 0;
+  de->ctime.tv_nsec = 0;
+  de->chmod = 0;
+  de->gid = 0;
+  de->uid = 0;
+  return de;
+}
+
+void copy_dir_entry(dir_entry* src, dir_entry* dst)
+{
+  dst->atime.tv_sec = src->atime.tv_sec;
+  dst->atime.tv_nsec = src->atime.tv_nsec;
+  dst->mtime.tv_sec = src->mtime.tv_sec;
+  dst->mtime.tv_nsec = src->mtime.tv_nsec;
+  dst->ctime.tv_sec = src->ctime.tv_sec;
+  dst->ctime.tv_nsec = src->ctime.tv_nsec;
+  dst->chmod = src->chmod;
+  //todo: copy md5sum as well
+}
+
+//check for file in cache, if found size will be updated, if not found
+//and this is a dir, a new dir cache entry is created
+void update_dir_cache(const char* path, off_t size, int isdir, int islink)
+{
+  debugf(DBG_LEVEL_EXTALL, KCYN "update_dir_cache(%s)", path);
+  pthread_mutex_lock(&dcachemut);
+  dir_cache* cw;
+  dir_entry* de;
+  char dir[MAX_PATH_SIZE];
+  dir_for(path, dir);
+  for (cw = dcache; cw; cw = cw->next)
+  {
+    if (!strcmp(cw->path, dir))
+    {
+      for (de = cw->entries; de; de = de->next)
+      {
+        if (!strcmp(de->full_name, path))
+        {
+          de->size = size;
+          pthread_mutex_unlock(&dcachemut);
+          debugf(DBG_LEVEL_EXTALL, "exit 0: update_dir_cache(%s)", path);
+          return;
+        }
+      }
+      de = init_dir_entry();
+      de->size = size;
+      de->isdir = isdir;
+      de->islink = islink;
+      de->name = strdup(&path[strlen(cw->path) + 1]);
+      de->full_name = strdup(path);
+      //fixed: the conditions below were mixed up dir -> link?
+      if (islink)
+        de->content_type = strdup("application/link");
+      if (isdir)
+        de->content_type = strdup("application/directory");
+      else
+        de->content_type = strdup("application/octet-stream");
+      de->next = cw->entries;
+      cw->entries = de;
+      if (isdir)
+        new_cache(path);
+      break;
+    }
+  }
+  debugf(DBG_LEVEL_EXTALL, "exit 1: update_dir_cache(%s)", path);
+  pthread_mutex_unlock(&dcachemut);
+}
+
+//returns first file entry in linked list. if not in cache will be downloaded.
+int caching_list_directory(const char* path, dir_entry** list)
+{
+  debugf(DBG_LEVEL_EXT, "caching_list_directory(%s)", path);
+  pthread_mutex_lock(&dcachemut);
+  bool new_entry = false;
+  if (!strcmp(path, "/"))
+    path = "";
+  dir_cache* cw;
+  for (cw = dcache; cw; cw = cw->next)
+  {
+    if (cw->was_deleted == true)
+    {
+      debugf(DBG_LEVEL_EXT,
+             KMAG"caching_list_directory status: dir(%s) is empty as cached expired, reload from cloud",
+             cw->path);
+      if (!cloudfs_list_directory(cw->path, list))
+        debugf(DBG_LEVEL_EXT,
+               KMAG"caching_list_directory status: cannot reload dir(%s)", cw->path);
+      else
+      {
+        debugf(DBG_LEVEL_EXT, KMAG"caching_list_directory status: reloaded dir(%s)",
+               cw->path);
+        //cw->entries = *list;
+        cw->was_deleted = false;
+        cw->cached = time(NULL);
+      }
+    }
+    if (cw->was_deleted == false)
+    {
+      if (!strcmp(cw->path, path))
+        break;
+    }
+  }
+  if (!cw)
+  {
+    //trying to download this entry from cloud, list will point to cached or downloaded entries
+    if (!cloudfs_list_directory(path, list))
+    {
+      //download was not ok
+      pthread_mutex_unlock(&dcachemut);
+      debugf(DBG_LEVEL_EXT,
+             "exit 0: caching_list_directory(%s) "KYEL"[CACHE-DIR-MISS]", path);
+      return  0;
+    }
+    debugf(DBG_LEVEL_EXT,
+           "caching_list_directory: new_cache(%s) "KYEL"[CACHE-CREATE]", path);
+    cw = new_cache(path);
+    new_entry = true;
+  }
+  else if (cache_timeout > 0 && (time(NULL) - cw->cached > cache_timeout))
+  {
+    if (!cloudfs_list_directory(path, list))
+    {
+      //mutex unlock was forgotten
+      pthread_mutex_unlock(&dcachemut);
+      debugf(DBG_LEVEL_EXT, "exit 1: caching_list_directory(%s)", path);
+      return  0;
+    }
+    //fixme: this frees dir subentries but leaves the dir parent entry, this confuses path_info
+    //which believes this dir has no entries
+    if (cw->entries != NULL)
+    {
+      cloudfs_free_dir_list(cw->entries);
+      cw->was_deleted = true;
+      cw->cached = time(NULL);
+      debugf(DBG_LEVEL_EXT, "caching_list_directory(%s) "KYEL"[CACHE-EXPIRED]",
+             path);
+    }
+    else
+    {
+      debugf(DBG_LEVEL_EXT,
+             "got NULL on caching_list_directory(%s) "KYEL"[CACHE-EXPIRED w NULL]", path);
+      pthread_mutex_unlock(&dcachemut);
+      return 0;
+    }
+  }
+  else
+  {
+    debugf(DBG_LEVEL_EXT, "caching_list_directory(%s) "KGRN"[CACHE-DIR-HIT]",
+           path);
+    *list = cw->entries;
+  }
+  //adding new dir file list to global cache, now this dir becomes visible in cache
+  cw->entries = *list;
+  pthread_mutex_unlock(&dcachemut);
+  debugf(DBG_LEVEL_EXT, "exit 2: caching_list_directory(%s)", path);
+  return 1;
+}
+
+dir_entry* path_info(const char* path)
+{
+  debugf(DBG_LEVEL_EXT, "path_info(%s)", path);
+  char dir[MAX_PATH_SIZE];
+  dir_for(path, dir);
+  dir_entry* tmp;
+  if (!caching_list_directory(dir, &tmp))
+  {
+    debugf(DBG_LEVEL_EXT, "exit 0: path_info(%s) "KYEL"[CACHE-DIR-MISS]", dir);
+    return NULL;
+  }
+  else
+    debugf(DBG_LEVEL_EXT, "path_info(%s) "KGRN"[CACHE-DIR-HIT]", dir);
+  //iterate in file list obtained from cache or downloaded
+  for (; tmp; tmp = tmp->next)
+  {
+    if (!strcmp(tmp->full_name, path))
+    {
+      debugf(DBG_LEVEL_EXT, "exit 1: path_info(%s) "KGRN"[CACHE-FILE-HIT]", path);
+      return tmp;
+    }
+  }
+  //miss in case the file is not found on a cached folder
+  debugf(DBG_LEVEL_EXT, "exit 2: path_info(%s) "KYEL"[CACHE-MISS]", path);
+  return NULL;
+}
+
+
+//retrieve folder from local cache if exists, return null if does not exist (rather than download)
+int check_caching_list_directory(const char* path, dir_entry** list)
+{
+  debugf(DBG_LEVEL_EXT, "check_caching_list_directory(%s)", path);
+  pthread_mutex_lock(&dcachemut);
+  if (!strcmp(path, "/"))
+    path = "";
+  dir_cache* cw;
+  for (cw = dcache; cw; cw = cw->next)
+    if (!strcmp(cw->path, path))
+    {
+      *list = cw->entries;
+      pthread_mutex_unlock(&dcachemut);
+      debugf(DBG_LEVEL_EXT,
+             "exit 0: check_caching_list_directory(%s) "KGRN"[CACHE-DIR-HIT]", path);
+      return 1;
+    }
+  pthread_mutex_unlock(&dcachemut);
+  debugf(DBG_LEVEL_EXT,
+         "exit 1: check_caching_list_directory(%s) "KYEL"[CACHE-DIR-MISS]", path);
+  return 0;
+}
+
+dir_entry* check_parent_folder_for_file(const char* path)
+{
+  char dir[MAX_PATH_SIZE];
+  dir_for(path, dir);
+  dir_entry* tmp;
+  if (!check_caching_list_directory(dir, &tmp))
+    return NULL;
+  else
+    return tmp;
+}
+
+//check if local path is in cache, without downloading from cloud if not in cache
+dir_entry* check_path_info(const char* path)
+{
+  debugf(DBG_LEVEL_EXT, "check_path_info(%s)", path);
+  char dir[MAX_PATH_SIZE];
+  dir_for(path, dir);
+  dir_entry* tmp;
+
+  //get parent folder cache entry
+  if (!check_caching_list_directory(dir, &tmp))
+  {
+    debugf(DBG_LEVEL_EXT, "exit 0: check_path_info(%s) "KYEL"[CACHE-MISS]", path);
+    return NULL;
+  }
+  for (; tmp; tmp = tmp->next)
+  {
+    if (!strcmp(tmp->full_name, path))
+    {
+      debugf(DBG_LEVEL_EXT, "exit 1: check_path_info(%s) "KGRN"[CACHE-HIT]", path);
+      return tmp;
+    }
+  }
+  if (!strcmp(path, "/"))
+    debugf(DBG_LEVEL_EXT,
+           "exit 2: check_path_info(%s) "KYEL"ignoring root [CACHE-MISS]", path);
+  else
+    debugf(DBG_LEVEL_EXT, "exit 3: check_path_info(%s) "KYEL"[CACHE-MISS]", path);
+  return NULL;
+}
+
+
+char* get_home_dir()
+{
+  char* home;
+  if ((home = getenv("HOME")) && !access(home, R_OK))
+    return home;
+  struct passwd* pwd = getpwuid(geteuid());
+  if ((home = pwd->pw_dir) && !access(home, R_OK))
+    return home;
+  return "~";
+}
+
+void cloudfs_debug(int dbg)
+{
+  debug = dbg;
+}
+
+void debugf(int level, char* fmt, ...)
+{
+  if (debug)
+  {
+    if (level <= option_debug_level)
+    {
+#ifdef SYS_gettid
+      pid_t thread_id = syscall(SYS_gettid);
+#else
+      int thread_id = 0;
+#error "SYS_gettid unavailable on this system"
+#endif
+      va_list args;
+      char prefix[] = "==DBG %d [%s]:%d==";
+      char line[4096];
+      char time_str[TIME_CHARS];
+      get_time_now_as_str(time_str, sizeof(time_str));
+      sprintf(line, prefix, level, time_str, thread_id);
+      fputs(line, stderr);
+      va_start(args, fmt);
+      vfprintf(stderr, fmt, args);
+      va_end(args);
+      fputs(KNRM, stderr);
+      putc('\n', stderr);
+      putc('\r', stderr);
+    }
+  }
+}

commonfs.h

@@ -0,0 +1,113 @@
+#ifndef _COMMONFS_H
+#define _COMMONFS_H
+#include <fuse.h>
+
+typedef enum { false, true } bool;
+#define MAX_PATH_SIZE (1024 + 256 + 3)
+#define THREAD_NAMELEN 16
+// 64 bit time + nanoseconds
+#define TIME_CHARS 32
+#define DBG_LEVEL_NORM 0
+#define DBG_LEVEL_EXT 1
+#define DBG_LEVEL_EXTALL 2
+#define INT_CHAR_LEN 16
+
+// utimens support
+#define HEADER_TEXT_MTIME "X-Object-Meta-Mtime"
+#define HEADER_TEXT_ATIME "X-Object-Meta-Atime"
+#define HEADER_TEXT_CTIME "X-Object-Meta-Ctime"
+#define HEADER_TEXT_MTIME_DISPLAY "X-Object-Meta-Mtime-Display"
+#define HEADER_TEXT_ATIME_DISPLAY "X-Object-Meta-Atime-Display"
+#define HEADER_TEXT_CTIME_DISPLAY "X-Object-Meta-Ctime-Display"
+#define HEADER_TEXT_CHMOD "X-Object-Meta-Chmod"
+#define HEADER_TEXT_UID "X-Object-Meta-Uid"
+#define HEADER_TEXT_GID "X-Object-Meta-Gid"
+#define HEADER_TEXT_FILEPATH "X-Object-Meta-FilePath"
+#define TEMP_FILE_NAME_FORMAT "%s/.cloudfuse_%s"
+#define HUBIC_DATE_FORMAT "%Y-%m-%d %T."
+
+#define KNRM  "\x1B[0m"
+#define KRED  "\x1B[31m"
+#define KGRN  "\x1B[32m"
+#define KYEL  "\x1B[33m"
+#define KBLU  "\x1B[34m"
+#define KMAG  "\x1B[35m"
+#define KCYN  "\x1B[36m"
+#define KWHT  "\x1B[37m"
+
+#define min(x, y) ({                \
+  typeof(x) _min1 = (x);          \
+  typeof(y) _min2 = (y);          \
+  (void)(&_min1 == &_min2);      \
+  _min1 < _min2 ? _min1 : _min2; })
+
+//linked list with files in a directory
+typedef struct dir_entry
+{
+  char* name;
+  char* full_name;
+  char* content_type;
+  off_t size;
+  time_t last_modified;
+  // implement utimens
+  struct timespec mtime;
+  struct timespec ctime;
+  struct timespec atime;
+  char* md5sum; //interesting capability for rsync/scrub
+  mode_t chmod;
+  uid_t uid;
+  gid_t gid;
+  bool issegmented;
+  time_t accessed_in_cache;//todo: cache support based on access time
+  bool metadata_downloaded;
+  // end change
+  int isdir;
+  int islink;
+  struct dir_entry* next;
+} dir_entry;
+
+// linked list with cached folder names
+typedef struct dir_cache
+{
+  char* path;
+  dir_entry* entries;
+  time_t cached;
+  //added cache support based on access time
+  time_t accessed_in_cache;
+  bool was_deleted;
+  //end change
+  struct dir_cache* next, *prev;
+} dir_cache;
+
+time_t my_timegm(struct tm* tm);
+time_t get_time_from_str_as_gmt(char* time_str);
+time_t get_time_as_local(time_t time_t_val, char time_str[],
+                         int char_buf_size);
+int get_time_as_string(time_t time_t_val, long nsec, char* time_str,
+                       int time_str_len);
+time_t get_time_now();
+int get_timespec_as_str(const struct timespec* times, char* time_str,
+                        int time_str_len);
+char* str2md5(const char* str, int length);
+int file_md5(FILE* file_handle, char* md5_file_str);
+void debug_print_descriptor(struct fuse_file_info* info);
+int get_safe_cache_file_path(const char* file_path, char* file_path_safe,
+                             char* temp_dir);
+dir_entry* init_dir_entry();
+void copy_dir_entry(dir_entry* src, dir_entry* dst);
+dir_cache* new_cache(const char* path);
+void dir_for(const char* path, char* dir);
+void debug_list_cache_content();
+void update_dir_cache(const char* path, off_t size, int isdir, int islink);
+dir_entry* path_info(const char* path);
+dir_entry* check_path_info(const char* path);
+dir_entry* check_parent_folder_for_file(const char* path);
+void dir_decache(const char* path);
+void cloudfs_free_dir_list(dir_entry* dir_list);
+extern int cloudfs_list_directory(const char* path, dir_entry**);
+int caching_list_directory(const char* path, dir_entry** list);
+char* get_home_dir();
+void cloudfs_debug(int dbg);
+void debugf(int level, char* fmt, ...);
+
+#endif

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for hubicfuse 1.0.1.
+# Generated by GNU Autoconf 2.69 for hubicfuse 3.0.0.
 #
 # Report bugs to <Pascal Obry <pascal@obry.net>>.
 #
@@ -580,8 +580,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='hubicfuse'
 PACKAGE_TARNAME='hubicfuse'
-PACKAGE_VERSION='1.0.1'
-PACKAGE_STRING='hubicfuse 1.0.1'
+PACKAGE_VERSION='3.0.0'
+PACKAGE_STRING='hubicfuse 3.0.0'
 PACKAGE_BUGREPORT='Pascal Obry <pascal@obry.net>'
 PACKAGE_URL=''
 
@@ -673,6 +673,7 @@ infodir
 docdir
 oldincludedir
 includedir
+runstatedir
 localstatedir
 sharedstatedir
 sysconfdir
@@ -756,6 +757,7 @@ datadir='${datarootdir}'
 sysconfdir='${prefix}/etc'
 sharedstatedir='${prefix}/com'
 localstatedir='${prefix}/var'
+runstatedir='${localstatedir}/run'
 includedir='${prefix}/include'
 oldincludedir='/usr/include'
 docdir='${datarootdir}/doc/${PACKAGE_TARNAME}'
@@ -1008,6 +1010,15 @@ do
   | -silent | --silent | --silen | --sile | --sil)
     silent=yes ;;
 
+  -runstatedir | --runstatedir | --runstatedi | --runstated \
+  | --runstate | --runstat | --runsta | --runst | --runs \
+  | --run | --ru | --r)
+    ac_prev=runstatedir ;;
+  -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \
+  | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \
+  | --run=* | --ru=* | --r=*)
+    runstatedir=$ac_optarg ;;
+
   -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
     ac_prev=sbindir ;;
   -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
@@ -1145,7 +1156,7 @@ fi
 for ac_var in	exec_prefix prefix bindir sbindir libexecdir datarootdir \
 		datadir sysconfdir sharedstatedir localstatedir includedir \
 		oldincludedir docdir infodir htmldir dvidir pdfdir psdir \
-		libdir localedir mandir
+		libdir localedir mandir runstatedir
 do
   eval ac_val=\$$ac_var
   # Remove trailing slashes.
@@ -1258,7 +1269,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures hubicfuse 1.0.1 to adapt to many kinds of systems.
+\`configure' configures hubicfuse 3.0.0 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1298,6 +1309,7 @@ Fine tuning of the installation directories:
   --sysconfdir=DIR        read-only single-machine data [PREFIX/etc]
   --sharedstatedir=DIR    modifiable architecture-independent data [PREFIX/com]
   --localstatedir=DIR     modifiable single-machine data [PREFIX/var]
+  --runstatedir=DIR       modifiable per-process data [LOCALSTATEDIR/run]
   --libdir=DIR            object code libraries [EPREFIX/lib]
   --includedir=DIR        C header files [PREFIX/include]
   --oldincludedir=DIR     C header files for non-gcc [/usr/include]
@@ -1319,7 +1331,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of hubicfuse 1.0.1:";;
+     short | recursive ) echo "Configuration of hubicfuse 3.0.0:";;
    esac
   cat <<\_ACEOF
 
@@ -1414,7 +1426,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-hubicfuse configure 1.0.1
+hubicfuse configure 3.0.0
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -1894,7 +1906,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by hubicfuse $as_me 1.0.1, which was
+It was created by hubicfuse $as_me 3.0.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -3609,8 +3621,8 @@ fi
 	# Put the nasty error message in config.log where it belongs
 	echo "$JSON_PKG_ERRORS" >&5
 
-	  #JSON_CFLAGS C compiler flags for JSON, overriding pkg-config
-	  #JSON_LIBS   linker flags for JSON, overriding pkg-config
+	  JSON_CFLAGS C compiler flags for JSON, overriding pkg-config
+  JSON_LIBS   linker flags for JSON, overriding pkg-config
 
 pkg_failed=no
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for JSON" >&5
@@ -5733,7 +5745,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by hubicfuse $as_me 1.0.1, which was
+This file was extended by hubicfuse $as_me 3.0.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -5795,7 +5807,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-hubicfuse config.status 1.0.1
+hubicfuse config.status 3.0.0
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ(2.61)
-AC_INIT([hubicfuse], [2.1.0], [Pascal Obry <pascal@obry.net>])
+AC_INIT([hubicfuse], [3.0.0], [Pascal Obry <pascal@obry.net>])
 AC_SUBST(CPPFLAGS, "$CPPFLAGS -D_FILE_OFFSET_BITS=64 -I/usr/include/libxml2")
 AC_LANG([C])
 AC_CONFIG_SRCDIR([cloudfuse.c])