From 68cf996c1df17ac1f53f5fd86b71bfdfcb9f2aba Mon Sep 17 00:00:00 2001 From: Elrond Date: Mon, 18 Jul 2011 14:07:03 +0200 Subject: [PATCH 1/7] First start at MountStorage. This includes the mounttab, a resolver and adding mount entries. --- mediagoblin/storage.py | 76 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/mediagoblin/storage.py b/mediagoblin/storage.py index 5d6faa4c..3d5ce9ab 100644 --- a/mediagoblin/storage.py +++ b/mediagoblin/storage.py @@ -216,6 +216,82 @@ class BasicFileStorage(StorageInterface): return self._resolve_filepath(filepath) +class MountStorage(StorageInterface): + def __init__(self, **kwargs): + self.mounttab = {} + + def mount(self, dirpath, backend): + """ + Mount a new backend under dirpath + """ + new_ent = clean_listy_filepath(dirpath) + new_ent.append(u'') + + print "Mounting:", repr(new_ent) + already, rem_1, table, rem_2 = self.resolve_to_backend(new_ent, True) + print "===", repr(already), repr(rem_1), repr(rem_2) + + assert rem_1.pop(-1) == u'', "Internal Error 1" + assert rem_2.pop(-1) == u'', "Internal Error 2" + assert (already is None) or (len(rem_2) > 0), "Already mounted" + for part in rem_2: + table[part] = {} + table = table[part] + table[None] = backend + + def resolve_to_backend(self, filepath, extra_info = False): + """ + extra_info = True is for internal use! + + Normally, returns the backend and the filepath inside that backend. + + With extra_info = True it returns the last directory node and the + remaining filepath from there in addition. + """ + table = self.mounttab + filepath = filepath[:] + res_fp = None + while True: + new_be = table.get(None) + if (new_be is not None) or res_fp is None: + res_be = new_be + res_fp = filepath[:] + res_extra = (table, filepath[:]) + # print "... New res: %r, %r, %r" % (res_be, res_fp, res_extra) + if len(filepath) == 0: + break + query = filepath.pop(0) + entry = table.get(query) + if entry is not None: + table = entry + res_extra = (table, filepath[:]) + else: + break + if extra_info: + return (res_be, res_fp) + res_extra + else: + return (res_be, res_fp) + + def __repr__(self, table = None, indent = ""): + res = [] + if table is None: + res.append("MountStorage<") + table = self.mounttab + v = table.get(None) + if v: + res.append(indent + "On this level: " + repr(v)) + for k, v in table.iteritems(): + if k == None: + continue + res.append(indent + repr(k) + ":") + res += self.__repr__(v, indent + " ") + if table is self.mounttab: + res.append(">") + return "\n".join(res) + else: + return res + + ########### # Utilities ########### From 93b2796c7e89e269d60db8e48bf84a0f2ca88d12 Mon Sep 17 00:00:00 2001 From: Elrond Date: Sat, 23 Jul 2011 15:27:02 +0200 Subject: [PATCH 2/7] MountStorage: Some small fixups/changes. 1) A bit more assert. 2) Change __repr__ to use lists for the recursion parameter. --- mediagoblin/storage.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mediagoblin/storage.py b/mediagoblin/storage.py index 3d5ce9ab..d994268b 100644 --- a/mediagoblin/storage.py +++ b/mediagoblin/storage.py @@ -237,6 +237,7 @@ class MountStorage(StorageInterface): for part in rem_2: table[part] = {} table = table[part] + assert not table.has_key(None), "Huh? Already mounted?!" table[None] = backend def resolve_to_backend(self, filepath, extra_info = False): @@ -272,19 +273,19 @@ class MountStorage(StorageInterface): else: return (res_be, res_fp) - def __repr__(self, table = None, indent = ""): + def __repr__(self, table = None, indent = []): res = [] if table is None: res.append("MountStorage<") table = self.mounttab v = table.get(None) if v: - res.append(indent + "On this level: " + repr(v)) + res.append(" " * len(indent) + repr(indent) + ": " + repr(v)) for k, v in table.iteritems(): if k == None: continue - res.append(indent + repr(k) + ":") - res += self.__repr__(v, indent + " ") + res.append(" " * len(indent) + repr(k) + ":") + res += self.__repr__(v, indent + [k]) if table is self.mounttab: res.append(">") return "\n".join(res) From 937e2c88112ee2f2ec71b9b38ccb1b473f32237e Mon Sep 17 00:00:00 2001 From: Elrond Date: Sat, 23 Jul 2011 15:29:22 +0200 Subject: [PATCH 3/7] MountStorage: Create all the wrappers All those methods just call the appropiate method of the relevant backend. --- mediagoblin/storage.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/mediagoblin/storage.py b/mediagoblin/storage.py index d994268b..e3d54a30 100644 --- a/mediagoblin/storage.py +++ b/mediagoblin/storage.py @@ -292,6 +292,34 @@ class MountStorage(StorageInterface): else: return res + def file_exists(self, filepath): + backend, filepath = self.resolve_to_backend(filepath) + return backend.file_exists(filepath) + + def get_file(self, filepath, mode='r'): + backend, filepath = self.resolve_to_backend(filepath) + return backend.get_file(filepath, mode) + + def delete_file(self, filepath): + backend, filepath = self.resolve_to_backend(filepath) + return backend.delete_file(filepath) + + def file_url(self, filepath): + backend, filepath = self.resolve_to_backend(filepath) + return backend.file_url(filepath) + + def get_local_path(self, filepath): + backend, filepath = self.resolve_to_backend(filepath) + return backend.get_local_path(filepath) + + def copy_locally(self, filepath, dest_path): + """ + Need to override copy_locally, because the local_storage + attribute is not correct. + """ + backend, filepath = self.resolve_to_backend(filepath) + backend.copy_locally(filepath, dest_path) + ########### # Utilities From 926aec14737d59160c0fac04a1af201550abe967 Mon Sep 17 00:00:00 2001 From: Elrond Date: Wed, 3 Aug 2011 23:11:42 +0200 Subject: [PATCH 4/7] MountStorage: drop u'' nonsense from mount method My old resolver couldn't handle empty lists, so for the mount resolving I appeneded a u'' as dummy element. Not needed any more. --- mediagoblin/storage.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/mediagoblin/storage.py b/mediagoblin/storage.py index e3d54a30..abbe16c9 100644 --- a/mediagoblin/storage.py +++ b/mediagoblin/storage.py @@ -225,14 +225,11 @@ class MountStorage(StorageInterface): Mount a new backend under dirpath """ new_ent = clean_listy_filepath(dirpath) - new_ent.append(u'') print "Mounting:", repr(new_ent) already, rem_1, table, rem_2 = self.resolve_to_backend(new_ent, True) print "===", repr(already), repr(rem_1), repr(rem_2) - assert rem_1.pop(-1) == u'', "Internal Error 1" - assert rem_2.pop(-1) == u'', "Internal Error 2" assert (already is None) or (len(rem_2) > 0), "Already mounted" for part in rem_2: table[part] = {} From eea315623fdff018f389dad2ab037a2b2aaf982b Mon Sep 17 00:00:00 2001 From: Elrond Date: Wed, 3 Aug 2011 23:13:58 +0200 Subject: [PATCH 5/7] MountStorage: Better error reporting Created a new resolver frontend function, that raises and error, when no backend is found. Useful for all the wrappers. --- mediagoblin/storage.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mediagoblin/storage.py b/mediagoblin/storage.py index abbe16c9..f9d2c3ea 100644 --- a/mediagoblin/storage.py +++ b/mediagoblin/storage.py @@ -227,7 +227,7 @@ class MountStorage(StorageInterface): new_ent = clean_listy_filepath(dirpath) print "Mounting:", repr(new_ent) - already, rem_1, table, rem_2 = self.resolve_to_backend(new_ent, True) + already, rem_1, table, rem_2 = self._resolve_to_backend(new_ent, True) print "===", repr(already), repr(rem_1), repr(rem_2) assert (already is None) or (len(rem_2) > 0), "Already mounted" @@ -237,7 +237,7 @@ class MountStorage(StorageInterface): assert not table.has_key(None), "Huh? Already mounted?!" table[None] = backend - def resolve_to_backend(self, filepath, extra_info = False): + def _resolve_to_backend(self, filepath, extra_info = False): """ extra_info = True is for internal use! @@ -270,6 +270,12 @@ class MountStorage(StorageInterface): else: return (res_be, res_fp) + def resolve_to_backend(self, filepath): + backend, filepath = self._resolve_to_backend(filepath) + if backend is None: + raise Error("Path not mounted") + return backend, filepath + def __repr__(self, table = None, indent = []): res = [] if table is None: From 255f02c48623533f6e781d9d1eade12d18ee0745 Mon Sep 17 00:00:00 2001 From: Elrond Date: Mon, 8 Aug 2011 20:11:28 +0200 Subject: [PATCH 6/7] MountStorage: Add docs. --- mediagoblin/storage.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/mediagoblin/storage.py b/mediagoblin/storage.py index 25598c82..bbf1c034 100644 --- a/mediagoblin/storage.py +++ b/mediagoblin/storage.py @@ -287,6 +287,21 @@ class CloudFilesStorage(StorageInterface): class MountStorage(StorageInterface): + """ + Experimental "Mount" virtual Storage Interface + + This isn't an interface to some real storage, instead + it's a redirecting interface, that redirects requests + to other "StorageInterface"s. + For example, requests for ["store1", "a"] to first + storage with the path ["a"], etc. + + To set this up, you currently need to call the mount() + method with the target path and a backend, that shall + be available under that target path. + You have to mount things in a sensible order, + especially you can't mount ["a", "b"] before ["a"]. + """ def __init__(self, **kwargs): self.mounttab = {} From 620fca54727e37ba1b3e5ba745fdd7d7186dcafd Mon Sep 17 00:00:00 2001 From: Elrond Date: Mon, 8 Aug 2011 21:51:11 +0200 Subject: [PATCH 7/7] MountStorage: Improve mounting asserts The asserts now differentiate between mounting on the same path and mounting over a shorter path. --- mediagoblin/storage.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mediagoblin/storage.py b/mediagoblin/storage.py index bbf1c034..88c748ce 100644 --- a/mediagoblin/storage.py +++ b/mediagoblin/storage.py @@ -313,13 +313,16 @@ class MountStorage(StorageInterface): print "Mounting:", repr(new_ent) already, rem_1, table, rem_2 = self._resolve_to_backend(new_ent, True) - print "===", repr(already), repr(rem_1), repr(rem_2) + print "===", repr(already), repr(rem_1), repr(rem_2), len(table) + + assert (len(rem_2) > 0) or (None not in table), \ + "That path is already mounted" + assert (len(rem_2) > 0) or (len(table)==0), \ + "A longer path is already mounted here" - assert (already is None) or (len(rem_2) > 0), "Already mounted" for part in rem_2: table[part] = {} table = table[part] - assert not table.has_key(None), "Huh? Already mounted?!" table[None] = backend def _resolve_to_backend(self, filepath, extra_info = False):