Makes DataFiles FileIO compatible (#106)

* added the AdvancedDataFile and AdvancedDataDirectory types

* fixed some test case failures (good thing I added tests!)

* removed debug statement

* added more if checks to ensure that we don't get type errors with older versions of python

* updated to use the filelike type as the default, with legacy support available under a flag

* swapped default of datafile and datadirectory types to advanced

* renamed AdvancedDataFile

* Update Algorithmia/datadirectory.py

Co-authored-by: lemonez <36384768+lemonez@users.noreply.github.com>

Co-authored-by: lemonez <36384768+lemonez@users.noreply.github.com>

Author: zeryx

Algorithmia/client.py

@@ -3,8 +3,8 @@
 import Algorithmia
 from Algorithmia.insights import Insights
 from Algorithmia.algorithm import Algorithm
-from Algorithmia.datafile import DataFile, LocalDataFile
-from Algorithmia.datadirectory import DataDirectory, LocalDataDirectory
+from Algorithmia.datafile import DataFile, LocalDataFile, AdvancedDataFile
+from Algorithmia.datadirectory import DataDirectory, LocalDataDirectory, AdvancedDataDirectory
 from algorithmia_api_client import Configuration, DefaultApi, ApiClient
 
 from tempfile import mkstemp
@@ -63,13 +63,17 @@ def username(self):
         username = next(self.dir("").list()).path
         return username
 
-    def file(self, dataUrl):
-        if dataUrl.startswith('file://'): return LocalDataFile(self, dataUrl)
-        else: return DataFile(self, dataUrl)
+    def file(self, dataUrl, cleanup=False):
+        if dataUrl.startswith('file://'):
+            return LocalDataFile(self, dataUrl)
+        else:
+            return AdvancedDataFile(self, dataUrl, cleanup)
 
     def dir(self, dataUrl):
-        if dataUrl.startswith('file://'): return LocalDataDirectory(self, dataUrl)
-        else: return DataDirectory(self, dataUrl)
+        if dataUrl.startswith('file://'):
+            return LocalDataDirectory(self, dataUrl)
+        else:
+            return AdvancedDataDirectory(self, dataUrl)
 
     def create_user(self, requestString):
         url = "/v1/users" 

Algorithmia/datadirectory.py

@@ -5,14 +5,15 @@
 import os
 import six
 import tempfile
-
 import Algorithmia
-from Algorithmia.datafile import DataFile
+
+from Algorithmia.datafile import DataFile, AdvancedDataFile, LocalDataFile
 from Algorithmia.data import DataObject, DataObjectType
 from Algorithmia.errors import DataApiError
 from Algorithmia.util import getParentAndBase, pathJoin
 from Algorithmia.acl import Acl
 
+
 class DataDirectory(DataObject):
     def __init__(self, client, dataUrl):
         super(DataDirectory, self).__init__(DataObjectType.directory)
@@ -41,7 +42,7 @@ def exists(self):
     def create(self, acl=None):
         '''Creates a directory, optionally include Acl argument to set permissions'''
         parent, name = getParentAndBase(self.path)
-        json = { 'name': name }
+        json = {'name': name}
         if acl is not None:
             json['acl'] = acl.to_api_param()
         response = self.client.postJsonHelper(DataDirectory._getUrl(parent), json, False)
@@ -90,7 +91,7 @@ def get_permissions(self):
             return None
 
     def update_permissions(self, acl):
-        params = {'acl':acl.to_api_param()}
+        params = {'acl': acl.to_api_param()}
         response = self.client.patchHelper(self.url, params)
         if response.status_code != 200:
             raise DataApiError('Unable to update permissions: ' + response.json()['error']['message'])
@@ -102,7 +103,7 @@ def _get_directory_iterator(self, type_filter=None):
         while first or (marker is not None and len(marker) > 0):
             first = False
             url = self.url
-            query_params= {}
+            query_params = {}
             if marker:
                 query_params['marker'] = marker
             response = self.client.getHelper(url, **query_params)
@@ -177,8 +178,17 @@ def list(self):
 
     def dirs(self, content):
         for x in os.listdir(self.path):
-            if os.path.isdir(self.path+'/'+x): yield x
+            if os.path.isdir(self.path + '/' + x): yield x
 
     def files(self, content):
         for x in os.listdir(self.path):
-            if os.path.isfile(self.path+'/'+x): yield x
+            if os.path.isfile(self.path + '/' + x):
+                yield x
+
+
+class AdvancedDataDirectory(DataDirectory):
+    def __init__(self, client, dataUrl):
+        super(AdvancedDataDirectory, self).__init__(client, dataUrl)
+
+    def file(self, name, cleanup=True):
+        return AdvancedDataFile(self.client, pathJoin(self.path, name), cleanup)

Algorithmia/datafile.py

@@ -11,6 +11,7 @@
 from Algorithmia.util import getParentAndBase
 from Algorithmia.data import DataObject, DataObjectType
 from Algorithmia.errors import DataApiError, raiseDataApiError
+from io import RawIOBase
 
 
 class DataFile(DataObject):
@@ -24,7 +25,7 @@ def __init__(self, client, dataUrl):
         self.size = None
 
     def set_attributes(self, attributes):
-        self.last_modified = datetime.strptime(attributes['last_modified'],'%Y-%m-%dT%H:%M:%S.%fZ')
+        self.last_modified = datetime.strptime(attributes['last_modified'], '%Y-%m-%dT%H:%M:%S.%fZ')
         self.size = attributes['size']
 
     # Deprecated:
@@ -38,13 +39,13 @@ def getFile(self):
             raise DataApiError('unable to get file {} - {}'.format(self.path, error))
         # Make HTTP get request
         response = self.client.getHelper(self.url)
-        with tempfile.NamedTemporaryFile(delete = False) as f:
+        with tempfile.NamedTemporaryFile(delete=False) as f:
             for block in response.iter_content(1024):
                 if not block:
-                    break;
+                    break
                 f.write(block)
             f.flush()
-            return open(f.name)
+        return open(f.name)
 
     def getName(self):
         _, name = getParentAndBase(self.path)
@@ -129,6 +130,7 @@ def putFile(self, path):
                 raise raiseDataApiError(result)
             else:
                 return self
+
     def putNumpy(self, array):
         # Post numpy array as json payload
         np_loader = pkgutil.find_loader('numpy')
@@ -148,6 +150,7 @@ def delete(self):
         else:
             return True
 
+
 class LocalDataFile():
     def __init__(self, client, filePath):
         self.client = client
@@ -158,7 +161,7 @@ def __init__(self, client, filePath):
         self.size = None
 
     def set_attributes(self, attributes):
-        self.last_modified = datetime.strptime(attributes['last_modified'],'%Y-%m-%dT%H:%M:%S.%fZ')
+        self.last_modified = datetime.strptime(attributes['last_modified'], '%Y-%m-%dT%H:%M:%S.%fZ')
         self.size = attributes['size']
 
     # Get file from the data api
@@ -229,9 +232,76 @@ def delete(self):
         except:
             raise DataApiError('Failed to delete local file ' + self.path)
 
+
 def localPutHelper(path, contents):
     try:
         with open(path, 'wb') as f:
             f.write(contents)
             return dict(status='success')
-    except Exception as e: return dict(error=str(e))
+    except Exception as e:
+        return dict(error=str(e))
+
+
+class AdvancedDataFile(DataFile, RawIOBase):
+    def __init__(self, client, dataUrl, cleanup=True):
+        super(AdvancedDataFile, self).__init__(client, dataUrl)
+        self.cleanup = cleanup
+        self.local_file = None
+
+    def __del__(self):
+        if self.local_file:
+            self.local_file.close()
+            if self.cleanup:
+                    os.remove(self.local_file)
+
+    def readable(self):
+        return True
+
+    def seekable(self):
+        return True
+
+    def writable(self):
+        return False
+
+    def read(self, __size=None):
+        if not self.local_file:
+            self.local_file = self.getFile()
+        if __size:
+            output = self.local_file.read(__size)
+        else:
+            output = self.local_file.read()
+        return output
+
+    def readline(self, __size=None):
+        if not self.local_file:
+            self.local_file = self.getFile()
+        with self.local_file as f:
+            if __size:
+                output = f.readline(__size)
+            else:
+                output = f.readline()
+        return output
+
+    def readlines(self, __hint=None):
+        if not self.local_file:
+            self.local_file = self.getFile()
+        if __hint:
+            output = self.local_file.readlines(__hint)
+        else:
+            output = self.local_file.readlines()
+        return output
+
+    def tell(self):
+        if not self.local_file:
+            self.local_file = self.getFile()
+        output = self.local_file.tell()
+        return output
+
+    def seek(self, __offset, __whence=None):
+        if not self.local_file:
+            self.local_file = self.getFile()
+        if __whence:
+            output = self.local_file.seek(__offset, __whence)
+        else:
+            output = self.local_file.seek(__offset)
+        return output

Test/datafile_test.py

@@ -7,7 +7,8 @@
 import unittest, os, uuid
 import numpy as np
 import Algorithmia
-from Algorithmia.datafile import DataFile, LocalDataFile
+import json
+from Algorithmia.datafile import DataFile, LocalDataFile, AdvancedDataFile
 
 class DataFileTest(unittest.TestCase):
     def setUp(self):
@@ -113,5 +114,42 @@ def test_read_types(self):
         txt = self.client.file(self.EXISTING_FILE).getFile().read()
         self.assertEqual(txt, self.EXISTING_TEXT)
 
+class AdvancedDataFileTest(unittest.TestCase):
+    def setUp(self):
+        self.client = Algorithmia.client()
+        if not self.client.dir("data://.my/empty").exists():
+            self.client.dir("data://.my/empty").create()
+
+    def test_get_nonexistant(self):
+        try:
+            with self.client.file('data://.my/nonexistant/nonreal') as f:
+                _ = f.read()
+            retrieved_file = True
+        except Exception as e:
+            retrieved_file = False
+        self.assertFalse(retrieved_file)
+
+    def test_get_str(self):
+        df = self.client.file('data://.my/nonexistant/nonreal', cleanup=True)
+        try:
+            print(df.getString())
+            retrieved_file = True
+        except Exception as e:
+            retrieved_file = False
+        self.assertFalse(retrieved_file)
+
+    def test_putJson_getJson(self):
+        file = '.my/empty/test.json'
+        df = AdvancedDataFile(self.client, 'data://' + file, cleanup=True)
+        if sys.version_info[0] < 3:
+            payload = {u"hello":u"world"}
+        else:
+            payload = {"hello": "world"}
+        response = df.putJson(payload)
+        self.assertEqual(response.path,file)
+        result = json.loads(df.read())
+        self.assertDictEqual(result, payload)
+        self.assertEqual(str(result), str(payload))
+
 if __name__ == '__main__':
     unittest.main()