Merge pull request #6 from heliumdatacommons/5-irods-ref-resource

5 irods ref resource

Author: hyi

hs_core/hydroshare/resource.py

@@ -295,7 +295,7 @@ def create_resource(
         resource_type, owner, title,
         edit_users=None, view_users=None, edit_groups=None, view_groups=None,
         keywords=(), metadata=None, extra_metadata=None,
-        files=(), source_names=[], fed_res_path='', move=False,
+        files=(), source_names=[], source_sizes=[], fed_res_path='', move=False, is_file_reference=False,
         create_metadata=True,
         create_bag=True, unpack_file=False, **kwargs):
     """
@@ -341,11 +341,16 @@ def create_resource(
     :param files: list of Django File or UploadedFile objects to be attached to the resource
     :param source_names: a list of file names from a federated zone to be
          used to create the resource in the federated zone, default is empty list
+    :param source_sizes: a list of file sizes corresponding to source_names if if_file_reference is True; otherwise,
+         it is not of any use and should be empty.
     :param fed_res_path: the federated zone path in the format of
          /federation_zone/home/localHydroProxy that indicate where the resource
          is stored, default is empty string
     :param move: a value of False or True indicating whether the content files
          should be erased from the source directory. default is False.
+    :param is_file_reference: a value of False or True indicating whether the files stored in
+        source_files are references to external files without being physically stored in
+        HydroShare internally. default is False.
     :param create_bag: whether to create a bag for the newly created resource or not.
         By default, the bag is created.
     :param unpack_file: boolean.  If files contains a single zip file, and unpack_file is True,
@@ -392,7 +397,7 @@ def create_resource(
             resource.save()
 
         # TODO: It would be safer to require an explicit zone path rather than harvesting file path
-        elif len(source_names) > 0:
+        elif len(source_names) > 0 and fed_res_path:
             fed_zone_home_path = utils.get_federated_zone_home_path(source_names[0])
             resource.resource_federation_path = fed_zone_home_path
             resource.save()
@@ -408,8 +413,8 @@ def create_resource(
             # few seconds.  We may want to add the option to do this
             # asynchronously if the file size is large and would take
             # more than ~15 seconds to complete.
-            add_resource_files(resource.short_id, *files, source_names=source_names,
-                               move=move)
+            add_resource_files(resource.short_id, *files, source_names=source_names, source_sizes=source_sizes,
+                               move=move, is_file_reference=is_file_reference)
 
         # by default resource is private
         resource_access = ResourceAccess(resource=resource)
@@ -625,6 +630,8 @@ def add_resource_files(pk, *files, **kwargs):
     resource = utils.get_resource_by_shortkey(pk)
     ret = []
     source_names = kwargs.pop('source_names', [])
+    source_sizes = kwargs.pop('source_sizes', [])
+    is_file_reference = kwargs.pop('is_file_reference', False)
 
     if __debug__:
         assert(isinstance(source_names, list))
@@ -641,14 +648,24 @@ def add_resource_files(pk, *files, **kwargs):
         ret.append(utils.add_file_to_resource(resource, f, folder=folder))
 
     if len(source_names) > 0:
+        if len(source_names) != len(source_sizes):
+            # if length is not equal, there is an issue with source_sizes input parameter, so it will not be
+            # used by setting it to be empty
+            source_sizes = []
+
+        idx = 0
         for ifname in source_names:
+            s_size = source_sizes[idx] if source_sizes else 0
+            idx += 1
             ret.append(utils.add_file_to_resource(resource, None,
                                                   folder=folder,
                                                   source_name=ifname,
-                                                  move=move))
-    if not ret:
-        # no file has been added, make sure data/contents directory exists if no file is added
-        utils.create_empty_contents_directory(resource)
+                                                  source_size=s_size,
+                                                  move=move,
+                                                  is_file_reference=is_file_reference))
+
+    # make sure data/contents directory exists if not exist already
+    utils.create_empty_contents_directory(resource)
 
     return ret
 

hs_core/hydroshare/utils.py

@@ -896,14 +896,16 @@ def resource_file_add_pre_process(resource, files, user, extract_metadata=False,
 
 # TODO: make this part of resource api. resource --> self.
 def resource_file_add_process(resource, files, user, extract_metadata=False,
-                              source_names=[], **kwargs):
+                              source_names=[], source_sizes=[], is_file_reference=False, **kwargs):
 
     from .resource import add_resource_files
     if __debug__:
         assert(isinstance(source_names, list))
     folder = kwargs.pop('folder', None)
     resource_file_objects = add_resource_files(resource.short_id, *files, folder=folder,
-                                               source_names=source_names)
+                                               source_names=source_names,
+                                               source_sizes=source_sizes,
+                                               is_file_reference=is_file_reference)
 
     # receivers need to change the values of this dict if file validation fails
     # in case of file validation failure it is assumed the resource type also deleted the file
@@ -929,8 +931,8 @@ def create_empty_contents_directory(resource):
         istorage.session.run("imkdir", None, '-p', res_contents_dir)
 
 
-def add_file_to_resource(resource, f, folder=None, source_name='',
-                         move=False):
+def add_file_to_resource(resource, f, folder=None, source_name='', source_size=0,
+                         move=False, is_file_reference=False):
     """
     Add a ResourceFile to a Resource.  Adds the 'format' metadata element to the resource.
     :param resource: Resource to which file should be added
@@ -943,12 +945,16 @@ def add_file_to_resource(resource, f, folder=None, source_name='',
                         disk, or from the federated zone directly where f is empty
                         but source_name has the whole data object
                         iRODS path in the federated zone
+    :param source_size: the size of the reference file in source_name if is_file_reference is True; otherwise, it is
+                        set to 0 and useless.
     :param move: indicate whether the file should be copied or moved from private user
                  account to proxy user account in federated zone; A value of False
                  indicates copy is needed, a value of True indicates no copy, but
                  the file will be moved from private user account to proxy user account.
                  The default value is False.
-
+    :param is_file_reference: indicate whether the file being added is a reference to an external
+                              file stored in an external zone or URL. source_name will hold
+                              the reference file path or url
     :return: The identifier of the ResourceFile added.
     """
 
@@ -966,7 +972,8 @@ def add_file_to_resource(resource, f, folder=None, source_name='',
     elif source_name:
         try:
             # create from existing iRODS file
-            ret = ResourceFile.create(resource, None, folder=folder, source=source_name, move=move)
+            ret = ResourceFile.create(resource, None, folder=folder, source=source_name, source_size=source_size,
+                                      is_file_reference=is_file_reference, move=move)
         except SessionException as ex:
             try:
                 ret.delete()

hs_core/migrations/0036_resourcefile_reference_file_path.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('hs_core', '0035_remove_deprecated_fields'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='resourcefile',
+            name='reference_file_path',
+            field=models.CharField(max_length=255, null=True, blank=True),
+        ),
+    ]

hs_core/migrations/0037_resourcefile_reference_file_size.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('hs_core', '0036_resourcefile_reference_file_path'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='resourcefile',
+            name='reference_file_size',
+            field=models.CharField(max_length=15, null=True, blank=True),
+        ),
+    ]

hs_core/models.py

@@ -2618,10 +2618,10 @@ class ResourceFile(ResourceFileIRODSMixin):
     fed_resource_file = models.FileField(upload_to=get_path, max_length=4096,
                                          null=True, blank=True, storage=FedStorage())
 
-    # DEPRECATED: utilize resfile.set_storage_path(path) and resfile.storage_path.
-    # fed_resource_file_name_or_path = models.CharField(max_length=255, null=True, blank=True)
-    # DEPRECATED: use native size routine
-    # fed_resource_file_size = models.CharField(max_length=15, null=True, blank=True)
+    # This is used to hold the reference path to an external file, e.g., a logical iRODS
+    # path refering to a file stored in an external iRODS zone, a URL that points to an external file
+    reference_file_path = models.CharField(max_length=255, null=True, blank=True)
+    reference_file_size = models.CharField(max_length=15, null=True, blank=True)
 
     # we are using GenericForeignKey to allow resource file to be associated with any
     # CommonsShare defined LogicalFile types (e.g., GeoRasterFile, NetCdfFile etc)
@@ -2640,7 +2640,7 @@ def __str__(self):
             return self.resource_file.name
 
     @classmethod
-    def create(cls, resource, file, folder=None, source=None, move=False):
+    def create(cls, resource, file, folder=None, source=None, source_size=0, is_file_reference=False, move=False):
         """Create custom create method for ResourceFile model.
 
         Create takes arguments that are invariant of storage medium.
@@ -2651,6 +2651,9 @@ def create(cls, resource, file, folder=None, source=None, move=False):
         :param file: a File or a iRODS path to an existing file already copied.
         :param folder: the folder in which to store the file.
         :param source: an iRODS path in the same zone from which to copy the file.
+        :param source_size: the size of the referenced source is if_file_reference is True, otherwise, it is set to 0
+               and useless.
+        :param is_file_reference: if True, source will hold the reference path to an external file
         :param move: if True, move the file rather than copying.
 
         There are two main usages to this constructor:
@@ -2697,23 +2700,31 @@ def create(cls, resource, file, folder=None, source=None, move=False):
             if file is None and source is not None:
                 if __debug__:
                     assert(isinstance(source, basestring))
-                # source is a path to an iRODS file to be copied here.
-                root, newfile = os.path.split(source)  # take file from source path
-                # newfile is where it should be copied to.
-                target = get_resource_file_path(resource, newfile, folder=folder)
-                istorage = resource.get_irods_storage()
-                if not istorage.exists(source):
-                    raise ValidationError("ResourceFile.create: source {} of copy not found"
-                                          .format(source))
-                if not move:
-                    istorage.copyFiles(source, target)
+
+                if is_file_reference:
+                    kwargs['resource_file'] = None
+                    kwargs['fed_resource_file'] = None
+                    kwargs['reference_file_path'] = source
+                    kwargs['reference_file_size'] = str(source_size)
+                    return ResourceFile.objects.create(**kwargs)
                 else:
-                    istorage.moveFile(source, target)
-                if not istorage.exists(target):
-                    raise ValidationError("ResourceFile.create: copy to target {} failed"
-                                          .format(target))
-                if move and istorage.exists(source):
-                    raise ValidationError("ResourceFile.create: move did not work")
+                    # source is a path to an iRODS file to be copied here.
+                    root, newfile = os.path.split(source)  # take file from source path
+                    # newfile is where it should be copied to.
+                    target = get_resource_file_path(resource, newfile, folder=folder)
+                    istorage = resource.get_irods_storage()
+                    if not istorage.exists(source):
+                        raise ValidationError("ResourceFile.create: source {} of copy not found"
+                                              .format(source))
+                    if not move:
+                        istorage.copyFiles(source, target)
+                    else:
+                        istorage.moveFile(source, target)
+                    if not istorage.exists(target):
+                        raise ValidationError("ResourceFile.create: copy to target {} failed"
+                                              .format(target))
+                    if move and istorage.exists(source):
+                        raise ValidationError("ResourceFile.create: move did not work")
             elif file is not None and source is None:
                 # file points to an existing iRODS file
                 # no need to verify whether the file exists in iRODS since the file
@@ -2766,11 +2777,13 @@ def size(self):
                 assert self.resource_file.name is None or \
                     self.resource_file.name == ''
             return self.fed_resource_file.size
-        else:
+        elif not self.reference_file_path:
             if __debug__:
                 assert self.fed_resource_file.name is None or \
                     self.fed_resource_file.name == ''
             return self.resource_file.size
+        else:
+            return 0
 
     # TODO: write unit test
     @property
@@ -2874,6 +2887,8 @@ def short_path(self):
         """
         if self.resource.is_federated:
             folder, base = self.path_is_acceptable(self.fed_resource_file.name, test_exists=False)
+        elif self.reference_file_path:
+            return self.reference_file_path
         else:
             folder, base = self.path_is_acceptable(self.resource_file.name, test_exists=False)
         if folder is not None:
@@ -2895,7 +2910,7 @@ def set_short_path(self, path):
         if self.resource.is_federated:
             self.resource_file = None
             self.fed_resource_file = get_path(self, base)
-        else:
+        elif not self.reference_file_path:
             self.resource_file = get_path(self, base)
             self.fed_resource_file = None
         self.save()