Merge pull request #8 from hydroshare:include-published

include published resources in batch update

Author: devincowan

hs_data_services/hs_data_services_sync/management/commands/copy_res_files_from_hydroshare.py

@@ -1,8 +1,6 @@
-import requests
-import json
 from django.core.management.base import BaseCommand
-from hs_data_services import settings
 from hs_data_services_sync import tasks
+from hs_data_services_sync.utilities import get_list_of_public_geo_resources
 
 
 class Command(BaseCommand):
@@ -19,7 +17,7 @@ def handle(self, *args, **options):
                 result = self.copy_files(resource_id)
                 print(result)
         else:
-            resources = self.get_list_of_public_geo_resources()
+            resources = get_list_of_public_geo_resources()
             num_resources = len(resources)
             print(f"Copying files from {num_resources} resources")
             counter = 1
@@ -30,36 +28,5 @@ def handle(self, *args, **options):
                 counter += 1
         print("Done copying files")
 
-    def get_list_of_public_geo_resources(self):
-        hydroshare_url = "/".join(settings.HYDROSHARE_URL.split("/")[:-1])
-        types = ["Geographic Feature (ESRI Shapefiles)", "Geographic Raster"]
-        # replace spaces with + for the query string
-        types = [t.replace(" ", "+") for t in types]
-        params = {
-            "type": types,
-            "availability": ["public"],
-            "geofilter": "false"
-        }
-        rest_url = f"{hydroshare_url}/discoverapi/?filter={json.dumps(params)}"
-        rest_url = rest_url.replace(" ", "")
-        print(f"Getting list of public geospatial resources from: {rest_url}")
-        response = requests.get(rest_url)
-        response_json = response.json()
-        page_count = response_json.get('pagecount', 0)
-        rescount = response_json.get('rescount', 0)
-        perpage = response_json.get('perpage', 0)
-        res_ids = []
-        print(f"Iterating over {page_count} pages of {perpage} resources each, total resources: {rescount}")
-        for i in range(1, page_count + 1):
-            print(f"Getting page {i}")
-            response = requests.get(f"{rest_url}&pnum={i}")
-            response_json = response.json()
-            resources = json.loads(response_json.get('resources', []))
-            new_ids = [resource["short_id"] for resource in resources if resource.get('short_id', None)]
-            res_ids.extend(new_ids)
-        print(f"Found {len(res_ids)} public geospatial resources")
-        print(f"Should match {rescount}")
-        return res_ids
-
     def copy_files(self, resource_id):
         return tasks.update_data_services_task.delay(resource_id)

hs_data_services/hs_data_services_sync/management/commands/unregister_resource_in_geoserver.py

@@ -1,7 +1,4 @@
-import requests
-import json
 from django.core.management.base import BaseCommand
-from hs_data_services import settings
 from hs_data_services_sync import utilities
 
 
@@ -19,7 +16,7 @@ def handle(self, *args, **options):
                 result = utilities.unregister_geoserver_databases(resource_id)
                 print(result)
         else:
-            resources = self.get_list_of_public_geo_resources()
+            resources = utilities.get_list_of_public_geo_resources()
             num_resources = len(resources)
             print(f"Unregistering {num_resources} public resources")
             counter = 1
@@ -29,34 +26,3 @@ def handle(self, *args, **options):
                 print(result)
                 counter += 1
         print("Done unregistering resources")
-
-    def get_list_of_public_geo_resources(self):
-        hydroshare_url = "/".join(settings.HYDROSHARE_URL.split("/")[:-1])
-        types = ["Geographic Feature (ESRI Shapefiles)", "Geographic Raster"]
-        # replace spaces with + for the query string
-        types = [t.replace(" ", "+") for t in types]
-        params = {
-            "type": types,
-            "availability": ["public"],
-            "geofilter": "false"
-        }
-        rest_url = f"{hydroshare_url}/discoverapi/?filter={json.dumps(params)}"
-        rest_url = rest_url.replace(" ", "")
-        print(f"Getting list of public geospatial resources from: {rest_url}")
-        response = requests.get(rest_url)
-        response_json = response.json()
-        page_count = response_json.get('pagecount', 0)
-        rescount = response_json.get('rescount', 0)
-        perpage = response_json.get('perpage', 0)
-        res_ids = []
-        print(f"Iterating over {page_count} pages of {perpage} resources each, total resources: {rescount}")
-        for i in range(1, page_count + 1):
-            print(f"Getting page {i}")
-            response = requests.get(f"{rest_url}&pnum={i}")
-            response_json = response.json()
-            resources = json.loads(response_json.get('resources', []))
-            new_ids = [resource["short_id"] for resource in resources if resource.get('short_id', None)]
-            res_ids.extend(new_ids)
-        print(f"Found {len(res_ids)} public geospatial resources")
-        print(f"Should match {rescount}")
-        return res_ids

hs_data_services/hs_data_services_sync/utilities.py

@@ -601,3 +601,35 @@ def get_layer_style(max_value, min_value, ndv_value, layer_id):
     </StyledLayerDescriptor>"""
 
     return layer_style
+
+
+def get_list_of_public_geo_resources():
+    hydroshare_url = "/".join(settings.HYDROSHARE_URL.split("/")[:-1])
+    types = ["Geographic Feature (ESRI Shapefiles)", "Geographic Raster"]
+    # replace spaces with + for the query string
+    types = [t.replace(" ", "+") for t in types]
+    params = {
+        "type": types,
+        "availability": ["public", "published"],
+        "geofilter": "false"
+    }
+    rest_url = f"{hydroshare_url}/discoverapi/?filter={json.dumps(params)}"
+    rest_url = rest_url.replace(" ", "")
+    print(f"Getting list of public geospatial resources from: {rest_url}")
+    response = requests.get(rest_url)
+    response_json = response.json()
+    page_count = response_json.get('pagecount', 0)
+    rescount = response_json.get('rescount', 0)
+    perpage = response_json.get('perpage', 0)
+    res_ids = []
+    print(f"Iterating over {page_count} pages of {perpage} resources each, total resources: {rescount}")
+    for i in range(1, page_count + 1):
+        print(f"Getting page {i}")
+        response = requests.get(f"{rest_url}&pnum={i}")
+        response_json = response.json()
+        resources = json.loads(response_json.get('resources', []))
+        new_ids = [resource["short_id"] for resource in resources if resource.get('short_id', None)]
+        res_ids.extend(new_ids)
+    print(f"Found {len(res_ids)} public geospatial resources")
+    print(f"Should match {rescount}")
+    return res_ids