first commit

Author: toddbirchard

.DS_Store

@@ -0,0 +1,2 @@
+HackersAndSlackers-d2a47db89384.json
+HackersAndSlackers-d2a47db89384.json

.gitignore

@@ -0,0 +1,14 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+google-cloud-storage="*"
+google-cloud-bigquery="*"
+pprint="*"
+
+[requires]
+python_version = "3.7"

Pipfile

@@ -0,0 +1 @@
+# bigquery-python-tutorial

Pipfile.lock

@@ -0,0 +1,15 @@
+id,initiated,hiredate,email,firstname,lastname,title,department,location,country,type
+100035435,2015-12-11T09:16:20.722-08:00,3/22/67,GretchenRMorrow@jourrapide.com,Gretchen,Morrow,Power plant operator,Physical Product,Britling Cafeterias,United Kingdom,Employee
+100056435,2015-12-15T10:11:24.604-08:00,6/22/99,ElizabethLSnow@armyspy.com,Elizabeth,Snow,Oxygen therapist,Physical Product,Grade A Investment,United States of America,Employee
+100037955,2015-12-16T14:31:32.765-08:00,5/31/74,AlbertMPeterson@einrot.com,Albert,Peterson,Psychologist,Physical Product,Grass Roots Yard Services,United States of America,Employee
+100035435,2016-01-20T11:15:47.249-08:00,9/9/69,JohnMLynch@dayrep.com,John,Lynch,Environmental hydrologist,Physical Product,Waccamaw's Homeplace,United States of America,Employee
+100057657,2016-01-21T12:45:38.261-08:00,4/9/83,TheresaJCahoon@teleworm.us,Theresa,Cahoon,Personal chef,Physical Product,Cala Foods,United States of America,Employee
+100056747,2016-02-01T11:25:39.317-08:00,6/26/98,KennethHPayne@dayrep.com,Kenneth,Payne,Central office operator,Frontline,Magna Consulting,United States of America,Employee
+100035435,2016-02-01T11:28:11.953-08:00,4/16/82,LeifTSpeights@fleckens.hu,Leif,Speights,Staff development director,Frontline,Rivera Property Maintenance,United States of America,Employee
+100035435,2016-02-01T12:21:01.756-08:00,8/6/80,JamesSRobinson@teleworm.us,James,Robinson,Scheduling clerk,Frontline,Discount Furniture Showcase,United States of America,Employee
+100074688,2016-02-01T13:29:19.147-08:00,12/14/74,AnnaDMoberly@jourrapide.com,Anna,Moberly,Playwright,Physical Product,The Wiz,United States of America,Employee
+100665778,2016-02-04T14:40:05.223-08:00,9/13/66,MarjorieBCrawford@armyspy.com,Marjorie,Crawford,"Court, municipal, and license clerk",Physical Product,The Serendipity Dip,United Kingdom,Employee
+100876876,2016-02-24T12:39:25.872-08:00,12/19/67,LyleCHackett@fleckens.hu,Lyle,Hackett,Airframe mechanic,Physical Product,Infinity Investment Plan,United States of America,Employee
+100658565,2016-02-29T15:52:12.933-08:00,11/17/83,MaryJDensmore@jourrapide.com,Mary,Densmore,Employer relations representative,Frontline,One-Up Realtors,United States of America,Employee
+100766547,2016-03-01T12:32:53.357-08:00,10/1/87,CindyRDiaz@armyspy.com,Cindy,Diaz,Student affairs administrator,Physical Product,Mr. AG's,United States of America,Employee
+100045677,2016-03-02T12:07:44.264-08:00,8/16/65,AndreaTLigon@einrot.com,Andrea,Ligon,Railroad engineer,Central Growth,Robinson Furniture,United States of America,Employee

README.md

@@ -0,0 +1,81 @@
+from google.cloud import storage
+from google.cloud import bigquery
+import pprint
+
+bucket_uri = 'gs://your-bucket/'
+bucket_name = 'your-bucket'
+bucket_target = 'datasets/data_upload.csv'
+local_dataset = 'data/test.csv'
+bucket_target_uri = bucket_uri + bucket_target
+bigquery_dataset = 'uploadtest'
+bigquery_table = 'my_table'
+
+
+def upload_blob(bucket_name, source_file_name, destination_blob_name):
+    """Upload a CSV to Google Cloud Storage.
+
+    1. Retrieve the target bucket.
+    2. Set destination of data to be uploaded.
+    3. Upload local CSV.
+    """
+    storage_client = storage.Client()
+    bucket = storage_client.get_bucket(bucket_name)
+    blob = bucket.blob(destination_blob_name)
+    # Commence Upload
+    blob.upload_from_filename(source_file_name)
+    print('File {} uploaded to {}.'.format(
+        source_file_name,
+        destination_blob_name))
+
+
+def insert_bigquery(target_uri, dataset_id, table_id):
+    """Insert CSV from Google Storage to BigQuery Table.
+
+    1. Specify target dataset within BigQuery.
+    2. Create a Job configuration.
+    3. Specify that we are autodetecting datatypes.
+    4. Reserve row #1 for headers.
+    5. Specify the source format of the file (defaults to CSV).
+    6. Pass the URI of the data storage on Google Cloud Storage from.
+    7. Load BigQuery Job.
+    8. Execute BigQuery Job.
+    """
+    bigquery_client = bigquery.Client()
+    dataset_ref = bigquery_client.dataset(dataset_id)
+    job_config = bigquery.LoadJobConfig()
+    job_config.autodetect = True
+    job_config.skip_leading_rows = 1
+    job_config.source_format = bigquery.SourceFormat.CSV
+    uri = target_uri
+    load_job = bigquery_client.load_table_from_uri(
+        uri,
+        dataset_ref.table(table_id),
+        job_config=job_config)  # API request
+    print('Starting job {}'.format(load_job.job_id))
+    # Waits for table load to complete.
+    load_job.result()
+    print('Job finished.')
+
+
+def get_schema(dataset_id, table_id):
+    """Get BigQuery Table Schema.
+
+    1. Specify target dataset within BigQuery.
+    2. Specify target table within given dataset.
+    3. Create Table class instance from existing BigQuery Table.
+    4. Print results to console.
+    5. Return the schema dict.
+    """
+    bigquery_client = bigquery.Client()
+    dataset_ref = bigquery_client.dataset(dataset_id)
+    bg_tableref = bigquery.table.TableReference(dataset_ref, table_id)
+    bg_table = bigquery_client.get_table(bg_tableref)
+    # Print Schema to Console
+    pp = pprint.PrettyPrinter(indent=4)
+    pp.pprint(bg_table.schema)
+    return bg_table.schema
+
+
+upload_blob(bucket_name, local_dataset, bucket_target)
+insert_bigquery(bucket_target_uri, bigquery_dataset, bigquery_table)
+bigquery_table_schema = get_schema(bigquery_dataset, bigquery_table)