data client inheritance

toddbirchard · toddbirchard · commit 0006ed31b0e8 · 2019-11-15T02:42:35.000-05:00
diff --git a/Pipfile b/Pipfile
@@ -6,7 +6,6 @@ verify_ssl = true
 [dev-packages]
 
 [packages]
-Google-Cloud-BigQuery = "*"
 SQLAlchemy = "*"
 PyBigQuery = "*"
 PyMySQL = "*"
diff --git a/Pipfile.lock b/Pipfile.lock
diff --git a/config.py b/config.py
diff --git a/config/bigquery_config.py b/config/bigquery_config.py
@@ -0,0 +1,9 @@
+from os import environ
+
+# Google Cloud config
+gcp_credentials = environ.get('GCP_CREDENTIALS')
+gcp_project = environ.get('GCP_PROJECT')
+
+# Google BigQuery config
+bigquery_dataset = environ.get('GCP_BIGQUERY_DATASET')
+bigquery_table = environ.get('GCP_BIGQUERY_TABLE')
diff --git a/config/sql_config.py b/config/sql_config.py
@@ -0,0 +1,10 @@
+from os import environ
+
+
+# SQL database config
+database_username = environ.get('DATABASE_USERNAME')
+database_password = environ.get('DATABASE_PASSWORD')
+database_host = environ.get('DATABASE_HOST')
+database_port = environ.get('DATABASE_PORT')
+database_name = environ.get('DATABASE_DATABASE_NAME')
+database_table = environ.get('DATABASE_TABLE_NAME')
diff --git a/requirements.txt b/requirements.txt
@@ -3,10 +3,10 @@ certifi==2019.9.11
 chardet==3.0.4
 future==0.18.2
 google-api-core==1.14.3
-google-auth==1.7.0
-google-cloud-bigquery==1.21.0
+google-auth==1.7.1
+google-cloud-bigquery==1.22.0
 google-cloud-core==1.0.3
-google-resumable-media==0.4.1
+google-resumable-media==0.5.0
 googleapis-common-protos==1.6.0
 idna==2.8
 protobuf==3.10.0
diff --git a/src/__init__.py b/src/__init__.py
@@ -1,27 +1,9 @@
-from config import Config
-from src.bigquery import BigQueryClient
-from src.database import SQLClient
-from src.util.read import get_query_from_file
+from src.sources.bigquery import bqc
+from src.sources.database import dbc
+from src.queries import analytics_query
 
 
 def main():
-    bigquery = init_bigquery()
-    database = init_database()
-    query = get_query_from_file(Config.bigquery_query)
-    rows = bigquery.fetch_rows(query)
-    updated = database.insert_rows(rows, replace=True)
+    rows = bqc.fetch_rows(analytics_query)
+    updated = dbc.insert_rows(rows, replace=True)
     print(updated)
-
-
-def init_bigquery():
-    """Initiate BigQuery Client."""
-    uri, creds = Config.get_bigquery_config()
-    bigquery = BigQueryClient(uri=uri, creds=creds)
-    return bigquery
-
-
-def init_database():
-    """Initiate SQL database Client."""
-    uri, table = Config.get_database_config(type='mysql')
-    bigquery = SQLClient(uri=uri, table=table)
-    return bigquery
diff --git a/src/bigquery.py b/src/bigquery.py
diff --git a/src/database.py b/src/database.py
diff --git a/src/queries/__init__.py b/src/queries/__init__.py
@@ -0,0 +1,4 @@
+from .read import read_query
+
+prefix = 'src/queries/sql'
+analytics_query = read_query(f'{prefix}/analytics.sql')
diff --git a/src/queries/read.py b/src/queries/read.py
@@ -1,7 +1,7 @@
 """Read SQL queries from local files."""
 
 
-def get_query_from_file(path):
+def read_query(path):
     """Read contents of file."""
     file = open(path, 'r')
     query = file.read()
diff --git a/src/queries/sql/analytics.sql b/src/queries/sql/analytics.sql
diff --git a/src/sources/__init__.py b/src/sources/__init__.py
@@ -0,0 +1,2 @@
+from .bigquery import bqc
+from .database import dbc
diff --git a/src/sources/bigquery/__init__.py b/src/sources/bigquery/__init__.py
@@ -0,0 +1,10 @@
+from config.bigquery_config import (gcp_credentials,
+                                    gcp_project,
+                                    bigquery_dataset,
+                                    bigquery_table)
+from .bigquery_client import BigQueryClient
+
+bqc = BigQueryClient(project=gcp_project,
+                     dataset=bigquery_dataset,
+                     table=bigquery_table,
+                     creds=gcp_credentials)
diff --git a/src/sources/bigquery/bigquery_client.py b/src/sources/bigquery/bigquery_client.py
@@ -0,0 +1,14 @@
+"""BigQuery Client."""
+from sqlalchemy.engine import create_engine
+from sqlalchemy import MetaData
+from src.sources.client import BaseClient
+
+
+class BigQueryClient(BaseClient):
+
+    def __init__(self, project=None, dataset=None, creds=None, table=None):
+        self.uri = f'bigquery://{project}/{dataset}'
+        self.engine = create_engine(self.uri, credentials_path=creds)
+        super().__init__(engine=self.engine,
+                         metadata=MetaData(bind=self.engine),
+                         table=table)
diff --git a/src/sources/client.py b/src/sources/client.py
@@ -0,0 +1,35 @@
+"""Base Client."""
+import logging
+from sqlalchemy import Table
+
+
+logging.basicConfig(filename='logs/queries.log',
+                    format='%(asctime)s %(message)s')
+logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
+
+
+class BaseClient:
+
+    def __init__(self, engine=None, metadata=None, table=None):
+        self.table = table
+        self.engine = engine
+        self.metadata = metadata
+
+    def insert_rows(self, rows, replace=None):
+        """Insert rows into table."""
+        if replace:
+            self.engine.execute(f'TRUNCATE TABLE {self.table}')
+        table = Table(self.table, self.metadata, autoload=True)
+        self.engine.execute(table.insert(), rows)
+        return self.__construct_response(rows)
+
+    def fetch_rows(self, query):
+        """Fetch all rows via query."""
+        rows = self.engine.execute(query).fetchall()
+        return rows
+
+    def __construct_response(self, rows):
+        columns = rows[0].keys()
+        column_names = ", ".join(columns)
+        num_rows = len(rows)
+        return f'Inserted {num_rows} rows into `{self.table}` with {len(columns)} columns: {column_names}'
diff --git a/src/sources/database/__init__.py b/src/sources/database/__init__.py
@@ -0,0 +1,15 @@
+from config.sql_config import (database_username,
+                               database_password,
+                               database_host,
+                               database_port,
+                               database_table,
+                               database_name)
+from .database_client import Database
+
+dbc = Database('mysql',
+               username=database_username,
+               password=database_password,
+               host=database_host,
+               port=database_port,
+               db_name=database_name,
+               table=database_table)
diff --git a/src/sources/database/database_client.py b/src/sources/database/database_client.py
@@ -0,0 +1,30 @@
+"""SQL Database Client."""
+from sqlalchemy.engine import create_engine
+from sqlalchemy import MetaData
+from src.sources.client import BaseClient
+
+
+class Database(BaseClient):
+
+    def __init__(self,
+                 type,
+                 username=None,
+                 password=None,
+                 host=None,
+                 port=None,
+                 table=None,
+                 db_name=None):
+        self.table = table
+        self.type = self.__conection_type(type)
+        self.engine = create_engine(f'{self.type}://{username}:{password}@{host}:{port}/{db_name}')
+        super().__init__(engine=self.engine,
+                         metadata=MetaData(bind=self.engine),
+                         table=table)
+
+    @staticmethod
+    def __conection_type(type):
+        if type.lower() == 'mysql':
+            return 'mysql+pymysql'
+        if type.lower() == 'postgres':
+            return 'postgresql+psycopg2'
+        return None
