minor tweks

toddbirchard · toddbirchard · commit 505936602f12 · 2020-02-11T06:59:19.000-05:00
diff --git a/Pipfile b/Pipfile
@@ -9,8 +9,7 @@ verify_ssl = true
 SQLAlchemy = "*"
 PyBigQuery = "*"
 PyMySQL = "*"
-Psycopg2-Binary = "*"
 Loguru = "*"
 
 [requires]
-python_version = "3.7"
+python_version = "3.8"
diff --git a/Pipfile.lock b/Pipfile.lock
diff --git a/README.md b/README.md
@@ -1,11 +1,10 @@
 # BigQuery SQLAlchemy Tutorial
 
-![Python](https://img.shields.io/badge/Python-v3.7-blue.svg?logo=python&longCache=true&logoColor=white&colorB=5e81ac&style=flat-square&colorA=4c566a)
-![Google Cloud BigQuery](https://img.shields.io/badge/Google--Cloud--BigQuery-v1.11.2-blue.svg?logo=Google&longCache=true&logoColor=white&colorB=5e81ac&style=flat-square&colorA=4c566a)
-![PyBigQuery](https://img.shields.io/badge/PyBigQuery-v0.4.11-blue.svg?logo=Google&longCache=true&logoColor=white&colorB=5e81ac&style=flat-square&colorA=4c566a)
-![SQLAlchemy](https://img.shields.io/badge/SQLAlchemy-v1.3.1-red.svg?longCache=true&style=flat-square&logo=scala&logoColor=white&colorA=4c566a&colorB=bf616a)
+![Python](https://img.shields.io/badge/Python-v^3.7-blue.svg?logo=python&longCache=true&logoColor=white&colorB=5e81ac&style=flat-square&colorA=4c566a)
+![Google Cloud BigQuery](https://img.shields.io/badge/Google--Cloud--BigQuery-v1.24.0-blue.svg?logo=Google&longCache=true&logoColor=white&colorB=5e81ac&style=flat-square&colorA=4c566a)
+![PyBigQuery](https://img.shields.io/badge/PyBigQuery-v0.4.13-blue.svg?logo=Google&longCache=true&logoColor=white&colorB=5e81ac&style=flat-square&colorA=4c566a)
+![SQLAlchemy](https://img.shields.io/badge/SQLAlchemy-v1.3.13-red.svg?longCache=true&style=flat-square&logo=scala&logoColor=white&colorA=4c566a&colorB=bf616a)
 ![PyMySQL](https://img.shields.io/badge/PyMySQL-v0.9.3-red.svg?longCache=true&style=flat-square&logo=mysql&logoColor=white&colorA=4c566a&colorB=bf616a)
-![Psycopg2-Binary](https://img.shields.io/badge/Psycopg2--Binary-v2.8.4-red.svg?longCache=true&style=flat-square&logo=postgresql&logoColor=white&colorA=4c566a&colorB=bf616a)
 ![GitHub Last Commit](https://img.shields.io/github/last-commit/google/skia.svg?style=flat-square&colorA=4c566a&colorB=a3be8c)
 [![GitHub Issues](https://img.shields.io/github/issues/hackersandslackers/bigquery-sqlalchemy-tutorial.svg?style=flat-square&colorA=4c566a&colorB=ebcb8b)](https://github.com/hackersandslackers/bigquery-python-tutorial/issues)
 [![GitHub Stars](https://img.shields.io/github/stars/hackersandslackers/bigquery-sqlalchemy-tutorial.svg?style=flat-square&colorB=ebcb8b&colorA=4c566a)](https://github.com/hackersandslackers/bigquery-python-tutorial/stargazers)
diff --git a/biquery_sql_etl/__init__.py b/biquery_sql_etl/__init__.py
@@ -1,18 +1,27 @@
+from loguru import logger
 from biquery_sql_etl.engines import bigquery_engine, rdbms_engine
 from biquery_sql_etl.queries import sql_queries
 from biquery_sql_etl.client import DataClient
-from loguru import logger
 from config import bigquery_table
 
 
 logger.add('logs/queries.log', format="{time} {message}", level="INFO")
 
 
-def main():
-    """Move data between sources."""
-    bqc = DataClient(bigquery_engine)
-    dbc = DataClient(rdbms_engine)
+def init_pipeline():
+    """Move data between Bigquery and MySQL."""
+    total_rows_affected = 0
+    bqc, dbc = data_sources()
     for table_name, query in sql_queries.items():
         fetched_rows = bqc.fetch_rows(query, table=bigquery_table)
         inserted_rows = dbc.insert_rows(fetched_rows, table_name, replace=True)
         logger.info(inserted_rows)
+        total_rows_affected += len(fetched_rows)
+    logger.info(f"Completed migration of {total_rows_affected} rows from BigQuery to MySQL.")
+
+
+def data_sources():
+    """Construct datasources."""
+    bqc = DataClient(bigquery_engine)
+    dbc = DataClient(rdbms_engine)
+    return bqc, dbc
diff --git a/biquery_sql_etl/client.py b/biquery_sql_etl/client.py
@@ -1,4 +1,4 @@
-"""Base Data Client."""
+"""Generic Client for interacting with data sources."""
 from sqlalchemy import MetaData, Table
 
 
diff --git a/biquery_sql_etl/engines/bigquery.py b/biquery_sql_etl/engines/bigquery.py
@@ -1,8 +1,8 @@
-"""SQL Database Engine."""
-
+"""BigQuery Engine."""
 from sqlalchemy.engine import create_engine
 from config import (gcp_credentials,
                     bigquery_uri)
 
 
-bigquery_engine = create_engine(bigquery_uri, credentials_path=gcp_credentials)
+bigquery_engine = create_engine(bigquery_uri,
+                                credentials_path=gcp_credentials)
diff --git a/biquery_sql_etl/queries.py b/biquery_sql_etl/queries.py
@@ -5,7 +5,7 @@
 
 
 def get_local_sql_files():
-    """Fetch all SQL query files in folder."""
+    """Fetch all .sql files from local folder."""
     files = [local_sql_folder + '/' + f for f in listdir(local_sql_folder) if isfile(join(local_sql_folder, f)) if '.sql' in f]
     return files
 
diff --git a/config.py b/config.py
@@ -15,5 +15,5 @@
 rdbms_name = environ.get('DATABASE_NAME')
 rdbms_uri = f'mysql+pymysql://{rdbms_user}:{rdbms_pass}@{rdbms_host}:{rdbms_port}/{rdbms_name}'
 
-# Local
+# Locally stored queries
 local_sql_folder = 'sql'
diff --git a/main.py b/main.py
@@ -1,5 +1,5 @@
 """Application entry point."""
-from biquery_sql_etl import main
+from biquery_sql_etl import init_pipeline
 
 if __name__ == '__main__':
-    main()
+    init_pipeline()
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,12 +1,12 @@
 [tool.poetry]
 name = "biquery_sql_etl"
-version = "0.1.0"
-description = "Simple Python project built with Poetry."
+version = "1.0.0"
+description = "Light ETL script to migrate data from BigQuery to SQL and vice-versa."
 authors = ["Todd Birchard <toddbirchard@gmail.com>"]
 maintainers = ["Todd Birchard <toddbirchard@gmail.com>"]
 license = "MIT"
 readme = "README.md"
-homepage = ""
+homepage = "https://hackersandslackers.com/bigquery-and-sql-databases/"
 repository = "https://github.com/hackersandslackers/bigquery-sqlalchemy-tutorial/"
 documentation = "https://hackersandslackers.com/bigquery-and-sql-databases/"
 keywords = ["BigQuery",
@@ -16,11 +16,10 @@ keywords = ["BigQuery",
             "Google Cloud"]
 
 [tool.poetry.dependencies]
-Python = "^3.7"
+Python = "^3.8"
 SQLAlchemy = "*"
 PyBigQuery = "*"
 PyMySQL = "*"
-Psycopg2-Binary = "*"
 Loguru = "*"
 
 [tool.poetry.dev-dependencies]
diff --git a/requirements.txt b/requirements.txt
@@ -1,4 +1,3 @@
-bigquery-sqlalchemy-tutorial==1.0.0
 cachetools==4.0.0
 certifi==2019.11.28
 chardet==3.0.4
@@ -12,7 +11,6 @@ googleapis-common-protos==1.51.0
 idna==2.8
 loguru==0.4.1
 protobuf==3.11.3
-psycopg2-binary==2.8.4
 pyasn1==0.4.8
 pyasn1-modules==0.2.8
 pybigquery==0.4.13
diff --git a/setup.py b/setup.py
@@ -22,13 +22,14 @@
         'Development Status :: 3 - Alpha',
         'Intended Audience :: Developers',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
     ],
-    keywords='SQLAlchemy Google BigQuery SQL ETL',
+    keywords='SQLAlchemy GCP Google BigQuery SQL ETL RDBMS',
     packages=find_packages(),
     install_requires=['SQLAlchemy',
                       'PyBigQuery',
                       'PyMySQL',
-                      'Psycopg2-Binary'],
+                      'Loguru'],
     extras_require={
         'dev': ['check-manifest'],
         'test': ['coverage'],

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-"""Base Data Client."""`
	`1`	`+"""Generic Client for interacting with data sources."""`
`2`	`2`	`from sqlalchemy import MetaData, Table`
`3`	`3`
`4`	`4`