First commit

Author: choldgraf

.github/workflows/release.yml

@@ -0,0 +1,50 @@
+name: Generate DB and Create Release
+
+on:
+  push:
+    branches:
+      - main
+  schedule:
+    - cron: '0 0 * * *' # Run daily at midnight UTC
+  workflow_dispatch: # Allows triggering the workflow manually from the GitHub UI
+
+jobs:
+  build_and_release:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4 # Checkout the repository code
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x' # Use your desired Python version
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Create data directory
+        run: mkdir -p data # Ensure the data directory exists
+
+      - name: Run database generation script
+        run: python scripts/download_issues.py # Execute your local script
+
+      - name: Get current date for release name
+        id: get_date
+        run: echo "RELEASE_DATE=$(date +'%Y-%m-%d-%H-%M')" >> $GITHUB_ENV # Set environment variable with current date and time
+
+      - name: Create Release and Upload Database File
+        uses: softprops/action-gh-release@v1
+        with:
+          # Use the current date for the release name
+          name: Database Release ${{ env.RELEASE_DATE }}
+          # A unique tag is required for releases.
+          # We'll use the date and the run ID to ensure uniqueness.
+          tag_name: db-${{ env.RELEASE_DATE }}-${{ github.run_id }}
+          # Attach the generated database file to the release
+          files: data/github.db
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # GitHub automatically provides this token

.gitignore

@@ -172,3 +172,6 @@ cython_debug/
 
 # PyPI configuration file
 .pypirc
+
+# Where we download the data
+data

requirements.txt

@@ -0,0 +1,4 @@
+pandas
+sqlite3
+github-to-sqlite
+rich

scripts/download_issues.py

@@ -0,0 +1,33 @@
+from subprocess import run
+import pandas as pd
+import sqlite3
+from rich.progress import track
+from pathlib import Path
+
+here = Path(__file__).parent
+
+def df_from_sql(query, db):
+    con = sqlite3.connect(db)
+    return pd.read_sql(query, con)
+    con.close()
+
+def download_issues_data(org, db):
+    # Load all repositories in a local DB
+    cmd = f"github-to-sqlite repos {db} {org}"
+    print(cmd)
+    run(cmd.split())
+    repos = df_from_sql("SELECT * FROM repos;", db)
+    repos = repos.set_index("id")
+    
+    # For each repository, download its issues
+    repos = repos["full_name"].tolist()
+    print(f"Downloading issues from {len(repos)} repositories...")
+    for repo in track(repos):
+        cmd = f"github-to-sqlite issues {db} {repo}"
+        print(cmd)
+        run(cmd.split())
+    print(f"Finished loading new issues to {db}")
+
+path_out = (here / ".." / "data" / "github.db").resolve()
+print(f"Downloading to {path_out}")
+download_issues_data("jupyter-book", path_out)