feat: repository analytics & repo populated & repo health score & health score refactor (IN-1054) (#3987)

Signed-off-by: Gašper Grom <gasper.grom@gmail.com>

Author: gaspergrom

services/libs/tinybird/datasources/project_insights_copy_ds.datasource

@@ -2,7 +2,9 @@ DESCRIPTION >
     - `project_insights_copy_ds` contains materialized project insights data.
     - Populated by `project_insights_copy.pipe` copy pipe.
     - Includes project metadata, health score, first commit, and activity metrics for last 365 days and previous 365 days.
-    - `id` column is the primary key identifier for the project.
+    - `id` column is the primary key identifier for the project or repository.
+    - `type` column indicates the record type: 'project' for project insights or 'repo' for repository insights.
+    - `repoUrl` column is the full repository URL for repo type records (empty string for project type).
     - `name` column is the human-readable project name.
     - `slug` column is the URL-friendly identifier used in routing and filtering.
     - `logoUrl` column is the URL to the project's logo image.
@@ -35,6 +37,8 @@ TAGS "Project insights", "Metrics"
 
 SCHEMA >
     `id` String,
+    `type` String,
+    `repoUrl` String,
     `name` String,
     `slug` String,
     `logoUrl` String,
@@ -64,4 +68,4 @@ SCHEMA >
     `activeOrganizationsPrevious365Days` UInt64
 
 ENGINE MergeTree
-ENGINE_SORTING_KEY id
+ENGINE_SORTING_KEY type, id

services/libs/tinybird/datasources/repo_health_score_copy_ds.datasource

@@ -0,0 +1,71 @@
+DESCRIPTION >
+    - `repo_health_score_copy_ds` contains comprehensive health score metrics and benchmarks per repository.
+    - Created via copy pipe with computed health metrics for repository-level analytics.
+    - Aggregates multiple health dimensions including contributors, popularity, development activity, and security.
+    - `channel` is the repository URL used as the primary key.
+    - `activeContributors` is the unique contributor count for the previous quarter.
+    - `activeContributorsBenchmark` is the benchmark score (0-5) for active contributors.
+    - `contributorDependencyCount` measures contributor concentration risk (bus factor).
+    - `contributorDependencyPercentage` is the combined contribution percentage of dependent contributors.
+    - `contributorDependencyBenchmark` is the benchmark score (0-5) for contributor dependency.
+    - `organizationDependencyCount` measures organizational concentration risk.
+    - `organizationDependencyPercentage` is the combined contribution percentage of dependent organizations.
+    - `organizationDependencyBenchmark` is the benchmark score (0-5) for organization dependency.
+    - `retentionRate` is the quarter-over-quarter contributor retention percentage.
+    - `retentionBenchmark` is the benchmark score (0-5) for retention.
+    - `stars` is the total star count for the repository.
+    - `starsBenchmark` is the benchmark score (0-5) for stars.
+    - `forks` is the total fork count for the repository.
+    - `forksBenchmark` is the benchmark score (0-5) for forks.
+    - `issueResolution` is the average days to close issues (nullable for repos without issues).
+    - `issueResolutionBenchmark` is the benchmark score (0-5) for issue resolution.
+    - `pullRequests` is the PR count in the last 365 days.
+    - `pullRequestsBenchmark` is the benchmark score (0-5) for pull requests.
+    - `mergeLeadTime` is the average days to merge PRs (nullable for repos without PRs).
+    - `mergeLeadTimeBenchmark` is the benchmark score (0-5) for merge lead time.
+    - `activeDaysCount` is the count of distinct active days in the last 365 days.
+    - `activeDaysBenchmark` is the benchmark score (0-5) for active days.
+    - `contributionsOutsideWorkHours` is the percentage of contributions outside work hours.
+    - `contributionsOutsideWorkHoursBenchmark` is the benchmark score (0-5) for outside work hours.
+    - `securityPercentage` is the health score percentage for the security category (0-100).
+    - `contributorPercentage` is the health score percentage for the contributors category (0-100).
+    - `popularityPercentage` is the health score percentage for the popularity category (0-100).
+    - `developmentPercentage` is the health score percentage for the development category (0-100).
+    - `overallScore` is the computed overall health score combining all dimensions.
+
+TAGS "Repository health", "Metrics"
+
+SCHEMA >
+    `channel` String,
+    `activeContributors` UInt64,
+    `activeContributorsBenchmark` UInt64,
+    `contributorDependencyCount` UInt64,
+    `contributorDependencyPercentage` Float64,
+    `contributorDependencyBenchmark` UInt64,
+    `organizationDependencyCount` UInt64,
+    `organizationDependencyPercentage` Float64,
+    `organizationDependencyBenchmark` UInt64,
+    `retentionRate` Float64,
+    `retentionBenchmark` UInt64,
+    `stars` UInt64,
+    `starsBenchmark` UInt64,
+    `forks` UInt64,
+    `forksBenchmark` UInt64,
+    `issueResolution` Nullable(Float64),
+    `issueResolutionBenchmark` UInt64,
+    `pullRequests` UInt64,
+    `pullRequestsBenchmark` UInt64,
+    `mergeLeadTime` Nullable(Float64),
+    `mergeLeadTimeBenchmark` UInt64,
+    `activeDaysCount` UInt64,
+    `activeDaysBenchmark` UInt64,
+    `contributionsOutsideWorkHours` Float64,
+    `contributionsOutsideWorkHoursBenchmark` UInt64,
+    `securityPercentage` Float64,
+    `contributorPercentage` Float64,
+    `popularityPercentage` Float64,
+    `developmentPercentage` Float64,
+    `overallScore` Float64
+
+ENGINE MergeTree
+ENGINE_SORTING_KEY channel

services/libs/tinybird/datasources/repositories_populated_ds.datasource

@@ -0,0 +1,27 @@
+DESCRIPTION >
+    - `repositories_populated_ds` contains enriched repository data with computed metrics.
+    - Populated by `repositories_populated_copy.pipe` copy pipe.
+    - Extends base repository data with contributor counts, software valuation, and first commit timestamp.
+    - `id` is the primary key identifier for the repository record.
+    - `url` is the full repository URL.
+    - `segmentId` links to the segment this repository belongs to.
+    - `insightsProjectId` links to the insights project this repository is associated with.
+    - `contributorCount` is the total number of unique contributors for the repository.
+    - `organizationCount` is the total number of unique organizations for the repository.
+    - `softwareValue` is the estimated economic value of the repository software.
+    - `firstCommit` is the timestamp of the first commit in the repository (nullable).
+
+TAGS "Repository metadata", "Analytics enrichment"
+
+SCHEMA >
+    `id` String,
+    `url` String,
+    `segmentId` String,
+    `insightsProjectId` String,
+    `contributorCount` UInt64,
+    `organizationCount` UInt64,
+    `softwareValue` UInt64,
+    `firstCommit` Nullable(DateTime64(3))
+
+ENGINE MergeTree
+ENGINE_SORTING_KEY id, url

services/libs/tinybird/pipes/health_score_active_contributors.pipe

@@ -1,4 +1,4 @@
-NODE health_score_active_contributors_score
+NODE health_score_active_contributors_score_something_else
 DESCRIPTION >
     Returns activeContributors for previous quarter per project
 
@@ -34,8 +34,9 @@ SQL >
         GROUP BY segmentId
     {% end %}
 
-NODE health_score_active_contributors_with_benchmark
+NODE health_score_active_contributors_benchmark
 SQL >
+    %
     SELECT
         segmentId,
         activeContributors,
@@ -54,4 +55,4 @@ SQL >
             THEN 5
             ELSE 0
         END AS activeContributorsBenchmark
-    FROM health_score_active_contributors_score
+    FROM health_score_active_contributors_score_something_else

services/libs/tinybird/pipes/health_score_active_days.pipe

@@ -27,8 +27,9 @@ SQL >
         GROUP BY segmentId
     {% end %}
 
-NODE health_score_active_days_with_benchmark
+NODE health_score_active_days_benchmark
 SQL >
+    %
     SELECT
         segmentId,
         activeDaysCount,

services/libs/tinybird/pipes/health_score_contributions_outside_work_hours.pipe

@@ -29,8 +29,9 @@ SQL >
         {% end %}
     GROUP BY segmentId
 
-NODE health_score_contributions_outside_work_hours_with_benchmark
+NODE health_score_contributions_outside_work_hours_benchmark
 SQL >
+    %
     SELECT
         segmentId,
         contributionsOutsideWorkHours,

services/libs/tinybird/pipes/health_score_contributor_dependency.pipe

@@ -33,8 +33,9 @@ SQL >
         ORDER by contributionCount DESC
     {% end %}
 
-NODE health_score_contributor_dependency_contribution_percentage
+NODE health_score_contributor_dependency_pct
 SQL >
+    %
     SELECT
         segmentId,
         memberId,
@@ -45,32 +46,34 @@ SQL >
     FROM health_score_contributor_dependency_contribution_count
     ORDER BY contributionPercentage DESC
 
-NODE health_score_contributor_dependency_contribution_runnning_total
+NODE health_score_contributor_dependency_running
 SQL >
+    %
     SELECT
         segmentId,
         memberId,
-        contributionCount,
         contributionPercentage,
         SUM(contributionPercentage) OVER (
             PARTITION BY segmentId ORDER BY contributionPercentage DESC, memberId
         ) AS contributionPercentageRunningTotal
-    FROM health_score_contributor_dependency_contribution_percentage
+    FROM health_score_contributor_dependency_pct
 
 NODE health_score_contributor_dependency_score
 SQL >
+    %
     SELECT
         segmentId,
         count() AS contributorDependencyCount,
         round(sum(contributionPercentage)) AS contributorDependencyPercentage
-    FROM health_score_contributor_dependency_contribution_runnning_total
+    FROM health_score_contributor_dependency_running
     WHERE
         contributionPercentageRunningTotal < 51
         OR (contributionPercentageRunningTotal - contributionPercentage < 51)
     GROUP BY segmentId
 
-NODE health_score_contributor_dependency_with_benchmark
+NODE health_score_contributor_dependency_benchmark
 SQL >
+    %
     SELECT
         segmentId,
         contributorDependencyCount,

services/libs/tinybird/pipes/health_score_forks.pipe

@@ -28,8 +28,9 @@ SQL >
         GROUP BY segmentId
     {% end %}
 
-NODE health_score_forks_with_benchmark
+NODE health_score_forks_benchmark
 SQL >
+    %
     SELECT
         segmentId,
         forks,

services/libs/tinybird/pipes/health_score_issues_resolution.pipe

@@ -38,8 +38,9 @@ SQL >
         GROUP BY segmentId
     {% end %}
 
-NODE health_score_issues_resolution_with_benchmark
+NODE health_score_issues_resolution_benchmark
 SQL >
+    %
     SELECT
         segmentId,
         issueResolution,

services/libs/tinybird/pipes/health_score_merge_lead_time.pipe

@@ -26,8 +26,9 @@ SQL >
         {% end %}
     GROUP BY segmentId
 
-NODE health_score_merge_lead_time_with_benchmark
+NODE health_score_merge_lead_time_benchmark
 SQL >
+    %
     SELECT
         segmentId,
         mergeLeadTime,