Merge pull request #4925 from linuxfoundation/unicron-add-stage-filter-to-ddog-scan-tools-prod

Address vulns, add DDog env/stage filtering, collapse more API URLs into single templates, add branch sync script

Author: lukaszgryglicki

cla-backend-go/events/repository.go

@@ -477,7 +477,10 @@ func (repo *repository) SearchEvents(params *eventOps.SearchEventsParams, pageSi
 	case params.ProjectSFID != nil:
 		// search by projectSFID
 		indexName = EventProjectSFIDEventTypeIndex
-		condition = expression.Key("event_project_sfid").Equal(expression.Value(params.ProjectSFID)).And(expression.Key("event_type").Equal(expression.Value(params.EventType)))
+		condition = expression.Key("event_project_sfid").Equal(expression.Value(params.ProjectSFID))
+		if params.EventType != nil {
+			condition = condition.And(expression.Key("event_type").Equal(expression.Value(params.EventType)))
+		}
 		pk = "event_project_sfid"
 		sk = "event_type"
 	}

cla-backend-go/swagger/.python-version

@@ -0,0 +1 @@
+3.11

cla-backend-go/telemetry/datadog_otlp.go

@@ -191,6 +191,7 @@ func WrapHTTPHandler(next http.Handler) http.Handler {
 		p = reUUIDLike.ReplaceAllString(p, "/{invalid-uuid}$1")
 		p = reUUIDHexDash36.ReplaceAllString(p, "/{invalid-uuid}$1")
 		p = reNumericID.ReplaceAllString(p, "/{id}$1")
+		p = reNumericID.ReplaceAllString(p, "/{id}$1")
 		// Salesforce IDs: valid vs invalid
 		p = reSFIDValid.ReplaceAllString(p, "/{sfid}$1")
 		p = reSFIDLike.ReplaceAllString(p, "/{invalid-sfid}$1")

cla-backend/cla/auth.py

@@ -7,7 +7,11 @@
 import os
 
 import requests
-from jose import jwt
+import json
+
+import jwt
+from jwt.algorithms import RSAAlgorithm
+from jwt.exceptions import ExpiredSignatureError, InvalidTokenError, PyJWTError
 
 import cla
 
@@ -81,7 +85,7 @@ def authenticate_user(headers):
 
     try:
         unverified_header = jwt.get_unverified_header(token)
-    except jwt.JWTError as e:
+    except PyJWTError as e:
         cla.log.error(e)
         raise AuthError('unable to decode claims')
 
@@ -99,19 +103,20 @@ def authenticate_user(headers):
     # print("JWKS kids:", [key["kid"] for key in jwks["keys"]])
     if rsa_key:
         try:
+            public_key = RSAAlgorithm.from_jwk(json.dumps(rsa_key))
+            jwt_algorithms = algorithms if isinstance(algorithms, (list, tuple, set)) else [algorithms]
             payload = jwt.decode(
                 token,
-                rsa_key,
-                algorithms=algorithms,
+                public_key,
+                algorithms=list(jwt_algorithms),
                 options={
-                    'verify_at_hash': False,
                     'verify_aud': False
                 }
             )
-        except jwt.ExpiredSignatureError as e:
+        except ExpiredSignatureError as e:
             cla.log.error(e)
             raise AuthError('token is expired')
-        except jwt.JWTClaimsError as e:
+        except InvalidTokenError as e:
             cla.log.error(e)
             raise AuthError('incorrect claims')
         except Exception as e:

cla-backend/cla/controllers/github_application.py

@@ -6,7 +6,7 @@
 
 import requests
 from github import BadCredentialsException, UnknownObjectException, GithubException, GithubIntegration, Github
-from jose import jwt
+import jwt
 from requests.exceptions import RequestException
 
 import cla
@@ -30,9 +30,7 @@ def repos(self):
     def __init__(self, installation_id):
         self.installation_id = installation_id
 
-        cla.log.debug('Initializing github application - installation_id: {}, app id: {}, private key'
-                      ' (minus header): {}...'.
-                      format(self.installation_id, self.app_id, self.private_key[32:38]))
+        cla.log.debug('Initializing github application - installation_id: {}, app id: {}'.format(self.installation_id, self.app_id))
 
         try:
             integration = GithubCLAIntegration(self.app_id, self.private_key)
@@ -77,15 +75,8 @@ def create_check_run(self, repository_name, data):
 
 
 class GithubCLAIntegration(GithubIntegration):
-    """
-    Custom GithubIntegration using python-jose instead of pyjwt for token creation.
-    """
 
     def create_jwt(self):
-        """
-        Overloaded to use python-jose instead of pyjwt.
-        Couldn't get it working with pyjwt.
-        """
         now = int(time.time())
         payload = {
             "iat": now,

cla-backend/cla/models/github_models.py

@@ -14,6 +14,7 @@
 import time
 import uuid
 from typing import List, Optional, Union, Tuple, Iterable
+from collections.abc import MutableMapping
 
 import cla
 import falcon
@@ -207,7 +208,7 @@ def user_from_session(self, request, get_redirect_url):
         fn = "github_models.user_from_session"
         cla.log.debug(f"{fn} - loading session from request: {request}...")
         session = self._get_request_session(request)
-        cla.log.debug(f"{fn} - session: {session}")
+        cla.log.debug(f"{fn} - session loaded (keys={list(session.keys())})")
 
         # We can already have token in the session
         if "github_oauth2_token" in session:
@@ -222,7 +223,7 @@ def user_from_session(self, request, get_redirect_url):
         authorization_url, csrf_token = self.get_authorization_url_and_state(None, None, None, ["user:email"], state='user-from-session')
         cla.log.debug(f"{fn} - obtained GitHub OAuth2 state from authorization - storing CSRF token in the session...")
         session["github_oauth2_state"] = csrf_token
-        cla.log.debug(f"{fn} - GitHub OAuth2 request with CSRF token {csrf_token} - sending user to {authorization_url}")
+        cla.log.debug(f"{fn} - redirecting user to GitHub OAuth2 authorization URL")
         # We must redirect to GitHub OAuth app for authentication, it will return you to /v2/github/installation which will handle returning user data
         if get_redirect_url:
             cla.log.debug(f"{fn} - sending redirect_url via 202 HTTP status JSON payload")
@@ -246,7 +247,7 @@ def sign_request(self, installation_id, github_repository_id, change_request_id,
         # Not sure if we need a different token for each installation ID...
         cla.log.debug(f"{fn} - Loading session from request: {request}...")
         session = self._get_request_session(request)
-        cla.log.debug(f"{fn} - Adding github details to session: {session} which is type: {type(session)}...")
+        cla.log.debug(f"{fn} - Adding github details to session: {list(session.keys())} which is type: {type(session)}...")
         session["github_installation_id"] = installation_id
         session["github_repository_id"] = github_repository_id
         session["github_change_request_id"] = change_request_id
@@ -267,7 +268,7 @@ def sign_request(self, installation_id, github_repository_id, change_request_id,
             )
             cla.log.debug(f"{fn} - Obtained GitHub OAuth2 state from authorization - storing state in the session...")
             session["github_oauth2_state"] = state
-            cla.log.debug(f"{fn} - GitHub OAuth2 request with state {state} - sending user to {authorization_url}")
+            cla.log.debug(f"{fn} - redirecting user to GitHub OAuth2 authorization URL")
             raise falcon.HTTPFound(authorization_url)
 
     def _get_request_session(self, request) -> dict:  # pylint: disable=no-self-use
@@ -277,17 +278,31 @@ def _get_request_session(self, request) -> dict:  # pylint: disable=no-self-use
         fn = "cla.models.github_models._get_request_session"
         session = request.context.get("session")
         if session is None:
-            cla.log.warning(f"Session is empty for request: {request}")
-        cla.log.debug(f"{fn} - loaded session: {session}")
+            cla.log.warning(f"{fn} - Session is empty for request: {request}")
+            session = {}
+            request.context["session"] = session
 
         # Ensure session is a dict - getting issue where session is a string
         if isinstance(session, str):
             # convert string to a dict
-            cla.log.debug(f"{fn} - session is type: {type(session)} - converting to dict...")
-            session = json.loads(session)
-            # Reset the session now that we have converted it to a dict
+            cla.log.warning(f"{fn} - session context is a string; attempting to parse JSON")
+            try:
+                session = json.loads(session)
+            except (ValueError, json.JSONDecodeError) as e:
+                cla.log.warning(f"{fn} - unable to parse session string as JSON: {e}")
+                session = {}
+
+            request.context["session"] = session
+
+        if not isinstance(session, MutableMapping):
+            try:
+                session = dict(session)
+            except Exception:
+                cla.log.warning(f"{fn} - session context has unsupported type {type(session)}; resetting to empty dict")
+                session = {}
             request.context["session"] = session
-            cla.log.debug(f"{fn} - session: {session} which is now type: {type(session)}...")
+
+        cla.log.debug(f"{fn} - loaded session (keys={list(session.keys())})")
 
         return session
 
@@ -342,7 +357,6 @@ def oauth2_redirect(self, state, code, request):  # pylint: disable=too-many-arg
         fn = "github_models.oauth2_redirect"
         cla.log.debug(f"{fn} - handling GitHub OAuth2 redirect with request: {dir(request)}")
         session = self._get_request_session(request)  # request.context['session']
-        cla.log.debug(f"{fn} - state: {state}, code: {code}, session: {session}")
 
         if "github_oauth2_state" in session:
             session_state = session["github_oauth2_state"]
@@ -353,29 +367,35 @@ def oauth2_redirect(self, state, code, request):  # pylint: disable=too-many-arg
         if state != session_state:
             # Eventually handle user-from-session API callback
             try:
-                state_data = json.loads(base64.urlsafe_b64decode(state.encode()).decode())
+                padded_state = state + "=" * (-len(state) % 4)
+                state_data = json.loads(base64.urlsafe_b64decode(padded_state.encode()).decode())
             except (ValueError, json.JSONDecodeError, binascii.Error) as err:
-                cla.log.warning(f"{fn} - failed to decode state: {state}, error: {err}")
-                raise falcon.HTTPBadRequest("Invalid OAuth2 state", state)
-            state_token = state_data["csrf"]
-            value = state_data["state"]
+                cla.log.warning(f"{fn} - failed to decode state, error: {err}")
+                raise falcon.HTTPBadRequest("Invalid OAuth2 state", "Invalid OAuth2 state")
+
+            state_token = state_data.get("csrf")
+            value = state_data.get("state")
+            if not state_token or not value:
+                cla.log.warning(f"{fn} - invalid OAuth2 state payload while handling callback")
+                raise falcon.HTTPBadRequest("Invalid OAuth2 state", "Invalid OAuth2 state")
+
             if value != "user-from-session":
-                cla.log.warning(f"{fn} - invalid GitHub OAuth2 state {session_state} expecting {state}, value: {value}")
-                raise falcon.HTTPBadRequest("Invalid OAuth2 state", state)
+                cla.log.warning(f"{fn} - invalid GitHub OAuth2 state while handling callback")
+                raise falcon.HTTPBadRequest("Invalid OAuth2 state", "Invalid OAuth2 state")
             if state_token != session_state:
-                cla.log.warning(f"{fn} - invalid GitHub OAuth2 state {session_state} expecting {state_token} while handling user-from-session callback")
-                raise falcon.HTTPBadRequest(f"Invalid OAuth2 state")
+                cla.log.warning(f"{fn} - invalid GitHub OAuth2 state while handling callback")
+                raise falcon.HTTPBadRequest(f"Invalid OAuth2 state", "Invalid OAuth2 state")
             cla.log.debug(f"handling user-from-session callback")
             token_url = cla.conf["GITHUB_OAUTH_TOKEN_URL"]
             client_id = os.environ["GH_OAUTH_CLIENT_ID"]
-            cla.log.debug(f"{fn} - using client ID {client_id}")
+            cla.log.debug(f"{fn} - using client ID {client_id[0:5]}...")
             client_secret = os.environ["GH_OAUTH_SECRET"]
             try:
                 token = self._fetch_token(client_id, state, token_url, client_secret, code)
             except Exception as err:
                 cla.log.warning(f"{fn} - GitHub OAuth2 error: {err}. Likely bad or expired code, returning HTTP 404 state.")
                 raise falcon.HTTPBadRequest("OAuth2 code is invalid or expired")
-            cla.log.debug(f"{fn} - oauth2 token received for state {state}: {token} - storing token in session")
+            cla.log.debug(f"{fn} - oauth2 token received - storing token in session")
             session["github_oauth2_token"] = token
             user = self.get_or_create_user(request)
             if user is None:
@@ -385,7 +405,7 @@ def oauth2_redirect(self, state, code, request):  # pylint: disable=too-many-arg
             return user.to_dict()
 
         # Get session information for this request.
-        cla.log.debug(f"{fn} - attempting to fetch OAuth2 token for state {state}")
+        cla.log.debug(f"{fn} - attempting to fetch OAuth2 token")
         installation_id = session.get("github_installation_id", None)
         github_repository_id = session.get("github_repository_id", None)
         change_request_id = session.get("github_change_request_id", None)
@@ -394,12 +414,9 @@ def oauth2_redirect(self, state, code, request):  # pylint: disable=too-many-arg
         token_url = cla.conf["GITHUB_OAUTH_TOKEN_URL"]
         client_id = os.environ["GH_OAUTH_CLIENT_ID"]
         client_secret = os.environ["GH_OAUTH_SECRET"]
-        cla.log.debug(
-            f"{fn} - fetching token using {client_id[0:5]}... with state={state}, token_url={token_url}, "
-            f"client_secret={client_secret[0:5]}, with code={code}"
-        )
+        cla.log.debug(f"{fn} - fetching oauth2 token with client ID: {client_id[0:5]}..., token_url: {token_url}")
         token = self._fetch_token(client_id, state, token_url, client_secret, code)
-        cla.log.debug(f"{fn} - oauth2 token received for state {state}: {token} - storing token in session")
+        cla.log.debug(f"{fn} - oauth2 token received - storing token in session")
         session["github_oauth2_token"] = token
         cla.log.debug(f"{fn} - redirecting the user back to the console: {origin_url}")
         return self.redirect_to_console(installation_id, github_repository_id, change_request_id, origin_url, request)
@@ -1498,7 +1515,7 @@ def get_user_data(self, session, client_id):  # pylint: disable=no-self-use
         fn = "cla.models.github_models.get_user_data"
         token = session.get("github_oauth2_token")
         if token is None:
-            cla.log.error(f"{fn} - unable to load github_oauth2_token from session, session is: {session}")
+            cla.log.error(f"{fn} - unable to load github_oauth2_token from session (keys={list(session.keys())})")
             return {"error": "could not get user data from session"}
 
         oauth2 = OAuth2Session(client_id, token=token)

cla-backend/cla/routes.py

@@ -108,6 +108,7 @@ def _sanitize_api_path(path: str) -> str:
     p = _RE_UUID_LIKE.sub(r"/{invalid-uuid}\1", p)
     p = _RE_UUID_HEXDASH_36.sub(r"/{invalid-uuid}\1", p)
     p = _RE_NUMERIC_ID.sub(r"/{id}\1", p)
+    p = _RE_NUMERIC_ID.sub(r"/{id}\1", p)
     p = _RE_SFID_VALID.sub(r"/{sfid}\1", p)
     p = _RE_SFID_LIKE.sub(r"/{invalid-sfid}\1", p)
     p = _RE_LFXID_VALID.sub(r"/{lfxid}\1", p)