Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix/improve guess cached pod name #1342

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions playbooks/robusta_playbooks/babysitter.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
FindingAggregationKey,
)
from robusta.core.reporting.base import EnrichmentType
from robusta.core.reporting.findings import FindingOwner


class BabysitterConfig(ActionParams):
Expand Down
2 changes: 2 additions & 0 deletions playbooks/robusta_playbooks/event_enrichments.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
)
from robusta.core.reporting import EventsBlock, EventRow
from robusta.core.reporting.base import EnrichmentType
from robusta.core.reporting.findings import FindingOwner
from robusta.core.reporting.custom_rendering import render_value


Expand Down Expand Up @@ -72,6 +73,7 @@ def event_report(event: EventChangeEvent):
subject_type=FindingSubjectType.from_kind(k8s_obj.kind),
namespace=k8s_obj.namespace,
node=KubeObjFindingSubject.get_node_name(k8s_obj),
owner=FindingOwner(owner_references=event.obj.metadata.ownerReferences)
),
)
event.add_finding(finding)
Expand Down
4 changes: 4 additions & 0 deletions scripts/generate_kubernetes_code.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ def autogenerate_events(f: TextIO):
from ..base_event import K8sBaseChangeEvent
from ....core.model.events import ExecutionBaseEvent, ExecutionEventBaseParams
from ....core.reporting.base import FindingSubject
from ....core.reporting.findings import FindingOwner
from ....core.reporting.consts import FindingSubjectType, FindingSource
from ....core.reporting.finding_subjects import KubeObjFindingSubject
from robusta.integrations.kubernetes.custom_models import {CUSTOM_MODELS_IMPORTS}
Expand Down Expand Up @@ -185,6 +186,7 @@ def get_subject(self) -> FindingSubject:
node=KubeObjFindingSubject.get_node_name(self.obj),
labels=self.obj.metadata.labels,
annotations=self.obj.metadata.annotations,
owner=FindingOwner(owner_references=self.obj.metadata.ownerReferences)
)

@classmethod
Expand Down Expand Up @@ -268,6 +270,7 @@ def get_subject(self) -> FindingSubject:
node=KubeObjFindingSubject.get_node_name(self.obj),
labels=self.obj.metadata.labels,
annotations=self.obj.metadata.annotations,
owner=FindingOwner(owner_references=self.obj.metadata.ownerReferences)
)


Expand All @@ -291,6 +294,7 @@ def get_subject(self) -> FindingSubject:
node=KubeObjFindingSubject.get_node_name(self.obj),
labels=self.obj.metadata.labels,
annotations=self.obj.metadata.annotations,
owner=FindingOwner(owner_references=self.obj.metadata.ownerReferences)
)


Expand Down
62 changes: 54 additions & 8 deletions src/robusta/core/discovery/top_service_resolver.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,12 @@
from collections import defaultdict
from typing import Dict, List, Optional

from hikaru.model.rel_1_26 import OwnerReference
from pydantic.main import BaseModel

from robusta.core.model.env_vars import RESOURCE_UPDATES_CACHE_TTL_SEC
from robusta.core.reporting.findings import FindingOwner
from robusta.integrations.kubernetes.custom_models import RobustaPod


class TopLevelResource(BaseModel):
Expand All @@ -26,6 +29,7 @@ class TopServiceResolver:
__recent_resource_updates: Dict[str, CachedResourceInfo] = {}
__namespace_to_resource: Dict[str, List[TopLevelResource]] = defaultdict(list)
__cached_updates_lock = threading.Lock()
__cached_owner_references: Dict[str, OwnerReference] = {}

@classmethod
def store_cached_resources(cls, resources: List[TopLevelResource]):
Expand All @@ -51,20 +55,62 @@ def store_cached_resources(cls, resources: List[TopLevelResource]):
# TODO remove this guess function
# temporary try to guess who the owner service is.
@classmethod
def guess_service_key(cls, name: str, namespace: str) -> str:
resource = cls.guess_cached_resource(name, namespace)
def guess_service_key(cls, name: str, namespace: str, kind: str, owner: Optional[FindingOwner]) -> str:
resource = cls.guess_cached_resource(name, namespace, kind=kind, owner=owner)
return resource.get_resource_key() if resource else ""

# TODO remove this guess function
# temporary try to guess who the owner service is.
@classmethod
def guess_cached_resource(cls, name: str, namespace: str) -> Optional[TopLevelResource]:
def get_pod_owner_reference(cls, name: str, namespace: str) -> Optional[OwnerReference]:
key = f"{namespace}/{name}"
if key in cls.__cached_owner_references:
return cls.__cached_owner_references[key]

robusta_pod = RobustaPod.find_pod(name, namespace)
if robusta_pod.metadata.ownerReferences:
cls.__cached_owner_references[key] = robusta_pod.metadata.ownerReferences[0]
return robusta_pod.metadata.ownerReferences[0]

return None

@classmethod
def guess_cached_resource(cls, name: str, namespace: str, kind: str, owner: Optional[FindingOwner]) \
-> Optional[TopLevelResource]:
if name is None or namespace is None:
return None

for cached_resource in cls.__namespace_to_resource[namespace]:
if name.startswith(cached_resource.name):
return cached_resource
kind = kind.lower()

# owner references available
if owner and owner.owner_references:
owner_kind = owner.owner_references[0].kind.lower()
owner_reference = owner.owner_references[0]

if owner_kind in ["deployment", "statefulset", "daemonset", "job", "deploymentconfig",
"argorollout"]:
return TopLevelResource(name=owner_reference.name, resource_type=owner_reference.kind,
namespace=namespace)

# replicset
if owner_kind == "replicaset":
new_owner_reference = cls.get_pod_owner_reference(name=owner_reference.name, namespace=namespace)
return TopLevelResource(name=new_owner_reference.name, resource_type=new_owner_reference.kind,
namespace=namespace)

# crd
if owner_kind not in ["deployment", "statefulset", "daemonset", "job", "deploymentconfig",
"argorollout", "pod"]:
return TopLevelResource(name=name, resource_type=kind, namespace=namespace)

# owner references NOT available
if owner is None or not owner.owner_references:
return TopLevelResource(name=name, resource_type=kind, namespace=namespace)

# unknown owner
if owner.unknown_owner:
for cached_resource in cls.__namespace_to_resource[namespace]:
if name.startswith(cached_resource.name):
return cached_resource

return None

@classmethod
Expand Down
7 changes: 6 additions & 1 deletion src/robusta/core/playbooks/internal/discovery_events.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from robusta.core.discovery.top_service_resolver import TopLevelResource, TopServiceResolver
from robusta.core.playbooks.common import get_event_timestamp, get_events_list
from robusta.core.reporting.base import EnrichmentType
from robusta.core.reporting.findings import FindingOwner


@action
Expand Down Expand Up @@ -65,6 +66,7 @@ def create_debug_event_finding(event: Event):
"""
k8s_obj = event.regarding
subject_type = FindingSubjectType.from_kind(k8s_obj.kind.lower()) if k8s_obj.kind else FindingSubjectType.TYPE_NONE

finding = Finding(
title=f"{event.reason} {event.type} for {k8s_obj.kind} {k8s_obj.namespace}/{k8s_obj.name}",
description=event.note,
Expand All @@ -76,10 +78,13 @@ def create_debug_event_finding(event: Event):
k8s_obj.name,
subject_type,
k8s_obj.namespace,
owner=FindingOwner(owner_references=event.metadata.ownerReferences)
),
creation_date=get_event_timestamp(event),
)
finding.service_key = TopServiceResolver.guess_service_key(name=k8s_obj.name, namespace=k8s_obj.namespace)
finding.service_key = TopServiceResolver.guess_service_key(name=k8s_obj.name, namespace=k8s_obj.namespace,
kind=k8s_obj.kind,
owner=finding.subject.owner)
return finding


Expand Down
7 changes: 6 additions & 1 deletion src/robusta/core/reporting/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from robusta.core.discovery.top_service_resolver import TopServiceResolver
from robusta.core.model.env_vars import ROBUSTA_UI_DOMAIN
from robusta.core.reporting.consts import FindingSource, FindingSubjectType, FindingType
from robusta.core.reporting.findings import FindingOwner
from robusta.utils.scope import BaseScopeMatcher


Expand Down Expand Up @@ -201,6 +202,7 @@ def __init__(
container: Optional[str] = None,
labels: Optional[Dict[str, str]] = None,
annotations: Optional[Dict[str, str]] = None,
owner: Optional[FindingOwner] = None,
):
self.name = name
self.subject_type = subject_type
Expand All @@ -209,6 +211,7 @@ def __init__(
self.container = container
self.labels = labels or {}
self.annotations = annotations or {}
self.owner = owner

def __str__(self):
if self.namespace is not None:
Expand Down Expand Up @@ -251,7 +254,9 @@ def __init__(
self.subject = subject
self.enrichments: List[Enrichment] = []
self.video_links: List[VideoLink] = []
self.service = TopServiceResolver.guess_cached_resource(name=subject.name, namespace=subject.namespace)
self.service = TopServiceResolver.guess_cached_resource(name=subject.name, namespace=subject.namespace,
kind=subject.subject_type.value,
owner=subject.owner)
self.service_key = self.service.get_resource_key() if self.service else ""
uri_path = f"services/{self.service_key}?tab=grouped" if self.service_key else "graphs"
self.investigate_uri = f"{ROBUSTA_UI_DOMAIN}/{uri_path}"
Expand Down
3 changes: 3 additions & 0 deletions src/robusta/core/reporting/consts.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ class FindingSubjectType(Enum):
TYPE_DAEMONSET = "daemonset"
TYPE_STATEFULSET = "statefulset"
TYPE_HPA = "horizontalpodautoscaler"
TYPE_REPLICASET = "replicaset"
TYPE_HELM_RELEASES = "helmreleases"

@staticmethod
Expand All @@ -62,6 +63,8 @@ def from_kind(kind: str):
return FindingSubjectType.TYPE_JOB
elif kind == "daemonset":
return FindingSubjectType.TYPE_DAEMONSET
elif kind == "replicaset":
return FindingSubjectType.TYPE_REPLICASET
elif kind == "statefulset":
return FindingSubjectType.TYPE_STATEFULSET
elif kind == "horizontalpodautoscaler":
Expand Down
3 changes: 3 additions & 0 deletions src/robusta/core/reporting/finding_subjects.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from hikaru.model.rel_1_26 import ObjectReference, Pod

from robusta.core.reporting.base import FindingSubject
from robusta.core.reporting.findings import FindingOwner
from robusta.core.reporting.consts import FindingSubjectType


Expand All @@ -23,6 +24,7 @@ def __init__(
node=node_name,
labels=obj.metadata.labels,
annotations=obj.metadata.annotations,
owner=FindingOwner(owner_references=obj.metadata.ownerReferences)
)

@staticmethod
Expand All @@ -49,4 +51,5 @@ def __init__(self, pod: Pod = None):
node=pod.spec.nodeName,
labels=pod.metadata.labels,
annotations=pod.metadata.annotations,
owner=FindingOwner(owner_references=pod.metadata.ownerReferences)
)
9 changes: 9 additions & 0 deletions src/robusta/core/reporting/findings.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
from typing import Optional, List

from hikaru.model.rel_1_26 import OwnerReference
from pydantic import BaseModel


class FindingOwner(BaseModel):
owner_references: Optional[List[OwnerReference]] = None
unknown_owner: bool = False
6 changes: 5 additions & 1 deletion src/robusta/core/triggers/error_event_trigger.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

from robusta.core.discovery.top_service_resolver import TopServiceResolver
from robusta.core.playbooks.base_trigger import TriggerEvent
from robusta.core.reporting.findings import FindingOwner
from robusta.integrations.kubernetes.autogenerated.triggers import EventAllChangesTrigger, EventChangeEvent
from robusta.integrations.kubernetes.base_triggers import K8sTriggerEvent
from robusta.utils.rate_limiter import RateLimiter
Expand Down Expand Up @@ -70,7 +71,10 @@ def should_fire(self, event: TriggerEvent, playbook_id: str, build_context: Dict
# Perform a rate limit for this service key according to the rate_limit parameter
name = exec_event.obj.regarding.name if exec_event.obj.regarding.name else ""
namespace = exec_event.obj.regarding.namespace if exec_event.obj.regarding.namespace else ""
service_key = TopServiceResolver.guess_service_key(name=name, namespace=namespace)
kind = exec_event.obj.regarding.kind if exec_event.obj.regarding.kind else ""
service_key = (TopServiceResolver.guess_service_key
(name=name, namespace=namespace, kind=kind,
owner=FindingOwner(owner_references=exec_event.obj.metadata.ownerReferences)))
return RateLimiter.mark_and_test(
f"WarningEventTrigger_{playbook_id}_{exec_event.obj.reason}",
service_key if service_key else namespace + ":" + name,
Expand Down
Loading
Loading