[sai-gen] Support merging SAI specs. (#577)

This change added the SAI spec merging support. This is to ensure any new SAI attribute changes will follow the rules below to help ABI compatibility:

- All new attributes or action enum values will be added in the end of the list.
- Existing attributes will be updated inline. In the future, we can add more check here, e.g., type changed in non-compatible way and etc.
- All old attributes will be marked as deprecated instead of removed.

Recently, the tunnel APIs are added, but because it merged with the SAI spec PR together without rebase, it is not captured in the previous SAI spec. With SAI spec merging, we can see the changes are showing up in the expected sequence. This would be an example that demos the several usages of SAI spec.

dash-pipeline/Makefile

@@ -177,8 +177,9 @@ sai: sai-headers sai-meta libsai
 sai-headers: p4 docker-saithrift-bldr-image-exists | SAI/SAI
 	@echo "Generate SAI library headers and implementation..."
 
-	# Once the specs are checked in, we can use this to revert any local changes before generating the new specs.
-	# git checkout SAI/specs/*
+	# Revert any local changes before generating the new specs.
+	git clean -xf SAI/specs
+	git checkout SAI/specs/*
 
 	mkdir -p SAI/lib
 

dash-pipeline/SAI/sai_api_gen.py

@@ -68,7 +68,8 @@
     print("Outputting new SAI spec to " + sai_spec_dir)
     yaml_inc_ctor.autoload = False
     new_sai_spec = dash_sai_exts.to_sai()
-    new_sai_spec.serialize(sai_spec_dir)
+    sai_spec.merge(new_sai_spec)
+    sai_spec.serialize(sai_spec_dir)
 
     # Generate and update all SAI files
     SAIGenerator(dash_sai_exts).generate()

dash-pipeline/SAI/specs/dash_outbound_ca_to_pa.yaml

@@ -217,4 +217,16 @@ sai_apis:
     allow_null: false
     valid_only: null
     deprecated: null
+  - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute
+    name: SAI_OUTBOUND_CA_TO_PA_ENTRY_DASH_TUNNEL_ID
+    description: Action parameter DASH_TUNNEL_ID
+    type: sai_object_id_t
+    attr_value_field: u16
+    default: SAI_NULL_OBJECT_ID
+    isresourcetype: false
+    flags: CREATE_AND_SET
+    object_name: SAI_OBJECT_TYPE_DASH_TUNNEL
+    allow_null: true
+    valid_only: null
+    deprecated: null
   stats: []

dash-pipeline/SAI/specs/dash_outbound_routing.yaml

@@ -241,4 +241,16 @@ sai_apis:
     allow_null: false
     valid_only: null
     deprecated: null
+  - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute
+    name: SAI_OUTBOUND_ROUTING_ENTRY_DASH_TUNNEL_ID
+    description: Action parameter DASH_TUNNEL_ID
+    type: sai_object_id_t
+    attr_value_field: u16
+    default: SAI_NULL_OBJECT_ID
+    isresourcetype: false
+    flags: CREATE_AND_SET
+    object_name: SAI_OBJECT_TYPE_DASH_TUNNEL
+    allow_null: true
+    valid_only: null
+    deprecated: null
   stats: []

dash-pipeline/SAI/specs/dash_tunnel.yaml

@@ -0,0 +1,48 @@
+!!python/object:utils.sai_spec.sai_api_group.SaiApiGroup
+name: dash_tunnel
+description: ''
+sai_apis:
+- !!python/object:utils.sai_spec.sai_api.SaiApi
+  name: dash_tunnel
+  description: ''
+  is_object: true
+  enums: []
+  structs: []
+  attributes:
+  - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute
+    name: SAI_DASH_TUNNEL_DIP
+    description: Action parameter DIP
+    type: sai_ip_address_t
+    attr_value_field: ipaddr
+    default: 0.0.0.0
+    isresourcetype: false
+    flags: CREATE_AND_SET
+    object_name: null
+    allow_null: false
+    valid_only: null
+    deprecated: null
+  - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute
+    name: SAI_DASH_TUNNEL_DASH_ENCAPSULATION
+    description: Action parameter DASH_ENCAPSULATION
+    type: sai_dash_encapsulation_t
+    attr_value_field: s32
+    default: SAI_DASH_ENCAPSULATION_VXLAN
+    isresourcetype: false
+    flags: CREATE_AND_SET
+    object_name: null
+    allow_null: false
+    valid_only: null
+    deprecated: null
+  - !!python/object:utils.sai_spec.sai_attribute.SaiAttribute
+    name: SAI_DASH_TUNNEL_TUNNEL_KEY
+    description: Action parameter TUNNEL_KEY
+    type: sai_uint32_t
+    attr_value_field: u32
+    default: '0'
+    isresourcetype: false
+    flags: CREATE_AND_SET
+    object_name: null
+    allow_null: false
+    valid_only: null
+    deprecated: null
+  stats: []

dash-pipeline/SAI/specs/sai_spec.yaml

@@ -12,6 +12,7 @@ api_types:
 - SAI_API_DASH_PA_VALIDATION
 - SAI_API_ROUTE
 - SAI_API_DASH_VIP
+- SAI_API_DASH_TUNNEL
 object_types:
 - SAI_OBJECT_TYPE_DASH_ACL_GROUP
 - SAI_OBJECT_TYPE_DASH_ACL_RULE
@@ -30,6 +31,7 @@ object_types:
 - SAI_OBJECT_TYPE_PA_VALIDATION_ENTRY
 - SAI_OBJECT_TYPE_ROUTE_ENTRY
 - SAI_OBJECT_TYPE_VIP_ENTRY
+- SAI_OBJECT_TYPE_DASH_TUNNEL
 object_entries:
 - !!python/object:utils.sai_spec.sai_struct_entry.SaiStructEntry
   name: direction_lookup_entry
@@ -97,81 +99,81 @@ enums:
     description: ''
     value: '2'
 - !!python/object:utils.sai_spec.sai_enum.SaiEnum
-  name: sai_dash_tunnel_dscp_mode_t
+  name: sai_dash_encapsulation_t
   description: ''
   members:
   - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: PRESERVE_MODEL
+    name: INVALID
     description: ''
     value: '0'
   - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: PIPE_MODEL
+    name: VXLAN
     description: ''
     value: '1'
+  - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
+    name: NVGRE
+    description: ''
+    value: '2'
 - !!python/object:utils.sai_spec.sai_enum.SaiEnum
-  name: sai_dash_ha_role_t
+  name: sai_dash_routing_actions_t
   description: ''
   members:
   - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: DEAD
-    description: ''
-    value: '0'
-  - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: ACTIVE
+    name: STATIC_ENCAP
     description: ''
     value: '1'
   - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: STANDBY
+    name: NAT
     description: ''
     value: '2'
   - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: STANDALONE
+    name: NAT46
     description: ''
-    value: '3'
+    value: '4'
   - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: SWITCHING_TO_ACTIVE
+    name: NAT64
     description: ''
-    value: '4'
+    value: '8'
+  - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
+    name: NAT_PORT
+    description: ''
+    value: '16'
 - !!python/object:utils.sai_spec.sai_enum.SaiEnum
-  name: sai_dash_encapsulation_t
+  name: sai_dash_tunnel_dscp_mode_t
   description: ''
   members:
   - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: INVALID
+    name: PRESERVE_MODEL
     description: ''
     value: '0'
   - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: VXLAN
+    name: PIPE_MODEL
     description: ''
     value: '1'
-  - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: NVGRE
-    description: ''
-    value: '2'
 - !!python/object:utils.sai_spec.sai_enum.SaiEnum
-  name: sai_dash_routing_actions_t
+  name: sai_dash_ha_role_t
   description: ''
   members:
   - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: STATIC_ENCAP
+    name: DEAD
     description: ''
-    value: '1'
+    value: '0'
   - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: NAT
+    name: ACTIVE
     description: ''
-    value: '2'
+    value: '1'
   - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: NAT46
+    name: STANDBY
     description: ''
-    value: '4'
+    value: '2'
   - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: NAT64
+    name: STANDALONE
     description: ''
-    value: '8'
+    value: '3'
   - !!python/object:utils.sai_spec.sai_enum_member.SaiEnumMember
-    name: NAT_PORT
+    name: SWITCHING_TO_ACTIVE
     description: ''
-    value: '16'
+    value: '4'
 port_extenstion: !!python/object:utils.sai_spec.sai_api_extension.SaiApiExtension
   attributes: []
   stats:
@@ -260,3 +262,4 @@ api_groups:
 - !inc '/SAI/specs/dash_pa_validation.yaml'
 - !inc '/SAI/specs/route.yaml'
 - !inc '/SAI/specs/dash_vip.yaml'
+- !inc '/SAI/specs/dash_tunnel.yaml'

dash-pipeline/SAI/utils/sai_spec/sai_api.py

@@ -3,6 +3,7 @@
 from .sai_attribute import SaiAttribute
 from .sai_enum import SaiEnum
 from .sai_struct import SaiStruct
+from . import sai_spec_utils
 
 
 class SaiApi(SaiCommon):
@@ -17,3 +18,12 @@ def __init__(self, name: str, description: str, is_object: bool = False):
         self.structs: List[SaiStruct] = []
         self.attributes: List[SaiAttribute] = []
         self.stats: List[SaiAttribute] = []
+
+    def merge(self, other: "SaiCommon"):
+        super().merge(other)
+
+        self.is_object = other.is_object
+        sai_spec_utils.merge_sai_common_lists(self.enums, other.enums)
+        sai_spec_utils.merge_sai_common_lists(self.structs, other.structs)
+        sai_spec_utils.merge_sai_common_lists(self.attributes, other.attributes)
+        sai_spec_utils.merge_sai_common_lists(self.stats, other.stats)

dash-pipeline/SAI/utils/sai_spec/sai_api_extension.py

@@ -1,5 +1,6 @@
 from typing import List
 from .sai_attribute import SaiAttribute
+from . import sai_spec_utils
 
 
 class SaiApiExtension:
@@ -12,3 +13,7 @@ class SaiApiExtension:
     def __init__(self):
         self.attributes: List[SaiAttribute] = []
         self.stats: List[SaiAttribute] = []
+
+    def merge(self, other: "SaiApiExtension"):
+        sai_spec_utils.merge_sai_common_lists(self.attributes, other.attributes)
+        sai_spec_utils.merge_sai_common_lists(self.stats, other.stats)

dash-pipeline/SAI/utils/sai_spec/sai_api_group.py

@@ -1,6 +1,7 @@
 from typing import List
 from .sai_common import SaiCommon
 from .sai_api import SaiApi
+from . import sai_spec_utils
 
 
 class SaiApiGroup(SaiCommon):
@@ -11,3 +12,20 @@ class SaiApiGroup(SaiCommon):
     def __init__(self, name: str, description: str):
         super().__init__(name, description)
         self.sai_apis: List[SaiApi] = []
+
+    def merge(self, other: "SaiCommon"):
+        super().merge(other)
+        sai_spec_utils.merge_sai_common_lists(self.sai_apis, other.sai_apis)
+
+    def deprecate(self) -> bool:
+        """
+        Deprecate API group.
+
+        When deprecating the API group, we can safely remove it from the list as the
+        net effect is the same as keeping it:
+        - The old API type, object type and object entries will not be changed.
+        - The SAI headers will not be changed, because their API groups are present.
+        - The DASH libsai will not be generated anymore, but it is ok, since we will not
+          use them in the BMv2 anyway.
+        """
+        return True

dash-pipeline/SAI/utils/sai_spec/sai_api_p4_meta.py

@@ -0,0 +1,19 @@
+from typing import Dict, List
+
+
+class SaiApiP4MetaAction:
+    def __init__(self, name: str, id: int):
+        self.name: str = name
+        self.id: int = id
+        self.attr_param_id: Dict[str, int] = {}
+
+
+class SaiApiP4MetaTable:
+    def __init__(self, id: int):
+        self.id: int = id
+        self.actions: Dict[str, SaiApiP4MetaAction] = {}
+
+
+class SaiApiP4Meta:
+    def __init__(self):
+        self.tables: List[SaiApiP4MetaTable] = []

dash-pipeline/SAI/utils/sai_spec/sai_attribute.py

@@ -31,3 +31,11 @@ def __init__(
         self.allow_null = allow_null
         self.valid_only = valid_only
         self.deprecated = deprecated
+
+    def merge(self, other: "SaiCommon"):
+        super().merge(other)
+        self.__dict__.update(other.__dict__)
+
+    def deprecate(self) -> bool:
+        self.deprecated = True
+        return False

dash-pipeline/SAI/utils/sai_spec/sai_common.py

@@ -6,3 +6,21 @@ class SaiCommon:
     def __init__(self, name: str, description: str):
         self.name: str = name
         self.description: str = description
+
+    def merge(self, other: "SaiCommon"):
+        """
+        Merge the other SaiCommon object into this object.
+        """
+        if not isinstance(other, type(self)):
+            raise TypeError(f"Cannot merge {type(self)} with {type(other)}")
+
+        self.description = other.description
+
+    def deprecate(self) -> bool:
+        """
+        Deprecate this object.
+        
+        If the value doesn't support deprecation marking, we don't do anything
+        but return False to keep it in the list.
+        """
+        return False

dash-pipeline/SAI/utils/sai_spec/sai_enum.py

@@ -1,6 +1,7 @@
 from typing import List
 from .sai_common import SaiCommon
 from .sai_enum_member import SaiEnumMember
+from . import sai_spec_utils
 
 
 class SaiEnum(SaiCommon):
@@ -11,3 +12,7 @@ class SaiEnum(SaiCommon):
     def __init__(self, name: str, description: str, members: List[SaiEnumMember] = []):
         super().__init__(name, description)
         self.members: List[SaiEnumMember] = members
+
+    def merge(self, other: "SaiCommon"):
+        super().merge(other)
+        sai_spec_utils.merge_sai_common_lists(self.members, other.members)

dash-pipeline/SAI/utils/sai_spec/sai_enum_member.py

@@ -10,3 +10,7 @@ class SaiEnumMember(SaiCommon):
     def __init__(self, name: str, description: str, value: str):
         super().__init__(name, description)
         self.value: str = value
+
+    def merge(self, other: "SaiCommon"):
+        super().merge(other)
+        self.value = other.value

dash-pipeline/SAI/utils/sai_spec/sai_spec.py

@@ -6,6 +6,7 @@
 from .sai_api_group import SaiApiGroup
 from .sai_api_extension import SaiApiExtension
 from .sai_struct_entry import SaiStructEntry
+from . import sai_spec_utils
 
 
 class SaiSpec:
@@ -24,12 +25,16 @@ def __init__(self):
     def serialize(self, spec_dir: str):
         yaml_inc_files = []
         for api_group in self.api_groups:
-            sai_api_group_spec_file_path = os.path.join(spec_dir, api_group.name + ".yaml")
+            sai_api_group_spec_file_path = os.path.join(
+                spec_dir, api_group.name + ".yaml"
+            )
 
             with open(sai_api_group_spec_file_path, "w") as f:
                 f.write(yaml.dump(api_group, indent=2, sort_keys=False))
-
-            yaml_inc_files.append(yaml_include.Data(urlpath=sai_api_group_spec_file_path))
+
+            yaml_inc_files.append(
+                yaml_include.Data(urlpath=sai_api_group_spec_file_path)
+            )
 
         api_groups = self.api_groups
         self.api_groups = yaml_inc_files
@@ -43,3 +48,21 @@ def serialize(self, spec_dir: str):
     def deserialize(spec_dir: str):
         with open(os.path.join(spec_dir, "sai_spec.yaml")) as f:
             return yaml.unsafe_load(f)
+
+    def merge(self, other: "SaiSpec"):
+        sai_spec_utils.merge_sai_value_lists(
+            self.api_types, other.api_types, lambda x: x
+        )
+        sai_spec_utils.merge_sai_value_lists(
+            self.object_types, other.object_types, lambda x: x
+        )
+        sai_spec_utils.merge_sai_common_lists(self.object_entries, other.object_entries)
+
+        # The global enums are generated from the P4 enum types, so we can respect whatever in the
+        # new spec and simply replace them, because:
+        # - It doesn't matter if the order of enum itself changes.
+        # - We cannot move the enum members as we want, as their order changes their values.
+        self.enums = other.enums
+
+        self.port_extenstion.merge(other.port_extenstion)
+        sai_spec_utils.merge_sai_common_lists(self.api_groups, other.api_groups)

dash-pipeline/SAI/utils/sai_spec/sai_spec_utils.py

@@ -0,0 +1,50 @@
+from typing import Any, List, Callable
+from .sai_common import SaiCommon
+
+
+def merge_sai_value_lists(
+    target: List[Any],
+    source: List[Any],
+    get_key: Callable[[Any], str],
+    on_conflict: Callable[[Any, Any], None] = lambda x, y: x,
+    on_deprecate: Callable[[Any], bool] = lambda x: False
+) -> None:
+    """
+    Merge 2 SAI value lists from source list into target.
+
+    Since we could not remove the old value or change the order of old values, the merge
+    is done as below:
+    - Any new values will be added in the end of the list.
+    - Any values that collapse with existing values will invoke on_conflict callback to resolve.
+    - Any values that needs to be removed will invoke on_deprecate function to deprecate. By default,
+      it will not be removed from the old list.
+    """
+    target_dict = {get_key(item): item for item in target}
+
+    source_keys = set()
+    for source_item in source:
+        source_key = get_key(source_item)
+        source_keys.add(source_key)
+
+        if source_key in target_dict:
+            target_item = target_dict[source_key]
+            on_conflict(target_item, source_item)
+        else:
+            target.append(source_item)
+            target_dict[source_key] = source_item
+
+    # Remove all items in target, if its key doesn't exist in source_keys and on_deprecate returns True.
+    target[:] = [item for item in target if get_key(item) in source_keys or not on_deprecate(item)]
+
+
+def merge_sai_common_lists(
+    target: List[SaiCommon],
+    source: List[SaiCommon],
+) -> None:
+    merge_sai_value_lists(
+        target,
+        source,
+        get_key=lambda x: x.name,
+        on_conflict=lambda x, y: x.merge(y),
+        on_deprecate=lambda x: x.deprecate(),
+    )

dash-pipeline/SAI/utils/sai_spec/sai_struct.py

@@ -1,7 +1,7 @@
 from typing import List
 from .sai_common import SaiCommon
 from .sai_struct_entry import SaiStructEntry
-
+from . import sai_spec_utils
 
 class SaiStruct(SaiCommon):
     """
@@ -11,3 +11,7 @@ class SaiStruct(SaiCommon):
     def __init__(self, name: str, description: str, members: List[SaiStructEntry] = []):
         super().__init__(name, description)
         self.members: List[SaiStructEntry] = members
+
+    def merge(self, other: "SaiCommon"):
+        super().merge(other)
+        sai_spec_utils.merge_sai_common_lists(self.members, other.members)

dash-pipeline/SAI/utils/sai_spec/sai_struct_entry.py

@@ -19,3 +19,7 @@ def __init__(
         self.type = type
         self.objects = objects
         self.valid_only = valid_only
+
+    def merge(self, other: "SaiCommon"):
+        super().merge(other)
+        self.__dict__.update(other.__dict__)