Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Network Drive] Add Dropdown for Drive type #1842

Merged
merged 5 commits into from
Oct 30, 2023
Merged

[Network Drive] Add Dropdown for Drive type #1842

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
798d980
add missing try except and dropdown for drive type
praveen-kukreja Oct 26, 2023
6c4101f
Merge branch 'main' into praveen/bugfix-dls-network-drive
praveen-kukreja Oct 26, 2023
1a1bcde
minor nit
praveen-kukreja Oct 26, 2023
869c83e
Merge branch 'main' into praveen/bugfix-dls-network-drive
praveen-kukreja Oct 27, 2023
f597878
Merge branch 'main' into praveen/bugfix-dls-network-drive
praveen-kukreja Oct 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 82 additions & 39 deletions connectors/sources/network_drive.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import fastjsonschema
import smbclient
import winrm
from requests.exceptions import ConnectionError
from smbprotocol.exceptions import SMBException, SMBOSError
from smbprotocol.file_info import (
InfoType,
Expand All @@ -38,7 +39,7 @@
AdvancedRulesValidator,
SyncRuleValidationResult,
)
from connectors.source import BaseDataSource
from connectors.source import BaseDataSource, ConfigurableFieldValueError
from connectors.utils import (
TIKA_SUPPORTED_FILETYPES,
RetryStrategy,
Expand All @@ -59,6 +60,9 @@
RETRIES = 3
RETRY_INTERVAL = 2

WINDOWS = "windows"
LINUX = "linux"


def _prefix_user(user):
return prefix_identity("user", user)
Expand Down Expand Up @@ -247,6 +251,7 @@ def __init__(self, configuration):
self.server_ip = self.configuration["server_ip"]
self.port = self.configuration["server_port"]
self.drive_path = self.configuration["drive_path"]
self.drive_type = self.configuration["drive_type"]
self.identity_mappings = self.configuration["identity_mappings"]
self.session = None
self.security_info = SecurityInfo(self.username, self.password, self.server_ip)
Expand Down Expand Up @@ -297,10 +302,28 @@ def get_default_configuration(cls):
"type": "bool",
"value": False,
},
"drive_type": {
"display": "dropdown",
"label": "Drive type",
"depends_on": [
{"field": "use_document_level_security", "value": True},
],
"options": [
{"label": "Windows", "value": WINDOWS},
{"label": "Linux", "value": LINUX},
],
"order": 7,
"type": "str",
"ui_restrictions": ["advanced"],
"value": WINDOWS,
},
"identity_mappings": {
"label": "Path of CSV file containing users and groups SID (For Linux Network Drive)",
"depends_on": [{"field": "use_document_level_security", "value": True}],
"order": 7,
"depends_on": [
{"field": "use_document_level_security", "value": True},
{"field": "drive_type", "value": LINUX},
],
"order": 8,
"type": "str",
"required": False,
"ui_restrictions": ["advanced"],
Expand Down Expand Up @@ -445,6 +468,8 @@ async def get_content(self, file, timestamp=None, doit=None):
executor=None, func=partial(self.fetch_file_content, path=file["path"])
)

if not content:
return
attachment = content.read()
content.close()
return {
Expand All @@ -454,17 +479,22 @@ async def get_content(self, file, timestamp=None, doit=None):
}

def list_file_permission(self, file_path, file_type, mode, access):
with smbclient.open_file(
file_path,
mode=mode,
buffering=0,
file_type=file_type,
desired_access=access,
) as file:
descriptor = self.security_info.get_descriptor(
file_descriptor=file.fd, info=SECURITY_INFO_DACL
try:
with smbclient.open_file(
file_path,
mode=mode,
buffering=0,
file_type=file_type,
desired_access=access,
) as file:
descriptor = self.security_info.get_descriptor(
file_descriptor=file.fd, info=SECURITY_INFO_DACL
)
return descriptor.get_dacl()["aces"]
except SMBOSError as error:
self._logger.error(
f"Cannot read the contents of file on path:{file_path}. Error {error}"
)
return descriptor.get_dacl()["aces"]

def _dls_enabled(self):
if (
Expand Down Expand Up @@ -521,8 +551,8 @@ def read_user_info_csv(self):
for row in csv_reader:
user_info.append(
{
"username": row[0],
"user_id": row[1],
"name": row[0],
"user_sid": row[1],
"groups": row[2].split(",") if len(row[2]) > 0 else [],
}
)
Expand All @@ -538,36 +568,49 @@ async def get_access_control(self):
return

# This if block fetches users, groups via local csv file path
if self.identity_mappings:
self._logger.info(
f"Fetching all groups and users from configured file path '{self.identity_mappings}'"
)
if self.drive_type == LINUX:
if self.identity_mappings:
self._logger.info(
f"Fetching all groups and users from configured file path '{self.identity_mappings}'"
)

for user in self.read_user_info_csv():
yield await self._user_access_control_doc(
user["name"], user["sid"], user["groups"]
for user in self.read_user_info_csv():
yield await self._user_access_control_doc(
user=user["name"],
sid=user["user_sid"],
groups_info=user["groups"],
)
else:
raise ConfigurableFieldValueError(
"CSV file path cannot be empty. Please provide a valid csv file path."
)
else:
self._logger.info(
f"Fetching all groups and members for drive at path '{self.drive_path}'"
)
groups_info = await asyncio.to_thread(self.security_info.fetch_groups)

groups_members = {}
for group_name, _ in groups_info.items():
groups_members[group_name] = await asyncio.to_thread(
self.security_info.fetch_members, group_name
try:
self._logger.info(
f"Fetching all groups and members for drive at path '{self.drive_path}'"
)
groups_info = await asyncio.to_thread(self.security_info.fetch_groups)

self._logger.info(
f"Fetching all users for drive at path '{self.drive_path}'"
)
users_info = await asyncio.to_thread(self.security_info.fetch_users)
groups_members = {}
for group_name, _ in groups_info.items():
groups_members[group_name] = await asyncio.to_thread(
self.security_info.fetch_members, group_name
)

for user, sid in users_info.items():
yield await self._user_access_control_doc(
user, sid, groups_info, groups_members
self._logger.info(
f"Fetching all users for drive at path '{self.drive_path}'"
)
users_info = await asyncio.to_thread(self.security_info.fetch_users)

for user, sid in users_info.items():
yield await self._user_access_control_doc(
user=user,
sid=sid,
groups_info=groups_info,
groups_members=groups_members,
)
except ConnectionError as exception:
raise ConnectionError("Something went wrong") from exception

async def get_entity_permission(self, file_path, file_type):
if not self._dls_enabled():
Expand All @@ -590,7 +633,7 @@ async def get_entity_permission(self, file_path, file_type):
mode="br",
access=DirectoryAccessMask.READ_CONTROL,
)
for permission in list_permissions:
for permission in list_permissions or []:
if (
permission["ace_type"].value == ACCESS_ALLOWED_TYPE
or permission["mask"].value == ACCESS_MASK_DENIED_WRITE_PERMISSION
Expand Down
17 changes: 15 additions & 2 deletions tests/sources/fixtures/network_drive/connector.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,10 +63,23 @@
"order": 6,
"ui_restrictions": []
},
"drive_type":{
"display": "dropdown",
"label": "Drive type",
"depends_on": [{"field": "use_document_level_security", "value": true}],
"options": [
{"label": "Windows", "value": "windows"},
{"label": "Linux", "value": "linux"}
],
"order":7,
"type": "str",
"ui_restrictions": ["advanced"],
"value":"windows"
},
"identity_mappings": {
"label": "Path of CSV file containing users and groups SID (For Linux Network Drive)",
"depends_on": [{"field": "use_document_level_security", "value": true}],
"order": 7,
"depends_on": [{"field": "use_document_level_security", "value": true}, {"field": "drive_type", "value": "linux"}],
"order": 8,
"type": "str",
"required": false,
"ui_restrictions": ["advanced"],
Expand Down
39 changes: 35 additions & 4 deletions tests/sources/test_network_drive.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

from connectors.filtering.validation import SyncRuleValidationResult
from connectors.protocol import Filter
from connectors.source import ConfigurableFieldValueError
from connectors.sources.network_drive import (
NASDataSource,
NetworkDriveAdvancedRulesValidator,
Expand All @@ -30,6 +31,9 @@
MAX_CHUNK_SIZE = 65536
ADVANCED_SNIPPET = "advanced_snippet"

WINDOWS = "windows"
LINUX = "linux"


def mock_permission(sid):
mock_response = {}
Expand Down Expand Up @@ -702,6 +706,16 @@ async def test_get_access_control_dls_disabled():
assert len(acl) == 0


@pytest.mark.asyncio
async def test_get_access_control_linux_empty_csv_file_path():
async with create_source(NASDataSource) as source:
source._dls_enabled = MagicMock(return_value=True)
source.drive_type = LINUX
source.identity_mappings = ""
with pytest.raises(ConfigurableFieldValueError):
await anext(source.get_access_control())


@pytest.mark.asyncio
async def test_get_access_control_dls_enabled():
expected_user_access_control = [
Expand All @@ -718,6 +732,7 @@ async def test_get_access_control_dls_enabled():

async with create_source(NASDataSource) as source:
source._dls_enabled = MagicMock(return_value=True)
source.drive_type = WINDOWS
mock_groups = {"Admins": "S-1-5-32-546"}
mock_group_members = {
"Administrator": "S-1-5-21-227823342-1368486282-703244805-500"
Expand Down Expand Up @@ -890,8 +905,8 @@ async def test_read_csv_with_valid_data():
):
user_info = source.read_user_info_csv()
expected_user_info = [
{"username": "user1", "user_id": "S-1", "groups": ["S-11", "S-22"]},
{"username": "user2", "user_id": "S-2", "groups": ["S-22"]},
{"name": "user1", "user_sid": "S-1", "groups": ["S-11", "S-22"]},
{"name": "user2", "user_sid": "S-2", "groups": ["S-22"]},
]
assert user_info == expected_user_info

Expand All @@ -913,8 +928,8 @@ async def test_read_csv_with_empty_groups():
):
user_info = source.read_user_info_csv()
expected_user_info = [
{"username": "user1", "user_id": "1", "groups": []},
{"username": "user2", "user_id": "2", "groups": []},
{"name": "user1", "user_sid": "1", "groups": []},
{"name": "user2", "user_sid": "2", "groups": []},
]
assert user_info == expected_user_info

Expand All @@ -938,3 +953,19 @@ async def test_list_file_permissions(mock_get_descriptor):
)

assert result == mock_dacl["aces"]


@pytest.mark.asyncio
async def test_list_file_permissions_with_inaccessible_file():
with mock.patch("smbclient.open_file", return_value=MagicMock()) as mock_file:
mock_file.side_effect = SMBOSError(ntstatus=0xC0000043, filename="file1.txt")

async with create_source(NASDataSource) as source:
result = source.list_file_permission(
file_path="/path/to/file.txt",
file_type="file",
mode="rb",
access="read",
)

assert result is None