From 7035434f6738eca1630c84c503f218b9bbc8dd92 Mon Sep 17 00:00:00 2001 From: Iakov GAN <82834333+iakov-aws@users.noreply.github.com> Date: Thu, 25 Apr 2024 11:01:39 +0200 Subject: [PATCH] better col management (#798) --- cid/helpers/cur.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cid/helpers/cur.py b/cid/helpers/cur.py index c9298190..40635285 100644 --- a/cid/helpers/cur.py +++ b/cid/helpers/cur.py @@ -4,7 +4,7 @@ import logging from cid.base import CidBase -from cid.utils import get_parameter, get_parameters +from cid.utils import get_parameter, get_parameters, cid_print from cid.exceptions import CidCritical logger = logging.getLogger(__name__) @@ -135,8 +135,9 @@ def ensure_column(self, column: str, column_type: str=None): self.athena.query(f'ALTER TABLE {self.table_name} ADD COLUMNS ({column} {column_type})') except (self.athena.client.exceptions.ClientError, CidCritical) as exc: raise CidCritical(f'Column {column} is not found in CUR and we were unable to add it. Please check FAQ.') from exc - self._metadata = self.athena.get_table_metadata(self.table_name) # refresh table metadata - logger.critical(f"Column '{column}' was added to CUR ({self.table_name}). Please make sure crawler do not override that columns. Crawler='{crawler_name}'") + # table takes time to update so just adding column to cached data + self._metadata.get('Columns', []).append({'Name': column, 'Type': column_type}) + cid_print(f"Column '{column}' was added to CUR ({self.table_name}).") return # if table cannot be updated, check if it is ri/sp case - let's hope dashboard views can handle it: @@ -145,7 +146,7 @@ def ensure_column(self, column: str, column_type: str=None): return # if a required column is not there and not ri/sp -> stop - logger.critical(f"Column '{column}' is not in ({self.table_name}). Cannot continue") + logger.critical(f"Column '{column}' is not in ({self.table_name}). Cannot continue.") def table_is_cur(self, table: dict=None, name: str=None, return_reason: bool=False) -> bool: