diff --git a/.gitignore b/.gitignore index 035941a..8ef5b05 100644 --- a/.gitignore +++ b/.gitignore @@ -102,3 +102,10 @@ pip-selfcheck.json # Generated folders public/ + +Pipfile + +Pipfile\.lock + +.DS_Store +statik.code-workspace diff --git a/.travis.yml b/.travis.yml index c85ed80..2815769 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,22 @@ language: python -python: - - "2.7" - - "3.5" - - "3.6" + +# Takes inspiration from https://github.com/tornadoweb/tornado/blob/master/.travis.yml +.mixins: +- &xenial-mixin + dist: xenial + sudo: true + addons: + apt: + packages: + - libgnutls-dev +jobs: + include: + - python: 2.7 + - python: 3.5 + - python: 3.6 + - <<: *xenial-mixin + python: 3.7 + install: - "pip install -r requirements.txt" script: diff --git a/CHANGELOG.md b/CHANGELOG.md index 1444bb2..deed71d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,25 @@ This is the **Statik** change log as of version `0.6.0`. ## Release History +### `v0.22.2` - 20 October 2018 + +* Merging #74 to better organise the growing list of CLI arguments. + +### `v0.22.1` - 8 October 2018 + +* Minor fix to add missing `paramiko` dependency + +### `v0.22.0` - 8 October 2018 + +* Merges #68 +* Merges #73 to add SFTP upload functionality +* Updates dependency versions in `requirements.txt` +* Updates Markdown package integration to use new API + +### `v0.21.3` - 15 February 2018 + +* Merging #65 to fix #64. + ### `v0.21.2` - 12 February 2018 * Attempting to fix issues #50 and #63 - automatic translation of special Unicode characters diff --git a/CONTRIBUTORS b/CONTRIBUTORS deleted file mode 100644 index c7126f1..0000000 --- a/CONTRIBUTORS +++ /dev/null @@ -1,4 +0,0 @@ -Thane Thomson -Patrick Paul -Kenton Hamaluik -Roman Vaughan \ No newline at end of file diff --git a/examples/blog/models/Tag.yml b/examples/blog/models/Tag.yml deleted file mode 100644 index e69de29..0000000 diff --git a/files.txt b/files.txt new file mode 100644 index 0000000..db603ce --- /dev/null +++ b/files.txt @@ -0,0 +1,10 @@ +/usr/local/lib/python3.7/site-packages/statik-0.21.2-py3.7.egg +/usr/local/bin/statik +/usr/local/bin/pystache +/usr/local/bin/pystache-test +/usr/local/bin/httpwatcher +/usr/local/bin/slugify +/usr/local/bin/markdown_py +/usr/local/bin/futurize +/usr/local/bin/pasteurize +/usr/local/bin/watchmedo diff --git a/requirements.txt b/requirements.txt index 8b7889d..ef30d27 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,21 @@ -future>=0.16.0 -jinja2>=2.8 -PyYAML>=3.11 -SQLAlchemy>=1.0.14 -markdown>=2.6.6 -python-slugify>=1.2.1 -six>=1.10.0 -lipsum>=0.1.1 -httpwatcher>=0.5.1 -mlalchemy>=0.2.1 -pystache>=0.5.4 -colorlog>=3.1.0 -python-dateutil>=2.6.1 \ No newline at end of file +wheel +argh==0.26.2 +colorlog==3.1.4 +future==0.16.0 +httpwatcher==0.5.1 +Jinja2==2.10 +lipsum==0.1.2 +Markdown==3.0.1 +MarkupSafe==1.0 +mlalchemy==0.2.2 +pathtools==0.1.2 +pystache==0.5.4 +python-dateutil==2.7.3 +python-slugify==1.2.6 +PyYAML==3.13 +six==1.11.0 +SQLAlchemy==1.2.12 +tornado==5.1.1 +Unidecode==1.0.22 +watchdog==0.9.0 +paramiko==2.4.2 diff --git a/setup.py b/setup.py index 3e59b8e..7831e82 100644 --- a/setup.py +++ b/setup.py @@ -54,6 +54,7 @@ def get_version(): "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", "Topic :: Internet :: WWW/HTTP", "Topic :: Utilities", ] diff --git a/statik/__init__.py b/statik/__init__.py index 95d4256..14a6482 100644 --- a/statik/__init__.py +++ b/statik/__init__.py @@ -1,3 +1,4 @@ # -*- coding:utf-8 -*- -__version__ = "0.21.2" +__version__ = "0.22.2" + diff --git a/statik/common.py b/statik/common.py index e862887..cc38b08 100644 --- a/statik/common.py +++ b/statik/common.py @@ -56,7 +56,8 @@ class ContentLoadable(object): loading content and metadata from a Markdown file. """ def __init__(self, filename=None, file_type=None, from_string=None, from_dict=None, - name=None, markdown_config=None, encoding='utf-8', error_context=None): + name=None, markdown_config=None, encoding='utf-8', error_context=None, + content_field = None): self.vars = None self.content = None self.file_content = None @@ -102,7 +103,6 @@ def __init__(self, filename=None, file_type=None, from_string=None, from_dict=No "filename", "from_string", "from_dict", context=self.error_context ) - if name is not None: self.name = name elif self.filename is not None: @@ -112,7 +112,23 @@ def __init__(self, filename=None, file_type=None, from_string=None, from_dict=No "name", "filename", context=self.error_context ) - + markdown_ext = [ + MarkdownYamlMetaExtension(), + MarkdownLoremIpsumExtension(error_context=self.error_context) + ] + if self.markdown_config.enable_permalinks: + markdown_ext.append( + MarkdownPermalinkExtension( + permalink_text=self.markdown_config.permalink_text, + permalink_class=self.markdown_config.permalink_class, + permalink_title=self.markdown_config.permalink_title, + ) + ) + markdown_ext.extend(self.markdown_config.extensions) + md = Markdown( + extensions=markdown_ext, + extension_configs=self.markdown_config.extension_config + ) # if it wasn't loaded from a dictionary if self.vars is None: if self.file_type is None: @@ -127,26 +143,11 @@ def __init__(self, filename=None, file_type=None, from_string=None, from_dict=No if not isinstance(self.vars, dict): self.vars = {} else: - markdown_ext = [ - MarkdownYamlMetaExtension(), - MarkdownLoremIpsumExtension(error_context=self.error_context) - ] - if self.markdown_config.enable_permalinks: - markdown_ext.append( - MarkdownPermalinkExtension( - permalink_text=self.markdown_config.permalink_text, - permalink_class=self.markdown_config.permalink_class, - permalink_title=self.markdown_config.permalink_title, - ) - ) - markdown_ext.extend(self.markdown_config.extensions) - - md = Markdown( - extensions=markdown_ext, - extension_configs=self.markdown_config.extension_config - ) self.content = md.convert(self.file_content) self.vars = md.meta + else: + if (content_field is not None) and (content_field in self.vars): + self.content = md.convert(self.vars[content_field]) if isinstance(self.vars, dict): self.vars = dict_strip(self.vars) diff --git a/statik/context.py b/statik/context.py index 80eb86c..a9be54c 100644 --- a/statik/context.py +++ b/statik/context.py @@ -44,12 +44,12 @@ def __repr__(self): def __str__(self): return repr(self) - def build_dynamic(self, db, safe_mode=False): + def build_dynamic(self, db, extra=None, safe_mode=False): """Builds the dynamic context based on our current dynamic context entity and the given database.""" result = dict() for var, query in iteritems(self.dynamic): - result[var] = db.query(query, safe_mode=safe_mode) + result[var] = db.query(query, safe_mode=safe_mode, additional_locals=extra) return result def build_for_each(self, db, safe_mode=False, extra=None): @@ -68,7 +68,7 @@ def build(self, db=None, safe_mode=False, for_each_inst=None, extra=None): result = copy(self.initial) result.update(self.static) if self.dynamic: - result.update(self.build_dynamic(db, safe_mode=safe_mode)) + result.update(self.build_dynamic(db, extra=extra, safe_mode=safe_mode)) if self.for_each and for_each_inst: result.update(self.build_for_each(db, safe_mode=safe_mode, extra=extra)) if isinstance(extra, dict): diff --git a/statik/database.py b/statik/database.py index 0864955..a3a33ae 100644 --- a/statik/database.py +++ b/statik/database.py @@ -6,11 +6,22 @@ from io import open import os.path +import glob import yaml -from sqlalchemy import String, Integer, Column, Table, ForeignKey, \ - Boolean, DateTime, Text, create_engine +from sqlalchemy import ( + String, + Integer, + Column, + Table, + ForeignKey, + Boolean, + DateTime, + Text, + create_engine, +) from sqlalchemy.orm import sessionmaker, relationship, backref +from sqlalchemy.orm.exc import NoResultFound from sqlalchemy.ext.declarative import declarative_base import mlalchemy @@ -31,40 +42,43 @@ import math import logging + logger = logging.getLogger(__name__) __all__ = [ - 'StatikDatabase', + "StatikDatabase", ] SQLALCHEMY_FIELD_MAPPER = { - 'String': String, - 'DateTime': DateTime, - 'Integer': Integer, - 'Boolean': Boolean, - 'Content': Text, - 'Text': Text + "String": String, + "DateTime": DateTime, + "Integer": Integer, + "Boolean": Boolean, + "Content": Text, + "Text": Text, } def set_global(name, val): - logger.debug('Setting global: %s = %s' % (name, val)) + logger.debug("Setting global: %s = %s" % (name, val)) globals()[name] = val set_global.tracked_globals.add(name) + + set_global.tracked_globals = set() def clear_tracked_globals(): for name in set_global.tracked_globals: - logger.debug('Clearing tracked global: %s' % name) + logger.debug("Clearing tracked global: %s" % name) del globals()[name] set_global.tracked_globals = set() class StatikDatabase(object): - - def __init__(self, data_path, models, encoding=None, markdown_config=None, - error_context=None): + def __init__( + self, data_path, models, encoding=None, markdown_config=None, error_context=None + ): """Constructor. Args: @@ -81,16 +95,16 @@ def __init__(self, data_path, models, encoding=None, markdown_config=None, self.models = models self.markdown_config = markdown_config self.error_context = error_context or StatikErrorContext() - self.engine = create_engine('sqlite:///:memory:') + self.engine = create_engine("sqlite:///:memory:") self.Base = declarative_base() self.session = sessionmaker(bind=self.engine)() - set_global('session', self.session) + set_global("session", self.session) self.find_backrefs() self.create_db(models) def find_backrefs(self): for model_name, model in iteritems(self.models): - logger.debug('Attempting to find backrefs for model: %s', model_name) + logger.debug("Attempting to find backrefs for model: %s", model_name) try: model.find_additional_rels(self.models) except Exception as exc: @@ -98,7 +112,7 @@ def find_backrefs(self): model_name, message="failed to accurately determine model cross-referencing.", orig_exc=exc, - context=self.error_context + context=self.error_context, ) def create_db(self, models): @@ -117,8 +131,7 @@ def create_db(self, models): self.Base.metadata.create_all(self.engine) except Exception as exc: raise StatikError( - message="Failed to create in-memory data model.", - orig_exc=exc + message="Failed to create in-memory data model.", orig_exc=exc ) self.load_all_model_data(models) @@ -144,7 +157,9 @@ def sort_models(self): A sorted list containing the names of the models. """ model_names = [ - table.name for table in self.Base.metadata.sorted_tables if table.name in self.models + table.name + for table in self.Base.metadata.sorted_tables + if table.name in self.models ] logger.debug("Unsorted models: %s", model_names) model_count = len(model_names) @@ -153,7 +168,7 @@ def sort_models(self): sort_round = 0 while swapped: sort_round += 1 - logger.debug('Sorting round: %d (%s)', sort_round, model_names) + logger.debug("Sorting round: %d (%s)", sort_round, model_names) sorted_models = [] for i in range(model_count): @@ -192,7 +207,7 @@ def create_model_table(self, model): model.name, message="failed to create in-memory table.", orig_exc=exc, - context=self.error_context + context=self.error_context, ) def load_model_data(self, path, model): @@ -200,72 +215,75 @@ def load_model_data(self, path, model): """ if os.path.isdir(path): # try find a model data collection - if os.path.isfile(os.path.join(path, '_all.yml')): + if glob.glob(os.path.join(path, "*.yml")): self.load_model_data_collection(path, model) - else: - self.load_model_data_from_files(path, model) + self.load_model_data_from_files(path, model) self.session.commit() def load_model_data_collection(self, path, model): - full_filename = os.path.join(path, '_all.yml') - self.error_context.update(filename=full_filename) + for full_filename in glob.iglob(os.path.join(path, "*.yml")): + # full_filename = os.path.join(path, '_all.yml') + self.error_context.update(filename=full_filename) - db_model = globals()[model.name] - # load the collection data from the collection file - with open(full_filename, mode='rt', encoding=self.encoding) as f: - collection = yaml.load(f.read()) + db_model = globals()[model.name] + # load the collection data from the collection file + with open(full_filename, mode="rt", encoding=self.encoding) as f: + collection = yaml.load(f.read(), Loader=yaml.FullLoader) - if not isinstance(collection, list): - raise InvalidModelCollectionDataError( - model.name, - context=self.error_context - ) - seen_entries = set() - logger.debug("Loading %d instance(s) for model: %s", len(collection), model.name) - for item in collection: - if not isinstance(item, dict) or 'pk' not in item: + if not isinstance(collection, list): raise InvalidModelCollectionDataError( - model.name, - context=self.error_context + model.name, context=self.error_context ) - - entry = StatikDatabaseInstance( - name=item['pk'], - from_dict=item, - model=model, - session=self.session, - encoding=self.encoding, - markdown_config=self.markdown_config + seen_entries = set() + logger.debug( + "Loading %d instance(s) for model: %s", len(collection), model.name ) - # duplicate primary key! - if entry.field_values['pk'] in seen_entries: - raise DuplicateModelInstanceError( - model.name, - pk=entry.field_values['pk'], - context=self.error_context - ) - else: - seen_entries.add(entry.field_values['pk']) + for item in collection: + if not isinstance(item, dict) or "pk" not in item: + raise InvalidModelCollectionDataError( + model.name, context=self.error_context + ) - try: - db_entry = db_model(**entry.field_values) - self.session.add(db_entry) - except Exception as exc: - raise DataError( - model.name, - pk=entry.field_values['pk'], - message="failed to insert entry into in-memory database.", - orig_exc=exc, - context=self.error_context + entry = StatikDatabaseInstance( + name=item["pk"], + from_dict=item, + model=model, + session=self.session, + encoding=self.encoding, + markdown_config=self.markdown_config, ) - + # duplicate primary key! + if entry.field_values["pk"] in seen_entries: + raise DuplicateModelInstanceError( + model.name, + pk=entry.field_values["pk"], + context=self.error_context, + ) + else: + seen_entries.add(entry.field_values["pk"]) + + try: + db_entry = db_model(**entry.field_values) + self.session.add(db_entry) + except Exception as exc: + raise DataError( + model.name, + pk=entry.field_values["pk"], + message="failed to insert entry into in-memory database.", + orig_exc=exc, + context=self.error_context, + ) + self.error_context.clear() def load_model_data_from_files(self, path, model): db_model = globals()[model.name] - entry_files = list_files(path, ['yml', 'yaml', 'md']) + entry_files = list_files(path, ["yml", "yaml", "md"]) + entry_files = [f for f in entry_files if not (f.endswith(".yml"))] seen_entries = set() - logger.debug("Loading %d instance(s) for model: %s", len(entry_files), model.name) + logger.debug( + "Loading %d instance(s) for model: %s", len(entry_files), model.name + ) for entry_file in entry_files: entry = StatikDatabaseInstance( filename=os.path.join(path, entry_file), @@ -273,17 +291,15 @@ def load_model_data_from_files(self, path, model): session=self.session, encoding=self.encoding, markdown_config=self.markdown_config, - error_context=self.error_context + error_context=self.error_context, ) # duplicate primary key! - if entry.field_values['pk'] in seen_entries: + if entry.field_values["pk"] in seen_entries: raise DuplicateModelInstanceError( - model.name, - pk=entry.field_values['pk'], - context=self.error_context + model.name, pk=entry.field_values["pk"], context=self.error_context ) else: - seen_entries.add(entry.field_values['pk']) + seen_entries.add(entry.field_values["pk"]) try: db_entry = db_model(**entry.field_values) @@ -291,10 +307,10 @@ def load_model_data_from_files(self, path, model): except Exception as exc: raise DataError( model.name, - pk=entry.field_values['pk'], + pk=entry.field_values["pk"], message="failed to insert entry into in-memory database.", orig_exc=exc, - context=self.error_context + context=self.error_context, ) self.error_context.clear() @@ -316,13 +332,15 @@ def query(self, query, additional_locals=None, safe_mode=False): logger.debug("Attempting to execute database query: %s", query) if safe_mode and not isinstance(query, dict): - raise SafetyViolationError( - context=self.error_context - ) + raise SafetyViolationError(context=self.error_context) if isinstance(query, dict): logger.debug("Executing query in safe mode (MLAlchemy)") - return mlalchemy.parse_query(query).to_sqlalchemy(self.session, self.tables).all() + return ( + mlalchemy.parse_query(query) + .to_sqlalchemy(self.session, self.tables) + .all() + ) else: logger.debug("Executing unsafe query (Python exec())") if additional_locals is not None: @@ -330,15 +348,11 @@ def query(self, query, additional_locals=None, safe_mode=False): locals()[k] = v exec( - compile( - 'result = %s' % query.strip(), - '', - 'exec' - ), + compile("result = %s" % query.strip(), "", "exec"), globals(), - locals() + locals(), ) - return locals()['result'] + return locals()["result"] def shutdown(self): """Shuts down the database engine.""" @@ -349,83 +363,159 @@ def shutdown(self): class StatikDatabaseInstance(ContentLoadable): - def __init__(self, model=None, session=None, **kwargs): - super(StatikDatabaseInstance, self).__init__(**kwargs) + super(StatikDatabaseInstance, self).__init__( + content_field=model.content_field, **kwargs + ) if model is None: raise MissingParameterError("model", context=self.error_context) self.model = model + self.implicit_data_items = [] + if session is None: raise MissingParameterError("session", context=self.error_context) self.session = session # convert the vars to their underscored representation self.field_values = underscore_var_names(self.vars) - self.field_values['pk'] = self.name + self.field_values["pk"] = self.name # run through the foreign key fields to check their assignment for field_name in self.model.field_names: field = self.model.fields[field_name] - if isinstance(field, StatikDateTimeField) and \ - isinstance(self.field_values.get(field_name), basestring): + if isinstance(field, StatikDateTimeField) and isinstance( + self.field_values.get(field_name), basestring + ): # attempt to perform an intelligent date/time parse operation - self.field_values[field_name] = dateutil_parse(self.field_values[field_name]) + self.field_values[field_name] = dateutil_parse( + self.field_values[field_name] + ) # if it's a foreign key elif isinstance(field, StatikForeignKeyField): # if we've got a pk value for a foreign key field if field_name in self.field_values: - self.field_values['%s_id' % field_name] = self.field_values[field_name] + self.field_values["%s_id" % field_name] = self.field_values[ + field_name + ] del self.field_values[field_name] - elif isinstance(field, StatikManyToManyField): + elif isinstance(field, StatikManyToManyField) and ( + field_name in self.field_values + ): if not isinstance(self.field_values[field_name], list): raise InvalidFieldTypeError( self.model.name, field_name, "a list", - context=self.error_context + context=self.error_context, ) - logger.debug( - "Attempting to look up primary keys for ManyToMany " + - "field relationship: %s", self.field_values[field_name] + "Attempting to look up primary keys for ManyToMany " + + "field relationship: %s", + self.field_values[field_name], + ) + + duplicates_in_array = find_duplicates_in_array( + self.field_values[field_name] ) + + if duplicates_in_array: + logger.warning( + "Duplicates found in %s: %s (field: %s)", + self.filename, + duplicates_in_array, + field_name, + ) + self.field_values[field_name] = list( + set(self.field_values[field_name]) + ) + + # check if non-string items are present + for item in self.field_values[field_name]: + if not isinstance(item, ("".__class__, "".__class__)): + logger.warning( + "Non-string values found in array " + + "(field: %s, instance: %s, model: %s): %s", + field_name, + self.field_values["pk"], + self.model.name, + item, + ) + # convert the list of field values to a query to look up the # primary keys of the corresponding table other_model = globals()[field.field_type] - self.field_values[field_name] = self.session.query( - other_model - ).filter( - other_model.pk.in_(self.field_values[field_name]) - ).all() + + missing_items = [] + + for item in self.field_values[field_name]: + try: + self.session.query(other_model).filter( + other_model.pk == item + ).one() + except NoResultFound: + # Only allow implicit tables if the model + # has no fields other than 'pk' + if len(other_model.__table__._columns) == 1: + missing_items.append({"pk": item}) + else: + logger.warning("%s not found in %s" % (item, other_model)) + else: + logger.debug("%s found", item) + + self.implicit_data_items.append((field.field_type, missing_items)) + + original_values = self.field_values[field_name] + + self.field_values[field_name] = ( + self.session.query(other_model) + .filter(other_model.pk.in_(self.field_values[field_name])) + .all() + ) + + # Ensure that values appear in original order + self.field_values[field_name].sort( + key=lambda x: original_values.index(x.pk) + ) + + # Ensure that values appear in original order + self.field_values[field_name].sort( + key=lambda x: original_values.index(x.pk) + ) # populate any Content field for this model if self.model.content_field is not None: self.field_values[self.model.content_field] = self.content - logger.debug('%s', self) + logger.debug("%s", self) def __repr__(self): result = ["StatikDatabaseInstance(model=%s" % self.model.name] for field_name, field_value in iteritems(self.field_values): model_field = self.model.fields.get(field_name, None) - if isinstance(model_field, StatikContentField) or isinstance(model_field, StatikTextField): + if isinstance(model_field, StatikContentField) or isinstance( + model_field, StatikTextField + ): result.append("%s=<...>" % field_name) else: result.append("%s=%s" % (field_name, field_value)) - result[-1] += ')' - return ', '.join(result) + result[-1] += ")" + return ", ".join(result) def __str__(self): return repr(self) def db_model_factory(Base, model, all_models): - def get_or_create_association_table(model1_name, model2_name): - _association_table_name = calculate_association_table_name(model1_name, model2_name) - logger.debug("Creating/getting ManyToMany relationship table: %s", _association_table_name) + _association_table_name = calculate_association_table_name( + model1_name, model2_name + ) + logger.debug( + "Creating/getting ManyToMany relationship table: %s", + _association_table_name, + ) if _association_table_name in globals(): return globals()[_association_table_name] @@ -433,35 +523,36 @@ def get_or_create_association_table(model1_name, model2_name): _association_table = Table( _association_table_name, Base.metadata, - Column('%s_pk' % model1_name.lower(), String, ForeignKey('%s.pk' % model1_name)), - Column('%s_pk' % model2_name.lower(), String, ForeignKey('%s.pk' % model2_name)) + Column( + "%s_pk" % model1_name.lower(), String, ForeignKey("%s.pk" % model1_name) + ), + Column( + "%s_pk" % model2_name.lower(), String, ForeignKey("%s.pk" % model2_name) + ), ) # track it in our globals set_global(_association_table_name, _association_table) return _association_table - logger.debug('-----') + logger.debug("-----") logger.debug("Generating model: %s", model.name) - model_fields = { - '__tablename__': model.name, - 'pk': Column(String, primary_key=True) - } + model_fields = {"__tablename__": model.name, "pk": Column(String, primary_key=True)} # populate all of the relevant additional relationships for this model for field_name, rel in iteritems(model.additional_rels): kwargs = {} - if rel.get('back_populates', None) is not None: - kwargs['back_populates'] = rel['back_populates'] - if rel.get('secondary', None) is not None: - kwargs['secondary'] = get_or_create_association_table(*rel['secondary']) + if rel.get("back_populates", None) is not None: + kwargs["back_populates"] = rel["back_populates"] + if rel.get("secondary", None) is not None: + kwargs["secondary"] = get_or_create_association_table(*rel["secondary"]) logger.debug( - 'Creating additional relationship %s.%s -> %s (%s)', + "Creating additional relationship %s.%s -> %s (%s)", model.name, field_name, - rel['to_model'], - kwargs + rel["to_model"], + kwargs, ) - model_fields[field_name] = relationship(rel['to_model'], **kwargs) + model_fields[field_name] = relationship(rel["to_model"], **kwargs) # now populate all of the standard fields for field_name in model.field_names: @@ -469,67 +560,66 @@ def get_or_create_association_table(model1_name, model2_name): if field.field_type in SQLALCHEMY_FIELD_MAPPER: # if it's a simple field model_fields[field.name] = Column( - field.name, - SQLALCHEMY_FIELD_MAPPER[field.field_type] + field.name, SQLALCHEMY_FIELD_MAPPER[field.field_type] ) elif field.field_type in all_models: # if it's a foreign key reference if isinstance(field, StatikForeignKeyField): - model_fields['%s_id' % field.name] = Column( - '%s_id' % field.name, - ForeignKey('%s.pk' % field.field_type) + model_fields["%s_id" % field.name] = Column( + "%s_id" % field.name, ForeignKey("%s.pk" % field.field_type) ) # if it's a self-referencing foreign key if field.field_type == model.name: - back_populates = field.back_populates or 'children' + back_populates = field.back_populates or "children" model_fields[back_populates] = relationship( field.field_type, - backref=backref(field_name, remote_side=[model_fields['pk']]) + backref=backref(field_name, remote_side=[model_fields["pk"]]), ) else: kwargs = {} if field.back_populates is not None: - kwargs['back_populates'] = field.back_populates - logger.debug('Field %s.%s has back-populates field name: %s', - model.name, field_name, field.back_populates + kwargs["back_populates"] = field.back_populates + logger.debug( + "Field %s.%s has back-populates field name: %s", + model.name, + field_name, + field.back_populates, ) else: - logger.debug('No back-populates field name for %s.%s', - model.name, field_name + logger.debug( + "No back-populates field name for %s.%s", + model.name, + field_name, ) + foreign_key = model_fields.get("%s_id" % field.name, None) model_fields[field.name] = relationship( - field.field_type, - **kwargs + field.field_type, foreign_keys=[foreign_key], **kwargs ) elif isinstance(field, StatikManyToManyField): - association_table = get_or_create_association_table(model.name, field.field_type) + association_table = get_or_create_association_table( + model.name, field.field_type + ) - kwargs = {'secondary': association_table} + kwargs = {"secondary": association_table} if field.back_populates is not None: - kwargs['back_populates'] = field.back_populates + kwargs["back_populates"] = field.back_populates - logger.debug("Creating model ManyToMany field %s.%s -> %s (%s)", - model.name, field.name, field.field_type, kwargs - ) - model_fields[field.name] = relationship( + logger.debug( + "Creating model ManyToMany field %s.%s -> %s (%s)", + model.name, + field.name, field.field_type, - **kwargs + kwargs, ) + model_fields[field.name] = relationship(field.field_type, **kwargs) else: - raise InvalidFieldTypeError( - model.name, - field.name - ) + raise InvalidFieldTypeError(model.name, field.name) - Model = type( - str(model.name), - (Base,), - model_fields - ) + Model = type(str(model.name), (Base,), model_fields) logger.debug("Model %s fields = %s", model.name, model_fields) diff --git a/statik/jinja2ext.py b/statik/jinja2ext.py index ec53d3e..91d199c 100644 --- a/statik/jinja2ext.py +++ b/statik/jinja2ext.py @@ -146,8 +146,8 @@ def __init__(self, environment): self.active_tag = None logger.debug("Loaded custom template tags: %s", ", ".join(self.tags)) - def _invoke_tag(self, context, *args, **kwargs): - return templatetags.store.invoke_tag(self.active_tag, context, *args, **kwargs) + def _invoke_tag(self, tag_name, context, *args, **kwargs): + return templatetags.store.invoke_tag(tag_name, context, *args, **kwargs) def parse(self, parser): lineno = next(parser.stream).lineno @@ -165,6 +165,7 @@ def parse(self, parser): # get the tag_name for use in looking up callable self.active_tag = parser._tag_stack[-1] + args.insert(0, nodes.Const(self.active_tag)) # create the node node = self.call_method('_invoke_tag', args=args, lineno=lineno) diff --git a/statik/markdown_config.py b/statik/markdown_config.py index 96c385d..5faf7f0 100644 --- a/statik/markdown_config.py +++ b/statik/markdown_config.py @@ -6,18 +6,17 @@ from statik.errors import * -__all__ = [ - 'MarkdownConfig' -] +__all__ = ["MarkdownConfig"] class MarkdownConfig(object): DEFAULT_MARKDOWN_EXTENSIONS = [ - 'markdown.extensions.fenced_code', - 'markdown.extensions.tables', - 'markdown.extensions.toc', - 'markdown.extensions.footnotes' + # 'markdown.extensions.fenced_code', + # 'markdown.extensions.tables', + # 'markdown.extensions.toc', + # 'markdown.extensions.footnotes' + # NoteExtension() ] def __init__(self, markdown_params=None, error_context=None): @@ -26,57 +25,62 @@ def __init__(self, markdown_params=None, error_context=None): if not isinstance(markdown_params, dict): raise ProjectConfigurationError( message="Markdown configuration parameters must be a dictionary.", - context=error_context + context=error_context, ) - permalinks_config = markdown_params.get('permalinks', dict()) + permalinks_config = markdown_params.get("permalinks", dict()) - self.enable_permalinks = permalinks_config.get('enabled', False) + self.enable_permalinks = permalinks_config.get("enabled", False) if self.enable_permalinks in {"true", "1", 1}: self.enable_permalinks = True elif self.enable_permalinks in {"false", "0", 0}: self.enable_permalinks = False - self.permalink_text = permalinks_config.get('text', "¶") - self.permalink_class = permalinks_config.get('class', None) - self.permalink_title = permalinks_config.get('title', None) + self.permalink_text = permalinks_config.get("text", "¶") + self.permalink_class = permalinks_config.get("class", None) + self.permalink_title = permalinks_config.get("title", None) # Required list of Markdown extensions self.extensions = copy(MarkdownConfig.DEFAULT_MARKDOWN_EXTENSIONS) # Configuration for the markdown extensions self.extension_config = {} - extension_list = markdown_params.get('extensions', []) + extension_list = markdown_params.get("extensions", []) # if it's a dictionary, first convert it to our list notation if isinstance(extension_list, dict): extension_list = [] - for ext_package, config in iteritems(markdown_params['extensions']): + for ext_package, config in iteritems(markdown_params["extensions"]): extension_list.append({ext_package: config}) # Try to load extensions as requested by config for extension in extension_list: if isinstance(extension, dict): - ext_package, config = next(iter(extension.keys())), next(iter(extension.values())) + ext_package, config = ( + next(iter(extension.keys())), + next(iter(extension.values())), + ) else: ext_package, config = extension, None if ext_package not in self.extensions: self.extensions.append(ext_package) - if config is not None: + if config is not None: if ext_package in self.extension_config: self.extension_config[ext_package].update(config) else: self.extension_config[ext_package] = config def __repr__(self): - return ("MarkdownConfig(enable_permalinks=%s, permalink_text=%s, permalink_class=%s, " + - "permalink_title=%s, extensions=%s, extension_config=%s)") % ( + return ( + "MarkdownConfig(enable_permalinks=%s, permalink_text=%s, permalink_class=%s, " + + "permalink_title=%s, extensions=%s, extension_config=%s)" + ) % ( self.enable_permalinks, self.permalink_text, self.permalink_class, self.permalink_title, self.extensions, - self.extension_config + self.extension_config, ) diff --git a/statik/markdown_exts.py b/statik/markdown_exts.py index 85290e6..20f84f4 100644 --- a/statik/markdown_exts.py +++ b/statik/markdown_exts.py @@ -28,11 +28,11 @@ class MarkdownYamlMetaExtension(Extension): - def extendMarkdown(self, md, md_globals): - md.preprocessors.add( - 'yaml-meta', + def extendMarkdown(self, md): + md.preprocessors.register( MarkdownYamlMetaPreprocessor(md), - ">normalize_whitespace", + 'yaml-meta', + 40 ) @@ -44,16 +44,16 @@ def __init__(self, *args, **kwargs): self.permalink_title = kwargs.pop('permalink_title', None) super(MarkdownPermalinkExtension, self).__init__(*args, **kwargs) - def extendMarkdown(self, md, md_globals): - md.treeprocessors.add( - 'permalink', + def extendMarkdown(self, md): + md.treeprocessors.register( MarkdownPermalinkProcessor( md, permalink_text=self.permalink_text, permalink_class=self.permalink_class, permalink_title=self.permalink_title ), - 'yaml-meta" + "lipsum", + 50 ) @@ -75,7 +75,7 @@ class MarkdownYamlMetaPreprocessor(Preprocessor): def run(self, lines): result = [] - self.markdown.meta = {} + self.md.meta = {} if len(lines) > 1: yaml_lines = [] @@ -96,7 +96,7 @@ def run(self, lines): result.append(line) if len(yaml_lines) > 0: - self.markdown.meta = yaml.safe_load( + self.md.meta = yaml.safe_load( '\n'.join(yaml_lines) ) diff --git a/statik/utils.py b/statik/utils.py index 5d83d65..2c4f7d0 100644 --- a/statik/utils.py +++ b/statik/utils.py @@ -42,7 +42,8 @@ '_str', '_unicode', 'find_first_file_with_ext', - 'uncapitalize' + 'uncapitalize', + 'find_duplicates_in_array', ] DEFAULT_CONFIG_CONTENT = """project-name: Your project name @@ -357,3 +358,26 @@ def find_first_file_with_ext(base_paths, prefix, exts): def uncapitalize(s): """If the given string begins with a capital letter, it converts it to lowercase.""" return (s[:1].lower() + s[1:]) if s else "" + +def find_duplicates_in_array(array): + """Runs through the array and returns the elements that contain + more than one duplicate + Args: + array: The array to check for duplicates. + Returns: + Array of the elements that are duplicates. Returns empty list if + there are no duplicates. + """ + duplicates = [] + non_duplicates = [] + + if len(array) != len(set(array)): + for item in array: + if item not in non_duplicates: + non_duplicates.append(item) + elif item in non_duplicates and item not in duplicates: + duplicates.append(item) + + return duplicates + + \ No newline at end of file diff --git a/tests/integration/data-non-root-base/assets/EMPTY b/tests/integration/data-non-root-base/assets/EMPTY deleted file mode 100644 index e69de29..0000000 diff --git a/tests/modular/data_test_database/Address/adanac.yml b/tests/modular/data_test_database/Address/adanac.yml new file mode 100644 index 0000000..d3134e3 --- /dev/null +++ b/tests/modular/data_test_database/Address/adanac.yml @@ -0,0 +1,2 @@ +street: Adanac +postal_code: V5K 2S6 diff --git a/tests/modular/data_test_database/Address/washington.yml b/tests/modular/data_test_database/Address/washington.yml new file mode 100644 index 0000000..596c21b --- /dev/null +++ b/tests/modular/data_test_database/Address/washington.yml @@ -0,0 +1,2 @@ +street: Washington +postal_code: V5M 2H9 diff --git a/tests/modular/data_test_database/Guest/gmerriweather.yml b/tests/modular/data_test_database/Guest/gmerriweather.yml index 0ae320f..1e24188 100644 --- a/tests/modular/data_test_database/Guest/gmerriweather.yml +++ b/tests/modular/data_test_database/Guest/gmerriweather.yml @@ -1,3 +1,5 @@ first-name: Gary last-name: Merriweather email: gmerriweather@somewhere.com +home_address: adanac +business_address: washington diff --git a/tests/modular/data_test_database/Guest/manderson.yml b/tests/modular/data_test_database/Guest/manderson.yml index 1593680..87543e9 100644 --- a/tests/modular/data_test_database/Guest/manderson.yml +++ b/tests/modular/data_test_database/Guest/manderson.yml @@ -1,3 +1,5 @@ first-name: Michael last-name: Anderson email: manderson@somewhere.com +home_address: adanac +business_address: washington diff --git a/tests/modular/test_context.py b/tests/modular/test_context.py new file mode 100644 index 0000000..ff3a0b4 --- /dev/null +++ b/tests/modular/test_context.py @@ -0,0 +1,16 @@ +# -*- coding:utf-8 -*- +import unittest + +from statik.context import StatikContext +from statik.database import StatikDatabase + + +class TestStatikContext(unittest.TestCase): + def test_build_context(self): + context = StatikContext(dynamic={'render_elm': 'True if my_var > 10 else False'}) + + result = context.build(db=StatikDatabase(models={}, data_path=''), extra={'my_var': 30}) + assert result.get('render_elm') is True + + result = context.build(db=StatikDatabase(models={}, data_path=''), extra={'my_var': 5}) + assert result.get('render_elm') is False diff --git a/tests/modular/test_database.py b/tests/modular/test_database.py index ff9d93b..5885754 100644 --- a/tests/modular/test_database.py +++ b/tests/modular/test_database.py @@ -10,9 +10,15 @@ from statik.models import * from statik.database import * +ADDRESS_MODEL = """street: String +postal_code: String +""" + GUEST_MODEL = """first-name: String last-name: String email: String +home_address: Address +business_address: Address """ GUESTHOUSE_MODEL = """guesthouse-name: String @@ -33,9 +39,10 @@ to-date: DateTime """ -MOCK_MODEL_NAMES = ['Guest', 'Guesthouse', 'GuesthouseRoom', 'Booking', 'RoomTag'] +MOCK_MODEL_NAMES = ['Address', 'Guest', 'Guesthouse', 'GuesthouseRoom', 'Booking', 'RoomTag'] MOCK_MODELS = { + 'Address': StatikModel(name='Address', from_string=ADDRESS_MODEL, model_names=MOCK_MODEL_NAMES), 'Guest': StatikModel(name='Guest', from_string=GUEST_MODEL, model_names=MOCK_MODEL_NAMES), 'Guesthouse': StatikModel(name='Guesthouse', from_string=GUESTHOUSE_MODEL, model_names=MOCK_MODEL_NAMES), 'GuesthouseRoom': StatikModel(name='GuesthouseRoom', from_string=GUESTHOUSE_ROOM_MODEL, model_names=MOCK_MODEL_NAMES), @@ -57,12 +64,14 @@ def setUp(self): def test_database(self): data_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data_test_database') db = StatikDatabase(data_path, MOCK_MODELS) + Address = db.tables['Address'] Guest = db.tables['Guest'] Guesthouse = db.tables['Guesthouse'] GuesthouseRoom = db.tables['GuesthouseRoom'] Booking = db.tables['Booking'] RoomTag = db.tables['RoomTag'] + addresses = db.session.query(Address).order_by(Address.street).all() guests = db.session.query(Guest).order_by(Guest.last_name).all() self.assertEqual(2, len(guests)) self.assertInstanceEqual({ @@ -70,12 +79,20 @@ def test_database(self): 'last_name': 'Anderson', 'pk': 'manderson', 'email': 'manderson@somewhere.com', + 'home_address': addresses[0], + 'home_address_id': addresses[0].pk, + 'business_address': addresses[1], + 'business_address_id': addresses[1].pk, }, guests[0]) self.assertInstanceEqual({ 'first_name': 'Gary', 'last_name': 'Merriweather', 'pk': 'gmerriweather', 'email': 'gmerriweather@somewhere.com', + 'home_address': addresses[0], + 'home_address_id': addresses[0].pk, + 'business_address': addresses[1], + 'business_address_id': addresses[1].pk, }, guests[1]) guesthouses = db.session.query(Guesthouse).order_by(Guesthouse.guesthouse_name).all() @@ -121,18 +138,15 @@ def test_database(self): room_tags = db.session.query(RoomTag).order_by(RoomTag.pk).all() self.assertEqual(5, len(room_tags)) - blueroom_tags = set([tag.pk for tag in guesthouse_rooms[0].tags]) + blueroom_tags = list([tag.pk for tag in guesthouse_rooms[0].tags]) self.assertEqual(4, len(blueroom_tags)) - self.assertIn('fireplace', blueroom_tags) - self.assertIn('double-bed', blueroom_tags) - self.assertIn('balcony', blueroom_tags) - self.assertIn('shower', blueroom_tags) + self.assertEqual( + ['fireplace', 'double-bed', 'balcony', 'shower'], blueroom_tags) - redroom_tags = set([tag.pk for tag in guesthouse_rooms[1].tags]) + redroom_tags = list([tag.pk for tag in guesthouse_rooms[1].tags]) self.assertEqual(3, len(redroom_tags)) - self.assertIn('fireplace', redroom_tags) - self.assertIn('single-bed', redroom_tags) - self.assertIn('shower', redroom_tags) + self.assertEqual( + ['fireplace', 'single-bed', 'shower'], redroom_tags) def assertInstanceEqual(self, expected, inst): for field_name, field_value in iteritems(expected):