fix parent bug in posts, only call tags once in posts. use Undefined …

…for cache misses in KVS

k8s.yml

@@ -14,7 +14,7 @@ spec:
     spec:
       containers:
       - name: fuzzly-backend
-        image: us-central1-docker.pkg.dev/kheinacom/fuzzly-repo/fuzzly-backend@sha256:a55b5e81eb296e501a3869dbc30971bc1d87686d3ac716e0b3c569829ba5c17c
+        image: us-central1-docker.pkg.dev/kheinacom/fuzzly-repo/fuzzly-backend@sha256:923f73cf75de6108b2cc05f730abd8657dd0d9fa4b292c90757d7008c9e2dce9
         env:
         - name: pod_ip
           valueFrom:

posts/repository.py

@@ -11,10 +11,11 @@
 from shared.datetime import datetime
 from shared.exceptions.http_error import BadRequest, NotFound
 from shared.maps import privacy_map
-from shared.models import InternalUser, UserPortable
+from shared.models import InternalUser, Undefined, UserPortable
 from shared.sql import SqlInterface
 from shared.sql.query import CTE, Field, Join, JoinType, Operator, Query, Table, Value, Where
 from shared.timing import timed
+from shared.utilities import flatten
 from tags.models import InternalTag, Tag, TagGroup
 from tags.repository import TagKVS, Tags
 from users.repository import Users
@@ -485,7 +486,7 @@ async def scores_many(self: Self, post_ids: list[PostId]) -> dict[PostId, Option
 			return { }
 
 		cached = await ScoreKVS.get_many_async(post_ids)
-		misses = [PostId(k) for k, v in cached.items() if v is None]
+		misses = [PostId(k) for k, v in cached.items() if v is Undefined]
 
 		if not misses :
 			return cached
@@ -551,7 +552,7 @@ async def votes_many(self: Self, user_id: int, post_ids: list[PostId]) -> dict[P
 			PostId(k[k.rfind('|') + 1:]): v
 			for k, v in (await VoteKVS.get_many_async([f'{user_id}|{post_id}' for post_id in post_ids])).items()
 		}
-		misses = [k for k, v in cached.items() if v is None]
+		misses = [k for k, v in cached.items() if v is Undefined]
 
 		if not misses :
 			return cached
@@ -819,7 +820,7 @@ async def _tags_many(self: Self, post_ids: list[PostId]) -> dict[PostId, list[In
 		misses: list[PostId] = []
 
 		for k, v in list(cached.items()) :
-			if v is not None :
+			if v is not Undefined :
 				continue
 
 			misses.append(k)
@@ -876,8 +877,13 @@ async def posts(self: Self, user: KhUser, iposts: list[InternalPost], assign_par
 		scores_task:    Task[dict[PostId, Optional[Score]]] = ensure_future(self._scores(user, iposts))
 
 		tags:      dict[PostId, list[InternalTag]] = await self._tags_many(list(map(lambda x : PostId(x.post_id), iposts)))
+		at_task:   Task[list[Tag]]                 = ensure_future(tagger.tags(user, [t for l in tags.values() for t in l]))
 		uploaders: dict[int, UserCombined]         = await uploaders_task
 		scores:    dict[PostId, Optional[Score]]   = await scores_task
+		all_tags:  dict[str, Tag]                  = {
+			tag.tag: tag
+			for tag in await at_task
+		}
 
 		# mapping of post_id -> parent post_id
 		parents:   dict[PostId, PostId] = { }
@@ -888,7 +894,6 @@ async def posts(self: Self, user: KhUser, iposts: list[InternalPost], assign_par
 			post_id:   PostId           = PostId(ipost.post_id)
 			parent_id: Optional[PostId] = None
 
-			t: Task[list[Tag]] = ensure_future(tagger.tags(user, tags[post_id]))
 			if ipost.parent :
 				parent_id = parents[post_id] = PostId(ipost.parent)
 
@@ -941,7 +946,7 @@ async def posts(self: Self, user: KhUser, iposts: list[InternalPost], assign_par
 
 				# only the first call retrieves blocked info, all the rest should be cached and not actually await
 				blocked = await is_post_blocked(user, uploaders[ipost.user_id].internal, [t.name for t in tags[post_id]]),
-				tags    = tagger.groups(await t)
+				tags    = tagger.groups([all_tags[t.name] for t in tags[post_id]])
 			)
 
 			if not assign_parents :
@@ -953,6 +958,9 @@ async def posts(self: Self, user: KhUser, iposts: list[InternalPost], assign_par
 
 		if assign_parents :
 			for post_id, parent in parents.items() :
+				if parent not in all_posts :
+					continue
+
 				all_posts[post_id].parent = all_posts[parent]
 
 		else :

posts/uploader.py

@@ -1,7 +1,7 @@
+import json
 from asyncio import Task, ensure_future
 from enum import Enum
 from io import BytesIO
-import json
 from os import path, remove
 from secrets import token_bytes
 from subprocess import PIPE, Popen
@@ -12,9 +12,9 @@
 import aerospike
 from aiohttp import ClientResponseError
 from exiftool import ExifToolAlpha as ExifTool
+from ffmpeg.asyncio import FFmpeg
 from wand import resource
 from wand.image import Image
-from ffmpeg.asyncio import FFmpeg
 
 from shared.auth import KhUser, Scope
 from shared.backblaze import B2Interface, MimeType

shared/caching/key_value_store.py

@@ -10,6 +10,7 @@
 
 from ..config.constants import environment
 from ..config.credentials import fetch
+from ..models import Undefined
 from ..timing import timed
 from ..utilities import __clear_cache__, coerse
 
@@ -109,7 +110,7 @@ def _get_many[T: KeyType](self: 'KeyValueStore', k: Iterable[T]) -> dict[T, Any]
 					self._cache[key] = (exp, value)
 
 				else :
-					data_map[key] = None
+					data_map[key] = Undefined
 
 			return {
 				**data_map,

shared/models/_shared.py

@@ -20,7 +20,6 @@
 """
 
 
-'''
 # insane shit
 class __undefined__(type) :
 	def __bool__(cls) :
@@ -31,6 +30,7 @@ class Undefined(metaclass=__undefined__) :
 	pass
 
 
+'''
 class BaseModel(PBM) :
 	"""
 	excludes any value set to Undefined from BaseModel.dict, as well as on encoded responses via fastapi

tags/repository.py

@@ -8,7 +8,7 @@
 from shared.auth import KhUser
 from shared.caching import AerospikeCache
 from shared.caching.key_value_store import KeyValueStore
-from shared.models import InternalUser
+from shared.models import InternalUser, Undefined
 from shared.sql import SqlInterface
 from shared.timing import timed
 from shared.utilities import flatten
@@ -139,7 +139,7 @@ async def _get_tag_counts(self, tags: Iterable[str]) -> dict[str, int] :
 
 		counts = await CountKVS.get_many_async(tags)
 		for k, v in counts.items() :
-			if v is None :
+			if v is Undefined :
 				counts[k] = await self._populate_tag_cache(k)
 
 		return counts

users/repository.py

@@ -9,7 +9,7 @@
 from shared.caching.key_value_store import KeyValueStore
 from shared.exceptions.http_error import BadRequest, NotFound
 from shared.maps import privacy_map
-from shared.models import Badge, InternalUser, Privacy, User, UserPortable, UserPrivacy, Verified
+from shared.models import Badge, InternalUser, Privacy, Undefined, User, UserPortable, UserPrivacy, Verified
 from shared.sql import SqlInterface
 from shared.timing import timed
 
@@ -200,7 +200,7 @@ async def _get_users(self, user_ids: Iterable[int]) -> dict[int, InternalUser] :
 			return { }
 
 		cached = await UserKVS.get_many_async(user_ids)
-		misses = [k for k, v in cached.items() if v is None]
+		misses = [k for k, v in cached.items() if v is Undefined]
 
 		if not misses :
 			return cached
@@ -330,7 +330,7 @@ async def following_many(self: Self, user_id: int, targets: list[int]) -> dict[i
 			int(k[k.rfind('|') + 1:]): v
 			for k, v in (await FollowKVS.get_many_async([f'{user_id}|{t}' for t in targets])).items()
 		}
-		misses = [k for k, v in cached.items() if v is None]
+		misses = [k for k, v in cached.items() if v is Undefined]
 
 		if not misses :
 			return cached