Preindex hashtag searches

Author: rhettinger

pubsub/pubsub.py

@@ -13,10 +13,12 @@
 from heapq import merge
 from bisect import bisect
 from sys import intern
+import re
 
 User = str
 Timestamp = float
 HashAndSalt = Tuple[bytes, bytes]
+HashTag = str
 
 class Post(NamedTuple):
     timestamp: float
@@ -32,16 +34,21 @@ class UserInfo(NamedTuple):
 
 posts = deque()                     # type: Deque[Post]     # Posts from newest to oldest
 user_posts = defaultdict(deque)     # type: DefaultDict[User, Deque[Post]]
+hashtag_index = defaultdict(deque)  # type: DefaultDict[HashTag, Deque[Post]]
 following = defaultdict(set)        # type: DefaultDict[User, Set[User]]
 followers = defaultdict(set)        # type: DefaultDict[User, Set[User]]
 user_info = dict()                  # type: Dict[User, UserInfo]
 
+hashtag_pattern = re.compile(r'#\w+')
+
 def post_message(user: User, text: str, timestamp: Optional[Timestamp]=None) -> None:
     user = intern(user)
     timestamp = timestamp or time()
     post = Post(timestamp, user, text)
     posts.appendleft(post)
     user_posts[user].appendleft(post)
+    for hashtag in hashtag_pattern.findall(text):
+        hashtag_index[hashtag].appendleft(post)
 
 def follow(user: User, followed_user: User) -> None:
     user, followed_user = intern(user), intern(followed_user)
@@ -62,7 +69,8 @@ def get_followed(user: User) -> List[User]:
     return sorted(following[user])
 
 def search(phrase:str, limit: Optional[int] = None) -> List[Post]:
-    # XXX this could benefit from caching and from preindexing
+    if hashtag_pattern.match(phrase):
+        return list(islice(hashtag_index[phrase], limit))
     return list(islice((post for post in posts if phrase in post.text), limit))
 
 def hash_password(password: str, salt: Optional[bytes] = None) -> HashAndSalt:

pubsub/session.py

@@ -34,7 +34,7 @@
 follow('raymondh', followed_user='barry')
 
 if __name__ == '__main__':
-    from pubsub import posts, followers, following, user_info
+    from pubsub import posts, followers, following, user_info, hashtag_index
     from pubsub import posts_by_user, posts_for_user, search
     from pubsub import get_followers, get_followed, get_user, check_user, age
 
@@ -57,6 +57,8 @@
     pprint(posts_for_user('davin', limit=6))
     print('\nSearch for #python (4)')
     pprint(search('#python', limit=4))
+    print('\nSearch for python (4)')
+    pprint(search('python', limit=4))
     print("\nRaymond's followers")
     pprint(get_followers('raymondh'))
     print("\nDavin follows")
@@ -65,3 +67,5 @@
     print(get_user('barry'))
     print("\nPost ages")
     pprint(list(map(age, posts)))
+    print('\nHash tag index:')
+    pprint(hashtag_index)