Complete rewrite with Polars. No more single loop to fetch all data; posts, queued posts, drafts, and activity are all fetched separately. Extra caution when querying API to prevent HTTP 429

model.py

@@ -0,0 +1,309 @@
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any, Dict, List
+
+import polars as pl
+
+
+class DateAgg(Enum):
+    DAY = 'DAY'
+    MONTH = 'MONTH'
+    QUARTER = 'QUARTER'
+    YEAR = 'YEAR'
+
+
+@dataclass
+class StatsModel:
+    """Class that models the output of the Tumblr stats script."""
+    # The blog in question.
+    blog_name: str
+
+    # Contains posts.
+    posts: pl.DataFrame
+
+    # Contains queued posts.
+    queued_posts: pl.DataFrame
+
+    # Contains drafted posts, both original and not original.
+    drafts: pl.DataFrame
+
+    # Contains activity (notifications).
+    activity: pl.DataFrame 
+
+    # Any tags used.
+    tags: List[str] = field(default_factory=list)
+
+    # Total count of posts processed.
+    total_posts: int = field(init=False)
+
+    # Total original posts (for blog_name) processed.
+    total_original_posts: int = field(init=False)
+
+    # Total original post (for blog_name) notes processed.
+    total_original_post_notes: int = field(init=False)
+
+    # Note stats for original posts by YYYY-MM-DD.
+    original_post_notes_by_date: List[Dict[str, Any]] = field(init=False)
+
+    # Note stats for original posts by month and year.
+    original_post_notes_by_month_and_year:  List[Dict[str, Any]] = field(init=False)
+
+    # Note stats for original posts by quarter and year.
+    original_post_notes_by_qtr_and_year:  List[Dict[str, Any]] = field(init=False)
+
+    # Note stats for original posts by year.
+    original_post_notes_by_year:  List[Dict[str, Any]] = field(init=False)
+
+    # Tags ranked from most popular to least popular by notes.
+    most_popular_tags: List[Dict[str, Any]] = field(init=False)
+
+    # Original posts ranked from most popular to least popular by notes.
+    ranked_original_posts: List[Dict[str, Any]] = field(init=False)
+
+    # Original post note stats by interaction month and year.
+    original_post_notes_by_interaction_date: List[Dict[str, Any]] = field(init=False)
+
+    # Original post note stats by interaction month and year.
+    original_post_notes_by_interaction_month_and_year: List[Dict[str, Any]] = field(init=False)
+
+    # Top blogs by original post note count.
+    top_interactors: List[Dict[str, Any]] = field(init=False)
+
+    # Queued posts (both original and not original), sorted in publish order.
+    ordered_queue: List[Dict[str, Any]] = field(init=False)
+
+    # Top tags in queued posts (both original and not original), sorted by post count.
+    top_queued_tags: List[Dict[str, Any]] = field(init=False)
+
+    # Top tags in drafts (both original and not original), sorted by post count.
+    top_draft_tags: List[Dict[str, Any]] = field(init=False)
+
+    def __post_init__(self):
+        self.total_posts = self.calculate_total_posts()
+        self.total_original_posts = self.calculate_total_original_posts()
+        self.total_original_post_notes = self.calculate_original_post_notes()
+        self.original_post_notes_by_date = self.collect_note_stats_by_original_post_year_month_day()
+        self.original_post_notes_by_month_and_year = self.collect_note_stats_by_original_post_month_and_year()
+        self.original_post_notes_by_qtr_and_year = self.collect_note_stats_by_original_post_qtr_and_year()
+        self.original_post_notes_by_year = self.collect_note_stats_by_original_post_year()
+        self.most_popular_tags = self.determine_most_popular_tags(
+            self.get_original_posts(), "note_count")
+        self.ranked_original_posts = self.rank_original_posts()
+        self.original_post_notes_by_interaction_month_and_year = self.collect_note_stats_by_op_interaction_month_and_year()
+        self.original_post_notes_by_interaction_date = self.collect_note_stats_by_op_interaction_year_month_day()
+        self.top_interactors = self.determine_top_interactors()
+        self.ordered_queue = self.determine_ordered_queue()
+        self.top_queued_tags = self.determine_most_popular_tags(
+            self.queued_posts, 'post_count')
+        self.top_draft_tags = self.determine_most_popular_tags(
+            self.drafts, 'post_count')
+
+    @staticmethod
+    def determine_most_popular_tags(df: pl.DataFrame, sort_key: str) -> List[Dict[str, Any]]:
+        if df.is_empty():
+            return []
+
+        grouped_tags = df.explode(
+            'tags', keep_nulls=False, empty_as_null=False).group_by('tags')
+        result: pl.DataFrame = pl.DataFrame()
+        by_post_count = grouped_tags.agg(pl.count().alias(
+            'post_count')).sort('post_count', descending=True)
+        if sort_key == 'post_count':
+            result = by_post_count.rename({'tags': 'tag'})
+        elif sort_key == 'note_count':
+            result = grouped_tags.agg(pl.sum('note_count'))
+            result = result.join(other=by_post_count, on='tags', how='inner')
+            result = result.select(
+                (pl.col('tags').alias('tag')),
+                (pl.col('note_count')),
+                (pl.col('post_count')),
+                (pl.col('note_count').truediv(pl.col('post_count'))).alias(
+                    "notes_to_posts_ratio")
+            )
+            result = result.sort('note_count', descending=True)
+        else:
+            print(f"WARN: {sort_key} is not a supported sort key!")
+        return result.to_dicts()
+
+    def get_original_posts(self) -> pl.DataFrame:
+        if self.posts.is_empty():
+            return pl.DataFrame()
+        return self.posts.filter(pl.col('parent_post_url').is_null())
+
+    def calculate_total_posts(self) -> int:
+        if self.posts.is_empty():
+            return 0
+        return len(self.posts)
+
+    def calculate_total_original_posts(self) -> int:
+        if self.posts.is_empty():
+            return 0
+        return len(self.get_original_posts())
+
+    def calculate_original_post_notes(self) -> int:
+        if self.posts.is_empty():
+            return 0
+        og_posts = self.get_original_posts()
+        return int(og_posts['note_count'].sum())
+
+    def rank_original_posts(self) -> List[Dict[str, Any]]:
+        og_posts = self.get_original_posts()
+
+        sorted_posts = og_posts.clone()
+        if not sorted_posts.is_empty():
+            sorted_posts = sorted_posts.sort(by='note_count', descending=True)
+        return sorted_posts.to_dicts()
+
+    def determine_top_interactors(self) -> List[Dict[str, Any]]:
+        if self.activity.is_empty(): return []
+        og_activity = self.activity.filter(pl.col('post').struct.field('is_og') == True)
+        interactor_group_by = og_activity.group_by(pl.col('from_tumblelog_name'))
+        agg = interactor_group_by.agg(pl.count().alias('interaction_count')).sort('interaction_count', descending=True)
+        return agg.to_dicts()
+
+    def determine_ordered_queue(self) -> List[Dict[str, Any]]:
+        if self.queued_posts.is_empty():
+            return []
+        sorted_queued_posts = self.queued_posts.sort(
+            by='scheduled_publish_time', descending=False)
+        return sorted_queued_posts.to_dicts()
+
+    def collect_note_stats_by_original_post_date(self, date_agg: DateAgg) -> List[Dict[str, Any]]:
+        og_posts = self.get_original_posts()
+        if og_posts.is_empty(): return []
+
+        sorted_og_posts = og_posts.sort(by='date')
+
+        group_key = self._get_group_key(sorted_og_posts, date_agg)
+        grouped_posts = sorted_og_posts.group_by(group_key)
+        base_stats = grouped_posts.agg(pl.col('id_string').n_unique().alias('post_count'),
+                                pl.sum('note_count').alias('note_count'))
+
+        # For breakdown by tag.
+        exploded_tags = sorted_og_posts.explode(
+            'tags', keep_nulls=False, empty_as_null=False)
+        exploded_tags = exploded_tags.rename({'tags': 'tag'})
+        grouped_tag_explosion = exploded_tags.group_by([group_key, pl.col('tag')])
+        by_tags = grouped_tag_explosion.agg(
+            pl.col('id_string').n_unique().alias('post_count'),
+            pl.sum('note_count'),
+            pl.sum('note_count').truediv(pl.col('id_string').n_unique())
+            .alias("notes_to_posts_ratio")).sort('note_count', descending=True)
+        by_tags = by_tags.group_by(pl.col('group_key')).agg(
+            pl.struct(pl.all()).alias('tags'))
+
+        # For breakdown by post.
+        grouped_by_posts = sorted_og_posts.group_by([group_key,
+                                                        pl.col('post_url').alias('url')])
+        by_posts = grouped_by_posts.agg(pl.sum('note_count')) \
+                                        .sort('note_count', descending=True)
+        by_posts = by_posts.group_by(pl.col('group_key')).agg(pl.struct(pl.all()).alias('posts'))
+
+        post_cols = [(pl.col('note_count')),
+                    (pl.col('post_count')),
+                    (pl.col('note_count').truediv(pl.col('post_count'))).alias(
+                        "notes_to_posts_ratio"),
+                    (pl.col('tags')),
+                    (pl.col('posts'))]
+
+        # Join to get final result.
+        result = base_stats.join(other=by_tags, on='group_key', how='inner')
+        result = result.join(other=by_posts, on='group_key', how='inner')
+        return self._date_agg_final_select(result, post_cols, date_agg).to_dicts()
+
+    def collect_note_stats_by_original_post_year_month_day(self) -> List[Dict[str, Any]]:
+        return self.collect_note_stats_by_original_post_date(DateAgg.DAY)
+
+    def collect_note_stats_by_original_post_month_and_year(self) -> List[Dict[str, Any]]:
+        return self.collect_note_stats_by_original_post_date(DateAgg.MONTH)
+
+    def collect_note_stats_by_original_post_qtr_and_year(self) -> List[Dict[str, Any]]:
+        return self.collect_note_stats_by_original_post_date(DateAgg.QUARTER)
+
+    def collect_note_stats_by_original_post_year(self) -> List[Dict[str, Any]]:
+        return self.collect_note_stats_by_original_post_date(DateAgg.YEAR)
+
+    def collect_note_stats_by_op_interaction_month_and_year(self) -> List[Dict[str, Any]]:
+        return self.collect_note_stats_by_op_interaction_date(DateAgg.MONTH)
+
+    def collect_note_stats_by_op_interaction_year_month_day(self) -> List[Dict[str, Any]]:
+        return self.collect_note_stats_by_op_interaction_date(DateAgg.DAY)
+
+    def collect_note_stats_by_op_interaction_date(self, date_agg: DateAgg) -> List[Dict[str, Any]]:
+        if self.activity.is_empty(): return []
+
+        og_activity = self.activity.filter(pl.col('post').struct.field('is_og') == True)
+        og_activity = og_activity.with_columns(pl.col('post').struct.field('tags').alias('og_post_tags'),
+                                               pl.col('post').struct.field('url').alias('og_post_url'))
+        sorted_og_activity = og_activity.sort(by='date')
+
+        group_key = self._get_group_key(sorted_og_activity, date_agg)
+        grouped_activity = sorted_og_activity.group_by(group_key)
+        base_stats = grouped_activity.agg(pl.col('id').n_unique().alias('interaction_count'))
+        base_stats.write_json('lesbo.json')
+
+        # For breakdown by tag.
+        exploded_tags = sorted_og_activity.explode(columns='og_post_tags', keep_nulls=False, empty_as_null=False)
+        exploded_tags = exploded_tags.rename({'og_post_tags': 'tag'})
+        grouped_tag_explosion = exploded_tags.group_by([group_key, pl.col('tag')])
+        by_tags = grouped_tag_explosion.agg(pl.col('id').n_unique().alias('interaction_count')) \
+                                        .sort('interaction_count', descending=True)
+        by_tags = by_tags.group_by(pl.col('group_key')).agg(pl.struct(pl.all()).alias('tags'))
+
+
+        # For breakdown by post.
+        grouped_by_posts = sorted_og_activity.group_by([group_key,
+                                                        pl.col('og_post_url').alias('url')])
+        by_posts = grouped_by_posts.agg(pl.col('id').n_unique().alias('interaction_count')) \
+                                        .sort('interaction_count', descending=True)
+        by_posts = by_posts.group_by(pl.col('group_key')).agg(pl.struct(pl.all()).alias('posts'))
+
+        actv_columns = [(pl.col('interaction_count')),
+                    (pl.col('tags')),
+                    (pl.col('posts'))]
+
+        # Join to get final result.
+        result = base_stats.join(other=by_tags, on='group_key', how='inner')
+        result = result.join(other=by_posts, on='group_key', how='inner')
+        return self._date_agg_final_select(result, actv_columns, date_agg).to_dicts()
+
+    def _get_group_key(self, df: pl.DataFrame, date_agg: DateAgg):
+        base_col = pl.col('date')
+        if df.get_column('date').dtype == pl.String:
+            base_col = pl.when(base_col.str.contains('GMT')) \
+                .then(base_col.str.to_date('%Y-%m-%d %H:%M:%S GMT')) \
+                .otherwise(base_col.str.to_date('%Y-%m-%d %H:%M:%S'))
+
+        match date_agg:
+            case DateAgg.DAY:
+                group_key = base_col.dt.date().alias('group_key')
+            case DateAgg.MONTH:
+                group_key = base_col.dt.month_start().dt.date().alias('group_key')
+            case DateAgg.QUARTER:
+                group_key = pl.struct(base_col.dt.year().alias('year'),
+                                                                   base_col.dt.quarter().alias('quarter')) \
+                                                                   .alias('group_key')
+            case DateAgg.YEAR:
+                group_key = base_col.dt.year().alias('group_key')
+        return group_key
+
+    def _date_agg_final_select(self, result: pl.DataFrame, cols: List[pl.Expr], date_agg: DateAgg) -> pl.DataFrame:
+        match date_agg:
+            case DateAgg.DAY:
+                final_cols = [(pl.col('group_key').dt.year().alias('year')),
+                                    (pl.col('group_key').dt.month().alias('month')),
+                                    (pl.col('group_key').dt.day().alias('day'))]
+                final_cols.extend(cols)
+                return result.select(final_cols).sort(by=['year', 'month', 'day'], descending=True)
+            case DateAgg.MONTH:
+                final_cols = [(pl.col('group_key').dt.year().alias('year')), (pl.col('group_key').dt.month().alias('month'))]
+                final_cols.extend(cols)
+                return result.select(final_cols).sort(by=['year', 'month'], descending=True)
+            case DateAgg.QUARTER:
+                final_cols = [(pl.col('group_key').alias('quarter'))]
+                final_cols.extend(cols)
+                return result.select(final_cols).sort(by='quarter', descending=True)
+            case DateAgg.YEAR:
+                final_cols = [(pl.col('group_key').alias('year'))]
+                final_cols.extend(cols)
+                return result.select(final_cols).sort(by='year', descending=True)