amber/tumblr-stats
1
0
Fork 0
Code Pull Requests Releases Activity

Complete rewrite with Polars. No more single loop to fetch all data; posts, queued posts, drafts, and activity are all fetched separately. Extra caution when querying API to prevent HTTP 429

Browse Source
This commit is contained in:
Amber
2026-10-21 22:12:31 -04:00
parent ea116ee0a3
commit a23bd35001
6 changed files with 964 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files

287
tumblr_stats.py
View File

@@ -1,46 +1,46 @@
import argparse
import csv
from dataclasses import asdict
from datetime import datetime
import dataclasses
from datetime import datetime as dt
import json
import os
from pathlib import Path
import sys
from typing import Any, Callable, Dict, List, Tuple
from typing import Any, Dict, List
import pytumblr
from build_deep_stats_model import BuildDeepStatsModel
from build_draft_stats_model import BuildDraftStatsModel
from build_tag_stats_model import BuildTagStatsModel
from build_total_stats_model import BuildTotalStatsModel
from build_queue_stats_model import BuildQueueStatsModel
from stats_model import StatsModel
import polars as pl
import fetcher
from model import StatsModel
def get_args() -> Dict[str, Any]:
"""Pull arguments from command line, turn them into a dictionary of <arg, value>"""
parser: argparse.ArgumentParser = argparse.ArgumentParser(
prog='tumblr_stats.py',
description='Use pytumblr to calculate stats after setting these enviroment variables: '
description='Use pytumblr and Polars to calculate stats after setting these enviroment variables: '
+ '$TUMBLR_CONSUMER_KEY, $TUMBLR_CONSUMER_SECRET, $TUMBLR_OAUTH_TOKEN, and $TUMBLR_OAUTH_SECRET',
epilog='— Be gay and do crime')
parser.add_argument('operation', type=str, nargs='+', metavar='OPERATION',
choices=['build_tag_stats', 'build_queue_stats', 'build_draft_stats',
'build_deep_stats'],
help="operation used to calculate stats")
parser.add_argument('-b', '--blog', type=str, required=True,
help='blog name for which to calculate stats')
parser.add_argument('-t', '--tags', type=str, nargs='+',
help='tag(s) to focus on in status (if applicable)')
parser.add_argument('-i', '--input', type=str,
help='Don\'t make API calls, just use a JSON input file')
# TODO: Make 'before' work, but it depends on https://github.com/tumblr/pytumblr/issues/174.
# parser.add_argument('--before', type=lambda s: datetime.strptime(s, '%Y-%m-%d'),
# help='only gather posts before YYYY-MM-DD')
parser.add_argument('--after', type=lambda s: datetime.strptime(s, '%Y-%m-%d'),
help='only gather posts after YYYY-MM-DD')
help='tag(s) on which to filter (only affects posts)')
parser.add_argument('-s', '--suppress', type=str, nargs='+',
choices=['none', 'posts', 'queued', 'drafts', 'activity'],
default='activity',
help='objects to skip when aggregating data; will render some stats empty')
parser.add_argument('--until', type=lambda s: dt.strptime(s, '%Y-%m-%d'),
help='only gather posts and activity until YYYY-MM-DD')
parser.add_argument('--since', type=lambda s: dt.strptime(s, '%Y-%m-%d'),
help='only fetch posts and activity since YYYY-MM-DD')
parser.add_argument('--queued-file', type=str,
help='read queued posts from this path instead of fetching them from APIs')
parser.add_argument('--drafts-file', type=str,
help='read drafts from this path instead of fetching them from APIs')
parser.add_argument('--posts-file', type=str,
help='read posts from this path instead of fetching them from APIs')
parser.add_argument('--activity-file', type=str,
help='read activity from this path instead of fetching it from APIs')
return vars(parser.parse_args())
@@ -68,205 +68,76 @@ def init_client() -> pytumblr.TumblrRestClient:
)
def filter_posts_for_after(post_list: List[Dict[str, Any]],
after: datetime) -> List[Dict[str, Any]]:
# Quick short circuit check.
if not post_list or not after:
return []
def json_encoder(obj):
if dataclasses.is_dataclass(obj):
return dataclasses.asdict(obj) # type: ignore
elif isinstance(obj, pl.DataFrame):
return json.loads(obj.write_json())
raise TypeError(f"Type {type(obj)} is not serializable!")
# Handle 'after'.
after_check: Callable[[Dict[str, Any]], bool] = lambda x: datetime.strptime(
x['date'], '%Y-%m-%d %H:%M:%S %Z') > after
return [post for post in post_list if after_check(post)]
def build_post_maps(client: pytumblr.TumblrRestClient,
args: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, Any]]:
og_post_map: Dict[str, Any] = {}
un_og_post_map: Dict[str, Any] = {}
blog_name: str = args['blog']
def file_is_used(obj_arg: str, args: Dict[str, Any]) -> bool:
return obj_arg in args and args[obj_arg]
# We populate params, starting with any tags for filtering.
params = {}
if args['tags']:
params.update({'tag': ','.join(args['tags'])})
# TODO: Make 'before' work.
# if args['before']:
# before: datetime = args['before']
# params.update({'before': int(before.timestamp())})
draft_url = f"/v2/blog/{blog_name}/posts/draft"
is_draft_stats: bool = 'build_draft_stats' in args['operation']
is_deep_stats: bool = 'build_deep_stats' in args['operation']
total: int = 0
offset: int = 0
limit: int = 20
# The request loop that pulls all data from the APIs.
while True:
# Begin LOOP
# Get me some posts via REST! 😈🍪🍪🍪
data: Dict[str, Any]
if 'build_queue_stats' in args['operation']:
data = client.queue(f"{blog_name}.tumblr.com",
offset=offset,
limit=limit,
**params)
elif is_draft_stats:
data = client.send_api_request("get", draft_url)
else: # Above is for queued + draft posts, below is for published posts.
if is_deep_stats:
params.update({'notes_info': 'true'})
data = client.posts(f"{blog_name}.tumblr.com",
offset=offset,
limit=limit,
**params)
# Stop the presses if we found no posts.
curr_posts: List[Dict[str, Any]] = data['posts']
if not curr_posts or len(curr_posts) < 1:
print('Stopping, as no more posts were found.')
break
next_off: int = 0
if '_links' in data and not is_draft_stats:
links = data['_links']
if 'next' in links and 'query_params' in links['next']:
next_off = int(links['next']['query_params']['offset'])
# Total init check for the first iteration, but always checked for sanity.
if not total and 'total_posts' in data:
total_posts = data['total_posts']
print(f"I'm working with {total_posts} total posts...")
total = total_posts
# Behavior for 'after'.
if args['after']:
after: datetime = args['after']
curr_posts = filter_posts_for_after(curr_posts, after)
if not curr_posts:
print(f"All posts after {after.year}-{after.month}-{after.day} processed.")
return (og_post_map, un_og_post_map)
# This block populates the local post_maps from the raw response data.
local_og_post_map: Dict[str, Any] = {
item['id_string']: item for item in curr_posts if 'parent_post_url' not in item
}
local_un_og_post_map: Dict[str, Any] = {
item['id_string']: item for item in curr_posts if 'parent_post_url' in item
}
# Update the maps with what we found.
og_post_map.update(local_og_post_map)
un_og_post_map.update(local_un_og_post_map)
# For build_draft_stats.
if is_draft_stats:
if '_links' in data:
draft_url = data['_links']['next']['href']
continue
else:
print('All draft posts processed.')
break
# The increment and status printing.
if next_off != 0 and next_off != offset:
offset = next_off
else:
offset += limit
if not args['after'] and total:
print(
f"Processed batch {offset // limit} of {(total // 20) + 1}...")
# End LOOP
# Return (og_posts, not_og_posts).
return (og_post_map, un_og_post_map)
def load_from_file(path: str) -> pl.DataFrame:
return pl.read_json(source=Path(path), infer_schema_length=None)
def main() -> None:
args: Dict[str, Any] = get_args()
suppress = args['suppress']
client: pytumblr.TumblrRestClient = init_client()
operation: List[str] = args['operation']
# Quick bail for bad use of build_queue_stats and build_draft_stats.
if (set(operation) & set(['build_queue_stats', 'build_draft_stats'])) and len(operation) > 1:
print(f"You can't mix operations {operation} together. Sorry.")
sys.exit(1)
pass
# Read queued posts from a file, or fetch them via API and write queued post JSON.
queued_posts: pl.DataFrame = pl.DataFrame()
if not suppress or 'queued' not in suppress:
if file_is_used('queued_file', args):
queued_posts = load_from_file(args['queued_file'])
else:
queued_posts = fetcher.fetch_queued_posts(client, args)
queued_posts.write_json("queued_posts.json")
# Handle JSON input (if you don't want to make API calls.)
if 'input' in args and args['input']:
input_path = Path(args['input'])
with open(input_path, "r") as f:
data = json.load(f)
og_post_map = data['original_post_map']
un_og_post_map = data['unoriginal_post_map']
for post_key in og_post_map.copy():
post = og_post_map[post_key]
date: datetime = datetime.strptime(
post['date'], '%Y-%m-%d %H:%M:%S %Z')
if date.year != 2025:
del og_post_map[post_key]
for post_key in un_og_post_map.copy():
post = un_og_post_map[post_key]
date: datetime = datetime.strptime(
post['date'], '%Y-%m-%d %H:%M:%S %Z')
if date.year != 2025:
del un_og_post_map[post_key]
else:
# Get the og_post_map (original posts) and un_og_post_map (not original posts).
og_post_map, un_og_post_map = build_post_maps(args=args, client=client)
# Read drafts from a file, or fetch them via API and write draft JSON.
drafts: pl.DataFrame = pl.DataFrame()
if not suppress or 'drafts' not in suppress:
if file_is_used('drafts_file', args):
drafts = load_from_file(args['drafts_file'])
else:
drafts = fetcher.fetch_drafts(client, args)
drafts.write_json("drafts.json")
# Pick a stats model, which will determine output.
stats_model: StatsModel
match args:
case {'operation': ['build_queue_stats']}:
stats_model = BuildQueueStatsModel(blog_name=args['blog'],
original_post_map=og_post_map,
unoriginal_post_map=un_og_post_map)
case {'operation': ['build_draft_stats']}:
stats_model = BuildDraftStatsModel(blog_name=args['blog'],
original_post_map=og_post_map,
unoriginal_post_map=un_og_post_map)
case {'operation': op} if 'build_tag_stats' in operation:
stats_model = BuildTagStatsModel(blog_name=args['blog'],
original_post_map=og_post_map,
unoriginal_post_map=un_og_post_map)
stats_model.tags = args['tags']
case {'operation': op} if 'build_total_stats' in operation:
if 'before' not in args: # or 'after' not in args:
print(f"You must specify a time range for {op}. " +
'You\'ll otherwise request TOO MUCH DATA!')
sys.exit(1)
stats_model = BuildTotalStatsModel(blog_name=args['blog'],
original_post_map=og_post_map,
unoriginal_post_map=un_og_post_map)
case {'operation': op} if 'build_deep_stats' in operation:
if 'after' not in args: # or 'before' not in args:
print(f"You must specify a time range for {op}. " +
'You\'ll otherwise request TOO MUCH DATA!')
sys.exit(1)
stats_model = BuildDeepStatsModel(blog_name=args['blog'],
original_post_map=og_post_map,
unoriginal_post_map=un_og_post_map)
case _:
print('Unsupported command. How did you even make it this far?!')
sys.exit(1)
# Read posts from a file, or fetch them via API and write post JSON.
posts: pl.DataFrame = pl.DataFrame()
if not suppress or 'posts' not in suppress:
if file_is_used('posts_file', args):
posts = load_from_file(args['posts_file'])
else:
posts = fetcher.fetch_posts(client, args)
posts.write_json("posts.json")
# Write the selected model as JSON output.
# Read activity from a file, or fetch them via API and write activity JSON.
activity: pl.DataFrame = pl.DataFrame(infer_schema_length=None)
if not suppress or 'activity' not in suppress:
if file_is_used('activity_file', args):
activity = load_from_file(args['activity_file'])
else:
activity = fetcher.fetch_activity(client, args, posts)
activity.write_json('activity.json')
# The model.
model: StatsModel = StatsModel(blog_name=args['blog'],
posts = posts,
queued_posts = queued_posts,
drafts = drafts,
activity = activity,
tags = args['tags'])
# Write the model as JSON output.
# https://stackoverflow.com/q/26244323
with open('./tumblr_stats.json', 'w') as f:
json.dump(asdict(stats_model), f, indent=1, default=str)
json.dump(asdict(model), f, indent=1, default=json_encoder)
# If there were original posts, create a CSV for them.
if og_post_map:
with open('./tumblr_original_posts.csv', 'w', newline='') as f:
post_list: List[Dict[str, Any]] = list(og_post_map.values())
wr = csv.DictWriter(f, quoting=csv.QUOTE_ALL, extrasaction='ignore',
fieldnames=post_list[0].keys())
wr.writeheader()
wr.writerows(post_list)
else:
print('No original posts were found, so a CSV of original posts was not written.')
return