amber/tumblr-stats
1
0
Fork 0
Code Pull Requests Releases Activity

Added a popular tags feature, greatly cleaned up code, commented spots

Browse Source
This commit is contained in:
Amber McCloughan
2025-12-30 18:40:24 -05:00
parent 4fbb99a3f6
commit 9a8144af68
4 changed files with 83 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
tumblr_stats.py
View File

@@ -2,7 +2,6 @@
import argparse
import csv
from dataclasses import asdict
from datetime import datetime
import json
import os
import sys
@@ -22,14 +21,17 @@ def get_args() -> Dict[str, Any]:
description='Use pytumblr to calculate stats after setting these enviroment variables: '
+ '$TUMBLR_CONSUMER_KEY, $TUMBLR_CONSUMER_SECRET, $TUMBLR_OAUTH_TOKEN, and $TUMBLR_OAUTH_SECRET',
epilog='— Be gay and do crime')
parser.add_argument('operation', type=str, metavar='OPERATION', choices=['build_tag_stats'],
parser.add_argument('operation', type=str, nargs = '+',
metavar='OPERATION', choices=['build_tag_stats'],
help="operation used to calculate stats")
parser.add_argument('-b', '--blog', type=str, required=True,
help='blog name for which to calculate stats')
parser.add_argument('-t', '--tags', type=str, nargs='+',
help='tag(s) to focus on in status (if applicable)')
# TODO: Make 'before' work, but it actually depends on https://github.com/tumblr/pytumblr/issues/174.
# parser.add_argument('--before', type=lambda s: datetime.strptime(s, '%Y-%m-%d'),
# help='only gather posts before YYYY-MM-DD')
# TODO: Make 'after' work if they add it to pytumblr.
# parser.add_argument('--after', type=lambda s: datetime.strptime(s, '%Y-%m-%d'),
# help='only gather posts after YYYY-MM-DD')
return vars(parser.parse_args())
@@ -64,12 +66,15 @@ def build_post_map_and_dumpster(client: pytumblr.TumblrRestClient, args: Dict[st
dumpster: Dict[str, Any] = {}
blog_name = args['blog']
# We populate params, starting with any tags for filtering.
params = {}
if args['tags']:
params.update({'tag': ','.join(args['tags'])})
# TODO: Make 'before' work.
# if args['before']:
# before: datetime = args['before']
# params.update({'before': int(before.timestamp())})
# TODO: Make 'after' work.
# if args['after']:
# after: datetime = args['after']
# params.update({'after': str(int(after.timestamp()))})
@@ -78,20 +83,21 @@ def build_post_map_and_dumpster(client: pytumblr.TumblrRestClient, args: Dict[st
offset = 0
limit = 20
# The request loop that pulls all data from the APIs.
while offset <= total:
# Begin LOOP
# Get me some posts! 😈🍪🍪🍪
# Get me some posts via REST! 😈🍪🍪🍪
data = client.posts(f"{blog_name}.tumblr.com",
offset=offset,
limit=limit,
**params)
# Sh**t it in the head if we found no posts.
# Stop the presses if we found no posts.
if not data['posts']:
print('Stopping, as no posts were found.')
break
# Total check for the first good iteration, but always checked for sanity.
# Total init check for the first iteration, but always checked for sanity.
if total == 0:
# Let's see what's in there,
total_posts = data['total_posts']
@@ -100,6 +106,7 @@ def build_post_map_and_dumpster(client: pytumblr.TumblrRestClient, args: Dict[st
print(f"I'm working with {total_posts} total posts...")
total = total_posts
# This block populates the local post_map from the raw response data.
curr_posts = data['posts']
local_post_map: Dict[str, Any] = {}
for curr_post in curr_posts:
@@ -107,47 +114,50 @@ def build_post_map_and_dumpster(client: pytumblr.TumblrRestClient, args: Dict[st
if curr_key not in local_post_map:
local_post_map[curr_key] = curr_post
# This block populates the local dumpster from the raw response data.
local_dumpster = {}
filtered_local_post_map = {}
for local_key in local_post_map:
local_post = local_post_map[local_key]
# Determines whether this is an OG post.
if 'parent_post_url' not in local_post:
filtered_local_post_map[local_key] = local_post
else:
else: # If it's not an OG post, into the local dumpster.
local_dumpster[local_key] = local_post
# The sacred should we add, and if we should, DO ADD, if statement.
has_og_posts = any(post not in post_map for post in filtered_local_post_map)
# The sacred "should we add, and if we should, DO ADD" conditional statements.
has_og_posts = any(
post not in post_map for post in filtered_local_post_map)
has_not_og_posts = any(post not in dumpster for post in local_dumpster)
if has_og_posts:
post_map.update(filtered_local_post_map)
if has_not_og_posts:
dumpster.update(local_dumpster)
# The increment and status printing. Should always end the loop!
offset += limit
if offset == limit:
print('Processed first batch...')
elif offset < total:
print(f"Processed batch {offset // limit} of {total // 20}...")
else:
print(f"Processed all {total} posts")
print(f"Processed batch {offset // limit} of {(total // 20) + 1}...")
# End LOOP
# Return (og_posts, not_og_posts).
return (post_map, dumpster)
def build_tag_stats_model(client: pytumblr.TumblrRestClient, args: Dict[str, Any]) -> BuildTagStatsModel:
post_map, dumpster = build_post_map_and_dumpster(client, args)
stats_model: BuildTagStatsModel = BuildTagStatsModel(blog_name=args['blog'], original_post_map=post_map,
def build_tag_stats_model(post_map: Dict[str, Any],
dumpster: Dict[str, Any],
args: Dict[str, Any]) -> BuildTagStatsModel:
stats_model: BuildTagStatsModel = BuildTagStatsModel(blog_name=args['blog'],
original_post_map=post_map,
unoriginal_post_map=dumpster)
stats_model.tags = args['tags']
return stats_model
def build_total_stats_model(client: pytumblr.TumblrRestClient, args: Dict[str, Any]) -> BuildTotalStatsModel:
post_map, dumpster = build_post_map_and_dumpster(client, args)
stats_model: BuildTotalStatsModel = BuildTotalStatsModel(blog_name=args['blog'], original_post_map=post_map,
def build_total_stats_model(post_map: Dict[str, Any],
dumpster: Dict[str, Any],
args: Dict[str, Any]) -> BuildTotalStatsModel:
stats_model: BuildTotalStatsModel = BuildTotalStatsModel(blog_name=args['blog'],
original_post_map=post_map,
unoriginal_post_map=dumpster)
return stats_model
@@ -156,32 +166,39 @@ def main() -> None:
args = get_args()
client = init_client()
stats_model = StatsModel(blog_name=args['blog'], operation='undefined',
original_post_map={}, unoriginal_post_map={})
# Get the post_map (original posts) and dumpster (not original posts).
post_map, dumpster = build_post_map_and_dumpster(args=args, client=client)
if args['operation'] == 'build_tag_stats':
stats_model = build_tag_stats_model(client, args)
elif args['operation'] == 'build_total_stats':
# Pick a stats model, which will determine output.
stats_model: StatsModel
if 'build_tag_stats' in args['operation']:
stats_model = build_tag_stats_model(post_map, dumpster, args)
if 'build_total_stats' in args['operation']:
if 'before' not in args: # or 'after' not in args:
print('You must specify a time range for build_total stats. ' +
'You\'ll otherwise request TOO MUCH DATA!')
sys.exit()
stats_model = build_total_stats_model(client, args)
stats_model = build_total_stats_model(post_map, dumpster, args)
# Write the chosen model as JSON output.
with open('./tumblr_stats.json', 'w') as f:
json.dump(asdict(stats_model), f, indent=2, sort_keys=True)
if stats_model.original_post_map:
json.dump(asdict(stats_model), f, indent=1)
# If there were original posts, create a CSV for them.
if post_map:
with open('./tumblr_original_posts.csv', 'w', newline='') as f:
post_list: List[Dict[str, Any]] = list(
stats_model.original_post_map.values())
post_list: List[Dict[str, Any]] = list(post_map.values())
wr = csv.DictWriter(f, quoting=csv.QUOTE_ALL, extrasaction='ignore',
fieldnames=post_list[0].keys())
wr.writeheader()
wr.writerows(post_list)
else:
print('No original posts were found, so no CSV of original posts was written.')
print('No original posts were found, so a CSV of original posts was not written.')
return
# DO NOT DELETE. The main if statement.
if __name__ == '__main__':
main()
print('All done.')
sys.exit(0)