#!/usr/bin/env python3 """Douyin posts into folders: text, author, counts and the original video. Run `python3 douyin.py ... --out `. Standard library only; needs share_common.py beside it and MY_API_KEY in the environment. TikHub get_aweme_id (skipped when the link already has /video/) and fetch_one_video_v3 for the metadata, 0.001 USD each; xiazaitool for the original video (0.005 CNY). If xiazaitool cannot parse it, the video comes from the largest of TikHub's play URLs, which change from call to call. Image posts save the unwatermarked JPEG from url_list. Folder layout and flags: see share_common.py. A profile link (douyin.com/user/, or a v.douyin.com short link landing there) pages through fetch_user_post_videos, 20 posts and 0.001 USD a page; each entry already carries what fetch_one_video_v3 gives, so the posts cost nothing more. """ from __future__ import annotations import re import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) from share_common import Profile, cli, first_url, host_in, https, iso, last_segment # noqa: E402 DOMAINS = ("douyin.com", "iesdouyin.com") USER_POSTS = "/api/v1/douyin/app/v3/fetch_user_post_videos" def aweme_id(url: str) -> str: match = re.search(r"/(?:video|note)/(\d{15,})|modal_id=(\d{15,})", url) return next((g for g in match.groups() if g), "") if match else "" def fetch(post, args) -> None: aid = aweme_id(post.url) or str(post.tikhub("/api/v1/douyin/web/get_aweme_id", {"url": post.url}) or "") post.meta["id"] = aid fill(post, post.tikhub("/api/v1/douyin/app/v3/fetch_one_video_v3", {"aweme_id": aid})["aweme_detail"], args) def fill(post, detail: dict, args) -> None: author = detail.get("author") or {} stats = detail.get("statistics") or {} desc = detail.get("desc") or "" images = detail.get("images") or [] video = detail.get("video") or {} duration = round((detail.get("duration") or video.get("duration") or 0) / 1000, 3) or None post.meta.update( id=detail.get("aweme_id") or post.meta["id"], kind="image" if images else "video", title=(detail.get("item_title") or detail.get("preview_title") or desc.split("\n")[0])[:80], desc=desc, author={"name": author.get("nickname"), "id": author.get("unique_id") or author.get("short_id"), "sec_uid": author.get("sec_uid")}, stats={ "like": stats.get("digg_count"), "collect": stats.get("collect_count"), "comment": stats.get("comment_count"), "share": stats.get("share_count"), }, published=iso(detail.get("create_time")), duration=duration, topics=[t["hashtag_name"] for t in detail.get("text_extra") or [] if t.get("hashtag_name")], ip_location=detail.get("ip_attribution"), ) if images: # url_list leads with HEIC, which browsers can't show; the same-size JPEG comes later in the list for i, image in enumerate(images, 1): urls = image.get("url_list") or [] if url := next((u for u in urls if ".heic" not in u.split("?")[0]), first_url(image)): post.meta["files"]["images"].append(post.download(https(url), f"images/{i:02d}.jpg")) return if cover := first_url(video.get("origin_cover")) or first_url(video.get("cover")): post.meta["files"]["cover"] = post.download(https(cover), "cover.jpg") part = {"page": 1, "title": post.meta["title"], "duration": duration, "video": None} post.meta["parts"] = [part] if args.no_video or post.video_via_xiazaitool(part): return offers = [video.get(k) or {} for k in ("play_addr", "play_addr_h264", "play_addr_265")] offers += [b.get("play_addr") or {} for b in video.get("bit_rate") or []] offers = [o for o in offers if first_url(o)] if offers: best = max(offers, key=lambda o: (int(o.get("width") or 0) * int(o.get("height") or 0), int(o.get("data_size") or 0))) post.warn(f"P1: video came from TikHub's play URL ({best.get('width')}x{best.get('height')}), not xiazaitool's original") try: part["video"] = post.download(https(first_url(best)), "video/p01.mp4") except Exception as exc: # noqa: BLE001 post.warn(f"P1: video download failed ({type(exc).__name__}: {exc})") class DouyinProfile(Profile): short = ("v.douyin.com",) def user_id(self, url: str) -> str: if aweme_id(url): # a post opened on someone's page: /user/?modal_id= return "" match = re.search(r"/user/([\w-]{20,})", url) return match.group(1) if match else "" def page(self, lister, uid, cursor, n): data = lister.tikhub(USER_POSTS, {"sec_user_id": uid, "max_cursor": cursor or 0, "count": 20}, raw_name=f"tikhub_fetch_user_post_videos_p{n:02d}") or {} return data.get("aweme_list") or [], data.get("max_cursor"), bool(data.get("has_more")) def item(self, raw: dict) -> dict: stats = raw.get("statistics") or {} kind = "image" if raw.get("images") else "video" return { "id": str(raw.get("aweme_id")), "url": f"https://www.douyin.com/{'note' if kind == 'image' else 'video'}/{raw.get('aweme_id')}", "title": (raw.get("item_title") or (raw.get("desc") or "").split("\n")[0])[:80], "published": iso(raw.get("create_time")), "pinned": bool(raw.get("is_top")), "kind": kind, "stats": {"like": stats.get("digg_count"), "collect": stats.get("collect_count"), "comment": stats.get("comment_count"), "share": stats.get("share_count")}, } def author(self, raw: dict) -> dict: author = raw.get("author") or {} return {"name": author.get("nickname"), "id": author.get("unique_id") or author.get("short_id"), "sec_uid": author.get("sec_uid")} def fetch(self, post, raw, args) -> None: if raw is None: fetch(post, args) return post.save_raw("tikhub_fetch_user_post_videos", raw) fill(post, raw, args) if __name__ == "__main__": raise SystemExit(cli(__doc__, "douyin", lambda u: host_in(u, DOMAINS), lambda u: aweme_id(u) or last_segment(u), fetch, profile=DouyinProfile()))