FlixRec/main.py

import requests
import os
from database import *
from tqdm import tqdm

from datetime import datetime

import time

trakt_id = os.getenv("TRAKT_ID")
trakt_se = os.getenv("TRAKT_SE")

max_requests = 5000  # How many requests do you want to make
req_count = 0

years = "1900-2021"
page = 1
extended = "full"  # Required to get additional information
limit = "10"  # No of entires per request
languages = "en"  # Limit to particular language

api_base = "https://api.trakt.tv"
database_url = "sqlite:///jlm.db"

headers = {
    "Content-Type": "application/json",
    "trakt-api-version": "2",
    "trakt-api-key": trakt_id,
}

params = {
    "query": "",
    "years": years,
    "page": page,
    "extended": extended,
    "limit": limit,
    "languages": languages,
}


def create_movie_dict(movie: dict):
    m = movie["movie"]
    movie_dict = {
        "title": m["title"],
        "overview": m["overview"],
        "genres": m["genres"],
        "language": m["language"],
        "year": int(m["year"]),
        "trakt_id": m["ids"]["trakt"],
        "released": m["released"],
        "runtime": int(m["runtime"]),
        "country": m["country"],
        "rating": int(m["rating"]),
        "votes": int(m["votes"]),
        "comment_count": int(m["comment_count"]),
        "tagline": m["tagline"],
    }
    return movie_dict


params["limit"] = 1
res = requests.get(f"{api_base}/search/movie", headers=headers, params=params)
total_items = res.headers["x-pagination-item-count"]

print(f"There are {total_items} movies")
print(f"Started from page {page}")

"""
movies = []
params["limit"] = limit
res = requests.get(f"{api_base}/search/movie",headers=headers,params=params)

if res.status_code == 200:
	for movie in res.json():
		movies.append(create_movie_dict(movie))
		print(create_movie_dict(movie)["title"])
"""
engine, Session = init_db_stuff(database_url)

start_time = datetime.now()

for page in tqdm(range(1, max_requests + 10)):
    if req_count == 999:
        seconds_to_sleep = 300 - (datetime.now() - start_time).seconds
        if seconds_to_sleep < 1:
            seconds_to_sleep = 60
        print(f"Sleeping {seconds_to_sleep}s")
        # Need to respect their rate limitting
        # Better to use x-ratelimit header
        time.sleep(seconds_to_sleep)
        start_time = datetime.now()
        req_count = 0

    params["page"] = page
    params["limit"] = int(int(total_items) / max_requests)
    movies = []
    res = requests.get(f"{api_base}/search/movie", headers=headers, params=params)

    if res.status_code == 500:
        break
    elif res.status_code == 200:
        None
    else:
        print(f"OwO Code {res.status_code}")

    for movie in res.json():
        movies.append(create_movie_dict(movie))

    with engine.connect() as conn:
        for movie in movies:
            with conn.begin() as trans:
                stmt = insert(movies_table).values(
                    trakt_id=movie["trakt_id"],
                    title=movie["title"],
                    genres=" ".join(movie["genres"]),
                    language=movie["language"],
                    year=movie["year"],
                    released=movie["released"],
                    runtime=movie["runtime"],
                    country=movie["country"],
                    overview=movie["overview"],
                    rating=movie["rating"],
                    votes=movie["votes"],
                    comment_count=movie["comment_count"],
                    tagline=movie["tagline"],
                )
                try:
                    result = conn.execute(stmt)
                    trans.commit()
                except IntegrityError:
                    trans.rollback()
    req_count += 1