cleaned with black

This commit is contained in:
navanchauhan 2022-05-22 11:42:00 -06:00
parent 801289d123
commit 404c6fb1a3
5 changed files with 158 additions and 125 deletions

44
app.py
View File

@ -19,8 +19,9 @@ index = pinecone.Index("movies")
app = Flask(__name__, template_folder="./templates") app = Flask(__name__, template_folder="./templates")
def title2trakt_id(title: str, df=df): def title2trakt_id(title: str, df=df):
#Matches Exact Title, Otherwise Returns None # Matches Exact Title, Otherwise Returns None
records = df[df["title"].str.lower() == title.lower()] records = df[df["title"].str.lower() == title.lower()]
if len(records) == 0: if len(records) == 0:
return 0, None return 0, None
@ -29,38 +30,48 @@ def title2trakt_id(title: str, df=df):
else: else:
return 2, records.trakt_id.tolist() return 2, records.trakt_id.tolist()
def get_vector_value(trakt_id: int): def get_vector_value(trakt_id: int):
fetch_response = index.fetch(ids=[str(trakt_id)]) fetch_response = index.fetch(ids=[str(trakt_id)])
return fetch_response["vectors"][str(trakt_id)]["values"] return fetch_response["vectors"][str(trakt_id)]["values"]
def query_vectors(vector: list, top_k: int = 20, include_values: bool = False, include_metada: bool = True):
def query_vectors(
vector: list,
top_k: int = 20,
include_values: bool = False,
include_metada: bool = True,
):
query_response = index.query( query_response = index.query(
queries=[ queries=[
(vector), (vector),
], ],
top_k=top_k, top_k=top_k,
include_values=include_values, include_values=include_values,
include_metadata=include_metada include_metadata=include_metada,
) )
return query_response return query_response
def query2ids(query_response): def query2ids(query_response):
trakt_ids = [] trakt_ids = []
for match in query_response["results"][0]["matches"]: for match in query_response["results"][0]["matches"]:
trakt_ids.append(int(match["id"])) trakt_ids.append(int(match["id"]))
return trakt_ids return trakt_ids
def get_deets_by_trakt_id(df, trakt_id: int): def get_deets_by_trakt_id(df, trakt_id: int):
df = df[df["trakt_id"]==trakt_id] df = df[df["trakt_id"] == trakt_id]
return { return {
"title": df.title.values[0], "title": df.title.values[0],
"overview": df.overview.values[0], "overview": df.overview.values[0],
"runtime": int(df.runtime.values[0]), "runtime": int(df.runtime.values[0]),
"year": int(df.year.values[0]), "year": int(df.year.values[0]),
"trakt_id": trakt_id, "trakt_id": trakt_id,
"tagline": df.tagline.values[0] "tagline": df.tagline.values[0],
} }
@app.route("/similar") @app.route("/similar")
def get_similar_titles(): def get_similar_titles():
trakt_id = request.args.get("trakt_id") trakt_id = request.args.get("trakt_id")
@ -99,10 +110,10 @@ def get_similar_titles():
except TypeError: except TypeError:
maxRuntime = 220 maxRuntime = 220
vector = get_vector_value(trakt_id) vector = get_vector_value(trakt_id)
movie_queries = query_vectors(vector, top_k = 69) movie_queries = query_vectors(vector, top_k=69)
movie_ids = query2ids(movie_queries) movie_ids = query2ids(movie_queries)
results = [] results = []
#for trakt_id in movie_ids: # for trakt_id in movie_ids:
# deets = get_deets_by_trakt_id(df, trakt_id) # deets = get_deets_by_trakt_id(df, trakt_id)
# results.append(deets) # results.append(deets)
max_res = 30 max_res = 30
@ -111,12 +122,15 @@ def get_similar_titles():
if cur_res >= max_res: if cur_res >= max_res:
break break
deets = get_deets_by_trakt_id(df, trakt_id) deets = get_deets_by_trakt_id(df, trakt_id)
if ((deets["year"]>=min_year) and (deets["year"]<=max_year)) and ((deets["runtime"]>=minRuntime) and (deets["runtime"]<=maxRuntime)): if ((deets["year"] >= min_year) and (deets["year"] <= max_year)) and (
(deets["runtime"] >= minRuntime) and (deets["runtime"] <= maxRuntime)
):
results.append(deets) results.append(deets)
cur_res += 1 cur_res += 1
return render_template("show_results.html",deets=results) return render_template("show_results.html", deets=results)
@app.route("/",methods=("GET","POST"))
@app.route("/", methods=("GET", "POST"))
def find_similar_title(): def find_similar_title():
if request.method == "GET": if request.method == "GET":
return render_template("index.html") return render_template("index.html")
@ -125,7 +139,9 @@ def find_similar_title():
code, values = title2trakt_id(to_search_title) code, values = title2trakt_id(to_search_title)
print(f"Code {code} for {to_search_title}") print(f"Code {code} for {to_search_title}")
if code == 0: if code == 0:
search_results = process.extract(to_search_title, movie_titles, scorer=fuzz.token_sort_ratio) search_results = process.extract(
to_search_title, movie_titles, scorer=fuzz.token_sort_ratio
)
to_search_titles = [] to_search_titles = []
to_search_ids = [] to_search_ids = []
results = [] results = []
@ -143,7 +159,7 @@ def find_similar_title():
deets = get_deets_by_trakt_id(df, int(trakt_id)) deets = get_deets_by_trakt_id(df, int(trakt_id))
deets["trakt_id"] = trakt_id deets["trakt_id"] = trakt_id
results.append(deets) results.append(deets)
return render_template("same_titles.html",deets=results) return render_template("same_titles.html", deets=results)
elif code == 1: elif code == 1:
vector = get_vector_value(values) vector = get_vector_value(values)
@ -153,11 +169,11 @@ def find_similar_title():
for trakt_id in movie_ids: for trakt_id in movie_ids:
deets = get_deets_by_trakt_id(df, trakt_id) deets = get_deets_by_trakt_id(df, trakt_id)
results.append(deets) results.append(deets)
return render_template("show_results.html",deets=results) return render_template("show_results.html", deets=results)
else: else:
results = [] results = []
for trakt_id in values: for trakt_id in values:
deets = get_deets_by_trakt_id(df, int(trakt_id)) deets = get_deets_by_trakt_id(df, int(trakt_id))
deets["trakt_id"] = trakt_id deets["trakt_id"] = trakt_id
results.append(deets) results.append(deets)
return render_template("same_titles.html",deets=results) return render_template("same_titles.html", deets=results)

View File

@ -5,7 +5,7 @@ from sqlalchemy import insert
from sqlalchemy.orm import sessionmaker from sqlalchemy.orm import sessionmaker
from sqlalchemy.exc import IntegrityError from sqlalchemy.exc import IntegrityError
#database_url = "sqlite:///jlm.db" # database_url = "sqlite:///jlm.db"
meta = MetaData() meta = MetaData()
@ -25,15 +25,17 @@ movies_table = Table(
Column("votes", Integer), Column("votes", Integer),
Column("comment_count", Integer), Column("comment_count", Integer),
Column("tagline", String), Column("tagline", String),
Column("embeddings", PickleType) Column("embeddings", PickleType),
) )
def init_db_stuff(database_url: str): def init_db_stuff(database_url: str):
engine = create_engine(database_url) engine = create_engine(database_url)
meta.create_all(engine) meta.create_all(engine)
Session = sessionmaker(bind=engine) Session = sessionmaker(bind=engine)
return engine, Session return engine, Session
""" """
movie = { movie = {
"title": movie["movie"]["title"], "title": movie["movie"]["title"],

View File

@ -20,18 +20,23 @@ model = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2")
batch_size = 32 batch_size = 32
df = pd.read_sql("Select * from movies", engine) df = pd.read_sql("Select * from movies", engine)
df["combined_text"] = df["title"] + ": " + df["overview"].fillna('') + " - " + df["tagline"].fillna('') + " Genres:- " + df["genres"].fillna('') df["combined_text"] = (
df["title"]
+ ": "
+ df["overview"].fillna("")
+ " - "
+ df["tagline"].fillna("")
+ " Genres:- "
+ df["genres"].fillna("")
)
print(f'Length of Combined Text: {len(df["combined_text"].tolist())}') print(f'Length of Combined Text: {len(df["combined_text"].tolist())}')
for x in tqdm(range(0,len(df),batch_size)): for x in tqdm(range(0, len(df), batch_size)):
to_send = [] to_send = []
trakt_ids = df["trakt_id"][x:x+batch_size].tolist() trakt_ids = df["trakt_id"][x : x + batch_size].tolist()
sentences = df["combined_text"][x:x+batch_size].tolist() sentences = df["combined_text"][x : x + batch_size].tolist()
embeddings = model.encode(sentences) embeddings = model.encode(sentences)
for idx, value in enumerate(trakt_ids): for idx, value in enumerate(trakt_ids):
to_send.append( to_send.append((str(value), embeddings[idx].tolist()))
(
str(value), embeddings[idx].tolist()
))
index.upsert(to_send) index.upsert(to_send)

34
main.py
View File

@ -25,7 +25,7 @@ database_url = "sqlite:///jlm.db"
headers = { headers = {
"Content-Type": "application/json", "Content-Type": "application/json",
"trakt-api-version": "2", "trakt-api-version": "2",
"trakt-api-key": trakt_id "trakt-api-key": trakt_id,
} }
params = { params = {
@ -34,7 +34,7 @@ params = {
"page": page, "page": page,
"extended": extended, "extended": extended,
"limit": limit, "limit": limit,
"languages": languages "languages": languages,
} }
@ -53,14 +53,13 @@ def create_movie_dict(movie: dict):
"rating": int(m["rating"]), "rating": int(m["rating"]),
"votes": int(m["votes"]), "votes": int(m["votes"]),
"comment_count": int(m["comment_count"]), "comment_count": int(m["comment_count"]),
"tagline": m["tagline"] "tagline": m["tagline"],
} }
return movie_dict return movie_dict
params["limit"] = 1 params["limit"] = 1
res = requests.get(f"{api_base}/search/movie",headers=headers,params=params) res = requests.get(f"{api_base}/search/movie", headers=headers, params=params)
total_items = res.headers["x-pagination-item-count"] total_items = res.headers["x-pagination-item-count"]
print(f"There are {total_items} movies") print(f"There are {total_items} movies")
@ -80,7 +79,7 @@ engine, Session = init_db_stuff(database_url)
start_time = datetime.now() start_time = datetime.now()
for page in tqdm(range(1,max_requests+10)): for page in tqdm(range(1, max_requests + 10)):
if req_count == 999: if req_count == 999:
seconds_to_sleep = 300 - (datetime.now() - start_time).seconds seconds_to_sleep = 300 - (datetime.now() - start_time).seconds
if seconds_to_sleep < 1: if seconds_to_sleep < 1:
@ -93,9 +92,9 @@ for page in tqdm(range(1,max_requests+10)):
req_count = 0 req_count = 0
params["page"] = page params["page"] = page
params["limit"] = int(int(total_items)/max_requests) params["limit"] = int(int(total_items) / max_requests)
movies = [] movies = []
res = requests.get(f"{api_base}/search/movie",headers=headers,params=params) res = requests.get(f"{api_base}/search/movie", headers=headers, params=params)
if res.status_code == 500: if res.status_code == 500:
break break
@ -111,11 +110,20 @@ for page in tqdm(range(1,max_requests+10)):
for movie in movies: for movie in movies:
with conn.begin() as trans: with conn.begin() as trans:
stmt = insert(movies_table).values( stmt = insert(movies_table).values(
trakt_id=movie["trakt_id"], title=movie["title"], genres=" ".join(movie["genres"]), trakt_id=movie["trakt_id"],
language=movie["language"], year=movie["year"], released=movie["released"], title=movie["title"],
runtime=movie["runtime"], country=movie["country"], overview=movie["overview"], genres=" ".join(movie["genres"]),
rating=movie["rating"], votes=movie["votes"], comment_count=movie["comment_count"], language=movie["language"],
tagline=movie["tagline"]) year=movie["year"],
released=movie["released"],
runtime=movie["runtime"],
country=movie["country"],
overview=movie["overview"],
rating=movie["rating"],
votes=movie["votes"],
comment_count=movie["comment_count"],
tagline=movie["tagline"],
)
try: try:
result = conn.execute(stmt) result = conn.execute(stmt)
trans.commit() trans.commit()

View File

@ -10,7 +10,9 @@ from sqlalchemy import func
movie_name = "Forrest Gump" movie_name = "Forrest Gump"
with engine.connect() as conn: with engine.connect() as conn:
movie_deets = select(movies_table).filter(func.lower(movies_table.columns.title)==func.lower(movie_name)) movie_deets = select(movies_table).filter(
func.lower(movies_table.columns.title) == func.lower(movie_name)
)
result = conn.execute(movie_deets) result = conn.execute(movie_deets)
for row in result: for row in result:
print(row) print(row)