cleaned with black
This commit is contained in:
parent
801289d123
commit
404c6fb1a3
44
app.py
44
app.py
|
@ -19,8 +19,9 @@ index = pinecone.Index("movies")
|
||||||
|
|
||||||
app = Flask(__name__, template_folder="./templates")
|
app = Flask(__name__, template_folder="./templates")
|
||||||
|
|
||||||
|
|
||||||
def title2trakt_id(title: str, df=df):
|
def title2trakt_id(title: str, df=df):
|
||||||
#Matches Exact Title, Otherwise Returns None
|
# Matches Exact Title, Otherwise Returns None
|
||||||
records = df[df["title"].str.lower() == title.lower()]
|
records = df[df["title"].str.lower() == title.lower()]
|
||||||
if len(records) == 0:
|
if len(records) == 0:
|
||||||
return 0, None
|
return 0, None
|
||||||
|
@ -29,38 +30,48 @@ def title2trakt_id(title: str, df=df):
|
||||||
else:
|
else:
|
||||||
return 2, records.trakt_id.tolist()
|
return 2, records.trakt_id.tolist()
|
||||||
|
|
||||||
|
|
||||||
def get_vector_value(trakt_id: int):
|
def get_vector_value(trakt_id: int):
|
||||||
fetch_response = index.fetch(ids=[str(trakt_id)])
|
fetch_response = index.fetch(ids=[str(trakt_id)])
|
||||||
return fetch_response["vectors"][str(trakt_id)]["values"]
|
return fetch_response["vectors"][str(trakt_id)]["values"]
|
||||||
|
|
||||||
def query_vectors(vector: list, top_k: int = 20, include_values: bool = False, include_metada: bool = True):
|
|
||||||
|
def query_vectors(
|
||||||
|
vector: list,
|
||||||
|
top_k: int = 20,
|
||||||
|
include_values: bool = False,
|
||||||
|
include_metada: bool = True,
|
||||||
|
):
|
||||||
query_response = index.query(
|
query_response = index.query(
|
||||||
queries=[
|
queries=[
|
||||||
(vector),
|
(vector),
|
||||||
],
|
],
|
||||||
top_k=top_k,
|
top_k=top_k,
|
||||||
include_values=include_values,
|
include_values=include_values,
|
||||||
include_metadata=include_metada
|
include_metadata=include_metada,
|
||||||
)
|
)
|
||||||
return query_response
|
return query_response
|
||||||
|
|
||||||
|
|
||||||
def query2ids(query_response):
|
def query2ids(query_response):
|
||||||
trakt_ids = []
|
trakt_ids = []
|
||||||
for match in query_response["results"][0]["matches"]:
|
for match in query_response["results"][0]["matches"]:
|
||||||
trakt_ids.append(int(match["id"]))
|
trakt_ids.append(int(match["id"]))
|
||||||
return trakt_ids
|
return trakt_ids
|
||||||
|
|
||||||
|
|
||||||
def get_deets_by_trakt_id(df, trakt_id: int):
|
def get_deets_by_trakt_id(df, trakt_id: int):
|
||||||
df = df[df["trakt_id"]==trakt_id]
|
df = df[df["trakt_id"] == trakt_id]
|
||||||
return {
|
return {
|
||||||
"title": df.title.values[0],
|
"title": df.title.values[0],
|
||||||
"overview": df.overview.values[0],
|
"overview": df.overview.values[0],
|
||||||
"runtime": int(df.runtime.values[0]),
|
"runtime": int(df.runtime.values[0]),
|
||||||
"year": int(df.year.values[0]),
|
"year": int(df.year.values[0]),
|
||||||
"trakt_id": trakt_id,
|
"trakt_id": trakt_id,
|
||||||
"tagline": df.tagline.values[0]
|
"tagline": df.tagline.values[0],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@app.route("/similar")
|
@app.route("/similar")
|
||||||
def get_similar_titles():
|
def get_similar_titles():
|
||||||
trakt_id = request.args.get("trakt_id")
|
trakt_id = request.args.get("trakt_id")
|
||||||
|
@ -99,10 +110,10 @@ def get_similar_titles():
|
||||||
except TypeError:
|
except TypeError:
|
||||||
maxRuntime = 220
|
maxRuntime = 220
|
||||||
vector = get_vector_value(trakt_id)
|
vector = get_vector_value(trakt_id)
|
||||||
movie_queries = query_vectors(vector, top_k = 69)
|
movie_queries = query_vectors(vector, top_k=69)
|
||||||
movie_ids = query2ids(movie_queries)
|
movie_ids = query2ids(movie_queries)
|
||||||
results = []
|
results = []
|
||||||
#for trakt_id in movie_ids:
|
# for trakt_id in movie_ids:
|
||||||
# deets = get_deets_by_trakt_id(df, trakt_id)
|
# deets = get_deets_by_trakt_id(df, trakt_id)
|
||||||
# results.append(deets)
|
# results.append(deets)
|
||||||
max_res = 30
|
max_res = 30
|
||||||
|
@ -111,12 +122,15 @@ def get_similar_titles():
|
||||||
if cur_res >= max_res:
|
if cur_res >= max_res:
|
||||||
break
|
break
|
||||||
deets = get_deets_by_trakt_id(df, trakt_id)
|
deets = get_deets_by_trakt_id(df, trakt_id)
|
||||||
if ((deets["year"]>=min_year) and (deets["year"]<=max_year)) and ((deets["runtime"]>=minRuntime) and (deets["runtime"]<=maxRuntime)):
|
if ((deets["year"] >= min_year) and (deets["year"] <= max_year)) and (
|
||||||
|
(deets["runtime"] >= minRuntime) and (deets["runtime"] <= maxRuntime)
|
||||||
|
):
|
||||||
results.append(deets)
|
results.append(deets)
|
||||||
cur_res += 1
|
cur_res += 1
|
||||||
return render_template("show_results.html",deets=results)
|
return render_template("show_results.html", deets=results)
|
||||||
|
|
||||||
@app.route("/",methods=("GET","POST"))
|
|
||||||
|
@app.route("/", methods=("GET", "POST"))
|
||||||
def find_similar_title():
|
def find_similar_title():
|
||||||
if request.method == "GET":
|
if request.method == "GET":
|
||||||
return render_template("index.html")
|
return render_template("index.html")
|
||||||
|
@ -125,7 +139,9 @@ def find_similar_title():
|
||||||
code, values = title2trakt_id(to_search_title)
|
code, values = title2trakt_id(to_search_title)
|
||||||
print(f"Code {code} for {to_search_title}")
|
print(f"Code {code} for {to_search_title}")
|
||||||
if code == 0:
|
if code == 0:
|
||||||
search_results = process.extract(to_search_title, movie_titles, scorer=fuzz.token_sort_ratio)
|
search_results = process.extract(
|
||||||
|
to_search_title, movie_titles, scorer=fuzz.token_sort_ratio
|
||||||
|
)
|
||||||
to_search_titles = []
|
to_search_titles = []
|
||||||
to_search_ids = []
|
to_search_ids = []
|
||||||
results = []
|
results = []
|
||||||
|
@ -143,7 +159,7 @@ def find_similar_title():
|
||||||
deets = get_deets_by_trakt_id(df, int(trakt_id))
|
deets = get_deets_by_trakt_id(df, int(trakt_id))
|
||||||
deets["trakt_id"] = trakt_id
|
deets["trakt_id"] = trakt_id
|
||||||
results.append(deets)
|
results.append(deets)
|
||||||
return render_template("same_titles.html",deets=results)
|
return render_template("same_titles.html", deets=results)
|
||||||
|
|
||||||
elif code == 1:
|
elif code == 1:
|
||||||
vector = get_vector_value(values)
|
vector = get_vector_value(values)
|
||||||
|
@ -153,11 +169,11 @@ def find_similar_title():
|
||||||
for trakt_id in movie_ids:
|
for trakt_id in movie_ids:
|
||||||
deets = get_deets_by_trakt_id(df, trakt_id)
|
deets = get_deets_by_trakt_id(df, trakt_id)
|
||||||
results.append(deets)
|
results.append(deets)
|
||||||
return render_template("show_results.html",deets=results)
|
return render_template("show_results.html", deets=results)
|
||||||
else:
|
else:
|
||||||
results = []
|
results = []
|
||||||
for trakt_id in values:
|
for trakt_id in values:
|
||||||
deets = get_deets_by_trakt_id(df, int(trakt_id))
|
deets = get_deets_by_trakt_id(df, int(trakt_id))
|
||||||
deets["trakt_id"] = trakt_id
|
deets["trakt_id"] = trakt_id
|
||||||
results.append(deets)
|
results.append(deets)
|
||||||
return render_template("same_titles.html",deets=results)
|
return render_template("same_titles.html", deets=results)
|
||||||
|
|
|
@ -5,7 +5,7 @@ from sqlalchemy import insert
|
||||||
from sqlalchemy.orm import sessionmaker
|
from sqlalchemy.orm import sessionmaker
|
||||||
from sqlalchemy.exc import IntegrityError
|
from sqlalchemy.exc import IntegrityError
|
||||||
|
|
||||||
#database_url = "sqlite:///jlm.db"
|
# database_url = "sqlite:///jlm.db"
|
||||||
|
|
||||||
meta = MetaData()
|
meta = MetaData()
|
||||||
|
|
||||||
|
@ -25,15 +25,17 @@ movies_table = Table(
|
||||||
Column("votes", Integer),
|
Column("votes", Integer),
|
||||||
Column("comment_count", Integer),
|
Column("comment_count", Integer),
|
||||||
Column("tagline", String),
|
Column("tagline", String),
|
||||||
Column("embeddings", PickleType)
|
Column("embeddings", PickleType),
|
||||||
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def init_db_stuff(database_url: str):
|
def init_db_stuff(database_url: str):
|
||||||
engine = create_engine(database_url)
|
engine = create_engine(database_url)
|
||||||
meta.create_all(engine)
|
meta.create_all(engine)
|
||||||
Session = sessionmaker(bind=engine)
|
Session = sessionmaker(bind=engine)
|
||||||
return engine, Session
|
return engine, Session
|
||||||
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
movie = {
|
movie = {
|
||||||
"title": movie["movie"]["title"],
|
"title": movie["movie"]["title"],
|
||||||
|
|
21
db2pc.py
21
db2pc.py
|
@ -20,18 +20,23 @@ model = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2")
|
||||||
batch_size = 32
|
batch_size = 32
|
||||||
|
|
||||||
df = pd.read_sql("Select * from movies", engine)
|
df = pd.read_sql("Select * from movies", engine)
|
||||||
df["combined_text"] = df["title"] + ": " + df["overview"].fillna('') + " - " + df["tagline"].fillna('') + " Genres:- " + df["genres"].fillna('')
|
df["combined_text"] = (
|
||||||
|
df["title"]
|
||||||
|
+ ": "
|
||||||
|
+ df["overview"].fillna("")
|
||||||
|
+ " - "
|
||||||
|
+ df["tagline"].fillna("")
|
||||||
|
+ " Genres:- "
|
||||||
|
+ df["genres"].fillna("")
|
||||||
|
)
|
||||||
|
|
||||||
print(f'Length of Combined Text: {len(df["combined_text"].tolist())}')
|
print(f'Length of Combined Text: {len(df["combined_text"].tolist())}')
|
||||||
|
|
||||||
for x in tqdm(range(0,len(df),batch_size)):
|
for x in tqdm(range(0, len(df), batch_size)):
|
||||||
to_send = []
|
to_send = []
|
||||||
trakt_ids = df["trakt_id"][x:x+batch_size].tolist()
|
trakt_ids = df["trakt_id"][x : x + batch_size].tolist()
|
||||||
sentences = df["combined_text"][x:x+batch_size].tolist()
|
sentences = df["combined_text"][x : x + batch_size].tolist()
|
||||||
embeddings = model.encode(sentences)
|
embeddings = model.encode(sentences)
|
||||||
for idx, value in enumerate(trakt_ids):
|
for idx, value in enumerate(trakt_ids):
|
||||||
to_send.append(
|
to_send.append((str(value), embeddings[idx].tolist()))
|
||||||
(
|
|
||||||
str(value), embeddings[idx].tolist()
|
|
||||||
))
|
|
||||||
index.upsert(to_send)
|
index.upsert(to_send)
|
34
main.py
34
main.py
|
@ -25,7 +25,7 @@ database_url = "sqlite:///jlm.db"
|
||||||
headers = {
|
headers = {
|
||||||
"Content-Type": "application/json",
|
"Content-Type": "application/json",
|
||||||
"trakt-api-version": "2",
|
"trakt-api-version": "2",
|
||||||
"trakt-api-key": trakt_id
|
"trakt-api-key": trakt_id,
|
||||||
}
|
}
|
||||||
|
|
||||||
params = {
|
params = {
|
||||||
|
@ -34,7 +34,7 @@ params = {
|
||||||
"page": page,
|
"page": page,
|
||||||
"extended": extended,
|
"extended": extended,
|
||||||
"limit": limit,
|
"limit": limit,
|
||||||
"languages": languages
|
"languages": languages,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -53,14 +53,13 @@ def create_movie_dict(movie: dict):
|
||||||
"rating": int(m["rating"]),
|
"rating": int(m["rating"]),
|
||||||
"votes": int(m["votes"]),
|
"votes": int(m["votes"]),
|
||||||
"comment_count": int(m["comment_count"]),
|
"comment_count": int(m["comment_count"]),
|
||||||
"tagline": m["tagline"]
|
"tagline": m["tagline"],
|
||||||
}
|
}
|
||||||
return movie_dict
|
return movie_dict
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
params["limit"] = 1
|
params["limit"] = 1
|
||||||
res = requests.get(f"{api_base}/search/movie",headers=headers,params=params)
|
res = requests.get(f"{api_base}/search/movie", headers=headers, params=params)
|
||||||
total_items = res.headers["x-pagination-item-count"]
|
total_items = res.headers["x-pagination-item-count"]
|
||||||
|
|
||||||
print(f"There are {total_items} movies")
|
print(f"There are {total_items} movies")
|
||||||
|
@ -80,7 +79,7 @@ engine, Session = init_db_stuff(database_url)
|
||||||
|
|
||||||
start_time = datetime.now()
|
start_time = datetime.now()
|
||||||
|
|
||||||
for page in tqdm(range(1,max_requests+10)):
|
for page in tqdm(range(1, max_requests + 10)):
|
||||||
if req_count == 999:
|
if req_count == 999:
|
||||||
seconds_to_sleep = 300 - (datetime.now() - start_time).seconds
|
seconds_to_sleep = 300 - (datetime.now() - start_time).seconds
|
||||||
if seconds_to_sleep < 1:
|
if seconds_to_sleep < 1:
|
||||||
|
@ -93,9 +92,9 @@ for page in tqdm(range(1,max_requests+10)):
|
||||||
req_count = 0
|
req_count = 0
|
||||||
|
|
||||||
params["page"] = page
|
params["page"] = page
|
||||||
params["limit"] = int(int(total_items)/max_requests)
|
params["limit"] = int(int(total_items) / max_requests)
|
||||||
movies = []
|
movies = []
|
||||||
res = requests.get(f"{api_base}/search/movie",headers=headers,params=params)
|
res = requests.get(f"{api_base}/search/movie", headers=headers, params=params)
|
||||||
|
|
||||||
if res.status_code == 500:
|
if res.status_code == 500:
|
||||||
break
|
break
|
||||||
|
@ -111,11 +110,20 @@ for page in tqdm(range(1,max_requests+10)):
|
||||||
for movie in movies:
|
for movie in movies:
|
||||||
with conn.begin() as trans:
|
with conn.begin() as trans:
|
||||||
stmt = insert(movies_table).values(
|
stmt = insert(movies_table).values(
|
||||||
trakt_id=movie["trakt_id"], title=movie["title"], genres=" ".join(movie["genres"]),
|
trakt_id=movie["trakt_id"],
|
||||||
language=movie["language"], year=movie["year"], released=movie["released"],
|
title=movie["title"],
|
||||||
runtime=movie["runtime"], country=movie["country"], overview=movie["overview"],
|
genres=" ".join(movie["genres"]),
|
||||||
rating=movie["rating"], votes=movie["votes"], comment_count=movie["comment_count"],
|
language=movie["language"],
|
||||||
tagline=movie["tagline"])
|
year=movie["year"],
|
||||||
|
released=movie["released"],
|
||||||
|
runtime=movie["runtime"],
|
||||||
|
country=movie["country"],
|
||||||
|
overview=movie["overview"],
|
||||||
|
rating=movie["rating"],
|
||||||
|
votes=movie["votes"],
|
||||||
|
comment_count=movie["comment_count"],
|
||||||
|
tagline=movie["tagline"],
|
||||||
|
)
|
||||||
try:
|
try:
|
||||||
result = conn.execute(stmt)
|
result = conn.execute(stmt)
|
||||||
trans.commit()
|
trans.commit()
|
||||||
|
|
|
@ -10,7 +10,9 @@ from sqlalchemy import func
|
||||||
movie_name = "Forrest Gump"
|
movie_name = "Forrest Gump"
|
||||||
|
|
||||||
with engine.connect() as conn:
|
with engine.connect() as conn:
|
||||||
movie_deets = select(movies_table).filter(func.lower(movies_table.columns.title)==func.lower(movie_name))
|
movie_deets = select(movies_table).filter(
|
||||||
|
func.lower(movies_table.columns.title) == func.lower(movie_name)
|
||||||
|
)
|
||||||
result = conn.execute(movie_deets)
|
result = conn.execute(movie_deets)
|
||||||
for row in result:
|
for row in result:
|
||||||
print(row)
|
print(row)
|
Loading…
Reference in New Issue