cleaned with black

This commit is contained in:
navanchauhan 2022-05-22 11:42:00 -06:00
parent 801289d123
commit 404c6fb1a3
5 changed files with 158 additions and 125 deletions

26
app.py
View File

@ -19,6 +19,7 @@ index = pinecone.Index("movies")
app = Flask(__name__, template_folder="./templates") app = Flask(__name__, template_folder="./templates")
def title2trakt_id(title: str, df=df): def title2trakt_id(title: str, df=df):
# Matches Exact Title, Otherwise Returns None # Matches Exact Title, Otherwise Returns None
records = df[df["title"].str.lower() == title.lower()] records = df[df["title"].str.lower() == title.lower()]
@ -29,27 +30,36 @@ def title2trakt_id(title: str, df=df):
else: else:
return 2, records.trakt_id.tolist() return 2, records.trakt_id.tolist()
def get_vector_value(trakt_id: int): def get_vector_value(trakt_id: int):
fetch_response = index.fetch(ids=[str(trakt_id)]) fetch_response = index.fetch(ids=[str(trakt_id)])
return fetch_response["vectors"][str(trakt_id)]["values"] return fetch_response["vectors"][str(trakt_id)]["values"]
def query_vectors(vector: list, top_k: int = 20, include_values: bool = False, include_metada: bool = True):
def query_vectors(
vector: list,
top_k: int = 20,
include_values: bool = False,
include_metada: bool = True,
):
query_response = index.query( query_response = index.query(
queries=[ queries=[
(vector), (vector),
], ],
top_k=top_k, top_k=top_k,
include_values=include_values, include_values=include_values,
include_metadata=include_metada include_metadata=include_metada,
) )
return query_response return query_response
def query2ids(query_response): def query2ids(query_response):
trakt_ids = [] trakt_ids = []
for match in query_response["results"][0]["matches"]: for match in query_response["results"][0]["matches"]:
trakt_ids.append(int(match["id"])) trakt_ids.append(int(match["id"]))
return trakt_ids return trakt_ids
def get_deets_by_trakt_id(df, trakt_id: int): def get_deets_by_trakt_id(df, trakt_id: int):
df = df[df["trakt_id"] == trakt_id] df = df[df["trakt_id"] == trakt_id]
return { return {
@ -58,9 +68,10 @@ def get_deets_by_trakt_id(df, trakt_id: int):
"runtime": int(df.runtime.values[0]), "runtime": int(df.runtime.values[0]),
"year": int(df.year.values[0]), "year": int(df.year.values[0]),
"trakt_id": trakt_id, "trakt_id": trakt_id,
"tagline": df.tagline.values[0] "tagline": df.tagline.values[0],
} }
@app.route("/similar") @app.route("/similar")
def get_similar_titles(): def get_similar_titles():
trakt_id = request.args.get("trakt_id") trakt_id = request.args.get("trakt_id")
@ -111,11 +122,14 @@ def get_similar_titles():
if cur_res >= max_res: if cur_res >= max_res:
break break
deets = get_deets_by_trakt_id(df, trakt_id) deets = get_deets_by_trakt_id(df, trakt_id)
if ((deets["year"]>=min_year) and (deets["year"]<=max_year)) and ((deets["runtime"]>=minRuntime) and (deets["runtime"]<=maxRuntime)): if ((deets["year"] >= min_year) and (deets["year"] <= max_year)) and (
(deets["runtime"] >= minRuntime) and (deets["runtime"] <= maxRuntime)
):
results.append(deets) results.append(deets)
cur_res += 1 cur_res += 1
return render_template("show_results.html", deets=results) return render_template("show_results.html", deets=results)
@app.route("/", methods=("GET", "POST")) @app.route("/", methods=("GET", "POST"))
def find_similar_title(): def find_similar_title():
if request.method == "GET": if request.method == "GET":
@ -125,7 +139,9 @@ def find_similar_title():
code, values = title2trakt_id(to_search_title) code, values = title2trakt_id(to_search_title)
print(f"Code {code} for {to_search_title}") print(f"Code {code} for {to_search_title}")
if code == 0: if code == 0:
search_results = process.extract(to_search_title, movie_titles, scorer=fuzz.token_sort_ratio) search_results = process.extract(
to_search_title, movie_titles, scorer=fuzz.token_sort_ratio
)
to_search_titles = [] to_search_titles = []
to_search_ids = [] to_search_ids = []
results = [] results = []

View File

@ -25,15 +25,17 @@ movies_table = Table(
Column("votes", Integer), Column("votes", Integer),
Column("comment_count", Integer), Column("comment_count", Integer),
Column("tagline", String), Column("tagline", String),
Column("embeddings", PickleType) Column("embeddings", PickleType),
) )
def init_db_stuff(database_url: str): def init_db_stuff(database_url: str):
engine = create_engine(database_url) engine = create_engine(database_url)
meta.create_all(engine) meta.create_all(engine)
Session = sessionmaker(bind=engine) Session = sessionmaker(bind=engine)
return engine, Session return engine, Session
""" """
movie = { movie = {
"title": movie["movie"]["title"], "title": movie["movie"]["title"],

View File

@ -20,7 +20,15 @@ model = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2")
batch_size = 32 batch_size = 32
df = pd.read_sql("Select * from movies", engine) df = pd.read_sql("Select * from movies", engine)
df["combined_text"] = df["title"] + ": " + df["overview"].fillna('') + " - " + df["tagline"].fillna('') + " Genres:- " + df["genres"].fillna('') df["combined_text"] = (
df["title"]
+ ": "
+ df["overview"].fillna("")
+ " - "
+ df["tagline"].fillna("")
+ " Genres:- "
+ df["genres"].fillna("")
)
print(f'Length of Combined Text: {len(df["combined_text"].tolist())}') print(f'Length of Combined Text: {len(df["combined_text"].tolist())}')
@ -30,8 +38,5 @@ for x in tqdm(range(0,len(df),batch_size)):
sentences = df["combined_text"][x : x + batch_size].tolist() sentences = df["combined_text"][x : x + batch_size].tolist()
embeddings = model.encode(sentences) embeddings = model.encode(sentences)
for idx, value in enumerate(trakt_ids): for idx, value in enumerate(trakt_ids):
to_send.append( to_send.append((str(value), embeddings[idx].tolist()))
(
str(value), embeddings[idx].tolist()
))
index.upsert(to_send) index.upsert(to_send)

26
main.py
View File

@ -25,7 +25,7 @@ database_url = "sqlite:///jlm.db"
headers = { headers = {
"Content-Type": "application/json", "Content-Type": "application/json",
"trakt-api-version": "2", "trakt-api-version": "2",
"trakt-api-key": trakt_id "trakt-api-key": trakt_id,
} }
params = { params = {
@ -34,7 +34,7 @@ params = {
"page": page, "page": page,
"extended": extended, "extended": extended,
"limit": limit, "limit": limit,
"languages": languages "languages": languages,
} }
@ -53,12 +53,11 @@ def create_movie_dict(movie: dict):
"rating": int(m["rating"]), "rating": int(m["rating"]),
"votes": int(m["votes"]), "votes": int(m["votes"]),
"comment_count": int(m["comment_count"]), "comment_count": int(m["comment_count"]),
"tagline": m["tagline"] "tagline": m["tagline"],
} }
return movie_dict return movie_dict
params["limit"] = 1 params["limit"] = 1
res = requests.get(f"{api_base}/search/movie", headers=headers, params=params) res = requests.get(f"{api_base}/search/movie", headers=headers, params=params)
total_items = res.headers["x-pagination-item-count"] total_items = res.headers["x-pagination-item-count"]
@ -111,11 +110,20 @@ for page in tqdm(range(1,max_requests+10)):
for movie in movies: for movie in movies:
with conn.begin() as trans: with conn.begin() as trans:
stmt = insert(movies_table).values( stmt = insert(movies_table).values(
trakt_id=movie["trakt_id"], title=movie["title"], genres=" ".join(movie["genres"]), trakt_id=movie["trakt_id"],
language=movie["language"], year=movie["year"], released=movie["released"], title=movie["title"],
runtime=movie["runtime"], country=movie["country"], overview=movie["overview"], genres=" ".join(movie["genres"]),
rating=movie["rating"], votes=movie["votes"], comment_count=movie["comment_count"], language=movie["language"],
tagline=movie["tagline"]) year=movie["year"],
released=movie["released"],
runtime=movie["runtime"],
country=movie["country"],
overview=movie["overview"],
rating=movie["rating"],
votes=movie["votes"],
comment_count=movie["comment_count"],
tagline=movie["tagline"],
)
try: try:
result = conn.execute(stmt) result = conn.execute(stmt)
trans.commit() trans.commit()

View File

@ -10,7 +10,9 @@ from sqlalchemy import func
movie_name = "Forrest Gump" movie_name = "Forrest Gump"
with engine.connect() as conn: with engine.connect() as conn:
movie_deets = select(movies_table).filter(func.lower(movies_table.columns.title)==func.lower(movie_name)) movie_deets = select(movies_table).filter(
func.lower(movies_table.columns.title) == func.lower(movie_name)
)
result = conn.execute(movie_deets) result = conn.execute(movie_deets)
for row in result: for row in result:
print(row) print(row)