Crypto Institutional Adoption

import json
import os

import pandas as pd
from dotenv import load_dotenv
from pathlib import Path
from openai import OpenAI

from src.bigdata_rest import BigdataRestClient, load_crypto_universe
from src.openai_utils import sampling_params_for_model
from src.search_entities import search_by_entities, post_process_dataframe
from src.visualization_tool import display_figures

# Load credentials
script_dir = Path(__file__).parent if '__file__' in globals() else Path.cwd()
load_dotenv(script_dir / '.env')

BIGDATA_API_KEY = os.getenv("BIGDATA_API_KEY")
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")

if not all([BIGDATA_API_KEY, OPENAI_API_KEY]):
    raise ValueError("Missing required environment variables. Check your .env file.")

rest_client = BigdataRestClient(api_key=BIGDATA_API_KEY)
openai_client = OpenAI(api_key=OPENAI_API_KEY)

# ===== Theme Definition =====
main_theme = "Crypto Institutional Adoption"
focus="Include know your customer (KYC) and anti-money laundering (AML) themes"

# ===== Entity Universe (Top 15 Cryptos CSV) =====
# Entity IDs resolved from Bigdata.com and saved under Screener_for_Crypto/data/
from src.bigdata_rest import load_crypto_universe

entity_ids, entity_names = load_crypto_universe("data/top_15_cryptos.csv")
control_entities = None

# ===== LLM Specification =====
llm_model = "gpt-5.6-luna"

# ===== Docs Configuration =====
document_type = "news"
fiscal_year = None

# ===== Enable/Disable Reranker =====
rerank_threshold = None

# ===== Specify Time Range =====
start_date = "2025-01-01"
end_date = "2025-09-08"

# ===== Source Selection =====
# Crypto Wire [D6D057] - Gold-standard crypto intelligence
# Premium dedicated source providing superior quality, topicality, volume, 
# source diversity, coverage and timeliness for cryptocurrency market analysis
sources = ["D6D057"]

def generate_themes(main_theme: str, focus: str = "", n_themes: int = 5) -> list[str]:
    prompt = f'''Generate {n_themes} specific, measurable sub-themes for analyzing: "{main_theme}"

{"Focus: " + focus if focus else ""}

Each sub-theme should be a short descriptive sentence that explicitly connects back to the main theme.
Return ONLY a JSON array of strings.'''
    response = openai_client.chat.completions.create(
        model=llm_model,
        messages=[{"role": "user", "content": prompt}],
        **sampling_params_for_model(llm_model, temperature=0.3),
    )
    text = response.choices[0].message.content.strip()
    if text.startswith("```"):
        text = text.split("```")[1]
        if text.startswith("json"):
            text = text[4:]
        text = text.strip()
    return json.loads(text)

node_summaries = generate_themes(main_theme, focus=focus, n_themes=5)

document_limit = 10

df_sentences = search_by_entities(
    entity_ids=entity_ids,
    entity_names=entity_names,
    sentences=node_summaries,
    start_date=start_date,
    end_date=end_date,
    rest_client=rest_client,
    document_limit=document_limit,
)

def label_crypto_text(text: str, entity_name: str, themes: list[str]) -> dict:
    prompt = f'''Analyze this text about {entity_name} and identify which theme it relates to most.

Themes: {", ".join(themes)}

Text: {text}

Return JSON: {{"label": "the closest matching theme, verbatim, or 'unclear'", "motivation": "brief one-sentence explanation"}}'''
    response = openai_client.chat.completions.create(
        model=llm_model,
        messages=[{"role": "user", "content": prompt}],
        response_format={"type": "json_object"},
        **sampling_params_for_model(llm_model, temperature=0.0),
    )
    return json.loads(response.choices[0].message.content)

label_results = df_sentences.apply(
    lambda row: label_crypto_text(row["text"], row["entity_name"], node_summaries),
    axis=1,
)
df_sentences["label"] = label_results.apply(lambda x: x.get("label", "unclear"))
df_sentences["motivation"] = label_results.apply(lambda x: x.get("motivation", ""))

df = post_process_dataframe(df_sentences)

def score_entities(df: pd.DataFrame, themes: list[str]) -> pd.DataFrame:
    rows = []
    for entity in df["Entity"].unique():
        entity_df = df[df["Entity"] == entity]
        theme_counts = {theme: int((entity_df["Theme"] == theme).sum()) for theme in themes}
        rows.append({
            "Entity": entity,
            "Total Mentions": len(entity_df),
            "Themes Covered": sum(1 for count in theme_counts.values() if count > 0),
            **theme_counts,
            "Composite Score": sum(theme_counts.values()),
        })
    return pd.DataFrame(rows).sort_values("Composite Score", ascending=False).reset_index(drop=True)

df_entity = score_entities(df, node_summaries)

display_figures(df_entity, interactive=False, n_entities=10)