Crude Oil Market Trend Analysis

from bigdata_client import Bigdata
from bigdata_client.models.search import DocumentType
from src.lexicon_generator import LexiconGenerator
from src.search_topics import search_by_keywords
from src.topics_extractor import (process_all_reports,
                                run_process_all_trending_topics,
                                run_add_advanced_novelty_scores,
                                add_market_impact_to_df,)
from src.report_generator import(prepare_data_for_report, generate_html_report,
                                save_html_report)
from IPython.display import display
from IPython.core.display import HTML

# Define output file paths for our results
output_dir = "report"
os.makedirs(output_dir, exist_ok=True)

export_path = f"{output_dir}/daily_digest_crude_oil.csv"

# ===== Context Definition =====
main_theme = 'Crude Oil'

point_of_view = "a crude oil trader, where price is influenced by supply-demand dynamics, geopolitical events, and market sentiment. \
As a trader, you focus on changes in production, inventories, and economic indicators from key markets."

# ===== Specify Time Range =====
start_query = '2025-06-20'
end_query = '2025-06-27'

# ===== Query Configuration =====
document_type = DocumentType.NEWS

# ===== LLM Specification =====
llm_model = "gpt-4o-mini"

# ===== Query Configuration =====
document_limit = 10 # Maximum number of documents to retrieve per query
frequency = 'D'  # Query frequency

lexicongenerator = LexiconGenerator(openai_key=OPENAI_API_KEY, model="gpt-4o", seeds=[123, 123456, 123456789, 456789, 789])

keywords = lexicongenerator.generate(theme=main_theme)

results, daily_keyword_count = search_by_keywords(
    keywords=keywords,
    start_date=start_query,
    end_date=end_query,
    scope=document_type,
    freq=frequency,
    document_limit=document_limit)

semaphore_size = 1000 # Maximum number of concurrent requests to OpenAI API

# Apply verification layer to remove irrelevant news
filtered_reports = process_all_reports(results, llm_model, OPENAI_API_KEY, main_theme, semaphore_size)

# Perform topic modeling and clustering
flattened_trending_topics_df = run_process_all_trending_topics(
    unique_reports=filtered_reports,
    model=llm_model,
    start_query=start_query,
    end_query=end_query,
    api_key=OPENAI_API_KEY,
    main_theme=main_theme,
    batches=20 # Number of batches to process the reports in parallel
)

flattened_trending_topics_df = run_add_advanced_novelty_scores(flattened_trending_topics_df, api_key=OPENAI_API_KEY, main_theme=main_theme)

flattened_trending_topics_df = add_market_impact_to_df(flattened_trending_topics_df, api_key=OPENAI_API_KEY, main_theme=main_theme, point_of_view=point_of_view)

specific_date = '2025-06-25'  # Example date, can be modified as needed
user_selected_ranking = ['novelty', 'volume', 'magnitude']  # User can modify this list
#impact_filter = 'positive_impact' #User can use the impact_filter to filter out the report

prepared_reports = prepare_data_for_report(flattened_trending_topics_df, user_selected_ranking, impact_filter=None, report_date=specific_date)

# Generate and display the HTML report for each date
for report in prepared_reports:
    html_content = generate_html_report(
        report['date'],
        report['day_in_review'],
        report['topics'],
        f'Daily {main_theme} market update'
    )
    display(HTML(html_content))
    save_html_report(html_content, report['date'], main_theme)

flattened_trending_topics_df.to_csv(export_path, index=False)