Regulation Risks

# ===== Fixed Parameters =====

# General regulatory theme
general_theme = 'Regulatory Issues'

# Specific focus areas within technology sectors
list_specific_focus = ['AI', 'Social Media', 'Hardware and Chips', 'E-commerce', 'Advertising']

# ===== Customizable Parameters =====

# Company Universe (from Watchlist)
my_watchlist_id = "ff900c90-1007-4971-a91d-49e6f9bb798c" # Magnificent 7
watchlist = bigdata.watchlists.get(my_watchlist_id)
companies = bigdata.knowledge_graph.get_entities(watchlist.items)
company_names = [company.name for company in companies]

# LLM Specification
llm_model = "openai::gpt-4o-mini"

# Search Frequency
search_frequency='M'

# Specify Time Range
start_date="2025-01-01"
end_date="2025-04-20"

# Fiscal Year
fiscal_year = 2025

# Document Limits
document_limit_news=10
document_limit_filings=5
document_limit_transcripts=5

# Others
batch_size=1

report_generator = GenerateReport(
        watchlist_id=my_watchlist_id,
        general_theme='Regulatory Issues',
        list_specific_focus=['AI', 'Social Media', 'Hardware and Chips', 'E-commerce', 'Advertising'],
        llm_model=llm_model,
        api_key=OPENAI_API_KEY,
        start_date=start_date,
        end_date=end_date,
        fiscal_year = fiscal_year,
        search_frequency=search_frequency,
        document_limit_news=document_limit_news,
        document_limit_filings=document_limit_filings,
        document_limit_transcripts=document_limit_transcripts,
        batch_size=batch_size,
        bigdata=bigdata
)

# Generate the Theme Tree
themes_tree_dict = {}
for focus in list_specific_focus:
    theme_tree = generate_theme_tree(
                        main_theme=general_theme,
                        focus=focus
                    )
    themes_tree_dict[focus] = theme_tree

theme_tree.visualize()

# Run searches on News, Filings, and Transcripts
df_sentences_news = []
df_sentences_filings = []
df_sentences_transcripts = []

scopes_config = [
    (DocumentType.NEWS, document_limit_news, df_sentences_news, None),
    (DocumentType.FILINGS, document_limit_filings, df_sentences_filings, fiscal_year),
    (DocumentType.TRANSCRIPTS, document_limit_transcripts, df_sentences_transcripts, fiscal_year)
]

# Search using summaries
for scope, document_limit, df_list, year in scopes_config:
    for focus in list_specific_focus:
        df_sentences = search_by_companies(
            companies=companies,
            sentences=list(themes_tree_dict[focus].get_terminal_label_summaries().values()),
            fiscal_year=year,
            start_date=start_date,
            end_date=end_date,
            scope=scope,
            freq=search_frequency,
            document_limit=document_limit,
            batch_size=batch_size
        )
        df_sentences['theme'] = general_theme + ' in ' + focus
        df_list.append(df_sentences)

# Concatenate results
df_sentences_news = pd.concat(df_sentences_news)
df_sentences_filings = pd.concat(df_sentences_filings)
df_sentences_transcripts = pd.concat(df_sentences_transcripts)

# Label the search results with our theme labels
labeler = ScreenerLabeler(llm_model=llm_model)

# Initialize empty lists for labeled data
df_news_labeled = []
df_filings_labeled = []
df_transcripts_labeled = []

# Configure data sources
sources_config = [
    (df_sentences_news, df_news_labeled),
    (df_sentences_filings, df_filings_labeled),
    (df_sentences_transcripts, df_transcripts_labeled)
]

for df_sentences, labeled_list in sources_config:
    for focus in list_specific_focus:
        df_sentences_theme = df_sentences.loc[(df_sentences.theme == general_theme + ' in ' + focus)]
        df_sentences_theme.reset_index(drop=True, inplace=True)
        df_labels = labeler.get_labels(
            main_theme=general_theme + ' in ' + focus,
            labels=list(themes_tree_dict[focus].get_terminal_label_summaries().keys()),
            texts=df_sentences_theme["masked_text"].tolist()
        )
        df_merged_labels = pd.merge(df_sentences_theme, df_labels, left_index=True, right_index=True)
        labeled_list.append(df_merged_labels)

# Concatenate results
df_news_labeled = pd.concat(df_news_labeled)
df_filings_labeled = pd.concat(df_filings_labeled)
df_transcripts_labeled = pd.concat(df_transcripts_labeled)

df_statistic_resources = pd.concat([df_news_labeled, df_filings_labeled, df_transcripts_labeled])
create_styled_table(df_statistic_resources, title='Retrieved Document Count by Company and Document Type', companies_list = company_names)

df_statistic_resources_relevant = df_statistic_resources.loc[~df_statistic_resources.label.isin(['', 'unassigned', 'unclear'])]
create_styled_table(df_statistic_resources_relevant, title='Relevant Document Count by Company and Document Type', companies_list = company_names)

# Run the process to summarize the documents and compute media attention by topic, sector-wide
summarizer_sector = TopicSummarizerSector(
   model=llm_model.split('::')[1],
   api_key=OPENAI_API_KEY,
   df_labeled=df_news_labeled,
   general_theme=general_theme,
   list_specific_focus=list_specific_focus,
   themes_tree_dict=themes_tree_dict,
   logger=GenerateReport.logger
)
df_by_theme = summarizer_sector.summarize()

# Run the process to summarize the documents and score media attention, risk and uncertainty by topic at company level
summarizer_company = TopicSummarizerCompany(
   model=llm_model.split('::')[1],
   api_key=OPENAI_API_KEY,
   logger=GenerateReport.logger,
   verbose=True
)
df_by_company = asyncio.run(
   summarizer_company.process_topic_by_company(
       df_labeled=df_news_labeled,
       list_entities=companies
   )
)

# Concatenate Filings and Transcripts dataframes
df_filings_labeled['doc_type'] = 'Filings'
df_transcripts_labeled['doc_type'] = 'Transcripts'
df_ft_labeled = pd.concat([df_filings_labeled, df_transcripts_labeled])
df_ft_labeled = df_ft_labeled.reset_index(drop=True)

# Run the process to extract company's mitigation plan from the documents (filings and transcripts)
response_processor = CompanyResponseProcessor(
   model=llm_model.split('::')[1],
   api_key=OPENAI_API_KEY,
   logger=GenerateReport.logger,
   verbose=True
)

df_response_by_company = asyncio.run(
   response_processor.process_response_by_company(
       df_labeled=df_ft_labeled,
       df_by_company=df_by_company,
       list_entities=companies
   )
)

# Merge the companies responses to the dataframe with issue summaries and scores
df_by_company_with_responses = pd.merge(df_by_company, df_response_by_company, on=['entity_id', 'entity_name', 'topic'], how='left')
df_by_company_with_responses['filings_response_summary'] = df_by_company_with_responses['response_summary']

# Extract the company's mitigation plan for each regulatory issue from the News documents
df_news_response_by_company = asyncio.run(
   response_processor.process_response_by_company(
       df_labeled=df_news_labeled,
       df_by_company=df_by_company,
       list_entities=companies
   )
)

df_news_response_by_company = df_news_response_by_company.rename(
   columns={'response_summary': 'news_response_summary', 'n_response_documents': 'news_n_response_documents'}
)
df_by_company_with_responses = pd.merge(df_by_company_with_responses, df_news_response_by_company,
                                       on=['entity_id', 'entity_name', 'topic'], how='left')

report_by_theme = df_by_theme
report_by_company = df_by_company_with_responses

# Generate the html report
top_by_theme, top_by_company = prepare_data_report_0(
     df_by_theme = df_by_theme,
     df_by_company_with_responses = df_by_company_with_responses,
     user_selected_nb_topics_themes = 3,
)

html_content = generate_html_report(top_by_theme, top_by_company, 'Regulatory Issues in the Tech Sector')

with open(output_dir+'/report.html', 'w') as file:
     file.write(html_content)

display(HTML(html_content))

try:
    # Create the Excel manager
    excel_manager = ExcelManager()

    # Define the dataframes and their sheet configurations
    df_args = [
        (df_by_company_with_responses, "Report Regulatory Issues by companies", (2, 3)),
        (df_by_theme, "Report Regulatory Issues by theme", (1, 1))
    ]

    # Save the workbook
    excel_manager.save_workbook(df_args, export_path)

except Exception as e:
    print(f"Warning while exporting to excel: {e}")