Code reorganisation to better use config files. Adapted code to use Gemma 3 as local model. Minor package updates

.dockerignore

@@ -14,4 +14,11 @@ dist/*
 logs/*
 usage/*
 feedback/*
-test_code/*
+test_code/*
+input/
+output/
+logs/
+usage/
+feedback/
+config/
+tmp/

.gitignore

@@ -6,6 +6,7 @@
 *.xls
 *.xlsx
 *.csv
+*.pyc
 examples/*
 output/*
 tools/__pycache__/*
@@ -14,4 +15,6 @@ dist/*
 logs/*
 usage/*
 feedback/*
-test_code/*
+test_code/*
+config/*
+tmp/*

Dockerfile

@@ -7,6 +7,8 @@ RUN apt-get update && apt-get install -y \
     gcc \
     g++ \
     cmake \
+    libopenblas-dev \
+    pkg-config \
     python3-dev \
     libffi-dev \
     && apt-get clean \
@@ -14,11 +16,15 @@ RUN apt-get update && apt-get install -y \
 
 WORKDIR /src
 
+# Optional: Set environment variables for OpenBLAS
+ENV OPENBLAS_VERBOSE=1
+ENV CMAKE_ARGS="-DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS"
+
 COPY requirements_aws.txt .
 
 RUN pip uninstall -y typing_extensions \
 && pip install --no-cache-dir --target=/install typing_extensions==4.12.2 \
-&& pip install --no-cache-dir --target=/install torch==2.6.0+cpu --extra-index-url https://download.pytorch.org/whl/cpu \
+&& pip install --no-cache-dir --target=/install torch==2.7.1+cpu --extra-index-url https://download.pytorch.org/whl/cpu \
 && pip install --no-cache-dir --target=/install -r requirements_aws.txt
 
 RUN rm requirements_aws.txt

README.md

@@ -6,7 +6,7 @@ colorTo: yellow
 sdk: gradio
 app_file: app.py
 pinned: true
-license: cc-by-nc-4.0
+license: agpl-3.0
 ---
 
 # Large language model topic modelling

app.py

@@ -1,38 +1,33 @@
 import os
-import socket
-import spaces
-from tools.helper_functions import ensure_output_folder_exists, add_folder_to_path, put_columns_in_df, get_connection_params, output_folder, get_or_create_env_var, reveal_feedback_buttons, wipe_logs, model_full_names, view_table, empty_output_vars_extract_topics, empty_output_vars_summarise, RUN_LOCAL_MODEL, load_in_previous_reference_file, join_cols_onto_reference_df, GEMINI_API_KEY
-from tools.aws_functions import upload_file_to_s3, RUN_AWS_FUNCTIONS
-from tools.llm_api_call import extract_topics, load_in_data_file, load_in_previous_data_files, sample_reference_table_summaries, summarise_output_topics, batch_size_default, deduplicate_topics, modify_existing_output_tables
-from tools.auth import authenticate_user
-from tools.prompts import initial_table_prompt, prompt2, prompt3, system_prompt, add_existing_topics_system_prompt, add_existing_topics_prompt, verify_titles_prompt, verify_titles_system_prompt, two_para_summary_format_prompt, single_para_summary_format_prompt
-from tools.verify_titles import verify_titles
-#from tools.aws_functions import load_data_from_aws
 import gradio as gr
 import pandas as pd
 from datetime import datetime
+from tools.helper_functions import put_columns_in_df, get_connection_params, get_or_create_env_var, reveal_feedback_buttons, wipe_logs, view_table, empty_output_vars_extract_topics, empty_output_vars_summarise, load_in_previous_reference_file, join_cols_onto_reference_df
+from tools.aws_functions import upload_file_to_s3
+from tools.llm_api_call import extract_topics, load_in_data_file, load_in_previous_data_files,  modify_existing_output_tables
+from tools.dedup_summaries import sample_reference_table_summaries, summarise_output_topics, deduplicate_topics
+from tools.auth import authenticate_user
+from tools.prompts import initial_table_prompt, prompt2, prompt3, system_prompt, add_existing_topics_system_prompt, add_existing_topics_prompt, verify_titles_prompt, verify_titles_system_prompt, two_para_summary_format_prompt, single_para_summary_format_prompt
+from tools.verify_titles import verify_titles
+from tools.config import RUN_AWS_FUNCTIONS, HOST_NAME, ACCESS_LOGS_FOLDER, FEEDBACK_LOGS_FOLDER, USAGE_LOGS_FOLDER, RUN_LOCAL_MODEL, AWS_USER_POOL_ID, FILE_INPUT_HEIGHT, GEMINI_API_KEY, model_full_names, BATCH_SIZE_DEFAULT, CHOSEN_LOCAL_MODEL_TYPE
 
 today_rev = datetime.now().strftime("%Y%m%d")
 
-ensure_output_folder_exists()
-
-host_name = socket.gethostname()
-# print("host_name is:", host_name)
-
-access_logs_data_folder = 'logs/' + today_rev + '/' + host_name + '/'
-feedback_data_folder = 'feedback/' + today_rev + '/' + host_name + '/'
-usage_data_folder = 'usage/' + today_rev + '/' + host_name + '/'
-file_input_height = 200
+host_name = HOST_NAME
+access_logs_data_folder = ACCESS_LOGS_FOLDER
+feedback_data_folder = FEEDBACK_LOGS_FOLDER
+usage_data_folder = USAGE_LOGS_FOLDER
+file_input_height = FILE_INPUT_HEIGHT
 
 if RUN_LOCAL_MODEL == "1":
-    default_model_choice = "gemma_2b_it_local"
+    default_model_choice = CHOSEN_LOCAL_MODEL_TYPE
 elif RUN_AWS_FUNCTIONS == "1":
     default_model_choice = "anthropic.claude-3-haiku-20240307-v1:0"
 else:
     default_model_choice = "gemini-2.0-flash-001"
 
 # Create the gradio interface
-app = gr.Blocks(theme = gr.themes.Base())
+app = gr.Blocks(theme = gr.themes.Default(primary_hue="blue"), fill_width=True)
 
 with app:
 
@@ -53,11 +48,11 @@ with app:
     master_reference_df_state = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=0, row_count = (0, "dynamic"), label="master_reference_df_state", visible=False, type="pandas")
 
     master_modify_unique_topics_df_state = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=0, row_count = (0, "dynamic"), label="master_modify_unique_topics_df_state", visible=False, type="pandas")
-    master_modify_reference_df_state = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=0, row_count = (0, "dynamic"), label="master_modify_reference_df_state", visible=False, type="pandas")
-    
+    master_modify_reference_df_state = gr.Dataframe(value=pd.DataFrame(), headers=None, col_count=0, row_count = (0, "dynamic"), label="master_modify_reference_df_state", visible=False, type="pandas")    
  
     session_hash_state = gr.State()
-    s3_output_folder_state = gr.State()
+    output_folder_state = gr.State()
+    input_folder_state = gr.State()
 
     # Logging state
     log_file_name = 'log.csv'
@@ -222,7 +217,7 @@ with app:
         """)
         with gr.Accordion("Settings for LLM generation", open = True):
             temperature_slide = gr.Slider(minimum=0.1, maximum=1.0, value=0.1, label="Choose LLM temperature setting")
-            batch_size_number = gr.Number(label = "Number of responses to submit in a single LLM query", value = batch_size_default, precision=0, minimum=1, maximum=100)
+            batch_size_number = gr.Number(label = "Number of responses to submit in a single LLM query", value = BATCH_SIZE_DEFAULT, precision=0, minimum=1, maximum=100)
             random_seed = gr.Number(value=42, label="Random seed for LLM generation", visible=False)            
 
         with gr.Accordion("Prompt settings", open = False):
@@ -279,13 +274,13 @@ with app:
     success(load_in_data_file,                           
         inputs = [in_data_files, in_colnames, batch_size_number, in_excel_sheets], outputs = [file_data_state, reference_data_file_name_textbox, total_number_of_batches], api_name="load_data").\
     success(fn=extract_topics,                           
-        inputs=[in_data_files, file_data_state, master_topic_df_state, master_reference_df_state, master_unique_topics_df_state, display_topic_table_markdown, reference_data_file_name_textbox, total_number_of_batches, in_api_key, temperature_slide, in_colnames, model_choice, candidate_topics, latest_batch_completed, display_topic_table_markdown, text_output_file_list_state, log_files_output_list_state, first_loop_state, conversation_metadata_textbox, initial_table_prompt_textbox, prompt_2_textbox, prompt_3_textbox, system_prompt_textbox, add_to_existing_topics_system_prompt_textbox, add_to_existing_topics_prompt_textbox, number_of_prompts, batch_size_number, context_textbox, estimated_time_taken_number, sentiment_checkbox, force_zero_shot_radio, in_excel_sheets, force_single_topic_radio],        
+        inputs=[in_data_files, file_data_state, master_topic_df_state, master_reference_df_state, master_unique_topics_df_state, display_topic_table_markdown, reference_data_file_name_textbox, total_number_of_batches, in_api_key, temperature_slide, in_colnames, model_choice, candidate_topics, latest_batch_completed, display_topic_table_markdown, text_output_file_list_state, log_files_output_list_state, first_loop_state, conversation_metadata_textbox, initial_table_prompt_textbox, prompt_2_textbox, prompt_3_textbox, system_prompt_textbox, add_to_existing_topics_system_prompt_textbox, add_to_existing_topics_prompt_textbox, number_of_prompts, batch_size_number, context_textbox, estimated_time_taken_number, sentiment_checkbox, force_zero_shot_radio, in_excel_sheets, force_single_topic_radio, output_folder_state],        
         outputs=[display_topic_table_markdown, master_topic_df_state, master_unique_topics_df_state, master_reference_df_state, topic_extraction_output_files, text_output_file_list_state, latest_batch_completed, log_files_output, log_files_output_list_state, conversation_metadata_textbox, estimated_time_taken_number, deduplication_input_files, summarisation_input_files, modifiable_unique_topics_df_state, modification_input_files, in_join_files], api_name="extract_topics")
     
     
     # If the output file count text box changes, keep going with redacting each data file until done. Then reveal the feedback buttons.
     # latest_batch_completed.change(fn=extract_topics,                                  
-    #     inputs=[in_data_files, file_data_state, master_topic_df_state, master_reference_df_state, master_unique_topics_df_state, display_topic_table_markdown, reference_data_file_name_textbox, total_number_of_batches, in_api_key, temperature_slide, in_colnames, model_choice, candidate_topics, latest_batch_completed, display_topic_table_markdown, text_output_file_list_state, log_files_output_list_state, second_loop_state, conversation_metadata_textbox, initial_table_prompt_textbox, prompt_2_textbox, prompt_3_textbox, system_prompt_textbox, add_to_existing_topics_system_prompt_textbox, add_to_existing_topics_prompt_textbox, number_of_prompts, batch_size_number, context_textbox, estimated_time_taken_number, sentiment_checkbox, force_zero_shot_radio, in_excel_sheets],
+    #     inputs=[in_data_files, file_data_state, master_topic_df_state, master_reference_df_state, master_unique_topics_df_state, display_topic_table_markdown, reference_data_file_name_textbox, total_number_of_batches, in_api_key, temperature_slide, in_colnames, model_choice, candidate_topics, latest_batch_completed, display_topic_table_markdown, text_output_file_list_state, log_files_output_list_state, second_loop_state, conversation_metadata_textbox, initial_table_prompt_textbox, prompt_2_textbox, prompt_3_textbox, system_prompt_textbox, add_to_existing_topics_system_prompt_textbox, add_to_existing_topics_prompt_textbox, number_of_prompts, batch_size_number, context_textbox, estimated_time_taken_number, sentiment_checkbox, force_zero_shot_radio, in_excel_sheets, force_single_topic_radio, output_folder_state],
     #     outputs=[display_topic_table_markdown, master_topic_df_state, master_unique_topics_df_state, master_reference_df_state, topic_extraction_output_files, text_output_file_list_state, latest_batch_completed, log_files_output, log_files_output_list_state, conversation_metadata_textbox, estimated_time_taken_number, deduplication_input_files, summarisation_input_files, modifiable_unique_topics_df_state, modification_input_files, in_join_files]).\
     #     success(fn = reveal_feedback_buttons,
     #         outputs=[data_feedback_radio, data_further_details_text, data_submit_feedback_btn, data_feedback_title], scroll_to_output=True)
@@ -293,24 +288,20 @@ with app:
     # If you upload data into the deduplication input box, the modifiable topic dataframe box is updated
     modification_input_files.change(fn=load_in_previous_data_files, inputs=[modification_input_files, modified_unique_table_change_bool], outputs=[modifiable_unique_topics_df_state, master_modify_reference_df_state, master_modify_unique_topics_df_state, reference_data_file_name_textbox, unique_topics_table_file_name_textbox, text_output_modify_file_list_state])
 
-    
-    
-   
-
     # Modify output table with custom topic names
-    save_modified_files_button.click(fn=modify_existing_output_tables, inputs=[master_modify_unique_topics_df_state, modifiable_unique_topics_df_state, master_modify_reference_df_state, text_output_modify_file_list_state], outputs=[master_unique_topics_df_state, master_reference_df_state, topic_extraction_output_files, text_output_file_list_state, deduplication_input_files, summarisation_input_files, reference_data_file_name_textbox, unique_topics_table_file_name_textbox, summarised_output_markdown])
+    save_modified_files_button.click(fn=modify_existing_output_tables, inputs=[master_modify_unique_topics_df_state, modifiable_unique_topics_df_state, master_modify_reference_df_state, text_output_modify_file_list_state, output_folder_state], outputs=[master_unique_topics_df_state, master_reference_df_state, topic_extraction_output_files, text_output_file_list_state, deduplication_input_files, summarisation_input_files, reference_data_file_name_textbox, unique_topics_table_file_name_textbox, summarised_output_markdown])
     
     # When button pressed, deduplicate data
     deduplicate_previous_data_btn.click(load_in_previous_data_files, inputs=[deduplication_input_files], outputs=[master_reference_df_state, master_unique_topics_df_state, latest_batch_completed_no_loop, deduplication_input_files_status, reference_data_file_name_textbox, unique_topics_table_file_name_textbox]).\
-        success(deduplicate_topics, inputs=[master_reference_df_state, master_unique_topics_df_state, reference_data_file_name_textbox, unique_topics_table_file_name_textbox, in_excel_sheets, merge_sentiment_drop, merge_general_topics_drop, deduplicate_score_threshold, in_data_files, in_colnames], outputs=[master_reference_df_state, master_unique_topics_df_state, summarisation_input_files, log_files_output, summarised_output_markdown], scroll_to_output=True)
+        success(deduplicate_topics, inputs=[master_reference_df_state, master_unique_topics_df_state, reference_data_file_name_textbox, unique_topics_table_file_name_textbox, in_excel_sheets, merge_sentiment_drop, merge_general_topics_drop, deduplicate_score_threshold, in_data_files, in_colnames, output_folder_state], outputs=[master_reference_df_state, master_unique_topics_df_state, summarisation_input_files, log_files_output, summarised_output_markdown], scroll_to_output=True, api_name="deduplicate_topics")
     
     # When button pressed, summarise previous data
     summarise_previous_data_btn.click(empty_output_vars_summarise, inputs=None, outputs=[summary_reference_table_sample_state, master_unique_topics_df_revised_summaries_state, master_reference_df_revised_summaries_state, summary_output_files, summarised_outputs_list, latest_summary_completed_num, conversation_metadata_textbox]).\
         success(load_in_previous_data_files, inputs=[summarisation_input_files], outputs=[master_reference_df_state, master_unique_topics_df_state, latest_batch_completed_no_loop, deduplication_input_files_status, reference_data_file_name_textbox, unique_topics_table_file_name_textbox]).\
-            success(sample_reference_table_summaries, inputs=[master_reference_df_state, master_unique_topics_df_state, random_seed], outputs=[summary_reference_table_sample_state, summarised_references_markdown, master_reference_df_state, master_unique_topics_df_state]).\
-                success(summarise_output_topics, inputs=[summary_reference_table_sample_state, master_unique_topics_df_state, master_reference_df_state, model_choice, in_api_key, summarised_references_markdown, temperature_slide, reference_data_file_name_textbox, summarised_outputs_list, latest_summary_completed_num, conversation_metadata_textbox, in_data_files, in_excel_sheets, in_colnames, log_files_output_list_state, summarise_format_radio], outputs=[summary_reference_table_sample_state, master_unique_topics_df_revised_summaries_state, master_reference_df_revised_summaries_state, summary_output_files, summarised_outputs_list, latest_summary_completed_num, conversation_metadata_textbox, summarised_output_markdown, log_files_output])
+            success(sample_reference_table_summaries, inputs=[master_reference_df_state, master_unique_topics_df_state, random_seed], outputs=[summary_reference_table_sample_state, summarised_references_markdown, master_reference_df_state, master_unique_topics_df_state], api_name="sample_summaries").\
+                success(summarise_output_topics, inputs=[summary_reference_table_sample_state, master_unique_topics_df_state, master_reference_df_state, model_choice, in_api_key, summarised_references_markdown, temperature_slide, reference_data_file_name_textbox, summarised_outputs_list, latest_summary_completed_num, conversation_metadata_textbox, in_data_files, in_excel_sheets, in_colnames, log_files_output_list_state, summarise_format_radio, output_folder_state], outputs=[summary_reference_table_sample_state, master_unique_topics_df_revised_summaries_state, master_reference_df_revised_summaries_state, summary_output_files, summarised_outputs_list, latest_summary_completed_num, conversation_metadata_textbox, summarised_output_markdown, log_files_output], api_name="summarise_topics")
 
-    latest_summary_completed_num.change(summarise_output_topics, inputs=[summary_reference_table_sample_state, master_unique_topics_df_state, master_reference_df_state, model_choice, in_api_key, summarised_references_markdown, temperature_slide, reference_data_file_name_textbox, summarised_outputs_list, latest_summary_completed_num, conversation_metadata_textbox, in_data_files, in_excel_sheets, in_colnames, log_files_output_list_state, summarise_format_radio], outputs=[summary_reference_table_sample_state, master_unique_topics_df_revised_summaries_state, master_reference_df_revised_summaries_state, summary_output_files, summarised_outputs_list, latest_summary_completed_num, conversation_metadata_textbox, summarised_output_markdown, log_files_output], scroll_to_output=True)
+    latest_summary_completed_num.change(summarise_output_topics, inputs=[summary_reference_table_sample_state, master_unique_topics_df_state, master_reference_df_state, model_choice, in_api_key, summarised_references_markdown, temperature_slide, reference_data_file_name_textbox, summarised_outputs_list, latest_summary_completed_num, conversation_metadata_textbox, in_data_files, in_excel_sheets, in_colnames, log_files_output_list_state, summarise_format_radio, output_folder_state], outputs=[summary_reference_table_sample_state, master_unique_topics_df_revised_summaries_state, master_reference_df_revised_summaries_state, summary_output_files, summarised_outputs_list, latest_summary_completed_num, conversation_metadata_textbox, summarised_output_markdown, log_files_output], scroll_to_output=True)
 
     # CONTINUE PREVIOUS TOPIC EXTRACTION PAGE
 
@@ -327,8 +318,8 @@ with app:
     verify_titles_btn.click(fn=empty_output_vars_extract_topics, inputs=None, outputs=[master_topic_df_state, master_unique_topics_df_state, master_reference_df_state, topic_extraction_output_files, text_output_file_list_state, latest_batch_completed, log_files_output, log_files_output_list_state, conversation_metadata_textbox, estimated_time_taken_number, file_data_state, reference_data_file_name_textbox, display_topic_table_markdown]).\
     success(load_in_data_file,
         inputs = [verify_in_data_files, verify_in_colnames, batch_size_number, verify_in_excel_sheets], outputs = [file_data_state, reference_data_file_name_textbox, total_number_of_batches], api_name="verify_load_data").\
-    success(fn=verify_titles,                           
-        inputs=[verify_in_data_files, file_data_state, master_topic_df_state, master_reference_df_state, master_unique_topics_df_state, display_topic_table_markdown, reference_data_file_name_textbox, total_number_of_batches, verify_in_api_key, temperature_slide, verify_in_colnames, verify_model_choice, candidate_topics, latest_batch_completed, display_topic_table_markdown, text_output_file_list_state, log_files_output_list_state, first_loop_state, conversation_metadata_textbox, verify_titles_prompt_textbox, prompt_2_textbox, prompt_3_textbox, verify_titles_system_prompt_textbox, verify_titles_system_prompt_textbox, verify_titles_prompt_textbox, number_of_prompts, batch_size_number, context_textbox, estimated_time_taken_number, sentiment_checkbox, force_zero_shot_radio, in_excel_sheets],        
+    success(fn=verify_titles,
+        inputs=[verify_in_data_files, file_data_state, master_topic_df_state, master_reference_df_state, master_unique_topics_df_state, display_topic_table_markdown, reference_data_file_name_textbox, total_number_of_batches, verify_in_api_key, temperature_slide, verify_in_colnames, verify_model_choice, candidate_topics, latest_batch_completed, display_topic_table_markdown, text_output_file_list_state, log_files_output_list_state, first_loop_state, conversation_metadata_textbox, verify_titles_prompt_textbox, prompt_2_textbox, prompt_3_textbox, verify_titles_system_prompt_textbox, verify_titles_system_prompt_textbox, verify_titles_prompt_textbox, number_of_prompts, batch_size_number, context_textbox, estimated_time_taken_number, sentiment_checkbox, force_zero_shot_radio, in_excel_sheets, output_folder_state],        
         outputs=[verify_display_topic_table_markdown, master_topic_df_state, master_unique_topics_df_state, master_reference_df_state, verify_titles_file_output, text_output_file_list_state, latest_batch_completed, log_files_output, log_files_output_list_state, conversation_metadata_textbox, estimated_time_taken_number, deduplication_input_files, summarisation_input_files, modifiable_unique_topics_df_state, verify_modification_input_files_placeholder], api_name="verify_descriptions")
     
     ###
@@ -346,7 +337,7 @@ with app:
     ###
     # LOGGING AND ON APP LOAD FUNCTIONS
     ###
-    app.load(get_connection_params, inputs=None, outputs=[session_hash_state, s3_output_folder_state, session_hash_textbox])
+    app.load(get_connection_params, inputs=None, outputs=[session_hash_state, output_folder_state, session_hash_textbox, input_folder_state])
 
     # Log usernames and times of access to file (to know who is using the app when running on AWS)
     access_callback = gr.CSVLogger(dataset_file_name=log_file_name)

requirements.txt

@@ -1,7 +1,7 @@
 pandas==2.2.3
-gradio==5.32.0
+gradio==5.34.2
 spaces==0.34.1
-boto3==1.38.5
+boto3==1.38.38
 pyarrow==19.0.1
 openpyxl==3.1.3
 markdown==3.7
@@ -11,11 +11,13 @@ google-generativeai==0.8.4
 html5lib==1.1
 beautifulsoup4==4.12.3
 rapidfuzz==3.10.1
-torch==2.6.0 --extra-index-url https://download.pytorch.org/whl/cpu
-#llama-cpp-python==0.2.90 --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
-#https://github.com/abetlen/llama-cpp-python/releases/download/v0.3.2/llama_cpp_python-0.3.2-cp311-cp311-win_amd64.whl
-llama-cpp-python==0.3.2 --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
-#llama-cpp-python==0.3.8
+torch==2.7.1 --extra-index-url https://download.pytorch.org/whl/cpu
+llama-cpp-python==0.3.9 -C cmake.args="-DGGML_BLAS=ON;-DGGML_BLAS_VENDOR=OpenBLAS"# Linux compatibility - for recent models like Gemma 3
+# For Windows try the following
+# llama-cpp-python==0.3.9 -C cmake.args="-DGGML_BLAS=ON;-DGGML_BLAS_VENDOR=OpenBLAS;-DBLAS_INCLUDE_DIRS=C:/<root-path-to-openblas>/OpenBLAS/include;-DBLAS_LIBRARIES=C:/<root-path-to-openblas>/OpenBLAS/lib/libopenblas.lib
+#https://github.com/abetlen/llama-cpp-python/releases/download/v0.3.2/llama_cpp_python-0.3.2-cp311-cp311-win_amd64.whl # Use this for Windows if abov doesn't work, enough for Gemma 2b
+#llama-cpp-python==0.3.2 --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu # Use this for guaranteed Linux compatibility - enough for Gemma 2b only
 transformers==4.51.1
+python-dotenv==1.1.0
 numpy==1.26.4
 typing_extensions==4.12.2

requirements_aws.txt

@@ -1,7 +1,7 @@
 pandas==2.2.3
-gradio==5.32.0
+gradio==5.34.2
 spaces==0.34.1
-boto3==1.38.5
+boto3==1.38.38
 pyarrow==19.0.1
 openpyxl==3.1.3
 markdown==3.7
@@ -11,8 +11,9 @@ google-generativeai==0.8.4
 html5lib==1.1
 beautifulsoup4==4.12.3
 rapidfuzz==3.10.1
-#llama-cpp-python==0.2.90 --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
-llama-cpp-python==0.3.2 --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
+llama-cpp-python==0.3.9
+#llama-cpp-python==0.3.2 --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
 transformers==4.51.1
+python-dotenv==1.1.0
 numpy==1.26.4
 typing_extensions==4.12.2

requirements_gpu.txt

@@ -1,7 +1,7 @@
 pandas==2.2.3
-gradio==5.32.0
+gradio==5.34.2
 spaces==0.34.1
-boto3==1.38.5
+boto3==1.38.38
 pyarrow==19.0.1
 openpyxl==3.1.3
 markdown==3.7
@@ -11,11 +11,14 @@ google-generativeai==0.8.4
 html5lib==1.1
 beautifulsoup4==4.12.3
 rapidfuzz==3.10.1
-torch==2.6.0 --extra-index-url https://download.pytorch.org/whl/cu121
+torch==2.6.0 --extra-index-url https://download.pytorch.org/whl/cu124
+llama-cpp-python==0.3.9 -C cmake.args="-DGGML_CUDA=on"
+# If the above doesn't work, try one of the following
 #llama-cpp-python==0.3.4 --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu121
 # Specify exact llama_cpp wheel for huggingface compatibility
 #https://github.com/abetlen/llama-cpp-python/releases/download/v0.3.4-cu121/llama_cpp_python-0.3.4-cp311-cp311-linux_x86_64.whl
-https://github.com/abetlen/llama-cpp-python/releases/download/v0.3.4-cu121/llama_cpp_python-0.3.4-cp311-cp311-win_amd64.whl # Windows
+#https://github.com/abetlen/llama-cpp-python/releases/download/v0.3.4-cu121/llama_cpp_python-0.3.4-cp311-cp311-win_amd64.whl # Windows
 transformers==4.51.1
+python-dotenv==1.1.0
 numpy==1.26.4
 typing_extensions==4.12.2

tools/auth.py

@@ -1,14 +1,20 @@
-
 import boto3
-from tools.helper_functions import get_or_create_env_var
-
-client_id = get_or_create_env_var('AWS_CLIENT_ID', '')
-print(f'The value of AWS_CLIENT_ID is {client_id}')
+import hmac
+import hashlib
+import base64
+from tools.config import AWS_CLIENT_ID, AWS_CLIENT_SECRET, AWS_USER_POOL_ID, AWS_REGION
 
-user_pool_id = get_or_create_env_var('AWS_USER_POOL_ID', '')
-print(f'The value of AWS_USER_POOL_ID is {user_pool_id}')
+def calculate_secret_hash(client_id:str, client_secret:str, username:str):
+    message = username + client_id
+    dig = hmac.new(
+        str(client_secret).encode('utf-8'),
+        msg=str(message).encode('utf-8'),
+        digestmod=hashlib.sha256
+    ).digest()
+    secret_hash = base64.b64encode(dig).decode()
+    return secret_hash
 
-def authenticate_user(username, password, user_pool_id=user_pool_id, client_id=client_id):
+def authenticate_user(username:str, password:str, user_pool_id:str=AWS_USER_POOL_ID, client_id:str=AWS_CLIENT_ID, client_secret:str=AWS_CLIENT_SECRET):
     """Authenticates a user against an AWS Cognito user pool.
 
     Args:
@@ -16,22 +22,39 @@ def authenticate_user(username, password, user_pool_id=user_pool_id, client_id=c
         client_id (str): The ID of the Cognito user pool client.
         username (str): The username of the user.
         password (str): The password of the user.
+        client_secret (str): The client secret of the app client
 
     Returns:
         bool: True if the user is authenticated, False otherwise.
     """
 
-    client = boto3.client('cognito-idp')  # Cognito Identity Provider client
+    client = boto3.client('cognito-idp', region_name=AWS_REGION)  # Cognito Identity Provider client
+
+    # Compute the secret hash
+    secret_hash = calculate_secret_hash(client_id, client_secret, username)
 
     try:
-        response = client.initiate_auth(
+
+        if client_secret == '':
+            response = client.initiate_auth(
+                AuthFlow='USER_PASSWORD_AUTH',
+                AuthParameters={
+                    'USERNAME': username,
+                    'PASSWORD': password,
+                },
+                ClientId=client_id
+            )
+
+        else:
+            response = client.initiate_auth(
             AuthFlow='USER_PASSWORD_AUTH',
             AuthParameters={
                 'USERNAME': username,
                 'PASSWORD': password,
+                'SECRET_HASH': secret_hash
             },
             ClientId=client_id
-        )
+            )
 
         # If successful, you'll receive an AuthenticationResult in the response
         if response.get('AuthenticationResult'):
@@ -44,5 +67,7 @@ def authenticate_user(username, password, user_pool_id=user_pool_id, client_id=c
     except client.exceptions.UserNotFoundException:
         return False
     except Exception as e:
-        print(f"An error occurred: {e}")
-        return False
+        out_message = f"An error occurred: {e}"
+        print(out_message)
+        raise Exception(out_message)
+        return False 

tools/aws_functions.py

@@ -3,21 +3,13 @@ import pandas as pd
 import boto3
 import tempfile
 import os
-from tools.helper_functions import get_or_create_env_var, RUN_AWS_FUNCTIONS
+from tools.config import RUN_AWS_FUNCTIONS, AWS_REGION, CONSULTATION_SUMMARY_BUCKET
 
 PandasDataFrame = Type[pd.DataFrame]
 
 # Get AWS credentials if required
 bucket_name=""
 
-AWS_REGION = get_or_create_env_var('AWS_REGION', 'eu-west-2')
-print(f'The value of AWS_REGION is {AWS_REGION}')
-
-CONSULTATION_SUMMARY_BUCKET = get_or_create_env_var('CONSULTATION_SUMMARY_BUCKET', '')
-print(f'The value of AWS_REGION is {CONSULTATION_SUMMARY_BUCKET}')
-
-
-
 if RUN_AWS_FUNCTIONS == "1":
     try:
         bucket_name = CONSULTATION_SUMMARY_BUCKET
@@ -41,11 +33,12 @@ if RUN_AWS_FUNCTIONS == "1":
     try:
         assumed_role_arn, assumed_role_name = get_assumed_role_info()
 
-        print("Assumed Role ARN:", assumed_role_arn)
-        print("Assumed Role Name:", assumed_role_name)
+        #print("Assumed Role ARN:", assumed_role_arn)
+        #print("Assumed Role Name:", assumed_role_name)
 
-    except Exception as e:
-        
+        print("Successfully assumed role with AWS STS")
+
+    except Exception as e:        
         print(e)
 
 # Download direct from S3 - requires login credentials
@@ -113,57 +106,6 @@ def download_files_from_s3(bucket_name, s3_folder, local_folder, filenames):
         except Exception as e:
             print(f"Error downloading 's3://{bucket_name}/{object_key}':", e)
 
-def load_data_from_aws(in_aws_keyword_file, aws_password="", bucket_name=bucket_name):
-
-    temp_dir = tempfile.mkdtemp()
-    local_address_stub = temp_dir + '/doc-redaction/'
-    files = []
-
-    if not 'LAMBETH_BOROUGH_PLAN_PASSWORD' in os.environ:
-        out_message = "Can't verify password for dataset access. Do you have a valid AWS connection? Data not loaded."
-        return files, out_message
-    
-    if aws_password:
-        if "Lambeth borough plan" in in_aws_keyword_file and aws_password == os.environ['LAMBETH_BOROUGH_PLAN_PASSWORD']:
-
-            s3_folder_stub = 'example-data/lambeth-borough-plan/latest/'
-
-            local_folder_path = local_address_stub                
-
-            # Check if folder exists
-            if not os.path.exists(local_folder_path):
-                print(f"Folder {local_folder_path} does not exist! Making folder.")
-
-                os.mkdir(local_folder_path)
-
-            # Check if folder is empty
-            if len(os.listdir(local_folder_path)) == 0:
-                print(f"Folder {local_folder_path} is empty")
-                # Download data
-                download_files_from_s3(bucket_name, s3_folder_stub, local_folder_path, filenames='*')
-
-                print("AWS data downloaded")
-
-            else:
-                print(f"Folder {local_folder_path} is not empty")
-
-            #files = os.listdir(local_folder_stub)
-            #print(files)
-
-            files = [os.path.join(local_folder_path, f) for f in os.listdir(local_folder_path) if os.path.isfile(os.path.join(local_folder_path, f))]
-
-            out_message = "Data successfully loaded from AWS"
-            print(out_message)
-
-        else:
-            out_message = "Data not loaded from AWS"
-            print(out_message)
-    else:
-        out_message = "No password provided. Please ask the data team for access if you need this."
-        print(out_message)
-
-    return files, out_message
-
 def upload_file_to_s3(local_file_paths:List[str], s3_key:str, s3_bucket:str=bucket_name, RUN_AWS_FUNCTIONS=RUN_AWS_FUNCTIONS):
     """
     Uploads a file from local machine to Amazon S3.

tools/chatfuncs.py

@@ -1,240 +0,0 @@
-from typing import TypeVar
-import torch.cuda
-import os
-import time
-from llama_cpp import Llama
-from huggingface_hub import hf_hub_download
-from tools.helper_functions import RUN_LOCAL_MODEL
-
-torch.cuda.empty_cache()
-
-PandasDataFrame = TypeVar('pd.core.frame.DataFrame')
-
-model_type = None # global variable setup
-
-full_text = "" # Define dummy source text (full text) just to enable highlight function to load
-
-model = [] # Define empty list for model functions to run
-tokenizer = [] #[] # Define empty list for model functions to run
-
-local_model_type = "Gemma 2b"
-
-# Both models are loaded on app initialisation so that users don't have to wait for the models to be downloaded
-
-# Check for torch cuda
-print("Is CUDA enabled? ", torch.cuda.is_available())
-print("Is a CUDA device available on this computer?", torch.backends.cudnn.enabled)
-if torch.cuda.is_available():
-    torch_device = "cuda"
-    gpu_layers = -1
-    os.system("nvidia-smi")
-else: 
-    torch_device =  "cpu"
-    gpu_layers = 0
-
-print("Device used is: ", torch_device)
-    
-
-print("Running on device:", torch_device)
-threads = torch.get_num_threads() # 8
-print("CPU threads:", threads)
-
-temperature: float = 0.1
-top_k: int = 3
-top_p: float = 1
-repetition_penalty: float = 1.2 # Mild repetition penalty to prevent repeating table rows
-last_n_tokens: int = 512
-max_new_tokens: int = 4096 # 200
-seed: int = 42
-reset: bool = True
-stream: bool = False
-threads: int = threads
-batch_size:int = 256
-context_length:int = 16384
-sample = True
-
-
-class llama_cpp_init_config_gpu:
-    def __init__(self,
-                 last_n_tokens=last_n_tokens,
-                 seed=seed,
-                 n_threads=threads,
-                 n_batch=batch_size,
-                 n_ctx=context_length,
-                 n_gpu_layers=gpu_layers):
-
-        self.last_n_tokens = last_n_tokens
-        self.seed = seed
-        self.n_threads = n_threads
-        self.n_batch = n_batch
-        self.n_ctx = n_ctx
-        self.n_gpu_layers = n_gpu_layers
-        # self.stop: list[str] = field(default_factory=lambda: [stop_string])
-
-    def update_gpu(self, new_value):
-        self.n_gpu_layers = new_value
-
-    def update_context(self, new_value):
-        self.n_ctx = new_value
-
-class llama_cpp_init_config_cpu(llama_cpp_init_config_gpu):
-    def __init__(self):
-        super().__init__()
-        self.n_gpu_layers = gpu_layers
-        self.n_ctx=context_length
-
-gpu_config = llama_cpp_init_config_gpu()
-cpu_config = llama_cpp_init_config_cpu()
-
-
-class LlamaCPPGenerationConfig:
-    def __init__(self, temperature=temperature,
-                 top_k=top_k,
-                 top_p=top_p,
-                 repeat_penalty=repetition_penalty,
-                 seed=seed,
-                 stream=stream,
-                 max_tokens=max_new_tokens
-                 ):
-        self.temperature = temperature
-        self.top_k = top_k
-        self.top_p = top_p
-        self.repeat_penalty = repeat_penalty
-        self.seed = seed
-        self.max_tokens=max_tokens
-        self.stream = stream
-
-    def update_temp(self, new_value):
-        self.temperature = new_value
-
-###
-# Load local model
-###
-def get_model_path():
-    repo_id = os.environ.get("REPO_ID", "lmstudio-community/gemma-2-2b-it-GGUF")# "bartowski/Llama-3.2-3B-Instruct-GGUF") # "lmstudio-community/gemma-2-2b-it-GGUF")#"QuantFactory/Phi-3-mini-128k-instruct-GGUF")
-    filename = os.environ.get("MODEL_FILE", "gemma-2-2b-it-Q8_0.gguf") # )"Llama-3.2-3B-Instruct-Q5_K_M.gguf") #"gemma-2-2b-it-Q8_0.gguf") #"Phi-3-mini-128k-instruct.Q4_K_M.gguf")
-    model_dir = "model/gemma" #"model/phi"  # Assuming this is your intended directory
-
-    # Construct the expected local path
-    local_path = os.path.join(model_dir, filename)
-
-    if os.path.exists(local_path):
-        print(f"Model already exists at: {local_path}")
-        return local_path
-    else:
-        print(f"Checking default Hugging Face folder. Downloading model from Hugging Face Hub if not found")
-        return hf_hub_download(repo_id=repo_id, filename=filename)
-
-def load_model(local_model_type:str=local_model_type, gpu_layers:int=gpu_layers, max_context_length:int=context_length, gpu_config:llama_cpp_init_config_gpu=gpu_config, cpu_config:llama_cpp_init_config_cpu=cpu_config, torch_device:str=torch_device):
-    '''
-    Load in a model from Hugging Face hub via the transformers package, or using llama_cpp_python by downloading a GGUF file from Huggingface Hub. 
-    '''
-    print("Loading model ", local_model_type)
-
-    if local_model_type == "Gemma 2b":
-        if torch_device == "cuda":
-            gpu_config.update_gpu(gpu_layers)
-            gpu_config.update_context(max_context_length)
-            print("Loading with", gpu_config.n_gpu_layers, "model layers sent to GPU. And a maximum context length of ", gpu_config.n_ctx)
-        else:
-            gpu_config.update_gpu(gpu_layers)
-            cpu_config.update_gpu(gpu_layers)
-
-            # Update context length according to slider
-            gpu_config.update_context(max_context_length)
-            cpu_config.update_context(max_context_length)
-
-            print("Loading with", cpu_config.n_gpu_layers, "model layers sent to GPU. And a maximum context length of ", gpu_config.n_ctx)
-
-        #print(vars(gpu_config))
-        #print(vars(cpu_config))
-            
-        model_path = get_model_path()        
-
-        try:
-            print("GPU load variables:" , vars(gpu_config))
-            llama_model = Llama(model_path=model_path, **vars(gpu_config)) #  type_k=8, type_v = 8, flash_attn=True, 
-        
-        except Exception as e:
-            print("GPU load failed")
-            print(e)
-            llama_model = Llama(model_path=model_path, type_k=8, **vars(cpu_config)) # type_v = 8, flash_attn=True, 
-
-        tokenizer = []
-
-    model = llama_model
-    tokenizer = tokenizer
-    local_model_type = local_model_type
-
-    load_confirmation = "Finished loading model: " + local_model_type
-
-    print(load_confirmation)
-    return model, tokenizer
-
-
-def call_llama_cpp_model(formatted_string:str, gen_config:str, model=model):
-    """
-    Calls your generation model with parameters from the LlamaCPPGenerationConfig object.
-
-    Args:
-        formatted_string (str): The formatted input text for the model.
-        gen_config (LlamaCPPGenerationConfig): An object containing generation parameters.
-    """
-    # Extracting parameters from the gen_config object
-    temperature = gen_config.temperature
-    top_k = gen_config.top_k
-    top_p = gen_config.top_p
-    repeat_penalty = gen_config.repeat_penalty
-    seed = gen_config.seed
-    max_tokens = gen_config.max_tokens
-    stream = gen_config.stream
-
-    # Now you can call your model directly, passing the parameters:
-    output = model(
-        formatted_string, 
-        temperature=temperature, 
-        top_k=top_k, 
-        top_p=top_p, 
-        repeat_penalty=repeat_penalty, 
-        seed=seed,
-        max_tokens=max_tokens,
-        stream=stream#,
-        #stop=["<|eot_id|>", "\n\n"]
-    )
-
-    return output
-
-
-# This function is not used in this app
-def llama_cpp_streaming(history, full_prompt, temperature=temperature):
-
-    gen_config = LlamaCPPGenerationConfig()
-    gen_config.update_temp(temperature)
-
-    print(vars(gen_config))
-
-    # Pull the generated text from the streamer, and update the model output.
-    start = time.time()
-    NUM_TOKENS=0
-    print('-'*4+'Start Generation'+'-'*4)
-
-    output = model(
-    full_prompt, **vars(gen_config))
-
-    history[-1][1] = ""
-    for out in output:
-
-        if "choices" in out and len(out["choices"]) > 0 and "text" in out["choices"][0]:
-            history[-1][1] += out["choices"][0]["text"]
-            NUM_TOKENS+=1
-            yield history
-        else:
-            print(f"Unexpected output structure: {out}") 
-
-    time_generate = time.time() - start
-    print('\n')
-    print('-'*4+'End Generation'+'-'*4)
-    print(f'Num of generated tokens: {NUM_TOKENS}')
-    print(f'Time for complete generation: {time_generate}s')
-    print(f'Tokens per secound: {NUM_TOKENS/time_generate}')
-    print(f'Time per token: {(time_generate/NUM_TOKENS)*1000}ms')

tools/config.py

@@ -0,0 +1,327 @@
+import os
+import tempfile
+import socket
+import logging
+from datetime import datetime
+from dotenv import load_dotenv
+
+today_rev = datetime.now().strftime("%Y%m%d")
+HOST_NAME = socket.gethostname()
+
+# Set or retrieve configuration variables for the redaction app
+
+def get_or_create_env_var(var_name:str, default_value:str, print_val:bool=False):
+    '''
+    Get an environmental variable, and set it to a default value if it doesn't exist
+    '''
+    # Get the environment variable if it exists
+    value = os.environ.get(var_name)
+    
+    # If it doesn't exist, set the environment variable to the default value
+    if value is None:
+        os.environ[var_name] = default_value
+        value = default_value
+
+    if print_val == True:
+        print(f'The value of {var_name} is {value}')
+    
+    return value
+
+def ensure_folder_exists(output_folder:str):
+    """Checks if the specified folder exists, creates it if not."""   
+
+    if not os.path.exists(output_folder):
+        # Create the folder if it doesn't exist
+        os.makedirs(output_folder, exist_ok=True)
+        print(f"Created the {output_folder} folder.")
+    else:
+        print(f"The {output_folder} folder already exists.")
+
+def add_folder_to_path(folder_path: str):
+    '''
+    Check if a folder exists on your system. If so, get the absolute path and then add it to the system Path variable if it doesn't already exist. Function is only relevant for locally-created executable files based on this app (when using pyinstaller it creates a _internal folder that contains tesseract and poppler. These need to be added to the system path to enable the app to run)
+    '''
+
+    if os.path.exists(folder_path) and os.path.isdir(folder_path):
+        print(folder_path, "folder exists.")
+
+        # Resolve relative path to absolute path
+        absolute_path = os.path.abspath(folder_path)
+
+        current_path = os.environ['PATH']
+        if absolute_path not in current_path.split(os.pathsep):
+            full_path_extension = absolute_path + os.pathsep + current_path
+            os.environ['PATH'] = full_path_extension
+            #print(f"Updated PATH with: ", full_path_extension)
+        else:
+            print(f"Directory {folder_path} already exists in PATH.")
+    else:
+        print(f"Folder not found at {folder_path} - not added to PATH")
+
+
+###
+# LOAD CONFIG FROM ENV FILE
+###
+
+CONFIG_FOLDER = get_or_create_env_var('CONFIG_FOLDER', 'config/')
+
+ensure_folder_exists(CONFIG_FOLDER)
+
+# If you have an aws_config env file in the config folder, you can load in app variables this way, e.g. 'config/app_config.env'
+APP_CONFIG_PATH = get_or_create_env_var('APP_CONFIG_PATH', CONFIG_FOLDER + 'app_config.env') # e.g. config/app_config.env
+
+if APP_CONFIG_PATH:
+    if os.path.exists(APP_CONFIG_PATH):
+        print(f"Loading app variables from config file {APP_CONFIG_PATH}")
+        load_dotenv(APP_CONFIG_PATH)
+    else: print("App config file not found at location:", APP_CONFIG_PATH)
+
+###
+# AWS OPTIONS
+###
+
+# If you have an aws_config env file in the config folder, you can load in AWS keys this way, e.g. 'env/aws_config.env'
+AWS_CONFIG_PATH = get_or_create_env_var('AWS_CONFIG_PATH', '') # e.g. config/aws_config.env
+
+if AWS_CONFIG_PATH:
+    if os.path.exists(AWS_CONFIG_PATH):
+        print(f"Loading AWS variables from config file {AWS_CONFIG_PATH}")
+        load_dotenv(AWS_CONFIG_PATH)
+    else: print("AWS config file not found at location:", AWS_CONFIG_PATH)
+
+RUN_AWS_FUNCTIONS = get_or_create_env_var("RUN_AWS_FUNCTIONS", "1")
+
+AWS_REGION = get_or_create_env_var('AWS_REGION', '')
+
+AWS_CLIENT_ID = get_or_create_env_var('AWS_CLIENT_ID', '')
+
+AWS_CLIENT_SECRET = get_or_create_env_var('AWS_CLIENT_SECRET', '')
+
+AWS_USER_POOL_ID = get_or_create_env_var('AWS_USER_POOL_ID', '')
+
+AWS_ACCESS_KEY = get_or_create_env_var('AWS_ACCESS_KEY', '')
+if AWS_ACCESS_KEY: print(f'AWS_ACCESS_KEY found in environment variables')
+
+AWS_SECRET_KEY = get_or_create_env_var('AWS_SECRET_KEY', '')
+if AWS_SECRET_KEY: print(f'AWS_SECRET_KEY found in environment variables')
+
+CONSULTATION_SUMMARY_BUCKET = get_or_create_env_var('CONSULTATION_SUMMARY_BUCKET', '')
+
+# Custom headers e.g. if routing traffic through Cloudfront
+# Retrieving or setting CUSTOM_HEADER
+CUSTOM_HEADER = get_or_create_env_var('CUSTOM_HEADER', '')
+
+# Retrieving or setting CUSTOM_HEADER_VALUE
+CUSTOM_HEADER_VALUE = get_or_create_env_var('CUSTOM_HEADER_VALUE', '')
+
+###
+# File I/O
+###
+SESSION_OUTPUT_FOLDER = get_or_create_env_var('SESSION_OUTPUT_FOLDER', 'False') # i.e. do you want your input and output folders saved within a subfolder based on session hash value within output/input folders 
+
+OUTPUT_FOLDER = get_or_create_env_var('GRADIO_OUTPUT_FOLDER', 'output/') # 'output/'
+INPUT_FOLDER = get_or_create_env_var('GRADIO_INPUT_FOLDER', 'input/') # 'input/'
+
+ensure_folder_exists(OUTPUT_FOLDER)
+ensure_folder_exists(INPUT_FOLDER)
+
+# Allow for files to be saved in a temporary folder for increased security in some instances
+if OUTPUT_FOLDER == "TEMP" or INPUT_FOLDER == "TEMP": 
+    # Create a temporary directory
+    with tempfile.TemporaryDirectory() as temp_dir:
+        print(f'Temporary directory created at: {temp_dir}')
+
+        if OUTPUT_FOLDER == "TEMP": OUTPUT_FOLDER = temp_dir + "/"
+        if INPUT_FOLDER == "TEMP": INPUT_FOLDER = temp_dir + "/"
+
+
+GRADIO_TEMP_DIR = get_or_create_env_var('GRADIO_TEMP_DIR', 'tmp/gradio_tmp/') # Default Gradio temp folder
+MPLCONFIGDIR = get_or_create_env_var('MPLCONFIGDIR', 'tmp/matplotlib_cache/') # Matplotlib cache folder
+
+ensure_folder_exists(GRADIO_TEMP_DIR)
+ensure_folder_exists(MPLCONFIGDIR)
+
+# TLDEXTRACT_CACHE = get_or_create_env_var('TLDEXTRACT_CACHE', 'tmp/tld/')
+# try:
+#     extract = TLDExtract(cache_dir=TLDEXTRACT_CACHE)
+# except:
+#     extract = TLDExtract(cache_dir=None)
+
+###
+# LOGGING OPTIONS
+###
+
+# By default, logs are put into a subfolder of today's date and the host name of the instance running the app. This is to avoid at all possible the possibility of log files from one instance overwriting the logs of another instance on S3. If running the app on one system always, or just locally, it is not necessary to make the log folders so specific.
+# Another way to address this issue would be to write logs to another type of storage, e.g. database such as dynamodb. I may look into this in future.
+
+SAVE_LOGS_TO_CSV = get_or_create_env_var('SAVE_LOGS_TO_CSV', 'True')
+
+USE_LOG_SUBFOLDERS = get_or_create_env_var('USE_LOG_SUBFOLDERS', 'True')
+
+if USE_LOG_SUBFOLDERS == "True":
+    day_log_subfolder = today_rev + '/'
+    host_name_subfolder = HOST_NAME + '/'
+    full_log_subfolder = day_log_subfolder + host_name_subfolder
+else:
+    full_log_subfolder = ""
+
+FEEDBACK_LOGS_FOLDER = get_or_create_env_var('FEEDBACK_LOGS_FOLDER', 'feedback/' + full_log_subfolder)
+ACCESS_LOGS_FOLDER = get_or_create_env_var('ACCESS_LOGS_FOLDER', 'logs/' + full_log_subfolder)
+USAGE_LOGS_FOLDER = get_or_create_env_var('USAGE_LOGS_FOLDER', 'usage/' + full_log_subfolder)
+
+ensure_folder_exists(FEEDBACK_LOGS_FOLDER)
+ensure_folder_exists(ACCESS_LOGS_FOLDER)
+ensure_folder_exists(USAGE_LOGS_FOLDER)
+
+# Should the redacted file name be included in the logs? In some instances, the names of the files themselves could be sensitive, and should not be disclosed beyond the app. So, by default this is false.
+DISPLAY_FILE_NAMES_IN_LOGS = get_or_create_env_var('DISPLAY_FILE_NAMES_IN_LOGS', 'False')
+
+# Further customisation options for CSV logs
+
+CSV_ACCESS_LOG_HEADERS = get_or_create_env_var('CSV_ACCESS_LOG_HEADERS', '') # If blank, uses component labels
+CSV_FEEDBACK_LOG_HEADERS = get_or_create_env_var('CSV_FEEDBACK_LOG_HEADERS', '') # If blank, uses component labels
+CSV_USAGE_LOG_HEADERS = get_or_create_env_var('CSV_USAGE_LOG_HEADERS', '["session_hash_textbox", "doc_full_file_name_textbox", "data_full_file_name_textbox", "actual_time_taken_number",	"total_page_count",	"textract_query_number", "pii_detection_method", "comprehend_query_number",  "cost_code", "textract_handwriting_signature", "host_name_textbox", "text_extraction_method", "is_this_a_textract_api_call"]') # If blank, uses component labels
+
+### DYNAMODB logs. Whether to save to DynamoDB, and the headers of the table
+
+SAVE_LOGS_TO_DYNAMODB = get_or_create_env_var('SAVE_LOGS_TO_DYNAMODB', 'False')
+
+ACCESS_LOG_DYNAMODB_TABLE_NAME = get_or_create_env_var('ACCESS_LOG_DYNAMODB_TABLE_NAME', 'redaction_access_log')
+DYNAMODB_ACCESS_LOG_HEADERS = get_or_create_env_var('DYNAMODB_ACCESS_LOG_HEADERS', '')
+
+FEEDBACK_LOG_DYNAMODB_TABLE_NAME = get_or_create_env_var('FEEDBACK_LOG_DYNAMODB_TABLE_NAME', 'redaction_feedback')
+DYNAMODB_FEEDBACK_LOG_HEADERS = get_or_create_env_var('DYNAMODB_FEEDBACK_LOG_HEADERS', '')
+
+USAGE_LOG_DYNAMODB_TABLE_NAME = get_or_create_env_var('USAGE_LOG_DYNAMODB_TABLE_NAME', 'redaction_usage')
+DYNAMODB_USAGE_LOG_HEADERS = get_or_create_env_var('DYNAMODB_USAGE_LOG_HEADERS', '')
+
+# Report logging to console?
+LOGGING = get_or_create_env_var('LOGGING', 'False')
+
+if LOGGING == 'True':
+    # Configure logging
+    logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+LOG_FILE_NAME = get_or_create_env_var('LOG_FILE_NAME', 'log.csv')
+
+###
+# LLM variables
+###
+
+MAX_TOKENS = int(get_or_create_env_var('MAX_TOKENS', '4096')) # Maximum number of output tokens
+TIMEOUT_WAIT = int(get_or_create_env_var('TIMEOUT_WAIT', '30')) # AWS now seems to have a 60 second minimum wait between API calls
+NUMBER_OF_RETRY_ATTEMPTS = int(get_or_create_env_var('NUMBER_OF_RETRY_ATTEMPTS', '5'))
+# Try up to 3 times to get a valid markdown table response with LLM calls, otherwise retry with temperature changed
+MAX_OUTPUT_VALIDATION_ATTEMPTS = int(get_or_create_env_var('MAX_OUTPUT_VALIDATION_ATTEMPTS', '3'))
+MAX_TIME_FOR_LOOP = int(get_or_create_env_var('MAX_TIME_FOR_LOOP', '99999'))
+BATCH_SIZE_DEFAULT = int(get_or_create_env_var('BATCH_SIZE_DEFAULT', '5'))
+DEDUPLICATION_THRESHOLD = int(get_or_create_env_var('DEDUPLICATION_THRESHOLD', '90'))
+MAX_COMMENT_CHARS = int(get_or_create_env_var('MAX_COMMENT_CHARS', '14000'))
+
+RUN_LOCAL_MODEL = get_or_create_env_var("RUN_LOCAL_MODEL", "1")
+RUN_GEMINI_MODELS = get_or_create_env_var("RUN_GEMINI_MODELS", "1")
+GEMINI_API_KEY = get_or_create_env_var('GEMINI_API_KEY', '')
+
+# Build up options for models
+
+model_full_names = []
+model_short_names = []
+
+CHOSEN_LOCAL_MODEL_TYPE = get_or_create_env_var("CHOSEN_LOCAL_MODEL_TYPE", "Gemma 3 4B") # Gemma 3 1B #  "Gemma 2b"
+
+if RUN_LOCAL_MODEL == "1" and CHOSEN_LOCAL_MODEL_TYPE:
+    model_full_names.append(CHOSEN_LOCAL_MODEL_TYPE)
+    model_short_names.append(CHOSEN_LOCAL_MODEL_TYPE)
+
+if RUN_AWS_FUNCTIONS == "1":
+    model_full_names.extend(["anthropic.claude-3-haiku-20240307-v1:0", "anthropic.claude-3-sonnet-20240229-v1:0"])
+    model_short_names.extend(["haiku", "sonnet"])
+
+if RUN_GEMINI_MODELS == "1":
+    model_full_names.extend(["gemini-2.0-flash-001", "gemini-2.5-flash-preview-05-20", "gemini-2.5-pro-exp-05-06" ]) # , # Gemini pro No longer available on free tier
+    model_short_names.extend(["gemini_flash_2", "gemini_flash_2.5", "gemini_pro"])
+
+print("model_short_names:", model_short_names)
+print("model_full_names:", model_full_names)
+
+model_name_map = {short: full for short, full in zip(model_full_names, model_short_names)}
+
+# HF token may or may not be needed for downloading models from Hugging Face
+HF_TOKEN = get_or_create_env_var('HF_TOKEN', '')
+
+GEMMA2_REPO_ID = get_or_create_env_var("GEMMA2_2B_REPO_ID", "lmstudio-community/gemma-2-2b-it-GGUF")# "bartowski/Llama-3.2-3B-Instruct-GGUF") # "lmstudio-community/gemma-2-2b-it-GGUF")#"QuantFactory/Phi-3-mini-128k-instruct-GGUF")
+GEMMA2_MODEL_FILE = get_or_create_env_var("GEMMA2_2B_MODEL_FILE", "gemma-2-2b-it-Q8_0.gguf") # )"Llama-3.2-3B-Instruct-Q5_K_M.gguf") #"gemma-2-2b-it-Q8_0.gguf") #"Phi-3-mini-128k-instruct.Q4_K_M.gguf")
+GEMMA2_MODEL_FOLDER = get_or_create_env_var("GEMMA2_2B_MODEL_FOLDER", "model/gemma") #"model/phi"  # Assuming this is your intended directory
+
+GEMMA3_REPO_ID = get_or_create_env_var("GEMMA3_REPO_ID", "ggml-org/gemma-3-1b-it-GGUF")# "bartowski/Llama-3.2-3B-Instruct-GGUF") # "lmstudio-community/gemma-2-2b-it-GGUF")#"QuantFactory/Phi-3-mini-128k-instruct-GGUF")
+GEMMA3_MODEL_FILE = get_or_create_env_var("GEMMA3_MODEL_FILE", "gemma-3-1b-it-Q8_0.gguf") # )"Llama-3.2-3B-Instruct-Q5_K_M.gguf") #"gemma-2-2b-it-Q8_0.gguf") #"Phi-3-mini-128k-instruct.Q4_K_M.gguf")
+GEMMA3_MODEL_FOLDER = get_or_create_env_var("GEMMA3_MODEL_FOLDER", "model/gemma")
+
+GEMMA3_4B_REPO_ID = get_or_create_env_var("GEMMA3_4B_REPO_ID", "ggml-org/gemma-3-4b-it-GGUF")# "bartowski/Llama-3.2-3B-Instruct-GGUF") # "lmstudio-community/gemma-2-2b-it-GGUF")#"QuantFactory/Phi-3-mini-128k-instruct-GGUF")
+GEMMA3_4B_MODEL_FILE = get_or_create_env_var("GEMMA3_4B_MODEL_FILE", "gemma-3-4b-it-Q4_K_M.gguf") # )"Llama-3.2-3B-Instruct-Q5_K_M.gguf") #"gemma-2-2b-it-Q8_0.gguf") #"Phi-3-mini-128k-instruct.Q4_K_M.gguf")
+GEMMA3_4B_MODEL_FOLDER = get_or_create_env_var("GEMMA3_4B_MODEL_FOLDER", "model/gemma3_4b")
+
+
+if CHOSEN_LOCAL_MODEL_TYPE == "Gemma 2b":
+    LOCAL_REPO_ID = GEMMA2_REPO_ID
+    LOCAL_MODEL_FILE = GEMMA2_MODEL_FILE
+    LOCAL_MODEL_FOLDER = GEMMA2_MODEL_FOLDER
+
+elif CHOSEN_LOCAL_MODEL_TYPE == "Gemma 3 1B":
+    LOCAL_REPO_ID = GEMMA3_REPO_ID
+    LOCAL_MODEL_FILE = GEMMA3_MODEL_FILE
+    LOCAL_MODEL_FOLDER = GEMMA3_MODEL_FOLDER
+
+elif CHOSEN_LOCAL_MODEL_TYPE == "Gemma 3 4B":
+    LOCAL_REPO_ID = GEMMA3_4B_REPO_ID
+    LOCAL_MODEL_FILE = GEMMA3_4B_MODEL_FILE
+    LOCAL_MODEL_FOLDER = GEMMA3_4B_MODEL_FOLDER
+
+    print("CHOSEN_LOCAL_MODEL_TYPE:", CHOSEN_LOCAL_MODEL_TYPE)
+    print("LOCAL_REPO_ID:", LOCAL_REPO_ID)
+    print("LOCAL_MODEL_FILE:", LOCAL_MODEL_FILE)
+    print("LOCAL_MODEL_FOLDER:", LOCAL_MODEL_FOLDER)
+
+LLM_TEMPERATURE = float(get_or_create_env_var('LLM_TEMPERATURE', '0.1'))
+LLM_TOP_K = int(get_or_create_env_var('LLM_TOP_K','3'))
+LLM_TOP_P = float(get_or_create_env_var('LLM_TOP_P', '1'))
+LLM_REPETITION_PENALTY = float(get_or_create_env_var('LLM_REPETITION_PENALTY', '1.2')) # Mild repetition penalty to prevent repeating table rows
+LLM_LAST_N_TOKENS = int(get_or_create_env_var('LLM_LAST_N_TOKENS', '512'))
+LLM_MAX_NEW_TOKENS = int(get_or_create_env_var('LLM_MAX_NEW_TOKENS', '4096'))
+LLM_SEED = int(get_or_create_env_var('LLM_SEED', '42'))
+LLM_RESET = get_or_create_env_var('LLM_RESET', 'True')
+LLM_STREAM = get_or_create_env_var('LLM_STREAM', 'False')
+LLM_THREADS = int(get_or_create_env_var('LLM_THREADS', '4'))
+LLM_BATCH_SIZE = int(get_or_create_env_var('LLM_BATCH_SIZE', '256'))
+LLM_CONTEXT_LENGTH = int(get_or_create_env_var('LLM_CONTEXT_LENGTH', '16384'))
+LLM_SAMPLE = get_or_create_env_var('LLM_SAMPLE', 'True')
+
+###
+# Gradio app variables
+###
+
+# Get some environment variables and Launch the Gradio app
+COGNITO_AUTH = get_or_create_env_var('COGNITO_AUTH', '0')
+
+RUN_DIRECT_MODE = get_or_create_env_var('RUN_DIRECT_MODE', '0')
+
+MAX_QUEUE_SIZE = int(get_or_create_env_var('MAX_QUEUE_SIZE', '5'))
+
+MAX_FILE_SIZE = get_or_create_env_var('MAX_FILE_SIZE', '250mb')
+
+GRADIO_SERVER_PORT = int(get_or_create_env_var('GRADIO_SERVER_PORT', '7860'))
+
+ROOT_PATH = get_or_create_env_var('ROOT_PATH', '')
+
+DEFAULT_CONCURRENCY_LIMIT = get_or_create_env_var('DEFAULT_CONCURRENCY_LIMIT', '3')
+
+GET_DEFAULT_ALLOW_LIST = get_or_create_env_var('GET_DEFAULT_ALLOW_LIST', '')
+
+ALLOW_LIST_PATH = get_or_create_env_var('ALLOW_LIST_PATH', '') # config/default_allow_list.csv
+
+S3_ALLOW_LIST_PATH = get_or_create_env_var('S3_ALLOW_LIST_PATH', '') # default_allow_list.csv # This is a path within the DOCUMENT_REDACTION_BUCKET
+
+if ALLOW_LIST_PATH: OUTPUT_ALLOW_LIST_PATH = ALLOW_LIST_PATH
+else: OUTPUT_ALLOW_LIST_PATH = 'config/default_allow_list.csv'
+
+FILE_INPUT_HEIGHT = get_or_create_env_var('FILE_INPUT_HEIGHT', '200')

tools/dedup_summaries.py

@@ -0,0 +1,602 @@
+import pandas as pd
+from rapidfuzz import process, fuzz
+from typing import List, Tuple
+import re
+import spaces
+import gradio as gr
+from time import time
+from tqdm import tqdm
+
+from tools.prompts import summarise_topic_descriptions_prompt, summarise_topic_descriptions_system_prompt, system_prompt
+from tools.llm_funcs import construct_gemini_generative_model, process_requests, ResponseObject, load_model
+from tools.helper_functions import create_unique_table_df_from_reference_table, load_in_data_file, get_basic_response_data, convert_reference_table_to_pivot_table, wrap_text
+from tools.config import OUTPUT_FOLDER, RUN_LOCAL_MODEL, MAX_COMMENT_CHARS, MAX_TOKENS, TIMEOUT_WAIT, NUMBER_OF_RETRY_ATTEMPTS, MAX_TIME_FOR_LOOP, BATCH_SIZE_DEFAULT, DEDUPLICATION_THRESHOLD, model_name_map
+
+max_tokens = MAX_TOKENS
+timeout_wait = TIMEOUT_WAIT
+number_of_api_retry_attempts = NUMBER_OF_RETRY_ATTEMPTS
+max_time_for_loop = MAX_TIME_FOR_LOOP
+batch_size_default = BATCH_SIZE_DEFAULT
+deduplication_threshold = DEDUPLICATION_THRESHOLD
+max_comment_character_length = MAX_COMMENT_CHARS
+
+# DEDUPLICATION/SUMMARISATION FUNCTIONS
+def deduplicate_categories(category_series: pd.Series, join_series: pd.Series, reference_df: pd.DataFrame, general_topic_series: pd.Series = None, merge_general_topics = "No", merge_sentiment:str="No", threshold: float = 90) -> pd.DataFrame:
+    """
+    Deduplicates similar category names in a pandas Series based on a fuzzy matching threshold,
+    merging smaller topics into larger topics.
+
+    Parameters:
+        category_series (pd.Series): Series containing category names to deduplicate.
+        join_series (pd.Series): Additional series used for joining back to original results.
+        reference_df (pd.DataFrame): DataFrame containing the reference data to count occurrences.
+        threshold (float): Similarity threshold for considering two strings as duplicates.
+
+    Returns:
+        pd.DataFrame: DataFrame with columns ['old_category', 'deduplicated_category'].
+    """
+    # Count occurrences of each category in the reference_df
+    category_counts = reference_df['Subtopic'].value_counts().to_dict()
+
+    # Initialize dictionaries for both category mapping and scores
+    deduplication_map = {}
+    match_scores = {}  # New dictionary to store match scores
+
+    # First pass: Handle exact matches
+    for category in category_series.unique():
+        if category in deduplication_map:
+            continue
+            
+        # Find all exact matches
+        exact_matches = category_series[category_series.str.lower() == category.lower()].index.tolist()
+        if len(exact_matches) > 1:
+            # Find the variant with the highest count
+            match_counts = {match: category_counts.get(category_series[match], 0) for match in exact_matches}
+            most_common = max(match_counts.items(), key=lambda x: x[1])[0]
+            most_common_category = category_series[most_common]
+            
+            # Map all exact matches to the most common variant and store score
+            for match in exact_matches:
+                deduplication_map[category_series[match]] = most_common_category
+                match_scores[category_series[match]] = 100  # Exact matches get score of 100
+
+    # Second pass: Handle fuzzy matches for remaining categories
+    # Create a DataFrame to maintain the relationship between categories and general topics
+    categories_df = pd.DataFrame({
+        'category': category_series,
+        'general_topic': general_topic_series
+    }).drop_duplicates()
+
+    for _, row in categories_df.iterrows():
+        category = row['category']
+        if category in deduplication_map:
+            continue
+
+        current_general_topic = row['general_topic']
+
+        # Filter potential matches to only those within the same General Topic if relevant
+        if merge_general_topics == "No":
+            potential_matches = categories_df[
+                (categories_df['category'] != category) & 
+                (categories_df['general_topic'] == current_general_topic)
+            ]['category'].tolist()
+        else:
+            potential_matches = categories_df[
+                (categories_df['category'] != category)
+            ]['category'].tolist()
+
+        matches = process.extract(category, 
+                                potential_matches,
+                                scorer=fuzz.WRatio,
+                                score_cutoff=threshold)
+
+        if matches:
+            best_match = max(matches, key=lambda x: x[1])
+            match, score, _ = best_match
+
+            if category_counts.get(category, 0) < category_counts.get(match, 0):
+                deduplication_map[category] = match
+                match_scores[category] = score
+            else:
+                deduplication_map[match] = category
+                match_scores[match] = score
+        else:
+            deduplication_map[category] = category
+            match_scores[category] = 100
+
+    # Create the result DataFrame with scores
+    result_df = pd.DataFrame({
+        'old_category': category_series + " | " + join_series,
+        'deduplicated_category': category_series.map(lambda x: deduplication_map.get(x, x)),
+        'match_score': category_series.map(lambda x: match_scores.get(x, 100))  # Add scores column
+    })
+
+    #print(result_df)
+
+    return result_df
+
+def deduplicate_topics(reference_df:pd.DataFrame,
+                       unique_topics_df:pd.DataFrame,
+                       reference_table_file_name:str,
+                       unique_topics_table_file_name:str,
+                       in_excel_sheets:str="",
+                       merge_sentiment:str= "No",
+                       merge_general_topics:str="No",
+                       score_threshold:int=90,
+                       in_data_files:List[str]=[],
+                       chosen_cols:List[str]="",
+                       deduplicate_topics:str="Yes",
+                       output_folder:str=OUTPUT_FOLDER
+                       ):
+    '''
+    Deduplicate topics based on a reference and unique topics table
+    '''
+    output_files = []
+    log_output_files = []
+    file_data = pd.DataFrame()
+
+    reference_table_file_name_no_ext = reference_table_file_name #get_file_name_no_ext(reference_table_file_name)
+    unique_topics_table_file_name_no_ext = unique_topics_table_file_name #get_file_name_no_ext(unique_topics_table_file_name)
+
+    # For checking that data is not lost during the process
+    initial_unique_references = len(reference_df["Response References"].unique())
+
+    if unique_topics_df.empty:
+        unique_topics_df = create_unique_table_df_from_reference_table(reference_df)
+
+        # Then merge the topic numbers back to the original dataframe
+        reference_df = reference_df.merge(
+            unique_topics_df[['General Topic', 'Subtopic', 'Sentiment', 'Topic_number']],
+            on=['General Topic', 'Subtopic', 'Sentiment'],
+            how='left'
+        )     
+
+    if in_data_files and chosen_cols:
+        file_data, data_file_names_textbox, total_number_of_batches = load_in_data_file(in_data_files, chosen_cols, 1, in_excel_sheets)
+    else:
+        out_message = "No file data found, pivot table output will not be created."
+        print(out_message)
+        #raise Exception(out_message)
+
+    # Run through this x times to try to get all duplicate topics
+    if deduplicate_topics == "Yes":
+        for i in range(0, 8):
+            if merge_sentiment == "No":    
+                if merge_general_topics == "No":
+                    reference_df["old_category"] = reference_df["Subtopic"] + " | " + reference_df["Sentiment"]
+                    reference_df_unique = reference_df.drop_duplicates("old_category")
+
+                    deduplicated_topic_map_df = reference_df_unique.groupby(["General Topic", "Sentiment"]).apply(
+                        lambda group: deduplicate_categories(
+                            group["Subtopic"], 
+                            group["Sentiment"], 
+                            reference_df, 
+                            general_topic_series=group["General Topic"],
+                            merge_general_topics="No",
+                            threshold=score_threshold
+                        )
+                    ).reset_index(drop=True)
+                else:
+                    # This case should allow cross-topic matching but is still grouping by Sentiment
+                    reference_df["old_category"] = reference_df["Subtopic"] + " | " + reference_df["Sentiment"]
+                    reference_df_unique = reference_df.drop_duplicates("old_category")
+
+                    deduplicated_topic_map_df = reference_df_unique.groupby("Sentiment").apply(
+                        lambda group: deduplicate_categories(
+                            group["Subtopic"], 
+                            group["Sentiment"], 
+                            reference_df, 
+                            general_topic_series=None,  # Set to None to allow cross-topic matching
+                            merge_general_topics="Yes",
+                            threshold=score_threshold
+                        )
+                    ).reset_index(drop=True)
+            else:
+                if merge_general_topics == "No":
+                    # Update this case to maintain general topic boundaries
+                    reference_df["old_category"] = reference_df["Subtopic"] + " | " + reference_df["Sentiment"]
+                    reference_df_unique = reference_df.drop_duplicates("old_category")
+
+                    deduplicated_topic_map_df = reference_df_unique.groupby("General Topic").apply(
+                        lambda group: deduplicate_categories(
+                            group["Subtopic"], 
+                            group["Sentiment"], 
+                            reference_df, 
+                            general_topic_series=group["General Topic"],
+                            merge_general_topics="No",
+                            merge_sentiment=merge_sentiment, 
+                            threshold=score_threshold
+                        )
+                    ).reset_index(drop=True)
+                else:
+                    # For complete merging across all categories
+                    reference_df["old_category"] = reference_df["Subtopic"] + " | " + reference_df["Sentiment"]
+                    reference_df_unique = reference_df.drop_duplicates("old_category")
+
+                    deduplicated_topic_map_df = deduplicate_categories(
+                        reference_df_unique["Subtopic"], 
+                        reference_df_unique["Sentiment"], 
+                        reference_df, 
+                        general_topic_series=None,  # Set to None to allow cross-topic matching
+                        merge_general_topics="Yes",
+                        merge_sentiment=merge_sentiment,
+                        threshold=score_threshold
+                    ).reset_index(drop=True)
+           
+            if deduplicated_topic_map_df['deduplicated_category'].isnull().all():
+            # Check if 'deduplicated_category' contains any values
+                print("No deduplicated categories found, skipping the following code.")
+
+            else:
+                # Remove rows where 'deduplicated_category' is blank or NaN
+                deduplicated_topic_map_df = deduplicated_topic_map_df.loc[(deduplicated_topic_map_df['deduplicated_category'].str.strip() != '') & ~(deduplicated_topic_map_df['deduplicated_category'].isnull()), ['old_category','deduplicated_category', 'match_score']]
+
+                #deduplicated_topic_map_df.to_csv(output_folder + "deduplicated_topic_map_df_" + str(i) + ".csv", index=None)
+
+                reference_df = reference_df.merge(deduplicated_topic_map_df, on="old_category", how="left")
+
+                reference_df.rename(columns={"Subtopic": "Subtopic_old", "Sentiment": "Sentiment_old"}, inplace=True)
+                # Extract subtopic and sentiment from deduplicated_category
+                reference_df["Subtopic"] = reference_df["deduplicated_category"].str.extract(r'^(.*?) \|')[0]  # Extract subtopic
+                reference_df["Sentiment"] = reference_df["deduplicated_category"].str.extract(r'\| (.*)$')[0]  # Extract sentiment
+
+                # Combine with old values to ensure no data is lost
+                reference_df["Subtopic"] = reference_df["deduplicated_category"].combine_first(reference_df["Subtopic_old"])
+                reference_df["Sentiment"] = reference_df["Sentiment"].combine_first(reference_df["Sentiment_old"])
+
+
+            reference_df.drop(['old_category', 'deduplicated_category', "Subtopic_old", "Sentiment_old"], axis=1, inplace=True, errors="ignore")
+
+            reference_df = reference_df[["Response References", "General Topic", "Subtopic", "Sentiment", "Summary", "Start row of group"]]
+
+            #reference_df["General Topic"] = reference_df["General Topic"].str.lower().str.capitalize() 
+            #reference_df["Subtopic"] = reference_df["Subtopic"].str.lower().str.capitalize() 
+            #reference_df["Sentiment"] = reference_df["Sentiment"].str.lower().str.capitalize() 
+
+            if merge_general_topics == "Yes":
+                # Replace General topic names for each Subtopic with that for the Subtopic with the most responses
+                # Step 1: Count the number of occurrences for each General Topic and Subtopic combination
+                count_df = reference_df.groupby(['Subtopic', 'General Topic']).size().reset_index(name='Count')
+
+                # Step 2: Find the General Topic with the maximum count for each Subtopic
+                max_general_topic = count_df.loc[count_df.groupby('Subtopic')['Count'].idxmax()]
+
+                # Step 3: Map the General Topic back to the original DataFrame
+                reference_df = reference_df.merge(max_general_topic[['Subtopic', 'General Topic']], on='Subtopic', suffixes=('', '_max'), how='left')
+
+                reference_df['General Topic'] = reference_df["General Topic_max"].combine_first(reference_df["General Topic"])        
+
+            if merge_sentiment == "Yes":
+                # Step 1: Count the number of occurrences for each General Topic and Subtopic combination
+                count_df = reference_df.groupby(['Subtopic', 'Sentiment']).size().reset_index(name='Count')
+
+                # Step 2: Determine the number of unique Sentiment values for each Subtopic
+                unique_sentiments = count_df.groupby('Subtopic')['Sentiment'].nunique().reset_index(name='UniqueCount')
+
+                # Step 3: Update Sentiment to 'Mixed' where there is more than one unique sentiment
+                reference_df = reference_df.merge(unique_sentiments, on='Subtopic', how='left')
+                reference_df['Sentiment'] = reference_df.apply(
+                    lambda row: 'Mixed' if row['UniqueCount'] > 1 else row['Sentiment'],
+                    axis=1
+                )
+
+                # Clean up the DataFrame by dropping the UniqueCount column
+                reference_df.drop(columns=['UniqueCount'], inplace=True)
+
+            reference_df = reference_df[["Response References", "General Topic", "Subtopic", "Sentiment", "Summary", "Start row of group"]]
+
+        # Update reference summary column with all summaries
+        reference_df["Summary"] = reference_df.groupby(
+        ["Response References", "General Topic", "Subtopic", "Sentiment"]
+        )["Summary"].transform(' <br> '.join)
+
+        # Check that we have not inadvertantly removed some data during the above process
+        end_unique_references = len(reference_df["Response References"].unique())
+
+        if initial_unique_references != end_unique_references:
+            raise Exception(f"Number of unique references changed during processing: Initial={initial_unique_references}, Final={end_unique_references}")
+        
+        # Drop duplicates in the reference table - each comment should only have the same topic referred to once
+        reference_df.drop_duplicates(['Response References', 'General Topic', 'Subtopic', 'Sentiment'], inplace=True)
+
+
+        # Remake unique_topics_df based on new reference_df
+        unique_topics_df = create_unique_table_df_from_reference_table(reference_df)
+
+        # Then merge the topic numbers back to the original dataframe
+        reference_df = reference_df.merge(
+            unique_topics_df[['General Topic', 'Subtopic', 'Sentiment', 'Topic_number']],
+            on=['General Topic', 'Subtopic', 'Sentiment'],
+            how='left'
+        ) 
+
+        if not file_data.empty:
+            basic_response_data = get_basic_response_data(file_data, chosen_cols)            
+            reference_df_pivot = convert_reference_table_to_pivot_table(reference_df, basic_response_data)
+
+            reference_pivot_file_path = output_folder + reference_table_file_name_no_ext + "_pivot_dedup.csv"
+            reference_df_pivot.to_csv(reference_pivot_file_path, index=None, encoding='utf-8')
+            log_output_files.append(reference_pivot_file_path)
+
+        #reference_table_file_name_no_ext = get_file_name_no_ext(reference_table_file_name)
+        #unique_topics_table_file_name_no_ext = get_file_name_no_ext(unique_topics_table_file_name)
+
+        reference_file_path = output_folder + reference_table_file_name_no_ext + "_dedup.csv"
+        unique_topics_file_path = output_folder + unique_topics_table_file_name_no_ext + "_dedup.csv"
+        reference_df.to_csv(reference_file_path, index = None, encoding='utf-8')
+        unique_topics_df.to_csv(unique_topics_file_path, index=None, encoding='utf-8')
+
+        output_files.append(reference_file_path)
+        output_files.append(unique_topics_file_path)        
+
+        # Outputs for markdown table output
+        unique_table_df_revised_display = unique_topics_df.apply(lambda col: col.map(lambda x: wrap_text(x, max_text_length=500)))
+
+        deduplicated_unique_table_markdown = unique_table_df_revised_display.to_markdown(index=False)
+
+    return reference_df, unique_topics_df, output_files, log_output_files, deduplicated_unique_table_markdown
+
+def sample_reference_table_summaries(reference_df:pd.DataFrame,
+                                     unique_topics_df:pd.DataFrame,
+                                     random_seed:int,
+                                     no_of_sampled_summaries:int=150):
+    
+    '''
+    Sample x number of summaries from which to produce summaries, so that the input token length is not too long.
+    '''
+    
+    all_summaries = pd.DataFrame()
+    output_files = []
+
+    reference_df_grouped = reference_df.groupby(["General Topic", "Subtopic", "Sentiment"])
+
+    if 'Revised summary' in reference_df.columns:
+        out_message = "Summary has already been created for this file"
+        print(out_message)
+        raise Exception(out_message)
+
+    for group_keys, reference_df_group in reference_df_grouped:
+        #print(f"Group: {group_keys}")
+        #print(f"Data: {reference_df_group}")
+
+        if len(reference_df_group["General Topic"]) > 1:
+
+            filtered_reference_df = reference_df_group.reset_index()
+
+            filtered_reference_df_unique = filtered_reference_df.drop_duplicates(["General Topic", "Subtopic", "Sentiment", "Summary"])
+
+            # Sample n of the unique topic summaries. To limit the length of the text going into the summarisation tool
+            filtered_reference_df_unique_sampled = filtered_reference_df_unique.sample(min(no_of_sampled_summaries, len(filtered_reference_df_unique)), random_state=random_seed)
+
+            #topic_summary_table_markdown = filtered_reference_df_unique_sampled.to_markdown(index=False)
+
+            #print(filtered_reference_df_unique_sampled)
+
+            all_summaries = pd.concat([all_summaries, filtered_reference_df_unique_sampled])
+    
+    summarised_references = all_summaries.groupby(["General Topic", "Subtopic", "Sentiment"]).agg({
+    'Response References': 'size',  # Count the number of references
+    'Summary': lambda x: '\n'.join([s.split(': ', 1)[1] for s in x if ': ' in s])  # Join substrings after ': '
+    }).reset_index()
+
+    summarised_references = summarised_references.loc[(summarised_references["Sentiment"] != "Not Mentioned") & (summarised_references["Response References"] > 1)]
+
+    summarised_references_markdown = summarised_references.to_markdown(index=False)
+
+    return summarised_references, summarised_references_markdown, reference_df, unique_topics_df
+
+def summarise_output_topics_query(model_choice:str, in_api_key:str, temperature:float, formatted_summary_prompt:str, summarise_topic_descriptions_system_prompt:str, local_model=[]):
+    conversation_history = []
+    whole_conversation_metadata = []
+
+    # Prepare Gemini models before query       
+    if "gemini" in model_choice:
+        print("Using Gemini model:", model_choice)
+        model, config = construct_gemini_generative_model(in_api_key=in_api_key, temperature=temperature, model_choice=model_choice, system_prompt=system_prompt, max_tokens=max_tokens)
+    else:
+        print("Using AWS Bedrock model:", model_choice)
+        model = model_choice
+        config = {}
+
+    whole_conversation = [summarise_topic_descriptions_system_prompt] 
+
+    # Process requests to large language model
+    responses, conversation_history, whole_conversation, whole_conversation_metadata, response_text = process_requests(formatted_summary_prompt, system_prompt, conversation_history, whole_conversation, whole_conversation_metadata, model, config, model_choice, temperature, local_model=local_model)
+
+    print("Finished summary query")
+
+    if isinstance(responses[-1], ResponseObject):
+        response_texts = [resp.text for resp in responses]
+    elif "choices" in responses[-1]:
+        response_texts = [resp["choices"][0]['text'] for resp in responses]
+    else:
+        response_texts = [resp.text for resp in responses]
+
+    latest_response_text = response_texts[-1]
+
+    #print("latest_response_text:", latest_response_text)
+    #print("Whole conversation metadata:", whole_conversation_metadata)
+
+    return latest_response_text, conversation_history, whole_conversation_metadata
+
+@spaces.GPU
+def summarise_output_topics(summarised_references:pd.DataFrame,
+                            unique_table_df:pd.DataFrame,
+                            reference_table_df:pd.DataFrame,
+                            model_choice:str,
+                            in_api_key:str,
+                            topic_summary_table_markdown:str,
+                            temperature:float,
+                            table_file_name:str,
+                            summarised_outputs:list = [],  
+                            latest_summary_completed:int = 0,
+                            out_metadata_str:str = "",
+                            in_data_files:List[str]=[],
+                            in_excel_sheets:str="",
+                            chosen_cols:List[str]=[],
+                            log_output_files:list[str]=[],
+                            summarise_format_radio:str="Return a summary up to two paragraphs long that includes as much detail as possible from the original text",
+                            output_folder:str=OUTPUT_FOLDER,
+                            output_files:list[str] = [],                            
+                            summarise_topic_descriptions_prompt:str=summarise_topic_descriptions_prompt, summarise_topic_descriptions_system_prompt:str=summarise_topic_descriptions_system_prompt,
+                            do_summaries:str="Yes",                            
+                            progress=gr.Progress(track_tqdm=True)):
+    '''
+    Create better summaries of the raw batch-level summaries created in the first run of the model.
+    '''
+    out_metadata = []
+    local_model = []
+    summarised_output_markdown = ""
+    
+
+    # Check for data for summarisations
+    if not unique_table_df.empty and not reference_table_df.empty:
+        print("Unique table and reference table data found.")
+    else:
+        out_message = "Please upload a unique topic table and reference table file to continue with summarisation."
+        print(out_message)
+        raise Exception(out_message)
+    
+    if 'Revised summary' in reference_table_df.columns:
+        out_message = "Summary has already been created for this file"
+        print(out_message)
+        raise Exception(out_message)
+    
+    # Load in data file and chosen columns if exists to create pivot table later
+    if in_data_files and chosen_cols:
+        file_data, data_file_names_textbox, total_number_of_batches = load_in_data_file(in_data_files, chosen_cols, 1, in_excel_sheets=in_excel_sheets)
+    else:
+        out_message = "No file data found, pivot table output will not be created."
+        print(out_message)
+        raise Exception(out_message)
+   
+    
+    all_summaries = summarised_references["Summary"].tolist()
+    length_all_summaries = len(all_summaries)
+
+    # If all summaries completed, make final outputs
+    if latest_summary_completed >= length_all_summaries:
+        print("All summaries completed. Creating outputs.")
+
+        model_choice_clean = model_name_map[model_choice]   
+        file_name = re.search(r'(.*?)(?:_batch_|_col_)', table_file_name).group(1) if re.search(r'(.*?)(?:_batch_|_col_)', table_file_name) else table_file_name
+        latest_batch_completed = int(re.search(r'batch_(\d+)_', table_file_name).group(1)) if 'batch_' in table_file_name else ""
+        batch_size_number = int(re.search(r'size_(\d+)_', table_file_name).group(1)) if 'size_' in table_file_name else ""
+        in_column_cleaned = re.search(r'col_(.*?)_reference', table_file_name).group(1) if 'col_' in table_file_name else ""
+
+        # Save outputs for each batch. If master file created, label file as master
+        if latest_batch_completed:
+            batch_file_path_details = f"{file_name}_batch_{latest_batch_completed}_size_{batch_size_number}_col_{in_column_cleaned}"
+        else:
+            batch_file_path_details = f"{file_name}_col_{in_column_cleaned}"
+
+        summarised_references["Revised summary"] = summarised_outputs        
+
+        join_cols = ["General Topic", "Subtopic", "Sentiment"]
+        join_plus_summary_cols = ["General Topic", "Subtopic", "Sentiment", "Revised summary"]
+
+        summarised_references_j = summarised_references[join_plus_summary_cols].drop_duplicates(join_plus_summary_cols)
+
+        unique_table_df_revised = unique_table_df.merge(summarised_references_j, on = join_cols, how = "left")
+
+        # If no new summary is available, keep the original
+        unique_table_df_revised["Revised summary"] = unique_table_df_revised["Revised summary"].combine_first(unique_table_df_revised["Summary"])
+
+        unique_table_df_revised = unique_table_df_revised[["General Topic",	"Subtopic",	"Sentiment", "Response References",	"Revised summary"]]        
+
+        reference_table_df_revised = reference_table_df.merge(summarised_references_j, on = join_cols, how = "left")
+        # If no new summary is available, keep the original
+        reference_table_df_revised["Revised summary"] = reference_table_df_revised["Revised summary"].combine_first(reference_table_df_revised["Summary"])
+        reference_table_df_revised = reference_table_df_revised.drop("Summary", axis=1)
+
+        # Remove topics that are tagged as 'Not Mentioned'
+        unique_table_df_revised = unique_table_df_revised.loc[unique_table_df_revised["Sentiment"] != "Not Mentioned", :]
+        reference_table_df_revised = reference_table_df_revised.loc[reference_table_df_revised["Sentiment"] != "Not Mentioned", :]
+        
+          
+            
+
+        if not file_data.empty:
+            basic_response_data = get_basic_response_data(file_data, chosen_cols)
+            reference_table_df_revised_pivot = convert_reference_table_to_pivot_table(reference_table_df_revised, basic_response_data)
+
+            ### Save pivot file to log area
+            reference_table_df_revised_pivot_path = output_folder + batch_file_path_details + "_summarised_reference_table_pivot_" + model_choice_clean + ".csv"
+            reference_table_df_revised_pivot.to_csv(reference_table_df_revised_pivot_path, index=None, encoding='utf-8')
+            log_output_files.append(reference_table_df_revised_pivot_path)
+
+        # Save to file
+        unique_table_df_revised_path = output_folder + batch_file_path_details + "_summarised_unique_topic_table_" + model_choice_clean + ".csv"
+        unique_table_df_revised.to_csv(unique_table_df_revised_path, index = None, encoding='utf-8')
+
+        reference_table_df_revised_path = output_folder + batch_file_path_details + "_summarised_reference_table_" + model_choice_clean + ".csv"
+        reference_table_df_revised.to_csv(reference_table_df_revised_path, index = None, encoding='utf-8')
+
+        output_files.extend([reference_table_df_revised_path, unique_table_df_revised_path])       
+
+        ###
+        unique_table_df_revised_display = unique_table_df_revised.apply(lambda col: col.map(lambda x: wrap_text(x, max_text_length=500)))
+
+        summarised_output_markdown = unique_table_df_revised_display.to_markdown(index=False)
+
+        # Ensure same file name not returned twice
+        output_files = list(set(output_files))
+        log_output_files = list(set(log_output_files))
+
+        return summarised_references, unique_table_df_revised, reference_table_df_revised, output_files, summarised_outputs, latest_summary_completed, out_metadata_str, summarised_output_markdown, log_output_files
+
+    tic = time.perf_counter()
+    
+    #print("Starting with:", latest_summary_completed)
+    #print("Last summary number:", length_all_summaries)
+
+    if (model_choice == "gemma_2b_it_local") & (RUN_LOCAL_MODEL == "1"):
+                progress(0.1, "Loading in Gemma 2b model")
+                local_model, tokenizer = load_model()
+                #print("Local model loaded:", local_model)
+
+    summary_loop_description = "Creating summaries. " + str(latest_summary_completed) + " summaries completed so far."
+    summary_loop = tqdm(range(latest_summary_completed, length_all_summaries), desc="Creating summaries", unit="summaries")   
+
+    if do_summaries == "Yes":
+        for summary_no in summary_loop:
+
+            print("Current summary number is:", summary_no)
+
+            summary_text = all_summaries[summary_no]
+            #print("summary_text:", summary_text)
+            formatted_summary_prompt = [summarise_topic_descriptions_prompt.format(summaries=summary_text, summary_format=summarise_format_radio)]
+
+            try:
+                response, conversation_history, metadata = summarise_output_topics_query(model_choice, in_api_key, temperature, formatted_summary_prompt, summarise_topic_descriptions_system_prompt, local_model)
+                summarised_output = response
+                summarised_output = re.sub(r'\n{2,}', '\n', summarised_output)  # Replace multiple line breaks with a single line break
+                summarised_output = re.sub(r'^\n{1,}', '', summarised_output)  # Remove one or more line breaks at the start
+                summarised_output = summarised_output.strip()
+            except Exception as e:
+                print(e)
+                summarised_output = ""
+
+            summarised_outputs.append(summarised_output)
+            out_metadata.extend(metadata)
+            out_metadata_str = '. '.join(out_metadata)
+
+            latest_summary_completed += 1
+
+            # Check if beyond max time allowed for processing and break if necessary
+            toc = time.perf_counter()
+            time_taken = tic - toc
+
+            if time_taken > max_time_for_loop:
+                print("Time taken for loop is greater than maximum time allowed. Exiting and restarting loop")
+                summary_loop.close()
+                tqdm._instances.clear()
+                break
+
+    # If all summaries completeed
+    if latest_summary_completed >= length_all_summaries:
+        print("At last summary.")
+
+    output_files = list(set(output_files))
+
+    return summarised_references, unique_table_df, reference_table_df, output_files, summarised_outputs, latest_summary_completed, out_metadata_str, summarised_output_markdown, log_output_files

tools/helper_functions.py

@@ -1,9 +1,13 @@
 import os
 import re
+import boto3
 import gradio as gr
 import pandas as pd
+import numpy as np
 from typing import List
 import math
+from botocore.exceptions import ClientError
+from tools.config import OUTPUT_FOLDER, INPUT_FOLDER, SESSION_OUTPUT_FOLDER, CUSTOM_HEADER, CUSTOM_HEADER_VALUE, AWS_USER_POOL_ID
 
 def empty_output_vars_extract_topics():
     # Empty output objects before processing a new file
@@ -37,7 +41,6 @@ def empty_output_vars_summarise():
 
     return summary_reference_table_sample_state, master_unique_topics_df_revised_summaries_state, master_reference_df_revised_summaries_state, summary_output_files, summarised_outputs_list, latest_summary_completed_num, conversation_metadata_textbox
 
-
 def get_or_create_env_var(var_name, default_value):
     # Get the environment variable if it exists
     value = os.environ.get(var_name)
@@ -49,45 +52,6 @@ def get_or_create_env_var(var_name, default_value):
     
     return value
 
-RUN_AWS_FUNCTIONS = get_or_create_env_var("RUN_AWS_FUNCTIONS", "1")
-print(f'The value of RUN_AWS_FUNCTIONS is {RUN_AWS_FUNCTIONS}')
-
-RUN_LOCAL_MODEL = get_or_create_env_var("RUN_LOCAL_MODEL", "1")
-print(f'The value of RUN_LOCAL_MODEL is {RUN_LOCAL_MODEL}')
-
-RUN_GEMINI_MODELS = get_or_create_env_var("RUN_GEMINI_MODELS", "1")
-print(f'The value of RUN_GEMINI_MODELS is {RUN_GEMINI_MODELS}')
-
-GEMINI_API_KEY = get_or_create_env_var('GEMINI_API_KEY', '')
-
-# Build up options for models
-model_full_names = []
-model_short_names = []
-
-if RUN_LOCAL_MODEL == "1":
-    model_full_names.append("gemma_2b_it_local")
-    model_short_names.append("gemma_local")
-
-if RUN_AWS_FUNCTIONS == "1":
-    model_full_names.extend(["anthropic.claude-3-haiku-20240307-v1:0", "anthropic.claude-3-sonnet-20240229-v1:0"])
-    model_short_names.extend(["haiku", "sonnet"])
-
-if RUN_GEMINI_MODELS == "1":
-    model_full_names.extend(["gemini-2.0-flash-001", "gemini-2.5-flash-preview-05-20", "gemini-2.5-pro-exp-05-06" ]) # , # Gemini pro No longer available on free tier
-    model_short_names.extend(["gemini_flash_2", "gemini_flash_2.5", "gemini_pro"])
-
-print("model_short_names:", model_short_names)
-print("model_full_names:", model_full_names)
-
-model_name_map = {short: full for short, full in zip(model_full_names, model_short_names)}
-
-# Retrieving or setting output folder
-env_var_name = 'GRADIO_OUTPUT_FOLDER'
-default_value = 'output/'
-
-output_folder = get_or_create_env_var(env_var_name, default_value)
-print(f'The value of {env_var_name} is {output_folder}')
-
 def get_file_path_with_extension(file_path):
     # First, get the basename of the file (e.g., "example.txt" from "/path/to/example.txt")
     basename = os.path.basename(file_path)
@@ -222,7 +186,7 @@ def load_in_previous_reference_file(file:str):
          
     return reference_file_data, reference_file_name
 
-def join_cols_onto_reference_df(reference_df:pd.DataFrame, original_data_df:pd.DataFrame, join_columns:List[str], original_file_name:str, output_folder:str=output_folder):
+def join_cols_onto_reference_df(reference_df:pd.DataFrame, original_data_df:pd.DataFrame, join_columns:List[str], original_file_name:str, output_folder:str=OUTPUT_FOLDER):
 
     #print("original_data_df columns:", original_data_df.columns)
     #print("original_data_df:", original_data_df)
@@ -246,6 +210,75 @@ def join_cols_onto_reference_df(reference_df:pd.DataFrame, original_data_df:pd.D
 
     return out_reference_df, file_data_outputs
 
+
+def get_basic_response_data(file_data:pd.DataFrame, chosen_cols:List[str], verify_titles:bool=False) -> pd.DataFrame:
+
+    if not isinstance(chosen_cols, list):
+        chosen_cols = [chosen_cols]
+    else:
+        chosen_cols = chosen_cols
+
+    basic_response_data = file_data[chosen_cols].reset_index(names="Reference")
+    basic_response_data["Reference"] = basic_response_data["Reference"].astype(int) + 1
+
+    if verify_titles == True:
+        basic_response_data = basic_response_data.rename(columns={chosen_cols[0]: "Response", chosen_cols[1]: "Title"})
+        basic_response_data["Title"] = basic_response_data["Title"].str.strip()
+        basic_response_data["Title"] = basic_response_data["Title"].apply(initial_clean)
+    else:
+        basic_response_data = basic_response_data.rename(columns={chosen_cols[0]: "Response"})
+
+    basic_response_data["Response"] = basic_response_data["Response"].str.strip()
+    basic_response_data["Response"] = basic_response_data["Response"].apply(initial_clean)
+
+    return basic_response_data
+
+def convert_reference_table_to_pivot_table(df:pd.DataFrame, basic_response_data:pd.DataFrame=pd.DataFrame()):
+
+    df_in = df[['Response References', 'General Topic', 'Subtopic', 'Sentiment']].copy()
+
+    df_in['Response References'] = df_in['Response References'].astype(int)
+
+    # Create a combined category column
+    df_in['Category'] = df_in['General Topic'] + ' - ' + df_in['Subtopic'] + ' - ' + df_in['Sentiment']
+    
+    # Create pivot table counting occurrences of each unique combination
+    pivot_table = pd.crosstab(
+        index=df_in['Response References'],
+        columns=[df_in['General Topic'], df_in['Subtopic'], df_in['Sentiment']],
+        margins=True
+    )
+    
+    # Flatten column names to make them more readable
+    pivot_table.columns = [' - '.join(col) for col in pivot_table.columns]
+
+    pivot_table.reset_index(inplace=True)
+
+    if not basic_response_data.empty:
+        pivot_table = basic_response_data.merge(pivot_table, right_on="Response References", left_on="Reference", how="left")
+
+        pivot_table.drop("Response References", axis=1, inplace=True)    
+
+    pivot_table.columns = pivot_table.columns.str.replace("Not assessed - ", "").str.replace("- Not assessed", "")
+
+    return pivot_table
+
+def create_unique_table_df_from_reference_table(reference_df:pd.DataFrame):
+
+    out_unique_topics_df = (reference_df.groupby(["General Topic", "Subtopic", "Sentiment"])
+            .agg({
+                'Response References': 'size',  # Count the number of references
+                'Summary': lambda x: '<br>'.join(
+                    sorted(set(x), key=lambda summary: reference_df.loc[reference_df['Summary'] == summary, 'Start row of group'].min())
+                )
+            })
+            .reset_index()
+            .sort_values('Response References', ascending=False)  # Sort by size, biggest first
+            .assign(Topic_number=lambda df: np.arange(1, len(df) + 1))  # Add numbering 1 to x
+        )
+
+    return out_unique_topics_df
+
 # Wrap text in each column to the specified max width, including whole words
 def wrap_text(text:str, max_width=60, max_text_length=None):
     if not isinstance(text, str):
@@ -332,7 +365,7 @@ def wrap_text(text:str, max_width=60, max_text_length=None):
     
     return '<br>'.join(wrapped_lines)
 
-def initial_clean(text):
+def initial_clean(text:str):
     #### Some of my cleaning functions
     html_pattern_regex = r'<.*?>|&([a-z0-9]+|#[0-9]{1,6}|#x[0-9a-f]{1,6});|\xa0|&nbsp;'
     html_start_pattern_end_dots_regex = r'<(.*?)\.\.'
@@ -445,7 +478,7 @@ def add_folder_to_path(folder_path: str):
 def reveal_feedback_buttons():
     return gr.Radio(visible=True), gr.Textbox(visible=True), gr.Button(visible=True), gr.Markdown(visible=True)
 
-def wipe_logs(feedback_logs_loc, usage_logs_loc):
+def wipe_logs(feedback_logs_loc:str, usage_logs_loc:str):
     try:
         os.remove(feedback_logs_loc)
     except Exception as e:
@@ -454,65 +487,67 @@ def wipe_logs(feedback_logs_loc, usage_logs_loc):
         os.remove(usage_logs_loc)
     except Exception as e:
         print("Could not remove usage logs file", e)
-   
-async def get_connection_params(request: gr.Request):
-    base_folder = ""
-
-    if request:
-        #print("request user:", request.username)
-
-        #request_data = await request.json()  # Parse JSON body
-        #print("All request data:", request_data)
-        #context_value = request_data.get('context') 
-        #if 'context' in request_data:
-        #     print("Request context dictionary:", request_data['context'])
-
-        # print("Request headers dictionary:", request.headers)
-        # print("All host elements", request.client)           
-        # print("IP address:", request.client.host)
-        # print("Query parameters:", dict(request.query_params))
-        # To get the underlying FastAPI items you would need to use await and some fancy @ stuff for a live query: https://fastapi.tiangolo.com/vi/reference/request/
-        #print("Request dictionary to object:", request.request.body())
-        print("Session hash:", request.session_hash)
-
-        # Retrieving or setting CUSTOM_CLOUDFRONT_HEADER
-        CUSTOM_CLOUDFRONT_HEADER_var = get_or_create_env_var('CUSTOM_CLOUDFRONT_HEADER', '')
-        #print(f'The value of CUSTOM_CLOUDFRONT_HEADER is {CUSTOM_CLOUDFRONT_HEADER_var}')
-
-        # Retrieving or setting CUSTOM_CLOUDFRONT_HEADER_VALUE
-        CUSTOM_CLOUDFRONT_HEADER_VALUE_var = get_or_create_env_var('CUSTOM_CLOUDFRONT_HEADER_VALUE', '')
-        #print(f'The value of CUSTOM_CLOUDFRONT_HEADER_VALUE_var is {CUSTOM_CLOUDFRONT_HEADER_VALUE_var}')
-
-        if CUSTOM_CLOUDFRONT_HEADER_var and CUSTOM_CLOUDFRONT_HEADER_VALUE_var:
-            if CUSTOM_CLOUDFRONT_HEADER_var in request.headers:
-                supplied_cloudfront_custom_value = request.headers[CUSTOM_CLOUDFRONT_HEADER_var]
-                if supplied_cloudfront_custom_value == CUSTOM_CLOUDFRONT_HEADER_VALUE_var:
-                    print("Custom Cloudfront header found:", supplied_cloudfront_custom_value)
+    
+async def get_connection_params(request: gr.Request,
+                                output_folder_textbox:str=OUTPUT_FOLDER,
+                                input_folder_textbox:str=INPUT_FOLDER,
+                                session_output_folder:str=SESSION_OUTPUT_FOLDER):
+
+    #print("Session hash:", request.session_hash)
+
+    if CUSTOM_HEADER and CUSTOM_HEADER_VALUE:
+            if CUSTOM_HEADER in request.headers:
+                supplied_custom_header_value = request.headers[CUSTOM_HEADER]
+                if supplied_custom_header_value == CUSTOM_HEADER_VALUE:
+                    print("Custom header supplied and matches CUSTOM_HEADER_VALUE")
                 else:
-                    raise(ValueError, "Custom Cloudfront header value does not match expected value.")
+                    print("Custom header value does not match expected value.")
+                    raise ValueError("Custom header value does not match expected value.")
+            else:
+                print("Custom header value not found.")
+                raise ValueError("Custom header value not found.")   
 
-        # Get output save folder from 1 - username passed in from direct Cognito login, 2 - Cognito ID header passed through a Lambda authenticator, 3 - the session hash.
+    # Get output save folder from 1 - username passed in from direct Cognito login, 2 - Cognito ID header passed through a Lambda authenticator, 3 - the session hash.
 
-        if request.username:
-            out_session_hash = request.username
-            base_folder = "user-files/"
-            print("Request username found:", out_session_hash)
+    if request.username:
+        out_session_hash = request.username
+        #print("Request username found:", out_session_hash)
 
-        elif 'x-cognito-id' in request.headers:
-            out_session_hash = request.headers['x-cognito-id']
-            base_folder = "user-files/"
-            print("Cognito ID found:", out_session_hash)
+    elif 'x-cognito-id' in request.headers:
+        out_session_hash = request.headers['x-cognito-id']
+        #print("Cognito ID found:", out_session_hash)
 
-        else:
-            out_session_hash = request.session_hash
-            base_folder = "temp-files/"
-            # print("Cognito ID not found. Using session hash as save folder:", out_session_hash)
+    elif 'x-amzn-oidc-identity' in request.headers:
+        out_session_hash = request.headers['x-amzn-oidc-identity']
+
+        # Fetch email address using Cognito client
+        cognito_client = boto3.client('cognito-idp')
+        try:
+            response = cognito_client.admin_get_user(
+                UserPoolId=AWS_USER_POOL_ID,  # Replace with your User Pool ID
+                Username=out_session_hash
+            )
+            email = next(attr['Value'] for attr in response['UserAttributes'] if attr['Name'] == 'email')
+            #print("Email address found:", email)
 
-        output_folder = base_folder + out_session_hash + "/"
-        #if bucket_name:
-        #    print("S3 output folder is: " + "s3://" + bucket_name + "/" + output_folder)
+            out_session_hash = email
+        except ClientError as e:
+            print("Error fetching user details:", e)
+            email = None
 
-        return out_session_hash, output_folder, out_session_hash
+        print("Cognito ID found:", out_session_hash)
+
+    else:
+        out_session_hash = request.session_hash
+
+    if session_output_folder == 'True':
+        output_folder = output_folder_textbox + out_session_hash + "/"
+        input_folder = input_folder_textbox + out_session_hash + "/"
     else:
-        print("No session parameters found.")
-        return "",""
+        output_folder = output_folder_textbox
+        input_folder = input_folder_textbox
+
+    if not os.path.exists(output_folder): os.mkdir(output_folder)
+    if not os.path.exists(input_folder): os.mkdir(input_folder)
+
+    return out_session_hash, output_folder, out_session_hash, input_folder

tools/llm_api_call.py

@@ -6,51 +6,41 @@ import gradio as gr
 import markdown
 import time
 import boto3
-import json
-import math
 import string
 import re
 import spaces
-from rapidfuzz import process, fuzz
 from tqdm import tqdm
+
 from gradio import Progress
 from typing import List, Tuple
 from io import StringIO
 
 GradioFileData = gr.FileData
 
-from tools.prompts import initial_table_prompt, prompt2, prompt3, system_prompt, summarise_topic_descriptions_prompt, summarise_topic_descriptions_system_prompt, add_existing_topics_system_prompt, add_existing_topics_prompt, create_general_topics_system_prompt, create_general_topics_prompt, force_existing_topics_prompt, allow_new_topics_prompt, force_single_topic_prompt
-from tools.helper_functions import output_folder, read_file, get_or_create_env_var, model_name_map, put_columns_in_df, wrap_text, initial_clean, load_in_data_file, load_in_file
-from tools.chatfuncs import LlamaCPPGenerationConfig, call_llama_cpp_model, load_model, RUN_LOCAL_MODEL
-
-# ResponseObject class for AWS Bedrock calls
-class ResponseObject:
-        def __init__(self, text, usage_metadata):
-            self.text = text
-            self.usage_metadata = usage_metadata
-
-max_tokens = 4096 # Maximum number of output tokens
-timeout_wait = 30 # AWS now seems to have a 60 second minimum wait between API calls
-number_of_api_retry_attempts = 5
-# Try up to 3 times to get a valid markdown table response with LLM calls, otherwise retry with temperature changed
-MAX_OUTPUT_VALIDATION_ATTEMPTS = 3
-max_time_for_loop = 99999
-batch_size_default = 5
-deduplication_threshold = 90
+from tools.prompts import initial_table_prompt, prompt2, prompt3, system_prompt,  add_existing_topics_system_prompt, add_existing_topics_prompt,  force_existing_topics_prompt, allow_new_topics_prompt, force_single_topic_prompt
+from tools.helper_functions import read_file, put_columns_in_df, wrap_text, initial_clean, load_in_data_file, load_in_file, create_unique_table_df_from_reference_table, convert_reference_table_to_pivot_table, get_basic_response_data
+from tools.llm_funcs import ResponseObject, process_requests, construct_gemini_generative_model
+from tools.config import RUN_LOCAL_MODEL, AWS_REGION, MAX_COMMENT_CHARS, MAX_OUTPUT_VALIDATION_ATTEMPTS, MAX_TOKENS, TIMEOUT_WAIT, NUMBER_OF_RETRY_ATTEMPTS, MAX_TIME_FOR_LOOP, BATCH_SIZE_DEFAULT, DEDUPLICATION_THRESHOLD, RUN_AWS_FUNCTIONS, model_name_map, OUTPUT_FOLDER, CHOSEN_LOCAL_MODEL_TYPE, LOCAL_REPO_ID, LOCAL_MODEL_FILE, LOCAL_MODEL_FOLDER
 
-MAX_COMMENT_CHARS = get_or_create_env_var('MAX_COMMENT_CHARS', '14000')
-print(f'The value of MAX_COMMENT_CHARS is {MAX_COMMENT_CHARS}')
+if RUN_LOCAL_MODEL == "1":
+    from tools.llm_funcs import load_model
 
-max_comment_character_length = int(MAX_COMMENT_CHARS)
+max_tokens = MAX_TOKENS
+timeout_wait = TIMEOUT_WAIT
+number_of_api_retry_attempts = NUMBER_OF_RETRY_ATTEMPTS
+max_time_for_loop = MAX_TIME_FOR_LOOP
+batch_size_default = BATCH_SIZE_DEFAULT
+deduplication_threshold = DEDUPLICATION_THRESHOLD
+max_comment_character_length = MAX_COMMENT_CHARS
 
-AWS_DEFAULT_REGION = get_or_create_env_var('AWS_DEFAULT_REGION', 'eu-west-2')
-print(f'The value of AWS_DEFAULT_REGION is {AWS_DEFAULT_REGION}')
-
-bedrock_runtime = boto3.client('bedrock-runtime', region_name=AWS_DEFAULT_REGION)
+if RUN_AWS_FUNCTIONS == '1':
+    bedrock_runtime = boto3.client('bedrock-runtime', region_name=AWS_REGION)
+else:
+    bedrock_runtime = []
 
 ### HELPER FUNCTIONS
 
-def normalise_string(text):
+def normalise_string(text:str):
     # Replace two or more dashes with a single dash
     text = re.sub(r'-{2,}', '-', text)
     
@@ -130,30 +120,7 @@ def load_in_previous_data_files(file_paths_partial_output:List[str], for_modifie
 
         return gr.Dataframe(value=unique_file_data, headers=None, col_count=(unique_file_data.shape[1], "fixed"), row_count = (unique_file_data.shape[0], "fixed"), visible=True, type="pandas"), reference_file_data, unique_file_data, reference_file_name, unique_file_name, out_file_names
 
-
-def get_basic_response_data(file_data:pd.DataFrame, chosen_cols:List[str], verify_titles:bool=False) -> pd.DataFrame:
-
-    if not isinstance(chosen_cols, list):
-        chosen_cols = [chosen_cols]
-    else:
-        chosen_cols = chosen_cols
-
-    basic_response_data = file_data[chosen_cols].reset_index(names="Reference")
-    basic_response_data["Reference"] = basic_response_data["Reference"].astype(int) + 1
-
-    if verify_titles == True:
-        basic_response_data = basic_response_data.rename(columns={chosen_cols[0]: "Response", chosen_cols[1]: "Title"})
-        basic_response_data["Title"] = basic_response_data["Title"].str.strip()
-        basic_response_data["Title"] = basic_response_data["Title"].apply(initial_clean)
-    else:
-        basic_response_data = basic_response_data.rename(columns={chosen_cols[0]: "Response"})
-
-    basic_response_data["Response"] = basic_response_data["Response"].str.strip()
-    basic_response_data["Response"] = basic_response_data["Response"].apply(initial_clean)
-
-    return basic_response_data
-
-def data_file_to_markdown_table(file_data:pd.DataFrame, file_name:str, chosen_cols: List[str], output_folder: str, batch_number: int, batch_size: int, verify_titles:bool=False) -> Tuple[str, str, str]:
+def data_file_to_markdown_table(file_data:pd.DataFrame, file_name:str, chosen_cols: List[str], batch_number: int, batch_size: int, verify_titles:bool=False) -> Tuple[str, str, str]:
     """
     Processes a file by simplifying its content based on chosen columns and saves the result to a specified output folder.
 
@@ -161,7 +128,6 @@ def data_file_to_markdown_table(file_data:pd.DataFrame, file_name:str, chosen_co
     - file_data (pd.DataFrame): Tabular data file with responses.
     - file_name (str): File name with extension.
     - chosen_cols (List[str]): A list of column names to include in the simplified file.
-    - output_folder (str): The directory where the simplified file will be saved.
     - batch_number (int): The current batch number for processing.
     - batch_size (int): The number of rows to process in each batch.
 
@@ -230,309 +196,6 @@ def replace_punctuation_with_underscore(input_string):
     # Translate the input string using the translation table
     return input_string.translate(translation_table)
 
-### LLM FUNCTIONS
-
-def construct_gemini_generative_model(in_api_key: str, temperature: float, model_choice: str, system_prompt: str, max_tokens: int) -> Tuple[object, dict]:
-    """
-    Constructs a GenerativeModel for Gemini API calls.
-
-    Parameters:
-    - in_api_key (str): The API key for authentication.
-    - temperature (float): The temperature parameter for the model, controlling the randomness of the output.
-    - model_choice (str): The choice of model to use for generation.
-    - system_prompt (str): The system prompt to guide the generation.
-    - max_tokens (int): The maximum number of tokens to generate.
-
-    Returns:
-    - Tuple[object, dict]: A tuple containing the constructed GenerativeModel and its configuration.
-    """
-    # Construct a GenerativeModel
-    try:
-        if in_api_key:
-            #print("Getting API key from textbox")
-            api_key = in_api_key
-            ai.configure(api_key=api_key)
-        elif "GOOGLE_API_KEY" in os.environ:
-            #print("Searching for API key in environmental variables")
-            api_key = os.environ["GOOGLE_API_KEY"]
-            ai.configure(api_key=api_key)
-        else:
-            print("No API key foound")
-            raise gr.Error("No API key found.")
-    except Exception as e:
-        print(e)
-    
-    config = ai.GenerationConfig(temperature=temperature, max_output_tokens=max_tokens)
-
-    #model = ai.GenerativeModel.from_cached_content(cached_content=cache, generation_config=config)
-    model = ai.GenerativeModel(model_name='models/' + model_choice, system_instruction=system_prompt, generation_config=config)
-    
-    # Upload CSV file (replace with your actual file path)
-    #file_id = ai.upload_file(upload_file_path)
-
-    
-    # if file_type == 'xlsx':
-    #     print("Running through all xlsx sheets")
-    #     #anon_xlsx = pd.ExcelFile(upload_file_path)
-    #     if not in_excel_sheets:
-    #         out_message.append("No Excel sheets selected. Please select at least one to anonymise.")
-    #         continue
-
-    #     anon_xlsx = pd.ExcelFile(upload_file_path)                
-
-    #     # Create xlsx file:
-    #     anon_xlsx_export_file_name = output_folder + file_name + "_redacted.xlsx"
-
-
-    ### QUERYING LARGE LANGUAGE MODEL ###
-    # Prompt caching the table and system prompt. See here: https://ai.google.dev/gemini-api/docs/caching?lang=python
-    # Create a cache with a 5 minute TTL. ONLY FOR CACHES OF AT LEAST 32k TOKENS!
-    # cache = ai.caching.CachedContent.create(
-    # model='models/' + model_choice,
-    # display_name=file_name, # used to identify the cache
-    # system_instruction=system_prompt,
-    # ttl=datetime.timedelta(minutes=5),
-    # )
-
-    return model, config
-
-def call_aws_claude(prompt: str, system_prompt: str, temperature: float, max_tokens: int, model_choice: str) -> ResponseObject:
-    """
-    This function sends a request to AWS Claude with the following parameters:
-    - prompt: The user's input prompt to be processed by the model.
-    - system_prompt: A system-defined prompt that provides context or instructions for the model.
-    - temperature: A value that controls the randomness of the model's output, with higher values resulting in more diverse responses.
-    - max_tokens: The maximum number of tokens (words or characters) in the model's response.
-    - model_choice: The specific model to use for processing the request.
-    
-    The function constructs the request configuration, invokes the model, extracts the response text, and returns a ResponseObject containing the text and metadata.
-    """
-
-    prompt_config = {
-        "anthropic_version": "bedrock-2023-05-31",
-        "max_tokens": max_tokens,
-        "top_p": 0.999,
-        "temperature":temperature,
-        "system": system_prompt,
-        "messages": [
-            {
-                "role": "user",
-                "content": [
-                    {"type": "text", "text": prompt},
-                ],
-            }
-        ],
-    }
-
-    body = json.dumps(prompt_config)
-
-    modelId = model_choice
-    accept = "application/json"
-    contentType = "application/json"
-
-    request = bedrock_runtime.invoke_model(
-        body=body, modelId=modelId, accept=accept, contentType=contentType
-    )
-
-    # Extract text from request
-    response_body = json.loads(request.get("body").read())
-    text = response_body.get("content")[0].get("text")
-
-    response = ResponseObject(
-    text=text,
-    usage_metadata=request['ResponseMetadata']
-    )
-
-    # Now you can access both the text and metadata
-    #print("Text:", response.text)
-    #print("Metadata:", response.usage_metadata)
-    #print("Text:", response.text)
-    
-    return response
-
-# Function to send a request and update history
-def send_request(prompt: str, conversation_history: List[dict], model: object, config: dict, model_choice: str, system_prompt: str, temperature: float, local_model=[], progress=Progress(track_tqdm=True)) -> Tuple[str, List[dict]]:
-    """
-    This function sends a request to a language model with the given prompt, conversation history, model configuration, model choice, system prompt, and temperature.
-    It constructs the full prompt by appending the new user prompt to the conversation history, generates a response from the model, and updates the conversation history with the new prompt and response.
-    If the model choice is specific to AWS Claude, it calls the `call_aws_claude` function; otherwise, it uses the `model.generate_content` method.
-    The function returns the response text and the updated conversation history.
-    """
-    # Constructing the full prompt from the conversation history
-    full_prompt = "Conversation history:\n"
-    
-    for entry in conversation_history:
-        role = entry['role'].capitalize()  # Assuming the history is stored with 'role' and 'parts'
-        message = ' '.join(entry['parts'])  # Combining all parts of the message
-        full_prompt += f"{role}: {message}\n"
-    
-    # Adding the new user prompt
-    full_prompt += f"\nUser: {prompt}"
-
-    # Clear any existing progress bars
-    tqdm._instances.clear()
-
-    progress_bar = range(0,number_of_api_retry_attempts)
-
-    # Generate the model's response
-    if "gemini" in model_choice:
-
-        for i in progress_bar:
-            try:
-                print("Calling Gemini model, attempt", i + 1)
-                #print("full_prompt:", full_prompt)
-                #print("generation_config:", config)
-
-                response = model.generate_content(contents=full_prompt, generation_config=config)
-
-                #progress_bar.close()
-                #tqdm._instances.clear()
-
-                print("Successful call to Gemini model.")
-                break
-            except Exception as e:
-                # If fails, try again after X seconds in case there is a throttle limit
-                print("Call to Gemini model failed:", e, " Waiting for ", str(timeout_wait), "seconds and trying again.")               
-
-                time.sleep(timeout_wait)
-            
-            if i == number_of_api_retry_attempts:
-                return ResponseObject(text="", usage_metadata={'RequestId':"FAILED"}), conversation_history
-    elif "anthropic.claude" in model_choice:
-        for i in progress_bar:
-            try:
-                print("Calling AWS Claude model, attempt", i + 1)
-                response = call_aws_claude(prompt, system_prompt, temperature, max_tokens, model_choice)
-
-                #progress_bar.close()
-                #tqdm._instances.clear()
-
-                print("Successful call to Claude model.")
-                break
-            except Exception as e:
-                # If fails, try again after X seconds in case there is a throttle limit
-                print("Call to Claude model failed:", e, " Waiting for ", str(timeout_wait), "seconds and trying again.")         
-
-                time.sleep(timeout_wait)
-                #response = call_aws_claude(prompt, system_prompt, temperature, max_tokens, model_choice)
-
-            if i == number_of_api_retry_attempts:
-                return ResponseObject(text="", usage_metadata={'RequestId':"FAILED"}), conversation_history
-    else:
-        # This is the Gemma model
-        for i in progress_bar:
-            try:
-                print("Calling Gemma 2B Instruct model, attempt", i + 1)
-
-                gen_config = LlamaCPPGenerationConfig()
-                gen_config.update_temp(temperature)
-
-                response = call_llama_cpp_model(prompt, gen_config, model=local_model)
-
-                #progress_bar.close()
-                #tqdm._instances.clear()
-
-                print("Successful call to Gemma model.")
-                print("Response:", response)
-                break
-            except Exception as e:
-                # If fails, try again after X seconds in case there is a throttle limit
-                print("Call to Gemma model failed:", e, " Waiting for ", str(timeout_wait), "seconds and trying again.")         
-
-                time.sleep(timeout_wait)
-                #response = call_aws_claude(prompt, system_prompt, temperature, max_tokens, model_choice)
-
-            if i == number_of_api_retry_attempts:
-                return ResponseObject(text="", usage_metadata={'RequestId':"FAILED"}), conversation_history       
-
-    # Update the conversation history with the new prompt and response
-    conversation_history.append({'role': 'user', 'parts': [prompt]})
-
-    # Check if is a LLama.cpp model response
-        # Check if the response is a ResponseObject
-    if isinstance(response, ResponseObject):
-        conversation_history.append({'role': 'assistant', 'parts': [response.text]})
-    elif 'choices' in response:
-        conversation_history.append({'role': 'assistant', 'parts': [response['choices'][0]['text']]})
-    else:
-        conversation_history.append({'role': 'assistant', 'parts': [response.text]})
-    
-    # Print the updated conversation history
-    #print("conversation_history:", conversation_history)
-    
-    return response, conversation_history
-
-def process_requests(prompts: List[str], system_prompt: str, conversation_history: List[dict], whole_conversation: List[str], whole_conversation_metadata: List[str], model: object, config: dict, model_choice: str, temperature: float, batch_no:int = 1, local_model = [], master:bool = False) -> Tuple[List[ResponseObject], List[dict], List[str], List[str]]:
-    """
-    Processes a list of prompts by sending them to the model, appending the responses to the conversation history, and updating the whole conversation and metadata.
-
-    Args:
-        prompts (List[str]): A list of prompts to be processed.
-        system_prompt (str): The system prompt.
-        conversation_history (List[dict]): The history of the conversation.
-        whole_conversation (List[str]): The complete conversation including prompts and responses.
-        whole_conversation_metadata (List[str]): Metadata about the whole conversation.
-        model (object): The model to use for processing the prompts.
-        config (dict): Configuration for the model.
-        model_choice (str): The choice of model to use.        
-        temperature (float): The temperature parameter for the model.
-        batch_no (int): Batch number of the large language model request.
-        local_model: Local gguf model (if loaded)
-        master (bool): Is this request for the master table.
-
-    Returns:
-        Tuple[List[ResponseObject], List[dict], List[str], List[str]]: A tuple containing the list of responses, the updated conversation history, the updated whole conversation, and the updated whole conversation metadata.
-    """
-    responses = []
-
-    # Clear any existing progress bars
-    tqdm._instances.clear()
-
-    for prompt in prompts:
-
-        #print("prompt to LLM:", prompt)
-
-        response, conversation_history = send_request(prompt, conversation_history, model=model, config=config, model_choice=model_choice, system_prompt=system_prompt, temperature=temperature, local_model=local_model)
-
-        if isinstance(response, ResponseObject):
-            response_text = response.text
-        elif 'choices' in response:
-            response_text = response['choices'][0]['text']
-        else:
-            response_text = response.text
-
-        responses.append(response)
-        whole_conversation.append(prompt)
-        whole_conversation.append(response_text)
-
-        # Create conversation metadata
-        if master == False:
-            whole_conversation_metadata.append(f"Query batch {batch_no} prompt {len(responses)} metadata:")
-        else:
-            whole_conversation_metadata.append(f"Query summary metadata:")
-
-        if not isinstance(response, str):
-            try:
-                print("model_choice:", model_choice)
-                if "claude" in model_choice:
-                    print("Appending selected metadata items to metadata")
-                    whole_conversation_metadata.append('x-amzn-bedrock-output-token-count:')
-                    whole_conversation_metadata.append(str(response.usage_metadata['HTTPHeaders']['x-amzn-bedrock-output-token-count']))
-                    whole_conversation_metadata.append('x-amzn-bedrock-input-token-count:')
-                    whole_conversation_metadata.append(str(response.usage_metadata['HTTPHeaders']['x-amzn-bedrock-input-token-count']))
-                elif "gemini" in model_choice:
-                    whole_conversation_metadata.append(str(response.usage_metadata))
-                else:
-                    whole_conversation_metadata.append(str(response['usage']))
-            except KeyError as e:
-                print(f"Key error: {e} - Check the structure of response.usage_metadata")
-        else:
-            print("Response is a string object.")
-            whole_conversation_metadata.append("Length prompt: " + str(len(prompt)) + ". Length response: " + str(len(response)))
-
-
-    return responses, conversation_history, whole_conversation, whole_conversation_metadata, response_text
-
 ### INITIAL TOPIC MODEL DEVELOPMENT FUNCTIONS
 
 def clean_markdown_table(text: str):
@@ -620,22 +283,6 @@ def clean_column_name(column_name, max_length=20):
     # Truncate to max_length
     return column_name[:max_length]
 
-def create_unique_table_df_from_reference_table(reference_df:pd.DataFrame):
-
-    out_unique_topics_df = (reference_df.groupby(["General Topic", "Subtopic", "Sentiment"])
-            .agg({
-                'Response References': 'size',  # Count the number of references
-                'Summary': lambda x: '<br>'.join(
-                    sorted(set(x), key=lambda summary: reference_df.loc[reference_df['Summary'] == summary, 'Start row of group'].min())
-                )
-            })
-            .reset_index()
-            .sort_values('Response References', ascending=False)  # Sort by size, biggest first
-            .assign(Topic_number=lambda df: np.arange(1, len(df) + 1))  # Add numbering 1 to x
-        )
-
-    return out_unique_topics_df
-
 # Convert output table to markdown and then to a pandas dataframe to csv
 def remove_before_last_term(input_string: str) -> str:
     # Use regex to find the last occurrence of the term
@@ -754,8 +401,9 @@ def call_llm_with_markdown_table_checks(batch_prompts: List[str],
                                         temperature: float,
                                         reported_batch_no: int,
                                         local_model: object,
-                                        MAX_OUTPUT_VALIDATION_ATTEMPTS: int,
-                                        master:bool=False) -> Tuple[List[ResponseObject], List[dict], List[str], List[str], str]:
+                                        MAX_OUTPUT_VALIDATION_ATTEMPTS: int,                                        
+                                        master:bool=False,
+                                        CHOSEN_LOCAL_MODEL_TYPE:str=CHOSEN_LOCAL_MODEL_TYPE) -> Tuple[List[ResponseObject], List[dict], List[str], List[str], str]:
     """
     Call the large language model with checks for a valid markdown table.
 
@@ -791,7 +439,7 @@ def call_llm_with_markdown_table_checks(batch_prompts: List[str],
             call_temperature, reported_batch_no, local_model, master=master
         )
 
-        if (model_choice != "gemma_local") & (model_choice != "gemma_2b_it_local"):
+        if model_choice != CHOSEN_LOCAL_MODEL_TYPE:
             stripped_response = responses[-1].text.strip()
         else:
             stripped_response = responses[-1]['choices'][0]['text'].strip()
@@ -824,8 +472,9 @@ def write_llm_output_and_logs(responses: List[ResponseObject],
                               existing_reference_df:pd.DataFrame,
                               existing_topics_df:pd.DataFrame,
                               batch_size_number:int,
-                              in_column:str,
-                              first_run: bool = False) -> None:
+                              in_column:str,                              
+                              first_run: bool = False,
+                              output_folder:str=OUTPUT_FOLDER) -> None:
     """
     Writes the output of the large language model requests and logs to files.
 
@@ -933,12 +582,24 @@ def write_llm_output_and_logs(responses: List[ResponseObject],
         # If no numbers found in the Response References column, check the Summary column in case reference numbers were put there by mistake
         if not references:
             references = re.findall(r'\d+', str(row.iloc[4])) if pd.notna(row.iloc[4]) else []
+        
+        # Filter out references that are outside the valid range
+        if references:
+            try:
+                # Convert all references to integers and keep only those within valid range
+                ref_numbers = [int(ref) for ref in references]
+                references = [str(ref) for ref in ref_numbers if 1 <= ref <= batch_size_number]
+            except ValueError:
+                # If any reference can't be converted to int, skip this row
+                print("Response value could not be converted to number:", references)
+                continue
+        
         topic = row.iloc[0] if pd.notna(row.iloc[0]) else ""
         subtopic = row.iloc[1] if pd.notna(row.iloc[1]) else ""
         sentiment = row.iloc[2] if pd.notna(row.iloc[2]) else ""
         summary = row.iloc[4] if pd.notna(row.iloc[4]) else ""
         # If the reference response column is very long, and there's nothing in the summary column, assume that the summary was put in the reference column
-        if not summary and len(str(row.iloc[3]) > 30):
+        if not summary and (len(str(row.iloc[3])) > 30):
             summary = row.iloc[3]        
 
         summary = row_number_string_start + summary
@@ -1172,10 +833,12 @@ def extract_topics(in_data_file,
               force_zero_shot_radio:str = "No",
               in_excel_sheets:List[str] = [],
               force_single_topic_radio:str = "No",
+              output_folder:str=OUTPUT_FOLDER,
               force_single_topic_prompt:str=force_single_topic_prompt,
               max_tokens:int=max_tokens,
               model_name_map:dict=model_name_map,              
-              max_time_for_loop:int=max_time_for_loop,              
+              max_time_for_loop:int=max_time_for_loop,
+              CHOSEN_LOCAL_MODEL_TYPE:str=CHOSEN_LOCAL_MODEL_TYPE,
               progress=Progress(track_tqdm=True)):
 
     '''
@@ -1215,10 +878,12 @@ def extract_topics(in_data_file,
     - force_zero_shot_radio (str, optional): Should responses be forced into a zero shot topic or not.
     - in_excel_sheets (List[str], optional): List of excel sheets to load from input file.
     - force_single_topic_radio (str, optional): Should the model be forced to assign only one single topic to each response (effectively a classifier).
+    - output_folder (str, optional): Output folder where results will be stored.
     - force_single_topic_prompt (str, optional): The prompt for forcing the model to assign only one single topic to each response.
     - max_tokens (int): The maximum number of tokens for the model.
     - model_name_map (dict, optional): A dictionary mapping full model name to shortened.
     - max_time_for_loop (int, optional): The number of seconds maximum that the function should run for before breaking (to run again, this is to avoid timeouts with some AWS services if deployed there).
+    - CHOSEN_LOCAL_MODEL_TYPE (str, optional): The name of the chosen local model.
     - progress (Progress): A progress tracker.
     '''
 
@@ -1265,9 +930,9 @@ def extract_topics(in_data_file,
             out_message = []
             out_file_paths = []
 
-            if (model_choice == "gemma_2b_it_local") & (RUN_LOCAL_MODEL == "1"):
-                progress(0.1, "Loading in Gemma 2b model")
-                local_model, tokenizer = load_model()
+            if (model_choice == CHOSEN_LOCAL_MODEL_TYPE) & (RUN_LOCAL_MODEL == "1"):
+                progress(0.1, f"Loading in local model: {CHOSEN_LOCAL_MODEL_TYPE}")
+                local_model, tokenizer = load_model(local_model_type=CHOSEN_LOCAL_MODEL_TYPE, repo_id=LOCAL_REPO_ID, model_filename=LOCAL_MODEL_FILE, model_dir=LOCAL_MODEL_FOLDER)
 
     if num_batches > 0:
         progress_measure = round(latest_batch_completed / num_batches, 1)
@@ -1305,7 +970,7 @@ def extract_topics(in_data_file,
             print("Running query batch", str(reported_batch_no))
 
             # Call the function to prepare the input table
-            simplified_csv_table_path, normalised_simple_markdown_table, start_row, end_row, batch_basic_response_df = data_file_to_markdown_table(file_data, file_name, chosen_cols, output_folder, latest_batch_completed, batch_size)
+            simplified_csv_table_path, normalised_simple_markdown_table, start_row, end_row, batch_basic_response_df = data_file_to_markdown_table(file_data, file_name, chosen_cols, latest_batch_completed, batch_size)
             #log_files_output_paths.append(simplified_csv_table_path)
 
             # Conversation history
@@ -1420,7 +1085,7 @@ def extract_topics(in_data_file,
                     responses, conversation_history, whole_conversation, whole_conversation_metadata, response_text = call_llm_with_markdown_table_checks(summary_prompt_list, system_prompt, conversation_history, whole_conversation, whole_conversation_metadata, model, config, model_choice, temperature, reported_batch_no, local_model, MAX_OUTPUT_VALIDATION_ATTEMPTS, master = True)
 
                     # Return output tables
-                    topic_table_out_path, reference_table_out_path, unique_topics_df_out_path, new_topic_df, new_markdown_table, new_reference_df, new_unique_topics_df, master_batch_out_file_part, is_error =  write_llm_output_and_logs(responses, whole_conversation, whole_conversation_metadata, file_name, latest_batch_completed, start_row, end_row, model_choice_clean, temperature, log_files_output_paths, existing_reference_df, existing_unique_topics_df, batch_size, chosen_cols, first_run=False)
+                    topic_table_out_path, reference_table_out_path, unique_topics_df_out_path, new_topic_df, new_markdown_table, new_reference_df, new_unique_topics_df, master_batch_out_file_part, is_error =  write_llm_output_and_logs(responses, whole_conversation, whole_conversation_metadata, file_name, latest_batch_completed, start_row, end_row, model_choice_clean, temperature, log_files_output_paths, existing_reference_df, existing_unique_topics_df, batch_size, chosen_cols, first_run=False, output_folder=output_folder)
 
                     # Write final output to text file for logging purposes
                     try:
@@ -1484,8 +1149,8 @@ def extract_topics(in_data_file,
                     if "gemini" in model_choice:
                         print("Using Gemini model:", model_choice)
                         model, config = construct_gemini_generative_model(in_api_key=in_api_key, temperature=temperature, model_choice=model_choice, system_prompt=system_prompt, max_tokens=max_tokens)
-                    elif "gemma" in model_choice:
-                        print("Using local Gemma model:", model_choice)
+                    elif model_choice == CHOSEN_LOCAL_MODEL_TYPE:
+                        print("Using local model:", model_choice)
                     else:
                         print("Using AWS Bedrock model:", model_choice)
 
@@ -1512,7 +1177,7 @@ def extract_topics(in_data_file,
                     responses, conversation_history, whole_conversation, whole_conversation_metadata, response_text = call_llm_with_markdown_table_checks(batch_prompts, system_prompt, conversation_history, whole_conversation, whole_conversation_metadata, model, config, model_choice, temperature, reported_batch_no, local_model, MAX_OUTPUT_VALIDATION_ATTEMPTS)
 
 
-                    topic_table_out_path, reference_table_out_path, unique_topics_df_out_path, topic_table_df, markdown_table, reference_df, new_unique_topics_df, batch_file_path_details, is_error =  write_llm_output_and_logs(responses, whole_conversation, whole_conversation_metadata, file_name, latest_batch_completed, start_row, end_row, model_choice_clean, temperature, log_files_output_paths, existing_reference_df, existing_unique_topics_df, batch_size, chosen_cols, first_run=True)
+                    topic_table_out_path, reference_table_out_path, unique_topics_df_out_path, topic_table_df, markdown_table, reference_df, new_unique_topics_df, batch_file_path_details, is_error =  write_llm_output_and_logs(responses, whole_conversation, whole_conversation_metadata, file_name, latest_batch_completed, start_row, end_row, model_choice_clean, temperature, log_files_output_paths, existing_reference_df, existing_unique_topics_df, batch_size, chosen_cols, first_run=True, output_folder=output_folder)
 
                     # If error in table parsing, leave function
                     if is_error == True:
@@ -1697,39 +1362,6 @@ def extract_topics(in_data_file,
 
     return unique_table_df_display_table_markdown, existing_topics_table, existing_unique_topics_df, existing_reference_df, out_file_paths, out_file_paths, latest_batch_completed, log_files_output_paths, log_files_output_paths, whole_conversation_metadata_str, final_time, out_file_paths, out_file_paths, gr.Dataframe(value=modifiable_unique_topics_df, headers=None, col_count=(modifiable_unique_topics_df.shape[1], "fixed"), row_count = (modifiable_unique_topics_df.shape[0], "fixed"), visible=True, type="pandas"), out_file_paths, join_file_paths
 
-def convert_reference_table_to_pivot_table(df:pd.DataFrame, basic_response_data:pd.DataFrame=pd.DataFrame()):
-
-    df_in = df[['Response References', 'General Topic', 'Subtopic', 'Sentiment']].copy()
-
-    df_in['Response References'] = df_in['Response References'].astype(int)
-
-    # Create a combined category column
-    df_in['Category'] = df_in['General Topic'] + ' - ' + df_in['Subtopic'] + ' - ' + df_in['Sentiment']
-    
-    # Create pivot table counting occurrences of each unique combination
-    pivot_table = pd.crosstab(
-        index=df_in['Response References'],
-        columns=[df_in['General Topic'], df_in['Subtopic'], df_in['Sentiment']],
-        margins=True
-    )
-    
-    # Flatten column names to make them more readable
-    pivot_table.columns = [' - '.join(col) for col in pivot_table.columns]
-
-    pivot_table.reset_index(inplace=True)
-
-    if not basic_response_data.empty:
-        pivot_table = basic_response_data.merge(pivot_table, right_on="Response References", left_on="Reference", how="left")
-
-        pivot_table.drop("Response References", axis=1, inplace=True)    
-
-    pivot_table.columns = pivot_table.columns.str.replace("Not assessed - ", "").str.replace("- Not assessed", "")
-
-    # print("pivot_table:", pivot_table)
-
-    return pivot_table
-
-
 def join_modified_topic_names_to_ref_table(modified_unique_topics_df:pd.DataFrame, original_unique_topics_df:pd.DataFrame, reference_df:pd.DataFrame):
     '''
     Take a unique topic table that has been modified by the user, and apply the topic name changes to the long-form reference table.
@@ -1770,7 +1402,7 @@ def join_modified_topic_names_to_ref_table(modified_unique_topics_df:pd.DataFram
     return modified_reference_df
 
 # MODIFY EXISTING TABLE
-def modify_existing_output_tables(original_unique_topics_df:pd.DataFrame, modifiable_unique_topics_df:pd.DataFrame, reference_df:pd.DataFrame, text_output_file_list_state:List[str]) -> Tuple:
+def modify_existing_output_tables(original_unique_topics_df:pd.DataFrame, modifiable_unique_topics_df:pd.DataFrame, reference_df:pd.DataFrame, text_output_file_list_state:List[str], output_folder:str=OUTPUT_FOLDER) -> Tuple:
     '''
     Take a unique_topics table that has been modified, apply these new topic names to the long-form reference_df, and save both tables to file.
     '''
@@ -1787,9 +1419,6 @@ def modify_existing_output_tables(original_unique_topics_df:pd.DataFrame, modifi
     reference_file_path = os.path.basename(reference_files[0]) if reference_files else None
     unique_table_file_path = os.path.basename(unique_files[0]) if unique_files else None
 
-    print("Reference File:", reference_file_path)
-    print("Unique Table File:", unique_table_file_path)
-
     output_file_list = []
 
     if reference_file_path and unique_table_file_path:
@@ -1833,583 +1462,3 @@ def modify_existing_output_tables(original_unique_topics_df:pd.DataFrame, modifi
     
 
     return modifiable_unique_topics_df, reference_df, output_file_list, output_file_list, output_file_list, output_file_list, reference_table_file_name, unique_table_file_name, deduplicated_unique_table_markdown
-
-
-# DEDUPLICATION/SUMMARISATION FUNCTIONS
-def deduplicate_categories(category_series: pd.Series, join_series: pd.Series, reference_df: pd.DataFrame, general_topic_series: pd.Series = None, merge_general_topics = "No", merge_sentiment:str="No", threshold: float = 90) -> pd.DataFrame:
-    """
-    Deduplicates similar category names in a pandas Series based on a fuzzy matching threshold,
-    merging smaller topics into larger topics.
-
-    Parameters:
-        category_series (pd.Series): Series containing category names to deduplicate.
-        join_series (pd.Series): Additional series used for joining back to original results.
-        reference_df (pd.DataFrame): DataFrame containing the reference data to count occurrences.
-        threshold (float): Similarity threshold for considering two strings as duplicates.
-
-    Returns:
-        pd.DataFrame: DataFrame with columns ['old_category', 'deduplicated_category'].
-    """
-    # Count occurrences of each category in the reference_df
-    category_counts = reference_df['Subtopic'].value_counts().to_dict()
-
-    # Initialize dictionaries for both category mapping and scores
-    deduplication_map = {}
-    match_scores = {}  # New dictionary to store match scores
-
-    # First pass: Handle exact matches
-    for category in category_series.unique():
-        if category in deduplication_map:
-            continue
-            
-        # Find all exact matches
-        exact_matches = category_series[category_series.str.lower() == category.lower()].index.tolist()
-        if len(exact_matches) > 1:
-            # Find the variant with the highest count
-            match_counts = {match: category_counts.get(category_series[match], 0) for match in exact_matches}
-            most_common = max(match_counts.items(), key=lambda x: x[1])[0]
-            most_common_category = category_series[most_common]
-            
-            # Map all exact matches to the most common variant and store score
-            for match in exact_matches:
-                deduplication_map[category_series[match]] = most_common_category
-                match_scores[category_series[match]] = 100  # Exact matches get score of 100
-
-    # Second pass: Handle fuzzy matches for remaining categories
-    # Create a DataFrame to maintain the relationship between categories and general topics
-    categories_df = pd.DataFrame({
-        'category': category_series,
-        'general_topic': general_topic_series
-    }).drop_duplicates()
-
-    for _, row in categories_df.iterrows():
-        category = row['category']
-        if category in deduplication_map:
-            continue
-
-        current_general_topic = row['general_topic']
-
-        # Filter potential matches to only those within the same General Topic if relevant
-        if merge_general_topics == "No":
-            potential_matches = categories_df[
-                (categories_df['category'] != category) & 
-                (categories_df['general_topic'] == current_general_topic)
-            ]['category'].tolist()
-        else:
-            potential_matches = categories_df[
-                (categories_df['category'] != category)
-            ]['category'].tolist()
-
-        matches = process.extract(category, 
-                                potential_matches,
-                                scorer=fuzz.WRatio,
-                                score_cutoff=threshold)
-
-        if matches:
-            best_match = max(matches, key=lambda x: x[1])
-            match, score, _ = best_match
-
-            if category_counts.get(category, 0) < category_counts.get(match, 0):
-                deduplication_map[category] = match
-                match_scores[category] = score
-            else:
-                deduplication_map[match] = category
-                match_scores[match] = score
-        else:
-            deduplication_map[category] = category
-            match_scores[category] = 100
-
-    # Create the result DataFrame with scores
-    result_df = pd.DataFrame({
-        'old_category': category_series + " | " + join_series,
-        'deduplicated_category': category_series.map(lambda x: deduplication_map.get(x, x)),
-        'match_score': category_series.map(lambda x: match_scores.get(x, 100))  # Add scores column
-    })
-
-    #print(result_df)
-
-    return result_df
-
-def deduplicate_topics(reference_df:pd.DataFrame,
-                       unique_topics_df:pd.DataFrame,
-                       reference_table_file_name:str,
-                       unique_topics_table_file_name:str,
-                       in_excel_sheets:str="",
-                       merge_sentiment:str= "No",
-                       merge_general_topics:str="No",
-                       score_threshold:int=90,
-                       in_data_files:List[str]=[],
-                       chosen_cols:List[str]="",
-                       deduplicate_topics:str="Yes"
-                       ):
-    '''
-    Deduplicate topics based on a reference and unique topics table
-    '''
-    output_files = []
-    log_output_files = []
-    file_data = pd.DataFrame()
-
-    reference_table_file_name_no_ext = reference_table_file_name #get_file_name_no_ext(reference_table_file_name)
-    unique_topics_table_file_name_no_ext = unique_topics_table_file_name #get_file_name_no_ext(unique_topics_table_file_name)
-
-    # For checking that data is not lost during the process
-    initial_unique_references = len(reference_df["Response References"].unique())
-
-    if unique_topics_df.empty:
-        unique_topics_df = create_unique_table_df_from_reference_table(reference_df)
-
-        # Then merge the topic numbers back to the original dataframe
-        reference_df = reference_df.merge(
-            unique_topics_df[['General Topic', 'Subtopic', 'Sentiment', 'Topic_number']],
-            on=['General Topic', 'Subtopic', 'Sentiment'],
-            how='left'
-        )     
-
-    if in_data_files and chosen_cols:
-        file_data, data_file_names_textbox, total_number_of_batches = load_in_data_file(in_data_files, chosen_cols, 1, in_excel_sheets)
-    else:
-        out_message = "No file data found, pivot table output will not be created."
-        print(out_message)
-        #raise Exception(out_message)
-
-    # Run through this x times to try to get all duplicate topics
-    if deduplicate_topics == "Yes":
-        for i in range(0, 8):
-            if merge_sentiment == "No":    
-                if merge_general_topics == "No":
-                    reference_df["old_category"] = reference_df["Subtopic"] + " | " + reference_df["Sentiment"]
-                    reference_df_unique = reference_df.drop_duplicates("old_category")
-
-                    deduplicated_topic_map_df = reference_df_unique.groupby(["General Topic", "Sentiment"]).apply(
-                        lambda group: deduplicate_categories(
-                            group["Subtopic"], 
-                            group["Sentiment"], 
-                            reference_df, 
-                            general_topic_series=group["General Topic"],
-                            merge_general_topics="No",
-                            threshold=score_threshold
-                        )
-                    ).reset_index(drop=True)
-                else:
-                    # This case should allow cross-topic matching but is still grouping by Sentiment
-                    reference_df["old_category"] = reference_df["Subtopic"] + " | " + reference_df["Sentiment"]
-                    reference_df_unique = reference_df.drop_duplicates("old_category")
-
-                    deduplicated_topic_map_df = reference_df_unique.groupby("Sentiment").apply(
-                        lambda group: deduplicate_categories(
-                            group["Subtopic"], 
-                            group["Sentiment"], 
-                            reference_df, 
-                            general_topic_series=None,  # Set to None to allow cross-topic matching
-                            merge_general_topics="Yes",
-                            threshold=score_threshold
-                        )
-                    ).reset_index(drop=True)
-            else:
-                if merge_general_topics == "No":
-                    # Update this case to maintain general topic boundaries
-                    reference_df["old_category"] = reference_df["Subtopic"] + " | " + reference_df["Sentiment"]
-                    reference_df_unique = reference_df.drop_duplicates("old_category")
-
-                    deduplicated_topic_map_df = reference_df_unique.groupby("General Topic").apply(
-                        lambda group: deduplicate_categories(
-                            group["Subtopic"], 
-                            group["Sentiment"], 
-                            reference_df, 
-                            general_topic_series=group["General Topic"],
-                            merge_general_topics="No",
-                            merge_sentiment=merge_sentiment, 
-                            threshold=score_threshold
-                        )
-                    ).reset_index(drop=True)
-                else:
-                    # For complete merging across all categories
-                    reference_df["old_category"] = reference_df["Subtopic"] + " | " + reference_df["Sentiment"]
-                    reference_df_unique = reference_df.drop_duplicates("old_category")
-
-                    deduplicated_topic_map_df = deduplicate_categories(
-                        reference_df_unique["Subtopic"], 
-                        reference_df_unique["Sentiment"], 
-                        reference_df, 
-                        general_topic_series=None,  # Set to None to allow cross-topic matching
-                        merge_general_topics="Yes",
-                        merge_sentiment=merge_sentiment,
-                        threshold=score_threshold
-                    ).reset_index(drop=True)
-           
-            if deduplicated_topic_map_df['deduplicated_category'].isnull().all():
-            # Check if 'deduplicated_category' contains any values
-                print("No deduplicated categories found, skipping the following code.")
-
-            else:
-                # Remove rows where 'deduplicated_category' is blank or NaN
-                deduplicated_topic_map_df = deduplicated_topic_map_df.loc[(deduplicated_topic_map_df['deduplicated_category'].str.strip() != '') & ~(deduplicated_topic_map_df['deduplicated_category'].isnull()), ['old_category','deduplicated_category', 'match_score']]
-
-                #deduplicated_topic_map_df.to_csv(output_folder + "deduplicated_topic_map_df_" + str(i) + ".csv", index=None)
-
-                reference_df = reference_df.merge(deduplicated_topic_map_df, on="old_category", how="left")
-
-                reference_df.rename(columns={"Subtopic": "Subtopic_old", "Sentiment": "Sentiment_old"}, inplace=True)
-                # Extract subtopic and sentiment from deduplicated_category
-                reference_df["Subtopic"] = reference_df["deduplicated_category"].str.extract(r'^(.*?) \|')[0]  # Extract subtopic
-                reference_df["Sentiment"] = reference_df["deduplicated_category"].str.extract(r'\| (.*)$')[0]  # Extract sentiment
-
-                # Combine with old values to ensure no data is lost
-                reference_df["Subtopic"] = reference_df["deduplicated_category"].combine_first(reference_df["Subtopic_old"])
-                reference_df["Sentiment"] = reference_df["Sentiment"].combine_first(reference_df["Sentiment_old"])
-
-
-            reference_df.drop(['old_category', 'deduplicated_category', "Subtopic_old", "Sentiment_old"], axis=1, inplace=True, errors="ignore")
-
-            reference_df = reference_df[["Response References", "General Topic", "Subtopic", "Sentiment", "Summary", "Start row of group"]]
-
-            #reference_df["General Topic"] = reference_df["General Topic"].str.lower().str.capitalize() 
-            #reference_df["Subtopic"] = reference_df["Subtopic"].str.lower().str.capitalize() 
-            #reference_df["Sentiment"] = reference_df["Sentiment"].str.lower().str.capitalize() 
-
-            if merge_general_topics == "Yes":
-                # Replace General topic names for each Subtopic with that for the Subtopic with the most responses
-                # Step 1: Count the number of occurrences for each General Topic and Subtopic combination
-                count_df = reference_df.groupby(['Subtopic', 'General Topic']).size().reset_index(name='Count')
-
-                # Step 2: Find the General Topic with the maximum count for each Subtopic
-                max_general_topic = count_df.loc[count_df.groupby('Subtopic')['Count'].idxmax()]
-
-                # Step 3: Map the General Topic back to the original DataFrame
-                reference_df = reference_df.merge(max_general_topic[['Subtopic', 'General Topic']], on='Subtopic', suffixes=('', '_max'), how='left')
-
-                reference_df['General Topic'] = reference_df["General Topic_max"].combine_first(reference_df["General Topic"])        
-
-            if merge_sentiment == "Yes":
-                # Step 1: Count the number of occurrences for each General Topic and Subtopic combination
-                count_df = reference_df.groupby(['Subtopic', 'Sentiment']).size().reset_index(name='Count')
-
-                # Step 2: Determine the number of unique Sentiment values for each Subtopic
-                unique_sentiments = count_df.groupby('Subtopic')['Sentiment'].nunique().reset_index(name='UniqueCount')
-
-                # Step 3: Update Sentiment to 'Mixed' where there is more than one unique sentiment
-                reference_df = reference_df.merge(unique_sentiments, on='Subtopic', how='left')
-                reference_df['Sentiment'] = reference_df.apply(
-                    lambda row: 'Mixed' if row['UniqueCount'] > 1 else row['Sentiment'],
-                    axis=1
-                )
-
-                # Clean up the DataFrame by dropping the UniqueCount column
-                reference_df.drop(columns=['UniqueCount'], inplace=True)
-
-            reference_df = reference_df[["Response References", "General Topic", "Subtopic", "Sentiment", "Summary", "Start row of group"]]
-
-        # Update reference summary column with all summaries
-        reference_df["Summary"] = reference_df.groupby(
-        ["Response References", "General Topic", "Subtopic", "Sentiment"]
-        )["Summary"].transform(' <br> '.join)
-
-        # Check that we have not inadvertantly removed some data during the above process
-        end_unique_references = len(reference_df["Response References"].unique())
-
-        if initial_unique_references != end_unique_references:
-            raise Exception(f"Number of unique references changed during processing: Initial={initial_unique_references}, Final={end_unique_references}")
-        
-        # Drop duplicates in the reference table - each comment should only have the same topic referred to once
-        reference_df.drop_duplicates(['Response References', 'General Topic', 'Subtopic', 'Sentiment'], inplace=True)
-
-
-        # Remake unique_topics_df based on new reference_df
-        unique_topics_df = create_unique_table_df_from_reference_table(reference_df)
-
-        # Then merge the topic numbers back to the original dataframe
-        reference_df = reference_df.merge(
-            unique_topics_df[['General Topic', 'Subtopic', 'Sentiment', 'Topic_number']],
-            on=['General Topic', 'Subtopic', 'Sentiment'],
-            how='left'
-        ) 
-
-        if not file_data.empty:
-            basic_response_data = get_basic_response_data(file_data, chosen_cols)            
-            reference_df_pivot = convert_reference_table_to_pivot_table(reference_df, basic_response_data)
-
-            reference_pivot_file_path = output_folder + reference_table_file_name_no_ext + "_pivot_dedup.csv"
-            reference_df_pivot.to_csv(reference_pivot_file_path, index=None, encoding='utf-8')
-            log_output_files.append(reference_pivot_file_path)
-
-        #reference_table_file_name_no_ext = get_file_name_no_ext(reference_table_file_name)
-        #unique_topics_table_file_name_no_ext = get_file_name_no_ext(unique_topics_table_file_name)
-
-        reference_file_path = output_folder + reference_table_file_name_no_ext + "_dedup.csv"
-        unique_topics_file_path = output_folder + unique_topics_table_file_name_no_ext + "_dedup.csv"
-        reference_df.to_csv(reference_file_path, index = None, encoding='utf-8')
-        unique_topics_df.to_csv(unique_topics_file_path, index=None, encoding='utf-8')
-
-        output_files.append(reference_file_path)
-        output_files.append(unique_topics_file_path)        
-
-        # Outputs for markdown table output
-        unique_table_df_revised_display = unique_topics_df.apply(lambda col: col.map(lambda x: wrap_text(x, max_text_length=500)))
-
-        deduplicated_unique_table_markdown = unique_table_df_revised_display.to_markdown(index=False)
-
-    return reference_df, unique_topics_df, output_files, log_output_files, deduplicated_unique_table_markdown
-
-def sample_reference_table_summaries(reference_df:pd.DataFrame,
-                                     unique_topics_df:pd.DataFrame,
-                                     random_seed:int,
-                                     no_of_sampled_summaries:int=150):
-    
-    '''
-    Sample x number of summaries from which to produce summaries, so that the input token length is not too long.
-    '''
-    
-    all_summaries = pd.DataFrame()
-    output_files = []
-
-    reference_df_grouped = reference_df.groupby(["General Topic", "Subtopic", "Sentiment"])
-
-    if 'Revised summary' in reference_df.columns:
-        out_message = "Summary has already been created for this file"
-        print(out_message)
-        raise Exception(out_message)
-
-    for group_keys, reference_df_group in reference_df_grouped:
-        #print(f"Group: {group_keys}")
-        #print(f"Data: {reference_df_group}")
-
-        if len(reference_df_group["General Topic"]) > 1:
-
-            filtered_reference_df = reference_df_group.reset_index()
-
-            filtered_reference_df_unique = filtered_reference_df.drop_duplicates(["General Topic", "Subtopic", "Sentiment", "Summary"])
-
-            # Sample n of the unique topic summaries. To limit the length of the text going into the summarisation tool
-            filtered_reference_df_unique_sampled = filtered_reference_df_unique.sample(min(no_of_sampled_summaries, len(filtered_reference_df_unique)), random_state=random_seed)
-
-            #topic_summary_table_markdown = filtered_reference_df_unique_sampled.to_markdown(index=False)
-
-            #print(filtered_reference_df_unique_sampled)
-
-            all_summaries = pd.concat([all_summaries, filtered_reference_df_unique_sampled])
-    
-    summarised_references = all_summaries.groupby(["General Topic", "Subtopic", "Sentiment"]).agg({
-    'Response References': 'size',  # Count the number of references
-    'Summary': lambda x: '\n'.join([s.split(': ', 1)[1] for s in x if ': ' in s])  # Join substrings after ': '
-    }).reset_index()
-
-    summarised_references = summarised_references.loc[(summarised_references["Sentiment"] != "Not Mentioned") & (summarised_references["Response References"] > 1)]
-
-    summarised_references_markdown = summarised_references.to_markdown(index=False)
-
-    return summarised_references, summarised_references_markdown, reference_df, unique_topics_df
-
-def summarise_output_topics_query(model_choice:str, in_api_key:str, temperature:float, formatted_summary_prompt:str, summarise_topic_descriptions_system_prompt:str, local_model=[]):
-    conversation_history = []
-    whole_conversation_metadata = []
-
-    # Prepare Gemini models before query       
-    if "gemini" in model_choice:
-        print("Using Gemini model:", model_choice)
-        model, config = construct_gemini_generative_model(in_api_key=in_api_key, temperature=temperature, model_choice=model_choice, system_prompt=system_prompt, max_tokens=max_tokens)
-    else:
-        print("Using AWS Bedrock model:", model_choice)
-        model = model_choice
-        config = {}
-
-    whole_conversation = [summarise_topic_descriptions_system_prompt] 
-
-    # Process requests to large language model
-    responses, conversation_history, whole_conversation, whole_conversation_metadata, response_text = process_requests(formatted_summary_prompt, system_prompt, conversation_history, whole_conversation, whole_conversation_metadata, model, config, model_choice, temperature, local_model=local_model)
-
-    print("Finished summary query")
-
-    if isinstance(responses[-1], ResponseObject):
-        response_texts = [resp.text for resp in responses]
-    elif "choices" in responses[-1]:
-        response_texts = [resp["choices"][0]['text'] for resp in responses]
-    else:
-        response_texts = [resp.text for resp in responses]
-
-    latest_response_text = response_texts[-1]
-
-    #print("latest_response_text:", latest_response_text)
-    #print("Whole conversation metadata:", whole_conversation_metadata)
-
-    return latest_response_text, conversation_history, whole_conversation_metadata
-
-@spaces.GPU
-def summarise_output_topics(summarised_references:pd.DataFrame,
-                            unique_table_df:pd.DataFrame,
-                            reference_table_df:pd.DataFrame,
-                            model_choice:str,
-                            in_api_key:str,
-                            topic_summary_table_markdown:str,
-                            temperature:float,
-                            table_file_name:str,
-                            summarised_outputs:list = [],  
-                            latest_summary_completed:int = 0,
-                            out_metadata_str:str = "",
-                            in_data_files:List[str]=[],
-                            in_excel_sheets:str="",
-                            chosen_cols:List[str]=[],
-                            log_output_files:list[str]=[],
-                            summarise_format_radio:str="Return a summary up to two paragraphs long that includes as much detail as possible from the original text",
-                            output_files:list[str] = [],                            
-                            summarise_topic_descriptions_prompt:str=summarise_topic_descriptions_prompt, summarise_topic_descriptions_system_prompt:str=summarise_topic_descriptions_system_prompt,
-                            do_summaries="Yes",
-                            progress=gr.Progress(track_tqdm=True)):
-    '''
-    Create better summaries of the raw batch-level summaries created in the first run of the model.
-    '''
-    out_metadata = []
-    local_model = []
-    summarised_output_markdown = ""
-    
-
-    # Check for data for summarisations
-    if not unique_table_df.empty and not reference_table_df.empty:
-        print("Unique table and reference table data found.")
-    else:
-        out_message = "Please upload a unique topic table and reference table file to continue with summarisation."
-        print(out_message)
-        raise Exception(out_message)
-    
-    if 'Revised summary' in reference_table_df.columns:
-        out_message = "Summary has already been created for this file"
-        print(out_message)
-        raise Exception(out_message)
-    
-    # Load in data file and chosen columns if exists to create pivot table later
-    if in_data_files and chosen_cols:
-        file_data, data_file_names_textbox, total_number_of_batches = load_in_data_file(in_data_files, chosen_cols, 1, in_excel_sheets=in_excel_sheets)
-    else:
-        out_message = "No file data found, pivot table output will not be created."
-        print(out_message)
-        raise Exception(out_message)
-   
-    
-    all_summaries = summarised_references["Summary"].tolist()
-    length_all_summaries = len(all_summaries)
-
-    # If all summaries completed, make final outputs
-    if latest_summary_completed >= length_all_summaries:
-        print("All summaries completed. Creating outputs.")
-
-        model_choice_clean = model_name_map[model_choice]   
-        file_name = re.search(r'(.*?)(?:_batch_|_col_)', table_file_name).group(1) if re.search(r'(.*?)(?:_batch_|_col_)', table_file_name) else table_file_name
-        latest_batch_completed = int(re.search(r'batch_(\d+)_', table_file_name).group(1)) if 'batch_' in table_file_name else ""
-        batch_size_number = int(re.search(r'size_(\d+)_', table_file_name).group(1)) if 'size_' in table_file_name else ""
-        in_column_cleaned = re.search(r'col_(.*?)_reference', table_file_name).group(1) if 'col_' in table_file_name else ""
-
-        # Save outputs for each batch. If master file created, label file as master
-        if latest_batch_completed:
-            batch_file_path_details = f"{file_name}_batch_{latest_batch_completed}_size_{batch_size_number}_col_{in_column_cleaned}"
-        else:
-            batch_file_path_details = f"{file_name}_col_{in_column_cleaned}"
-
-        summarised_references["Revised summary"] = summarised_outputs        
-
-        join_cols = ["General Topic", "Subtopic", "Sentiment"]
-        join_plus_summary_cols = ["General Topic", "Subtopic", "Sentiment", "Revised summary"]
-
-        summarised_references_j = summarised_references[join_plus_summary_cols].drop_duplicates(join_plus_summary_cols)
-
-        unique_table_df_revised = unique_table_df.merge(summarised_references_j, on = join_cols, how = "left")
-
-        # If no new summary is available, keep the original
-        unique_table_df_revised["Revised summary"] = unique_table_df_revised["Revised summary"].combine_first(unique_table_df_revised["Summary"])
-
-        unique_table_df_revised = unique_table_df_revised[["General Topic",	"Subtopic",	"Sentiment", "Response References",	"Revised summary"]]        
-
-        reference_table_df_revised = reference_table_df.merge(summarised_references_j, on = join_cols, how = "left")
-        # If no new summary is available, keep the original
-        reference_table_df_revised["Revised summary"] = reference_table_df_revised["Revised summary"].combine_first(reference_table_df_revised["Summary"])
-        reference_table_df_revised = reference_table_df_revised.drop("Summary", axis=1)
-
-        # Remove topics that are tagged as 'Not Mentioned'
-        unique_table_df_revised = unique_table_df_revised.loc[unique_table_df_revised["Sentiment"] != "Not Mentioned", :]
-        reference_table_df_revised = reference_table_df_revised.loc[reference_table_df_revised["Sentiment"] != "Not Mentioned", :]
-        
-          
-            
-
-        if not file_data.empty:
-            basic_response_data = get_basic_response_data(file_data, chosen_cols)
-            reference_table_df_revised_pivot = convert_reference_table_to_pivot_table(reference_table_df_revised, basic_response_data)
-
-            ### Save pivot file to log area
-            reference_table_df_revised_pivot_path = output_folder + batch_file_path_details + "_summarised_reference_table_pivot_" + model_choice_clean + ".csv"
-            reference_table_df_revised_pivot.to_csv(reference_table_df_revised_pivot_path, index=None, encoding='utf-8')
-            log_output_files.append(reference_table_df_revised_pivot_path)
-
-        # Save to file
-        unique_table_df_revised_path = output_folder + batch_file_path_details + "_summarised_unique_topic_table_" + model_choice_clean + ".csv"
-        unique_table_df_revised.to_csv(unique_table_df_revised_path, index = None, encoding='utf-8')
-
-        reference_table_df_revised_path = output_folder + batch_file_path_details + "_summarised_reference_table_" + model_choice_clean + ".csv"
-        reference_table_df_revised.to_csv(reference_table_df_revised_path, index = None, encoding='utf-8')
-
-        output_files.extend([reference_table_df_revised_path, unique_table_df_revised_path])       
-
-        ###
-        unique_table_df_revised_display = unique_table_df_revised.apply(lambda col: col.map(lambda x: wrap_text(x, max_text_length=500)))
-
-        summarised_output_markdown = unique_table_df_revised_display.to_markdown(index=False)
-
-        # Ensure same file name not returned twice
-        output_files = list(set(output_files))
-        log_output_files = list(set(log_output_files))
-
-        return summarised_references, unique_table_df_revised, reference_table_df_revised, output_files, summarised_outputs, latest_summary_completed, out_metadata_str, summarised_output_markdown, log_output_files
-
-    tic = time.perf_counter()
-    
-    #print("Starting with:", latest_summary_completed)
-    #print("Last summary number:", length_all_summaries)
-
-    if (model_choice == "gemma_2b_it_local") & (RUN_LOCAL_MODEL == "1"):
-                progress(0.1, "Loading in Gemma 2b model")
-                local_model, tokenizer = load_model()
-                #print("Local model loaded:", local_model)
-
-    summary_loop_description = "Creating summaries. " + str(latest_summary_completed) + " summaries completed so far."
-    summary_loop = tqdm(range(latest_summary_completed, length_all_summaries), desc="Creating summaries", unit="summaries")   
-
-    if do_summaries == "Yes":
-        for summary_no in summary_loop:
-
-            print("Current summary number is:", summary_no)
-
-            summary_text = all_summaries[summary_no]
-            #print("summary_text:", summary_text)
-            formatted_summary_prompt = [summarise_topic_descriptions_prompt.format(summaries=summary_text, summary_format=summarise_format_radio)]
-
-            try:
-                response, conversation_history, metadata = summarise_output_topics_query(model_choice, in_api_key, temperature, formatted_summary_prompt, summarise_topic_descriptions_system_prompt, local_model)
-                summarised_output = response
-                summarised_output = re.sub(r'\n{2,}', '\n', summarised_output)  # Replace multiple line breaks with a single line break
-                summarised_output = re.sub(r'^\n{1,}', '', summarised_output)  # Remove one or more line breaks at the start
-                summarised_output = summarised_output.strip()
-            except Exception as e:
-                print(e)
-                summarised_output = ""
-
-            summarised_outputs.append(summarised_output)
-            out_metadata.extend(metadata)
-            out_metadata_str = '. '.join(out_metadata)
-
-            latest_summary_completed += 1
-
-            # Check if beyond max time allowed for processing and break if necessary
-            toc = time.perf_counter()
-            time_taken = tic - toc
-
-            if time_taken > max_time_for_loop:
-                print("Time taken for loop is greater than maximum time allowed. Exiting and restarting loop")
-                summary_loop.close()
-                tqdm._instances.clear()
-                break
-
-    # If all summaries completeed
-    if latest_summary_completed >= length_all_summaries:
-        print("At last summary.")
-
-    output_files = list(set(output_files))
-
-    return summarised_references, unique_table_df, reference_table_df, output_files, summarised_outputs, latest_summary_completed, out_metadata_str, summarised_output_markdown, log_output_files

tools/llm_funcs.py

@@ -0,0 +1,579 @@
+from typing import TypeVar
+import torch.cuda
+import os
+import time
+import boto3
+import json
+from tqdm import tqdm
+
+from huggingface_hub import hf_hub_download
+from typing import List, Tuple
+import google.generativeai as ai
+import gradio as gr
+from gradio import Progress
+
+torch.cuda.empty_cache()
+
+PandasDataFrame = TypeVar('pd.core.frame.DataFrame')
+
+model_type = None # global variable setup
+full_text = "" # Define dummy source text (full text) just to enable highlight function to load
+model = [] # Define empty list for model functions to run
+tokenizer = [] #[] # Define empty list for model functions to run
+
+
+# Both models are loaded on app initialisation so that users don't have to wait for the models to be downloaded
+# Check for torch cuda
+print("Is CUDA enabled? ", torch.cuda.is_available())
+print("Is a CUDA device available on this computer?", torch.backends.cudnn.enabled)
+if torch.cuda.is_available():
+    torch_device = "cuda"
+    gpu_layers = -1
+    os.system("nvidia-smi")
+else: 
+    torch_device =  "cpu"
+    gpu_layers = 0
+
+print("Device used is: ", torch_device)
+print("Running on device:", torch_device)
+
+from tools.config import RUN_AWS_FUNCTIONS, AWS_REGION, LLM_TEMPERATURE, LLM_TOP_K, LLM_TOP_P, LLM_REPETITION_PENALTY, LLM_LAST_N_TOKENS, LLM_MAX_NEW_TOKENS, LLM_SEED, LLM_RESET, LLM_STREAM, LLM_THREADS, LLM_BATCH_SIZE, LLM_CONTEXT_LENGTH, LLM_SAMPLE, MAX_TOKENS, TIMEOUT_WAIT, NUMBER_OF_RETRY_ATTEMPTS, MAX_TIME_FOR_LOOP, BATCH_SIZE_DEFAULT, DEDUPLICATION_THRESHOLD, MAX_COMMENT_CHARS, RUN_LOCAL_MODEL, CHOSEN_LOCAL_MODEL_TYPE, LOCAL_REPO_ID, LOCAL_MODEL_FILE, LOCAL_MODEL_FOLDER, HF_TOKEN
+
+if RUN_LOCAL_MODEL == "1":
+    print("Running local model - importing llama-cpp-python")
+    from llama_cpp import Llama
+
+max_tokens = MAX_TOKENS
+timeout_wait = TIMEOUT_WAIT
+number_of_api_retry_attempts = NUMBER_OF_RETRY_ATTEMPTS
+max_time_for_loop = MAX_TIME_FOR_LOOP
+batch_size_default = BATCH_SIZE_DEFAULT
+deduplication_threshold = DEDUPLICATION_THRESHOLD
+max_comment_character_length = MAX_COMMENT_CHARS
+
+if RUN_AWS_FUNCTIONS == '1':
+    bedrock_runtime = boto3.client('bedrock-runtime', region_name=AWS_REGION)
+else:
+    bedrock_runtime = []
+
+if not LLM_THREADS:
+    threads = torch.get_num_threads() # 8
+else: threads = LLM_THREADS
+print("CPU threads:", threads)
+
+if LLM_RESET == 'True': reset = True
+else: reset = False
+
+if LLM_STREAM == 'True': stream = True
+else: stream = False
+
+if LLM_SAMPLE == 'True': sample = True
+else: sample = False
+
+temperature = LLM_TEMPERATURE
+top_k = LLM_TOP_K
+top_p = LLM_TOP_P
+repetition_penalty = LLM_REPETITION_PENALTY # Mild repetition penalty to prevent repeating table rows
+last_n_tokens = LLM_LAST_N_TOKENS
+max_new_tokens: int = LLM_MAX_NEW_TOKENS
+seed: int = LLM_SEED
+reset: bool = reset
+stream: bool = stream
+threads: int = threads
+batch_size:int = LLM_BATCH_SIZE
+context_length:int = LLM_CONTEXT_LENGTH
+sample = LLM_SAMPLE
+
+class llama_cpp_init_config_gpu:
+    def __init__(self,
+                 last_n_tokens=last_n_tokens,
+                 seed=seed,
+                 n_threads=threads,
+                 n_batch=batch_size,
+                 n_ctx=context_length,
+                 n_gpu_layers=gpu_layers):
+
+        self.last_n_tokens = last_n_tokens
+        self.seed = seed
+        self.n_threads = n_threads
+        self.n_batch = n_batch
+        self.n_ctx = n_ctx
+        self.n_gpu_layers = n_gpu_layers
+        # self.stop: list[str] = field(default_factory=lambda: [stop_string])
+
+    def update_gpu(self, new_value):
+        self.n_gpu_layers = new_value
+
+    def update_context(self, new_value):
+        self.n_ctx = new_value
+
+class llama_cpp_init_config_cpu(llama_cpp_init_config_gpu):
+    def __init__(self):
+        super().__init__()
+        self.n_gpu_layers = gpu_layers
+        self.n_ctx=context_length
+
+gpu_config = llama_cpp_init_config_gpu()
+cpu_config = llama_cpp_init_config_cpu()
+
+
+class LlamaCPPGenerationConfig:
+    def __init__(self, temperature=temperature,
+                 top_k=top_k,
+                 top_p=top_p,
+                 repeat_penalty=repetition_penalty,
+                 seed=seed,
+                 stream=stream,
+                 max_tokens=max_new_tokens
+                 ):
+        self.temperature = temperature
+        self.top_k = top_k
+        self.top_p = top_p
+        self.repeat_penalty = repeat_penalty
+        self.seed = seed
+        self.max_tokens=max_tokens
+        self.stream = stream
+
+    def update_temp(self, new_value):
+        self.temperature = new_value
+
+# ResponseObject class for AWS Bedrock calls
+class ResponseObject:
+        def __init__(self, text, usage_metadata):
+            self.text = text
+            self.usage_metadata = usage_metadata
+
+###
+# LOCAL MODEL FUNCTIONS
+###
+
+def get_model_path(repo_id=LOCAL_REPO_ID, model_filename=LOCAL_MODEL_FILE, model_dir=LOCAL_MODEL_FOLDER, hf_token=HF_TOKEN):
+    # Construct the expected local path
+    local_path = os.path.join(model_dir, model_filename)
+
+    print("local path for model load:", local_path)
+
+    if os.path.exists(local_path):
+        print(f"Model already exists at: {local_path}")
+
+        return local_path
+    else:
+        print(f"Checking default Hugging Face folder. Downloading model from Hugging Face Hub if not found")
+        if hf_token:
+            downloaded_model_path = hf_hub_download(repo_id=repo_id, token=hf_token, filename=model_filename)
+        else:
+            downloaded_model_path = hf_hub_download(repo_id=repo_id, filename=model_filename)
+
+        return downloaded_model_path
+
+def load_model(local_model_type:str=CHOSEN_LOCAL_MODEL_TYPE, gpu_layers:int=gpu_layers, max_context_length:int=context_length, gpu_config:llama_cpp_init_config_gpu=gpu_config, cpu_config:llama_cpp_init_config_cpu=cpu_config, torch_device:str=torch_device, repo_id=LOCAL_REPO_ID, model_filename=LOCAL_MODEL_FILE, model_dir=LOCAL_MODEL_FOLDER):
+    '''
+    Load in a model from Hugging Face hub via the transformers package, or using llama_cpp_python by downloading a GGUF file from Huggingface Hub. 
+    '''
+    print("Loading model ", local_model_type)
+    model_path = get_model_path(repo_id=repo_id, model_filename=model_filename, model_dir=model_dir)  
+
+    print("model_path:", model_path)
+
+    # GPU mode    
+    if torch_device == "cuda":
+        gpu_config.update_gpu(gpu_layers)
+        gpu_config.update_context(max_context_length)        
+
+        try:
+            print("GPU load variables:" , vars(gpu_config))
+            llama_model = Llama(model_path=model_path, **vars(gpu_config)) #  type_k=8, type_v = 8, flash_attn=True, 
+    
+        except Exception as e:
+            print("GPU load failed due to:", e)
+            llama_model = Llama(model_path=model_path, type_k=8, **vars(cpu_config)) # type_v = 8, flash_attn=True, 
+        
+        print("Loading with", gpu_config.n_gpu_layers, "model layers sent to GPU. And a maximum context length of ", gpu_config.n_ctx)
+    
+    # CPU mode
+    else:
+        gpu_config.update_gpu(gpu_layers)
+        cpu_config.update_gpu(gpu_layers)
+
+        # Update context length according to slider
+        gpu_config.update_context(max_context_length)
+        cpu_config.update_context(max_context_length)
+
+        llama_model = Llama(model_path=model_path, type_k=8, **vars(cpu_config)) # type_v = 8, flash_attn=True, 
+
+        print("Loading with", cpu_config.n_gpu_layers, "model layers sent to GPU. And a maximum context length of ", gpu_config.n_ctx)
+    
+    tokenizer = []
+
+    print("Finished loading model:", local_model_type)
+    return llama_model, tokenizer
+
+def call_llama_cpp_model(formatted_string:str, gen_config:str, model=model):
+    """
+    Calls your generation model with parameters from the LlamaCPPGenerationConfig object.
+
+    Args:
+        formatted_string (str): The formatted input text for the model.
+        gen_config (LlamaCPPGenerationConfig): An object containing generation parameters.
+    """
+    # Extracting parameters from the gen_config object
+    temperature = gen_config.temperature
+    top_k = gen_config.top_k
+    top_p = gen_config.top_p
+    repeat_penalty = gen_config.repeat_penalty
+    seed = gen_config.seed
+    max_tokens = gen_config.max_tokens
+    stream = gen_config.stream
+
+    # Now you can call your model directly, passing the parameters:
+    output = model(
+        formatted_string, 
+        temperature=temperature, 
+        top_k=top_k, 
+        top_p=top_p, 
+        repeat_penalty=repeat_penalty, 
+        seed=seed,
+        max_tokens=max_tokens,
+        stream=stream#,
+        #stop=["<|eot_id|>", "\n\n"]
+    )
+
+    return output
+
+# This function is not used in this app
+def llama_cpp_streaming(history, full_prompt, temperature=temperature):
+
+    gen_config = LlamaCPPGenerationConfig()
+    gen_config.update_temp(temperature)
+
+    print(vars(gen_config))
+
+    # Pull the generated text from the streamer, and update the model output.
+    start = time.time()
+    NUM_TOKENS=0
+    print('-'*4+'Start Generation'+'-'*4)
+
+    output = model(
+    full_prompt, **vars(gen_config))
+
+    history[-1][1] = ""
+    for out in output:
+
+        if "choices" in out and len(out["choices"]) > 0 and "text" in out["choices"][0]:
+            history[-1][1] += out["choices"][0]["text"]
+            NUM_TOKENS+=1
+            yield history
+        else:
+            print(f"Unexpected output structure: {out}") 
+
+    time_generate = time.time() - start
+    print('\n')
+    print('-'*4+'End Generation'+'-'*4)
+    print(f'Num of generated tokens: {NUM_TOKENS}')
+    print(f'Time for complete generation: {time_generate}s')
+    print(f'Tokens per secound: {NUM_TOKENS/time_generate}')
+    print(f'Time per token: {(time_generate/NUM_TOKENS)*1000}ms')
+
+###
+# LLM FUNCTIONS
+###
+
+def construct_gemini_generative_model(in_api_key: str, temperature: float, model_choice: str, system_prompt: str, max_tokens: int) -> Tuple[object, dict]:
+    """
+    Constructs a GenerativeModel for Gemini API calls.
+
+    Parameters:
+    - in_api_key (str): The API key for authentication.
+    - temperature (float): The temperature parameter for the model, controlling the randomness of the output.
+    - model_choice (str): The choice of model to use for generation.
+    - system_prompt (str): The system prompt to guide the generation.
+    - max_tokens (int): The maximum number of tokens to generate.
+
+    Returns:
+    - Tuple[object, dict]: A tuple containing the constructed GenerativeModel and its configuration.
+    """
+    # Construct a GenerativeModel
+    try:
+        if in_api_key:
+            #print("Getting API key from textbox")
+            api_key = in_api_key
+            ai.configure(api_key=api_key)
+        elif "GOOGLE_API_KEY" in os.environ:
+            #print("Searching for API key in environmental variables")
+            api_key = os.environ["GOOGLE_API_KEY"]
+            ai.configure(api_key=api_key)
+        else:
+            print("No API key foound")
+            raise gr.Error("No API key found.")
+    except Exception as e:
+        print(e)
+    
+    config = ai.GenerationConfig(temperature=temperature, max_output_tokens=max_tokens)
+
+    #model = ai.GenerativeModel.from_cached_content(cached_content=cache, generation_config=config)
+    model = ai.GenerativeModel(model_name='models/' + model_choice, system_instruction=system_prompt, generation_config=config)
+    
+    # Upload CSV file (replace with your actual file path)
+    #file_id = ai.upload_file(upload_file_path)
+
+    
+    # if file_type == 'xlsx':
+    #     print("Running through all xlsx sheets")
+    #     #anon_xlsx = pd.ExcelFile(upload_file_path)
+    #     if not in_excel_sheets:
+    #         out_message.append("No Excel sheets selected. Please select at least one to anonymise.")
+    #         continue
+
+    #     anon_xlsx = pd.ExcelFile(upload_file_path)                
+
+    #     # Create xlsx file:
+    #     anon_xlsx_export_file_name = output_folder + file_name + "_redacted.xlsx"
+
+
+    ### QUERYING LARGE LANGUAGE MODEL ###
+    # Prompt caching the table and system prompt. See here: https://ai.google.dev/gemini-api/docs/caching?lang=python
+    # Create a cache with a 5 minute TTL. ONLY FOR CACHES OF AT LEAST 32k TOKENS!
+    # cache = ai.caching.CachedContent.create(
+    # model='models/' + model_choice,
+    # display_name=file_name, # used to identify the cache
+    # system_instruction=system_prompt,
+    # ttl=datetime.timedelta(minutes=5),
+    # )
+
+    return model, config
+
+def call_aws_claude(prompt: str, system_prompt: str, temperature: float, max_tokens: int, model_choice: str) -> ResponseObject:
+    """
+    This function sends a request to AWS Claude with the following parameters:
+    - prompt: The user's input prompt to be processed by the model.
+    - system_prompt: A system-defined prompt that provides context or instructions for the model.
+    - temperature: A value that controls the randomness of the model's output, with higher values resulting in more diverse responses.
+    - max_tokens: The maximum number of tokens (words or characters) in the model's response.
+    - model_choice: The specific model to use for processing the request.
+    
+    The function constructs the request configuration, invokes the model, extracts the response text, and returns a ResponseObject containing the text and metadata.
+    """
+
+    prompt_config = {
+        "anthropic_version": "bedrock-2023-05-31",
+        "max_tokens": max_tokens,
+        "top_p": 0.999,
+        "temperature":temperature,
+        "system": system_prompt,
+        "messages": [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": prompt},
+                ],
+            }
+        ],
+    }
+
+    body = json.dumps(prompt_config)
+
+    modelId = model_choice
+    accept = "application/json"
+    contentType = "application/json"
+
+    request = bedrock_runtime.invoke_model(
+        body=body, modelId=modelId, accept=accept, contentType=contentType
+    )
+
+    # Extract text from request
+    response_body = json.loads(request.get("body").read())
+    text = response_body.get("content")[0].get("text")
+
+    response = ResponseObject(
+    text=text,
+    usage_metadata=request['ResponseMetadata']
+    )
+
+    # Now you can access both the text and metadata
+    #print("Text:", response.text)
+    #print("Metadata:", response.usage_metadata)
+    #print("Text:", response.text)
+    
+    return response
+
+# Function to send a request and update history
+def send_request(prompt: str, conversation_history: List[dict], model: object, config: dict, model_choice: str, system_prompt: str, temperature: float, local_model=[], progress=Progress(track_tqdm=True)) -> Tuple[str, List[dict]]:
+    """
+    This function sends a request to a language model with the given prompt, conversation history, model configuration, model choice, system prompt, and temperature.
+    It constructs the full prompt by appending the new user prompt to the conversation history, generates a response from the model, and updates the conversation history with the new prompt and response.
+    If the model choice is specific to AWS Claude, it calls the `call_aws_claude` function; otherwise, it uses the `model.generate_content` method.
+    The function returns the response text and the updated conversation history.
+    """
+    # Constructing the full prompt from the conversation history
+    full_prompt = "Conversation history:\n"
+    
+    for entry in conversation_history:
+        role = entry['role'].capitalize()  # Assuming the history is stored with 'role' and 'parts'
+        message = ' '.join(entry['parts'])  # Combining all parts of the message
+        full_prompt += f"{role}: {message}\n"
+    
+    # Adding the new user prompt
+    full_prompt += f"\nUser: {prompt}"
+
+    # Clear any existing progress bars
+    tqdm._instances.clear()
+
+    progress_bar = range(0,number_of_api_retry_attempts)
+
+    # Generate the model's response
+    if "gemini" in model_choice:
+
+        for i in progress_bar:
+            try:
+                print("Calling Gemini model, attempt", i + 1)
+                #print("full_prompt:", full_prompt)
+                #print("generation_config:", config)
+
+                response = model.generate_content(contents=full_prompt, generation_config=config)
+
+                #progress_bar.close()
+                #tqdm._instances.clear()
+
+                print("Successful call to Gemini model.")
+                break
+            except Exception as e:
+                # If fails, try again after X seconds in case there is a throttle limit
+                print("Call to Gemini model failed:", e, " Waiting for ", str(timeout_wait), "seconds and trying again.")               
+
+                time.sleep(timeout_wait)
+            
+            if i == number_of_api_retry_attempts:
+                return ResponseObject(text="", usage_metadata={'RequestId':"FAILED"}), conversation_history
+    elif "anthropic.claude" in model_choice:
+        for i in progress_bar:
+            try:
+                print("Calling AWS Claude model, attempt", i + 1)
+                response = call_aws_claude(prompt, system_prompt, temperature, max_tokens, model_choice)
+
+                #progress_bar.close()
+                #tqdm._instances.clear()
+
+                print("Successful call to Claude model.")
+                break
+            except Exception as e:
+                # If fails, try again after X seconds in case there is a throttle limit
+                print("Call to Claude model failed:", e, " Waiting for ", str(timeout_wait), "seconds and trying again.")         
+
+                time.sleep(timeout_wait)
+                #response = call_aws_claude(prompt, system_prompt, temperature, max_tokens, model_choice)
+
+            if i == number_of_api_retry_attempts:
+                return ResponseObject(text="", usage_metadata={'RequestId':"FAILED"}), conversation_history
+    else:
+        # This is the local model
+        for i in progress_bar:
+            try:
+                print("Calling local model, attempt", i + 1)
+
+                gen_config = LlamaCPPGenerationConfig()
+                gen_config.update_temp(temperature)
+
+                response = call_llama_cpp_model(prompt, gen_config, model=local_model)
+
+                #progress_bar.close()
+                #tqdm._instances.clear()
+
+                print("Successful call to local model. Response:", response)
+                break
+            except Exception as e:
+                # If fails, try again after X seconds in case there is a throttle limit
+                print("Call to Gemma model failed:", e, " Waiting for ", str(timeout_wait), "seconds and trying again.")         
+
+                time.sleep(timeout_wait)
+                #response = call_aws_claude(prompt, system_prompt, temperature, max_tokens, model_choice)
+
+            if i == number_of_api_retry_attempts:
+                return ResponseObject(text="", usage_metadata={'RequestId':"FAILED"}), conversation_history       
+
+    # Update the conversation history with the new prompt and response
+    conversation_history.append({'role': 'user', 'parts': [prompt]})
+
+    # Check if is a LLama.cpp model response
+        # Check if the response is a ResponseObject
+    if isinstance(response, ResponseObject):
+        conversation_history.append({'role': 'assistant', 'parts': [response.text]})
+    elif 'choices' in response:
+        conversation_history.append({'role': 'assistant', 'parts': [response['choices'][0]['text']]})
+    else:
+        conversation_history.append({'role': 'assistant', 'parts': [response.text]})
+    
+    # Print the updated conversation history
+    #print("conversation_history:", conversation_history)
+    
+    return response, conversation_history
+
+def process_requests(prompts: List[str], system_prompt: str, conversation_history: List[dict], whole_conversation: List[str], whole_conversation_metadata: List[str], model: object, config: dict, model_choice: str, temperature: float, batch_no:int = 1, local_model = [], master:bool = False) -> Tuple[List[ResponseObject], List[dict], List[str], List[str]]:
+    """
+    Processes a list of prompts by sending them to the model, appending the responses to the conversation history, and updating the whole conversation and metadata.
+
+    Args:
+        prompts (List[str]): A list of prompts to be processed.
+        system_prompt (str): The system prompt.
+        conversation_history (List[dict]): The history of the conversation.
+        whole_conversation (List[str]): The complete conversation including prompts and responses.
+        whole_conversation_metadata (List[str]): Metadata about the whole conversation.
+        model (object): The model to use for processing the prompts.
+        config (dict): Configuration for the model.
+        model_choice (str): The choice of model to use.        
+        temperature (float): The temperature parameter for the model.
+        batch_no (int): Batch number of the large language model request.
+        local_model: Local gguf model (if loaded)
+        master (bool): Is this request for the master table.
+
+    Returns:
+        Tuple[List[ResponseObject], List[dict], List[str], List[str]]: A tuple containing the list of responses, the updated conversation history, the updated whole conversation, and the updated whole conversation metadata.
+    """
+    responses = []
+
+    # Clear any existing progress bars
+    tqdm._instances.clear()
+
+    for prompt in prompts:
+
+        #print("prompt to LLM:", prompt)
+
+        response, conversation_history = send_request(prompt, conversation_history, model=model, config=config, model_choice=model_choice, system_prompt=system_prompt, temperature=temperature, local_model=local_model)
+
+        if isinstance(response, ResponseObject):
+            response_text = response.text
+        elif 'choices' in response:
+            response_text = response['choices'][0]['text']
+        else:
+            response_text = response.text
+
+        responses.append(response)
+        whole_conversation.append(prompt)
+        whole_conversation.append(response_text)
+
+        # Create conversation metadata
+        if master == False:
+            whole_conversation_metadata.append(f"Query batch {batch_no} prompt {len(responses)} metadata:")
+        else:
+            whole_conversation_metadata.append(f"Query summary metadata:")
+
+        if not isinstance(response, str):
+            try:
+                print("model_choice:", model_choice)
+                if "claude" in model_choice:
+                    print("Appending selected metadata items to metadata")
+                    whole_conversation_metadata.append('x-amzn-bedrock-output-token-count:')
+                    whole_conversation_metadata.append(str(response.usage_metadata['HTTPHeaders']['x-amzn-bedrock-output-token-count']))
+                    whole_conversation_metadata.append('x-amzn-bedrock-input-token-count:')
+                    whole_conversation_metadata.append(str(response.usage_metadata['HTTPHeaders']['x-amzn-bedrock-input-token-count']))
+                elif "gemini" in model_choice:
+                    whole_conversation_metadata.append(str(response.usage_metadata))
+                else:
+                    whole_conversation_metadata.append(str(response['usage']))
+            except KeyError as e:
+                print(f"Key error: {e} - Check the structure of response.usage_metadata")
+        else:
+            print("Response is a string object.")
+            whole_conversation_metadata.append("Length prompt: " + str(len(prompt)) + ". Length response: " + str(len(response)))
+
+
+    return responses, conversation_history, whole_conversation, whole_conversation_metadata, response_text

tools/verify_titles.py

@@ -17,12 +17,12 @@ from gradio import Progress
 from typing import List, Tuple
 from io import StringIO
 
-from tools.prompts import initial_table_prompt, prompt2, prompt3, system_prompt, summarise_topic_descriptions_prompt, summarise_topic_descriptions_system_prompt, add_existing_topics_system_prompt, add_existing_topics_prompt, create_general_topics_system_prompt, create_general_topics_prompt, force_existing_topics_prompt, allow_new_topics_prompt, verify_titles_prompt, verify_titles_system_prompt
-from tools.helper_functions import output_folder, detect_file_type, get_file_name_no_ext, read_file, get_or_create_env_var, model_name_map, put_columns_in_df, wrap_text
-from tools.chatfuncs import LlamaCPPGenerationConfig, call_llama_cpp_model, load_model, RUN_LOCAL_MODEL
-from tools.llm_api_call import normalise_string, load_in_file, load_in_data_file, load_in_previous_data_files, get_basic_response_data, data_file_to_markdown_table,replace_punctuation_with_underscore, construct_gemini_generative_model, call_aws_claude, send_request, process_requests, clean_markdown_table, clean_column_name, create_unique_table_df_from_reference_table, remove_before_last_term, convert_response_text_to_markdown_table, call_llm_with_markdown_table_checks, write_llm_output_and_logs, convert_reference_table_to_pivot_table, ResponseObject, max_tokens, timeout_wait, number_of_api_retry_attempts, MAX_OUTPUT_VALIDATION_ATTEMPTS, max_time_for_loop, batch_size_default, MAX_COMMENT_CHARS, max_comment_character_length, AWS_DEFAULT_REGION, bedrock_runtime, GradioFileData
+from tools.prompts import initial_table_prompt, prompt2, prompt3, system_prompt,add_existing_topics_system_prompt, add_existing_topics_prompt
+from tools.helper_functions import put_columns_in_df, wrap_text
+from tools.llm_funcs import load_model, construct_gemini_generative_model
+from tools.llm_api_call import load_in_data_file, get_basic_response_data, data_file_to_markdown_table, clean_column_name,  convert_response_text_to_markdown_table, call_llm_with_markdown_table_checks,  ResponseObject, max_tokens, max_time_for_loop, batch_size_default,  GradioFileData
 
-#
+from tools.config import MAX_OUTPUT_VALIDATION_ATTEMPTS,  RUN_LOCAL_MODEL, model_name_map, OUTPUT_FOLDER
 
 def write_llm_output_and_logs_verify(responses: List[ResponseObject],
                               whole_conversation: List[str],
@@ -37,8 +37,9 @@ def write_llm_output_and_logs_verify(responses: List[ResponseObject],
                               existing_reference_df:pd.DataFrame,
                               existing_topics_df:pd.DataFrame,
                               batch_size_number:int,
-                              in_column:str,
-                              first_run: bool = False) -> None:
+                              in_column:str,                              
+                              first_run: bool = False,
+                              output_folder:str=OUTPUT_FOLDER) -> None:
     """
     Writes the output of the large language model requests and logs to files.
 
@@ -56,6 +57,7 @@ def write_llm_output_and_logs_verify(responses: List[ResponseObject],
     - existing_reference_df (pd.DataFrame): The existing reference dataframe mapping response numbers to topics.
     - existing_topics_df (pd.DataFrame): The existing unique topics dataframe 
     - first_run (bool): A boolean indicating if this is the first run through this function in this process. Defaults to False.
+    - output_folder (str): A string indicating the folder to output to
     """
     unique_topics_df_out_path = []
     topic_table_out_path = "topic_table_error.csv"
@@ -236,6 +238,7 @@ def verify_titles(in_data_file,
               sentiment_checkbox:str = "Negative, Neutral, or Positive",
               force_zero_shot_radio:str = "No",
               in_excel_sheets:List[str] = [],
+              output_folder:str=OUTPUT_FOLDER,
               max_tokens:int=max_tokens,
               model_name_map:dict=model_name_map,              
               max_time_for_loop:int=max_time_for_loop,              
@@ -276,7 +279,8 @@ def verify_titles(in_data_file,
     - time_taken (float, optional): The amount of time taken to process the responses up until this point.
     - sentiment_checkbox (str, optional): What type of sentiment analysis should the topic modeller do?
     - force_zero_shot_radio (str, optional): Should responses be forced into a zero shot topic or not.
-    - in_excel_sheets (List[str], optional): List of excel sheets to load from input file
+    - in_excel_sheets (List[str], optional): List of excel sheets to load from input file.
+    - output_folder (str): The output folder where files will be saved.
     - max_tokens (int): The maximum number of tokens for the model.
     - model_name_map (dict, optional): A dictionary mapping full model name to shortened.
     - max_time_for_loop (int, optional): The number of seconds maximum that the function should run for before breaking (to run again, this is to avoid timeouts with some AWS services if deployed there).

windows_install_llama-cpp-python.txt

@@ -0,0 +1,118 @@
+---
+
+set PKG_CONFIG_PATH=C:\<path-to-openblas>\OpenBLAS\lib\pkgconfig # Set this in environment variables
+
+
+pip install llama-cpp-python==0.3.9 --force-reinstall --verbose --no-cache-dir -Ccmake.args="-DGGML_BLAS=ON;-DGGML_BLAS_VENDOR=OpenBLAS;-DBLAS_INCLUDE_DIRS=C:/<path-to-openblas>/OpenBLAS/include;-DBLAS_LIBRARIES=C:/<path-to-openblas>/OpenBLAS/lib/libopenblas.lib"
+---
+
+# With CUDA
+
+pip install llama-cpp-python==0.3.9 --force-reinstall --no-cache-dir --verbose -C cmake.args="-DGGML_CUDA=on"
+
+
+---
+
+
+How to Make it Work: Step-by-Step Guide
+To successfully run your command, you need to set up a proper C++ development environment.
+
+Step 1: Install the C++ Compiler
+Go to the Visual Studio downloads page.
+
+Scroll down to "Tools for Visual Studio" and download the "Build Tools for Visual Studio". This is a standalone installer that gives you the C++ compiler and libraries without installing the full Visual Studio IDE.
+
+Run the installer. In the "Workloads" tab, check the box for "Desktop development with C++".
+
+MSVC v143
+C++ ATL
+C++ Profiling tools
+C++ CMake tools for Windows
+C++ MFC
+C++ Modules
+Windows 10 SDK (10.0.20348.0)
+
+Proceed with the installation.
+
+
+Need to use 'x64 Native Tools Command Prompt for VS 2022' to install. Run as administrator
+
+Step 2: Install CMake
+Go to the CMake download page.
+
+Download the latest Windows installer (e.g., cmake-x.xx.x-windows-x86_64.msi).
+
+Run the installer. Crucially, when prompted, select the option to "Add CMake to the system PATH for all users" or "for the current user." This allows you to run cmake from any command prompt.
+
+
+Step 3: Download and Place OpenBLAS
+This is often the trickiest part.
+
+Go to the OpenBLAS releases on GitHub.
+
+Find a recent release and download the pre-compiled version for Windows. It will typically be a file named something like OpenBLAS-0.3.21-x64.zip (the version number will change). Make sure you get the 64-bit (x64) version if you are using 64-bit Python.
+
+Create a folder somewhere easily accessible, for example, C:\libs\.
+
+Extract the contents of the OpenBLAS zip file into that folder. Your final directory structure should look something like this:
+
+Generated code
+C:\libs\OpenBLAS\
+├── bin\
+├── include\
+└── lib\
+Use code with caution.
+
+3.b. Install Chocolatey
+https://chocolatey.org/install
+
+Step 1: Install Chocolatey (if you don't already have it)
+Open PowerShell as an Administrator. (Right-click the Start Menu -> "Windows PowerShell (Admin)" or "Terminal (Admin)").
+
+Run the following command to install Chocolatey. It's a single, long line:
+
+Generated powershell
+Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))
+Use code with caution.
+Powershell
+Wait for it to finish. Once it's done, close the Administrator PowerShell window.
+
+Step 2: Install pkg-config-lite using Chocolatey
+IMPORTANT: Open a NEW command prompt or PowerShell window (as a regular user is fine). This is necessary so it recognizes the new choco command.
+
+Run the following command to install a lightweight version of pkg-config:
+
+Generated cmd
+choco install pkgconfiglite
+Use code with caution.
+Cmd
+Approve the installation by typing Y or A if prompted.
+
+
+Step 4: Run the Installation Command
+Now you have all the pieces. The final step is to run the command in a terminal that is aware of your new build environment.
+
+Open the "Developer Command Prompt for VS" from your Start Menu. This is important! This special command prompt automatically configures all the necessary paths for the C++ compiler.
+
+## For CPU
+
+set PKG_CONFIG_PATH=C:\<path-to-openblas>\OpenBLAS\lib\pkgconfig # Set this in environment variables
+
+
+pip install llama-cpp-python==0.3.9 --force-reinstall --verbose --no-cache-dir -Ccmake.args="-DGGML_BLAS=ON;-DGGML_BLAS_VENDOR=OpenBLAS;-DBLAS_INCLUDE_DIRS=C:/<path-to-openblas>/OpenBLAS/include;-DBLAS_LIBRARIES=C:/<path-to-openblas>/OpenBLAS/lib/libopenblas.lib"
+
+## With Cuda
+
+
+Use NVIDIA GPU (cuBLAS): If you have an NVIDIA GPU, using cuBLAS is often easier because the CUDA Toolkit installer handles most of the setup.
+
+Install the NVIDIA CUDA Toolkit.
+
+Run the install command specifying cuBLAS:
+
+
+set PKG_CONFIG_PATH=C:\<path-to-openblas>\OpenBLAS\lib\pkgconfig # Set this in environment variables
+
+pip install llama-cpp-python==0.3.9 --force-reinstall --no-cache-dir --verbose -C cmake.args="-DGGML_CUDA=on"
+
+