update

app.py

@@ -1,7 +1,7 @@
 import gradio as gr
 
 from pages.arena import create_arena
-from pages.summarization_example import create_summarization_interface
+from pages.summarization_playground import create_summarization_interface
 from pages.leaderboard import create_leaderboard
 
 def welcome_message():
@@ -22,12 +22,14 @@ with gr.Blocks() as demo:
         )
 
         with gr.Tabs() as tabs:
-            with gr.TabItem("Demo_of_Streaming"):
-                create_arena()
             with gr.TabItem("Summarization"):
                 create_summarization_interface()
             with gr.TabItem("Leaderboard"):
                 create_leaderboard()
+            with gr.TabItem("Batch_Evaluation"):
+                create_arena()
+            with gr.TabItem("Demo_of_Streaming"):
+                create_arena()
 
 
 if __name__ == "__main__":

pages/__init__.py

@@ -0,0 +1,4 @@
+# This is the __init__.py file for the utils package
+# You can add any initialization code or import statements here
+
+__all__ = ['arena', 'batch_evaluation', 'leaderboard', 'summarization_playground']

pages/batch_evaluation.py

@@ -0,0 +1,57 @@
+from dotenv import load_dotenv
+import gradio as gr
+import random
+
+from utils.model import Model
+from utils.data import dataset
+from utils.metric import metric_rouge_score
+
+from summarization_playground import model, generate_answer
+
+load_dotenv()
+
+def process(seed, model_selection, prompt, num=10):
+    random.seed(seed)
+    response_list = []
+
+    for data in random.choices(dataset, k=num):
+        dialogue = data['dialogue']
+        summary = data['summary']
+        response = generate_answer(dialogue, model, model_selection, prompt)
+
+        rouge_score = metric_rouge_score(response, summary)
+
+        response_list.append(
+            {
+                'dialogue': dialogue,
+                'summary': summary,
+                'response': response,
+                'metric_score': {
+                    'rouge_score': rouge_score
+                }
+            }
+        )
+
+    return response_list
+
+def create_batch_evaluation_interface():
+    with gr.blocks() as demo:
+        gr.Markdown("## Here are evaluation setups")
+        with gr.Row():
+            seed = gr.Number(value=8, placeholder="pick your favoriate random seed")
+            model_dropdown = gr.Dropdown(choices=Model.__model_list__, label="Choose a model", value=Model.__model_list__[0])
+        Template_text = gr.Textbox(value="""Summariza the following dialogue""", label='Input Prompting Template', lines=8, placeholder='Input your prompts')
+        submit_button = gr.Button("✨ Submit ✨")
+        output = gr.Markdown()
+
+        submit_button.click(
+            process,
+            inputs=[seed, model_dropdown, Template_text],
+            outputs=output
+        )
+
+    return demo
+
+if __name__ == "__main__":
+    demo = create_batch_evaluation_interface()
+    demo.launch()

pages/{summarization_example.py → summarization_playground.py}

@@ -7,11 +7,7 @@ from utils.data import dataset
 
 load_dotenv()
 
-__model_list__ = [
-    "lmsys/vicuna-7b-v1.5",
-    "tiiuae/falcon-7b-instruct"
-]
-model = {model_name: Model(model_name) for model_name in __model_list__}
+model = {model_name: Model(model_name) for model_name in Model.__model_list__}
 
 random_label = '🔀 Random dialogue from dataset'
 examples = {
@@ -31,7 +27,7 @@ Back in Boston, Kidd is going to rely on Lively even more. He'll play close to 3
     random_label: ""
 }
 
-def generate_answer(sources, model_name, prompt):
+def generate_answer(sources, model, model_name, prompt):
     assert "{sources}" in prompt, ValueError("No {sources} Found")
 
     meta_prompt = prompt
@@ -42,7 +38,7 @@ def generate_answer(sources, model_name, prompt):
 
 def process_input(input_text, model_selection, prompt):
     if input_text:
-        response = generate_answer(input_text, model_selection, prompt)
+        response = generate_answer(input_text, model, model_selection, prompt)
         return f"## Original Article:\n\n{input_text}\n\n## Summarization:\n\n{response}"
     else:
         return "Please fill the input to generate outputs."
@@ -54,15 +50,13 @@ def update_input(example):
 
 def create_summarization_interface():
     with gr.Blocks() as demo:
-        gr.Markdown("## This is an example to show summarization")
+        gr.Markdown("## This is a playground to test summarizations")
         
         with gr.Row():
             example_dropdown = gr.Dropdown(choices=list(examples.keys()), label="Choose an example")
-            model_dropdown = gr.Dropdown(choices=["lmsys/vicuna-7b-v1.5", "tiiuae/falcon-7b-instruct"], label="Choose a model", value="lmsys/vicuna-7b-v1.5")
+            model_dropdown = gr.Dropdown(choices=Model.__model_list__, label="Choose a model", value=Model.__model_list__[0])
         
-        Template_text = gr.Textbox(value="""{sources}
-
-summarization: """, label='Input Prompting Template', lines=8, placeholder='Input your prompts, must include \{sources\}')
+        Template_text = gr.Textbox(value="""Summariza the following dialogue""", label='Input Prompting Template', lines=8, placeholder='Input your prompts, must include \{sources\}')
 
         input_text = gr.Textbox(label="Input Text", lines=10, placeholder="Enter text here...")
         submit_button = gr.Button("✨ Submit ✨")

utils/__init__.py

@@ -1,4 +1,4 @@
 # This is the __init__.py file for the utils package
 # You can add any initialization code or import statements here
 
-__all__ = ['multiple_stream', 'model']
+__all__ = ['multiple_stream', 'model', 'data', 'metric']

utils/metric.py

@@ -0,0 +1,6 @@
+from rouge_score import rouge_scorer
+
+scorer = rouge_scorer.RougeScorer(['rougeL'], use_stemmer=True)
+
+def metric_rouge_score(pred, ref):
+    return scorer.score(pred, ref)['rougeL'].fmeasure

utils/model.py

@@ -4,6 +4,10 @@ import torch
 
 class Model():
     number_of_models = 0
+    __model_list__ = [
+        "lmsys/vicuna-7b-v1.5",
+        "tiiuae/falcon-7b-instruct"
+    ]
 
     def __init__(self, model_name="lmsys/vicuna-7b-v1.5") -> None:
         self.tokenizer = AutoTokenizer.from_pretrained(model_name)