File size: 3,651 Bytes
056e156 2b8f89d e22e877 d1ca5fe 056e156 e1043c6 e22e877 741eae9 e1043c6 056e156 fd87b41 056e156 e22e877 056e156 e22e877 056e156 e22e877 056e156 e22e877 056e156 e22e877 056e156 8011593 056e156 d1ca5fe 056e156 e22e877 056e156 e22e877 8011593 056e156 e22e877 e1043c6 e22e877 e1043c6 b1ae936 e1043c6 e22e877 e1043c6 6eae533 e1043c6 e22e877 e1043c6 056e156 e22e877 2b8f89d 9a20624 e22e877 741eae9 2b8f89d e22e877 741eae9 7be8121 e22e877 741eae9 1eda6aa 741eae9 056e156 e22e877 8011593 056e156 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 |
import os
import json
import datetime
from email.utils import parseaddr
from io import BytesIO
from huggingface_hub import HfApi
import gradio as gr
from eval_utils import get_evaluation_scores
# Constants
LEADERBOARD_PATH = "Exploration-Lab/IL-TUR-Leaderboard"
SUBMISSION_FORMAT = "predictions"
TOKEN = os.environ.get("TOKEN", None)
YEAR_VERSION = "2024"
api = HfApi(token=TOKEN)
# Helper functions for formatting messages
def format_message(msg, color):
return f"<p style='color: {color}; font-size: 20px; text-align: center;'>{msg}</p>"
def format_error(msg):
return format_message(msg, "red")
def format_warning(msg):
return format_message(msg, "orange")
def format_log(msg):
return format_message(msg, "green")
def model_hyperlink(link, model_name):
return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
def input_verification(method_name, url, path_to_file, organisation, mail):
"""Verify the input fields for submission."""
# Check if any field is empty
if any(
input == "" for input in [method_name, url, path_to_file, organisation, mail]
):
return format_warning("Please fill all the fields.")
# Check if file is attached
if path_to_file is None:
return format_warning("Please attach a file.")
return
def add_new_eval(
method_name: str,
submitted_by: str,
url: str,
path_to_file: str,
organisation: str,
mail: str,
):
"""Add a new evaluation to the leaderboard."""
# Verify input
# Check if any field is empty
if any(
input == "" for input in [method_name, url, path_to_file, organisation, mail]
):
return format_warning("Please fill all the fields.")
# Check if file is attached
if path_to_file is None:
return format_warning("Please attach a file.")
# Verify email format
_, parsed_mail = parseaddr(mail)
if "@" not in parsed_mail:
print(parseaddr(mail))
return format_warning("Please provide a valid email address.")
# Process submission
if SUBMISSION_FORMAT == "predictions":
# Read submission and gold data
with open(path_to_file, "r") as f:
submission_data = json.load(f)
with open("submissions/baseline/IL_TUR_eval_gold.json", "r") as f:
gold_data = json.load(f)
# Get evaluation scores
submission = get_evaluation_scores(gold_data, submission_data)
# Add metadata
submission["Method"] = method_name
submission["Submitted By"] = submitted_by
# submission["Organisation"] = organisation
# submission["Email"] = mail
submission["Github Link"] = url
else:
# Read submission directly if it's not in predictions format
with open(path_to_file, "r") as f:
submission = json.load(f)
# Update results
with open("submissions/baseline/results.json", "r") as f:
results = json.load(f)
results.append(submission[0])
# Prepare buffer for upload
leaderboard_buffer = BytesIO(json.dumps(results).encode())
leaderboard_buffer.seek(0)
# Upload to Hugging Face
api.upload_file(
repo_id=LEADERBOARD_PATH,
path_in_repo="submissions/baseline/results.json",
path_or_fileobj=leaderboard_buffer,
token=TOKEN,
repo_type="space",
)
return format_log(
f"Method {method_name} submitted by {organisation} successfully. \n"
"Please refresh the leaderboard, and wait for the evaluation results."
)
|