diff --git a/client/api/assignment.py b/client/api/assignment.py index f771f168..727470ac 100644 --- a/client/api/assignment.py +++ b/client/api/assignment.py @@ -261,6 +261,7 @@ def server_url(self): # "rate_limit", uncomment to turn rate limiting back on! "file_contents", "grading", + "help", "analytics", "autostyle", "collaborate", diff --git a/client/cli/ok.py b/client/cli/ok.py index 35408a9e..58b9acdf 100644 --- a/client/cli/ok.py +++ b/client/cli/ok.py @@ -131,6 +131,8 @@ def parse_input(command_input=None): help="run AutoStyle feedback system") experiment.add_argument('--collab', action='store_true', help="launch collaborative programming environment") + experiment.add_argument('--get-help', action='store_true', + help="receive 61A-bot feedback on your code") # Debug information debug = parser.add_argument_group('ok developer debugging options') @@ -273,6 +275,7 @@ def main(): try: msgs = messages.Messages() + msgs['email'] = assign.get_student_email() for name, proto in assign.protocol_map.items(): log.info('Execute {}.run()'.format(name)) proto.run(msgs) diff --git a/client/protocols/grading.py b/client/protocols/grading.py index ab8d970f..0d3f287f 100644 --- a/client/protocols/grading.py +++ b/client/protocols/grading.py @@ -8,8 +8,11 @@ from client.protocols.common import models from client.utils import format from client.utils import storage +from client.utils import output +from client.utils import config as config_utils import logging import sys +import re log = logging.getLogger(__name__) @@ -43,10 +46,10 @@ def run(self, messages, env=None): 'Suite number must be valid.({})'.format(len(test.suites)))) if self.args.case: suite.run_only = [int(c) for c in self.args.case] - grade(tests, messages, env, verbose=self.args.verbose) + grade(tests, messages, env, verbose=self.args.verbose, get_help=self.args.get_help, config=self.args.config) -def grade(questions, messages, env=None, verbose=True): +def grade(questions, messages, env=None, verbose=True, get_help=False, config=None): format.print_line('~') print('Running tests') print() @@ -57,6 +60,8 @@ def grade(questions, messages, env=None, verbose=True): analytics = {} # The environment in which to run the tests. + + log_id = output.new_log() for test in questions: log.info('Running tests for {}'.format(test.name)) results = test.run(env) @@ -78,6 +83,30 @@ def grade(questions, messages, env=None, verbose=True): verbose=verbose) print() + autograder_output = ''.join(output.get_log(log_id)) + messages['grading'] = analytics + ### Fa23 Helper Bot ### + HELP_KEY = 'jfv97pd8ogybhilq3;orfuwyhiulae' + config = config_utils._get_config(config) + if (failed > 0 or get_help) and (config['src'][0][:2] == 'hw'): + res = input("Would you like to receive 61A-bot feedback on your code (y/N)? ") + print() + if res == "y": + filename = config['src'][0] + code = open(filename, 'r').read() + messages['gpt'] = { + 'email': messages.get('email') or '', + 'promptLabel': 'Get_help', + 'hwId': re.findall(r'hw(\d+)\.(py|scm|sql)', filename)[0][0], + 'activeFunction': questions[0].name, + 'code': code, + 'codeError': autograder_output, + 'version': 'v2', + 'key': HELP_KEY + } + else: + messages['gpt'] = False + protocol = GradingProtocol diff --git a/client/protocols/help.py b/client/protocols/help.py new file mode 100644 index 00000000..e1c9fb99 --- /dev/null +++ b/client/protocols/help.py @@ -0,0 +1,70 @@ +from client.protocols.common import models + +import requests +import random + +import itertools +import threading +import time +import sys + +import logging + +from client.utils.printer import print_error + +log = logging.getLogger(__name__) + +class HelpProtocol(models.Protocol): + + SERVER = 'https://61a-bot-backend.zamfi.net' + HELP_ENDPOINT = SERVER + '/get-help-cli' + FEEDBACK_PROBABILITY = 0.25 + FEEDBACK_ENDPOINT = SERVER + '/feedback' + FEEDBACK_KEY = 'jfv97pd8ogybhilq3;orfuwyhiulae' + + def run(self, messages): + help_payload = messages.get('gpt') + if help_payload: + help_response = None + def animate(): + for c in itertools.cycle(["|", "/", "-", "\\"]): + if help_response: + break + sys.stdout.write("\rLoading " + c + " ") + sys.stdout.write('\033[2K\033[1G') + time.sleep(0.1) + # sys.stdout.write("\033[K") + t = threading.Thread(target=animate) + t.daemon = True + t.start() + try: + help_response = requests.post(self.HELP_ENDPOINT, json=help_payload).json() + except Exception as e: + print_error("Error generating hint. Please try again later.") + return + print(help_response.get('output', "An error occurred. Please try again later.")) + print() + + random.seed(int(time.time())) + if random.random() < self.FEEDBACK_PROBABILITY: + print("Please indicate whether the feedback you received was helpful or not.") + print("1) It was helpful.") + print("-1) It was not helpful.") + feedback = None + while feedback not in {"1", "-1"}: + if feedback is None: + feedback = input("? ") + else: + feedback = input("-- Please select a provided option. --\n? ") + print("\nThank you for your feedback.\n") + req_id = help_response.get('requestId') + if req_id: + feedback_payload = { + 'version': 'v2', + 'key': self.FEEDBACK_KEY, + 'requestId': req_id, + 'feedback': feedback + } + feedback_response = requests.post(self.FEEDBACK_ENDPOINT, json=feedback_payload).json() + +protocol = HelpProtocol diff --git a/client/utils/config.py b/client/utils/config.py index 66f67ffe..a4f9055e 100644 --- a/client/utils/config.py +++ b/client/utils/config.py @@ -8,3 +8,32 @@ def create_config_directory(): if not os.path.exists(CONFIG_DIRECTORY): os.makedirs(CONFIG_DIRECTORY) return CONFIG_DIRECTORY + +def _get_config(config): + if config is None: + configs = glob.glob(CONFIG_EXTENSION) + if len(configs) > 1: + raise ex.LoadingException('\n'.join([ + 'Multiple .ok files found:', + ' ' + ' '.join(configs), + "Please specify a particular assignment's config file with", + ' python3 ok --config ' + ])) + elif not configs: + raise ex.LoadingException('No .ok configuration file found') + config = configs[0] + elif not os.path.isfile(config): + raise ex.LoadingException( + 'Could not find config file: {}'.format(config)) + + try: + with open(config, 'r') as f: + result = json.load(f, object_pairs_hook=collections.OrderedDict) + except IOError: + raise ex.LoadingException('Error loading config: {}'.format(config)) + except ValueError: + raise ex.LoadingException( + '{0} is a malformed .ok configuration file. ' + 'Please re-download {0}.'.format(config)) + else: + return result